From 147d4d6db39763b5555519e9d282d26e6068d023 Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Sun, 22 Mar 2020 17:03:00 +0530 Subject: [PATCH 01/28] Implemented R2 Score --- src/mlpack/core/cv/metrics/CMakeLists.txt | 2 + src/mlpack/core/cv/metrics/r2_score.hpp | 71 ++++++++++++++++++++ src/mlpack/core/cv/metrics/r2_score_impl.hpp | 60 +++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100644 src/mlpack/core/cv/metrics/r2_score.hpp create mode 100644 src/mlpack/core/cv/metrics/r2_score_impl.hpp diff --git a/src/mlpack/core/cv/metrics/CMakeLists.txt b/src/mlpack/core/cv/metrics/CMakeLists.txt index 4cf1027874..b9edacaf9a 100644 --- a/src/mlpack/core/cv/metrics/CMakeLists.txt +++ b/src/mlpack/core/cv/metrics/CMakeLists.txt @@ -13,6 +13,8 @@ set(SOURCES precision_impl.hpp recall.hpp recall_impl.hpp + r2_score.hpp + r2_score_impl.hpp ) # Add directory name to sources. diff --git a/src/mlpack/core/cv/metrics/r2_score.hpp b/src/mlpack/core/cv/metrics/r2_score.hpp new file mode 100644 index 0000000000..a1ff04bf0f --- /dev/null +++ b/src/mlpack/core/cv/metrics/r2_score.hpp @@ -0,0 +1,71 @@ +/** + * @file r2_score.hpp + * @author Bisakh Mondal + * + * The R^2 (Coefficient of determination) regression metric. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_R2SCORE_HPP +#define MLPACK_CORE_CV_METRICS_R2SCORE_HPP + +#include + +namespace mlpack { +namespace cv { + +/** + * The R2Score is a metric of performance for regression algorithms + * that represents the proportion of variance (of y) that has been + * explained by the independent variables in the model. It provides + * an indication of goodness of fit and therefore a measure of how + * well unseen samples are likely to be predicted by the model, + * through the proportion of explained variance. + * As R2Score is dataset dependent it can have wide range of values, + * best possible score is @f$R^2 =1.0@f$, and it can be negative too for an + * arbitraryly worse model. For a model which predicts exactly the expected + * value of y, disregarding the input features, gets a R2Score equals to 0.0. + * If a model predicts @f$ \hat{y}_i $@f of the @f$ i $@f-th sample for a true + * @f$ y_i $@f for total n samples, the R2Score is calculated by + * @f{eqnarray*}{ + * R^{2} \left( y, \hat{y} \right) &=& 1-\frac{\sum_{i=1}^{n} + * \left( y_i - \hat{y_i} \right)^2 } + * {\sum_{i=1}^{n} \left( y_i - \bar{y}\right)^2}\\ + * @f} + * where @f$ \bar{y} = frac{1}{y}\sum_{i=1}^{n} y_i $@f. + * For example, a model having R2Score = 0.85, explains 85 \% variability of + * the response data around its mean. + */ +class R2Score +{ + public: + /** + * Run prediction and calculate the R squared error. + * + * @param model A regression model. + * @param data Column-major data containing test items. + * @param responses Ground truth (correct) target values for the test items, + * should be either a row vector or a column-major matrix. + */ + template + static double Evaluate(MLAlgorithm& model, + const DataType& data, + const ResponsesType& responses); + + /** + * Information for hyper-parameter tuning code. It indicates that we want + * to maximize the measurement. + */ + static const bool NeedsMinimization = false; +}; + +} // namespace cv +} // namespace mlpack + +// Include implementation. +#include "r2_score_impl.hpp" + +#endif diff --git a/src/mlpack/core/cv/metrics/r2_score_impl.hpp b/src/mlpack/core/cv/metrics/r2_score_impl.hpp new file mode 100644 index 0000000000..b047709b12 --- /dev/null +++ b/src/mlpack/core/cv/metrics/r2_score_impl.hpp @@ -0,0 +1,60 @@ +/** + * @file r2_score_impl.hpp + * @author Bisakh Mondal + * + * The implementation of the class R2Score. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_R2SCORE_IMPL_HPP +#define MLPACK_CORE_CV_METRICS_R2SCORE_IMPL_HPP + +namespace mlpack { +namespace cv { + +template +double R2Score::Evaluate(MLAlgorithm& model, + const DataType& data, + const ResponsesType& responses) +{ + if (data.n_cols != responses.n_cols) + { + std::ostringstream oss; + oss << "R2Score::Evaluate(): number of points (" << data.n_cols << ") " + << "does not match number of responses (" << responses.n_cols << ")!" + << std::endl; + throw std::invalid_argument(oss.str()); + } + + ResponsesType predictedResponses; + // Taking Predicted Output from the model. + model.Predict(data, predictedResponses); + // Mean value of response. + double mean_responses = arma::mean(responses); + + // Calculate the numerator i.e. residual sum of squares. + double ss_res = arma::accu(arma::square(responses, predictedResponses)); + + // Calculate the denominator i.e.total sum of squares. + double ss_tot = arma::accu(arma::square(responses, mean_responses)); + + // Handling undefined R2Score when both denominator and numerator is 0.0 + if (ss_res == 0.0) + { + if (ss_tot !=0.0) + return 1.0; + else + return DBL_MIN; + + } + + return 1 - ss_res/ss_tot; +} + +} // namespace cv +} // namespace mlpack + +#endif From ab261514f744c2243fddabc103dbef084610754e Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Sun, 22 Mar 2020 17:34:09 +0530 Subject: [PATCH 02/28] R2Score tests added --- src/mlpack/tests/cv_test.cpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index b4182ac1fb..fe7f0aa8d2 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -164,6 +165,29 @@ BOOST_AUTO_TEST_CASE(MSETest) BOOST_REQUIRE_CLOSE(MSE::Evaluate(lr, data, responses), expectedMSE, 1e-5); } +/** + * Test the R squared metric (R2Score). + */ +BOOST_AUTO_TEST_CASE(R2ScoreTest) +{ + // Making two points that define the linear function f(x) = x - 1 + arma::mat trainingData("0 1"); + arma::rowvec trainingResponses("-1 0"); + + LinearRegression lr(trainingData, trainingResponses); + + // Making three responses that differ from the correct ones by 0, 1, and 2 + // respectively. Original Responses mean (1 + 2 + 3) / 3 = 2 + arma::mat data("2 3 4"); + arma::rowvec responses("1 3 5");// Mean_responses= (1+ 3 + 5) /3 = 3 + + double ss_reg = (0 + 1 * 1 + 2 * 2); + double ss_tot = ((1 - 2) * (1- 2) + (2 -2) * (2 - 2) + (3 -2) * (3 - 2)); + double expectedR2 = 1 - ss_reg / ss_tot; + + BOOST_REQUIRE_CLOSE(R2Score::Evaluate(lr, data, responses), expectedR2, 1e-5); +} + /** * Test the mean squared error with matrix responses. */ From c0d6259b4a9ee574cf3719da477f39ff26d5d90c Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Sun, 22 Mar 2020 20:08:12 +0530 Subject: [PATCH 03/28] Typo fixed --- src/mlpack/core/cv/metrics/r2_score_impl.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/mlpack/core/cv/metrics/r2_score_impl.hpp b/src/mlpack/core/cv/metrics/r2_score_impl.hpp index b047709b12..6fecfac378 100644 --- a/src/mlpack/core/cv/metrics/r2_score_impl.hpp +++ b/src/mlpack/core/cv/metrics/r2_score_impl.hpp @@ -36,10 +36,10 @@ double R2Score::Evaluate(MLAlgorithm& model, double mean_responses = arma::mean(responses); // Calculate the numerator i.e. residual sum of squares. - double ss_res = arma::accu(arma::square(responses, predictedResponses)); + double ss_res = arma::accu(arma::square(responses - predictedResponses)); // Calculate the denominator i.e.total sum of squares. - double ss_tot = arma::accu(arma::square(responses, mean_responses)); + double ss_tot = arma::accu(arma::square(responses - mean_responses)); // Handling undefined R2Score when both denominator and numerator is 0.0 if (ss_res == 0.0) @@ -48,7 +48,6 @@ double R2Score::Evaluate(MLAlgorithm& model, return 1.0; else return DBL_MIN; - } return 1 - ss_res/ss_tot; From 56982ac742cdf18de065921e43eff883c0935120 Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Sun, 22 Mar 2020 20:12:41 +0530 Subject: [PATCH 04/28] Tests slighltly modified --- src/mlpack/tests/cv_test.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index fe7f0aa8d2..8886f68c4b 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -176,13 +176,17 @@ BOOST_AUTO_TEST_CASE(R2ScoreTest) LinearRegression lr(trainingData, trainingResponses); - // Making three responses that differ from the correct ones by 0, 1, and 2 - // respectively. Original Responses mean (1 + 2 + 3) / 3 = 2 - arma::mat data("2 3 4"); - arma::rowvec responses("1 3 5");// Mean_responses= (1+ 3 + 5) /3 = 3 - - double ss_reg = (0 + 1 * 1 + 2 * 2); - double ss_tot = ((1 - 2) * (1- 2) + (2 -2) * (2 - 2) + (3 -2) * (3 - 2)); + // Making five responses that are the output of regression function f(x) + // with some responses having a slight deviation of 0.005 + // Mean Responses = (1 + 2 + 3 + 6 + 8)/5 = 4 + arma::mat data("2 3 4 7 9"); + arma::rowvec responses("1 2.005 3 6.005 8.005"); + + double ss_reg = (0 * 0 + 0.005 * 0.005 + 0 * 0 + + 0.005 * 0.005 + 0.005 *0.005); + double ss_tot = ((1 - 4) * (1- 4) + (2.005 - 4) * + (2.005 - 4) + (3 - 4) * (3 - 4) + (6.005 - 4) * + (6.005 - 4) + (8.005 - 4) * (8.005 -4)); double expectedR2 = 1 - ss_reg / ss_tot; BOOST_REQUIRE_CLOSE(R2Score::Evaluate(lr, data, responses), expectedR2, 1e-5); From edca4cd82b1965299fa4811d69fb5350c3203889 Mon Sep 17 00:00:00 2001 From: Bisakh Mondal <41498427+bisakhmondal@users.noreply.github.com> Date: Mon, 23 Mar 2020 09:55:18 +0530 Subject: [PATCH 05/28] Apply suggestions from code review Co-Authored-By: favre49 <40389657+favre49@users.noreply.github.com> --- src/mlpack/core/cv/metrics/r2_score.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/cv/metrics/r2_score.hpp b/src/mlpack/core/cv/metrics/r2_score.hpp index a1ff04bf0f..7ffb608760 100644 --- a/src/mlpack/core/cv/metrics/r2_score.hpp +++ b/src/mlpack/core/cv/metrics/r2_score.hpp @@ -24,11 +24,11 @@ namespace cv { * an indication of goodness of fit and therefore a measure of how * well unseen samples are likely to be predicted by the model, * through the proportion of explained variance. - * As R2Score is dataset dependent it can have wide range of values, + * As R2Score is dataset dependent it can have wide range of values, * best possible score is @f$R^2 =1.0@f$, and it can be negative too for an * arbitraryly worse model. For a model which predicts exactly the expected * value of y, disregarding the input features, gets a R2Score equals to 0.0. - * If a model predicts @f$ \hat{y}_i $@f of the @f$ i $@f-th sample for a true + * If a model predicts @f$ \hat{y}_i $@f of the @f$ i $@f-th sample for a true * @f$ y_i $@f for total n samples, the R2Score is calculated by * @f{eqnarray*}{ * R^{2} \left( y, \hat{y} \right) &=& 1-\frac{\sum_{i=1}^{n} From ee7206576e8968db900612264dd8d5a289eeb72e Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Mon, 23 Mar 2020 11:27:39 +0530 Subject: [PATCH 06/28] style fixes --- src/mlpack/core/cv/metrics/r2_score.hpp | 13 ++++++++----- src/mlpack/core/cv/metrics/r2_score_impl.hpp | 11 ++++++----- src/mlpack/tests/cv_test.cpp | 11 ++++++----- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/mlpack/core/cv/metrics/r2_score.hpp b/src/mlpack/core/cv/metrics/r2_score.hpp index 7ffb608760..0654899498 100644 --- a/src/mlpack/core/cv/metrics/r2_score.hpp +++ b/src/mlpack/core/cv/metrics/r2_score.hpp @@ -25,15 +25,18 @@ namespace cv { * well unseen samples are likely to be predicted by the model, * through the proportion of explained variance. * As R2Score is dataset dependent it can have wide range of values, - * best possible score is @f$R^2 =1.0@f$, and it can be negative too for an - * arbitraryly worse model. For a model which predicts exactly the expected - * value of y, disregarding the input features, gets a R2Score equals to 0.0. + * best possible score is @f$R^2 =1.0@f$.arbitraryly worse model. Values + * of R2 outside the range 0 to 1 can occur when the model fits the data + * worse than a horizontal hyperplane. This would occur when the wrong model + * was chosen, or nonsensical constraints were applied by mistake. For a model + * which predicts exactly the expected value of y, disregarding the input + * features, gets a R2Score equals to 0.0. * If a model predicts @f$ \hat{y}_i $@f of the @f$ i $@f-th sample for a true * @f$ y_i $@f for total n samples, the R2Score is calculated by * @f{eqnarray*}{ * R^{2} \left( y, \hat{y} \right) &=& 1-\frac{\sum_{i=1}^{n} - * \left( y_i - \hat{y_i} \right)^2 } - * {\sum_{i=1}^{n} \left( y_i - \bar{y}\right)^2}\\ + * \left( y_i - \hat{y_i} \right)^2 } + * {\sum_{i=1}^{n} \left( y_i - \bar{y}\right)^2}\\ * @f} * where @f$ \bar{y} = frac{1}{y}\sum_{i=1}^{n} y_i $@f. * For example, a model having R2Score = 0.85, explains 85 \% variability of diff --git a/src/mlpack/core/cv/metrics/r2_score_impl.hpp b/src/mlpack/core/cv/metrics/r2_score_impl.hpp index 6fecfac378..578c73d840 100644 --- a/src/mlpack/core/cv/metrics/r2_score_impl.hpp +++ b/src/mlpack/core/cv/metrics/r2_score_impl.hpp @@ -16,9 +16,10 @@ namespace mlpack { namespace cv { template -double R2Score::Evaluate(MLAlgorithm& model, - const DataType& data, - const ResponsesType& responses) +double R2Score::Evaluate( + MLAlgorithm& model, + const DataType& data, + const ResponsesType& responses) { if (data.n_cols != responses.n_cols) { @@ -44,13 +45,13 @@ double R2Score::Evaluate(MLAlgorithm& model, // Handling undefined R2Score when both denominator and numerator is 0.0 if (ss_res == 0.0) { - if (ss_tot !=0.0) + if (ss_tot != 0.0) return 1.0; else return DBL_MIN; } - return 1 - ss_res/ss_tot; + return 1 - ss_res / ss_tot; } } // namespace cv diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index 8886f68c4b..b93ea13e3b 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -181,12 +181,13 @@ BOOST_AUTO_TEST_CASE(R2ScoreTest) // Mean Responses = (1 + 2 + 3 + 6 + 8)/5 = 4 arma::mat data("2 3 4 7 9"); arma::rowvec responses("1 2.005 3 6.005 8.005"); - - double ss_reg = (0 * 0 + 0.005 * 0.005 + 0 * 0 - + 0.005 * 0.005 + 0.005 *0.005); + + double ss_reg = (0 * 0 + 0.005 * 0.005 + 0 * 0 + + 0.005 * 0.005 + 0.005 *0.005); double ss_tot = ((1 - 4) * (1- 4) + (2.005 - 4) * - (2.005 - 4) + (3 - 4) * (3 - 4) + (6.005 - 4) * - (6.005 - 4) + (8.005 - 4) * (8.005 -4)); + (2.005 - 4) + (3 - 4) * (3 - 4) + (6.005 - 4) * + (6.005 - 4) + (8.005 - 4) * (8.005 -4)); + double expectedR2 = 1 - ss_reg / ss_tot; BOOST_REQUIRE_CLOSE(R2Score::Evaluate(lr, data, responses), expectedR2, 1e-5); From 2e5110a20daa41a24f16928a25bc790e367dfcbc Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Tue, 24 Mar 2020 09:03:52 +0530 Subject: [PATCH 07/28] Update src/mlpack/core/cv/metrics/r2_score.hpp Co-Authored-By: favre49 <40389657+favre49@users.noreply.github.com> --- src/mlpack/core/cv/metrics/r2_score.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/cv/metrics/r2_score.hpp b/src/mlpack/core/cv/metrics/r2_score.hpp index 0654899498..8a08aa6f74 100644 --- a/src/mlpack/core/cv/metrics/r2_score.hpp +++ b/src/mlpack/core/cv/metrics/r2_score.hpp @@ -25,7 +25,7 @@ namespace cv { * well unseen samples are likely to be predicted by the model, * through the proportion of explained variance. * As R2Score is dataset dependent it can have wide range of values, - * best possible score is @f$R^2 =1.0@f$.arbitraryly worse model. Values + * best possible score is @f$R^2 =1.0@f$. Values * of R2 outside the range 0 to 1 can occur when the model fits the data * worse than a horizontal hyperplane. This would occur when the wrong model * was chosen, or nonsensical constraints were applied by mistake. For a model From 0086654e92aa9ff3955167079a413e5fdb881efa Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Tue, 24 Mar 2020 09:32:47 +0530 Subject: [PATCH 08/28] R2Score tests hardcoded --- src/mlpack/core/cv/metrics/r2_score.hpp | 10 +++++----- src/mlpack/core/cv/metrics/r2_score_impl.hpp | 7 +++---- src/mlpack/tests/cv_test.cpp | 8 +------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/mlpack/core/cv/metrics/r2_score.hpp b/src/mlpack/core/cv/metrics/r2_score.hpp index 8a08aa6f74..b96745937e 100644 --- a/src/mlpack/core/cv/metrics/r2_score.hpp +++ b/src/mlpack/core/cv/metrics/r2_score.hpp @@ -25,11 +25,11 @@ namespace cv { * well unseen samples are likely to be predicted by the model, * through the proportion of explained variance. * As R2Score is dataset dependent it can have wide range of values, - * best possible score is @f$R^2 =1.0@f$. Values - * of R2 outside the range 0 to 1 can occur when the model fits the data - * worse than a horizontal hyperplane. This would occur when the wrong model - * was chosen, or nonsensical constraints were applied by mistake. For a model - * which predicts exactly the expected value of y, disregarding the input + * best possible score is @f$R^2 =1.0@f$. Values of R2 outside the range + * 0 to 1 can occur when the model fits the data worse than a horizontal + * hyperplane. This would occur when the wrong model was chosen, or + * nonsensical constraints were applied by mistake. For a model which + * predicts exactly the expected value of y, disregarding the input * features, gets a R2Score equals to 0.0. * If a model predicts @f$ \hat{y}_i $@f of the @f$ i $@f-th sample for a true * @f$ y_i $@f for total n samples, the R2Score is calculated by diff --git a/src/mlpack/core/cv/metrics/r2_score_impl.hpp b/src/mlpack/core/cv/metrics/r2_score_impl.hpp index 578c73d840..0bc398fd2d 100644 --- a/src/mlpack/core/cv/metrics/r2_score_impl.hpp +++ b/src/mlpack/core/cv/metrics/r2_score_impl.hpp @@ -16,10 +16,9 @@ namespace mlpack { namespace cv { template -double R2Score::Evaluate( - MLAlgorithm& model, - const DataType& data, - const ResponsesType& responses) +double R2Score::Evaluate(MLAlgorithm& model, + const DataType& data, + const ResponsesType& responses) { if (data.n_cols != responses.n_cols) { diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index b93ea13e3b..f896a209d3 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -182,13 +182,7 @@ BOOST_AUTO_TEST_CASE(R2ScoreTest) arma::mat data("2 3 4 7 9"); arma::rowvec responses("1 2.005 3 6.005 8.005"); - double ss_reg = (0 * 0 + 0.005 * 0.005 + 0 * 0 + - 0.005 * 0.005 + 0.005 *0.005); - double ss_tot = ((1 - 4) * (1- 4) + (2.005 - 4) * - (2.005 - 4) + (3 - 4) * (3 - 4) + (6.005 - 4) * - (6.005 - 4) + (8.005 - 4) * (8.005 -4)); - - double expectedR2 = 1 - ss_reg / ss_tot; + double expectedR2 = 0.99999779; BOOST_REQUIRE_CLOSE(R2Score::Evaluate(lr, data, responses), expectedR2, 1e-5); } From 8841e1cb29b0385ac919e8730e718f88dae41d58 Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Tue, 24 Mar 2020 09:41:21 +0530 Subject: [PATCH 09/28] full stop added --- src/mlpack/tests/cv_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index f896a209d3..a1240fe69f 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -170,15 +170,15 @@ BOOST_AUTO_TEST_CASE(MSETest) */ BOOST_AUTO_TEST_CASE(R2ScoreTest) { - // Making two points that define the linear function f(x) = x - 1 + // Making two points that define the linear function f(x) = x - 1. arma::mat trainingData("0 1"); arma::rowvec trainingResponses("-1 0"); LinearRegression lr(trainingData, trainingResponses); // Making five responses that are the output of regression function f(x) - // with some responses having a slight deviation of 0.005 - // Mean Responses = (1 + 2 + 3 + 6 + 8)/5 = 4 + // with some responses having a slight deviation of 0.005. + // Mean Responses = (1 + 2 + 3 + 6 + 8)/5 = 4. arma::mat data("2 3 4 7 9"); arma::rowvec responses("1 2.005 3 6.005 8.005"); From 30577238cc40e0d6015292ae3b96f40beb6b8561 Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Tue, 24 Mar 2020 21:59:57 +0530 Subject: [PATCH 10/28] Apply suggestions from code review Co-Authored-By: favre49 <40389657+favre49@users.noreply.github.com> --- src/mlpack/core/cv/metrics/r2_score.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/cv/metrics/r2_score.hpp b/src/mlpack/core/cv/metrics/r2_score.hpp index b96745937e..26b4fb25d9 100644 --- a/src/mlpack/core/cv/metrics/r2_score.hpp +++ b/src/mlpack/core/cv/metrics/r2_score.hpp @@ -24,11 +24,11 @@ namespace cv { * an indication of goodness of fit and therefore a measure of how * well unseen samples are likely to be predicted by the model, * through the proportion of explained variance. - * As R2Score is dataset dependent it can have wide range of values, + * As R2Score is dataset dependent it can have wide range of values. The * best possible score is @f$R^2 =1.0@f$. Values of R2 outside the range * 0 to 1 can occur when the model fits the data worse than a horizontal * hyperplane. This would occur when the wrong model was chosen, or - * nonsensical constraints were applied by mistake. For a model which + * nonsensical constraints were applied by mistake. A model which * predicts exactly the expected value of y, disregarding the input * features, gets a R2Score equals to 0.0. * If a model predicts @f$ \hat{y}_i $@f of the @f$ i $@f-th sample for a true From af930b79358635a79ff484105e4e1b47b239db32 Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Tue, 24 Mar 2020 22:12:30 +0530 Subject: [PATCH 11/28] Update HISTORY.md --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 9dba6a517d..01e5493f38 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? + * Added `R2Score` regression metric (#2323). + * Added `mean squared logarithmic error` loss function for neural networks (#2210). From cc7a370b31bff9370e2c3bfab8c8111a4dca68ef Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Wed, 25 Mar 2020 10:26:11 +0530 Subject: [PATCH 12/28] updates following review suggestions --- HISTORY.md | 2 +- src/mlpack/core/cv/metrics/r2_score.hpp | 12 +++++++----- src/mlpack/core/cv/metrics/r2_score_impl.hpp | 18 ++++++++---------- src/mlpack/tests/cv_test.cpp | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 01e5493f38..81729e3292 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,6 @@ ### mlpack ?.?.? ###### ????-??-?? - * Added `R2Score` regression metric (#2323). + * Added `R2 Score` regression metric (#2323). * Added `mean squared logarithmic error` loss function for neural networks (#2210). diff --git a/src/mlpack/core/cv/metrics/r2_score.hpp b/src/mlpack/core/cv/metrics/r2_score.hpp index 26b4fb25d9..6fcac955aa 100644 --- a/src/mlpack/core/cv/metrics/r2_score.hpp +++ b/src/mlpack/core/cv/metrics/r2_score.hpp @@ -18,26 +18,27 @@ namespace mlpack { namespace cv { /** - * The R2Score is a metric of performance for regression algorithms - * that represents the proportion of variance (of y) that has been + * The R2 Score is a metric of performance for regression algorithms + * that represents the proportion of variance (here y) that has been * explained by the independent variables in the model. It provides * an indication of goodness of fit and therefore a measure of how * well unseen samples are likely to be predicted by the model, * through the proportion of explained variance. - * As R2Score is dataset dependent it can have wide range of values. The + * As R2 Score is dataset dependent it can have wide range of values. The * best possible score is @f$R^2 =1.0@f$. Values of R2 outside the range * 0 to 1 can occur when the model fits the data worse than a horizontal * hyperplane. This would occur when the wrong model was chosen, or * nonsensical constraints were applied by mistake. A model which * predicts exactly the expected value of y, disregarding the input - * features, gets a R2Score equals to 0.0. + * features, gets a R2 Score equals to 0.0. * If a model predicts @f$ \hat{y}_i $@f of the @f$ i $@f-th sample for a true - * @f$ y_i $@f for total n samples, the R2Score is calculated by + * @f$ y_i $@f for total n samples, the R2 Score is calculated by * @f{eqnarray*}{ * R^{2} \left( y, \hat{y} \right) &=& 1-\frac{\sum_{i=1}^{n} * \left( y_i - \hat{y_i} \right)^2 } * {\sum_{i=1}^{n} \left( y_i - \bar{y}\right)^2}\\ * @f} + * * where @f$ \bar{y} = frac{1}{y}\sum_{i=1}^{n} y_i $@f. * For example, a model having R2Score = 0.85, explains 85 \% variability of * the response data around its mean. @@ -52,6 +53,7 @@ class R2Score * @param data Column-major data containing test items. * @param responses Ground truth (correct) target values for the test items, * should be either a row vector or a column-major matrix. + * @return calculated R2 Score. */ template static double Evaluate(MLAlgorithm& model, diff --git a/src/mlpack/core/cv/metrics/r2_score_impl.hpp b/src/mlpack/core/cv/metrics/r2_score_impl.hpp index 0bc398fd2d..1dd6df2278 100644 --- a/src/mlpack/core/cv/metrics/r2_score_impl.hpp +++ b/src/mlpack/core/cv/metrics/r2_score_impl.hpp @@ -33,24 +33,22 @@ double R2Score::Evaluate(MLAlgorithm& model, // Taking Predicted Output from the model. model.Predict(data, predictedResponses); // Mean value of response. - double mean_responses = arma::mean(responses); + double meanResponses = arma::mean(responses); // Calculate the numerator i.e. residual sum of squares. - double ss_res = arma::accu(arma::square(responses - predictedResponses)); + double residualSumSquared = arma::accu(arma::square(responses - + predictedResponses)); // Calculate the denominator i.e.total sum of squares. - double ss_tot = arma::accu(arma::square(responses - mean_responses)); + double totalSumSquared = arma::accu(arma::square(responses - meanResponses)); - // Handling undefined R2Score when both denominator and numerator is 0.0 - if (ss_res == 0.0) + // Handling undefined R2 Score when both denominator and numerator is 0.0. + if (residualSumSquared == 0.0) { - if (ss_tot != 0.0) - return 1.0; - else - return DBL_MIN; + return totalSumSquared ? 1.0 : DBL_MIN; } - return 1 - ss_res / ss_tot; + return 1 - residualSumSquared / totalSumSquared; } } // namespace cv diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index a1240fe69f..4210065887 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -166,7 +166,7 @@ BOOST_AUTO_TEST_CASE(MSETest) } /** - * Test the R squared metric (R2Score). + * Test the R squared metric (R2 Score). */ BOOST_AUTO_TEST_CASE(R2ScoreTest) { From 75b5e235a6082a06ea6141652cd9707d9ceba8d8 Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Fri, 27 Mar 2020 01:58:19 +0530 Subject: [PATCH 13/28] added asynchronous learning tutorial --- .../reinforcement_learning.txt | 167 +++++++++++++++++- 1 file changed, 166 insertions(+), 1 deletion(-) diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt index 1c837ef070..df9a5ae02f 100644 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -1,6 +1,7 @@ /*! @file rl.txt @author Sriram S K +@author Joel Joseph @brief Tutorial for how to use the Reinforcement Learning module in mlpack. @page rltutorial Reinforcement Learning Tutorial @@ -29,6 +30,7 @@ This tutorial is split into the following sections: - \ref environment_rltut - \ref agent_components_rltut - \ref q_learning_rltut + - \ref async_learning_rltut - \ref further_rltut @section environment_rltut Reinforcement Learning Environments @@ -231,9 +233,172 @@ to have converged when the average return reaches a predetermined value (i.e. > Conversely, if the average return does not go beyond that amount even after a thousand episodes, we can conclude that the agent will not converge and exit the training loop. +@section async_learning_rltut + +In 2016, Researchers at Deepmind and University of Montreal published their paper +"Asynchronous Methods for Deep Reinforcement Learning". In it they described asynchronous +variants of four standard reinforcement learning algorithms: + - One-Step SARSA + - One-Step Q-Learning + - N-Step Q-Learning + - Advantage Actor-Critic(A3C) + +Online RL algorithms and Deep Neural Networks make an unstable combination because of the +non-stationary and correlated nature of online updates. Although this is solved by Experience Replay, +it has several drawbacks: it uses more memory and computation per real interaction; and it requires +off-policy learning algorithms. + +Asynchronous methods, instead of experience replay, asynchronously executes multiple agents +in parallel, on multiple instances of the environment, which solves all the above problems. + +Here, we demonstrate Asynchronous Learning methods in mlpack through the training of an async +agent. Asynchronous learning involves training several agents simultaneously. Here, each of the +agents are referred to as "workers". Currently mlpack has One-Step Q-Learning worker, N-Step +Q-Learning worker and One-Step SARSA worker. + +Lets examine the sample code in chunks. + +Apart from the includes used for the q-learning example, two more have to be included: + +@code +#include +#include +@endcode + +Here we don't use experience replay. And instead of a single policy, we use three different +policies, each corresponding to its worker. Number of workers created, depends on the number of +policies given in the Aggregated Policy. The column vector contains the probability distribution +for each child policy. We should make sure its size is same as the number of policies and the sum +of its elements is equal to 1. + +@code +AggregatedPolicy> policy({GreedyPolicy(0.7, 5000, 0.1), + GreedyPolicy(0.7, 5000, 0.01), + GreedyPolicy(0.7, 5000, 0.5)}, + arma::colvec("0.4 0.3 0.3")); +@endcode + +Now, we will create the "OneStepQLearning" agent. We could have used "NStepQLearning" or "OneStepSarsa" +here according to our requirement. + +@code +OneStepQLearning< + CartPole, decltype(model), ens::AdamUpdate, decltype(policy)> + agent(std::move(config), std::move(model), std::move(policy)); +@endcode + +Here, unlike the Q-Learning example, instead of the entire while loop, we use the Train method of the Asynchronous +Learning class inside a for loop which runs for 100 training episodes. + +@code +for(int i=0;i<100;i++) +{ + agent.Train(measure); +} +@endcode + +What is "measure" here? It can be a lambda function which returns a boolean value(indicating the end of training) +and accepts the episode return(total reward of a deterministic test episode) as parameter. +So, lets create that. + +@code +arma::vec returns(20, arma::fill::zeros); +size_t position = 0; +size_t episode = 0; + +auto measure = [&returns, &position, &episode](double episodeReturn) +{ + if(episode > 10000) return true; + + returns[position++] = episodeReturn; + position = position % returns.n_elem; + episode++; + + std::cout << "Episode No.: " << episode + << "; Episode Return: " << episodeReturn + << "; Average Return: " << arma::mean(returns) << endl; +}; +@endcode + +This will train three different agents on three CPU threads asynchronously and use this data to update the +action value estimate. +Voila, thats all there is to it. + +Here is the full code, to try this right away: + +@code +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace mlpack; +using namespace mlpack::ann; +using namespace mlpack::rl; +int main() +{ + // Set up the network. + FFN, GaussianInitialization> model(MeanSquaredError<>(), + GaussianInitialization(0, 0.001)); + model.Add>(4, 128); + model.Add>(); + model.Add>(128, 128); + model.Add>(); + model.Add>(128, 2); + + AggregatedPolicy> policy({GreedyPolicy(0.7, 5000, 0.1), + GreedyPolicy(0.7, 5000, 0.01), + GreedyPolicy(0.7, 5000, 0.5)}, + arma::colvec("0.4 0.3 0.3")); + + TrainingConfig config; + config.StepSize() = 0.01; + config.Discount() = 0.9; + config.TargetNetworkSyncInterval() = 100; + config.ExplorationSteps() = 100; + config.DoubleQLearning() = false; + config.StepLimit() = 200; + + OneStepQLearning< + CartPole, decltype(model), ens::VanillaUpdate, decltype(policy)> + agent(std::move(config), std::move(model), std::move(policy)); + + arma::vec returns(20, arma::fill::zeros); + size_t position = 0; + size_t episode = 0; + + auto measure = [&returns, &position, &episode](double episodeReturn) + { + if(episode > 10000) return true; + + returns[position++] = episodeReturn; + position = position % returns.n_elem; + episode++; + + std::cout << "Episode No.: " << episode + << "; Episode Return: " << episodeReturn + << "; Average Return: " << arma::mean(returns) << endl; + }; + + for(int i=0;i<100;i++) + { + agent.Train(measure); + } +} +@endcode + +It will train for 100 episodes, which will take around 50 seconds. + @section further_rltut Further documentation For further documentation on the rl classes, consult the \ref mlpack::rl "complete API documentation". -*/ \ No newline at end of file +*/ From dd29a5ef41e6d5a03aec4f078335a7a58ee8817c Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Fri, 27 Mar 2020 10:00:46 +0530 Subject: [PATCH 14/28] Update r2_score_impl.hpp Removing braces on 47-49 --- src/mlpack/core/cv/metrics/r2_score_impl.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mlpack/core/cv/metrics/r2_score_impl.hpp b/src/mlpack/core/cv/metrics/r2_score_impl.hpp index 1dd6df2278..86c57e11fb 100644 --- a/src/mlpack/core/cv/metrics/r2_score_impl.hpp +++ b/src/mlpack/core/cv/metrics/r2_score_impl.hpp @@ -44,9 +44,7 @@ double R2Score::Evaluate(MLAlgorithm& model, // Handling undefined R2 Score when both denominator and numerator is 0.0. if (residualSumSquared == 0.0) - { return totalSumSquared ? 1.0 : DBL_MIN; - } return 1 - residualSumSquared / totalSumSquared; } From d367f0a4bfcc73c6c6f50f603db8c2a4abf9e25f Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Fri, 27 Mar 2020 11:35:40 +0530 Subject: [PATCH 15/28] Update doc/tutorials/reinforcement_learning/reinforcement_learning.txt Co-Authored-By: favre49 <40389657+favre49@users.noreply.github.com> --- doc/tutorials/reinforcement_learning/reinforcement_learning.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt index df9a5ae02f..10cab01a55 100644 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -256,7 +256,7 @@ agent. Asynchronous learning involves training several agents simultaneously. He agents are referred to as "workers". Currently mlpack has One-Step Q-Learning worker, N-Step Q-Learning worker and One-Step SARSA worker. -Lets examine the sample code in chunks. +Let's examine the sample code in chunks. Apart from the includes used for the q-learning example, two more have to be included: From 7f68d6eeaac07a320a508b39fe2b12464d3396cf Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Fri, 27 Mar 2020 11:36:05 +0530 Subject: [PATCH 16/28] Update doc/tutorials/reinforcement_learning/reinforcement_learning.txt Co-Authored-By: favre49 <40389657+favre49@users.noreply.github.com> --- doc/tutorials/reinforcement_learning/reinforcement_learning.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt index 10cab01a55..783097fb8e 100644 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -265,7 +265,7 @@ Apart from the includes used for the q-learning example, two more have to be inc #include @endcode -Here we don't use experience replay. And instead of a single policy, we use three different +Here we don't use experience replay, and instead of a single policy, we use three different policies, each corresponding to its worker. Number of workers created, depends on the number of policies given in the Aggregated Policy. The column vector contains the probability distribution for each child policy. We should make sure its size is same as the number of policies and the sum From ac291035480a78840af4b13f34ffbc90bd52bd68 Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Fri, 27 Mar 2020 11:37:02 +0530 Subject: [PATCH 17/28] Update doc/tutorials/reinforcement_learning/reinforcement_learning.txt Co-Authored-By: favre49 <40389657+favre49@users.noreply.github.com> --- doc/tutorials/reinforcement_learning/reinforcement_learning.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt index 783097fb8e..50d17baad5 100644 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -297,7 +297,7 @@ for(int i=0;i<100;i++) } @endcode -What is "measure" here? It can be a lambda function which returns a boolean value(indicating the end of training) +What is "measure" here? It is a lambda function which returns a boolean value (indicating the end of training) and accepts the episode return(total reward of a deterministic test episode) as parameter. So, lets create that. From 6dcdd941c7700bbfd43d17eebd667488816c3b5b Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Fri, 27 Mar 2020 11:37:12 +0530 Subject: [PATCH 18/28] Update doc/tutorials/reinforcement_learning/reinforcement_learning.txt Co-Authored-By: favre49 <40389657+favre49@users.noreply.github.com> --- doc/tutorials/reinforcement_learning/reinforcement_learning.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt index 50d17baad5..7d450b3b69 100644 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -299,7 +299,7 @@ for(int i=0;i<100;i++) What is "measure" here? It is a lambda function which returns a boolean value (indicating the end of training) and accepts the episode return(total reward of a deterministic test episode) as parameter. -So, lets create that. +So, let's create that. @code arma::vec returns(20, arma::fill::zeros); From 0e7c466d7c8ac3ef58604afe013ccb3da715cf14 Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Fri, 27 Mar 2020 11:37:21 +0530 Subject: [PATCH 19/28] Update doc/tutorials/reinforcement_learning/reinforcement_learning.txt Co-Authored-By: favre49 <40389657+favre49@users.noreply.github.com> --- doc/tutorials/reinforcement_learning/reinforcement_learning.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt index 7d450b3b69..139112662b 100644 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -314,7 +314,7 @@ auto measure = [&returns, &position, &episode](double episodeReturn) position = position % returns.n_elem; episode++; - std::cout << "Episode No.: " << episode + std::cout << "Episode No.: " << episode << "; Episode Return: " << episodeReturn << "; Average Return: " << arma::mean(returns) << endl; }; From ddff7d375bdfe51c748a95b7874127548fbaa511 Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Fri, 27 Mar 2020 11:38:08 +0530 Subject: [PATCH 20/28] Update doc/tutorials/reinforcement_learning/reinforcement_learning.txt Co-Authored-By: favre49 <40389657+favre49@users.noreply.github.com> --- doc/tutorials/reinforcement_learning/reinforcement_learning.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt index 139112662b..8b817cc1e5 100644 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -298,7 +298,7 @@ for(int i=0;i<100;i++) @endcode What is "measure" here? It is a lambda function which returns a boolean value (indicating the end of training) -and accepts the episode return(total reward of a deterministic test episode) as parameter. +and accepts the episode return (total reward of a deterministic test episode) as parameter. So, let's create that. @code From c388fac8ce3bc916ea46bafb531cb2b1bd7d3009 Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Fri, 27 Mar 2020 12:04:13 +0530 Subject: [PATCH 21/28] fixed the styling issues at lines 286,294 and 397 --- .../reinforcement_learning.txt | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt index 8b817cc1e5..ed745c0ccd 100644 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -282,16 +282,15 @@ Now, we will create the "OneStepQLearning" agent. We could have used "NStepQLear here according to our requirement. @code -OneStepQLearning< - CartPole, decltype(model), ens::AdamUpdate, decltype(policy)> +OneStepQLearning agent(std::move(config), std::move(model), std::move(policy)); @endcode Here, unlike the Q-Learning example, instead of the entire while loop, we use the Train method of the Asynchronous -Learning class inside a for loop which runs for 100 training episodes. +Learning class inside a for loop. 100 training episodes will take around 50 seconds. @code -for(int i=0;i<100;i++) +for (int i = 0; i < 100; i++) { agent.Train(measure); } @@ -315,8 +314,8 @@ auto measure = [&returns, &position, &episode](double episodeReturn) episode++; std::cout << "Episode No.: " << episode - << "; Episode Return: " << episodeReturn - << "; Average Return: " << arma::mean(returns) << endl; + << "; Episode Return: " << episodeReturn + << "; Average Return: " << arma::mean(returns) << endl; }; @endcode @@ -324,7 +323,7 @@ This will train three different agents on three CPU threads asynchronously and u action value estimate. Voila, thats all there is to it. -Here is the full code, to try this right away: +Here is the full code to try this right away: @code #include @@ -394,8 +393,6 @@ int main() } @endcode -It will train for 100 episodes, which will take around 50 seconds. - @section further_rltut Further documentation For further documentation on the rl classes, consult the \ref mlpack::rl From e7e2e86fd50b6bd601930594c345281496a0dc51 Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Fri, 27 Mar 2020 12:10:07 +0530 Subject: [PATCH 22/28] fixed styling issues of the test code --- .../reinforcement_learning.txt | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt index ed745c0ccd..dc8c066553 100644 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -344,8 +344,7 @@ using namespace mlpack::rl; int main() { // Set up the network. - FFN, GaussianInitialization> model(MeanSquaredError<>(), - GaussianInitialization(0, 0.001)); + FFN, GaussianInitialization> model(MeanSquaredError<>(), GaussianInitialization(0, 0.001)); model.Add>(4, 128); model.Add>(); model.Add>(128, 128); @@ -365,9 +364,8 @@ int main() config.DoubleQLearning() = false; config.StepLimit() = 200; - OneStepQLearning< - CartPole, decltype(model), ens::VanillaUpdate, decltype(policy)> - agent(std::move(config), std::move(model), std::move(policy)); + OneStepQLearning + agent(std::move(config), std::move(model), std::move(policy)); arma::vec returns(20, arma::fill::zeros); size_t position = 0; @@ -382,11 +380,11 @@ int main() episode++; std::cout << "Episode No.: " << episode - << "; Episode Return: " << episodeReturn - << "; Average Return: " << arma::mean(returns) << endl; + << "; Episode Return: " << episodeReturn + << "; Average Return: " << arma::mean(returns) << endl; }; - for(int i=0;i<100;i++) + for (int i = 0; i < 100; i++) { agent.Train(measure); } From 82e52d762330d77e790d4d633653637d4c5d5406 Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Sat, 28 Mar 2020 11:22:23 +0530 Subject: [PATCH 23/28] Update COPYRIGHT.txt --- COPYRIGHT.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index ce5c944e0d..db423bb3b8 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -130,6 +130,7 @@ Copyright: Copyright 2020, Saraansh Tandon Copyright 2020, Gaurav Singh Copyright 2020, Lakshya Ojha + Copyright 2020, Bisakh Mondal License: BSD-3-clause All rights reserved. From 2f23fe84bbceb9717540fc8ffb80d0c8e69ef086 Mon Sep 17 00:00:00 2001 From: Mrityunjay Tripathi Date: Wed, 25 Mar 2020 17:34:29 +0530 Subject: [PATCH 24/28] templating return type of loss functions --- HISTORY.md | 2 ++ .../ann/loss_functions/cross_entropy_error.hpp | 3 ++- .../loss_functions/cross_entropy_error_impl.hpp | 6 ++++-- .../methods/ann/loss_functions/dice_loss.hpp | 3 ++- .../ann/loss_functions/dice_loss_impl.hpp | 5 +++-- .../ann/loss_functions/earth_mover_distance.hpp | 3 ++- .../earth_mover_distance_impl.hpp | 6 ++++-- .../ann/loss_functions/hinge_embedding_loss.hpp | 3 ++- .../hinge_embedding_loss_impl.hpp | 6 ++++-- .../methods/ann/loss_functions/huber_loss.hpp | 3 ++- .../ann/loss_functions/huber_loss_impl.hpp | 17 +++++++++++------ .../ann/loss_functions/kl_divergence.hpp | 3 ++- .../ann/loss_functions/kl_divergence_impl.hpp | 5 +++-- .../ann/loss_functions/log_cosh_loss.hpp | 5 +++-- .../ann/loss_functions/log_cosh_loss_impl.hpp | 5 +++-- .../ann/loss_functions/mean_bias_error.hpp | 3 ++- .../ann/loss_functions/mean_bias_error_impl.hpp | 5 +++-- .../ann/loss_functions/mean_squared_error.hpp | 3 ++- .../loss_functions/mean_squared_error_impl.hpp | 6 ++++-- .../mean_squared_logarithmic_error.hpp | 3 ++- .../mean_squared_logarithmic_error_impl.hpp | 6 ++++-- .../loss_functions/negative_log_likelihood.hpp | 3 ++- .../negative_log_likelihood_impl.hpp | 9 ++++++--- .../ann/loss_functions/reconstruction_loss.hpp | 3 ++- .../loss_functions/reconstruction_loss_impl.hpp | 3 ++- .../sigmoid_cross_entropy_error.hpp | 4 ++-- .../sigmoid_cross_entropy_error_impl.hpp | 9 ++++++--- 27 files changed, 86 insertions(+), 46 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 9dba6a517d..849422955a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? + * Templated return type of `Forward function` of loss functions (#2339). + * Added `mean squared logarithmic error` loss function for neural networks (#2210). diff --git a/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp index 3041ae3099..1811f99998 100644 --- a/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp @@ -49,7 +49,8 @@ class CrossEntropyError * @param target The target vector. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. diff --git a/src/mlpack/methods/ann/loss_functions/cross_entropy_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/cross_entropy_error_impl.hpp index 9c6d73e481..7d0d80357b 100644 --- a/src/mlpack/methods/ann/loss_functions/cross_entropy_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/cross_entropy_error_impl.hpp @@ -27,8 +27,10 @@ CrossEntropyError::CrossEntropyError( template template -double CrossEntropyError::Forward( - const InputType& input, const TargetType& target) +typename InputType::elem_type +CrossEntropyError::Forward( + const InputType& input, + const TargetType& target) { return -arma::accu(target % arma::log(input + eps) + (1. - target) % arma::log(1. - input + eps)); diff --git a/src/mlpack/methods/ann/loss_functions/dice_loss.hpp b/src/mlpack/methods/ann/loss_functions/dice_loss.hpp index a30618e424..42551388aa 100644 --- a/src/mlpack/methods/ann/loss_functions/dice_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/dice_loss.hpp @@ -62,7 +62,8 @@ class DiceLoss * @param target The target vector. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. diff --git a/src/mlpack/methods/ann/loss_functions/dice_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/dice_loss_impl.hpp index 708c1102f7..904d699c21 100644 --- a/src/mlpack/methods/ann/loss_functions/dice_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/dice_loss_impl.hpp @@ -27,8 +27,9 @@ DiceLoss::DiceLoss( template template -double DiceLoss::Forward( - const InputType& input, const TargetType& target) +typename InputType::elem_type DiceLoss::Forward( + const InputType& input, + const TargetType& target) { return 1 - ((2 * arma::accu(target % input) + smooth) / (arma::accu(target % target) + arma::accu( diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp index 0683715bd4..71c0aa8e4b 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp @@ -45,7 +45,8 @@ class EarthMoverDistance * @param target The target vector. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp index 9ba7ac8f47..44d5e24c4e 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp @@ -26,8 +26,10 @@ EarthMoverDistance::EarthMoverDistance() template template -double EarthMoverDistance::Forward( - const InputType& input, const TargetType& target) +typename InputType::elem_type +EarthMoverDistance::Forward( + const InputType& input, + const TargetType& target) { return -arma::accu(target % input); } diff --git a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp index bcb9b17d6d..3e3eac19ec 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp @@ -48,7 +48,8 @@ class HingeEmbeddingLoss * @param target Target data to compare with. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. diff --git a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp index b0e71632cb..f3f420b3ca 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp @@ -27,8 +27,10 @@ HingeEmbeddingLoss::HingeEmbeddingLoss() template template -double HingeEmbeddingLoss::Forward( - const InputType& input, const TargetType& target) +typename InputType::elem_type +HingeEmbeddingLoss::Forward( + const InputType& input, + const TargetType& target) { TargetType temp = target - (target == 0); return (arma::accu(arma::max(1-input % temp, 0.))) / target.n_elem; diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp index 9a18cfea1b..60e8d96b10 100644 --- a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp @@ -52,7 +52,8 @@ class HuberLoss * @param target The target vector. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp index 25da492bcf..c9c5ee9145 100644 --- a/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp @@ -30,13 +30,15 @@ HuberLoss::HuberLoss( template template -double HuberLoss::Forward( - const InputType& input, const TargetType& target) +typename InputType::elem_type +HuberLoss::Forward(const InputType& input, + const TargetType& target) { - double loss = 0; + typedef typename InputType::elem_type ElemType; + ElemType loss = 0; for (size_t i = 0; i < input.n_elem; ++i) { - const double absError = std::abs(target[i] - input[i]); + const ElemType absError = std::abs(target[i] - input[i]); loss += absError > delta ? delta * (absError - 0.5 * delta) : 0.5 * std::pow(absError, 2); } @@ -50,13 +52,16 @@ void HuberLoss::Backward( const TargetType& target, OutputType& output) { + typedef typename InputType::elem_type ElemType; + output.set_size(size(input)); for (size_t i = 0; i < output.n_elem; ++i) { - const double absError = std::abs(target[i] - input[i]); + const ElemType absError = std::abs(target[i] - input[i]); output[i] = absError > delta ? - delta * (target[i] - input[i]) / absError : input[i] - target[i]; - if (mean) output[i] /= output.n_elem; + if (mean) + output[i] /= output.n_elem; } } diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp index ad39d16c62..4640dce43e 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp @@ -60,7 +60,8 @@ class KLDivergence * @param target Target data to compare with. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp index dc72dc1645..bc9b44ea09 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp @@ -28,8 +28,9 @@ KLDivergence::KLDivergence(const bool takeMean) : template template -double KLDivergence::Forward( - const InputType& input, const TargetType& target) +typename InputType::elem_type +KLDivergence::Forward(const InputType& input, + const TargetType& target) { if (takeMean) { diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp index d528157b11..2fc859c5c7 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp @@ -20,7 +20,7 @@ namespace ann /** Artificial Neural Network. */ { /** * The Log-Hyperbolic-Cosine loss function is often used to improve - * variational auto encoder. This function is the log of hyperbolic + * variational auto encoder. This function is the log of hyperbolic * cosine of difference between true values and predicted values. * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, @@ -55,7 +55,8 @@ class LogCoshLoss * @param target Target data to compare with. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp index 5de18340e6..1fd13c922f 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp @@ -28,8 +28,9 @@ LogCoshLoss::LogCoshLoss(const double a) : template template -double LogCoshLoss::Forward( - const InputType& input, const TargetType& target) +typename InputType::elem_type +LogCoshLoss::Forward(const InputType& input, + const TargetType& target) { return arma::accu(arma::log(arma::cosh(a * (target - input)))) / a; } diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp index 40418980d3..a238ac50cb 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp @@ -45,7 +45,8 @@ class MeanBiasError * @param target The target vector. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp index 8488ae487a..9d014585f4 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp @@ -27,8 +27,9 @@ MeanBiasError::MeanBiasError() template template -double MeanBiasError::Forward( - const InputType& input, const TargetType& target) +typename InputType::elem_type +MeanBiasError::Forward(const InputType& input, + const TargetType& target) { return arma::accu(target - input) / target.n_cols; } diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp index 6dc6642a0d..0315c10b5c 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp @@ -46,7 +46,8 @@ class MeanSquaredError * @param target The target vector. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp index d7203b2499..81cf2281cf 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp @@ -26,8 +26,10 @@ MeanSquaredError::MeanSquaredError() template template -double MeanSquaredError::Forward( - const InputType& input, const TargetType& target) +typename InputType::elem_type +MeanSquaredError::Forward( + const InputType& input, + const TargetType& target) { return arma::accu(arma::square(input - target)) / target.n_cols; } diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp index 54b74d17d5..49b94d1d4f 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp @@ -45,7 +45,8 @@ class MeanSquaredLogarithmicError * @param target The target vector. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp index 4a2aa75d8a..92ead103a7 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp @@ -27,8 +27,10 @@ MeanSquaredLogarithmicError template template -double MeanSquaredLogarithmicError::Forward( - const InputType& input, const TargetType& target) +typename InputType::elem_type +MeanSquaredLogarithmicError::Forward( + const InputType& input, + const TargetType& target) { return arma::accu(arma::square(arma::log(1. + target) - arma::log(1. + input))) / target.n_cols; diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp index 200fa1c5f3..6c28321cb9 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp @@ -48,7 +48,8 @@ class NegativeLogLikelihood * between 1 and the number of classes. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. The negative log diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp index d006b17912..1eb47280c5 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp @@ -26,10 +26,13 @@ NegativeLogLikelihood::NegativeLogLikelihood() template template -double NegativeLogLikelihood::Forward( - const InputType& input, const TargetType& target) +typename InputType::elem_type +NegativeLogLikelihood::Forward( + const InputType& input, + const TargetType& target) { - double output = 0; + typedef typename InputType::elem_type ElemType; + ElemType output = 0; for (size_t i = 0; i < input.n_cols; ++i) { size_t currentTarget = target(i) - 1; diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp index d8c775efc0..7d7c8e7da6 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp @@ -49,7 +49,8 @@ class ReconstructionLoss * @param target The target matrix. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp index 02ea50f4a7..b47d5bfcdb 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp @@ -30,7 +30,8 @@ ReconstructionLoss< template template -double ReconstructionLoss::Forward( +typename InputType::elem_type +ReconstructionLoss::Forward( const InputType& input, const TargetType& target) { dist = DistType(input); diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp index fec584de8c..0d70d2d29d 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp @@ -64,8 +64,8 @@ class SigmoidCrossEntropyError * @param target The target vector. */ template - inline double Forward(const InputType& input, - const TargetType& target); + inline typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. * diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp index 1ab976874a..e5cf69188c 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp @@ -29,10 +29,13 @@ SigmoidCrossEntropyError template template -inline double SigmoidCrossEntropyError::Forward( - const InputType& input, const TargetType& target) +inline typename InputType::elem_type +SigmoidCrossEntropyError::Forward( + const InputType& input, + const TargetType& target) { - double maximum = 0; + typedef typename InputType::elem_type ElemType; + ElemType maximum = 0; for (size_t i = 0; i < input.n_elem; ++i) { maximum += std::max(input[i], 0.0) + From b7108a3a86f4466ac483b76b7ced0e4832bfb85c Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Wed, 1 Apr 2020 18:13:18 +0530 Subject: [PATCH 25/28] removed the 'example' code i will do another pull request at mlpack/examples to put it there. --- .../reinforcement_learning.txt | 68 +------------------ 1 file changed, 1 insertion(+), 67 deletions(-) diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt index dc8c066553..9fda6c258f 100644 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -323,73 +323,7 @@ This will train three different agents on three CPU threads asynchronously and u action value estimate. Voila, thats all there is to it. -Here is the full code to try this right away: - -@code -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace mlpack; -using namespace mlpack::ann; -using namespace mlpack::rl; -int main() -{ - // Set up the network. - FFN, GaussianInitialization> model(MeanSquaredError<>(), GaussianInitialization(0, 0.001)); - model.Add>(4, 128); - model.Add>(); - model.Add>(128, 128); - model.Add>(); - model.Add>(128, 2); - - AggregatedPolicy> policy({GreedyPolicy(0.7, 5000, 0.1), - GreedyPolicy(0.7, 5000, 0.01), - GreedyPolicy(0.7, 5000, 0.5)}, - arma::colvec("0.4 0.3 0.3")); - - TrainingConfig config; - config.StepSize() = 0.01; - config.Discount() = 0.9; - config.TargetNetworkSyncInterval() = 100; - config.ExplorationSteps() = 100; - config.DoubleQLearning() = false; - config.StepLimit() = 200; - - OneStepQLearning - agent(std::move(config), std::move(model), std::move(policy)); - - arma::vec returns(20, arma::fill::zeros); - size_t position = 0; - size_t episode = 0; - - auto measure = [&returns, &position, &episode](double episodeReturn) - { - if(episode > 10000) return true; - - returns[position++] = episodeReturn; - position = position % returns.n_elem; - episode++; - - std::cout << "Episode No.: " << episode - << "; Episode Return: " << episodeReturn - << "; Average Return: " << arma::mean(returns) << endl; - }; - - for (int i = 0; i < 100; i++) - { - agent.Train(measure); - } -} -@endcode +If you want the example code to try this right away, see [mlpack/examples](https://github.com/mlpack/examples) @section further_rltut Further documentation From 1f9429cc5184cabbb299e5a84a48e0d0a3564402 Mon Sep 17 00:00:00 2001 From: Joel Joseph <34275997+joeljosephjin@users.noreply.github.com> Date: Wed, 1 Apr 2020 19:45:01 +0530 Subject: [PATCH 26/28] reverted the last commit --- .../reinforcement_learning.txt | 68 ++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt index 9fda6c258f..dc8c066553 100644 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -323,7 +323,73 @@ This will train three different agents on three CPU threads asynchronously and u action value estimate. Voila, thats all there is to it. -If you want the example code to try this right away, see [mlpack/examples](https://github.com/mlpack/examples) +Here is the full code to try this right away: + +@code +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace mlpack; +using namespace mlpack::ann; +using namespace mlpack::rl; +int main() +{ + // Set up the network. + FFN, GaussianInitialization> model(MeanSquaredError<>(), GaussianInitialization(0, 0.001)); + model.Add>(4, 128); + model.Add>(); + model.Add>(128, 128); + model.Add>(); + model.Add>(128, 2); + + AggregatedPolicy> policy({GreedyPolicy(0.7, 5000, 0.1), + GreedyPolicy(0.7, 5000, 0.01), + GreedyPolicy(0.7, 5000, 0.5)}, + arma::colvec("0.4 0.3 0.3")); + + TrainingConfig config; + config.StepSize() = 0.01; + config.Discount() = 0.9; + config.TargetNetworkSyncInterval() = 100; + config.ExplorationSteps() = 100; + config.DoubleQLearning() = false; + config.StepLimit() = 200; + + OneStepQLearning + agent(std::move(config), std::move(model), std::move(policy)); + + arma::vec returns(20, arma::fill::zeros); + size_t position = 0; + size_t episode = 0; + + auto measure = [&returns, &position, &episode](double episodeReturn) + { + if(episode > 10000) return true; + + returns[position++] = episodeReturn; + position = position % returns.n_elem; + episode++; + + std::cout << "Episode No.: " << episode + << "; Episode Return: " << episodeReturn + << "; Average Return: " << arma::mean(returns) << endl; + }; + + for (int i = 0; i < 100; i++) + { + agent.Train(measure); + } +} +@endcode @section further_rltut Further documentation From b60c33bae4090b58b467c5d874653a07070e7444 Mon Sep 17 00:00:00 2001 From: favre49 <40389657+favre49@users.noreply.github.com> Date: Thu, 2 Apr 2020 10:34:09 +0530 Subject: [PATCH 27/28] Update doc/tutorials/reinforcement_learning/reinforcement_learning.txt --- doc/tutorials/reinforcement_learning/reinforcement_learning.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt index dc8c066553..a91dc27671 100644 --- a/doc/tutorials/reinforcement_learning/reinforcement_learning.txt +++ b/doc/tutorials/reinforcement_learning/reinforcement_learning.txt @@ -379,7 +379,7 @@ int main() position = position % returns.n_elem; episode++; - std::cout << "Episode No.: " << episode + std::cout << "Episode No.: " << episode << "; Episode Return: " << episodeReturn << "; Average Return: " << arma::mean(returns) << endl; }; From 5ff26b029f88453c910f7784c868c866cb847a3c Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Fri, 3 Apr 2020 15:41:11 +0530 Subject: [PATCH 28/28] Windows build updated --- .ci/ci.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.ci/ci.yaml b/.ci/ci.yaml index 739ba9906c..e0654b49e1 100644 --- a/.ci/ci.yaml +++ b/.ci/ci.yaml @@ -57,23 +57,6 @@ jobs: steps: - template: macos-steps.yaml -- job: WindowsVS14 - timeoutInMinutes: 360 - displayName: Windows VS14 - pool: - vmImage: vs2015-win2012r2 - strategy: - matrix: - Plain: - CMakeArgs: '-DDEBUG=ON -DPROFILE=OFF -DBUILD_PYTHON_BINDINGS=OFF' - CMakeGenerator: '-G "Visual Studio 14 2015 Win64"' - MSBuildVersion: '14.0' - ArchiveNoLibs: 'mlpack-windows-vs14-no-libs.zip' - ArchiveLibs: 'mlpack-windows-vs14.zip' - ArchiveTests: 'mlpack_test-vs14.xml' - steps: - - template: windows-steps.yaml - - job: WindowsVS15 timeoutInMinutes: 360 displayName: Windows VS15