From 3716a369e8445afda140f3dc731b370fd5406890 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Wed, 2 Feb 2022 21:57:47 +0530 Subject: [PATCH 01/34] L1 Loss. --- .../methods/ann/loss_functions/l1_loss.hpp | 25 +++++++++------ .../ann/loss_functions/l1_loss_impl.hpp | 18 +++++++---- src/mlpack/tests/loss_functions_test.cpp | 32 +++++++------------ 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/l1_loss.hpp b/src/mlpack/methods/ann/loss_functions/l1_loss.hpp index 552089bbd0..0256e532d2 100644 --- a/src/mlpack/methods/ann/loss_functions/l1_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/l1_loss.hpp @@ -18,8 +18,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * The L1 loss is a loss function that measures the mean absolute error (MAE) - * between each element in the input x and target y + * The L1 loss is a loss function that measures the mean absolute error (MAE) + * between each element in the input x and target y. * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). @@ -36,8 +36,12 @@ class L1Loss /** * Create the L1Loss object. * - * @param mean Reduction type. If true, it returns the mean of - * the loss. Else, it returns the sum. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. + * */ L1Loss(const bool mean = true); @@ -70,10 +74,10 @@ class L1Loss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the value of reduction type. - bool Mean() const { return mean; } - //! Set the value of reduction type. - bool& Mean() { return mean; } + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } /** * Serialize the layer. @@ -85,8 +89,9 @@ class L1Loss //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Reduction type. If true, performs mean of loss else sum. - bool mean; + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class L1Loss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp index 100823ad90..74a7ce987e 100644 --- a/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp @@ -19,8 +19,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -L1Loss::L1Loss(const bool mean): - mean(mean) +L1Loss::L1Loss(const bool reduction): + reduction(reduction) { // Nothing to do here. } @@ -32,10 +32,13 @@ L1Loss::Forward( const PredictionType& prediction, const TargetType& target) { - if (mean) - return arma::accu(arma::mean(prediction - target)); + PredictionType loss = arma::abs(prediction - target); + typename PredictionType::elem_type lossSum = arma::accu(loss); - return arma::accu(prediction - target); + if (reduction) + return lossSum; + + return lossSum / prediction.n_elem; } template @@ -46,6 +49,9 @@ void L1Loss::Backward( LossType& loss) { loss = arma::sign(prediction - target); + + if (!reduction) + loss = loss / prediction.n_elem; } template @@ -54,7 +60,7 @@ void L1Loss::serialize( Archive& ar, const uint32_t /* version */) { - ar(CEREAL_NVP(mean)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 37467c3119..6b3c3a33d1 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -667,35 +667,25 @@ TEST_CASE("HingeEmbeddingLossTest", "[LossFunctionsTest]") */ TEST_CASE("SimpleL1LossTest", "[LossFunctionsTest]") { - arma::mat input1, input2, output, target1, target2; - L1Loss<> module(false); + arma::mat input, output, target; + double loss; + L1Loss<> module(true); // Test the Forward function on a user generator input and compare it against // the manually calculated result. - input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5"); - target1 = arma::zeros(1, 7); - double error1 = module.Forward(input1, target1); - REQUIRE(error1 == 3.5); - - input2 = arma::mat("0 1 1 0 1 0 0 1"); - target2 = arma::mat("0 1 1 0 1 0 0 1"); - double error2 = module.Forward(input2, target2); - REQUIRE(error2 == Approx(0.0).epsilon(1e-5)); + input = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5"); + target = arma::zeros(1, 7); + loss = module.Forward(input, target); + // Value calculated using torch.nn.L1Loss(reduction='sum'). + REQUIRE(loss == 3.5); // Test the Backward function. - module.Backward(input1, target1, output); + module.Backward(input, target, output); for (double el : output) REQUIRE(el == 1); - REQUIRE(output.n_rows == input1.n_rows); - REQUIRE(output.n_cols == input1.n_cols); - - module.Backward(input2, target2, output); - for (double el : output) - REQUIRE(el == 0); - - REQUIRE(output.n_rows == input2.n_rows); - REQUIRE(output.n_cols == input2.n_cols); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); } /** From 57d489a2c8e997a9aa68539f880502a0a461385e Mon Sep 17 00:00:00 2001 From: Anwaar Date: Thu, 3 Feb 2022 11:40:01 +0530 Subject: [PATCH 02/34] Mean Bias Error. --- .../ann/loss_functions/mean_bias_error.hpp | 20 ++++++- .../loss_functions/mean_bias_error_impl.hpp | 20 +++++-- src/mlpack/tests/loss_functions_test.cpp | 54 +++++++++++-------- 3 files changed, 67 insertions(+), 27 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp index b9836f856f..244abc5d3d 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp @@ -35,8 +35,16 @@ class MeanBiasError public: /** * Create the MeanBiasError object. + * + * @param reduction Specifies the reduction to apply to + * the output. If false, 'mean' reduction + * is used, where sum of the output will + * be divided by the number of elements + * in the output. If true, 'sum' reduction + * is used and the output will be summed. + * It is set to true by default. */ - MeanBiasError(); + MeanBiasError(const bool reduction = true); /** * Computes the mean bias error function. @@ -67,6 +75,12 @@ class MeanBiasError //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + //! Get the type of reduction used. + bool Reduction() const {return reduction; } + + //! Modify the type of reduction used. + bool& Reduction() {return reduction; } + /** * Serialize the layer. */ @@ -76,6 +90,10 @@ class MeanBiasError private: //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! The boolen value that tells if reduction + //! is 'sum' or 'mean'. + bool reduction; }; // class MeanBiasError } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp index 0343558693..b6cec44f46 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp @@ -20,9 +20,10 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -MeanBiasError::MeanBiasError() +MeanBiasError:: + MeanBiasError(const bool reduction) : reduction(reduction) { - // Nothing to do here. + // Nothing to do here } template @@ -32,7 +33,13 @@ MeanBiasError::Forward( const PredictionType& prediction, const TargetType& target) { - return arma::accu(target - prediction) / target.n_cols; + PredictionType loss = target - prediction; + typename PredictionType::elem_type lossSum = arma::accu(loss); + + if (reduction) + return lossSum; + + return lossSum/ prediction.n_elem; } template @@ -44,15 +51,18 @@ void MeanBiasError::Backward( { loss.set_size(arma::size(prediction)); loss.fill(-1.0); + + if (!reduction) + loss = loss / prediction.n_elem; } template template void MeanBiasError::serialize( - Archive& /* ar */, + Archive& ar, const uint32_t /* version */) { - // Nothing to do here. + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 6b3c3a33d1..efccd0d814 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -551,37 +551,49 @@ TEST_CASE("DiceLossTest", "[LossFunctionsTest]") */ TEST_CASE("SimpleMeanBiasErrorTest", "[LossFunctionsTest]") { - arma::mat input, output, target; + arma::mat input, target, output; + double loss; MeanBiasError<> module; - // Test the Forward function on a user generator input and compare it against - // the manually calculated result. - input = arma::mat("1.0 0.0 1.0 -1.0 -1.0 0.0 -1.0 0.0"); - target = arma::zeros(1, 8); - double error = module.Forward(input, target); - REQUIRE(error == 0.125); + // Test for sum reduction. + input = arma::mat("-0.0494 -1.1958 -1.0486 -0.2121 1.6028 0.0737 -0.7091 " + "0.8612 0.9639 0.9648 0.0745 0.5924"); + target = arma::mat("0.4316 0.0164 -0.4478 1.1452 0.5106 0.9255 0.5571 0.0864 " + "0.7059 -0.8288 -0.0231 -1.0526"); - // Test the Backward function. + input.reshape(4, 3); + target.reshape(4, 3); + + // Test the forward function. + // Loss should be 0.1081. + loss = module.Forward(input, target); + REQUIRE(loss == Approx(0.1081).epsilon(1e-5)); + + // Test the backward function. module.Backward(input, target, output); - // We should get a vector with -1 everywhere. - for (double el : output) - { + + for(double el : output) REQUIRE(el == -1); - } REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); - // Test the error function on a single input. - input = arma::mat("2"); - target = arma::mat("3"); - error = module.Forward(input, target); - REQUIRE(error == 1.0); + // Test for mean reduction by modifying + // reduction parameter using accessor. + module.Reduction() = false; - // Test the Backward function on a single input. + // Test the forward function + // loss should be 0.00900833 + loss = module.Forward(input, target); + REQUIRE(loss == Approx(0.00900833).epsilon(1e-5)); + + // Test the backward function module.Backward(input, target, output); - // Test whether the output is negative. - REQUIRE(arma::accu(output) == -1); - REQUIRE(output.n_elem == 1); + + for(double el : output) + REQUIRE(el == Approx(-0.0833).epsilon(1e-3)); + REQUIRE(arma::accu(output) == Approx(-1).epsilon(1e-5)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); } /** From e20d03d8eb4800beb769cb4d467dd0aad0124c59 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Thu, 3 Feb 2022 13:08:43 +0530 Subject: [PATCH 03/34] change mean to reduction in L1 Loss. --- src/mlpack/methods/ann/loss_functions/l1_loss.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/loss_functions/l1_loss.hpp b/src/mlpack/methods/ann/loss_functions/l1_loss.hpp index 0256e532d2..ec47252fcd 100644 --- a/src/mlpack/methods/ann/loss_functions/l1_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/l1_loss.hpp @@ -43,7 +43,7 @@ class L1Loss * is set to true by default. * */ - L1Loss(const bool mean = true); + L1Loss(const bool reduction = true); /** * Computes the L1 Loss function. From ad69821f601e0088f057914b14df2df4a93d0d0e Mon Sep 17 00:00:00 2001 From: Anwaar Date: Thu, 3 Feb 2022 14:02:15 +0530 Subject: [PATCH 04/34] Hinge Embedding Loss --- .../loss_functions/hinge_embedding_loss.hpp | 17 ++++++- .../hinge_embedding_loss_impl.hpp | 22 ++++++--- src/mlpack/tests/loss_functions_test.cpp | 49 ++++++++++++------- 3 files changed, 62 insertions(+), 26 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp index 99e5d50ff2..3159e2f014 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp @@ -38,8 +38,14 @@ class HingeEmbeddingLoss public: /** * Create the Hinge Embedding object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - HingeEmbeddingLoss(); + HingeEmbeddingLoss(const bool reduction = true); /** * Computes the Hinge Embedding loss function. @@ -70,6 +76,11 @@ class HingeEmbeddingLoss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + //! Get the type of reduction used. + bool Reduction() const {return reduction; } + //! Modify the type of reduction used. + bool& Reduction() {return reduction; } + /** * Serialize the loss function. */ @@ -79,6 +90,10 @@ class HingeEmbeddingLoss private: //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean values that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class HingeEmbeddingLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp index 1f6456c71f..39be4aa738 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp @@ -20,7 +20,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -HingeEmbeddingLoss::HingeEmbeddingLoss() +HingeEmbeddingLoss + ::HingeEmbeddingLoss(const bool reduction) : reduction(reduction) { // Nothing to do here. } @@ -32,8 +33,13 @@ HingeEmbeddingLoss::Forward( const PredictionType& prediction, const TargetType& target) { - TargetType temp = target - (target == 0); - return (arma::accu(arma::max(1 - prediction % temp, 0.))) / target.n_elem; + PredictionType loss = (1 - target) / 2 + prediction % (target); + typename PredictionType::elem_type lossSum = arma::accu(loss); + + if (reduction) + return lossSum; + + return lossSum / prediction.n_elem; } template @@ -43,17 +49,19 @@ void HingeEmbeddingLoss::Backward( const TargetType& target, LossType& loss) { - TargetType temp = target - (target == 0); - loss = (prediction < 1 / temp) % -temp; + loss = target; + + if (!reduction) + loss = loss / prediction.n_elem; } template template void HingeEmbeddingLoss::serialize( - Archive& /* ar */, + Archive& ar, const uint32_t /* version */) { - // Nothing to do here. + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index efccd0d814..1d2491e0ae 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -640,38 +640,51 @@ TEST_CASE("LogCoshLossTest", "[LossFunctionsTest]") */ TEST_CASE("HingeEmbeddingLossTest", "[LossFunctionsTest]") { - arma::mat input, target, output; + arma::mat input, target, output, expectedOutput; double loss; HingeEmbeddingLoss<> module; - // Test the Forward function. Loss should be 0 if input = target. - input = arma::ones(10, 1); - target = arma::ones(10, 1); + // Test for sum reduction + input = arma::mat("0.1778 0.0957 0.1397 0.2256 0.1203 0.2403 0.1925 0.3144 " + "-0.2264 -0.3400 -0.3336 -0.8695"); + target = arma::mat("1 1 -1 1 1 -1 1 1 -1 1 1 1"); + expectedOutput = arma::mat("1 1 -1 1 1 -1 1 1 -1 1 1 1"); + input.reshape(4, 3); + target.reshape(4, 3); + expectedOutput.reshape(4, 3); + + // Test the forward function + // Loss should be 2.4296 + // Value calculated using torch.nn.HingeEmbeddingLoss(reduction='sum') loss = module.Forward(input, target); - REQUIRE(loss == 0); + REQUIRE(loss == Approx(2.4296).epsilon(1e-3)); - // Test the Backward function for input = target. + // Test the Backward function module.Backward(input, target, output); - for (double el : output) - { - // For input = target we should get 0.0 everywhere. - REQUIRE(el == Approx(0.0).epsilon(1e-5)); - } - + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(6).epsilon(1e-3)); REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 1e-3); - // Test the Forward function. Loss should be 0.84. - input = arma::mat("0.1 0.8 0.6 0.0 0.5"); - target = arma::mat("0 1.0 1.0 0 0"); + // Test for mean reduction by modifying reduction + // parameter through the accessor. + module.Reduction() = false; + expectedOutput = arma::mat("0.0833 0.0833 -0.0833 0.0833 0.0833 -0.0833 " + "0.0833 0.0833 -0.0833 0.0833 0.0833 0.0833"); + expectedOutput.reshape(4, 3); + + // Test the forward function + // Loss should be 0.202467 + // Value calculated using torch.nn.HingeEmbeddingLoss(reduction='mean') loss = module.Forward(input, target); - REQUIRE(loss == Approx(0.84).epsilon(1e-3)); + REQUIRE(loss == Approx(0.202467).epsilon(1e-3)); - // Test the Backward function. + // Test the backward function module.Backward(input, target, output); - REQUIRE(arma::accu(output) == Approx(-2).epsilon(1e-3)); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(0.5).epsilon(1e-3)); REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); } /** From dfcee974e12b097c9a186b639bb0c954266cde1b Mon Sep 17 00:00:00 2001 From: Anwaar Date: Thu, 3 Feb 2022 17:40:56 +0530 Subject: [PATCH 05/34] Huber Loss. --- .../methods/ann/loss_functions/huber_loss.hpp | 21 +++++--- .../ann/loss_functions/huber_loss_impl.hpp | 21 +++++--- src/mlpack/tests/loss_functions_test.cpp | 53 ++++++++++++++----- 3 files changed, 66 insertions(+), 29 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp index f5ce03dba4..7665253a24 100644 --- a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp @@ -41,9 +41,13 @@ class HuberLoss * * @param delta The threshold value upto which squared error is followed and * after which absolute error is considered. - * @param mean If true then mean loss is computed otherwise sum. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - HuberLoss(const double delta = 1.0, const bool mean = true); + HuberLoss(const double delta = 1.0, const bool reduction = true); /** * Computes the Huber Loss function. @@ -79,10 +83,10 @@ class HuberLoss //! Set the value of delta. double& Delta() { return delta; } - //! Get the value of reduction type. - bool Mean() const { return mean; } - //! Set the value of reduction type. - bool& Mean() { return mean; } + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } /** * Serialize the layer. @@ -97,8 +101,9 @@ class HuberLoss //! Hyperparameter `delta` defines the point upto which MSE is considered. double delta; - //! Reduction type. If true, performs mean of loss else sum. - bool mean; + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class HuberLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp index 50b2c61858..8c5bf87cae 100644 --- a/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp @@ -21,9 +21,9 @@ namespace ann /** Artificial Neural Network. */ { template HuberLoss::HuberLoss( const double delta, - const bool mean): + const bool reduction): delta(delta), - mean(mean) + reduction(reduction) { // Nothing to do here. } @@ -36,14 +36,18 @@ HuberLoss::Forward( const TargetType& target) { typedef typename PredictionType::elem_type ElemType; - ElemType loss = 0; + ElemType lossSum = 0; for (size_t i = 0; i < prediction.n_elem; ++i) { const ElemType absError = std::abs(target[i] - prediction[i]); - loss += absError > delta ? + lossSum += absError > delta ? delta * (absError - 0.5 * delta) : 0.5 * std::pow(absError, 2); } - return mean ? loss / prediction.n_elem : loss; + + if (reduction) + return lossSum; + + return lossSum / prediction.n_elem; } template @@ -62,9 +66,10 @@ void HuberLoss::Backward( loss[i] = absError > delta ? -delta * (target[i] - prediction[i]) / absError : prediction[i] - target[i]; - if (mean) - loss[i] /= loss.n_elem; } + + if (!reduction) + loss = loss / prediction.n_elem; } template @@ -74,7 +79,7 @@ void HuberLoss::serialize( const uint32_t /* version */) { ar(CEREAL_NVP(delta)); - ar(CEREAL_NVP(mean)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 1d2491e0ae..87373b241a 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -52,25 +52,52 @@ using namespace mlpack::ann; TEST_CASE("HuberLossTest", "[LossFunctionsTest]") { arma::mat input, target, output; + arma::mat expectedOutput; + double loss; HuberLoss<> module; - // Test the Forward function. - input = arma::mat("17.45 12.91 13.63 29.01 7.12 15.47 31.52 31.97"); - target = arma::mat("16.52 13.11 13.67 29.51 24.31 15.03 30.72 34.07"); - double loss = module.Forward(input, target); - REQUIRE(loss == Approx(2.410631).epsilon(1e-5)); + // Test for sum reduction. + input = arma::mat("-0.0494 -1.1958 -1.0486 -0.2121 1.6028 0.0737 -0.7091 " + "0.8612 0.9639 0.9648 0.0745 0.5924"); + target = arma::mat("0.4316 0.0164 -0.4478 1.1452 0.5106 0.9255 0.5571 0.0864 " + "0.7059 -0.8288 -0.0231 -1.0526"); + expectedOutput = arma::mat("-0.4810 -1.0000 -0.6008 -1.0000 1.0000 -0.8518 " + "-1.0000 0.7748 0.2580 1.0000 0.0976 1.0000"); + input.reshape(4, 3); + target.reshape(4, 3); + expectedOutput.reshape(4, 3); - // Test the backward function. + // Test the Forward function. Loss should be 6.36364. + // Value calculated using torch.nn.SmoothL1Loss(reduction='sum'). + loss = module.Forward(input, target); + REQUIRE(loss == Approx(6.36364).epsilon(1e-3)); + + // Test the Backward function. module.Backward(input, target, output); - - // Expected Output: - // [0.1162 -0.0250 -0.0050 -0.0625 -0.1250 0.0550 0.1000 -0.1250] - // Sum of Expected Output = -0.07125. - double expectedOutputSum = arma::accu(output); - REQUIRE(expectedOutputSum == Approx(-0.07125).epsilon(1e-5)); - + REQUIRE(arma::as_scalar(arma::accu(output)) == + Approx(-0.8032).epsilon(1e-3)); REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); + + // Test for mean reduction by modifying reduction parameter using accessor. + module.Reduction() = false; + expectedOutput = arma::mat("-0.0401 -0.0833 -0.0501 -0.0833 0.0833 -0.0710 " + "-0.0833 0.0646 0.0215 0.0833 0.0081 0.0833"); + expectedOutput.reshape(4, 3); + + // Test the Forward function. Loss should be 0.530304. + // Value calculated using torch.nn.SmoothL1Loss(reduction='mean'). + loss = module.Forward(input, target); + REQUIRE(loss == Approx(0.530304).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input, target, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == + Approx(-0.0669333).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); } /** From 9f162f991a70c3d61162c045ea3c598083322d69 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Fri, 4 Feb 2022 14:05:38 +0530 Subject: [PATCH 06/34] KL Divergence. --- .../ann/loss_functions/kl_divergence.hpp | 22 +++-- .../ann/loss_functions/kl_divergence_impl.hpp | 35 +++----- src/mlpack/tests/loss_functions_test.cpp | 90 +++++++++---------- 3 files changed, 69 insertions(+), 78 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp index 5eacb03b36..ffbb0c1f7f 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp @@ -49,9 +49,13 @@ class KLDivergence * Create the Kullback–Leibler Divergence object with the specified * parameters. * - * @param takeMean Boolean variable to specify whether to take mean or not. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - KLDivergence(const bool takeMean = false); + KLDivergence(const bool reduction = true); /** * Computes the Kullback–Leibler divergence error function. @@ -82,11 +86,10 @@ class KLDivergence //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the value of takeMean. - bool TakeMean() const { return takeMean; } - //! Modify the value of takeMean. - bool& TakeMean() { return takeMean; } - + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } /** * Serialize the loss function */ @@ -97,8 +100,9 @@ class KLDivergence //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Boolean variable for taking mean or not. - bool takeMean; + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class KLDivergence } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp index 9c74453a21..f5e72bc7e5 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp @@ -20,8 +20,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -KLDivergence::KLDivergence(const bool takeMean) : - takeMean(takeMean) +KLDivergence::KLDivergence(const bool reduction): + reduction(reduction) { // Nothing to do here. } @@ -33,15 +33,13 @@ KLDivergence::Forward( const PredictionType& prediction, const TargetType& target) { - if (takeMean) - { - return arma::as_scalar(arma::mean( - arma::mean(prediction % (arma::log(prediction) - arma::log(target))))); - } - else - { - return arma::accu(prediction % (arma::log(prediction) - arma::log(target))); - } + PredictionType loss = target % (arma::log(target) - prediction); + typename PredictionType::elem_type lossSum = arma::accu(loss); + + if (reduction) + return lossSum; + + return lossSum / prediction.n_elem; } template @@ -51,15 +49,10 @@ void KLDivergence::Backward( const TargetType& target, LossType& loss) { - if (takeMean) - { - loss = arma::mean(arma::mean( - arma::log(prediction) - arma::log(target) + 1)); - } - else - { - loss = arma::accu(arma::log(prediction) - arma::log(target) + 1); - } + loss = - target; + + if (!reduction) + loss = loss / prediction.n_elem; } template @@ -68,7 +61,7 @@ void KLDivergence::serialize( Archive& ar, const uint32_t /* version */) { - ar(CEREAL_NVP(takeMean)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 87373b241a..701d5d14c8 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -168,19 +168,55 @@ TEST_CASE("PoissonNLLLossTest", "[LossFunctionsTest]") } /** - * Simple KL Divergence test. The loss should be zero if input = target. + * Simple KL Divergence test. */ TEST_CASE("SimpleKLDivergenceTest", "[LossFunctionsTest]") { arma::mat input, target, output; + arma::mat expectedOutput; double loss; - KLDivergence<> module(true); + KLDivergence<> module; - // Test the Forward function. Loss should be 0 if input = target. - input = arma::ones(10, 1); - target = arma::ones(10, 1); + // Test for sum reduction. + input = arma::mat("-0.7007 -2.0247 -0.7132 -0.4584 -0.2637 -1.1795 -0.1093 " + "-1.0530 -2.4250 -0.4556 -0.7861 -0.9120"); + target = arma::mat("0.0223 0.5185 0.1610 0.9152 0.1689 0.6977 0.2823 0.3971 " + "0.2939 0.8000 0.6816 0.8742"); + expectedOutput = arma::mat("-0.0223 -0.5185 -0.1610 -0.9152 -0.1689 -0.6977 " + "-0.2823 -0.3971 -0.2939 -0.8000 -0.6816 -0.8742"); + input.reshape(4, 3); + target.reshape(4, 3); + expectedOutput.reshape(4, 3); + + // Test the Forward function. Loss should be 2.33349. + // Value calculated using torch.nn.KLDivLoss(reduction='sum'). loss = module.Forward(input, target); - REQUIRE(loss == Approx(0.0).margin(1e-5)); + REQUIRE(loss == Approx(2.33349).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input, target, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-5.8127).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); + + // Test for mean reduction by modifying reduction parameter using accessor. + module.Reduction() = false; + expectedOutput = arma::mat("-0.0019 -0.0432 -0.0134 -0.0763 -0.0141 -0.0581 " + "-0.0235 -0.0331 -0.0245 -0.0667 -0.0568 -0.0728"); + expectedOutput.reshape(4, 3); + + // Test the Forward function. Loss should be 0.194458. + // Value calculated using torch.nn.KLDivLoss(reduction='mean'). + loss = module.Forward(input, target); + REQUIRE(loss == Approx(0.194458).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input, target, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-0.484392).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); } /* @@ -217,48 +253,6 @@ TEST_CASE("SimpleMeanSquaredLogarithmicErrorTest", "[LossFunctionsTest]") REQUIRE(output.n_elem == 1); } -/** - * Test to check KL Divergence loss function when we take mean. - */ -TEST_CASE("KLDivergenceMeanTest", "[LossFunctionsTest]") -{ - arma::mat input, target, output; - double loss; - KLDivergence<> module(true); - - // Test the Forward function. - input = arma::mat("1 1 1 1 1 1 1 1 1 1"); - target = arma::exp(arma::mat("2 1 1 1 1 1 1 1 1 1")); - - loss = module.Forward(input, target); - REQUIRE(loss == Approx(-1.1).epsilon(1e-5)); - - // Test the Backward function. - module.Backward(input, target, output); - REQUIRE(arma::as_scalar(output) == Approx(-0.1).epsilon(1e-5)); -} - -/** - * Test to check KL Divergence loss function when we do not take mean. - */ -TEST_CASE("KLDivergenceNoMeanTest", "[LossFunctionsTest]") -{ - arma::mat input, target, output; - double loss; - KLDivergence<> module(false); - - // Test the Forward function. - input = arma::mat("1 1 1 1 1 1 1 1 1 1"); - target = arma::exp(arma::mat("2 1 1 1 1 1 1 1 1 1")); - - loss = module.Forward(input, target); - REQUIRE(loss == Approx(-11).epsilon(1e-5)); - - // Test the Backward function. - module.Backward(input, target, output); - REQUIRE(arma::as_scalar(output) == Approx(-1).epsilon(1e-5)); -} - /* * Simple test for the mean squared error performance function. */ From 46d491bc2f41f4ce640d0ca884af9e6b679ac0a9 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Fri, 4 Feb 2022 17:17:46 +0530 Subject: [PATCH 07/34] Margin Ranking Loss. --- .../loss_functions/margin_ranking_loss.hpp | 19 +++++- .../margin_ranking_loss_impl.hpp | 28 ++++++-- src/mlpack/tests/loss_functions_test.cpp | 65 +++++++++++-------- 3 files changed, 74 insertions(+), 38 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index 28971c89f0..de512500ed 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -37,10 +37,14 @@ class MarginRankingLoss public: /** * Create the MarginRankingLoss object with Hyperparameter margin. - * Hyperparameter margin defines a minimum distance between correctly ranked - * samples. + * @param margin defines a minimum distance between correctly ranked samples. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - MarginRankingLoss(const double margin = 1.0); + MarginRankingLoss(const double margin = 1.0, const bool reduction = true); /** * Computes the Margin Ranking Loss function. @@ -80,6 +84,11 @@ class MarginRankingLoss //! Modify the margin parameter. double& Margin() { return margin; } + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the layer. */ @@ -92,6 +101,10 @@ class MarginRankingLoss //! The margin value used in calculating Margin Ranking Loss. double margin; + + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class MarginRankingLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp index 17c63ca8e8..033357d9e8 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp @@ -20,7 +20,8 @@ namespace ann /** Artifical Neural Network. */ { template MarginRankingLoss::MarginRankingLoss( - const double margin) : margin(margin) + const double margin, const bool reduction): + margin(margin), reduction(reduction) { // Nothing to do here. } @@ -37,8 +38,14 @@ MarginRankingLoss::Forward( predictionRows / 2 - 1); const PredictionType& prediction2 = prediction.rows(predictionRows / 2, predictionRows - 1); - return arma::accu(arma::max(arma::zeros(size(target)), - -target % (prediction1 - prediction2) + margin)) / target.n_cols; + + double lossSum = arma::accu(arma::max(arma::zeros(size(target)), + -target % (prediction1 - prediction2) + margin)); + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -57,10 +64,16 @@ void MarginRankingLoss::Backward( predictionRows / 2 - 1); const PredictionType& prediction2 = prediction.rows(predictionRows / 2, predictionRows - 1); - loss = -target % (prediction1 - prediction2) + margin; - loss.elem(arma::find(loss >= 0)).ones(); - loss.elem(arma::find(loss < 0)).zeros(); - loss = (prediction2 - prediction1) % loss / target.n_cols; + LossType lossPrediction1 = -target % (prediction1 - prediction2) + margin; + lossPrediction1.elem(arma::find(lossPrediction1 >= 0)).ones(); + lossPrediction1.elem(arma::find(lossPrediction1 < 0)).zeros(); + LossType lossPrediction2 = lossPrediction1; + lossPrediction1 = -target % lossPrediction1; + lossPrediction2 = target % lossPrediction2; + loss = arma::join_cols(lossPrediction1, lossPrediction2); + + if (!reduction) + loss = loss / target.n_elem; } template @@ -70,6 +83,7 @@ void MarginRankingLoss::serialize( const uint32_t /* version */) { ar(CEREAL_NVP(margin)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 701d5d14c8..1c73298d0c 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -74,7 +74,7 @@ TEST_CASE("HuberLossTest", "[LossFunctionsTest]") // Test the Backward function. module.Backward(input, target, output); - REQUIRE(arma::as_scalar(arma::accu(output)) == + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-0.8032).epsilon(1e-3)); REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); @@ -584,7 +584,7 @@ TEST_CASE("SimpleMeanBiasErrorTest", "[LossFunctionsTest]") input.reshape(4, 3); target.reshape(4, 3); - + // Test the forward function. // Loss should be 0.1081. loss = module.Forward(input, target); @@ -826,42 +826,51 @@ TEST_CASE("CosineEmbeddingLossTest", "[LossFunctionsTest]") */ TEST_CASE("MarginRankingLossTest", "[LossFunctionsTest]") { - arma::mat input, input1, input2, target, output; + arma::mat input, input1, input2, target, output, expectedOutput; + double loss; + // Test sum reduction MarginRankingLoss<> module; - - // Test the Forward function on a user generator input and compare it against - // the manually calculated result. - input1 = arma::mat("1 2 5 7 -1 -3"); - input2 = arma::mat("-1 3 -4 11 3 -3"); - input = arma::join_cols(input1, input2); - target = arma::mat("1 -1 -1 1 -1 1"); - double error = module.Forward(input, target); - // Computed using torch.nn.functional.margin_ranking_loss() - REQUIRE(error == Approx(2.66667).epsilon(1e-3)); - - // Test the Backward function. - module.Backward(input, target, output); - - CheckMatrices(output, arma::mat("-0.000000 0.166667 -1.500000 0.666667 " - "0.000000 -0.000000"), 1e-3); - REQUIRE(output.n_rows == target.n_rows); - REQUIRE(output.n_cols == target.n_cols); - - // Test the error function on another input. input1 = arma::mat("0.4287 -1.6208 -1.5006 -0.4473 1.5208 -4.5184 9.3574 " "-4.8090 4.3455 5.2070"); input2 = arma::mat("-4.5288 -9.2766 -0.5882 -5.6643 -6.0175 8.8506 3.4759 " "-9.4886 2.2755 8.4951"); + expectedOutput << 0.0000 << 0.0000 << 1.0000 << 0.0000 << 1.0000 << -1.0000 + << 0.0000 << 0.0000 << 1.0000 << -1.0000 < Date: Sat, 5 Feb 2022 15:21:04 +0530 Subject: [PATCH 08/34] Log CosH Error. --- .../ann/loss_functions/log_cosh_loss.hpp | 27 ++++++++++++++----- .../ann/loss_functions/log_cosh_loss_impl.hpp | 17 +++++++++--- src/mlpack/tests/loss_functions_test.cpp | 16 ++++++++++- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp index db3090488e..a9bab96fe5 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp @@ -39,14 +39,18 @@ class LogCoshLoss * Create the Log-Hyperbolic-Cosine object with the specified * parameters. * - * @param a A double type value for smoothening loss function. - * It must be positive a real number, Sharpness of loss - * function is directly proportional to a. It can also - * act as a scaling factor hence making the loss - * function more sensitive to small losses around the - * origin. Default value = 1.0. + * @param a A double type value for smoothening loss function. It must be a + * positive real number. Sharpness of loss function is directly + * proportional to a. It can also act as a scaling factor, hence + * making the loss function more sensitive to small losses around + * the origin. Default value = 1.0. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - LogCoshLoss(const double a = 1.0); + LogCoshLoss(const double a = 1.0, const bool reduction = true); /** * Computes the Log-Hyperbolic-Cosine loss function. @@ -82,6 +86,11 @@ class LogCoshLoss //! Modify the value of hyperparameter a. double& A() { return a; } + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the loss function. */ @@ -94,6 +103,10 @@ class LogCoshLoss //! Hyperparameter a for smoothening function curve. double a; + + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class LogCoshLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp index 63a72cfa97..e5be1d43d7 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp @@ -20,8 +20,9 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -LogCoshLoss::LogCoshLoss(const double a) : - a(a) +LogCoshLoss::LogCoshLoss + (const double a, const bool reduction) : + a(a) , reduction(reduction) { Log::Assert(a > 0, "Hyper-Parameter \'a\' must be positive"); } @@ -33,7 +34,13 @@ LogCoshLoss::Forward( const PredictionType& prediction, const TargetType& target) { - return arma::accu(arma::log(arma::cosh(a * (target - prediction)))) / a; + typename PredictionType::elem_type lossSum = + arma::accu(arma::log(arma::cosh(a * (target - prediction)))) / a; + + if (reduction) + return lossSum; + + return lossSum / prediction.n_elem; } template @@ -44,6 +51,9 @@ void LogCoshLoss::Backward( LossType& loss) { loss = arma::tanh(a * (target - prediction)); + + if (!reduction) + loss = loss / prediction.n_elem; } template @@ -53,6 +63,7 @@ void LogCoshLoss::serialize( const uint32_t /* version */) { ar(CEREAL_NVP(a)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 1c73298d0c..415a5ede13 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -643,9 +643,10 @@ TEST_CASE("LogCoshLossTest", "[LossFunctionsTest]") REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); - // Test the Forward function. Loss should be 0.546621. + // Test for sum reduction input = arma::mat("1 2 3 4 5"); target = arma::mat("1 2.4 3.4 4.2 5.5"); + // Test the Forward function. Loss should be 0.546621. loss = module.Forward(input, target); REQUIRE(loss == Approx(0.546621).epsilon(1e-3)); @@ -653,6 +654,19 @@ TEST_CASE("LogCoshLossTest", "[LossFunctionsTest]") module.Backward(input, target, output); REQUIRE(arma::accu(output) == Approx(2.46962).epsilon(1e-3)); REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + + // Test for mean reduction by modifying reduction parameter using accessor. + module.Reduction() = false; + + // Test the Forward function. Loss should be 0.109324. + loss = module.Forward(input, target); + REQUIRE(loss == Approx(0.109324).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input, target, output); + REQUIRE(arma::accu(output) == Approx(0.49392).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); } From 786711b906351b70c9872c681d0eeb35052a6f22 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Sun, 6 Feb 2022 17:31:38 +0530 Subject: [PATCH 09/34] Mean Squared Logarithmic Error. --- .../mean_squared_logarithmic_error.hpp | 17 ++++++- .../mean_squared_logarithmic_error_impl.hpp | 21 +++++--- src/mlpack/tests/loss_functions_test.cpp | 50 ++++++++++++------- 3 files changed, 64 insertions(+), 24 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp index 14a7a08ad0..347dc61142 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp @@ -35,8 +35,14 @@ class MeanSquaredLogarithmicError public: /** * Create the MeanSquaredLogarithmicError object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - MeanSquaredLogarithmicError(); + MeanSquaredLogarithmicError(const bool reduction = true); /** * Computes the mean squared logarithmic error function. @@ -67,6 +73,11 @@ class MeanSquaredLogarithmicError //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the layer */ @@ -76,6 +87,10 @@ class MeanSquaredLogarithmicError private: //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class MeanSquaredLogarithmicError } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp index ffb1ea7cd8..42e24d65a2 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp @@ -20,7 +20,7 @@ namespace ann /** Artificial Neural Network. */ { template MeanSquaredLogarithmicError -::MeanSquaredLogarithmicError() +::MeanSquaredLogarithmicError(const bool reduction) : reduction(reduction) { // Nothing to do here. } @@ -32,8 +32,14 @@ MeanSquaredLogarithmicError::Forward( const PredictionType& prediction, const TargetType& target) { - return arma::accu(arma::square(arma::log(1. + target) - - arma::log(1. + prediction))) / target.n_cols; + typename PredictionType::elem_type lossSum = + arma::accu(arma::square(arma::log(1. + target) - + arma::log(1. + prediction))) ; + + if (reduction) + return lossSum; + + return lossSum / prediction.n_elem; } template @@ -44,16 +50,19 @@ void MeanSquaredLogarithmicError::Backward( LossType& loss) { loss = 2 * (arma::log(1. + prediction) - arma::log(1. + target)) / - ((1. + prediction) * target.n_cols); + (1. + prediction); + + if (!reduction) + loss = loss / prediction.n_elem; } template template void MeanSquaredLogarithmicError::serialize( - Archive& /* ar */, + Archive& ar, const uint32_t /* version */) { - // Nothing to do here. + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 415a5ede13..7631489049 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -224,33 +224,49 @@ TEST_CASE("SimpleKLDivergenceTest", "[LossFunctionsTest]") */ TEST_CASE("SimpleMeanSquaredLogarithmicErrorTest", "[LossFunctionsTest]") { - arma::mat input, output, target; + arma::mat input, target, output, expectedOutput; + double loss; MeanSquaredLogarithmicError<> module; - // Test the Forward function on a user generator input and compare it against - // the manually calculated result. - input = arma::zeros(1, 8); - target = arma::zeros(1, 8); - double error = module.Forward(input, target); - REQUIRE(error == Approx(0.0).margin(1e-5)); + // Test for sum reduction. + input = arma::mat("-0.0494 1.1958 1.0486 -0.2121 1.6028 0.0737 -0.7091 " + "0.8612 0.9639 0.9648 0.0745 0.5924"); + target = arma::mat("0.4316 0.0164 -0.4478 1.1452 0.5106 0.9255 0.5571 0.0864 " + "0.7059 -0.8288 -0.0231 1.0526"); + expectedOutput = arma::mat("-0.8615 0.7016 1.2799 -2.5425 0.4181 -1.0880 " + "-11.5339 0.5785 0.1434 2.4840 0.1772 -0.3188"); + input.reshape(4, 3); + target.reshape(4, 3); + expectedOutput.reshape(4, 3); + + // Test the Forward function. Loss should be 13.2728. + loss = module.Forward(input, target); + REQUIRE(loss == Approx(13.2728).epsilon(1e-3)); // Test the Backward function. module.Backward(input, target, output); - // The output should be equal to 0. - CheckMatrices(input, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-10.5619).epsilon(1e-3)); REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); - // Test the error function on a single input. - input = arma::mat("2"); - target = arma::mat("3"); - error = module.Forward(input, target); - REQUIRE(error == Approx(0.082760974810151655).epsilon(1e-3)); + // Test for mean reduction by modifying reduction parameter using accessor. + module.Reduction() = false; + expectedOutput = arma::mat("-0.0718 0.0585 0.1067 -0.2119 0.0348 -0.0907 " + "-0.9612 0.0482 0.0120 0.2070 0.0148 -0.0266"); + expectedOutput.reshape(4, 3); - // Test the Backward function on a single input. + // Test the Forward function. Loss should be 1.10606. + loss = module.Forward(input, target); + REQUIRE(loss == Approx(1.10606).epsilon(1e-3)); + + // Test the Backward function. module.Backward(input, target, output); - REQUIRE(arma::accu(output) == Approx(-0.1917880483011872).epsilon(1e-3)); - REQUIRE(output.n_elem == 1); + REQUIRE(arma::as_scalar(arma::accu(output)) == + Approx(-0.880156).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); } /* From c03075da022541ffbee0d7e0a983f2807ab3a72c Mon Sep 17 00:00:00 2001 From: Anwaar Date: Sun, 6 Feb 2022 18:02:35 +0530 Subject: [PATCH 10/34] cosine embeddding loss --- .../loss_functions/cosine_embedding_loss.hpp | 23 +++++++++++-------- .../cosine_embedding_loss_impl.hpp | 22 ++++++++++-------- src/mlpack/tests/loss_functions_test.cpp | 4 ++-- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp index c7df95e4a4..df40c823fd 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp @@ -46,13 +46,15 @@ class CosineEmbeddingLoss * Refer definition of cosine-embedding-loss above. * @param similarity Determines whether to use similarity or dissimilarity for * comparision. - * @param takeMean Boolean variable to specify whether to take mean or not. - * Specifies reduction method i.e. sum or mean corresponding - * to 0 and 1 respectively. Default value = 0. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ CosineEmbeddingLoss(const double margin = 0.0, const bool similarity = true, - const bool takeMean = false); + const bool reduction = true); /** * Ordinary feed forward pass of a neural network. @@ -93,10 +95,10 @@ class CosineEmbeddingLoss //! Modify the delta. OutputDataType& Delta() { return delta; } - //! Get the value of takeMean. - bool TakeMean() const { return takeMean; } - //! Modify the value of takeMean. - bool& TakeMean() { return takeMean; } + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } //! Get the value of margin. double Margin() const { return margin; } @@ -130,8 +132,9 @@ class CosineEmbeddingLoss //! Locally-stored value of similarity hyper-parameter. bool similarity; - //! Locally-stored value of takeMean hyper-parameter. - bool takeMean; +//! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class CosineEmbeddingLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp index 129a12c26c..bee9906032 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp @@ -20,8 +20,8 @@ namespace ann /** Artificial Neural Network. */ { template CosineEmbeddingLoss::CosineEmbeddingLoss( - const double margin, const bool similarity, const bool takeMean): - margin(margin), similarity(similarity), takeMean(takeMean) + const double margin, const bool similarity, const bool reduction): + margin(margin), similarity(similarity), reduction(reduction) { // Nothing to do here. } @@ -42,7 +42,7 @@ CosineEmbeddingLoss::Forward( arma::colvec inputTemp1 = arma::vectorise(prediction); arma::colvec inputTemp2 = arma::vectorise(target); - ElemType loss = 0.0; + ElemType lossSum = 0.0; for (size_t i = 0; i < inputTemp1.n_elem; i += cols) { @@ -50,18 +50,18 @@ CosineEmbeddingLoss::Forward( inputTemp1(arma::span(i, i + cols - 1)), inputTemp2(arma::span(i, i + cols - 1))); if (similarity) - loss += 1 - cosDist; + lossSum += 1 - cosDist; else { const ElemType currentLoss = cosDist - margin; - loss += currentLoss > 0 ? currentLoss : 0; + lossSum += currentLoss > 0 ? currentLoss : 0; } } - if (takeMean) - loss = (ElemType) loss / batchSize; + if (reduction) + return lossSum; - return loss; + return (ElemType) lossSum / batchSize; } template @@ -74,6 +74,7 @@ void CosineEmbeddingLoss::Backward( typedef typename PredictionType::elem_type ElemType; const size_t cols = prediction.n_cols; + const size_t batchSize = prediction.n_elem / cols; if (arma::size(prediction) != arma::size(target)) Log::Fatal << "Input Tensors must have same dimensions." << std::endl; @@ -99,6 +100,9 @@ void CosineEmbeddingLoss::Backward( 1)))) / std::sqrt(arma::accu(arma::pow(inputTemp1(arma::span(i, i + cols - 1)), 2))); } + + if (!reduction) + outputTemp = outputTemp / batchSize; } } @@ -109,7 +113,7 @@ void CosineEmbeddingLoss::serialize( { ar(CEREAL_NVP(margin)); ar(CEREAL_NVP(similarity)); - ar(CEREAL_NVP(takeMean)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 7631489049..769e051ff8 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -837,7 +837,7 @@ TEST_CASE("CosineEmbeddingLossTest", "[LossFunctionsTest]") REQUIRE(arma::accu(output) == Approx(-0.36649111).epsilon(1e-3)); // Check Output for mean type of reduction. - CosineEmbeddingLoss<> module3(0.0, true, true); + CosineEmbeddingLoss<> module3(0.0, true, false); loss = module3.Forward(input3, input4); REQUIRE(loss == Approx(0.092325).epsilon(1e-3)); @@ -848,7 +848,7 @@ TEST_CASE("CosineEmbeddingLossTest", "[LossFunctionsTest]") // Test the Backward function. module3.Backward(input3, input4, output); - REQUIRE(arma::accu(output) == Approx(0.36649111).epsilon(1e-4)); + REQUIRE(arma::accu(output) == Approx(0.0236749374).epsilon(1e-4)); } /* From 420f89313ef6eff136151741211928f0d1f1310f Mon Sep 17 00:00:00 2001 From: Anwaar Date: Sun, 6 Feb 2022 18:49:48 +0530 Subject: [PATCH 11/34] earth mover's distance --- .../loss_functions/earth_mover_distance.hpp | 17 ++++++++- .../earth_mover_distance_impl.hpp | 18 +++++++--- src/mlpack/tests/loss_functions_test.cpp | 35 ++++++++++++++++--- 3 files changed, 61 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp index a5afbf37c2..2afedc60a6 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp @@ -35,8 +35,14 @@ class EarthMoverDistance public: /** * Create the EarthMoverDistance object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - EarthMoverDistance(); + EarthMoverDistance(const bool reduction = true); /** * Ordinary feed forward pass of a neural network. @@ -67,6 +73,11 @@ class EarthMoverDistance //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the layer. */ @@ -76,6 +87,10 @@ class EarthMoverDistance private: //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class EarthMoverDistance } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp index 8f6ab6f52e..7cf360f17e 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp @@ -19,7 +19,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -EarthMoverDistance::EarthMoverDistance() +EarthMoverDistance + ::EarthMoverDistance(const bool reduction) : reduction(reduction) { // Nothing to do here. } @@ -31,7 +32,13 @@ EarthMoverDistance::Forward( const PredictionType& prediction, const TargetType& target) { - return -arma::accu(target % prediction); + typename PredictionType::elem_type lossSum = + -arma::accu(target % prediction); + + if (reduction) + return lossSum; + + return lossSum / prediction.n_elem; } template @@ -42,15 +49,18 @@ void EarthMoverDistance::Backward( LossType& loss) { loss = -target; + + if (!reduction) + loss = loss / target.n_elem; } template template void EarthMoverDistance::serialize( - Archive& /* ar */, + Archive& ar, const uint32_t /* version */) { - /* Nothing to do here */ + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 769e051ff8..fbd1258994 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -424,21 +424,23 @@ TEST_CASE("SimpleSigmoidCrossEntropyErrorTest", "[LossFunctionsTest]") TEST_CASE("SimpleEarthMoverDistanceLayerTest", "[LossFunctionsTest]") { arma::mat input1, input2, output, target1, target2, expectedOutput; + arma::mat input3, target3; + double loss; EarthMoverDistance<> module; // Test the Forward function on a user generator input and compare it against // the manually calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); target1 = arma::zeros(1, 8); - double error1 = module.Forward(input1, target1); + loss = module.Forward(input1, target1); double expected = 0.0; - REQUIRE(error1 / input1.n_elem - expected == Approx(0.0).margin(1e-7)); + REQUIRE(loss / input1.n_elem - expected == Approx(0.0).margin(1e-7)); input2 = arma::mat("1 2 3 4 5"); target2 = arma::mat("1 0 1 0 1"); - double error2 = module.Forward(input2, target2); + loss = module.Forward(input2, target2); expected = -1.8; - REQUIRE(error2 / input2.n_elem - expected == Approx(0.0).margin(1e-6)); + REQUIRE(loss / input2.n_elem - expected == Approx(0.0).margin(1e-6)); // Test the Backward function. module.Backward(input1, target1, output); @@ -454,6 +456,31 @@ TEST_CASE("SimpleEarthMoverDistanceLayerTest", "[LossFunctionsTest]") REQUIRE(output(i) - expectedOutput(i) == Approx(0.0).margin(1e-5)); REQUIRE(output.n_rows == input2.n_rows); REQUIRE(output.n_cols == input2.n_cols); + + // Test for mean reduction. + module.Reduction() = false; + input3 = arma::mat("-0.0494 -1.1958 -1.0486 -0.2121 1.6028 0.0737 -0.7091 " + "0.8612 0.9639 0.9648 0.0745 0.5924"); + target3 = arma::mat("0.4316 0.0164 -0.4478 1.1452 0.5106 0.9255 0.5571 " + "0.0864 0.7059 -0.8288 -0.0231 -1.0526"); + expectedOutput = arma::mat("-0.0360 -0.0014 0.0373 -0.0954 -0.0426 -0.0771 " + "-0.0464 -0.0072 -0.0588 0.0691 0.0019 0.0877"); + input3.reshape(4, 3); + target3.reshape(4, 3); + expectedOutput.reshape(4, 3); + + // Test the Forward function. Loss should be -0.00060089. + // Value calculated manually. + loss = module.Forward(input3, target3); + REQUIRE(loss == Approx(-0.00060089).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input3, target3, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == + Approx(-0.168867).epsilon(1e-3)); + REQUIRE(output.n_rows == input3.n_rows); + REQUIRE(output.n_cols == input3.n_cols); + CheckMatrices(output, expectedOutput, 0.1); } /* From 6363350bbef6b67fd0a4a4f5496cdb0e538c3029 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Sun, 6 Feb 2022 20:14:23 +0530 Subject: [PATCH 12/34] mean squared error --- .../ann/loss_functions/mean_squared_error.hpp | 17 +++++++++++++++- .../mean_squared_error_impl.hpp | 20 ++++++++++++++----- src/mlpack/tests/loss_functions_test.cpp | 17 ++++++++++++++-- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp index 0cc3f6378d..ec3a1c239d 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp @@ -36,8 +36,14 @@ class MeanSquaredError public: /** * Create the MeanSquaredError object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - MeanSquaredError(); + MeanSquaredError(const bool reduction = true); /** * Computes the mean squared error function. @@ -67,6 +73,11 @@ class MeanSquaredError OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } /** * Serialize the layer @@ -77,6 +88,10 @@ class MeanSquaredError private: //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class MeanSquaredError } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp index ee4ae8c021..f6a6345b40 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp @@ -19,7 +19,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -MeanSquaredError::MeanSquaredError() +MeanSquaredError + ::MeanSquaredError(const bool reduction) : reduction(reduction) { // Nothing to do here. } @@ -31,7 +32,13 @@ MeanSquaredError::Forward( const PredictionType& prediction, const TargetType& target) { - return arma::accu(arma::square(prediction - target)) / target.n_cols; + typename PredictionType::elem_type lossSum = + arma::accu(arma::square(prediction - target)); + + if (reduction) + return lossSum; + + return lossSum / prediction.n_elem; } template @@ -41,16 +48,19 @@ void MeanSquaredError::Backward( const TargetType& target, LossType& loss) { - loss = 2 * (prediction - target) / target.n_cols; + loss = 2 * (prediction - target) ; + + if (!reduction) + loss = loss / prediction.n_elem; } template template void MeanSquaredError::serialize( - Archive& /* ar */, + Archive& ar, const uint32_t /* version */) { - // Nothing to do here. + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index fbd1258994..1b6cf800d9 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -275,9 +275,9 @@ TEST_CASE("SimpleMeanSquaredLogarithmicErrorTest", "[LossFunctionsTest]") TEST_CASE("SimpleMeanSquaredErrorTest", "[LossFunctionsTest]") { arma::mat input, output, target; - MeanSquaredError<> module; + MeanSquaredError<> module(false); - // Test the Forward function on a user generator input and compare it against + // Test the Forward function on a user generated input and compare it against // the manually calculated result. input = arma::mat("1.0 0.0 1.0 0.0 -1.0 0.0 -1.0 0.0"); target = arma::zeros(1, 8); @@ -304,6 +304,19 @@ TEST_CASE("SimpleMeanSquaredErrorTest", "[LossFunctionsTest]") // Test whether the output is negative. REQUIRE(arma::accu(output) == -2); REQUIRE(output.n_elem == 1); + + // Test for sum reduction + module.Reduction() = true; + + // Test the Forward function + error = module.Forward(input, target); + REQUIRE(error == Approx(1.0).epsilon(1e-5)); + + // Test the Backward function on a single input. + module.Backward(input, target, output); + // Test whether the output is negative. + REQUIRE(arma::accu(output) == -2); + REQUIRE(output.n_elem == 1); } /* From 3edd592bc3c76d641a4aad8d8591ce6db7c828a9 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Mon, 7 Feb 2022 12:50:45 +0530 Subject: [PATCH 13/34] negative log likelihood --- .../negative_log_likelihood.hpp | 19 +++++++- .../negative_log_likelihood_impl.hpp | 19 +++++--- src/mlpack/tests/loss_functions_test.cpp | 48 +++++++++++++++++++ 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp index 4f97c152f5..70efee5c45 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp @@ -19,7 +19,7 @@ namespace ann /** Artificial Neural Network. */ { /** * Implementation of the negative log likelihood layer. The negative log - * likelihood layer expectes that the input contains log-probabilities for each + * likelihood layer expects that the input contains log-probabilities for each * class. The layer also expects a class index, in the range between 1 and the * number of classes, as target when calling the Forward function. * @@ -37,8 +37,14 @@ class NegativeLogLikelihood public: /** * Create the NegativeLogLikelihoodLayer object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - NegativeLogLikelihood(); + NegativeLogLikelihood(const bool reduction = true); /** * Computes the Negative log likelihood. @@ -84,6 +90,11 @@ class NegativeLogLikelihood //! Modify the delta. OutputDataType& Delta() { return delta; } + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the layer */ @@ -99,6 +110,10 @@ class NegativeLogLikelihood //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class NegativeLogLikelihood } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp index 1eace1d772..b33979d130 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp @@ -19,7 +19,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -NegativeLogLikelihood::NegativeLogLikelihood() +NegativeLogLikelihood + ::NegativeLogLikelihood(const bool reduction) : reduction(reduction) { // Nothing to do here. } @@ -32,16 +33,19 @@ NegativeLogLikelihood::Forward( const TargetType& target) { typedef typename PredictionType::elem_type ElemType; - ElemType output = 0; + ElemType lossSum = 0; for (size_t i = 0; i < prediction.n_cols; ++i) { Log::Assert(target(i) >= 0 && target(i) < prediction.n_rows, "Target class out of range."); - output -= prediction(target(i), i); + lossSum -= prediction(target(i), i); } - return output; + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -59,14 +63,17 @@ void NegativeLogLikelihood::Backward( loss(target(i), i) = -1; } + + if (!reduction) + loss = loss / target.n_elem; } template template void NegativeLogLikelihood::serialize( - Archive& /* ar */, const uint32_t /* version */) + Archive& ar, const uint32_t /* version */) { - // Nothing to do here. + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 1b6cf800d9..88df022c75 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -1265,3 +1265,51 @@ TEST_CASE("MultiLabelSoftMarginLossWeightedTest", "[LossFunctionsTest]") REQUIRE(output.n_cols == input.n_cols); CheckMatrices(output, expectedOutput, 0.1); } + +/** + * Simple Negative Log Likelihood Loss test. + */ +TEST_CASE("NegativeLogLikelihoodLossTest", "[LossFunctionsTest]") +{ + arma::mat input, target, output; + arma::mat expectedOutput; + double loss; + NegativeLogLikelihood<> module; + + // Test for sum reduction. + input = arma::mat("-0.1689 -0.2862 -1.0543 -1.2865 -2.0033 -1.9392 -0.6196 " + "-1.4797 -3.8886 -2.2532 -2.1769 -0.7011"); + target = arma::mat("2 2 1 2"); + expectedOutput = arma::mat("0 0 0 0 0 0 -1.0000 0 -1.0000 -1.0000 0 -1.0000"); + input.reshape(4, 3); + expectedOutput.reshape(4, 3); + + // Test the Forward function. Loss should be 7.4625. + // Value calculated using torch.nn.NLLLoss(reduction='sum'). + loss = module.Forward(input, target); + REQUIRE(loss == Approx(7.4625).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input, target, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-4).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); + + // Test for mean reduction by modifying reduction parameter using accessor. + module.Reduction() = false; + expectedOutput = arma::mat("0 0 0 0 0 0 -0.2500 0 -0.2500 -0.2500 0 -0.2500"); + expectedOutput.reshape(4, 3); + + // Test the Forward function. Loss should be 1.86562. + // Value calculated using torch.nn.NLLLoss(reduction='mean'). + loss = module.Forward(input, target); + REQUIRE(loss == Approx(1.86562).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input, target, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-1).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); +} From 2a202f40a666923e0c41a767571f76d4870f9202 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Mon, 7 Feb 2022 16:19:09 +0530 Subject: [PATCH 14/34] NLL: fixed the forward function --- .../negative_log_likelihood_impl.hpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp index b33979d130..f76210a9bc 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp @@ -32,16 +32,20 @@ NegativeLogLikelihood::Forward( const PredictionType& prediction, const TargetType& target) { - typedef typename PredictionType::elem_type ElemType; - ElemType lossSum = 0; - for (size_t i = 0; i < prediction.n_cols; ++i) + PredictionType loss; + loss.zeros(size(target)); + typename TargetType::elem_type currentTarget; + for (size_t i = 0; i < target.n_cols; ++i) { - Log::Assert(target(i) >= 0 && target(i) < prediction.n_rows, + currentTarget = target(i); + Log::Assert(currentTarget >=0 && currentTarget < prediction.n_rows, "Target class out of range."); - lossSum -= prediction(target(i), i); + loss(i) -= prediction(i, currentTarget); } + typename PredictionType::elem_type lossSum= arma::accu(loss); + if (reduction) return lossSum; From 31298616e2ff66e5a6786109451975e53358cf06 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Mon, 7 Feb 2022 18:10:15 +0530 Subject: [PATCH 15/34] NLL: fixed the backward function --- .../loss_functions/negative_log_likelihood_impl.hpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp index f76210a9bc..c81ebe001f 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp @@ -59,13 +59,15 @@ void NegativeLogLikelihood::Backward( const TargetType& target, LossType& loss) { - loss = arma::zeros(prediction.n_rows, prediction.n_cols); - for (size_t i = 0; i < prediction.n_cols; ++i) + loss.zeros(size(prediction)); + typename TargetType::elem_type currentTarget; + for (size_t i = 0; i < target.n_cols; ++i) { - Log::Assert(target(i) >= 0 && target(i) < prediction.n_rows, + currentTarget = target(i); + Log::Assert(currentTarget >= 0 && currentTarget < prediction.n_rows, "Target class out of range."); - loss(target(i), i) = -1; + loss(i, currentTarget) = -1; } if (!reduction) From 600d207aa25ecece5133f65f77232e7e03b4a18d Mon Sep 17 00:00:00 2001 From: Anwaar Date: Mon, 7 Feb 2022 18:10:55 +0530 Subject: [PATCH 16/34] Poisson NLL. --- .../ann/loss_functions/poisson_nll_loss.hpp | 25 +++++++++++-------- .../loss_functions/poisson_nll_loss_impl.hpp | 16 +++++++----- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp index 31e1cb5620..aa0098761a 100644 --- a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp @@ -45,12 +45,16 @@ class PoissonNLLLoss * @param full Boolean value that determines whether to include Stirling's * approximation term. * @param eps A small value to prevent 0 in denominators and logarithms. - * @param mean When true, mean loss is computed otherwise total loss. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ PoissonNLLLoss(const bool logInput = true, const bool full = false, const typename InputDataType::elem_type eps = 1e-08, - const bool mean = true); + const bool reduction = true); /** * Computes the Poisson negative log likelihood Loss. @@ -112,13 +116,10 @@ class PoissonNLLLoss //! logarithms and denominators. typename InputDataType::elem_type& Eps() { return eps; } - //! Get the value of mean. It's a boolean value that tells if - //! mean of the total loss has to be taken. - bool Mean() const { return mean; } - //! Modify the value of mean. It's a boolean value that tells if - //! mean of the total loss has to be taken. - bool& Mean() { return mean; } - + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } /** * Serialize the layer. */ @@ -154,8 +155,10 @@ class PoissonNLLLoss //! eps is a small value required to prevent 0 in logarithms and denominators. typename InputDataType::elem_type eps; - //! Boolean value that tells if mean of the total loss has to be taken. - bool mean; + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; + }; // class PoissonNLLLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp index 05d2d79886..3152d73f56 100644 --- a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp @@ -24,11 +24,11 @@ PoissonNLLLoss::PoissonNLLLoss( const bool logInput, const bool full, const typename InputDataType::elem_type eps, - const bool mean): + const bool reduction): logInput(logInput), full(full), eps(eps), - mean(mean) + reduction(reduction) { Log::Assert(eps >= 0, "Epsilon (eps) must be greater than or equal to zero."); } @@ -57,8 +57,12 @@ PoissonNLLLoss::Forward( + 0.5 * arma::log(2 * M_PI * target); loss.elem(arma::find(mask)) += approx.elem(arma::find(mask)); } - - return mean ? arma::accu(loss) / loss.n_elem : arma::accu(loss); + typename PredictionType::elem_type lossSum = arma::accu(loss); + + if (reduction) + return lossSum; + + return lossSum / loss.n_elem; } template @@ -75,7 +79,7 @@ void PoissonNLLLoss::Backward( else loss = (1 - target / (prediction + eps)); - if (mean) + if (!reduction) loss = loss / loss.n_elem; } @@ -88,7 +92,7 @@ void PoissonNLLLoss::serialize( ar(CEREAL_NVP(logInput)); ar(CEREAL_NVP(full)); ar(CEREAL_NVP(eps)); - ar(CEREAL_NVP(mean)); + ar(CEREAL_NVP(reduction)); } } // namespace ann From c7a17eba8ee285562142f72bbfd0114bcb811d20 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Mon, 7 Feb 2022 18:12:00 +0530 Subject: [PATCH 17/34] Update tests for Poisson NLL. --- src/mlpack/tests/loss_functions_test.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 88df022c75..1e15622db3 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -108,10 +108,10 @@ TEST_CASE("PoissonNLLLossTest", "[LossFunctionsTest]") arma::mat input, target, input4, target4; arma::mat output1, output2, output3, output4; arma::mat expOutput1, expOutput2, expOutput3, expOutput4; - PoissonNLLLoss<> module1; - PoissonNLLLoss<> module2(true, true, 1e-08, false); - PoissonNLLLoss<> module3(true, true, 1e-08, true); - PoissonNLLLoss<> module4(false, true, 1e-08, true); + PoissonNLLLoss<> module1(true, false, 1e-8, false); + PoissonNLLLoss<> module2(true, true, 1e-08, true); + PoissonNLLLoss<> module3(true, true, 1e-08, false); + PoissonNLLLoss<> module4(false, true, 1e-08, false); // Test the Forward function on a user generated input. input = arma::mat("1.0 1.0 1.9 1.6 -1.9 3.7 -1.0 0.5"); From d0ecb07389f8194d95bb9bdfb1170c9c04df12ae Mon Sep 17 00:00:00 2001 From: Anwaar Date: Wed, 9 Feb 2022 14:28:15 +0530 Subject: [PATCH 18/34] Revert NLL: Current Implementation is correct --- .../negative_log_likelihood.hpp | 4 ++-- .../negative_log_likelihood_impl.hpp | 24 +++++++------------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp index 70efee5c45..b207a80037 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp @@ -20,8 +20,8 @@ namespace ann /** Artificial Neural Network. */ { /** * Implementation of the negative log likelihood layer. The negative log * likelihood layer expects that the input contains log-probabilities for each - * class. The layer also expects a class index, in the range between 1 and the - * number of classes, as target when calling the Forward function. + * class. The layer also expects a class index, in the range between 0 and + * number of classes -1, as target when calling the Forward function. * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp index c81ebe001f..b33979d130 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp @@ -32,20 +32,16 @@ NegativeLogLikelihood::Forward( const PredictionType& prediction, const TargetType& target) { - PredictionType loss; - loss.zeros(size(target)); - typename TargetType::elem_type currentTarget; - for (size_t i = 0; i < target.n_cols; ++i) + typedef typename PredictionType::elem_type ElemType; + ElemType lossSum = 0; + for (size_t i = 0; i < prediction.n_cols; ++i) { - currentTarget = target(i); - Log::Assert(currentTarget >=0 && currentTarget < prediction.n_rows, + Log::Assert(target(i) >= 0 && target(i) < prediction.n_rows, "Target class out of range."); - loss(i) -= prediction(i, currentTarget); + lossSum -= prediction(target(i), i); } - typename PredictionType::elem_type lossSum= arma::accu(loss); - if (reduction) return lossSum; @@ -59,15 +55,13 @@ void NegativeLogLikelihood::Backward( const TargetType& target, LossType& loss) { - loss.zeros(size(prediction)); - typename TargetType::elem_type currentTarget; - for (size_t i = 0; i < target.n_cols; ++i) + loss = arma::zeros(prediction.n_rows, prediction.n_cols); + for (size_t i = 0; i < prediction.n_cols; ++i) { - currentTarget = target(i); - Log::Assert(currentTarget >= 0 && currentTarget < prediction.n_rows, + Log::Assert(target(i) >= 0 && target(i) < prediction.n_rows, "Target class out of range."); - loss(i, currentTarget) = -1; + loss(target(i), i) = -1; } if (!reduction) From 521ff80189841650b76a52fbef116a3958de0047 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Wed, 9 Feb 2022 14:40:04 +0530 Subject: [PATCH 19/34] Update test for NLL --- src/mlpack/tests/loss_functions_test.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 1e15622db3..7561fa4c00 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -1277,12 +1277,12 @@ TEST_CASE("NegativeLogLikelihoodLossTest", "[LossFunctionsTest]") NegativeLogLikelihood<> module; // Test for sum reduction. - input = arma::mat("-0.1689 -0.2862 -1.0543 -1.2865 -2.0033 -1.9392 -0.6196 " - "-1.4797 -3.8886 -2.2532 -2.1769 -0.7011"); + input = arma::mat("-0.1689 -2.0033 -3.8886 -0.2862 -1.9392 -2.2532" + " -1.0543 -0.6196 -2.1769 -1.2865 -1.4797 -0.7011"); target = arma::mat("2 2 1 2"); - expectedOutput = arma::mat("0 0 0 0 0 0 -1.0000 0 -1.0000 -1.0000 0 -1.0000"); - input.reshape(4, 3); - expectedOutput.reshape(4, 3); + expectedOutput = arma::mat("0 0 -1.0000 0 0 -1.0000 0 -1.0000 0 0 0 -1.0000"); + input.reshape(3, 4); + expectedOutput.reshape(3, 4); // Test the Forward function. Loss should be 7.4625. // Value calculated using torch.nn.NLLLoss(reduction='sum'). @@ -1298,8 +1298,8 @@ TEST_CASE("NegativeLogLikelihoodLossTest", "[LossFunctionsTest]") // Test for mean reduction by modifying reduction parameter using accessor. module.Reduction() = false; - expectedOutput = arma::mat("0 0 0 0 0 0 -0.2500 0 -0.2500 -0.2500 0 -0.2500"); - expectedOutput.reshape(4, 3); + expectedOutput = arma::mat("0 0 -0.2500 0 0 -0.2500 0 -0.2500 0 0 0 -0.2500"); + expectedOutput.reshape(3, 4); // Test the Forward function. Loss should be 1.86562. // Value calculated using torch.nn.NLLLoss(reduction='mean'). From f4a6bd257a433fffe096c657e462008e2980e003 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Thu, 10 Feb 2022 10:06:47 +0530 Subject: [PATCH 20/34] Reconstruction Loss. --- .../ann/loss_functions/reconstruction_loss.hpp | 17 ++++++++++++++++- .../loss_functions/reconstruction_loss_impl.hpp | 16 ++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp index 56f6f39cc3..1f0b606521 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp @@ -39,8 +39,14 @@ class ReconstructionLoss public: /** * Create the ReconstructionLoss object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - ReconstructionLoss(); + ReconstructionLoss(const bool reduction = true); /** * Computes the reconstruction loss. @@ -71,6 +77,11 @@ class ReconstructionLoss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the layer */ @@ -83,6 +94,10 @@ class ReconstructionLoss //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class ReconstructionLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp index ca5c986f05..ccb70b90f0 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp @@ -23,7 +23,7 @@ ReconstructionLoss< InputDataType, OutputDataType, DistType ->::ReconstructionLoss() +>::ReconstructionLoss(const bool reduction) : reduction(reduction) { // Nothing to do here. } @@ -35,7 +35,12 @@ ReconstructionLoss::Forward( const PredictionType& prediction, const TargetType& target) { dist = DistType(prediction); - return -dist.LogProbability(target); + typename PredictionType::elem_type lossSum = -dist.LogProbability(target); + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -47,15 +52,18 @@ void ReconstructionLoss::Backward( { dist.LogProbBackward(target, loss); loss *= -1; + + if (!reduction) + loss = loss / target.n_elem; } template template void ReconstructionLoss::serialize( - Archive& /* ar */, + Archive& ar, const uint32_t /* version */) { - // Nothing to do here. + ar(CEREAL_NVP(reduction)); } } // namespace ann From ae887d948b173924f9363d4494f8e2493af7bae0 Mon Sep 17 00:00:00 2001 From: Anwaar Date: Fri, 11 Feb 2022 10:22:35 +0530 Subject: [PATCH 21/34] Sigmoid Cross Entropy Error --- .../sigmoid_cross_entropy_error.hpp | 21 ++++++++++++++++--- .../sigmoid_cross_entropy_error_impl.hpp | 17 +++++++++++---- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp index a1f4384e7f..0eaf1fe555 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp @@ -3,7 +3,7 @@ * @author Kris Singh * @author Shikhar Jaiswal * - * Definition of the cross-entropy with logit performance function. + * Definition of the cross-entropy with logits performance function. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -54,8 +54,14 @@ class SigmoidCrossEntropyError public: /** * Create the SigmoidCrossEntropyError object. - */ - SigmoidCrossEntropyError(); + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. + */ + SigmoidCrossEntropyError(const bool reduction = true); /** * Computes the Sigmoid CrossEntropy Error functions. @@ -87,6 +93,11 @@ class SigmoidCrossEntropyError //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the layer. */ @@ -96,6 +107,10 @@ class SigmoidCrossEntropyError private: //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class SigmoidCrossEntropy } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp index 93b3775e6a..a108bc30df 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp @@ -22,7 +22,7 @@ namespace ann /** Artificial Neural Network. */ { template SigmoidCrossEntropyError -::SigmoidCrossEntropyError() +::SigmoidCrossEntropyError(const bool reduction): reduction(reduction) { // Nothing to do here. } @@ -42,7 +42,13 @@ SigmoidCrossEntropyError::Forward( std::log(1 + std::exp(-std::abs(prediction[i]))); } - return maximum - arma::accu(prediction % target); + ElemType lossSum = + maximum - arma::accu(prediction % target); + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -53,15 +59,18 @@ inline void SigmoidCrossEntropyError::Backward( LossType& loss) { loss = 1.0 / (1.0 + arma::exp(-prediction)) - target; + + if (!reduction) + loss = loss / target.n_elem; } template template void SigmoidCrossEntropyError::serialize( - Archive& /* ar */, + Archive& ar , const uint32_t /* version */) { - // Nothing to do here + ar(CEREAL_NVP(reduction)); } } // namespace ann From c2bd93ab7daf859bca9a59f00f756c147e05f02e Mon Sep 17 00:00:00 2001 From: Anwaar Date: Fri, 11 Feb 2022 10:30:29 +0530 Subject: [PATCH 22/34] BCE Loss. --- .../loss_functions/binary_cross_entropy_loss.hpp | 15 ++++++++++----- .../binary_cross_entropy_loss_impl.hpp | 12 ++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp index cc114da81f..6b4853c4ec 100644 --- a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp @@ -38,8 +38,12 @@ class BCELoss * * @param eps The minimum value used for computing logarithms * and denominators in a numerically stable way. - * @param reduction Reduction type. If true, it returns the mean of - * the loss. Else, it returns the sum. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ BCELoss(const double eps = 1e-10, const bool reduction = true); @@ -77,9 +81,9 @@ class BCELoss //! Modify the epsilon. double& Eps() { return eps; } - //! Get the reduction. + //! Get the type of reduction used. bool Reduction() const { return reduction; } - //! Set the reduction. + //! Modify the type of reduction used. bool& Reduction() { return reduction; } /** @@ -95,7 +99,8 @@ class BCELoss //! The minimum value used for computing logarithms and denominators double eps; - //! Reduction type. If true, performs mean of loss else sum. + //! Boolean value that tells if reduction + // is 'sum' or 'mean'. bool reduction; }; // class BCELoss diff --git a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp index 4555240b88..666d285763 100644 --- a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp @@ -34,11 +34,13 @@ BCELoss::Forward( { typedef typename PredictionType::elem_type ElemType; - ElemType loss = -arma::accu(target % arma::log(prediction + eps) + + ElemType lossSum = -arma::accu(target % arma::log(prediction + eps) + (1. - target) % arma::log(1. - prediction + eps)); + if (reduction) - loss /= prediction.n_elem; - return loss; + return lossSum; + + return lossSum / target.n_elem; } template @@ -49,7 +51,8 @@ void BCELoss::Backward( LossType& loss) { loss = (1. - target) / (1. - prediction + eps) - target / (prediction + eps); - if (reduction) + + if (!reduction) loss /= prediction.n_elem; } @@ -60,6 +63,7 @@ void BCELoss::serialize( const uint32_t /* version */) { ar(CEREAL_NVP(eps)); + ar(CEREAL_NVP(reduction)); } } // namespace ann From 377cd49aef476850fd64633b1d23466d37711a4d Mon Sep 17 00:00:00 2001 From: Anwaar Date: Fri, 11 Feb 2022 10:32:36 +0530 Subject: [PATCH 23/34] BCE Loss: Update tests. --- src/mlpack/tests/loss_functions_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 7561fa4c00..2c800b1e21 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -325,8 +325,8 @@ TEST_CASE("SimpleMeanSquaredErrorTest", "[LossFunctionsTest]") TEST_CASE("SimpleBinaryCrossEntropyLossTest", "[LossFunctionsTest]") { arma::mat input1, input2, input3, output, target1, target2, target3; - BCELoss<> module1(1e-6, false); - BCELoss<> module2(1e-6, true); + BCELoss<> module1(1e-6, true); + BCELoss<> module2(1e-6, false); // Test the Forward function on a user generator input and compare it against // the manually calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); From d13ff3f4a806e8d903fb8221a0a1d59c9f227baf Mon Sep 17 00:00:00 2001 From: hello-fri-end Date: Tue, 22 Feb 2022 09:41:27 +0530 Subject: [PATCH 24/34] Apply documentation suggestion to all loss functions --- .../methods/ann/loss_functions/binary_cross_entropy_loss.hpp | 3 ++- .../methods/ann/loss_functions/cosine_embedding_loss.hpp | 3 ++- .../methods/ann/loss_functions/earth_mover_distance.hpp | 3 ++- .../methods/ann/loss_functions/hinge_embedding_loss.hpp | 3 ++- src/mlpack/methods/ann/loss_functions/hinge_loss.hpp | 3 ++- src/mlpack/methods/ann/loss_functions/huber_loss.hpp | 3 ++- src/mlpack/methods/ann/loss_functions/kl_divergence.hpp | 3 ++- src/mlpack/methods/ann/loss_functions/l1_loss.hpp | 3 ++- src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp | 3 ++- src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp | 3 ++- src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp | 4 ++-- src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp | 3 ++- .../ann/loss_functions/mean_squared_logarithmic_error.hpp | 3 ++- .../methods/ann/loss_functions/multilabel_softmargin_loss.hpp | 3 ++- .../methods/ann/loss_functions/negative_log_likelihood.hpp | 3 ++- src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp | 3 ++- src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp | 3 ++- .../ann/loss_functions/sigmoid_cross_entropy_error.hpp | 3 ++- src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp | 3 ++- 19 files changed, 38 insertions(+), 20 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp index 6b4853c4ec..9951d57c81 100644 --- a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp @@ -81,7 +81,8 @@ class BCELoss //! Modify the epsilon. double& Eps() { return eps; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp index df40c823fd..89c2f2fc4a 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp @@ -95,7 +95,8 @@ class CosineEmbeddingLoss //! Modify the delta. OutputDataType& Delta() { return delta; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp index 2afedc60a6..45853cade1 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp @@ -73,7 +73,8 @@ class EarthMoverDistance //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp index 3159e2f014..74def487b8 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp @@ -76,7 +76,8 @@ class HingeEmbeddingLoss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const {return reduction; } //! Modify the type of reduction used. bool& Reduction() {return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp b/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp index 60a2002782..b9f47d2e91 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp @@ -77,7 +77,8 @@ class HingeLoss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp index 7665253a24..a36cf936d2 100644 --- a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp @@ -83,7 +83,8 @@ class HuberLoss //! Set the value of delta. double& Delta() { return delta; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp index ffbb0c1f7f..ff4949b395 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp @@ -86,7 +86,8 @@ class KLDivergence //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/l1_loss.hpp b/src/mlpack/methods/ann/loss_functions/l1_loss.hpp index ec47252fcd..65b64a568c 100644 --- a/src/mlpack/methods/ann/loss_functions/l1_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/l1_loss.hpp @@ -74,7 +74,8 @@ class L1Loss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp index a9bab96fe5..6318788e08 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp @@ -86,7 +86,8 @@ class LogCoshLoss //! Modify the value of hyperparameter a. double& A() { return a; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index de512500ed..69df60b5e5 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -84,7 +84,8 @@ class MarginRankingLoss //! Modify the margin parameter. double& Margin() { return margin; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp index 244abc5d3d..f5037c6483 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp @@ -75,9 +75,9 @@ class MeanBiasError //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const {return reduction; } - //! Modify the type of reduction used. bool& Reduction() {return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp index ec3a1c239d..96b7946e64 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp @@ -74,7 +74,8 @@ class MeanSquaredError //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp index 347dc61142..bfae3eb881 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp @@ -73,7 +73,8 @@ class MeanSquaredLogarithmicError //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp b/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp index 8721bd7d4e..485e82571e 100644 --- a/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp @@ -80,7 +80,8 @@ class MultiLabelSoftMarginLoss //! Modify the weights assigned to each class. arma::rowvec& ClassWeights() { return classWeights; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp index b207a80037..bb825aca6e 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp @@ -90,7 +90,8 @@ class NegativeLogLikelihood //! Modify the delta. OutputDataType& Delta() { return delta; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp index aa0098761a..3cd5bd869b 100644 --- a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp @@ -116,7 +116,8 @@ class PoissonNLLLoss //! logarithms and denominators. typename InputDataType::elem_type& Eps() { return eps; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp index 1f0b606521..1c09423019 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp @@ -77,7 +77,8 @@ class ReconstructionLoss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp index 0eaf1fe555..c3ebfc6f59 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp @@ -93,7 +93,8 @@ class SigmoidCrossEntropyError //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } diff --git a/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp b/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp index a35db04d14..4fea3a0fe8 100644 --- a/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp @@ -74,7 +74,8 @@ class SoftMarginLoss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } From be855fa29b5af6796c6c1c3b270be024e5dfbde6 Mon Sep 17 00:00:00 2001 From: hello-fri-end Date: Thu, 24 Feb 2022 10:51:12 +0530 Subject: [PATCH 25/34] Apply documentation suggestion to all loss functions --- .../methods/ann/loss_functions/binary_cross_entropy_loss.hpp | 3 +-- .../methods/ann/loss_functions/cosine_embedding_loss.hpp | 3 +-- src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp | 3 +-- src/mlpack/methods/ann/loss_functions/hinge_loss.hpp | 2 +- src/mlpack/methods/ann/loss_functions/huber_loss.hpp | 3 +-- src/mlpack/methods/ann/loss_functions/l1_loss.hpp | 3 +-- src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp | 3 +-- src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp | 3 +-- src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp | 3 +-- src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp | 3 +-- .../ann/loss_functions/mean_squared_logarithmic_error.hpp | 3 +-- .../methods/ann/loss_functions/multilabel_softmargin_loss.hpp | 2 +- src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp | 3 +-- src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp | 3 +-- .../methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp | 3 +-- src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp | 2 +- 16 files changed, 16 insertions(+), 29 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp index 9951d57c81..8893227bc7 100644 --- a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp @@ -100,8 +100,7 @@ class BCELoss //! The minimum value used for computing logarithms and denominators double eps; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class BCELoss diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp index 89c2f2fc4a..362d0d4e3b 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp @@ -133,8 +133,7 @@ class CosineEmbeddingLoss //! Locally-stored value of similarity hyper-parameter. bool similarity; -//! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class CosineEmbeddingLoss diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp index 45853cade1..e8013aac57 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp @@ -89,8 +89,7 @@ class EarthMoverDistance //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class EarthMoverDistance diff --git a/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp b/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp index b9f47d2e91..7829b20348 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp @@ -93,7 +93,7 @@ class HingeLoss //! Locally-stored output parameter object. OutputDataType outputParameter; - //! The boolean value that tells if reduction is sum or mean. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class HingeLoss diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp index a36cf936d2..c1533c84f6 100644 --- a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp @@ -102,8 +102,7 @@ class HuberLoss //! Hyperparameter `delta` defines the point upto which MSE is considered. double delta; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class HuberLoss diff --git a/src/mlpack/methods/ann/loss_functions/l1_loss.hpp b/src/mlpack/methods/ann/loss_functions/l1_loss.hpp index 65b64a568c..e8494ca915 100644 --- a/src/mlpack/methods/ann/loss_functions/l1_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/l1_loss.hpp @@ -90,8 +90,7 @@ class L1Loss //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class L1Loss diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp index 6318788e08..23aa6baa4e 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp @@ -105,8 +105,7 @@ class LogCoshLoss //! Hyperparameter a for smoothening function curve. double a; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class LogCoshLoss diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index 69df60b5e5..fabb65d78a 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -103,8 +103,7 @@ class MarginRankingLoss //! The margin value used in calculating Margin Ranking Loss. double margin; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class MarginRankingLoss diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp index f5037c6483..fb84b2ba4c 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp @@ -91,8 +91,7 @@ class MeanBiasError //! Locally-stored output parameter object. OutputDataType outputParameter; - //! The boolen value that tells if reduction - //! is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class MeanBiasError diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp index 96b7946e64..6006d76d12 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp @@ -90,8 +90,7 @@ class MeanSquaredError //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class MeanSquaredError diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp index bfae3eb881..bdd702ef54 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp @@ -89,8 +89,7 @@ class MeanSquaredLogarithmicError //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class MeanSquaredLogarithmicError diff --git a/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp b/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp index 485e82571e..e707410155 100644 --- a/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp @@ -96,7 +96,7 @@ class MultiLabelSoftMarginLoss //! Locally-stored output parameter object. OutputDataType outputParameter; - //! The boolean value that tells if reduction is sum or mean. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; //! A (1, numClasses) shaped vector with weights for each class. diff --git a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp index 3cd5bd869b..e3ba107423 100644 --- a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp @@ -156,8 +156,7 @@ class PoissonNLLLoss //! eps is a small value required to prevent 0 in logarithms and denominators. typename InputDataType::elem_type eps; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class PoissonNLLLoss diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp index 1c09423019..454c22cb31 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp @@ -96,8 +96,7 @@ class ReconstructionLoss //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class ReconstructionLoss diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp index c3ebfc6f59..04f9419521 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp @@ -109,8 +109,7 @@ class SigmoidCrossEntropyError //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class SigmoidCrossEntropy diff --git a/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp b/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp index 4fea3a0fe8..a44e321fcd 100644 --- a/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp @@ -90,7 +90,7 @@ class SoftMarginLoss //! Locally-stored output parameter object. OutputDataType outputParameter; - //! The boolean value that tells if reduction is sum or mean. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class SoftMarginLoss From a876744548ed829dea7df857086e530ba97f6d1b Mon Sep 17 00:00:00 2001 From: Shah Anwaar Khalid Date: Fri, 25 Feb 2022 08:56:34 +0530 Subject: [PATCH 26/34] apply suggestion Co-authored-by: Marcus Edel --- src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp index e5be1d43d7..75cb924e2d 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp @@ -20,8 +20,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -LogCoshLoss::LogCoshLoss - (const double a, const bool reduction) : +LogCoshLoss::LogCoshLoss( + const double a, const bool reduction) : a(a) , reduction(reduction) { Log::Assert(a > 0, "Hyper-Parameter \'a\' must be positive"); From 5f26cd444e1b8abf407c18ec472552544f95b5fc Mon Sep 17 00:00:00 2001 From: Shah Anwaar Khalid Date: Fri, 25 Feb 2022 08:56:59 +0530 Subject: [PATCH 27/34] apply suggestion Co-authored-by: Marcus Edel --- src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp index 75cb924e2d..91de00668e 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp @@ -35,7 +35,7 @@ LogCoshLoss::Forward( const TargetType& target) { typename PredictionType::elem_type lossSum = - arma::accu(arma::log(arma::cosh(a * (target - prediction)))) / a; + arma::accu(arma::log(arma::cosh(a * (target - prediction)))) / a; if (reduction) return lossSum; From ef462f03cf033327614c60c8db8673c0e3e12741 Mon Sep 17 00:00:00 2001 From: Shah Anwaar Khalid Date: Fri, 25 Feb 2022 08:57:14 +0530 Subject: [PATCH 28/34] apply suggestion Co-authored-by: Marcus Edel --- .../methods/ann/loss_functions/mean_squared_error_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp index f6a6345b40..cee7191310 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp @@ -48,7 +48,7 @@ void MeanSquaredError::Backward( const TargetType& target, LossType& loss) { - loss = 2 * (prediction - target) ; + loss = 2 * (prediction - target); if (!reduction) loss = loss / prediction.n_elem; From 825a3a35dcde71709675249697f5e4dc13ef5cb0 Mon Sep 17 00:00:00 2001 From: Shah Anwaar Khalid Date: Fri, 25 Feb 2022 08:57:26 +0530 Subject: [PATCH 29/34] apply suggestion Co-authored-by: Marcus Edel --- .../methods/ann/loss_functions/mean_squared_error_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp index cee7191310..ffe1f49d0a 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp @@ -33,7 +33,7 @@ MeanSquaredError::Forward( const TargetType& target) { typename PredictionType::elem_type lossSum = - arma::accu(arma::square(prediction - target)); + arma::accu(arma::square(prediction - target)); if (reduction) return lossSum; From cfef1a0df27cc1c3933854582fa6e3f3a79713aa Mon Sep 17 00:00:00 2001 From: Shah Anwaar Khalid Date: Fri, 25 Feb 2022 08:57:39 +0530 Subject: [PATCH 30/34] apply suggestion Co-authored-by: Marcus Edel --- .../loss_functions/mean_squared_logarithmic_error_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp index 42e24d65a2..9fae31c342 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp @@ -33,8 +33,8 @@ MeanSquaredLogarithmicError::Forward( const TargetType& target) { typename PredictionType::elem_type lossSum = - arma::accu(arma::square(arma::log(1. + target) - - arma::log(1. + prediction))) ; + arma::accu(arma::square(arma::log(1.0 + target) - + arma::log(1.0 + prediction))); if (reduction) return lossSum; From d74c03c40d89191bbe2d82789e63679c10925e54 Mon Sep 17 00:00:00 2001 From: Shah Anwaar Khalid Date: Fri, 25 Feb 2022 08:57:54 +0530 Subject: [PATCH 31/34] apply suggestion Co-authored-by: Marcus Edel --- .../ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp index a108bc30df..e29e4a7478 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp @@ -42,8 +42,7 @@ SigmoidCrossEntropyError::Forward( std::log(1 + std::exp(-std::abs(prediction[i]))); } - ElemType lossSum = - maximum - arma::accu(prediction % target); + ElemType lossSum = maximum - arma::accu(prediction % target); if (reduction) return lossSum; From cbe29821238010ab68538d2a1db4e0670f8b3747 Mon Sep 17 00:00:00 2001 From: hello-fri-end Date: Fri, 25 Feb 2022 18:13:55 +0530 Subject: [PATCH 32/34] Should divide by target.n_elem and not predictions.n_elem in mean reduction --- .../ann/loss_functions/binary_cross_entropy_loss_impl.hpp | 2 +- .../methods/ann/loss_functions/earth_mover_distance_impl.hpp | 2 +- .../methods/ann/loss_functions/hinge_embedding_loss_impl.hpp | 4 ++-- src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp | 4 ++-- src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp | 4 ++-- src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp | 4 ++-- src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp | 4 ++-- .../methods/ann/loss_functions/mean_bias_error_impl.hpp | 4 ++-- .../methods/ann/loss_functions/mean_squared_error_impl.hpp | 4 ++-- .../loss_functions/mean_squared_logarithmic_error_impl.hpp | 4 ++-- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp index 666d285763..a217141564 100644 --- a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp @@ -53,7 +53,7 @@ void BCELoss::Backward( loss = (1. - target) / (1. - prediction + eps) - target / (prediction + eps); if (!reduction) - loss /= prediction.n_elem; + loss /= target.n_elem; } template diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp index 7cf360f17e..a049e1dc5a 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp @@ -38,7 +38,7 @@ EarthMoverDistance::Forward( if (reduction) return lossSum; - return lossSum / prediction.n_elem; + return lossSum / target.n_elem; } template diff --git a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp index 39be4aa738..701891d089 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp @@ -39,7 +39,7 @@ HingeEmbeddingLoss::Forward( if (reduction) return lossSum; - return lossSum / prediction.n_elem; + return lossSum / target.n_elem; } template @@ -52,7 +52,7 @@ void HingeEmbeddingLoss::Backward( loss = target; if (!reduction) - loss = loss / prediction.n_elem; + loss = loss / target.n_elem; } template diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp index 8c5bf87cae..f2e496aaed 100644 --- a/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp @@ -47,7 +47,7 @@ HuberLoss::Forward( if (reduction) return lossSum; - return lossSum / prediction.n_elem; + return lossSum / target.n_elem; } template @@ -69,7 +69,7 @@ void HuberLoss::Backward( } if (!reduction) - loss = loss / prediction.n_elem; + loss = loss / target.n_elem; } template diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp index f5e72bc7e5..6ff54ac21b 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp @@ -39,7 +39,7 @@ KLDivergence::Forward( if (reduction) return lossSum; - return lossSum / prediction.n_elem; + return lossSum / target.n_elem; } template @@ -52,7 +52,7 @@ void KLDivergence::Backward( loss = - target; if (!reduction) - loss = loss / prediction.n_elem; + loss = loss / target.n_elem; } template diff --git a/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp index 74a7ce987e..dfdfbc573d 100644 --- a/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp @@ -38,7 +38,7 @@ L1Loss::Forward( if (reduction) return lossSum; - return lossSum / prediction.n_elem; + return lossSum / target.n_elem; } template @@ -51,7 +51,7 @@ void L1Loss::Backward( loss = arma::sign(prediction - target); if (!reduction) - loss = loss / prediction.n_elem; + loss = loss / target.n_elem; } template diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp index 91de00668e..7c4bba7b9b 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp @@ -40,7 +40,7 @@ LogCoshLoss::Forward( if (reduction) return lossSum; - return lossSum / prediction.n_elem; + return lossSum / target.n_elem; } template @@ -53,7 +53,7 @@ void LogCoshLoss::Backward( loss = arma::tanh(a * (target - prediction)); if (!reduction) - loss = loss / prediction.n_elem; + loss = loss / target.n_elem; } template diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp index b6cec44f46..adfca2bcf7 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp @@ -39,7 +39,7 @@ MeanBiasError::Forward( if (reduction) return lossSum; - return lossSum/ prediction.n_elem; + return lossSum/ target.n_elem; } template @@ -53,7 +53,7 @@ void MeanBiasError::Backward( loss.fill(-1.0); if (!reduction) - loss = loss / prediction.n_elem; + loss = loss / target.n_elem; } template diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp index ffe1f49d0a..3bd64bc638 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp @@ -38,7 +38,7 @@ MeanSquaredError::Forward( if (reduction) return lossSum; - return lossSum / prediction.n_elem; + return lossSum / target.n_elem; } template @@ -51,7 +51,7 @@ void MeanSquaredError::Backward( loss = 2 * (prediction - target); if (!reduction) - loss = loss / prediction.n_elem; + loss = loss / target.n_elem; } template diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp index 9fae31c342..4b1d3740c7 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp @@ -39,7 +39,7 @@ MeanSquaredLogarithmicError::Forward( if (reduction) return lossSum; - return lossSum / prediction.n_elem; + return lossSum / target.n_elem; } template @@ -53,7 +53,7 @@ void MeanSquaredLogarithmicError::Backward( (1. + prediction); if (!reduction) - loss = loss / prediction.n_elem; + loss = loss / target.n_elem; } template From f5c80237c8619512cf4c5f7a9f01828103eb7066 Mon Sep 17 00:00:00 2001 From: hello-fri-end Date: Fri, 25 Feb 2022 19:46:33 +0530 Subject: [PATCH 33/34] small fix in mean_bias_error --- .../methods/ann/loss_functions/mean_bias_error_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp index adfca2bcf7..fa801ff275 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp @@ -39,7 +39,7 @@ MeanBiasError::Forward( if (reduction) return lossSum; - return lossSum/ target.n_elem; + return lossSum / target.n_elem; } template @@ -53,7 +53,7 @@ void MeanBiasError::Backward( loss.fill(-1.0); if (!reduction) - loss = loss / target.n_elem; + loss = loss / loss.n_elem; } template From 1978faadbbc82f5a65d0857d19d94423a71fafeb Mon Sep 17 00:00:00 2001 From: hello-fri-end Date: Sat, 26 Feb 2022 09:24:02 +0530 Subject: [PATCH 34/34] apply documentation suggestion --- src/mlpack/methods/ann/loss_functions/kl_divergence.hpp | 3 +-- .../methods/ann/loss_functions/negative_log_likelihood.hpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp index ff4949b395..0962c3c4e4 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp @@ -101,8 +101,7 @@ class KLDivergence //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class KLDivergence diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp index bb825aca6e..49e8d06957 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp @@ -112,8 +112,7 @@ class NegativeLogLikelihood //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Boolean value that tells if reduction - // is 'sum' or 'mean'. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class NegativeLogLikelihood