diff --git a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp index cc114da81f..8893227bc7 100644 --- a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp @@ -38,8 +38,12 @@ class BCELoss * * @param eps The minimum value used for computing logarithms * and denominators in a numerically stable way. - * @param reduction Reduction type. If true, it returns the mean of - * the loss. Else, it returns the sum. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ BCELoss(const double eps = 1e-10, const bool reduction = true); @@ -77,9 +81,10 @@ class BCELoss //! Modify the epsilon. double& Eps() { return eps; } - //! Get the reduction. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } - //! Set the reduction. + //! Modify the type of reduction used. bool& Reduction() { return reduction; } /** @@ -95,7 +100,7 @@ class BCELoss //! The minimum value used for computing logarithms and denominators double eps; - //! Reduction type. If true, performs mean of loss else sum. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class BCELoss diff --git a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp index 4555240b88..a217141564 100644 --- a/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp @@ -34,11 +34,13 @@ BCELoss::Forward( { typedef typename PredictionType::elem_type ElemType; - ElemType loss = -arma::accu(target % arma::log(prediction + eps) + + ElemType lossSum = -arma::accu(target % arma::log(prediction + eps) + (1. - target) % arma::log(1. - prediction + eps)); + if (reduction) - loss /= prediction.n_elem; - return loss; + return lossSum; + + return lossSum / target.n_elem; } template @@ -49,8 +51,9 @@ void BCELoss::Backward( LossType& loss) { loss = (1. - target) / (1. - prediction + eps) - target / (prediction + eps); - if (reduction) - loss /= prediction.n_elem; + + if (!reduction) + loss /= target.n_elem; } template @@ -60,6 +63,7 @@ void BCELoss::serialize( const uint32_t /* version */) { ar(CEREAL_NVP(eps)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp index c7df95e4a4..362d0d4e3b 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp @@ -46,13 +46,15 @@ class CosineEmbeddingLoss * Refer definition of cosine-embedding-loss above. * @param similarity Determines whether to use similarity or dissimilarity for * comparision. - * @param takeMean Boolean variable to specify whether to take mean or not. - * Specifies reduction method i.e. sum or mean corresponding - * to 0 and 1 respectively. Default value = 0. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ CosineEmbeddingLoss(const double margin = 0.0, const bool similarity = true, - const bool takeMean = false); + const bool reduction = true); /** * Ordinary feed forward pass of a neural network. @@ -93,10 +95,11 @@ class CosineEmbeddingLoss //! Modify the delta. OutputDataType& Delta() { return delta; } - //! Get the value of takeMean. - bool TakeMean() const { return takeMean; } - //! Modify the value of takeMean. - bool& TakeMean() { return takeMean; } + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } //! Get the value of margin. double Margin() const { return margin; } @@ -130,8 +133,8 @@ class CosineEmbeddingLoss //! Locally-stored value of similarity hyper-parameter. bool similarity; - //! Locally-stored value of takeMean hyper-parameter. - bool takeMean; + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class CosineEmbeddingLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp index 129a12c26c..bee9906032 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp @@ -20,8 +20,8 @@ namespace ann /** Artificial Neural Network. */ { template CosineEmbeddingLoss::CosineEmbeddingLoss( - const double margin, const bool similarity, const bool takeMean): - margin(margin), similarity(similarity), takeMean(takeMean) + const double margin, const bool similarity, const bool reduction): + margin(margin), similarity(similarity), reduction(reduction) { // Nothing to do here. } @@ -42,7 +42,7 @@ CosineEmbeddingLoss::Forward( arma::colvec inputTemp1 = arma::vectorise(prediction); arma::colvec inputTemp2 = arma::vectorise(target); - ElemType loss = 0.0; + ElemType lossSum = 0.0; for (size_t i = 0; i < inputTemp1.n_elem; i += cols) { @@ -50,18 +50,18 @@ CosineEmbeddingLoss::Forward( inputTemp1(arma::span(i, i + cols - 1)), inputTemp2(arma::span(i, i + cols - 1))); if (similarity) - loss += 1 - cosDist; + lossSum += 1 - cosDist; else { const ElemType currentLoss = cosDist - margin; - loss += currentLoss > 0 ? currentLoss : 0; + lossSum += currentLoss > 0 ? currentLoss : 0; } } - if (takeMean) - loss = (ElemType) loss / batchSize; + if (reduction) + return lossSum; - return loss; + return (ElemType) lossSum / batchSize; } template @@ -74,6 +74,7 @@ void CosineEmbeddingLoss::Backward( typedef typename PredictionType::elem_type ElemType; const size_t cols = prediction.n_cols; + const size_t batchSize = prediction.n_elem / cols; if (arma::size(prediction) != arma::size(target)) Log::Fatal << "Input Tensors must have same dimensions." << std::endl; @@ -99,6 +100,9 @@ void CosineEmbeddingLoss::Backward( 1)))) / std::sqrt(arma::accu(arma::pow(inputTemp1(arma::span(i, i + cols - 1)), 2))); } + + if (!reduction) + outputTemp = outputTemp / batchSize; } } @@ -109,7 +113,7 @@ void CosineEmbeddingLoss::serialize( { ar(CEREAL_NVP(margin)); ar(CEREAL_NVP(similarity)); - ar(CEREAL_NVP(takeMean)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp index a5afbf37c2..e8013aac57 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp @@ -35,8 +35,14 @@ class EarthMoverDistance public: /** * Create the EarthMoverDistance object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - EarthMoverDistance(); + EarthMoverDistance(const bool reduction = true); /** * Ordinary feed forward pass of a neural network. @@ -67,6 +73,12 @@ class EarthMoverDistance //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the layer. */ @@ -76,6 +88,9 @@ class EarthMoverDistance private: //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class EarthMoverDistance } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp index 8f6ab6f52e..a049e1dc5a 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp @@ -19,7 +19,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -EarthMoverDistance::EarthMoverDistance() +EarthMoverDistance + ::EarthMoverDistance(const bool reduction) : reduction(reduction) { // Nothing to do here. } @@ -31,7 +32,13 @@ EarthMoverDistance::Forward( const PredictionType& prediction, const TargetType& target) { - return -arma::accu(target % prediction); + typename PredictionType::elem_type lossSum = + -arma::accu(target % prediction); + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -42,15 +49,18 @@ void EarthMoverDistance::Backward( LossType& loss) { loss = -target; + + if (!reduction) + loss = loss / target.n_elem; } template template void EarthMoverDistance::serialize( - Archive& /* ar */, + Archive& ar, const uint32_t /* version */) { - /* Nothing to do here */ + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp index 99e5d50ff2..74def487b8 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp @@ -38,8 +38,14 @@ class HingeEmbeddingLoss public: /** * Create the Hinge Embedding object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - HingeEmbeddingLoss(); + HingeEmbeddingLoss(const bool reduction = true); /** * Computes the Hinge Embedding loss function. @@ -70,6 +76,12 @@ class HingeEmbeddingLoss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const {return reduction; } + //! Modify the type of reduction used. + bool& Reduction() {return reduction; } + /** * Serialize the loss function. */ @@ -79,6 +91,10 @@ class HingeEmbeddingLoss private: //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean values that tells if reduction + // is 'sum' or 'mean'. + bool reduction; }; // class HingeEmbeddingLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp index 1f6456c71f..701891d089 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp @@ -20,7 +20,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -HingeEmbeddingLoss::HingeEmbeddingLoss() +HingeEmbeddingLoss + ::HingeEmbeddingLoss(const bool reduction) : reduction(reduction) { // Nothing to do here. } @@ -32,8 +33,13 @@ HingeEmbeddingLoss::Forward( const PredictionType& prediction, const TargetType& target) { - TargetType temp = target - (target == 0); - return (arma::accu(arma::max(1 - prediction % temp, 0.))) / target.n_elem; + PredictionType loss = (1 - target) / 2 + prediction % (target); + typename PredictionType::elem_type lossSum = arma::accu(loss); + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -43,17 +49,19 @@ void HingeEmbeddingLoss::Backward( const TargetType& target, LossType& loss) { - TargetType temp = target - (target == 0); - loss = (prediction < 1 / temp) % -temp; + loss = target; + + if (!reduction) + loss = loss / target.n_elem; } template template void HingeEmbeddingLoss::serialize( - Archive& /* ar */, + Archive& ar, const uint32_t /* version */) { - // Nothing to do here. + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp b/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp index 60a2002782..7829b20348 100644 --- a/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/hinge_loss.hpp @@ -77,7 +77,8 @@ class HingeLoss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } @@ -92,7 +93,7 @@ class HingeLoss //! Locally-stored output parameter object. OutputDataType outputParameter; - //! The boolean value that tells if reduction is sum or mean. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class HingeLoss diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp index f5ce03dba4..c1533c84f6 100644 --- a/src/mlpack/methods/ann/loss_functions/huber_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/huber_loss.hpp @@ -41,9 +41,13 @@ class HuberLoss * * @param delta The threshold value upto which squared error is followed and * after which absolute error is considered. - * @param mean If true then mean loss is computed otherwise sum. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - HuberLoss(const double delta = 1.0, const bool mean = true); + HuberLoss(const double delta = 1.0, const bool reduction = true); /** * Computes the Huber Loss function. @@ -79,10 +83,11 @@ class HuberLoss //! Set the value of delta. double& Delta() { return delta; } - //! Get the value of reduction type. - bool Mean() const { return mean; } - //! Set the value of reduction type. - bool& Mean() { return mean; } + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } /** * Serialize the layer. @@ -97,8 +102,8 @@ class HuberLoss //! Hyperparameter `delta` defines the point upto which MSE is considered. double delta; - //! Reduction type. If true, performs mean of loss else sum. - bool mean; + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class HuberLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp index 50b2c61858..f2e496aaed 100644 --- a/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp @@ -21,9 +21,9 @@ namespace ann /** Artificial Neural Network. */ { template HuberLoss::HuberLoss( const double delta, - const bool mean): + const bool reduction): delta(delta), - mean(mean) + reduction(reduction) { // Nothing to do here. } @@ -36,14 +36,18 @@ HuberLoss::Forward( const TargetType& target) { typedef typename PredictionType::elem_type ElemType; - ElemType loss = 0; + ElemType lossSum = 0; for (size_t i = 0; i < prediction.n_elem; ++i) { const ElemType absError = std::abs(target[i] - prediction[i]); - loss += absError > delta ? + lossSum += absError > delta ? delta * (absError - 0.5 * delta) : 0.5 * std::pow(absError, 2); } - return mean ? loss / prediction.n_elem : loss; + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -62,9 +66,10 @@ void HuberLoss::Backward( loss[i] = absError > delta ? -delta * (target[i] - prediction[i]) / absError : prediction[i] - target[i]; - if (mean) - loss[i] /= loss.n_elem; } + + if (!reduction) + loss = loss / target.n_elem; } template @@ -74,7 +79,7 @@ void HuberLoss::serialize( const uint32_t /* version */) { ar(CEREAL_NVP(delta)); - ar(CEREAL_NVP(mean)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp index 5eacb03b36..0962c3c4e4 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp @@ -49,9 +49,13 @@ class KLDivergence * Create the Kullback–Leibler Divergence object with the specified * parameters. * - * @param takeMean Boolean variable to specify whether to take mean or not. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - KLDivergence(const bool takeMean = false); + KLDivergence(const bool reduction = true); /** * Computes the Kullback–Leibler divergence error function. @@ -82,11 +86,11 @@ class KLDivergence //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the value of takeMean. - bool TakeMean() const { return takeMean; } - //! Modify the value of takeMean. - bool& TakeMean() { return takeMean; } - + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } /** * Serialize the loss function */ @@ -97,8 +101,8 @@ class KLDivergence //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Boolean variable for taking mean or not. - bool takeMean; + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class KLDivergence } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp index 9c74453a21..6ff54ac21b 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp @@ -20,8 +20,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -KLDivergence::KLDivergence(const bool takeMean) : - takeMean(takeMean) +KLDivergence::KLDivergence(const bool reduction): + reduction(reduction) { // Nothing to do here. } @@ -33,15 +33,13 @@ KLDivergence::Forward( const PredictionType& prediction, const TargetType& target) { - if (takeMean) - { - return arma::as_scalar(arma::mean( - arma::mean(prediction % (arma::log(prediction) - arma::log(target))))); - } - else - { - return arma::accu(prediction % (arma::log(prediction) - arma::log(target))); - } + PredictionType loss = target % (arma::log(target) - prediction); + typename PredictionType::elem_type lossSum = arma::accu(loss); + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -51,15 +49,10 @@ void KLDivergence::Backward( const TargetType& target, LossType& loss) { - if (takeMean) - { - loss = arma::mean(arma::mean( - arma::log(prediction) - arma::log(target) + 1)); - } - else - { - loss = arma::accu(arma::log(prediction) - arma::log(target) + 1); - } + loss = - target; + + if (!reduction) + loss = loss / target.n_elem; } template @@ -68,7 +61,7 @@ void KLDivergence::serialize( Archive& ar, const uint32_t /* version */) { - ar(CEREAL_NVP(takeMean)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/l1_loss.hpp b/src/mlpack/methods/ann/loss_functions/l1_loss.hpp index 552089bbd0..e8494ca915 100644 --- a/src/mlpack/methods/ann/loss_functions/l1_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/l1_loss.hpp @@ -18,8 +18,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * The L1 loss is a loss function that measures the mean absolute error (MAE) - * between each element in the input x and target y + * The L1 loss is a loss function that measures the mean absolute error (MAE) + * between each element in the input x and target y. * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). @@ -36,10 +36,14 @@ class L1Loss /** * Create the L1Loss object. * - * @param mean Reduction type. If true, it returns the mean of - * the loss. Else, it returns the sum. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. + * */ - L1Loss(const bool mean = true); + L1Loss(const bool reduction = true); /** * Computes the L1 Loss function. @@ -70,10 +74,11 @@ class L1Loss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the value of reduction type. - bool Mean() const { return mean; } - //! Set the value of reduction type. - bool& Mean() { return mean; } + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } /** * Serialize the layer. @@ -85,8 +90,8 @@ class L1Loss //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Reduction type. If true, performs mean of loss else sum. - bool mean; + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class L1Loss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp index 100823ad90..dfdfbc573d 100644 --- a/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp @@ -19,8 +19,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -L1Loss::L1Loss(const bool mean): - mean(mean) +L1Loss::L1Loss(const bool reduction): + reduction(reduction) { // Nothing to do here. } @@ -32,10 +32,13 @@ L1Loss::Forward( const PredictionType& prediction, const TargetType& target) { - if (mean) - return arma::accu(arma::mean(prediction - target)); + PredictionType loss = arma::abs(prediction - target); + typename PredictionType::elem_type lossSum = arma::accu(loss); - return arma::accu(prediction - target); + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -46,6 +49,9 @@ void L1Loss::Backward( LossType& loss) { loss = arma::sign(prediction - target); + + if (!reduction) + loss = loss / target.n_elem; } template @@ -54,7 +60,7 @@ void L1Loss::serialize( Archive& ar, const uint32_t /* version */) { - ar(CEREAL_NVP(mean)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp index db3090488e..23aa6baa4e 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp @@ -39,14 +39,18 @@ class LogCoshLoss * Create the Log-Hyperbolic-Cosine object with the specified * parameters. * - * @param a A double type value for smoothening loss function. - * It must be positive a real number, Sharpness of loss - * function is directly proportional to a. It can also - * act as a scaling factor hence making the loss - * function more sensitive to small losses around the - * origin. Default value = 1.0. + * @param a A double type value for smoothening loss function. It must be a + * positive real number. Sharpness of loss function is directly + * proportional to a. It can also act as a scaling factor, hence + * making the loss function more sensitive to small losses around + * the origin. Default value = 1.0. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - LogCoshLoss(const double a = 1.0); + LogCoshLoss(const double a = 1.0, const bool reduction = true); /** * Computes the Log-Hyperbolic-Cosine loss function. @@ -82,6 +86,12 @@ class LogCoshLoss //! Modify the value of hyperparameter a. double& A() { return a; } + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the loss function. */ @@ -94,6 +104,9 @@ class LogCoshLoss //! Hyperparameter a for smoothening function curve. double a; + + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class LogCoshLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp index 63a72cfa97..7c4bba7b9b 100644 --- a/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp @@ -20,8 +20,9 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -LogCoshLoss::LogCoshLoss(const double a) : - a(a) +LogCoshLoss::LogCoshLoss( + const double a, const bool reduction) : + a(a) , reduction(reduction) { Log::Assert(a > 0, "Hyper-Parameter \'a\' must be positive"); } @@ -33,7 +34,13 @@ LogCoshLoss::Forward( const PredictionType& prediction, const TargetType& target) { - return arma::accu(arma::log(arma::cosh(a * (target - prediction)))) / a; + typename PredictionType::elem_type lossSum = + arma::accu(arma::log(arma::cosh(a * (target - prediction)))) / a; + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -44,6 +51,9 @@ void LogCoshLoss::Backward( LossType& loss) { loss = arma::tanh(a * (target - prediction)); + + if (!reduction) + loss = loss / target.n_elem; } template @@ -53,6 +63,7 @@ void LogCoshLoss::serialize( const uint32_t /* version */) { ar(CEREAL_NVP(a)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index 28971c89f0..fabb65d78a 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -37,10 +37,14 @@ class MarginRankingLoss public: /** * Create the MarginRankingLoss object with Hyperparameter margin. - * Hyperparameter margin defines a minimum distance between correctly ranked - * samples. + * @param margin defines a minimum distance between correctly ranked samples. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - MarginRankingLoss(const double margin = 1.0); + MarginRankingLoss(const double margin = 1.0, const bool reduction = true); /** * Computes the Margin Ranking Loss function. @@ -80,6 +84,12 @@ class MarginRankingLoss //! Modify the margin parameter. double& Margin() { return margin; } + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the layer. */ @@ -92,6 +102,9 @@ class MarginRankingLoss //! The margin value used in calculating Margin Ranking Loss. double margin; + + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class MarginRankingLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp index 17c63ca8e8..033357d9e8 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp @@ -20,7 +20,8 @@ namespace ann /** Artifical Neural Network. */ { template MarginRankingLoss::MarginRankingLoss( - const double margin) : margin(margin) + const double margin, const bool reduction): + margin(margin), reduction(reduction) { // Nothing to do here. } @@ -37,8 +38,14 @@ MarginRankingLoss::Forward( predictionRows / 2 - 1); const PredictionType& prediction2 = prediction.rows(predictionRows / 2, predictionRows - 1); - return arma::accu(arma::max(arma::zeros(size(target)), - -target % (prediction1 - prediction2) + margin)) / target.n_cols; + + double lossSum = arma::accu(arma::max(arma::zeros(size(target)), + -target % (prediction1 - prediction2) + margin)); + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -57,10 +64,16 @@ void MarginRankingLoss::Backward( predictionRows / 2 - 1); const PredictionType& prediction2 = prediction.rows(predictionRows / 2, predictionRows - 1); - loss = -target % (prediction1 - prediction2) + margin; - loss.elem(arma::find(loss >= 0)).ones(); - loss.elem(arma::find(loss < 0)).zeros(); - loss = (prediction2 - prediction1) % loss / target.n_cols; + LossType lossPrediction1 = -target % (prediction1 - prediction2) + margin; + lossPrediction1.elem(arma::find(lossPrediction1 >= 0)).ones(); + lossPrediction1.elem(arma::find(lossPrediction1 < 0)).zeros(); + LossType lossPrediction2 = lossPrediction1; + lossPrediction1 = -target % lossPrediction1; + lossPrediction2 = target % lossPrediction2; + loss = arma::join_cols(lossPrediction1, lossPrediction2); + + if (!reduction) + loss = loss / target.n_elem; } template @@ -70,6 +83,7 @@ void MarginRankingLoss::serialize( const uint32_t /* version */) { ar(CEREAL_NVP(margin)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp index b9836f856f..fb84b2ba4c 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp @@ -35,8 +35,16 @@ class MeanBiasError public: /** * Create the MeanBiasError object. + * + * @param reduction Specifies the reduction to apply to + * the output. If false, 'mean' reduction + * is used, where sum of the output will + * be divided by the number of elements + * in the output. If true, 'sum' reduction + * is used and the output will be summed. + * It is set to true by default. */ - MeanBiasError(); + MeanBiasError(const bool reduction = true); /** * Computes the mean bias error function. @@ -67,6 +75,12 @@ class MeanBiasError //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const {return reduction; } + //! Modify the type of reduction used. + bool& Reduction() {return reduction; } + /** * Serialize the layer. */ @@ -76,6 +90,9 @@ class MeanBiasError private: //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class MeanBiasError } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp index 0343558693..fa801ff275 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp @@ -20,9 +20,10 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -MeanBiasError::MeanBiasError() +MeanBiasError:: + MeanBiasError(const bool reduction) : reduction(reduction) { - // Nothing to do here. + // Nothing to do here } template @@ -32,7 +33,13 @@ MeanBiasError::Forward( const PredictionType& prediction, const TargetType& target) { - return arma::accu(target - prediction) / target.n_cols; + PredictionType loss = target - prediction; + typename PredictionType::elem_type lossSum = arma::accu(loss); + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -44,15 +51,18 @@ void MeanBiasError::Backward( { loss.set_size(arma::size(prediction)); loss.fill(-1.0); + + if (!reduction) + loss = loss / loss.n_elem; } template template void MeanBiasError::serialize( - Archive& /* ar */, + Archive& ar, const uint32_t /* version */) { - // Nothing to do here. + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp index 0cc3f6378d..6006d76d12 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp @@ -36,8 +36,14 @@ class MeanSquaredError public: /** * Create the MeanSquaredError object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - MeanSquaredError(); + MeanSquaredError(const bool reduction = true); /** * Computes the mean squared error function. @@ -67,6 +73,12 @@ class MeanSquaredError OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } /** * Serialize the layer @@ -77,6 +89,9 @@ class MeanSquaredError private: //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class MeanSquaredError } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp index ee4ae8c021..3bd64bc638 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp @@ -19,7 +19,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -MeanSquaredError::MeanSquaredError() +MeanSquaredError + ::MeanSquaredError(const bool reduction) : reduction(reduction) { // Nothing to do here. } @@ -31,7 +32,13 @@ MeanSquaredError::Forward( const PredictionType& prediction, const TargetType& target) { - return arma::accu(arma::square(prediction - target)) / target.n_cols; + typename PredictionType::elem_type lossSum = + arma::accu(arma::square(prediction - target)); + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -41,16 +48,19 @@ void MeanSquaredError::Backward( const TargetType& target, LossType& loss) { - loss = 2 * (prediction - target) / target.n_cols; + loss = 2 * (prediction - target); + + if (!reduction) + loss = loss / target.n_elem; } template template void MeanSquaredError::serialize( - Archive& /* ar */, + Archive& ar, const uint32_t /* version */) { - // Nothing to do here. + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp index 14a7a08ad0..bdd702ef54 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp @@ -35,8 +35,14 @@ class MeanSquaredLogarithmicError public: /** * Create the MeanSquaredLogarithmicError object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - MeanSquaredLogarithmicError(); + MeanSquaredLogarithmicError(const bool reduction = true); /** * Computes the mean squared logarithmic error function. @@ -67,6 +73,12 @@ class MeanSquaredLogarithmicError //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the layer */ @@ -76,6 +88,9 @@ class MeanSquaredLogarithmicError private: //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class MeanSquaredLogarithmicError } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp index ffb1ea7cd8..4b1d3740c7 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp @@ -20,7 +20,7 @@ namespace ann /** Artificial Neural Network. */ { template MeanSquaredLogarithmicError -::MeanSquaredLogarithmicError() +::MeanSquaredLogarithmicError(const bool reduction) : reduction(reduction) { // Nothing to do here. } @@ -32,8 +32,14 @@ MeanSquaredLogarithmicError::Forward( const PredictionType& prediction, const TargetType& target) { - return arma::accu(arma::square(arma::log(1. + target) - - arma::log(1. + prediction))) / target.n_cols; + typename PredictionType::elem_type lossSum = + arma::accu(arma::square(arma::log(1.0 + target) - + arma::log(1.0 + prediction))); + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -44,16 +50,19 @@ void MeanSquaredLogarithmicError::Backward( LossType& loss) { loss = 2 * (arma::log(1. + prediction) - arma::log(1. + target)) / - ((1. + prediction) * target.n_cols); + (1. + prediction); + + if (!reduction) + loss = loss / target.n_elem; } template template void MeanSquaredLogarithmicError::serialize( - Archive& /* ar */, + Archive& ar, const uint32_t /* version */) { - // Nothing to do here. + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp b/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp index 8721bd7d4e..e707410155 100644 --- a/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp @@ -80,7 +80,8 @@ class MultiLabelSoftMarginLoss //! Modify the weights assigned to each class. arma::rowvec& ClassWeights() { return classWeights; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } @@ -95,7 +96,7 @@ class MultiLabelSoftMarginLoss //! Locally-stored output parameter object. OutputDataType outputParameter; - //! The boolean value that tells if reduction is sum or mean. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; //! A (1, numClasses) shaped vector with weights for each class. diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp index 4f97c152f5..49e8d06957 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp @@ -19,9 +19,9 @@ namespace ann /** Artificial Neural Network. */ { /** * Implementation of the negative log likelihood layer. The negative log - * likelihood layer expectes that the input contains log-probabilities for each - * class. The layer also expects a class index, in the range between 1 and the - * number of classes, as target when calling the Forward function. + * likelihood layer expects that the input contains log-probabilities for each + * class. The layer also expects a class index, in the range between 0 and + * number of classes -1, as target when calling the Forward function. * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). @@ -37,8 +37,14 @@ class NegativeLogLikelihood public: /** * Create the NegativeLogLikelihoodLayer object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - NegativeLogLikelihood(); + NegativeLogLikelihood(const bool reduction = true); /** * Computes the Negative log likelihood. @@ -84,6 +90,12 @@ class NegativeLogLikelihood //! Modify the delta. OutputDataType& Delta() { return delta; } + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the layer */ @@ -99,6 +111,9 @@ class NegativeLogLikelihood //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class NegativeLogLikelihood } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp index 1eace1d772..b33979d130 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp @@ -19,7 +19,8 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { template -NegativeLogLikelihood::NegativeLogLikelihood() +NegativeLogLikelihood + ::NegativeLogLikelihood(const bool reduction) : reduction(reduction) { // Nothing to do here. } @@ -32,16 +33,19 @@ NegativeLogLikelihood::Forward( const TargetType& target) { typedef typename PredictionType::elem_type ElemType; - ElemType output = 0; + ElemType lossSum = 0; for (size_t i = 0; i < prediction.n_cols; ++i) { Log::Assert(target(i) >= 0 && target(i) < prediction.n_rows, "Target class out of range."); - output -= prediction(target(i), i); + lossSum -= prediction(target(i), i); } - return output; + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -59,14 +63,17 @@ void NegativeLogLikelihood::Backward( loss(target(i), i) = -1; } + + if (!reduction) + loss = loss / target.n_elem; } template template void NegativeLogLikelihood::serialize( - Archive& /* ar */, const uint32_t /* version */) + Archive& ar, const uint32_t /* version */) { - // Nothing to do here. + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp index 31e1cb5620..e3ba107423 100644 --- a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp @@ -45,12 +45,16 @@ class PoissonNLLLoss * @param full Boolean value that determines whether to include Stirling's * approximation term. * @param eps A small value to prevent 0 in denominators and logarithms. - * @param mean When true, mean loss is computed otherwise total loss. + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ PoissonNLLLoss(const bool logInput = true, const bool full = false, const typename InputDataType::elem_type eps = 1e-08, - const bool mean = true); + const bool reduction = true); /** * Computes the Poisson negative log likelihood Loss. @@ -112,13 +116,11 @@ class PoissonNLLLoss //! logarithms and denominators. typename InputDataType::elem_type& Eps() { return eps; } - //! Get the value of mean. It's a boolean value that tells if - //! mean of the total loss has to be taken. - bool Mean() const { return mean; } - //! Modify the value of mean. It's a boolean value that tells if - //! mean of the total loss has to be taken. - bool& Mean() { return mean; } - + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } /** * Serialize the layer. */ @@ -154,8 +156,9 @@ class PoissonNLLLoss //! eps is a small value required to prevent 0 in logarithms and denominators. typename InputDataType::elem_type eps; - //! Boolean value that tells if mean of the total loss has to be taken. - bool mean; + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; + }; // class PoissonNLLLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp index 05d2d79886..3152d73f56 100644 --- a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp @@ -24,11 +24,11 @@ PoissonNLLLoss::PoissonNLLLoss( const bool logInput, const bool full, const typename InputDataType::elem_type eps, - const bool mean): + const bool reduction): logInput(logInput), full(full), eps(eps), - mean(mean) + reduction(reduction) { Log::Assert(eps >= 0, "Epsilon (eps) must be greater than or equal to zero."); } @@ -57,8 +57,12 @@ PoissonNLLLoss::Forward( + 0.5 * arma::log(2 * M_PI * target); loss.elem(arma::find(mask)) += approx.elem(arma::find(mask)); } - - return mean ? arma::accu(loss) / loss.n_elem : arma::accu(loss); + typename PredictionType::elem_type lossSum = arma::accu(loss); + + if (reduction) + return lossSum; + + return lossSum / loss.n_elem; } template @@ -75,7 +79,7 @@ void PoissonNLLLoss::Backward( else loss = (1 - target / (prediction + eps)); - if (mean) + if (!reduction) loss = loss / loss.n_elem; } @@ -88,7 +92,7 @@ void PoissonNLLLoss::serialize( ar(CEREAL_NVP(logInput)); ar(CEREAL_NVP(full)); ar(CEREAL_NVP(eps)); - ar(CEREAL_NVP(mean)); + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp index 56f6f39cc3..454c22cb31 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp @@ -39,8 +39,14 @@ class ReconstructionLoss public: /** * Create the ReconstructionLoss object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. */ - ReconstructionLoss(); + ReconstructionLoss(const bool reduction = true); /** * Computes the reconstruction loss. @@ -71,6 +77,12 @@ class ReconstructionLoss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the layer */ @@ -83,6 +95,9 @@ class ReconstructionLoss //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class ReconstructionLoss } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp index ca5c986f05..ccb70b90f0 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp @@ -23,7 +23,7 @@ ReconstructionLoss< InputDataType, OutputDataType, DistType ->::ReconstructionLoss() +>::ReconstructionLoss(const bool reduction) : reduction(reduction) { // Nothing to do here. } @@ -35,7 +35,12 @@ ReconstructionLoss::Forward( const PredictionType& prediction, const TargetType& target) { dist = DistType(prediction); - return -dist.LogProbability(target); + typename PredictionType::elem_type lossSum = -dist.LogProbability(target); + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -47,15 +52,18 @@ void ReconstructionLoss::Backward( { dist.LogProbBackward(target, loss); loss *= -1; + + if (!reduction) + loss = loss / target.n_elem; } template template void ReconstructionLoss::serialize( - Archive& /* ar */, + Archive& ar, const uint32_t /* version */) { - // Nothing to do here. + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp index a1f4384e7f..04f9419521 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp @@ -3,7 +3,7 @@ * @author Kris Singh * @author Shikhar Jaiswal * - * Definition of the cross-entropy with logit performance function. + * Definition of the cross-entropy with logits performance function. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -54,8 +54,14 @@ class SigmoidCrossEntropyError public: /** * Create the SigmoidCrossEntropyError object. - */ - SigmoidCrossEntropyError(); + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If true, + * 'sum' reduction is used and the output will be summed. It + * is set to true by default. + */ + SigmoidCrossEntropyError(const bool reduction = true); /** * Computes the Sigmoid CrossEntropy Error functions. @@ -87,6 +93,12 @@ class SigmoidCrossEntropyError //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + /** * Serialize the layer. */ @@ -96,6 +108,9 @@ class SigmoidCrossEntropyError private: //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Boolean value that tells if reduction is 'sum' or 'mean'. + bool reduction; }; // class SigmoidCrossEntropy } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp index 93b3775e6a..e29e4a7478 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp @@ -22,7 +22,7 @@ namespace ann /** Artificial Neural Network. */ { template SigmoidCrossEntropyError -::SigmoidCrossEntropyError() +::SigmoidCrossEntropyError(const bool reduction): reduction(reduction) { // Nothing to do here. } @@ -42,7 +42,12 @@ SigmoidCrossEntropyError::Forward( std::log(1 + std::exp(-std::abs(prediction[i]))); } - return maximum - arma::accu(prediction % target); + ElemType lossSum = maximum - arma::accu(prediction % target); + + if (reduction) + return lossSum; + + return lossSum / target.n_elem; } template @@ -53,15 +58,18 @@ inline void SigmoidCrossEntropyError::Backward( LossType& loss) { loss = 1.0 / (1.0 + arma::exp(-prediction)) - target; + + if (!reduction) + loss = loss / target.n_elem; } template template void SigmoidCrossEntropyError::serialize( - Archive& /* ar */, + Archive& ar , const uint32_t /* version */) { - // Nothing to do here + ar(CEREAL_NVP(reduction)); } } // namespace ann diff --git a/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp b/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp index a35db04d14..a44e321fcd 100644 --- a/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp @@ -74,7 +74,8 @@ class SoftMarginLoss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the type of reduction used. + //! Get the reduction type, represented as boolean + //! (false 'mean' reduction, true 'sum' reduction). bool Reduction() const { return reduction; } //! Modify the type of reduction used. bool& Reduction() { return reduction; } @@ -89,7 +90,7 @@ class SoftMarginLoss //! Locally-stored output parameter object. OutputDataType outputParameter; - //! The boolean value that tells if reduction is sum or mean. + //! Boolean value that tells if reduction is 'sum' or 'mean'. bool reduction; }; // class SoftMarginLoss diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 37467c3119..2c800b1e21 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -52,25 +52,52 @@ using namespace mlpack::ann; TEST_CASE("HuberLossTest", "[LossFunctionsTest]") { arma::mat input, target, output; + arma::mat expectedOutput; + double loss; HuberLoss<> module; - // Test the Forward function. - input = arma::mat("17.45 12.91 13.63 29.01 7.12 15.47 31.52 31.97"); - target = arma::mat("16.52 13.11 13.67 29.51 24.31 15.03 30.72 34.07"); - double loss = module.Forward(input, target); - REQUIRE(loss == Approx(2.410631).epsilon(1e-5)); + // Test for sum reduction. + input = arma::mat("-0.0494 -1.1958 -1.0486 -0.2121 1.6028 0.0737 -0.7091 " + "0.8612 0.9639 0.9648 0.0745 0.5924"); + target = arma::mat("0.4316 0.0164 -0.4478 1.1452 0.5106 0.9255 0.5571 0.0864 " + "0.7059 -0.8288 -0.0231 -1.0526"); + expectedOutput = arma::mat("-0.4810 -1.0000 -0.6008 -1.0000 1.0000 -0.8518 " + "-1.0000 0.7748 0.2580 1.0000 0.0976 1.0000"); + input.reshape(4, 3); + target.reshape(4, 3); + expectedOutput.reshape(4, 3); - // Test the backward function. + // Test the Forward function. Loss should be 6.36364. + // Value calculated using torch.nn.SmoothL1Loss(reduction='sum'). + loss = module.Forward(input, target); + REQUIRE(loss == Approx(6.36364).epsilon(1e-3)); + + // Test the Backward function. module.Backward(input, target, output); - - // Expected Output: - // [0.1162 -0.0250 -0.0050 -0.0625 -0.1250 0.0550 0.1000 -0.1250] - // Sum of Expected Output = -0.07125. - double expectedOutputSum = arma::accu(output); - REQUIRE(expectedOutputSum == Approx(-0.07125).epsilon(1e-5)); - + REQUIRE(arma::as_scalar(arma::accu(output)) == + Approx(-0.8032).epsilon(1e-3)); REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); + + // Test for mean reduction by modifying reduction parameter using accessor. + module.Reduction() = false; + expectedOutput = arma::mat("-0.0401 -0.0833 -0.0501 -0.0833 0.0833 -0.0710 " + "-0.0833 0.0646 0.0215 0.0833 0.0081 0.0833"); + expectedOutput.reshape(4, 3); + + // Test the Forward function. Loss should be 0.530304. + // Value calculated using torch.nn.SmoothL1Loss(reduction='mean'). + loss = module.Forward(input, target); + REQUIRE(loss == Approx(0.530304).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input, target, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == + Approx(-0.0669333).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); } /** @@ -81,10 +108,10 @@ TEST_CASE("PoissonNLLLossTest", "[LossFunctionsTest]") arma::mat input, target, input4, target4; arma::mat output1, output2, output3, output4; arma::mat expOutput1, expOutput2, expOutput3, expOutput4; - PoissonNLLLoss<> module1; - PoissonNLLLoss<> module2(true, true, 1e-08, false); - PoissonNLLLoss<> module3(true, true, 1e-08, true); - PoissonNLLLoss<> module4(false, true, 1e-08, true); + PoissonNLLLoss<> module1(true, false, 1e-8, false); + PoissonNLLLoss<> module2(true, true, 1e-08, true); + PoissonNLLLoss<> module3(true, true, 1e-08, false); + PoissonNLLLoss<> module4(false, true, 1e-08, false); // Test the Forward function on a user generated input. input = arma::mat("1.0 1.0 1.9 1.6 -1.9 3.7 -1.0 0.5"); @@ -141,19 +168,55 @@ TEST_CASE("PoissonNLLLossTest", "[LossFunctionsTest]") } /** - * Simple KL Divergence test. The loss should be zero if input = target. + * Simple KL Divergence test. */ TEST_CASE("SimpleKLDivergenceTest", "[LossFunctionsTest]") { arma::mat input, target, output; + arma::mat expectedOutput; double loss; - KLDivergence<> module(true); + KLDivergence<> module; - // Test the Forward function. Loss should be 0 if input = target. - input = arma::ones(10, 1); - target = arma::ones(10, 1); + // Test for sum reduction. + input = arma::mat("-0.7007 -2.0247 -0.7132 -0.4584 -0.2637 -1.1795 -0.1093 " + "-1.0530 -2.4250 -0.4556 -0.7861 -0.9120"); + target = arma::mat("0.0223 0.5185 0.1610 0.9152 0.1689 0.6977 0.2823 0.3971 " + "0.2939 0.8000 0.6816 0.8742"); + expectedOutput = arma::mat("-0.0223 -0.5185 -0.1610 -0.9152 -0.1689 -0.6977 " + "-0.2823 -0.3971 -0.2939 -0.8000 -0.6816 -0.8742"); + input.reshape(4, 3); + target.reshape(4, 3); + expectedOutput.reshape(4, 3); + + // Test the Forward function. Loss should be 2.33349. + // Value calculated using torch.nn.KLDivLoss(reduction='sum'). loss = module.Forward(input, target); - REQUIRE(loss == Approx(0.0).margin(1e-5)); + REQUIRE(loss == Approx(2.33349).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input, target, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-5.8127).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); + + // Test for mean reduction by modifying reduction parameter using accessor. + module.Reduction() = false; + expectedOutput = arma::mat("-0.0019 -0.0432 -0.0134 -0.0763 -0.0141 -0.0581 " + "-0.0235 -0.0331 -0.0245 -0.0667 -0.0568 -0.0728"); + expectedOutput.reshape(4, 3); + + // Test the Forward function. Loss should be 0.194458. + // Value calculated using torch.nn.KLDivLoss(reduction='mean'). + loss = module.Forward(input, target); + REQUIRE(loss == Approx(0.194458).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input, target, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-0.484392).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); } /* @@ -161,75 +224,49 @@ TEST_CASE("SimpleKLDivergenceTest", "[LossFunctionsTest]") */ TEST_CASE("SimpleMeanSquaredLogarithmicErrorTest", "[LossFunctionsTest]") { - arma::mat input, output, target; + arma::mat input, target, output, expectedOutput; + double loss; MeanSquaredLogarithmicError<> module; - // Test the Forward function on a user generator input and compare it against - // the manually calculated result. - input = arma::zeros(1, 8); - target = arma::zeros(1, 8); - double error = module.Forward(input, target); - REQUIRE(error == Approx(0.0).margin(1e-5)); + // Test for sum reduction. + input = arma::mat("-0.0494 1.1958 1.0486 -0.2121 1.6028 0.0737 -0.7091 " + "0.8612 0.9639 0.9648 0.0745 0.5924"); + target = arma::mat("0.4316 0.0164 -0.4478 1.1452 0.5106 0.9255 0.5571 0.0864 " + "0.7059 -0.8288 -0.0231 1.0526"); + expectedOutput = arma::mat("-0.8615 0.7016 1.2799 -2.5425 0.4181 -1.0880 " + "-11.5339 0.5785 0.1434 2.4840 0.1772 -0.3188"); + input.reshape(4, 3); + target.reshape(4, 3); + expectedOutput.reshape(4, 3); + + // Test the Forward function. Loss should be 13.2728. + loss = module.Forward(input, target); + REQUIRE(loss == Approx(13.2728).epsilon(1e-3)); // Test the Backward function. module.Backward(input, target, output); - // The output should be equal to 0. - CheckMatrices(input, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-10.5619).epsilon(1e-3)); REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); - // Test the error function on a single input. - input = arma::mat("2"); - target = arma::mat("3"); - error = module.Forward(input, target); - REQUIRE(error == Approx(0.082760974810151655).epsilon(1e-3)); - - // Test the Backward function on a single input. - module.Backward(input, target, output); - REQUIRE(arma::accu(output) == Approx(-0.1917880483011872).epsilon(1e-3)); - REQUIRE(output.n_elem == 1); -} - -/** - * Test to check KL Divergence loss function when we take mean. - */ -TEST_CASE("KLDivergenceMeanTest", "[LossFunctionsTest]") -{ - arma::mat input, target, output; - double loss; - KLDivergence<> module(true); - - // Test the Forward function. - input = arma::mat("1 1 1 1 1 1 1 1 1 1"); - target = arma::exp(arma::mat("2 1 1 1 1 1 1 1 1 1")); + // Test for mean reduction by modifying reduction parameter using accessor. + module.Reduction() = false; + expectedOutput = arma::mat("-0.0718 0.0585 0.1067 -0.2119 0.0348 -0.0907 " + "-0.9612 0.0482 0.0120 0.2070 0.0148 -0.0266"); + expectedOutput.reshape(4, 3); + // Test the Forward function. Loss should be 1.10606. loss = module.Forward(input, target); - REQUIRE(loss == Approx(-1.1).epsilon(1e-5)); + REQUIRE(loss == Approx(1.10606).epsilon(1e-3)); // Test the Backward function. module.Backward(input, target, output); - REQUIRE(arma::as_scalar(output) == Approx(-0.1).epsilon(1e-5)); -} - -/** - * Test to check KL Divergence loss function when we do not take mean. - */ -TEST_CASE("KLDivergenceNoMeanTest", "[LossFunctionsTest]") -{ - arma::mat input, target, output; - double loss; - KLDivergence<> module(false); - - // Test the Forward function. - input = arma::mat("1 1 1 1 1 1 1 1 1 1"); - target = arma::exp(arma::mat("2 1 1 1 1 1 1 1 1 1")); - - loss = module.Forward(input, target); - REQUIRE(loss == Approx(-11).epsilon(1e-5)); - - // Test the Backward function. - module.Backward(input, target, output); - REQUIRE(arma::as_scalar(output) == Approx(-1).epsilon(1e-5)); + REQUIRE(arma::as_scalar(arma::accu(output)) == + Approx(-0.880156).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); } /* @@ -238,9 +275,9 @@ TEST_CASE("KLDivergenceNoMeanTest", "[LossFunctionsTest]") TEST_CASE("SimpleMeanSquaredErrorTest", "[LossFunctionsTest]") { arma::mat input, output, target; - MeanSquaredError<> module; + MeanSquaredError<> module(false); - // Test the Forward function on a user generator input and compare it against + // Test the Forward function on a user generated input and compare it against // the manually calculated result. input = arma::mat("1.0 0.0 1.0 0.0 -1.0 0.0 -1.0 0.0"); target = arma::zeros(1, 8); @@ -267,6 +304,19 @@ TEST_CASE("SimpleMeanSquaredErrorTest", "[LossFunctionsTest]") // Test whether the output is negative. REQUIRE(arma::accu(output) == -2); REQUIRE(output.n_elem == 1); + + // Test for sum reduction + module.Reduction() = true; + + // Test the Forward function + error = module.Forward(input, target); + REQUIRE(error == Approx(1.0).epsilon(1e-5)); + + // Test the Backward function on a single input. + module.Backward(input, target, output); + // Test whether the output is negative. + REQUIRE(arma::accu(output) == -2); + REQUIRE(output.n_elem == 1); } /* @@ -275,8 +325,8 @@ TEST_CASE("SimpleMeanSquaredErrorTest", "[LossFunctionsTest]") TEST_CASE("SimpleBinaryCrossEntropyLossTest", "[LossFunctionsTest]") { arma::mat input1, input2, input3, output, target1, target2, target3; - BCELoss<> module1(1e-6, false); - BCELoss<> module2(1e-6, true); + BCELoss<> module1(1e-6, true); + BCELoss<> module2(1e-6, false); // Test the Forward function on a user generator input and compare it against // the manually calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); @@ -387,21 +437,23 @@ TEST_CASE("SimpleSigmoidCrossEntropyErrorTest", "[LossFunctionsTest]") TEST_CASE("SimpleEarthMoverDistanceLayerTest", "[LossFunctionsTest]") { arma::mat input1, input2, output, target1, target2, expectedOutput; + arma::mat input3, target3; + double loss; EarthMoverDistance<> module; // Test the Forward function on a user generator input and compare it against // the manually calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); target1 = arma::zeros(1, 8); - double error1 = module.Forward(input1, target1); + loss = module.Forward(input1, target1); double expected = 0.0; - REQUIRE(error1 / input1.n_elem - expected == Approx(0.0).margin(1e-7)); + REQUIRE(loss / input1.n_elem - expected == Approx(0.0).margin(1e-7)); input2 = arma::mat("1 2 3 4 5"); target2 = arma::mat("1 0 1 0 1"); - double error2 = module.Forward(input2, target2); + loss = module.Forward(input2, target2); expected = -1.8; - REQUIRE(error2 / input2.n_elem - expected == Approx(0.0).margin(1e-6)); + REQUIRE(loss / input2.n_elem - expected == Approx(0.0).margin(1e-6)); // Test the Backward function. module.Backward(input1, target1, output); @@ -417,6 +469,31 @@ TEST_CASE("SimpleEarthMoverDistanceLayerTest", "[LossFunctionsTest]") REQUIRE(output(i) - expectedOutput(i) == Approx(0.0).margin(1e-5)); REQUIRE(output.n_rows == input2.n_rows); REQUIRE(output.n_cols == input2.n_cols); + + // Test for mean reduction. + module.Reduction() = false; + input3 = arma::mat("-0.0494 -1.1958 -1.0486 -0.2121 1.6028 0.0737 -0.7091 " + "0.8612 0.9639 0.9648 0.0745 0.5924"); + target3 = arma::mat("0.4316 0.0164 -0.4478 1.1452 0.5106 0.9255 0.5571 " + "0.0864 0.7059 -0.8288 -0.0231 -1.0526"); + expectedOutput = arma::mat("-0.0360 -0.0014 0.0373 -0.0954 -0.0426 -0.0771 " + "-0.0464 -0.0072 -0.0588 0.0691 0.0019 0.0877"); + input3.reshape(4, 3); + target3.reshape(4, 3); + expectedOutput.reshape(4, 3); + + // Test the Forward function. Loss should be -0.00060089. + // Value calculated manually. + loss = module.Forward(input3, target3); + REQUIRE(loss == Approx(-0.00060089).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input3, target3, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == + Approx(-0.168867).epsilon(1e-3)); + REQUIRE(output.n_rows == input3.n_rows); + REQUIRE(output.n_cols == input3.n_cols); + CheckMatrices(output, expectedOutput, 0.1); } /* @@ -551,37 +628,49 @@ TEST_CASE("DiceLossTest", "[LossFunctionsTest]") */ TEST_CASE("SimpleMeanBiasErrorTest", "[LossFunctionsTest]") { - arma::mat input, output, target; + arma::mat input, target, output; + double loss; MeanBiasError<> module; - // Test the Forward function on a user generator input and compare it against - // the manually calculated result. - input = arma::mat("1.0 0.0 1.0 -1.0 -1.0 0.0 -1.0 0.0"); - target = arma::zeros(1, 8); - double error = module.Forward(input, target); - REQUIRE(error == 0.125); + // Test for sum reduction. + input = arma::mat("-0.0494 -1.1958 -1.0486 -0.2121 1.6028 0.0737 -0.7091 " + "0.8612 0.9639 0.9648 0.0745 0.5924"); + target = arma::mat("0.4316 0.0164 -0.4478 1.1452 0.5106 0.9255 0.5571 0.0864 " + "0.7059 -0.8288 -0.0231 -1.0526"); - // Test the Backward function. + input.reshape(4, 3); + target.reshape(4, 3); + + // Test the forward function. + // Loss should be 0.1081. + loss = module.Forward(input, target); + REQUIRE(loss == Approx(0.1081).epsilon(1e-5)); + + // Test the backward function. module.Backward(input, target, output); - // We should get a vector with -1 everywhere. - for (double el : output) - { + + for(double el : output) REQUIRE(el == -1); - } REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); - // Test the error function on a single input. - input = arma::mat("2"); - target = arma::mat("3"); - error = module.Forward(input, target); - REQUIRE(error == 1.0); + // Test for mean reduction by modifying + // reduction parameter using accessor. + module.Reduction() = false; - // Test the Backward function on a single input. + // Test the forward function + // loss should be 0.00900833 + loss = module.Forward(input, target); + REQUIRE(loss == Approx(0.00900833).epsilon(1e-5)); + + // Test the backward function module.Backward(input, target, output); - // Test whether the output is negative. - REQUIRE(arma::accu(output) == -1); - REQUIRE(output.n_elem == 1); + + for(double el : output) + REQUIRE(el == Approx(-0.0833).epsilon(1e-3)); + REQUIRE(arma::accu(output) == Approx(-1).epsilon(1e-5)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); } /** @@ -610,9 +699,10 @@ TEST_CASE("LogCoshLossTest", "[LossFunctionsTest]") REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); - // Test the Forward function. Loss should be 0.546621. + // Test for sum reduction input = arma::mat("1 2 3 4 5"); target = arma::mat("1 2.4 3.4 4.2 5.5"); + // Test the Forward function. Loss should be 0.546621. loss = module.Forward(input, target); REQUIRE(loss == Approx(0.546621).epsilon(1e-3)); @@ -620,6 +710,19 @@ TEST_CASE("LogCoshLossTest", "[LossFunctionsTest]") module.Backward(input, target, output); REQUIRE(arma::accu(output) == Approx(2.46962).epsilon(1e-3)); REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + + // Test for mean reduction by modifying reduction parameter using accessor. + module.Reduction() = false; + + // Test the Forward function. Loss should be 0.109324. + loss = module.Forward(input, target); + REQUIRE(loss == Approx(0.109324).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input, target, output); + REQUIRE(arma::accu(output) == Approx(0.49392).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); } @@ -628,38 +731,51 @@ TEST_CASE("LogCoshLossTest", "[LossFunctionsTest]") */ TEST_CASE("HingeEmbeddingLossTest", "[LossFunctionsTest]") { - arma::mat input, target, output; + arma::mat input, target, output, expectedOutput; double loss; HingeEmbeddingLoss<> module; - // Test the Forward function. Loss should be 0 if input = target. - input = arma::ones(10, 1); - target = arma::ones(10, 1); + // Test for sum reduction + input = arma::mat("0.1778 0.0957 0.1397 0.2256 0.1203 0.2403 0.1925 0.3144 " + "-0.2264 -0.3400 -0.3336 -0.8695"); + target = arma::mat("1 1 -1 1 1 -1 1 1 -1 1 1 1"); + expectedOutput = arma::mat("1 1 -1 1 1 -1 1 1 -1 1 1 1"); + input.reshape(4, 3); + target.reshape(4, 3); + expectedOutput.reshape(4, 3); + + // Test the forward function + // Loss should be 2.4296 + // Value calculated using torch.nn.HingeEmbeddingLoss(reduction='sum') loss = module.Forward(input, target); - REQUIRE(loss == 0); + REQUIRE(loss == Approx(2.4296).epsilon(1e-3)); - // Test the Backward function for input = target. + // Test the Backward function module.Backward(input, target, output); - for (double el : output) - { - // For input = target we should get 0.0 everywhere. - REQUIRE(el == Approx(0.0).epsilon(1e-5)); - } - + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(6).epsilon(1e-3)); REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 1e-3); - // Test the Forward function. Loss should be 0.84. - input = arma::mat("0.1 0.8 0.6 0.0 0.5"); - target = arma::mat("0 1.0 1.0 0 0"); + // Test for mean reduction by modifying reduction + // parameter through the accessor. + module.Reduction() = false; + expectedOutput = arma::mat("0.0833 0.0833 -0.0833 0.0833 0.0833 -0.0833 " + "0.0833 0.0833 -0.0833 0.0833 0.0833 0.0833"); + expectedOutput.reshape(4, 3); + + // Test the forward function + // Loss should be 0.202467 + // Value calculated using torch.nn.HingeEmbeddingLoss(reduction='mean') loss = module.Forward(input, target); - REQUIRE(loss == Approx(0.84).epsilon(1e-3)); + REQUIRE(loss == Approx(0.202467).epsilon(1e-3)); - // Test the Backward function. + // Test the backward function module.Backward(input, target, output); - REQUIRE(arma::accu(output) == Approx(-2).epsilon(1e-3)); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(0.5).epsilon(1e-3)); REQUIRE(output.n_rows == input.n_rows); REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); } /** @@ -667,35 +783,25 @@ TEST_CASE("HingeEmbeddingLossTest", "[LossFunctionsTest]") */ TEST_CASE("SimpleL1LossTest", "[LossFunctionsTest]") { - arma::mat input1, input2, output, target1, target2; - L1Loss<> module(false); + arma::mat input, output, target; + double loss; + L1Loss<> module(true); // Test the Forward function on a user generator input and compare it against // the manually calculated result. - input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5"); - target1 = arma::zeros(1, 7); - double error1 = module.Forward(input1, target1); - REQUIRE(error1 == 3.5); - - input2 = arma::mat("0 1 1 0 1 0 0 1"); - target2 = arma::mat("0 1 1 0 1 0 0 1"); - double error2 = module.Forward(input2, target2); - REQUIRE(error2 == Approx(0.0).epsilon(1e-5)); + input = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5"); + target = arma::zeros(1, 7); + loss = module.Forward(input, target); + // Value calculated using torch.nn.L1Loss(reduction='sum'). + REQUIRE(loss == 3.5); // Test the Backward function. - module.Backward(input1, target1, output); + module.Backward(input, target, output); for (double el : output) REQUIRE(el == 1); - REQUIRE(output.n_rows == input1.n_rows); - REQUIRE(output.n_cols == input1.n_cols); - - module.Backward(input2, target2, output); - for (double el : output) - REQUIRE(el == 0); - - REQUIRE(output.n_rows == input2.n_rows); - REQUIRE(output.n_cols == input2.n_cols); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); } /** @@ -771,7 +877,7 @@ TEST_CASE("CosineEmbeddingLossTest", "[LossFunctionsTest]") REQUIRE(arma::accu(output) == Approx(-0.36649111).epsilon(1e-3)); // Check Output for mean type of reduction. - CosineEmbeddingLoss<> module3(0.0, true, true); + CosineEmbeddingLoss<> module3(0.0, true, false); loss = module3.Forward(input3, input4); REQUIRE(loss == Approx(0.092325).epsilon(1e-3)); @@ -782,7 +888,7 @@ TEST_CASE("CosineEmbeddingLossTest", "[LossFunctionsTest]") // Test the Backward function. module3.Backward(input3, input4, output); - REQUIRE(arma::accu(output) == Approx(0.36649111).epsilon(1e-4)); + REQUIRE(arma::accu(output) == Approx(0.0236749374).epsilon(1e-4)); } /* @@ -790,42 +896,51 @@ TEST_CASE("CosineEmbeddingLossTest", "[LossFunctionsTest]") */ TEST_CASE("MarginRankingLossTest", "[LossFunctionsTest]") { - arma::mat input, input1, input2, target, output; + arma::mat input, input1, input2, target, output, expectedOutput; + double loss; + // Test sum reduction MarginRankingLoss<> module; - - // Test the Forward function on a user generator input and compare it against - // the manually calculated result. - input1 = arma::mat("1 2 5 7 -1 -3"); - input2 = arma::mat("-1 3 -4 11 3 -3"); - input = arma::join_cols(input1, input2); - target = arma::mat("1 -1 -1 1 -1 1"); - double error = module.Forward(input, target); - // Computed using torch.nn.functional.margin_ranking_loss() - REQUIRE(error == Approx(2.66667).epsilon(1e-3)); - - // Test the Backward function. - module.Backward(input, target, output); - - CheckMatrices(output, arma::mat("-0.000000 0.166667 -1.500000 0.666667 " - "0.000000 -0.000000"), 1e-3); - REQUIRE(output.n_rows == target.n_rows); - REQUIRE(output.n_cols == target.n_cols); - - // Test the error function on another input. input1 = arma::mat("0.4287 -1.6208 -1.5006 -0.4473 1.5208 -4.5184 9.3574 " "-4.8090 4.3455 5.2070"); input2 = arma::mat("-4.5288 -9.2766 -0.5882 -5.6643 -6.0175 8.8506 3.4759 " "-9.4886 2.2755 8.4951"); + expectedOutput << 0.0000 << 0.0000 << 1.0000 << 0.0000 << 1.0000 << -1.0000 + << 0.0000 << 0.0000 << 1.0000 << -1.0000 < module; + + // Test for sum reduction. + input = arma::mat("-0.1689 -2.0033 -3.8886 -0.2862 -1.9392 -2.2532" + " -1.0543 -0.6196 -2.1769 -1.2865 -1.4797 -0.7011"); + target = arma::mat("2 2 1 2"); + expectedOutput = arma::mat("0 0 -1.0000 0 0 -1.0000 0 -1.0000 0 0 0 -1.0000"); + input.reshape(3, 4); + expectedOutput.reshape(3, 4); + + // Test the Forward function. Loss should be 7.4625. + // Value calculated using torch.nn.NLLLoss(reduction='sum'). + loss = module.Forward(input, target); + REQUIRE(loss == Approx(7.4625).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input, target, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-4).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); + + // Test for mean reduction by modifying reduction parameter using accessor. + module.Reduction() = false; + expectedOutput = arma::mat("0 0 -0.2500 0 0 -0.2500 0 -0.2500 0 0 0 -0.2500"); + expectedOutput.reshape(3, 4); + + // Test the Forward function. Loss should be 1.86562. + // Value calculated using torch.nn.NLLLoss(reduction='mean'). + loss = module.Forward(input, target); + REQUIRE(loss == Approx(1.86562).epsilon(1e-3)); + + // Test the Backward function. + module.Backward(input, target, output); + REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-1).epsilon(1e-3)); + REQUIRE(output.n_rows == input.n_rows); + REQUIRE(output.n_cols == input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); +}