apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3144 from hello-fri-end/fixLossFunctions

Browse Source 
Fix bugs in ANN loss functions.
This commit is contained in:
Marcus Edel
2022-03-10 22:40:54 -05:00
committed by GitHub
parent 7b7f721e84 1978faadbb
commit b5dfa9561b
36 changed files with 816 additions and 365 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss.hpp
+10 -5
View File
@@ -38,8 +38,12 @@ class BCELoss
*
* @param eps The minimum value used for computing logarithms
* and denominators in a numerically stable way.
* @param reduction Reduction type. If true, it returns the mean of
* the loss. Else, it returns the sum.
*
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
BCELoss(const double eps = 1e-10, const bool reduction = true);
@@ -77,9 +81,10 @@ class BCELoss
//! Modify the epsilon.
double& Eps() { return eps; }
//! Get the reduction.
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Set the reduction.
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
@@ -95,7 +100,7 @@ class BCELoss
//! The minimum value used for computing logarithms and denominators
double eps;
//! Reduction type. If true, performs mean of loss else sum.
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class BCELoss
src/mlpack/methods/ann/loss_functions/binary_cross_entropy_loss_impl.hpp
+9 -5
View File
@@ -34,11 +34,13 @@ BCELoss<InputDataType, OutputDataType>::Forward(
{
typedef typename PredictionType::elem_type ElemType;
ElemType loss = -arma::accu(target % arma::log(prediction + eps) +
ElemType lossSum = -arma::accu(target % arma::log(prediction + eps) +
(1. - target) % arma::log(1. - prediction + eps));
if (reduction)
loss /= prediction.n_elem;
return loss;
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -49,8 +51,9 @@ void BCELoss<InputDataType, OutputDataType>::Backward(
LossType& loss)
{
loss = (1. - target) / (1. - prediction + eps) - target / (prediction + eps);
if (reduction)
loss /= prediction.n_elem;
if (!reduction)
loss /= target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -60,6 +63,7 @@ void BCELoss<InputDataType, OutputDataType>::serialize(
const uint32_t /* version */)
{
ar(CEREAL_NVP(eps));
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp
+13 -10
View File
@@ -46,13 +46,15 @@ class CosineEmbeddingLoss
* Refer definition of cosine-embedding-loss above.
* @param similarity Determines whether to use similarity or dissimilarity for
* comparision.
* @param takeMean Boolean variable to specify whether to take mean or not.
* Specifies reduction method i.e. sum or mean corresponding
* to 0 and 1 respectively. Default value = 0.
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
CosineEmbeddingLoss(const double margin = 0.0,
const bool similarity = true,
const bool takeMean = false);
const bool reduction = true);
/**
* Ordinary feed forward pass of a neural network.
@@ -93,10 +95,11 @@ class CosineEmbeddingLoss
//! Modify the delta.
OutputDataType& Delta() { return delta; }
//! Get the value of takeMean.
bool TakeMean() const { return takeMean; }
//! Modify the value of takeMean.
bool& TakeMean() { return takeMean; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
//! Get the value of margin.
double Margin() const { return margin; }
@@ -130,8 +133,8 @@ class CosineEmbeddingLoss
//! Locally-stored value of similarity hyper-parameter.
bool similarity;
//! Locally-stored value of takeMean hyper-parameter.
bool takeMean;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class CosineEmbeddingLoss
} // namespace ann
src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp
+13 -9
View File
@@ -20,8 +20,8 @@ namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
CosineEmbeddingLoss<InputDataType, OutputDataType>::CosineEmbeddingLoss(
const double margin, const bool similarity, const bool takeMean):
margin(margin), similarity(similarity), takeMean(takeMean)
const double margin, const bool similarity, const bool reduction):
margin(margin), similarity(similarity), reduction(reduction)
{
// Nothing to do here.
}
@@ -42,7 +42,7 @@ CosineEmbeddingLoss<InputDataType, OutputDataType>::Forward(
arma::colvec inputTemp1 = arma::vectorise(prediction);
arma::colvec inputTemp2 = arma::vectorise(target);
ElemType loss = 0.0;
ElemType lossSum = 0.0;
for (size_t i = 0; i < inputTemp1.n_elem; i += cols)
{
@@ -50,18 +50,18 @@ CosineEmbeddingLoss<InputDataType, OutputDataType>::Forward(
inputTemp1(arma::span(i, i + cols - 1)), inputTemp2(arma::span(i,
i + cols - 1)));
if (similarity)
loss += 1 - cosDist;
lossSum += 1 - cosDist;
else
{
const ElemType currentLoss = cosDist - margin;
loss += currentLoss > 0 ? currentLoss : 0;
lossSum += currentLoss > 0 ? currentLoss : 0;
}
}
if (takeMean)
loss = (ElemType) loss / batchSize;
if (reduction)
return lossSum;
return loss;
return (ElemType) lossSum / batchSize;
}
template<typename InputDataType, typename OutputDataType>
@@ -74,6 +74,7 @@ void CosineEmbeddingLoss<InputDataType, OutputDataType>::Backward(
typedef typename PredictionType::elem_type ElemType;
const size_t cols = prediction.n_cols;
const size_t batchSize = prediction.n_elem / cols;
if (arma::size(prediction) != arma::size(target))
Log::Fatal << "Input Tensors must have same dimensions." << std::endl;
@@ -99,6 +100,9 @@ void CosineEmbeddingLoss<InputDataType, OutputDataType>::Backward(
1)))) / std::sqrt(arma::accu(arma::pow(inputTemp1(arma::span(i, i +
cols - 1)), 2)));
}
if (!reduction)
outputTemp = outputTemp / batchSize;
}
}
@@ -109,7 +113,7 @@ void CosineEmbeddingLoss<InputDataType, OutputDataType>::serialize(
{
ar(CEREAL_NVP(margin));
ar(CEREAL_NVP(similarity));
ar(CEREAL_NVP(takeMean));
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp
+16 -1
View File
@@ -35,8 +35,14 @@ class EarthMoverDistance
public:
/**
* Create the EarthMoverDistance object.
*
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
EarthMoverDistance();
EarthMoverDistance(const bool reduction = true);
/**
* Ordinary feed forward pass of a neural network.
@@ -67,6 +73,12 @@ class EarthMoverDistance
//! Modify the output parameter.
OutputDataType& OutputParameter() { return outputParameter; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
* Serialize the layer.
*/
@@ -76,6 +88,9 @@ class EarthMoverDistance
private:
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class EarthMoverDistance
} // namespace ann
src/mlpack/methods/ann/loss_functions/earth_mover_distance_impl.hpp
+14 -4
View File
@@ -19,7 +19,8 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
EarthMoverDistance<InputDataType, OutputDataType>::EarthMoverDistance()
EarthMoverDistance<InputDataType, OutputDataType>
::EarthMoverDistance(const bool reduction) : reduction(reduction)
{
// Nothing to do here.
}
@@ -31,7 +32,13 @@ EarthMoverDistance<InputDataType, OutputDataType>::Forward(
const PredictionType& prediction,
const TargetType& target)
{
return -arma::accu(target % prediction);
typename PredictionType::elem_type lossSum =
-arma::accu(target % prediction);
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -42,15 +49,18 @@ void EarthMoverDistance<InputDataType, OutputDataType>::Backward(
LossType& loss)
{
loss = -target;
if (!reduction)
loss = loss / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
template<typename Archive>
void EarthMoverDistance<InputDataType, OutputDataType>::serialize(
Archive& /* ar */,
Archive& ar,
const uint32_t /* version */)
{
/* Nothing to do here */
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp
+17 -1
View File
@@ -38,8 +38,14 @@ class HingeEmbeddingLoss
public:
/**
* Create the Hinge Embedding object.
*
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
HingeEmbeddingLoss();
HingeEmbeddingLoss(const bool reduction = true);
/**
* Computes the Hinge Embedding loss function.
@@ -70,6 +76,12 @@ class HingeEmbeddingLoss
//! Modify the output parameter.
OutputDataType& OutputParameter() { return outputParameter; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const {return reduction; }
//! Modify the type of reduction used.
bool& Reduction() {return reduction; }
/**
* Serialize the loss function.
*/
@@ -79,6 +91,10 @@ class HingeEmbeddingLoss
private:
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Boolean values that tells if reduction
// is 'sum' or 'mean'.
bool reduction;
}; // class HingeEmbeddingLoss
} // namespace ann
src/mlpack/methods/ann/loss_functions/hinge_embedding_loss_impl.hpp
+15 -7
View File
@@ -20,7 +20,8 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
HingeEmbeddingLoss<InputDataType, OutputDataType>::HingeEmbeddingLoss()
HingeEmbeddingLoss<InputDataType, OutputDataType>
::HingeEmbeddingLoss(const bool reduction) : reduction(reduction)
{
// Nothing to do here.
}
@@ -32,8 +33,13 @@ HingeEmbeddingLoss<InputDataType, OutputDataType>::Forward(
const PredictionType& prediction,
const TargetType& target)
{
TargetType temp = target - (target == 0);
return (arma::accu(arma::max(1 - prediction % temp, 0.))) / target.n_elem;
PredictionType loss = (1 - target) / 2 + prediction % (target);
typename PredictionType::elem_type lossSum = arma::accu(loss);
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -43,17 +49,19 @@ void HingeEmbeddingLoss<InputDataType, OutputDataType>::Backward(
const TargetType& target,
LossType& loss)
{
TargetType temp = target - (target == 0);
loss = (prediction < 1 / temp) % -temp;
loss = target;
if (!reduction)
loss = loss / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
template<typename Archive>
void HingeEmbeddingLoss<InputDataType, OutputDataType>::serialize(
Archive& /* ar */,
Archive& ar,
const uint32_t /* version */)
{
// Nothing to do here.
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/hinge_loss.hpp
+3 -2
View File
@@ -77,7 +77,8 @@ class HingeLoss
//! Modify the output parameter.
OutputDataType& OutputParameter() { return outputParameter; }
//! Get the type of reduction used.
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
@@ -92,7 +93,7 @@ class HingeLoss
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! The boolean value that tells if reduction is sum or mean.
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class HingeLoss
src/mlpack/methods/ann/loss_functions/huber_loss.hpp
+13 -8
View File
@@ -41,9 +41,13 @@ class HuberLoss
*
* @param delta The threshold value upto which squared error is followed and
* after which absolute error is considered.
* @param mean If true then mean loss is computed otherwise sum.
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
HuberLoss(const double delta = 1.0, const bool mean = true);
HuberLoss(const double delta = 1.0, const bool reduction = true);
/**
* Computes the Huber Loss function.
@@ -79,10 +83,11 @@ class HuberLoss
//! Set the value of delta.
double& Delta() { return delta; }
//! Get the value of reduction type.
bool Mean() const { return mean; }
//! Set the value of reduction type.
bool& Mean() { return mean; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
* Serialize the layer.
@@ -97,8 +102,8 @@ class HuberLoss
//! Hyperparameter `delta` defines the point upto which MSE is considered.
double delta;
//! Reduction type. If true, performs mean of loss else sum.
bool mean;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class HuberLoss
} // namespace ann
src/mlpack/methods/ann/loss_functions/huber_loss_impl.hpp
+13 -8
View File
@@ -21,9 +21,9 @@ namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
HuberLoss<InputDataType, OutputDataType>::HuberLoss(
const double delta,
const bool mean):
const bool reduction):
delta(delta),
mean(mean)
reduction(reduction)
{
// Nothing to do here.
}
@@ -36,14 +36,18 @@ HuberLoss<InputDataType, OutputDataType>::Forward(
const TargetType& target)
{
typedef typename PredictionType::elem_type ElemType;
ElemType loss = 0;
ElemType lossSum = 0;
for (size_t i = 0; i < prediction.n_elem; ++i)
{
const ElemType absError = std::abs(target[i] - prediction[i]);
loss += absError > delta ?
lossSum += absError > delta ?
delta * (absError - 0.5 * delta) : 0.5 * std::pow(absError, 2);
}
return mean ? loss / prediction.n_elem : loss;
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -62,9 +66,10 @@ void HuberLoss<InputDataType, OutputDataType>::Backward(
loss[i] = absError > delta ?
-delta * (target[i] - prediction[i]) / absError :
prediction[i] - target[i];
if (mean)
loss[i] /= loss.n_elem;
}
if (!reduction)
loss = loss / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -74,7 +79,7 @@ void HuberLoss<InputDataType, OutputDataType>::serialize(
const uint32_t /* version */)
{
ar(CEREAL_NVP(delta));
ar(CEREAL_NVP(mean));
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/kl_divergence.hpp
+13 -9
View File
@@ -49,9 +49,13 @@ class KLDivergence
* Create the Kullback–Leibler Divergence object with the specified
* parameters.
*
* @param takeMean Boolean variable to specify whether to take mean or not.
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
KLDivergence(const bool takeMean = false);
KLDivergence(const bool reduction = true);
/**
* Computes the Kullback–Leibler divergence error function.
@@ -82,11 +86,11 @@ class KLDivergence
//! Modify the output parameter.
OutputDataType& OutputParameter() { return outputParameter; }
//! Get the value of takeMean.
bool TakeMean() const { return takeMean; }
//! Modify the value of takeMean.
bool& TakeMean() { return takeMean; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
* Serialize the loss function
*/
@@ -97,8 +101,8 @@ class KLDivergence
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Boolean variable for taking mean or not.
bool takeMean;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class KLDivergence
} // namespace ann
src/mlpack/methods/ann/loss_functions/kl_divergence_impl.hpp
+14 -21
View File
@@ -20,8 +20,8 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
KLDivergence<InputDataType, OutputDataType>::KLDivergence(const bool takeMean) :
takeMean(takeMean)
KLDivergence<InputDataType, OutputDataType>::KLDivergence(const bool reduction):
reduction(reduction)
{
// Nothing to do here.
}
@@ -33,15 +33,13 @@ KLDivergence<InputDataType, OutputDataType>::Forward(
const PredictionType& prediction,
const TargetType& target)
{
if (takeMean)
{
return arma::as_scalar(arma::mean(
arma::mean(prediction % (arma::log(prediction) - arma::log(target)))));
}
else
{
return arma::accu(prediction % (arma::log(prediction) - arma::log(target)));
}
PredictionType loss = target % (arma::log(target) - prediction);
typename PredictionType::elem_type lossSum = arma::accu(loss);
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -51,15 +49,10 @@ void KLDivergence<InputDataType, OutputDataType>::Backward(
const TargetType& target,
LossType& loss)
{
if (takeMean)
{
loss = arma::mean(arma::mean(
arma::log(prediction) - arma::log(target) + 1));
}
else
{
loss = arma::accu(arma::log(prediction) - arma::log(target) + 1);
}
loss = - target;
if (!reduction)
loss = loss / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -68,7 +61,7 @@ void KLDivergence<InputDataType, OutputDataType>::serialize(
Archive& ar,
const uint32_t /* version */)
{
ar(CEREAL_NVP(takeMean));
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/l1_loss.hpp
+16 -11
View File
@@ -18,8 +18,8 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
/**
* The L1 loss is a loss function that measures the mean absolute error (MAE)
* between each element in the input x and target y
* The L1 loss is a loss function that measures the mean absolute error (MAE)
* between each element in the input x and target y.
*
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
* arma::sp_mat or arma::cube).
@@ -36,10 +36,14 @@ class L1Loss
/**
* Create the L1Loss object.
*
* @param mean Reduction type. If true, it returns the mean of
* the loss. Else, it returns the sum.
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*
*/
L1Loss(const bool mean = true);
L1Loss(const bool reduction = true);
/**
* Computes the L1 Loss function.
@@ -70,10 +74,11 @@ class L1Loss
//! Modify the output parameter.
OutputDataType& OutputParameter() { return outputParameter; }
//! Get the value of reduction type.
bool Mean() const { return mean; }
//! Set the value of reduction type.
bool& Mean() { return mean; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
* Serialize the layer.
@@ -85,8 +90,8 @@ class L1Loss
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Reduction type. If true, performs mean of loss else sum.
bool mean;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class L1Loss
} // namespace ann
src/mlpack/methods/ann/loss_functions/l1_loss_impl.hpp
+12 -6
View File
@@ -19,8 +19,8 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
L1Loss<InputDataType, OutputDataType>::L1Loss(const bool mean):
mean(mean)
L1Loss<InputDataType, OutputDataType>::L1Loss(const bool reduction):
reduction(reduction)
{
// Nothing to do here.
}
@@ -32,10 +32,13 @@ L1Loss<InputDataType, OutputDataType>::Forward(
const PredictionType& prediction,
const TargetType& target)
{
if (mean)
return arma::accu(arma::mean(prediction - target));
PredictionType loss = arma::abs(prediction - target);
typename PredictionType::elem_type lossSum = arma::accu(loss);
return arma::accu(prediction - target);
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -46,6 +49,9 @@ void L1Loss<InputDataType, OutputDataType>::Backward(
LossType& loss)
{
loss = arma::sign(prediction - target);
if (!reduction)
loss = loss / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -54,7 +60,7 @@ void L1Loss<InputDataType, OutputDataType>::serialize(
Archive& ar,
const uint32_t /* version */)
{
ar(CEREAL_NVP(mean));
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/log_cosh_loss.hpp
+20 -7
View File
@@ -39,14 +39,18 @@ class LogCoshLoss
* Create the Log-Hyperbolic-Cosine object with the specified
* parameters.
*
* @param a A double type value for smoothening loss function.
* It must be positive a real number, Sharpness of loss
* function is directly proportional to a. It can also
* act as a scaling factor hence making the loss
* function more sensitive to small losses around the
* origin. Default value = 1.0.
* @param a A double type value for smoothening loss function. It must be a
* positive real number. Sharpness of loss function is directly
* proportional to a. It can also act as a scaling factor, hence
* making the loss function more sensitive to small losses around
* the origin. Default value = 1.0.
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
LogCoshLoss(const double a = 1.0);
LogCoshLoss(const double a = 1.0, const bool reduction = true);
/**
* Computes the Log-Hyperbolic-Cosine loss function.
@@ -82,6 +86,12 @@ class LogCoshLoss
//! Modify the value of hyperparameter a.
double& A() { return a; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
* Serialize the loss function.
*/
@@ -94,6 +104,9 @@ class LogCoshLoss
//! Hyperparameter a for smoothening function curve.
double a;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class LogCoshLoss
} // namespace ann
src/mlpack/methods/ann/loss_functions/log_cosh_loss_impl.hpp
+14 -3
View File
@@ -20,8 +20,9 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
LogCoshLoss<InputDataType, OutputDataType>::LogCoshLoss(const double a) :
a(a)
LogCoshLoss<InputDataType, OutputDataType>::LogCoshLoss(
const double a, const bool reduction) :
a(a) , reduction(reduction)
{
Log::Assert(a > 0, "Hyper-Parameter \'a\' must be positive");
}
@@ -33,7 +34,13 @@ LogCoshLoss<InputDataType, OutputDataType>::Forward(
const PredictionType& prediction,
const TargetType& target)
{
return arma::accu(arma::log(arma::cosh(a * (target - prediction)))) / a;
typename PredictionType::elem_type lossSum =
arma::accu(arma::log(arma::cosh(a * (target - prediction)))) / a;
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -44,6 +51,9 @@ void LogCoshLoss<InputDataType, OutputDataType>::Backward(
LossType& loss)
{
loss = arma::tanh(a * (target - prediction));
if (!reduction)
loss = loss / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -53,6 +63,7 @@ void LogCoshLoss<InputDataType, OutputDataType>::serialize(
const uint32_t /* version */)
{
ar(CEREAL_NVP(a));
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp
+16 -3
View File
@@ -37,10 +37,14 @@ class MarginRankingLoss
public:
/**
* Create the MarginRankingLoss object with Hyperparameter margin.
* Hyperparameter margin defines a minimum distance between correctly ranked
* samples.
* @param margin defines a minimum distance between correctly ranked samples.
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
MarginRankingLoss(const double margin = 1.0);
MarginRankingLoss(const double margin = 1.0, const bool reduction = true);
/**
* Computes the Margin Ranking Loss function.
@@ -80,6 +84,12 @@ class MarginRankingLoss
//! Modify the margin parameter.
double& Margin() { return margin; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
* Serialize the layer.
*/
@@ -92,6 +102,9 @@ class MarginRankingLoss
//! The margin value used in calculating Margin Ranking Loss.
double margin;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class MarginRankingLoss
} // namespace ann
src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp
+21 -7
View File
@@ -20,7 +20,8 @@ namespace ann /** Artifical Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
MarginRankingLoss<InputDataType, OutputDataType>::MarginRankingLoss(
const double margin) : margin(margin)
const double margin, const bool reduction):
margin(margin), reduction(reduction)
{
// Nothing to do here.
}
@@ -37,8 +38,14 @@ MarginRankingLoss<InputDataType, OutputDataType>::Forward(
predictionRows / 2 - 1);
const PredictionType& prediction2 = prediction.rows(predictionRows / 2,
predictionRows - 1);
return arma::accu(arma::max(arma::zeros(size(target)),
-target % (prediction1 - prediction2) + margin)) / target.n_cols;
double lossSum = arma::accu(arma::max(arma::zeros(size(target)),
-target % (prediction1 - prediction2) + margin));
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -57,10 +64,16 @@ void MarginRankingLoss<InputDataType, OutputDataType>::Backward(
predictionRows / 2 - 1);
const PredictionType& prediction2 = prediction.rows(predictionRows / 2,
predictionRows - 1);
loss = -target % (prediction1 - prediction2) + margin;
loss.elem(arma::find(loss >= 0)).ones();
loss.elem(arma::find(loss < 0)).zeros();
loss = (prediction2 - prediction1) % loss / target.n_cols;
LossType lossPrediction1 = -target % (prediction1 - prediction2) + margin;
lossPrediction1.elem(arma::find(lossPrediction1 >= 0)).ones();
lossPrediction1.elem(arma::find(lossPrediction1 < 0)).zeros();
LossType lossPrediction2 = lossPrediction1;
lossPrediction1 = -target % lossPrediction1;
lossPrediction2 = target % lossPrediction2;
loss = arma::join_cols(lossPrediction1, lossPrediction2);
if (!reduction)
loss = loss / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -70,6 +83,7 @@ void MarginRankingLoss<InputDataType, OutputDataType>::serialize(
const uint32_t /* version */)
{
ar(CEREAL_NVP(margin));
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/mean_bias_error.hpp
+18 -1
View File
@@ -35,8 +35,16 @@ class MeanBiasError
public:
/**
* Create the MeanBiasError object.
*
* @param reduction Specifies the reduction to apply to
* the output. If false, 'mean' reduction
* is used, where sum of the output will
* be divided by the number of elements
* in the output. If true, 'sum' reduction
* is used and the output will be summed.
* It is set to true by default.
*/
MeanBiasError();
MeanBiasError(const bool reduction = true);
/**
* Computes the mean bias error function.
@@ -67,6 +75,12 @@ class MeanBiasError
//! Modify the output parameter.
OutputDataType& OutputParameter() { return outputParameter; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const {return reduction; }
//! Modify the type of reduction used.
bool& Reduction() {return reduction; }
/**
* Serialize the layer.
*/
@@ -76,6 +90,9 @@ class MeanBiasError
private:
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class MeanBiasError
} // namespace ann
src/mlpack/methods/ann/loss_functions/mean_bias_error_impl.hpp
+15 -5
View File
@@ -20,9 +20,10 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
MeanBiasError<InputDataType, OutputDataType>::MeanBiasError()
MeanBiasError<InputDataType, OutputDataType>::
MeanBiasError(const bool reduction) : reduction(reduction)
{
// Nothing to do here.
// Nothing to do here
}
template<typename InputDataType, typename OutputDataType>
@@ -32,7 +33,13 @@ MeanBiasError<InputDataType, OutputDataType>::Forward(
const PredictionType& prediction,
const TargetType& target)
{
return arma::accu(target - prediction) / target.n_cols;
PredictionType loss = target - prediction;
typename PredictionType::elem_type lossSum = arma::accu(loss);
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -44,15 +51,18 @@ void MeanBiasError<InputDataType, OutputDataType>::Backward(
{
loss.set_size(arma::size(prediction));
loss.fill(-1.0);
if (!reduction)
loss = loss / loss.n_elem;
}
template<typename InputDataType, typename OutputDataType>
template<typename Archive>
void MeanBiasError<InputDataType, OutputDataType>::serialize(
Archive& /* ar */,
Archive& ar,
const uint32_t /* version */)
{
// Nothing to do here.
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp
+16 -1
View File
@@ -36,8 +36,14 @@ class MeanSquaredError
public:
/**
* Create the MeanSquaredError object.
*
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
MeanSquaredError();
MeanSquaredError(const bool reduction = true);
/**
* Computes the mean squared error function.
@@ -67,6 +73,12 @@ class MeanSquaredError
OutputDataType& OutputParameter() const { return outputParameter; }
//! Modify the output parameter.
OutputDataType& OutputParameter() { return outputParameter; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
* Serialize the layer
@@ -77,6 +89,9 @@ class MeanSquaredError
private:
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class MeanSquaredError
} // namespace ann
src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp
+15 -5
View File
@@ -19,7 +19,8 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
MeanSquaredError<InputDataType, OutputDataType>::MeanSquaredError()
MeanSquaredError<InputDataType, OutputDataType>
::MeanSquaredError(const bool reduction) : reduction(reduction)
{
// Nothing to do here.
}
@@ -31,7 +32,13 @@ MeanSquaredError<InputDataType, OutputDataType>::Forward(
const PredictionType& prediction,
const TargetType& target)
{
return arma::accu(arma::square(prediction - target)) / target.n_cols;
typename PredictionType::elem_type lossSum =
arma::accu(arma::square(prediction - target));
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -41,16 +48,19 @@ void MeanSquaredError<InputDataType, OutputDataType>::Backward(
const TargetType& target,
LossType& loss)
{
loss = 2 * (prediction - target) / target.n_cols;
loss = 2 * (prediction - target);
if (!reduction)
loss = loss / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
template<typename Archive>
void MeanSquaredError<InputDataType, OutputDataType>::serialize(
Archive& /* ar */,
Archive& ar,
const uint32_t /* version */)
{
// Nothing to do here.
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error.hpp
+16 -1
View File
@@ -35,8 +35,14 @@ class MeanSquaredLogarithmicError
public:
/**
* Create the MeanSquaredLogarithmicError object.
*
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
MeanSquaredLogarithmicError();
MeanSquaredLogarithmicError(const bool reduction = true);
/**
* Computes the mean squared logarithmic error function.
@@ -67,6 +73,12 @@ class MeanSquaredLogarithmicError
//! Modify the output parameter.
OutputDataType& OutputParameter() { return outputParameter; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
* Serialize the layer
*/
@@ -76,6 +88,9 @@ class MeanSquaredLogarithmicError
private:
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class MeanSquaredLogarithmicError
} // namespace ann
src/mlpack/methods/ann/loss_functions/mean_squared_logarithmic_error_impl.hpp
+15 -6
View File
@@ -20,7 +20,7 @@ namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
MeanSquaredLogarithmicError<InputDataType, OutputDataType>
::MeanSquaredLogarithmicError()
::MeanSquaredLogarithmicError(const bool reduction) : reduction(reduction)
{
// Nothing to do here.
}
@@ -32,8 +32,14 @@ MeanSquaredLogarithmicError<InputDataType, OutputDataType>::Forward(
const PredictionType& prediction,
const TargetType& target)
{
return arma::accu(arma::square(arma::log(1. + target) -
arma::log(1. + prediction))) / target.n_cols;
typename PredictionType::elem_type lossSum =
arma::accu(arma::square(arma::log(1.0 + target) -
arma::log(1.0 + prediction)));
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -44,16 +50,19 @@ void MeanSquaredLogarithmicError<InputDataType, OutputDataType>::Backward(
LossType& loss)
{
loss = 2 * (arma::log(1. + prediction) - arma::log(1. + target)) /
((1. + prediction) * target.n_cols);
(1. + prediction);
if (!reduction)
loss = loss / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
template<typename Archive>
void MeanSquaredLogarithmicError<InputDataType, OutputDataType>::serialize(
Archive& /* ar */,
Archive& ar,
const uint32_t /* version */)
{
// Nothing to do here.
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/multilabel_softmargin_loss.hpp
+3 -2
View File
@@ -80,7 +80,8 @@ class MultiLabelSoftMarginLoss
//! Modify the weights assigned to each class.
arma::rowvec& ClassWeights() { return classWeights; }
//! Get the type of reduction used.
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
@@ -95,7 +96,7 @@ class MultiLabelSoftMarginLoss
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! The boolean value that tells if reduction is sum or mean.
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
//! A (1, numClasses) shaped vector with weights for each class.
src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp
+19 -4
View File
@@ -19,9 +19,9 @@ namespace ann /** Artificial Neural Network. */ {
/**
* Implementation of the negative log likelihood layer. The negative log
* likelihood layer expectes that the input contains log-probabilities for each
* class. The layer also expects a class index, in the range between 1 and the
* number of classes, as target when calling the Forward function.
* likelihood layer expects that the input contains log-probabilities for each
* class. The layer also expects a class index, in the range between 0 and
* number of classes -1, as target when calling the Forward function.
*
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
* arma::sp_mat or arma::cube).
@@ -37,8 +37,14 @@ class NegativeLogLikelihood
public:
/**
* Create the NegativeLogLikelihoodLayer object.
*
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
NegativeLogLikelihood();
NegativeLogLikelihood(const bool reduction = true);
/**
* Computes the Negative log likelihood.
@@ -84,6 +90,12 @@ class NegativeLogLikelihood
//! Modify the delta.
OutputDataType& Delta() { return delta; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
* Serialize the layer
*/
@@ -99,6 +111,9 @@ class NegativeLogLikelihood
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class NegativeLogLikelihood
} // namespace ann
src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp
+13 -6
View File
@@ -19,7 +19,8 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
NegativeLogLikelihood<InputDataType, OutputDataType>::NegativeLogLikelihood()
NegativeLogLikelihood<InputDataType, OutputDataType>
::NegativeLogLikelihood(const bool reduction) : reduction(reduction)
{
// Nothing to do here.
}
@@ -32,16 +33,19 @@ NegativeLogLikelihood<InputDataType, OutputDataType>::Forward(
const TargetType& target)
{
typedef typename PredictionType::elem_type ElemType;
ElemType output = 0;
ElemType lossSum = 0;
for (size_t i = 0; i < prediction.n_cols; ++i)
{
Log::Assert(target(i) >= 0 && target(i) < prediction.n_rows,
"Target class out of range.");
output -= prediction(target(i), i);
lossSum -= prediction(target(i), i);
}
return output;
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -59,14 +63,17 @@ void NegativeLogLikelihood<InputDataType, OutputDataType>::Backward(
loss(target(i), i) = -1;
}
if (!reduction)
loss = loss / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
template<typename Archive>
void NegativeLogLikelihood<InputDataType, OutputDataType>::serialize(
Archive& /* ar */, const uint32_t /* version */)
Archive& ar, const uint32_t /* version */)
{
// Nothing to do here.
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp
+14 -11
View File
@@ -45,12 +45,16 @@ class PoissonNLLLoss
* @param full Boolean value that determines whether to include Stirling's
* approximation term.
* @param eps A small value to prevent 0 in denominators and logarithms.
* @param mean When true, mean loss is computed otherwise total loss.
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
PoissonNLLLoss(const bool logInput = true,
const bool full = false,
const typename InputDataType::elem_type eps = 1e-08,
const bool mean = true);
const bool reduction = true);
/**
* Computes the Poisson negative log likelihood Loss.
@@ -112,13 +116,11 @@ class PoissonNLLLoss
//! logarithms and denominators.
typename InputDataType::elem_type& Eps() { return eps; }
//! Get the value of mean. It's a boolean value that tells if
//! mean of the total loss has to be taken.
bool Mean() const { return mean; }
//! Modify the value of mean. It's a boolean value that tells if
//! mean of the total loss has to be taken.
bool& Mean() { return mean; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
* Serialize the layer.
*/
@@ -154,8 +156,9 @@ class PoissonNLLLoss
//! eps is a small value required to prevent 0 in logarithms and denominators.
typename InputDataType::elem_type eps;
//! Boolean value that tells if mean of the total loss has to be taken.
bool mean;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class PoissonNLLLoss
} // namespace ann
src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp
+10 -6
View File
@@ -24,11 +24,11 @@ PoissonNLLLoss<InputDataType, OutputDataType>::PoissonNLLLoss(
const bool logInput,
const bool full,
const typename InputDataType::elem_type eps,
const bool mean):
const bool reduction):
logInput(logInput),
full(full),
eps(eps),
mean(mean)
reduction(reduction)
{
Log::Assert(eps >= 0, "Epsilon (eps) must be greater than or equal to zero.");
}
@@ -57,8 +57,12 @@ PoissonNLLLoss<InputDataType, OutputDataType>::Forward(
+ 0.5 * arma::log(2 * M_PI * target);
loss.elem(arma::find(mask)) += approx.elem(arma::find(mask));
}
return mean ? arma::accu(loss) / loss.n_elem : arma::accu(loss);
typename PredictionType::elem_type lossSum = arma::accu(loss);
if (reduction)
return lossSum;
return lossSum / loss.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -75,7 +79,7 @@ void PoissonNLLLoss<InputDataType, OutputDataType>::Backward(
else
loss = (1 - target / (prediction + eps));
if (mean)
if (!reduction)
loss = loss / loss.n_elem;
}
@@ -88,7 +92,7 @@ void PoissonNLLLoss<InputDataType, OutputDataType>::serialize(
ar(CEREAL_NVP(logInput));
ar(CEREAL_NVP(full));
ar(CEREAL_NVP(eps));
ar(CEREAL_NVP(mean));
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp
+16 -1
View File
@@ -39,8 +39,14 @@ class ReconstructionLoss
public:
/**
* Create the ReconstructionLoss object.
*
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
ReconstructionLoss();
ReconstructionLoss(const bool reduction = true);
/**
* Computes the reconstruction loss.
@@ -71,6 +77,12 @@ class ReconstructionLoss
//! Modify the output parameter.
OutputDataType& OutputParameter() { return outputParameter; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
* Serialize the layer
*/
@@ -83,6 +95,9 @@ class ReconstructionLoss
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class ReconstructionLoss
} // namespace ann
src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp
+12 -4
View File
@@ -23,7 +23,7 @@ ReconstructionLoss<
InputDataType,
OutputDataType,
DistType
>::ReconstructionLoss()
>::ReconstructionLoss(const bool reduction) : reduction(reduction)
{
// Nothing to do here.
}
@@ -35,7 +35,12 @@ ReconstructionLoss<InputDataType, OutputDataType, DistType>::Forward(
const PredictionType& prediction, const TargetType& target)
{
dist = DistType(prediction);
return -dist.LogProbability(target);
typename PredictionType::elem_type lossSum = -dist.LogProbability(target);
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType, typename DistType>
@@ -47,15 +52,18 @@ void ReconstructionLoss<InputDataType, OutputDataType, DistType>::Backward(
{
dist.LogProbBackward(target, loss);
loss *= -1;
if (!reduction)
loss = loss / target.n_elem;
}
template<typename InputDataType, typename OutputDataType, typename DistType>
template<typename Archive>
void ReconstructionLoss<InputDataType, OutputDataType, DistType>::serialize(
Archive& /* ar */,
Archive& ar,
const uint32_t /* version */)
{
// Nothing to do here.
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp
+18 -3
View File
@@ -3,7 +3,7 @@
* @author Kris Singh
* @author Shikhar Jaiswal
*
* Definition of the cross-entropy with logit performance function.
* Definition of the cross-entropy with logits performance function.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
@@ -54,8 +54,14 @@ class SigmoidCrossEntropyError
public:
/**
* Create the SigmoidCrossEntropyError object.
*/
SigmoidCrossEntropyError();
*
* @param reduction Specifies the reduction to apply to the output. If false,
* 'mean' reduction is used, where sum of the output will be
* divided by the number of elements in the output. If true,
* 'sum' reduction is used and the output will be summed. It
* is set to true by default.
*/
SigmoidCrossEntropyError(const bool reduction = true);
/**
* Computes the Sigmoid CrossEntropy Error functions.
@@ -87,6 +93,12 @@ class SigmoidCrossEntropyError
//! Modify the output parameter.
OutputDataType& OutputParameter() { return outputParameter; }
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
/**
* Serialize the layer.
*/
@@ -96,6 +108,9 @@ class SigmoidCrossEntropyError
private:
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class SigmoidCrossEntropy
} // namespace ann
src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error_impl.hpp
+12 -4
View File
@@ -22,7 +22,7 @@ namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
SigmoidCrossEntropyError<InputDataType, OutputDataType>
::SigmoidCrossEntropyError()
::SigmoidCrossEntropyError(const bool reduction): reduction(reduction)
{
// Nothing to do here.
}
@@ -42,7 +42,12 @@ SigmoidCrossEntropyError<InputDataType, OutputDataType>::Forward(
std::log(1 + std::exp(-std::abs(prediction[i])));
}
return maximum - arma::accu(prediction % target);
ElemType lossSum = maximum - arma::accu(prediction % target);
if (reduction)
return lossSum;
return lossSum / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
@@ -53,15 +58,18 @@ inline void SigmoidCrossEntropyError<InputDataType, OutputDataType>::Backward(
LossType& loss)
{
loss = 1.0 / (1.0 + arma::exp(-prediction)) - target;
if (!reduction)
loss = loss / target.n_elem;
}
template<typename InputDataType, typename OutputDataType>
template<typename Archive>
void SigmoidCrossEntropyError<InputDataType, OutputDataType>::serialize(
Archive& /* ar */,
Archive& ar ,
const uint32_t /* version */)
{
// Nothing to do here
ar(CEREAL_NVP(reduction));
}
} // namespace ann
src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp
+3 -2
View File
@@ -74,7 +74,8 @@ class SoftMarginLoss
//! Modify the output parameter.
OutputDataType& OutputParameter() { return outputParameter; }
//! Get the type of reduction used.
//! Get the reduction type, represented as boolean
//! (false 'mean' reduction, true 'sum' reduction).
bool Reduction() const { return reduction; }
//! Modify the type of reduction used.
bool& Reduction() { return reduction; }
@@ -89,7 +90,7 @@ class SoftMarginLoss
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! The boolean value that tells if reduction is sum or mean.
//! Boolean value that tells if reduction is 'sum' or 'mean'.
bool reduction;
}; // class SoftMarginLoss
src/mlpack/tests/loss_functions_test.cpp
+339 -176
View File
@@ -52,25 +52,52 @@ using namespace mlpack::ann;
TEST_CASE("HuberLossTest", "[LossFunctionsTest]")
{
arma::mat input, target, output;
arma::mat expectedOutput;
double loss;
HuberLoss<> module;
// Test the Forward function.
input = arma::mat("17.45 12.91 13.63 29.01 7.12 15.47 31.52 31.97");
target = arma::mat("16.52 13.11 13.67 29.51 24.31 15.03 30.72 34.07");
double loss = module.Forward(input, target);
REQUIRE(loss == Approx(2.410631).epsilon(1e-5));
// Test for sum reduction.
input = arma::mat("-0.0494 -1.1958 -1.0486 -0.2121 1.6028 0.0737 -0.7091 "
"0.8612 0.9639 0.9648 0.0745 0.5924");
target = arma::mat("0.4316 0.0164 -0.4478 1.1452 0.5106 0.9255 0.5571 0.0864 "
"0.7059 -0.8288 -0.0231 -1.0526");
expectedOutput = arma::mat("-0.4810 -1.0000 -0.6008 -1.0000 1.0000 -0.8518 "
"-1.0000 0.7748 0.2580 1.0000 0.0976 1.0000");
input.reshape(4, 3);
target.reshape(4, 3);
expectedOutput.reshape(4, 3);
// Test the backward function.
// Test the Forward function. Loss should be 6.36364.
// Value calculated using torch.nn.SmoothL1Loss(reduction='sum').
loss = module.Forward(input, target);
REQUIRE(loss == Approx(6.36364).epsilon(1e-3));
// Test the Backward function.
module.Backward(input, target, output);
// Expected Output:
// [0.1162 -0.0250 -0.0050 -0.0625 -0.1250 0.0550 0.1000 -0.1250]
// Sum of Expected Output = -0.07125.
double expectedOutputSum = arma::accu(output);
REQUIRE(expectedOutputSum == Approx(-0.07125).epsilon(1e-5));
REQUIRE(arma::as_scalar(arma::accu(output)) ==
Approx(-0.8032).epsilon(1e-3));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
// Test for mean reduction by modifying reduction parameter using accessor.
module.Reduction() = false;
expectedOutput = arma::mat("-0.0401 -0.0833 -0.0501 -0.0833 0.0833 -0.0710 "
"-0.0833 0.0646 0.0215 0.0833 0.0081 0.0833");
expectedOutput.reshape(4, 3);
// Test the Forward function. Loss should be 0.530304.
// Value calculated using torch.nn.SmoothL1Loss(reduction='mean').
loss = module.Forward(input, target);
REQUIRE(loss == Approx(0.530304).epsilon(1e-3));
// Test the Backward function.
module.Backward(input, target, output);
REQUIRE(arma::as_scalar(arma::accu(output)) ==
Approx(-0.0669333).epsilon(1e-3));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
}
/**
@@ -81,10 +108,10 @@ TEST_CASE("PoissonNLLLossTest", "[LossFunctionsTest]")
arma::mat input, target, input4, target4;
arma::mat output1, output2, output3, output4;
arma::mat expOutput1, expOutput2, expOutput3, expOutput4;
PoissonNLLLoss<> module1;
PoissonNLLLoss<> module2(true, true, 1e-08, false);
PoissonNLLLoss<> module3(true, true, 1e-08, true);
PoissonNLLLoss<> module4(false, true, 1e-08, true);
PoissonNLLLoss<> module1(true, false, 1e-8, false);
PoissonNLLLoss<> module2(true, true, 1e-08, true);
PoissonNLLLoss<> module3(true, true, 1e-08, false);
PoissonNLLLoss<> module4(false, true, 1e-08, false);
// Test the Forward function on a user generated input.
input = arma::mat("1.0 1.0 1.9 1.6 -1.9 3.7 -1.0 0.5");
@@ -141,19 +168,55 @@ TEST_CASE("PoissonNLLLossTest", "[LossFunctionsTest]")
}
/**
* Simple KL Divergence test. The loss should be zero if input = target.
* Simple KL Divergence test.
*/
TEST_CASE("SimpleKLDivergenceTest", "[LossFunctionsTest]")
{
arma::mat input, target, output;
arma::mat expectedOutput;
double loss;
KLDivergence<> module(true);
KLDivergence<> module;
// Test the Forward function. Loss should be 0 if input = target.
input = arma::ones(10, 1);
target = arma::ones(10, 1);
// Test for sum reduction.
input = arma::mat("-0.7007 -2.0247 -0.7132 -0.4584 -0.2637 -1.1795 -0.1093 "
"-1.0530 -2.4250 -0.4556 -0.7861 -0.9120");
target = arma::mat("0.0223 0.5185 0.1610 0.9152 0.1689 0.6977 0.2823 0.3971 "
"0.2939 0.8000 0.6816 0.8742");
expectedOutput = arma::mat("-0.0223 -0.5185 -0.1610 -0.9152 -0.1689 -0.6977 "
"-0.2823 -0.3971 -0.2939 -0.8000 -0.6816 -0.8742");
input.reshape(4, 3);
target.reshape(4, 3);
expectedOutput.reshape(4, 3);
// Test the Forward function. Loss should be 2.33349.
// Value calculated using torch.nn.KLDivLoss(reduction='sum').
loss = module.Forward(input, target);
REQUIRE(loss == Approx(0.0).margin(1e-5));
REQUIRE(loss == Approx(2.33349).epsilon(1e-3));
// Test the Backward function.
module.Backward(input, target, output);
REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-5.8127).epsilon(1e-3));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
// Test for mean reduction by modifying reduction parameter using accessor.
module.Reduction() = false;
expectedOutput = arma::mat("-0.0019 -0.0432 -0.0134 -0.0763 -0.0141 -0.0581 "
"-0.0235 -0.0331 -0.0245 -0.0667 -0.0568 -0.0728");
expectedOutput.reshape(4, 3);
// Test the Forward function. Loss should be 0.194458.
// Value calculated using torch.nn.KLDivLoss(reduction='mean').
loss = module.Forward(input, target);
REQUIRE(loss == Approx(0.194458).epsilon(1e-3));
// Test the Backward function.
module.Backward(input, target, output);
REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-0.484392).epsilon(1e-3));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
}
/*
@@ -161,75 +224,49 @@ TEST_CASE("SimpleKLDivergenceTest", "[LossFunctionsTest]")
*/
TEST_CASE("SimpleMeanSquaredLogarithmicErrorTest", "[LossFunctionsTest]")
{
arma::mat input, output, target;
arma::mat input, target, output, expectedOutput;
double loss;
MeanSquaredLogarithmicError<> module;
// Test the Forward function on a user generator input and compare it against
// the manually calculated result.
input = arma::zeros(1, 8);
target = arma::zeros(1, 8);
double error = module.Forward(input, target);
REQUIRE(error == Approx(0.0).margin(1e-5));
// Test for sum reduction.
input = arma::mat("-0.0494 1.1958 1.0486 -0.2121 1.6028 0.0737 -0.7091 "
"0.8612 0.9639 0.9648 0.0745 0.5924");
target = arma::mat("0.4316 0.0164 -0.4478 1.1452 0.5106 0.9255 0.5571 0.0864 "
"0.7059 -0.8288 -0.0231 1.0526");
expectedOutput = arma::mat("-0.8615 0.7016 1.2799 -2.5425 0.4181 -1.0880 "
"-11.5339 0.5785 0.1434 2.4840 0.1772 -0.3188");
input.reshape(4, 3);
target.reshape(4, 3);
expectedOutput.reshape(4, 3);
// Test the Forward function. Loss should be 13.2728.
loss = module.Forward(input, target);
REQUIRE(loss == Approx(13.2728).epsilon(1e-3));
// Test the Backward function.
module.Backward(input, target, output);
// The output should be equal to 0.
CheckMatrices(input, output);
REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-10.5619).epsilon(1e-3));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
// Test the error function on a single input.
input = arma::mat("2");
target = arma::mat("3");
error = module.Forward(input, target);
REQUIRE(error == Approx(0.082760974810151655).epsilon(1e-3));
// Test the Backward function on a single input.
module.Backward(input, target, output);
REQUIRE(arma::accu(output) == Approx(-0.1917880483011872).epsilon(1e-3));
REQUIRE(output.n_elem == 1);
}
/**
* Test to check KL Divergence loss function when we take mean.
*/
TEST_CASE("KLDivergenceMeanTest", "[LossFunctionsTest]")
{
arma::mat input, target, output;
double loss;
KLDivergence<> module(true);
// Test the Forward function.
input = arma::mat("1 1 1 1 1 1 1 1 1 1");
target = arma::exp(arma::mat("2 1 1 1 1 1 1 1 1 1"));
// Test for mean reduction by modifying reduction parameter using accessor.
module.Reduction() = false;
expectedOutput = arma::mat("-0.0718 0.0585 0.1067 -0.2119 0.0348 -0.0907 "
"-0.9612 0.0482 0.0120 0.2070 0.0148 -0.0266");
expectedOutput.reshape(4, 3);
// Test the Forward function. Loss should be 1.10606.
loss = module.Forward(input, target);
REQUIRE(loss == Approx(-1.1).epsilon(1e-5));
REQUIRE(loss == Approx(1.10606).epsilon(1e-3));
// Test the Backward function.
module.Backward(input, target, output);
REQUIRE(arma::as_scalar(output) == Approx(-0.1).epsilon(1e-5));
}
/**
* Test to check KL Divergence loss function when we do not take mean.
*/
TEST_CASE("KLDivergenceNoMeanTest", "[LossFunctionsTest]")
{
arma::mat input, target, output;
double loss;
KLDivergence<> module(false);
// Test the Forward function.
input = arma::mat("1 1 1 1 1 1 1 1 1 1");
target = arma::exp(arma::mat("2 1 1 1 1 1 1 1 1 1"));
loss = module.Forward(input, target);
REQUIRE(loss == Approx(-11).epsilon(1e-5));
// Test the Backward function.
module.Backward(input, target, output);
REQUIRE(arma::as_scalar(output) == Approx(-1).epsilon(1e-5));
REQUIRE(arma::as_scalar(arma::accu(output)) ==
Approx(-0.880156).epsilon(1e-3));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
}
/*
@@ -238,9 +275,9 @@ TEST_CASE("KLDivergenceNoMeanTest", "[LossFunctionsTest]")
TEST_CASE("SimpleMeanSquaredErrorTest", "[LossFunctionsTest]")
{
arma::mat input, output, target;
MeanSquaredError<> module;
MeanSquaredError<> module(false);
// Test the Forward function on a user generator input and compare it against
// Test the Forward function on a user generated input and compare it against
// the manually calculated result.
input = arma::mat("1.0 0.0 1.0 0.0 -1.0 0.0 -1.0 0.0");
target = arma::zeros(1, 8);
@@ -267,6 +304,19 @@ TEST_CASE("SimpleMeanSquaredErrorTest", "[LossFunctionsTest]")
// Test whether the output is negative.
REQUIRE(arma::accu(output) == -2);
REQUIRE(output.n_elem == 1);
// Test for sum reduction
module.Reduction() = true;
// Test the Forward function
error = module.Forward(input, target);
REQUIRE(error == Approx(1.0).epsilon(1e-5));
// Test the Backward function on a single input.
module.Backward(input, target, output);
// Test whether the output is negative.
REQUIRE(arma::accu(output) == -2);
REQUIRE(output.n_elem == 1);
}
/*
@@ -275,8 +325,8 @@ TEST_CASE("SimpleMeanSquaredErrorTest", "[LossFunctionsTest]")
TEST_CASE("SimpleBinaryCrossEntropyLossTest", "[LossFunctionsTest]")
{
arma::mat input1, input2, input3, output, target1, target2, target3;
BCELoss<> module1(1e-6, false);
BCELoss<> module2(1e-6, true);
BCELoss<> module1(1e-6, true);
BCELoss<> module2(1e-6, false);
// Test the Forward function on a user generator input and compare it against
// the manually calculated result.
input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5");
@@ -387,21 +437,23 @@ TEST_CASE("SimpleSigmoidCrossEntropyErrorTest", "[LossFunctionsTest]")
TEST_CASE("SimpleEarthMoverDistanceLayerTest", "[LossFunctionsTest]")
{
arma::mat input1, input2, output, target1, target2, expectedOutput;
arma::mat input3, target3;
double loss;
EarthMoverDistance<> module;
// Test the Forward function on a user generator input and compare it against
// the manually calculated result.
input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5");
target1 = arma::zeros(1, 8);
double error1 = module.Forward(input1, target1);
loss = module.Forward(input1, target1);
double expected = 0.0;
REQUIRE(error1 / input1.n_elem - expected == Approx(0.0).margin(1e-7));
REQUIRE(loss / input1.n_elem - expected == Approx(0.0).margin(1e-7));
input2 = arma::mat("1 2 3 4 5");
target2 = arma::mat("1 0 1 0 1");
double error2 = module.Forward(input2, target2);
loss = module.Forward(input2, target2);
expected = -1.8;
REQUIRE(error2 / input2.n_elem - expected == Approx(0.0).margin(1e-6));
REQUIRE(loss / input2.n_elem - expected == Approx(0.0).margin(1e-6));
// Test the Backward function.
module.Backward(input1, target1, output);
@@ -417,6 +469,31 @@ TEST_CASE("SimpleEarthMoverDistanceLayerTest", "[LossFunctionsTest]")
REQUIRE(output(i) - expectedOutput(i) == Approx(0.0).margin(1e-5));
REQUIRE(output.n_rows == input2.n_rows);
REQUIRE(output.n_cols == input2.n_cols);
// Test for mean reduction.
module.Reduction() = false;
input3 = arma::mat("-0.0494 -1.1958 -1.0486 -0.2121 1.6028 0.0737 -0.7091 "
"0.8612 0.9639 0.9648 0.0745 0.5924");
target3 = arma::mat("0.4316 0.0164 -0.4478 1.1452 0.5106 0.9255 0.5571 "
"0.0864 0.7059 -0.8288 -0.0231 -1.0526");
expectedOutput = arma::mat("-0.0360 -0.0014 0.0373 -0.0954 -0.0426 -0.0771 "
"-0.0464 -0.0072 -0.0588 0.0691 0.0019 0.0877");
input3.reshape(4, 3);
target3.reshape(4, 3);
expectedOutput.reshape(4, 3);
// Test the Forward function. Loss should be -0.00060089.
// Value calculated manually.
loss = module.Forward(input3, target3);
REQUIRE(loss == Approx(-0.00060089).epsilon(1e-3));
// Test the Backward function.
module.Backward(input3, target3, output);
REQUIRE(arma::as_scalar(arma::accu(output)) ==
Approx(-0.168867).epsilon(1e-3));
REQUIRE(output.n_rows == input3.n_rows);
REQUIRE(output.n_cols == input3.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
}
/*
@@ -551,37 +628,49 @@ TEST_CASE("DiceLossTest", "[LossFunctionsTest]")
*/
TEST_CASE("SimpleMeanBiasErrorTest", "[LossFunctionsTest]")
{
arma::mat input, output, target;
arma::mat input, target, output;
double loss;
MeanBiasError<> module;
// Test the Forward function on a user generator input and compare it against
// the manually calculated result.
input = arma::mat("1.0 0.0 1.0 -1.0 -1.0 0.0 -1.0 0.0");
target = arma::zeros(1, 8);
double error = module.Forward(input, target);
REQUIRE(error == 0.125);
// Test for sum reduction.
input = arma::mat("-0.0494 -1.1958 -1.0486 -0.2121 1.6028 0.0737 -0.7091 "
"0.8612 0.9639 0.9648 0.0745 0.5924");
target = arma::mat("0.4316 0.0164 -0.4478 1.1452 0.5106 0.9255 0.5571 0.0864 "
"0.7059 -0.8288 -0.0231 -1.0526");
// Test the Backward function.
input.reshape(4, 3);
target.reshape(4, 3);
// Test the forward function.
// Loss should be 0.1081.
loss = module.Forward(input, target);
REQUIRE(loss == Approx(0.1081).epsilon(1e-5));
// Test the backward function.
module.Backward(input, target, output);
// We should get a vector with -1 everywhere.
for (double el : output)
{
for(double el : output)
REQUIRE(el == -1);
}
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
// Test the error function on a single input.
input = arma::mat("2");
target = arma::mat("3");
error = module.Forward(input, target);
REQUIRE(error == 1.0);
// Test for mean reduction by modifying
// reduction parameter using accessor.
module.Reduction() = false;
// Test the Backward function on a single input.
// Test the forward function
// loss should be 0.00900833
loss = module.Forward(input, target);
REQUIRE(loss == Approx(0.00900833).epsilon(1e-5));
// Test the backward function
module.Backward(input, target, output);
// Test whether the output is negative.
REQUIRE(arma::accu(output) == -1);
REQUIRE(output.n_elem == 1);
for(double el : output)
REQUIRE(el == Approx(-0.0833).epsilon(1e-3));
REQUIRE(arma::accu(output) == Approx(-1).epsilon(1e-5));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
}
/**
@@ -610,9 +699,10 @@ TEST_CASE("LogCoshLossTest", "[LossFunctionsTest]")
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
// Test the Forward function. Loss should be 0.546621.
// Test for sum reduction
input = arma::mat("1 2 3 4 5");
target = arma::mat("1 2.4 3.4 4.2 5.5");
// Test the Forward function. Loss should be 0.546621.
loss = module.Forward(input, target);
REQUIRE(loss == Approx(0.546621).epsilon(1e-3));
@@ -620,6 +710,19 @@ TEST_CASE("LogCoshLossTest", "[LossFunctionsTest]")
module.Backward(input, target, output);
REQUIRE(arma::accu(output) == Approx(2.46962).epsilon(1e-3));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
// Test for mean reduction by modifying reduction parameter using accessor.
module.Reduction() = false;
// Test the Forward function. Loss should be 0.109324.
loss = module.Forward(input, target);
REQUIRE(loss == Approx(0.109324).epsilon(1e-3));
// Test the Backward function.
module.Backward(input, target, output);
REQUIRE(arma::accu(output) == Approx(0.49392).epsilon(1e-3));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
}
@@ -628,38 +731,51 @@ TEST_CASE("LogCoshLossTest", "[LossFunctionsTest]")
*/
TEST_CASE("HingeEmbeddingLossTest", "[LossFunctionsTest]")
{
arma::mat input, target, output;
arma::mat input, target, output, expectedOutput;
double loss;
HingeEmbeddingLoss<> module;
// Test the Forward function. Loss should be 0 if input = target.
input = arma::ones(10, 1);
target = arma::ones(10, 1);
// Test for sum reduction
input = arma::mat("0.1778 0.0957 0.1397 0.2256 0.1203 0.2403 0.1925 0.3144 "
"-0.2264 -0.3400 -0.3336 -0.8695");
target = arma::mat("1 1 -1 1 1 -1 1 1 -1 1 1 1");
expectedOutput = arma::mat("1 1 -1 1 1 -1 1 1 -1 1 1 1");
input.reshape(4, 3);
target.reshape(4, 3);
expectedOutput.reshape(4, 3);
// Test the forward function
// Loss should be 2.4296
// Value calculated using torch.nn.HingeEmbeddingLoss(reduction='sum')
loss = module.Forward(input, target);
REQUIRE(loss == 0);
REQUIRE(loss == Approx(2.4296).epsilon(1e-3));
// Test the Backward function for input = target.
// Test the Backward function
module.Backward(input, target, output);
for (double el : output)
{
// For input = target we should get 0.0 everywhere.
REQUIRE(el == Approx(0.0).epsilon(1e-5));
}
REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(6).epsilon(1e-3));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 1e-3);
// Test the Forward function. Loss should be 0.84.
input = arma::mat("0.1 0.8 0.6 0.0 0.5");
target = arma::mat("0 1.0 1.0 0 0");
// Test for mean reduction by modifying reduction
// parameter through the accessor.
module.Reduction() = false;
expectedOutput = arma::mat("0.0833 0.0833 -0.0833 0.0833 0.0833 -0.0833 "
"0.0833 0.0833 -0.0833 0.0833 0.0833 0.0833");
expectedOutput.reshape(4, 3);
// Test the forward function
// Loss should be 0.202467
// Value calculated using torch.nn.HingeEmbeddingLoss(reduction='mean')
loss = module.Forward(input, target);
REQUIRE(loss == Approx(0.84).epsilon(1e-3));
REQUIRE(loss == Approx(0.202467).epsilon(1e-3));
// Test the Backward function.
// Test the backward function
module.Backward(input, target, output);
REQUIRE(arma::accu(output) == Approx(-2).epsilon(1e-3));
REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(0.5).epsilon(1e-3));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
}
/**
@@ -667,35 +783,25 @@ TEST_CASE("HingeEmbeddingLossTest", "[LossFunctionsTest]")
*/
TEST_CASE("SimpleL1LossTest", "[LossFunctionsTest]")
{
arma::mat input1, input2, output, target1, target2;
L1Loss<> module(false);
arma::mat input, output, target;
double loss;
L1Loss<> module(true);
// Test the Forward function on a user generator input and compare it against
// the manually calculated result.
input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5");
target1 = arma::zeros(1, 7);
double error1 = module.Forward(input1, target1);
REQUIRE(error1 == 3.5);
input2 = arma::mat("0 1 1 0 1 0 0 1");
target2 = arma::mat("0 1 1 0 1 0 0 1");
double error2 = module.Forward(input2, target2);
REQUIRE(error2 == Approx(0.0).epsilon(1e-5));
input = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5");
target = arma::zeros(1, 7);
loss = module.Forward(input, target);
// Value calculated using torch.nn.L1Loss(reduction='sum').
REQUIRE(loss == 3.5);
// Test the Backward function.
module.Backward(input1, target1, output);
module.Backward(input, target, output);
for (double el : output)
REQUIRE(el == 1);
REQUIRE(output.n_rows == input1.n_rows);
REQUIRE(output.n_cols == input1.n_cols);
module.Backward(input2, target2, output);
for (double el : output)
REQUIRE(el == 0);
REQUIRE(output.n_rows == input2.n_rows);
REQUIRE(output.n_cols == input2.n_cols);
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
}
/**
@@ -771,7 +877,7 @@ TEST_CASE("CosineEmbeddingLossTest", "[LossFunctionsTest]")
REQUIRE(arma::accu(output) == Approx(-0.36649111).epsilon(1e-3));
// Check Output for mean type of reduction.
CosineEmbeddingLoss<> module3(0.0, true, true);
CosineEmbeddingLoss<> module3(0.0, true, false);
loss = module3.Forward(input3, input4);
REQUIRE(loss == Approx(0.092325).epsilon(1e-3));
@@ -782,7 +888,7 @@ TEST_CASE("CosineEmbeddingLossTest", "[LossFunctionsTest]")
// Test the Backward function.
module3.Backward(input3, input4, output);
REQUIRE(arma::accu(output) == Approx(0.36649111).epsilon(1e-4));
REQUIRE(arma::accu(output) == Approx(0.0236749374).epsilon(1e-4));
}
/*
@@ -790,42 +896,51 @@ TEST_CASE("CosineEmbeddingLossTest", "[LossFunctionsTest]")
*/
TEST_CASE("MarginRankingLossTest", "[LossFunctionsTest]")
{
arma::mat input, input1, input2, target, output;
arma::mat input, input1, input2, target, output, expectedOutput;
double loss;
// Test sum reduction
MarginRankingLoss<> module;
// Test the Forward function on a user generator input and compare it against
// the manually calculated result.
input1 = arma::mat("1 2 5 7 -1 -3");
input2 = arma::mat("-1 3 -4 11 3 -3");
input = arma::join_cols(input1, input2);
target = arma::mat("1 -1 -1 1 -1 1");
double error = module.Forward(input, target);
// Computed using torch.nn.functional.margin_ranking_loss()
REQUIRE(error == Approx(2.66667).epsilon(1e-3));
// Test the Backward function.
module.Backward(input, target, output);
CheckMatrices(output, arma::mat("-0.000000 0.166667 -1.500000 0.666667 "
"0.000000 -0.000000"), 1e-3);
REQUIRE(output.n_rows == target.n_rows);
REQUIRE(output.n_cols == target.n_cols);
// Test the error function on another input.
input1 = arma::mat("0.4287 -1.6208 -1.5006 -0.4473 1.5208 -4.5184 9.3574 "
"-4.8090 4.3455 5.2070");
input2 = arma::mat("-4.5288 -9.2766 -0.5882 -5.6643 -6.0175 8.8506 3.4759 "
"-9.4886 2.2755 8.4951");
expectedOutput << 0.0000 << 0.0000 << 1.0000 << 0.0000 << 1.0000 << -1.0000
<< 0.0000 << 0.0000 << 1.0000 << -1.0000 <<arma::endr << 0.0000
<< 0.0000 << -1.0000 << 0.0000 << -1.0000 << 1.0000 << 0.0000
<< 0.0000 << -1.0000 << 1.0000 << arma::endr;
input = arma::join_cols(input1, input2);
target = arma::mat("1 1 -1 1 -1 1 1 1 -1 1");
error = module.Forward(input, target);
REQUIRE(error == Approx(3.03530).epsilon(1e-3));
// Test the Backward function on the second input.
// Test the forward function
// loss should be 30.3530
// value calculated using torch.nn.MarginRankingLoss(margin=1.0,reduction='sum')
loss = module.Forward(input, target);
REQUIRE(loss == Approx(30.3530).epsilon(1e-3));
// Test the Backward function.
module.Backward(input, target, output);
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
CheckMatrices(output, arma::mat("0.000000 0.000000 0.091240 0.000000 "
"-0.753830 1.336900 0.000000 0.000000 -0.207000 0.328810"), 1e-6);
// Test for mean reduction by modifying reduction parameter using accessor.
module.Reduction() = false;
// Test the forward function
// loss should be 3.0353
// value calculated using torch.nn.MarginRankingLoss(margin=1.0,reduction='mean')
loss = module.Forward(input, target);
REQUIRE(loss == Approx(3.0353).epsilon(1e-5));
// Test the backward function
module.Backward(input, target, output);
expectedOutput << 0.0000 << 0.0000 << 0.1000 << 0.0000 << 0.1000 << -0.1000
<< 0.0000 << 0.0000 << 0.1000 << -0.1000 <<arma::endr << 0.0000
<< 0.0000 << -0.1000 << 0.0000 << -0.1000 << 0.1000 << 0.0000
<< 0.0000 << -0.1000 << 0.1000 << arma::endr;
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
}
/**
@@ -1150,3 +1265,51 @@ TEST_CASE("MultiLabelSoftMarginLossWeightedTest", "[LossFunctionsTest]")
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
}
/**
* Simple Negative Log Likelihood Loss test.
*/
TEST_CASE("NegativeLogLikelihoodLossTest", "[LossFunctionsTest]")
{
arma::mat input, target, output;
arma::mat expectedOutput;
double loss;
NegativeLogLikelihood<> module;
// Test for sum reduction.
input = arma::mat("-0.1689 -2.0033 -3.8886 -0.2862 -1.9392 -2.2532"
" -1.0543 -0.6196 -2.1769 -1.2865 -1.4797 -0.7011");
target = arma::mat("2 2 1 2");
expectedOutput = arma::mat("0 0 -1.0000 0 0 -1.0000 0 -1.0000 0 0 0 -1.0000");
input.reshape(3, 4);
expectedOutput.reshape(3, 4);
// Test the Forward function. Loss should be 7.4625.
// Value calculated using torch.nn.NLLLoss(reduction='sum').
loss = module.Forward(input, target);
REQUIRE(loss == Approx(7.4625).epsilon(1e-3));
// Test the Backward function.
module.Backward(input, target, output);
REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-4).epsilon(1e-3));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
// Test for mean reduction by modifying reduction parameter using accessor.
module.Reduction() = false;
expectedOutput = arma::mat("0 0 -0.2500 0 0 -0.2500 0 -0.2500 0 0 0 -0.2500");
expectedOutput.reshape(3, 4);
// Test the Forward function. Loss should be 1.86562.
// Value calculated using torch.nn.NLLLoss(reduction='mean').
loss = module.Forward(input, target);
REQUIRE(loss == Approx(1.86562).epsilon(1e-3));
// Test the Backward function.
module.Backward(input, target, output);
REQUIRE(arma::as_scalar(arma::accu(output)) == Approx(-1).epsilon(1e-3));
REQUIRE(output.n_rows == input.n_rows);
REQUIRE(output.n_cols == input.n_cols);
CheckMatrices(output, expectedOutput, 0.1);
}