diff --git a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt index a4e2cc590c..5dbfe5a1c7 100644 --- a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt @@ -33,6 +33,8 @@ set(SOURCES reconstruction_loss_impl.hpp sigmoid_cross_entropy_error.hpp sigmoid_cross_entropy_error_impl.hpp + soft_margin_loss.hpp + soft_margin_loss_impl.hpp hinge_embedding_loss.hpp hinge_embedding_loss_impl.hpp empty_loss.hpp diff --git a/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp b/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp new file mode 100644 index 0000000000..6050875113 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/soft_margin_loss.hpp @@ -0,0 +1,100 @@ +/** + * @file methods/ann/loss_functions/soft_margin_loss.hpp + * @author Anjishnu Mukherjee + * + * Definition of the Soft Margin Loss function. + * + * It is a criterion that optimizes a two-class classification logistic loss, + * between input x and target y, both having the same shape, with the target + * containing only the values 1 or -1. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_ANN_LOSS_FUNCTION_SOFT_MARGIN_LOSS_HPP +#define MLPACK_ANN_LOSS_FUNCTION_SOFT_MARGIN_LOSS_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class SoftMarginLoss +{ + public: + /** + * Create the SoftMarginLoss object. + * + * @param reduction Specifies the reduction to apply to the output. If false, + * 'mean' reduction is used, where sum of the output will be + * divided by the number of elements in the output. If + * true, 'sum' reduction is used and the output will be + * summed. It is set to true by default. + */ + SoftMarginLoss(const bool reduction = true); + + /** + * Computes the Soft Margin Loss function. + * + * @param input Input data used for evaluating the specified function. + * @param target The target vector with same shape as input. + */ + template + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); + + /** + * Ordinary feed backward pass of a neural network. + * + * @param input The propagated input activation. + * @param target The target vector. + * @param output The calculated error. + */ + template + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the type of reduction used. + bool Reduction() const { return reduction; } + //! Modify the type of reduction used. + bool& Reduction() { return reduction; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! The boolean value that tells if reduction is sum or mean. + bool reduction; +}; // class SoftMarginLoss + +} // namespace ann +} // namespace mlpack + +// include implementation. +#include "soft_margin_loss_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/soft_margin_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/soft_margin_loss_impl.hpp new file mode 100644 index 0000000000..87fdb3f801 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/soft_margin_loss_impl.hpp @@ -0,0 +1,72 @@ +/** + * @file methods/ann/loss_functions/soft_margin_loss_impl.hpp + * @author Anjishnu Mukherjee + * + * Implementation of the Soft Margin Loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_SOFT_MARGIN_LOSS_IMPL_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_SOFT_MARGIN_LOSS_IMPL_HPP + +// In case it hasn't been included. +#include "soft_margin_loss.hpp" + +namespace mlpack { +namespace ann /** Artifical Neural Network. */ { + +template +SoftMarginLoss:: +SoftMarginLoss(const bool reduction) : reduction(reduction) +{ + // Nothing to do here. +} + +template +template +typename InputType::elem_type +SoftMarginLoss::Forward( + const InputType& input, const TargetType& target) +{ + InputType loss = arma::log(1 + arma::exp(-target % input)); + typename InputType::elem_type lossSum = arma::accu(loss); + + if (reduction) + return lossSum; + + return lossSum / input.n_elem; +} + +template +template +void SoftMarginLoss::Backward( + const InputType& input, + const TargetType& target, + OutputType& output) +{ + output.set_size(size(input)); + InputType temp = arma::exp(-target % input); + InputType numerator = -target % temp; + InputType denominator = 1 + temp; + output = numerator / denominator; + + if (!reduction) + output = output / input.n_elem; +} + +template +template +void SoftMarginLoss::serialize( + Archive& ar, + const unsigned int /* version */) +{ + ar & BOOST_SERIALIZATION_NVP(reduction); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 180e374402..5af527ac8c 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -816,4 +817,59 @@ BOOST_AUTO_TEST_CASE(MarginRankingLossTest) "-0.753830 1.336900 0.000000 0.000000 -0.207000 0.328810"), 1e-6); } +/** + * Simple test for the Softmargin Loss function. + */ +BOOST_AUTO_TEST_CASE(SoftMarginLossTest) +{ + arma::mat input, target, output, expectedOutput; + double loss; + SoftMarginLoss<> module1; + SoftMarginLoss<> module2(false); + + input = arma::mat("0.1778 0.0957 0.1397 0.1203 0.2403 0.1925 -0.2264 -0.3400 " + "-0.3336"); + target = arma::mat("1 1 -1 1 -1 1 -1 1 1"); + input.reshape(3, 3); + target.reshape(3, 3); + + // Test for sum reduction. + + // Calculated using torch.nn.SoftMarginLoss(reduction='sum'). + expectedOutput = arma::mat("-0.4557 -0.4761 0.5349 -0.4700 0.5598 -0.4520 " + "0.4436 -0.5842 -0.5826"); + expectedOutput.reshape(3, 3); + + // Test the Forward function. Loss should be 6.41456. + // Value calculated using torch.nn.SoftMarginLoss(reduction='sum'). + loss = module1.Forward(input, target); + BOOST_REQUIRE_CLOSE(loss, 6.41456, 1e-3); + + // Test the Backward function. + module1.Backward(input, target, output); + BOOST_REQUIRE_CLOSE(arma::as_scalar(arma::accu(output)), -1.48227, 1e-3); + BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); + + // Test for mean reduction. + + // Calculated using torch.nn.SoftMarginLoss(reduction='mean'). + expectedOutput = arma::mat("-0.0506 -0.0529 0.0594 -0.0522 0.0622 -0.0502 " + "0.0493 -0.0649 -0.0647"); + expectedOutput.reshape(3, 3); + + // Test the Forward function. Loss should be 0.712729. + // Value calculated using torch.nn.SoftMarginLoss(reduction='mean'). + loss = module2.Forward(input, target); + BOOST_REQUIRE_CLOSE(loss, 0.712729, 1e-3); + + // Test the Backward function. + module2.Backward(input, target, output); + BOOST_REQUIRE_CLOSE(arma::as_scalar(arma::accu(output)), -0.164697, 1e-3); + BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); + CheckMatrices(output, expectedOutput, 0.1); +} + BOOST_AUTO_TEST_SUITE_END();