Merge pull request #2494 from iamshnoo/soft_margin_loss

Soft Margin Loss function
2020-08-05 00:31:50 -04:00
parent a9d9430f9b be400f36d9
commit 9e706011bb
4 changed files with 230 additions and 0 deletions
@@ -33,6 +33,8 @@ set(SOURCES
  reconstruction_loss_impl.hpp
  sigmoid_cross_entropy_error.hpp
  sigmoid_cross_entropy_error_impl.hpp
+  soft_margin_loss.hpp
+  soft_margin_loss_impl.hpp
  hinge_embedding_loss.hpp
  hinge_embedding_loss_impl.hpp
  empty_loss.hpp
@@ -0,0 +1,100 @@
+/**
+ * @file methods/ann/loss_functions/soft_margin_loss.hpp
+ * @author Anjishnu Mukherjee
+ *
+ * Definition of the Soft Margin Loss function.
+ *
+ * It is a criterion that optimizes a two-class classification logistic loss,
+ * between input x and target y, both having the same shape, with the target
+ * containing only the values 1 or -1.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_ANN_LOSS_FUNCTION_SOFT_MARGIN_LOSS_HPP
+#define MLPACK_ANN_LOSS_FUNCTION_SOFT_MARGIN_LOSS_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ */
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class SoftMarginLoss
+{
+ public:
+  /**
+   * Create the SoftMarginLoss object.
+   *
+   * @param reduction Specifies the reduction to apply to the output. If false,
+   *                  'mean' reduction is used, where sum of the output will be
+   *                  divided by the number of elements in the output. If
+   *                  true, 'sum' reduction is used and the output will be
+   *                  summed. It is set to true by default.
+   */
+  SoftMarginLoss(const bool reduction = true);
+
+  /**
+   * Computes the Soft Margin Loss function.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param target The target vector with same shape as input.
+   */
+  template<typename InputType, typename TargetType>
+  typename InputType::elem_type Forward(const InputType& input,
+                                        const TargetType& target);
+
+  /**
+   * Ordinary feed backward pass of a neural network.
+   *
+   * @param input The propagated input activation.
+   * @param target The target vector.
+   * @param output The calculated error.
+   */
+  template<typename InputType, typename TargetType, typename OutputType>
+  void Backward(const InputType& input,
+                const TargetType& target,
+                OutputType& output);
+
+  //! Get the output parameter.
+  OutputDataType& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the type of reduction used.
+  bool Reduction() const { return reduction; }
+  //! Modify the type of reduction used.
+  bool& Reduction() { return reduction; }
+
+  /**
+   * Serialize the layer.
+   */
+  template<typename Archive>
+  void serialize(Archive& ar, const unsigned int /* version */);
+
+ private:
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! The boolean value that tells if reduction is sum or mean.
+  bool reduction;
+}; // class SoftMarginLoss
+
+} // namespace ann
+} // namespace mlpack
+
+// include implementation.
+#include "soft_margin_loss_impl.hpp"
+
+#endif
@@ -0,0 +1,72 @@
+/**
+ * @file methods/ann/loss_functions/soft_margin_loss_impl.hpp
+ * @author Anjishnu Mukherjee
+ *
+ * Implementation of the Soft Margin Loss function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_SOFT_MARGIN_LOSS_IMPL_HPP
+#define MLPACK_METHODS_ANN_LOSS_FUNCTION_SOFT_MARGIN_LOSS_IMPL_HPP
+
+// In case it hasn't been included.
+#include "soft_margin_loss.hpp"
+
+namespace mlpack {
+namespace ann /** Artifical Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType>
+SoftMarginLoss<InputDataType, OutputDataType>::
+SoftMarginLoss(const bool reduction) : reduction(reduction)
+{
+  // Nothing to do here.
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename InputType, typename TargetType>
+typename InputType::elem_type
+SoftMarginLoss<InputDataType, OutputDataType>::Forward(
+    const InputType& input, const TargetType& target)
+{
+  InputType loss = arma::log(1 + arma::exp(-target % input));
+  typename InputType::elem_type lossSum = arma::accu(loss);
+
+  if (reduction)
+    return lossSum;
+
+  return lossSum / input.n_elem;
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename InputType, typename TargetType, typename OutputType>
+void SoftMarginLoss<InputDataType, OutputDataType>::Backward(
+    const InputType& input,
+    const TargetType& target,
+    OutputType& output)
+{
+  output.set_size(size(input));
+  InputType temp = arma::exp(-target % input);
+  InputType numerator = -target % temp;
+  InputType denominator = 1 + temp;
+  output = numerator / denominator;
+
+  if (!reduction)
+    output = output / input.n_elem;
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename Archive>
+void SoftMarginLoss<InputDataType, OutputDataType>::serialize(
+    Archive& ar,
+    const unsigned int /* version */)
+{
+  ar & BOOST_SERIALIZATION_NVP(reduction);
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif
@@ -31,6 +31,7 @@
 #include <mlpack/methods/ann/loss_functions/hinge_embedding_loss.hpp>
 #include <mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp>
 #include <mlpack/methods/ann/loss_functions/l1_loss.hpp>
+#include <mlpack/methods/ann/loss_functions/soft_margin_loss.hpp>
 #include <mlpack/methods/ann/init_rules/nguyen_widrow_init.hpp>
 #include <mlpack/methods/ann/ffn.hpp>

@@ -816,4 +817,59 @@ BOOST_AUTO_TEST_CASE(MarginRankingLossTest)
      "-0.753830 1.336900 0.000000 0.000000 -0.207000 0.328810"), 1e-6);
 }

+/**
+ * Simple test for the Softmargin Loss function.
+ */
+BOOST_AUTO_TEST_CASE(SoftMarginLossTest)
+{
+  arma::mat input, target, output, expectedOutput;
+  double loss;
+  SoftMarginLoss<> module1;
+  SoftMarginLoss<> module2(false);
+
+  input = arma::mat("0.1778 0.0957 0.1397 0.1203 0.2403 0.1925 -0.2264 -0.3400 "
+      "-0.3336");
+  target = arma::mat("1 1 -1 1 -1 1 -1 1 1");
+  input.reshape(3, 3);
+  target.reshape(3, 3);
+
+  // Test for sum reduction.
+
+  // Calculated using torch.nn.SoftMarginLoss(reduction='sum').
+  expectedOutput = arma::mat("-0.4557 -0.4761 0.5349 -0.4700 0.5598 -0.4520 "
+      "0.4436 -0.5842 -0.5826");
+  expectedOutput.reshape(3, 3);
+
+  // Test the Forward function. Loss should be 6.41456.
+  // Value calculated using torch.nn.SoftMarginLoss(reduction='sum').
+  loss = module1.Forward(input, target);
+  BOOST_REQUIRE_CLOSE(loss, 6.41456, 1e-3);
+
+  // Test the Backward function.
+  module1.Backward(input, target, output);
+  BOOST_REQUIRE_CLOSE(arma::as_scalar(arma::accu(output)), -1.48227, 1e-3);
+  BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows);
+  BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols);
+  CheckMatrices(output, expectedOutput, 0.1);
+
+  // Test for mean reduction.
+
+  // Calculated using torch.nn.SoftMarginLoss(reduction='mean').
+  expectedOutput = arma::mat("-0.0506 -0.0529 0.0594 -0.0522 0.0622 -0.0502 "
+      "0.0493 -0.0649 -0.0647");
+  expectedOutput.reshape(3, 3);
+
+  // Test the Forward function. Loss should be 0.712729.
+  // Value calculated using torch.nn.SoftMarginLoss(reduction='mean').
+  loss = module2.Forward(input, target);
+  BOOST_REQUIRE_CLOSE(loss, 0.712729, 1e-3);
+
+  // Test the Backward function.
+  module2.Backward(input, target, output);
+  BOOST_REQUIRE_CLOSE(arma::as_scalar(arma::accu(output)), -0.164697, 1e-3);
+  BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows);
+  BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols);
+  CheckMatrices(output, expectedOutput, 0.1);
+}
+
 BOOST_AUTO_TEST_SUITE_END();