From 496c18ac52fa710ad73f4f1998b680cdb7769dfa Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Thu, 14 Nov 2013 16:31:41 +0000
Subject: [PATCH] Update Gradient() for the separable case.

---
 .../logistic_regression_function.cpp          | 21 ++++++++++++++
 .../logistic_regression_function.hpp          | 28 ++++++++++++-------
 2 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp b/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp
index 6db18af557..f6e68b42a1 100644
--- a/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp
+++ b/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp
@@ -101,6 +101,7 @@ double LogisticRegressionFunction::Evaluate(const arma::mat& parameters,
     return -(log(1.0 - sigmoid) + regularization);
 }
 
+//! Evaluate the gradient of the logistic regression objective function.
 void LogisticRegressionFunction::Gradient(const arma::mat& parameters,
                                           arma::mat& gradient) const
 {
@@ -113,3 +114,23 @@ void LogisticRegressionFunction::Gradient(const arma::mat& parameters,
       - (1 / (1 + arma::exp(-predictors.t() * parameters))))
       - regularization;
 }
+
+/**
+ * Evaluate the individual gradients of the logistic regression objective
+ * function with respect to individual points.  This is useful for optimizers
+ * that use a separable objective function, such as SGD.
+ */
+void LogisticRegressionFunction::Gradient(const arma::mat& parameters,
+                                          const size_t i,
+                                          arma::mat& gradient) const
+{
+  // Calculate the regularization term.
+  arma::mat regularization = arma::zeros<arma::mat>(predictors.n_rows, 1);
+  regularization.rows(1, predictors.n_rows - 1) = lambda *
+      parameters.col(0).subvec(1, predictors.n_rows - 1) / predictors.n_cols;
+
+  const double sigmoid = 1.0 /
+      (1.0 + std::exp(-arma::dot(predictors.col(i), parameters)));
+
+  gradient = -predictors.col(i) * (responses[i] - sigmoid) - regularization;
+}
diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp b/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp
index 0b57aedaa9..00bd4f9eb2 100644
--- a/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp
+++ b/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp
@@ -57,8 +57,8 @@ class LogisticRegressionFunction
   /**
    * Evaluate the logistic regression log-likelihood function with the given
    * parameters, but using only one data point.  This is useful for optimizers
-   * such as SGD, that require a separable objective function.  Note that if the
-   * point has 0 probability of being classified correctly with the given
+   * such as SGD, which require a separable objective function.  Note that if
+   * the point has 0 probability of being classified correctly with the given
    * parameters, then Evaluate() will return nan (this is kind of a corner case
    * and should not happen for reasonable models).
    *
@@ -68,7 +68,7 @@ class LogisticRegressionFunction
    * @param parameters Vector of logistic regression parameters.
    * @param i Index of point to use for objective function evaluation.
    */
-  double Evaluate(const arma::mat& values, const size_t i) const;
+  double Evaluate(const arma::mat& parameters, const size_t i) const;
 
   /**
    * Evaluate the gradient of the logistic regression log-likelihood function
@@ -79,16 +79,24 @@ class LogisticRegressionFunction
    */
   void Gradient(const arma::mat& parameters, arma::mat& gradient) const;
 
+  /**
+   * Evaluate the gradient of the logistic regression log-likelihood function
+   * with the given parameters, and with respect to only one point in the
+   * dataset.  This is useful for optimizers such as SGD, which require a
+   * separable objective function.
+   *
+   * @param parameters Vector of logistic regression parameters.
+   * @param i Index of points to use for objective function gradient evaluation.
+   * @param gradient Vector to output gradient into.
+   */
+  void Gradient(const arma::mat& parameters,
+                const size_t i,
+                arma::mat& gradient) const;
+
   //! Return the initial point for the optimization.
   const arma::mat& GetInitialPoint() const { return initialPoint; }
 
-  void Gradient(const arma::mat& values,
-                const size_t i,
-                arma::mat& gradient)
-  {
-    Gradient(values,gradient);
-  }
-
+  //! Return the number of separable functions (the number of predictor points).
   size_t NumFunctions() const { return predictors.n_cols; }
 
  private: