Refactor Evaluate() function so that it works alright, and eliminate unnecessary

parameters to the function.  Move LogisticRegressionFunction implementation into
a .cpp file because it is not templatized (for now).

src/mlpack/methods/logistic_regression/CMakeLists.txt

@@ -5,7 +5,7 @@ set(SOURCES
   logistic_regression.hpp
   logistic_regression_impl.hpp
   logistic_regression_function.hpp
-  logistic_regression_function_impl.hpp
+  logistic_regression_function.cpp
 )
 
 # add directory name to sources
@@ -17,10 +17,10 @@ endforeach()
 # the parent scope)
 set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
 
-add_executable(logistic_regression
-  logistic_regression_main.cpp
-)
-target_link_libraries(logistic_regression
-  mlpack
-)
-install(TARGETS logistic_regression RUNTIME DESTINATION bin)
+#add_executable(logistic_regression
+#  logistic_regression_main.cpp
+#)
+#target_link_libraries(logistic_regression
+#  mlpack
+#)
+#install(TARGETS logistic_regression RUNTIME DESTINATION bin)

src/mlpack/methods/logistic_regression/logistic_regression_function.cpp

@@ -0,0 +1,102 @@
+/**
+ * @file logistic_regression_function.cpp
+ * @author Sumedh Ghaisas
+ *
+ * Implementation of hte LogisticRegressionFunction class.
+ */
+#include "logistic_regression_function.hpp"
+
+using namespace mlpack;
+using namespace mlpack::regression;
+
+LogisticRegressionFunction::LogisticRegressionFunction(
+    arma::mat& predictors,
+    arma::vec& responses,
+    const double lambda) :
+    predictors(predictors),
+    responses(responses),
+    lambda(lambda)
+{
+  initialPoint = arma::zeros<arma::mat>(predictors.n_rows + 1, 1);
+}
+
+LogisticRegressionFunction::LogisticRegressionFunction(
+    arma::mat& predictors,
+    arma::vec& responses,
+    const arma::mat& initialPoint,
+    const double lambda) :
+    initialPoint(initialPoint),
+    predictors(predictors),
+    responses(responses),
+    lambda(lambda)
+{
+  //to check if initialPoint is compatible with predictors
+  if (initialPoint.n_rows != (predictors.n_rows + 1) ||
+      initialPoint.n_cols != 1)
+    this->initialPoint = arma::zeros<arma::mat>(predictors.n_rows + 1, 1);
+}
+
+/*
+arma::vec LogisticRegressionFunction::getSigmoid(const arma::vec& values,
+                                       arma::vec& output) const
+{
+  arma::vec out = arma::ones<arma::vec>(values.n_rows,1) /
+      (arma::ones<arma::vec>(values.n_rows,1) + arma::exp(-values));
+  return out;
+}
+*/
+
+/**
+ * Evaluate the logistic regression objective function given the estimated
+ * parameters.
+ */
+double LogisticRegressionFunction::Evaluate(const arma::mat& parameters)
+    const
+{
+  // The objective function is the log-likelihood function (w is the parameters
+  // vector for the model; y is the responses; x is the predictors; sig() is the
+  // sigmoid function):
+  //   f(w) = sum(y log(sig(w'x)) + (1 - y) log(sig(1 - w'x))).
+  // We want to minimize this function.  L2-regularization is just lambda
+  // multiplied by the squared l2-norm of the parameters then divided by two.
+
+  // For the regularization, we ignore the first term, which is the intercept
+  // term.
+  const double regularization = 0.5 * lambda *
+      arma::dot(parameters.col(0).subvec(1, parameters.n_elem - 1),
+                parameters.col(0).subvec(1, parameters.n_elem - 1));
+
+  // Calculate vectors of sigmoids.
+  const arma::vec exponents = predictors.t() * parameters;
+  const arma::vec sigmoid = 1.0 / (1.0 + arma::exp(-exponents));
+
+  // Assemble full objective function.  Often the objective function and the
+  // regularization as given are divided by the number of features, but this
+  // doesn't actually affect the optimization result, so we'll just ignore those
+  // terms for computational efficiency.
+  double result = 0.0;
+  for (size_t i = 0; i < responses.n_elem; ++i)
+  {
+    if (responses[i] == 1)
+      result += responses[i] * log(sigmoid[i]);
+    else
+      result += (1 - responses[i]) * log(1.0 - sigmoid[i]);
+  }
+
+  // Invert the result, because it's a minimization.
+  return -(result + regularization);
+}
+
+void LogisticRegressionFunction::Gradient(const arma::mat& values,
+                                arma::mat& gradient)
+{
+  //regularization
+//  arma::mat regularization = arma::zeros<arma::mat>(predictors.n_rows, 1);
+//  regularization.rows(1, predictors.n_rows - 1) = lambda *
+//      values.rows(1, predictors.n_rows - 1) / responses.n_rows;
+
+  //gradient =
+//  gradient = -(predictors * (responses
+//      - (1 / (1 + arma::exp(-(arma::trans(predictors) * values))))
+//      / responses.n_rows + regularization;
+}

src/mlpack/methods/logistic_regression/logistic_regression_function.hpp

@@ -38,8 +38,18 @@ class LogisticRegressionFunction
   //! Modify the lambda
   double& Lambda() { return lambda; }
 
-  //functions to optimize by l-bfgs
-  double Evaluate(const arma::mat& values) const;
+  /**
+   * Evaluate the logistic regression log-likelihood function with the given
+   * parameters.  Note that if a point has 0 probability of being classified
+   * directly with the given parameters, then Evaluate() will return nan (this
+   * is kind of a corner case and should not happen for reasonable models).
+   *
+   * The optimum (minimum) of this function is 0.0, and occurs when each point
+   * is classified correctly with very high probability.
+   *
+   * @param parameters Vector of logistic regression parameters.
+   */
+  double Evaluate(const arma::mat& parameters) const;
 
   void Gradient(const arma::mat& values, arma::mat& gradient);
 

src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp

@@ -1,88 +0,0 @@
-/**
- * @file logistic_regression_function_impl.hpp
- * @author Sumedh Ghaisas
- *
- * Implementation of hte LogisticRegressionFunction class.
- */
-#ifndef __MLPACK_METHODS_LOGISTIC_REGRESSION_FUNCTION_IMPL_HPP
-#define __MLPACK_METHODS_LOGISTIC_REGRESSION_FUNCTION_IMPL_HPP
-
-// In case it hasn't been done yet.
-#include "logistic_regression_function.hpp"
-
-namespace mlpack {
-namespace regression {
-
-LogisticRegressionFunction::LogisticRegressionFunction(
-    arma::mat& predictors,
-    arma::vec& responses,
-    const double lambda) :
-    predictors(predictors),
-    responses(responses),
-    lambda(lambda)
-{
-  initialPoint = arma::zeros<arma::mat>(predictors.n_rows + 1, 1);
-}
-
-LogisticRegressionFunction::LogisticRegressionFunction(
-    arma::mat& predictors,
-    arma::vec& responses,
-    const arma::mat& initialPoint,
-    const double lambda) :
-    initialPoint(initialPoint),
-    predictors(predictors),
-    responses(responses),
-    lambda(lambda)
-{
-  //to check if initialPoint is compatible with predictors
-  if (initialPoint.n_rows != (predictors.n_rows + 1) ||
-      initialPoint.n_cols != 1)
-    this->initialPoint = arma::zeros<arma::mat>(predictors.n_rows + 1, 1);
-}
-
-arma::vec LogisticRegressionFunction::getSigmoid(const arma::vec& values,
-                                       arma::vec& output) const
-{
-  arma::vec out = arma::ones<arma::vec>(values.n_rows,1) /
-      (arma::ones<arma::vec>(values.n_rows,1) + arma::exp(-values));
-  return out;
-}
-
-double LogisticRegressionFunction::Evaluate(const arma::mat& values) const
-{
-  const size_t nCols = predictors.n_cols;
-
-  //sigmoid = Sigmoid(X' * values)
-  arma::vec sigmoid = 1 / (1 + arma::exp(-(arma::trans(predictors) * values)));
-
-  //l2-regularization(considering only values(2:end) in regularization
-  arma::vec temp = arma::trans(values) * values;
-  double regularization = lambda * (temp(0,0) - values(0,0) * values(0,0)) /
-      (2 * responses.n_rows);
-
-  //J = -(sum(y' * log(sigmoid)) + sum((ones(m,1) - y)' * log(ones(m,1)
-  //   - sigmoid))) + regularization
-  return -(sum(arma::trans(responses) * arma::log(sigmoid)) +
-      sum(arma::trans(arma::ones<arma::vec>(nCols, 1) - responses) *
-      arma::log(arma::ones<arma::vec>(nCols,1) - sigmoid))) /
-      predictors.n_cols + regularization;
-}
-
-void LogisticRegressionFunction::Gradient(const arma::mat& values,
-                                arma::mat& gradient)
-{
-  //regularization
-  arma::mat regularization = arma::zeros<arma::mat>(predictors.n_rows, 1);
-  regularization.rows(1, predictors.n_rows - 1) = lambda *
-      values.rows(1, predictors.n_rows - 1) / responses.n_rows;
-
-  //gradient =
-  gradient = -(predictors * (responses
-      - (1 / (1 + arma::exp(-(arma::trans(predictors) * values))))
-      / responses.n_rows + regularization;
-}
-
-}; // namespace regression
-}; // namespace mlpack
-
-#endif

src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp

@@ -72,11 +72,12 @@ double LogisticRegression<OptimizerType>::ComputeError(
   ones.ones(predictors.n_cols);
   predictors.insert_rows(0, ones);
 
-  double out = errorFunction.Evaluate(predictors, responses, parameters);
+//  double out = errorFunction.Evaluate(predictors, responses, parameters);
 
   predictors.shed_row(0);
 
-  return out;
+//  return out;
+  return 0.0;
 }
 
 template <template<typename> class OptimizerType>