Initial commit of logistic regression by Sumedh Ghaisas (#305).

src/mlpack/methods/logistic_regression/CMakeLists.txt

@@ -0,0 +1,26 @@
+# Define the files we need to compile
+# Anything not in this list will not be compiled into the output library
+# Do not include test programs here
+set(SOURCES
+  logistic_regression.hpp
+  logistic_regression_impl.hpp
+  logistic_regression_function.hpp
+  logistic_regression_function_impl.hpp
+)
+
+# add directory name to sources
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+# append sources (with directory name) to list of all MLPACK sources (used at
+# the parent scope)
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
+
+add_executable(logistic_regression
+  logistic_regression_main.cpp
+)
+target_link_libraries(logistic_regression
+  mlpack
+)
+install(TARGETS logistic_regression RUNTIME DESTINATION bin)

src/mlpack/methods/logistic_regression/logistic_regression.hpp

@@ -0,0 +1,71 @@
+/**
+ * @file logistic_regression.hpp
+ * @author Sumedh Ghaisas
+ *
+ * The LogisticRegression class, which implements logistic regression.
+ */
+#ifndef __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_HPP
+#define __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_HPP
+
+#include <mlpack/core.hpp>
+#include <mlpack/core/optimizers/lbfgs/lbfgs.hpp>
+
+#include "logistic_regression_function.hpp"
+
+namespace mlpack {
+namespace regression {
+
+template<
+  template<typename> class OptimizerType = mlpack::optimization::L_BFGS
+>
+class LogisticRegression
+{
+ public:
+  LogisticRegression(arma::mat& predictors,
+                     arma::vec& responses,
+                     const double lambda = 0);
+
+  LogisticRegression(arma::mat& predictors,
+                     arma::vec& responses,
+                     const arma::mat& initialPoint,
+                     const double lambda = 0);
+
+  //! Return the parameters (the b vector).
+  const arma::vec& Parameters() const { return parameters; }
+  //! Modify the parameters (the b vector).
+  arma::vec& Parameters() { return parameters; }
+
+  //! Return the lambda value
+  const double& Lambda() const { return lambda; }
+  //! Modify the lambda value
+  double& Lambda() { return lambda; }
+
+  double LearnModel();
+
+  //predict functions
+  void Predict(arma::mat& predictors,
+               arma::vec& responses,
+               const double decisionBoundary = 0.5);
+
+  double ComputeAccuracy(arma::mat& predictors,
+                         const arma::vec& responses,
+                         const double decisionBoundary = 0.5);
+
+  double ComputeError(arma::mat& predictors,const arma::vec& responses);
+
+ private:
+  arma::vec parameters;
+  arma::mat& predictors;
+  arma::vec& responses;
+  LogisticRegressionFunction errorFunction;
+  OptimizerType<LogisticRegressionFunction> optimizer;
+  double lambda;
+};
+
+}; // namespace regression
+}; // namespace mlpack
+
+// Include implementation.
+#include "logistic_regression_impl.hpp"
+
+#endif // __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_HPP

src/mlpack/methods/logistic_regression/logistic_regression_function.hpp

@@ -0,0 +1,80 @@
+/**
+ * @file logistic_regression_function.hpp
+ * @author Sumedh Ghaisas
+ *
+ * Implementation of the logistic regression function, which is meant to be
+ * optimized by a separate optimizer class that takes LogisticRegressionFunction
+ * as its FunctionType class.
+ */
+#ifndef __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_FUNCTION_HPP
+#define __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_FUNCTION_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace regression {
+
+class LogisticRegressionFunction
+{
+ public:
+  LogisticRegressionFunction(arma::mat& predictors,
+                             arma::vec& responses,
+                             const double lambda = 0);
+
+  LogisticRegressionFunction(arma::mat& predictors,
+                             arma::vec& responses,
+                             const arma::mat& initialPoint,
+                             const double lambda = 0);
+
+  arma::vec getSigmoid(const arma::vec& values) const;
+
+  //evaluates the logistic function with given parameters
+  double Evaluate(const arma::mat& predictors,
+                  const arma::vec& responses,
+                  const arma::mat& values) const;
+
+  //!Return the initial point
+  const arma::mat& InitialPoint() const { return initialPoint; }
+  //! Modify the initial point
+  arma::mat& InitialPoint() { return initialPoint; }
+
+  //!Return the lambda
+  const double& Lambda() const { return lambda; }
+  //! Modify the lambda
+  double& Lambda() { return lambda; }
+
+  //functions to optimize by l-bfgs
+  double Evaluate(const arma::mat& values) const
+  {
+      return Evaluate(predictors, responses, values);
+  }
+
+  void Gradient(const arma::mat& values, arma::mat& gradient);
+
+  const arma::mat& GetInitialPoint() const { return initialPoint; }
+
+  //functions to optimize by sgd
+  double Evaluate(const arma::mat& values, const size_t i) const
+  {
+    return Evaluate(values);
+  }
+  void Gradient(const arma::mat& values,
+                const size_t i,
+                arma::mat& gradient)
+  {
+    Gradient(values,gradient);
+  }
+
+  size_t NumFunctions() { return 1; }
+
+ private:
+  arma::mat initialPoint;
+  arma::mat& predictors;
+  arma::vec& responses;
+  double lambda;
+};
+
+}; // namespace regression
+}; // namespace mlpack
+
+#endif // __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_FUNCTION_HPP

src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp

@@ -0,0 +1,89 @@
+/**
+ * @file logistic_regression_function_impl.hpp
+ * @author Sumedh Ghaisas
+ *
+ * Implementation of hte LogisticRegressionFunction class.
+ */
+#ifndef __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_FUNCTION_IMPL_HPP
+#define __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_FUNCTION_IMPL_HPP
+
+// In case it hasn't been done yet.
+#include "logistic_regression_function.hpp"
+
+namespace mlpack {
+namespace regression {
+
+LogisticRegressionFunction::LogisticRegressionFunction(
+    arma::mat& predictors,
+    arma::vec& responses,
+    const double lambda) :
+    predictors(predictors),
+    responses(responses),
+    lambda(lambda)
+{
+  initialPoint = arma::zeros<arma::mat>(predictors.n_rows + 1, 1);
+}
+
+LogisticRegressionFunction::LogisticRegressionFunction(
+    arma::mat& predictors,
+    arma::vec& responses,
+    const arma::mat& initialPoint,
+    const double lambda) :
+    initialPoint(initialPoint),
+    predictors(predictors),
+    responses(responses),
+    lambda(lambda)
+{
+  //to check if initialPoint is compatible with predictors
+  if(initialPoint.n_rows != (predictors.n_rows + 1) || initialPoint.n_cols != 1)
+    this->initialPoint = arma::zeros<arma::mat>(predictors.n_rows + 1,1);
+}
+
+arma::vec LogisticRegressionFunction::getSigmoid(const arma::vec& values) const
+{
+  arma::vec out = arma::ones<arma::vec>(values.n_rows,1) /
+      (arma::ones<arma::vec>(values.n_rows,1) + arma::exp(-values));
+  return out;
+}
+
+double LogisticRegressionFunction::Evaluate(
+    const arma::mat& predictors,
+    const arma::vec& responses,
+    const arma::mat& values) const
+{
+  size_t nCols = predictors.n_cols;
+
+  //sigmoid = Sigmoid(X' * values)
+  arma::vec sigmoid = getSigmoid(arma::trans(predictors) * values);
+
+  //l2-regularization(considering only values(2:end) in regularization
+  arma::vec temp = arma::trans(values) * values;
+  double regularization = lambda * (temp(0,0) - values(0,0) * values(0,0)) /
+      (2 * responses.n_rows);
+
+  //J = -(sum(y' * log(sigmoid)) + sum((ones(m,1) - y)' * log(ones(m,1)
+  //   - sigmoid))) + regularization
+  return -(sum(arma::trans(responses) * arma::log(sigmoid)) +
+      sum(arma::trans(arma::ones<arma::vec>(nCols, 1) - responses) *
+      arma::log(arma::ones<arma::vec>(nCols,1) - sigmoid))) /
+      predictors.n_cols + regularization;
+}
+
+void LogisticRegressionFunction::Gradient(
+    const arma::mat& values,
+    arma::mat& gradient)
+{
+  //regularization
+  arma::mat regularization = arma::zeros<arma::mat>(predictors.n_rows, 1);
+  regularization.rows(1, predictors.n_rows - 1) = lambda *
+      values.rows(1, predictors.n_rows - 1) / responses.n_rows;
+
+  //gradient =
+  gradient = -(predictors * (responses - getSigmoid(arma::trans(predictors) *
+      values))) / responses.n_rows + regularization;
+}
+
+}; // namespace regression
+}; // namespace mlpack
+
+#endif

src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp

@@ -0,0 +1,120 @@
+/**
+ * @file logistic_regression_impl.hpp
+ * @author Sumedh Ghaisas
+ *
+ * Implementation of the LogisticRegression class.
+ */
+#ifndef __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_IMPL_HPP
+#define __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "logistic_regression.hpp"
+
+namespace mlpack {
+namespace regression {
+
+template<template<typename> class OptimizerType>
+LogisticRegression<OptimizerType>::LogisticRegression(
+    arma::mat& predictors,
+    arma::vec& responses,
+    const double lambda) :
+    predictors(predictors),
+    responses(responses),
+    errorFunction(LogisticRegressionFunction(predictors,responses,lambda)),
+    optimizer(OptimizerType<LogisticRegressionFunction>(errorFunction)),
+    lambda(lambda)
+{
+  parameters.zeros(predictors.n_rows + 1);
+}
+
+template<template<typename> class OptimizerType>
+LogisticRegression<OptimizerType>::LogisticRegression(
+    arma::mat& predictors,
+    arma::vec& responses,
+    const arma::mat& initialPoint,
+    const double lambda) :
+    predictors(predictors),
+    responses(responses),
+    errorFunction(LogisticRegressionFunction(predictors,responses)),
+    optimizer(OptimizerType<LogisticRegressionFunction>(errorFunction)),
+    lambda(lambda)
+{
+  parameters.zeros(predictors.n_rows + 1);
+}
+
+template <template<typename> class OptimizerType>
+double LogisticRegression<OptimizerType>::LearnModel()
+{
+  //add rows of ones to predictors
+  arma::rowvec ones;
+  ones.ones(predictors.n_cols);
+  predictors.insert_rows(0, ones);
+
+  double out;
+  Timer::Start("logistic_regression_optimization");
+  out = optimizer.Optimize(parameters);
+  Timer::Stop("logistic_regression_optimization");
+
+  //shed the added rows
+  predictors.shed_row(0);
+
+  return out;
+}
+
+template <template<typename> class OptimizerType>
+double LogisticRegression<OptimizerType>::ComputeError(
+    arma::mat& predictors,
+    const arma::vec& responses)
+{
+  // Here we add the row of ones to the predictors.
+  arma::rowvec ones;
+  ones.ones(predictors.n_cols);
+  predictors.insert_rows(0, ones);
+
+  double out = errorFunction.Evaluate(predictors,responses,parameters);
+
+  predictors.shed_row(0);
+
+  return out;
+}
+
+template <template<typename> class OptimizerType>
+double LogisticRegression<OptimizerType>::ComputeAccuracy(
+    arma::mat& predictors,
+    const arma::vec& responses,
+    const double decisionBoundary)
+{
+  arma::vec temp_responses;
+  Predict(predictors,temp_responses,decisionBoundary);
+  int count = 0;
+  for (size_t i = 0; i < responses.n_rows; i++)
+    if (responses(i, 0) == temp_responses(i, 0))
+      count++;
+
+  return (double) (count * 100) / responses.n_rows;
+}
+
+template <template<typename> class OptimizerType>
+void LogisticRegression<OptimizerType>::Predict(
+    arma::mat& predictors,
+    arma::vec& responses,
+    const double decisionBoundary)
+{
+  //add rows of ones to predictors
+  arma::rowvec ones;
+  ones.ones(predictors.n_cols);
+  predictors.insert_rows(0, ones);
+
+  responses = arma::floor(
+      errorFunction.getSigmoid(arma::trans(predictors) * parameters) -
+      decisionBoundary * arma::ones<arma::vec>(predictors.n_cols, 1)) +
+      arma::ones<arma::vec>(predictors.n_cols);
+
+  //shed the added rows
+  predictors.shed_row(0);
+}
+
+}; // namespace regression
+}; // namespace mlpack
+
+#endif // __MLPACK_METHODS_LOGISTIC_REGRESSION_LOGISTIC_REGRESSION_IMPL_HPP