From c40a059e770e012bdae5fb05b06f7d73c64cd915 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 13 Nov 2013 16:42:29 +0000 Subject: [PATCH] Refactor Evaluate() function so that it works alright, and eliminate unnecessary parameters to the function. Move LogisticRegressionFunction implementation into a .cpp file because it is not templatized (for now). --- .../logistic_regression/CMakeLists.txt | 16 +-- .../logistic_regression_function.cpp | 102 ++++++++++++++++++ .../logistic_regression_function.hpp | 14 ++- .../logistic_regression_function_impl.hpp | 88 --------------- .../logistic_regression_impl.hpp | 5 +- 5 files changed, 125 insertions(+), 100 deletions(-) create mode 100644 src/mlpack/methods/logistic_regression/logistic_regression_function.cpp delete mode 100644 src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp diff --git a/src/mlpack/methods/logistic_regression/CMakeLists.txt b/src/mlpack/methods/logistic_regression/CMakeLists.txt index d1477ff713..b175c6dd1c 100644 --- a/src/mlpack/methods/logistic_regression/CMakeLists.txt +++ b/src/mlpack/methods/logistic_regression/CMakeLists.txt @@ -5,7 +5,7 @@ set(SOURCES logistic_regression.hpp logistic_regression_impl.hpp logistic_regression_function.hpp - logistic_regression_function_impl.hpp + logistic_regression_function.cpp ) # add directory name to sources @@ -17,10 +17,10 @@ endforeach() # the parent scope) set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) -add_executable(logistic_regression - logistic_regression_main.cpp -) -target_link_libraries(logistic_regression - mlpack -) -install(TARGETS logistic_regression RUNTIME DESTINATION bin) +#add_executable(logistic_regression +# logistic_regression_main.cpp +#) +#target_link_libraries(logistic_regression +# mlpack +#) +#install(TARGETS logistic_regression RUNTIME DESTINATION bin) diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp b/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp new file mode 100644 index 0000000000..3888b6c16c --- /dev/null +++ b/src/mlpack/methods/logistic_regression/logistic_regression_function.cpp @@ -0,0 +1,102 @@ +/** + * @file logistic_regression_function.cpp + * @author Sumedh Ghaisas + * + * Implementation of hte LogisticRegressionFunction class. + */ +#include "logistic_regression_function.hpp" + +using namespace mlpack; +using namespace mlpack::regression; + +LogisticRegressionFunction::LogisticRegressionFunction( + arma::mat& predictors, + arma::vec& responses, + const double lambda) : + predictors(predictors), + responses(responses), + lambda(lambda) +{ + initialPoint = arma::zeros(predictors.n_rows + 1, 1); +} + +LogisticRegressionFunction::LogisticRegressionFunction( + arma::mat& predictors, + arma::vec& responses, + const arma::mat& initialPoint, + const double lambda) : + initialPoint(initialPoint), + predictors(predictors), + responses(responses), + lambda(lambda) +{ + //to check if initialPoint is compatible with predictors + if (initialPoint.n_rows != (predictors.n_rows + 1) || + initialPoint.n_cols != 1) + this->initialPoint = arma::zeros(predictors.n_rows + 1, 1); +} + +/* +arma::vec LogisticRegressionFunction::getSigmoid(const arma::vec& values, + arma::vec& output) const +{ + arma::vec out = arma::ones(values.n_rows,1) / + (arma::ones(values.n_rows,1) + arma::exp(-values)); + return out; +} +*/ + +/** + * Evaluate the logistic regression objective function given the estimated + * parameters. + */ +double LogisticRegressionFunction::Evaluate(const arma::mat& parameters) + const +{ + // The objective function is the log-likelihood function (w is the parameters + // vector for the model; y is the responses; x is the predictors; sig() is the + // sigmoid function): + // f(w) = sum(y log(sig(w'x)) + (1 - y) log(sig(1 - w'x))). + // We want to minimize this function. L2-regularization is just lambda + // multiplied by the squared l2-norm of the parameters then divided by two. + + // For the regularization, we ignore the first term, which is the intercept + // term. + const double regularization = 0.5 * lambda * + arma::dot(parameters.col(0).subvec(1, parameters.n_elem - 1), + parameters.col(0).subvec(1, parameters.n_elem - 1)); + + // Calculate vectors of sigmoids. + const arma::vec exponents = predictors.t() * parameters; + const arma::vec sigmoid = 1.0 / (1.0 + arma::exp(-exponents)); + + // Assemble full objective function. Often the objective function and the + // regularization as given are divided by the number of features, but this + // doesn't actually affect the optimization result, so we'll just ignore those + // terms for computational efficiency. + double result = 0.0; + for (size_t i = 0; i < responses.n_elem; ++i) + { + if (responses[i] == 1) + result += responses[i] * log(sigmoid[i]); + else + result += (1 - responses[i]) * log(1.0 - sigmoid[i]); + } + + // Invert the result, because it's a minimization. + return -(result + regularization); +} + +void LogisticRegressionFunction::Gradient(const arma::mat& values, + arma::mat& gradient) +{ + //regularization +// arma::mat regularization = arma::zeros(predictors.n_rows, 1); +// regularization.rows(1, predictors.n_rows - 1) = lambda * +// values.rows(1, predictors.n_rows - 1) / responses.n_rows; + + //gradient = +// gradient = -(predictors * (responses +// - (1 / (1 + arma::exp(-(arma::trans(predictors) * values)))) +// / responses.n_rows + regularization; +} diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp b/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp index ccc7a4ed57..3ba05f4f0e 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_function.hpp @@ -38,8 +38,18 @@ class LogisticRegressionFunction //! Modify the lambda double& Lambda() { return lambda; } - //functions to optimize by l-bfgs - double Evaluate(const arma::mat& values) const; + /** + * Evaluate the logistic regression log-likelihood function with the given + * parameters. Note that if a point has 0 probability of being classified + * directly with the given parameters, then Evaluate() will return nan (this + * is kind of a corner case and should not happen for reasonable models). + * + * The optimum (minimum) of this function is 0.0, and occurs when each point + * is classified correctly with very high probability. + * + * @param parameters Vector of logistic regression parameters. + */ + double Evaluate(const arma::mat& parameters) const; void Gradient(const arma::mat& values, arma::mat& gradient); diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp b/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp deleted file mode 100644 index 604252b8d5..0000000000 --- a/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp +++ /dev/null @@ -1,88 +0,0 @@ -/** - * @file logistic_regression_function_impl.hpp - * @author Sumedh Ghaisas - * - * Implementation of hte LogisticRegressionFunction class. - */ -#ifndef __MLPACK_METHODS_LOGISTIC_REGRESSION_FUNCTION_IMPL_HPP -#define __MLPACK_METHODS_LOGISTIC_REGRESSION_FUNCTION_IMPL_HPP - -// In case it hasn't been done yet. -#include "logistic_regression_function.hpp" - -namespace mlpack { -namespace regression { - -LogisticRegressionFunction::LogisticRegressionFunction( - arma::mat& predictors, - arma::vec& responses, - const double lambda) : - predictors(predictors), - responses(responses), - lambda(lambda) -{ - initialPoint = arma::zeros(predictors.n_rows + 1, 1); -} - -LogisticRegressionFunction::LogisticRegressionFunction( - arma::mat& predictors, - arma::vec& responses, - const arma::mat& initialPoint, - const double lambda) : - initialPoint(initialPoint), - predictors(predictors), - responses(responses), - lambda(lambda) -{ - //to check if initialPoint is compatible with predictors - if (initialPoint.n_rows != (predictors.n_rows + 1) || - initialPoint.n_cols != 1) - this->initialPoint = arma::zeros(predictors.n_rows + 1, 1); -} - -arma::vec LogisticRegressionFunction::getSigmoid(const arma::vec& values, - arma::vec& output) const -{ - arma::vec out = arma::ones(values.n_rows,1) / - (arma::ones(values.n_rows,1) + arma::exp(-values)); - return out; -} - -double LogisticRegressionFunction::Evaluate(const arma::mat& values) const -{ - const size_t nCols = predictors.n_cols; - - //sigmoid = Sigmoid(X' * values) - arma::vec sigmoid = 1 / (1 + arma::exp(-(arma::trans(predictors) * values))); - - //l2-regularization(considering only values(2:end) in regularization - arma::vec temp = arma::trans(values) * values; - double regularization = lambda * (temp(0,0) - values(0,0) * values(0,0)) / - (2 * responses.n_rows); - - //J = -(sum(y' * log(sigmoid)) + sum((ones(m,1) - y)' * log(ones(m,1) - // - sigmoid))) + regularization - return -(sum(arma::trans(responses) * arma::log(sigmoid)) + - sum(arma::trans(arma::ones(nCols, 1) - responses) * - arma::log(arma::ones(nCols,1) - sigmoid))) / - predictors.n_cols + regularization; -} - -void LogisticRegressionFunction::Gradient(const arma::mat& values, - arma::mat& gradient) -{ - //regularization - arma::mat regularization = arma::zeros(predictors.n_rows, 1); - regularization.rows(1, predictors.n_rows - 1) = lambda * - values.rows(1, predictors.n_rows - 1) / responses.n_rows; - - //gradient = - gradient = -(predictors * (responses - - (1 / (1 + arma::exp(-(arma::trans(predictors) * values)))) - / responses.n_rows + regularization; -} - -}; // namespace regression -}; // namespace mlpack - -#endif diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp b/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp index 8a7671992e..87c151dfb7 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp @@ -72,11 +72,12 @@ double LogisticRegression::ComputeError( ones.ones(predictors.n_cols); predictors.insert_rows(0, ones); - double out = errorFunction.Evaluate(predictors, responses, parameters); +// double out = errorFunction.Evaluate(predictors, responses, parameters); predictors.shed_row(0); - return out; +// return out; + return 0.0; } template class OptimizerType>