diff --git a/fastlib/branches/fastlib-stl/mlpack/naive_bayes/CMakeLists.txt b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/CMakeLists.txt index be9ead94a1..c0442364a7 100644 --- a/fastlib/branches/fastlib-stl/mlpack/naive_bayes/CMakeLists.txt +++ b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/CMakeLists.txt @@ -4,8 +4,8 @@ cmake_minimum_required(VERSION 2.8) # Anything not in this list will not be compiled into the output library # Do not include test programs here set(SOURCES - simple_nbc.h - phi.h + simple_nbc.cc + phi.cc ) # add directory name to sources diff --git a/fastlib/branches/fastlib-stl/mlpack/naive_bayes/README.txt b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/README.txt index d56d26a507..cabcf15db3 100644 --- a/fastlib/branches/fastlib-stl/mlpack/naive_bayes/README.txt +++ b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/README.txt @@ -6,15 +6,13 @@ The files are the following: --train : the file that contains the training data, the last column being the class of the data point --nbc/classes : the number of classes the data provided has been classified into --test : this file contains the testing data, this still contains its actual labels on the last column, but it is not used. - --output : the file into which you want the output to be written into, defaults to "output.csv" (but it does not output in .csv format, it outputs in the "pretty-print" format) + --output : the file into which you want the output to be written into, defaults to "output.csv" 2. simple_nbc.h - this is the file that contains the definition of the class SimpleNaiveBayesClassifier. The rest of the details are present in the file itself. 3. phi.h - this contains the function that calculates the value of the univariate Gaussian pdf -4. math_functions.h - this file contains functions like returning the highest element and its index in an array/vector. - -5. test_simple_nbc_main.cc - this file contains the class which tests the class SimpleNaiveBayesClassifier. The executable formed is "test_simple_nbc_main". +4. test_simple_nbc_main.cc - this file contains the class which tests the class SimpleNaiveBayesClassifier. The executable formed is "test_simple_nbc_main". - the parameters taken in by this are the following: --training_set : the training set, defaults to the file "trainSet.arff" --training_results : the training results, defaults to the file "trainRes.arff" @@ -22,9 +20,9 @@ The files are the following: --testing_results : the testing results, defaults to the file "testRes.arff" --num_classes : the number of classes in the data, defaults to "2", which in the number of classes of the data present in the default files -6. the .arff files, whose use has been described above. +5. the .arff files, whose use has been described above. -7. build.py - you might want to take a look at them for the purpose of seeing what all files are used where. +6. build.py - you might want to take a look at them for the purpose of seeing what all files are used where. -> An example run would the following: fl-build nbc_main diff --git a/fastlib/branches/fastlib-stl/mlpack/naive_bayes/nbc_main.cc b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/nbc_main.cc index c9e7dd3284..ddcb87fed4 100644 --- a/fastlib/branches/fastlib-stl/mlpack/naive_bayes/nbc_main.cc +++ b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/nbc_main.cc @@ -31,8 +31,6 @@ #include #include -#include -#include const fx_entry_doc parm_nbc_main_entries[] = { {"train", FX_REQUIRED, FX_STR, NULL, @@ -95,7 +93,7 @@ int main(int argc, char* argv[]) { fx_timer_start(nbc_module, "testing"); ////// Calling the function that classifies the test data - nbc.Classify(testing_data, &results); + nbc.Classify(testing_data, results); fx_timer_stop(nbc_module, "testing"); diff --git a/fastlib/branches/fastlib-stl/mlpack/naive_bayes/phi.cc b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/phi.cc new file mode 100644 index 0000000000..2eb8715c92 --- /dev/null +++ b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/phi.cc @@ -0,0 +1,114 @@ +/** + * @author Parikshit Ram (pram@cc.gatech.edu) + * @file phi.h + * + * This file computes the Gaussian probability + * density function + */ +#include "fastlib/fastlib.h" +#include "fastlib/fastlib_int.h" +#include +#include "phi.h" + +long double phi(const arma::vec& x, const arma::vec& mean, const arma::mat& cov) { + + long double det, f; + double exponent; + index_t dim; + arma::mat inv; + arma::vec diff, tmp; + + dim = x.n_rows; + inv = arma::inv(cov); + det = arma::det(cov); + + if( det < 0){ + det = -det; + } + + diff = mean - x; + tmp = inv*diff; + exponent = arma::dot(diff,tmp); + + long double tmp1, tmp2, tmp3; + tmp1 = 1; + tmp2 = dim; + tmp2 = tmp2/2; + tmp2 = pow((2*(math::PI)),tmp2); + tmp1 = tmp1/tmp2; + tmp3 = 1; + tmp2 = sqrt(det); + tmp3 = tmp3/tmp2; + tmp2 = -exponent; + tmp2 = tmp2 / 2; + + f = (tmp1*tmp3*exp(tmp2)); + + return f; +} + +long double phi(const double x, const double mean, const double var) { + + long double f; + + f = exp( -1.0*( (x-mean)*(x-mean)/(2*var) ) )/sqrt(2*math::PI*var); + return f; +} + +long double phi(const arma::vec& x, const arma::vec& mean, const arma::mat& cov, const std::vector& d_cov, arma::vec& g_mean, arma::vec& g_cov){ + + long double det, f; + double exponent; + index_t dim; + arma::mat inv; + arma::vec diff, tmp; + + dim = x.n_rows; + inv = arma::inv(cov); + det = arma::det(cov); + + if( det < 0){ + det = -det; + } + + diff = mean - x; + tmp = inv*diff; + exponent = arma::dot(diff,tmp); + + long double tmp1, tmp2, tmp3; + tmp1 = 1; + tmp2 = dim; + tmp2 = tmp2/2; + tmp2 = pow((2*(math::PI)),tmp2); + tmp1 = tmp1/tmp2; + tmp3 = 1; + tmp2 = sqrt(det); + tmp3 = tmp3/tmp2; + tmp2 = -exponent; + tmp2 = tmp2 / 2; + + f = (tmp1*tmp3*exp(tmp2)); + + // Calculating the g_mean values which would be a (1 X dim) vector + g_mean = f*tmp; + + // Calculating the g_cov values which would be a (1 X (dim*(dim+1)/2)) vector + arma::vec g_cov_tmp(d_cov.size()); + for(index_t i = 0; i < d_cov.size(); i++){ + arma::vec tmp_d; + arma::mat inv_d; + long double tmp_d_cov_d_r; + + tmp_d = d_cov[i]*tmp; + tmp_d_cov_d_r = arma::dot(tmp_d,tmp); + inv_d = inv*d_cov[i]; + + for(index_t j = 0; j < dim; j++) + tmp_d_cov_d_r += inv_d(j,j); + + g_cov_tmp[i] = f*tmp_d_cov_d_r/2; + } + g_cov = g_cov_tmp; + + return f; +} diff --git a/fastlib/branches/fastlib-stl/mlpack/naive_bayes/phi.h b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/phi.h index 3c421918e7..a295d2b6cd 100644 --- a/fastlib/branches/fastlib-stl/mlpack/naive_bayes/phi.h +++ b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/phi.h @@ -5,6 +5,10 @@ * This file computes the Gaussian probability * density function */ + +#ifndef MLPACK_PHI_H +#define MLPACK_PHI_H + #include "fastlib/fastlib.h" #include "fastlib/fastlib_int.h" #include @@ -21,39 +25,7 @@ * @endcode */ -long double phi(arma::vec& x , arma::vec& mean , arma::mat& cov) { - - long double det, f; - double exponent; - index_t dim; - arma::mat inv; - arma::vec diff, tmp; - - dim = x.n_rows; - inv = arma::inv(cov); - det = arma::det(cov); - - if( det < 0){ - det = -det; - } - diff = mean - x; - tmp = inv*diff; - exponent = arma::dot(diff,tmp); - long double tmp1, tmp2, tmp3; - tmp1 = 1; - tmp2 = dim; - tmp2 = tmp2/2; - tmp2 = pow((2*(math::PI)),tmp2); - tmp1 = tmp1/tmp2; - tmp3 = 1; - tmp2 = sqrt(det); - tmp3 = tmp3/tmp2; - tmp2 = -exponent; - tmp2 = tmp2 / 2; - f = (tmp1*tmp3*exp(tmp2)); - - return f; -} +long double phi(const arma::vec& x, const arma::vec& mean, const arma::mat& cov); /** * Calculates the univariate Gaussian probability density function @@ -66,13 +38,7 @@ long double phi(arma::vec& x , arma::vec& mean , arma::mat& cov) { * @endcode */ -long double phi(const double x, const double mean, const double var) { - - long double f = 0.0; - - f = exp(-1.0*((x-mean)*(x-mean)/(2*var)))/sqrt(2*math::PI*var); - return f; -} +long double phi(const double x, const double mean, const double var); /** * Calculates the multivariate Gaussian probability density function @@ -87,57 +53,6 @@ long double phi(const double x, const double mean, const double var) { * @endcode */ -long double phi(arma::vec& x, arma::vec& mean, arma::mat& cov, std::vector& d_cov, arma::vec *g_mean, arma::vec *g_cov){ - - long double det, f; - double exponent; - index_t dim; - arma::mat inv; - arma::vec diff, tmp; - - dim = x.n_rows; - inv = arma::inv(cov); - inv = arma::inv(cov); - det = arma::det(cov); +long double phi(const arma::vec& x, const arma::vec& mean, const arma::mat& cov, const std::vector& d_cov, arma::vec& g_mean, arma::vec& g_cov); - if( det < 0){ - det = -det; - } - diff = mean - x; - tmp = inv*diff; - exponent = arma::dot(diff,tmp); - long double tmp1, tmp2, tmp3; - tmp1 = 1; - tmp2 = dim; - tmp2 = tmp2/2; - tmp2 = pow((2*(math::PI)),tmp2); - tmp1 = tmp1/tmp2; - tmp3 = 1; - tmp2 = sqrt(det); - tmp3 = tmp3/tmp2; - tmp2 = -exponent; - tmp2 = tmp2 / 2; - f = (tmp1*tmp3*exp(tmp2)); - - // Calculating the g_mean values which would be a (1 X dim) vector - *g_mean = f*tmp; - - // Calculating the g_cov values which would be a (1 X (dim*(dim+1)/2)) vector - arma::vec g_cov_tmp(d_cov.size()); - for(index_t i = 0; i < d_cov.size(); i++){ - arma::vec tmp_d; - arma::mat inv_d; - long double tmp_d_cov_d_r; - - tmp_d = d_cov[i]*tmp; - tmp_d_cov_d_r = arma::dot(tmp_d,tmp); - inv_d = inv*d_cov[i]; - for(index_t j = 0; j < dim; j++) - tmp_d_cov_d_r += inv_d(j,j); - g_cov_tmp[i] = f*tmp_d_cov_d_r/2; - } - *g_cov = g_cov_tmp; // Is g_cov larger than g_cov_tmp?? -// g_cov->Copy(g_cov_tmp,d_cov.size()); - - return f; -} +#endif // MLPACK_PHI_H diff --git a/fastlib/branches/fastlib-stl/mlpack/naive_bayes/simple_nbc.cc b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/simple_nbc.cc new file mode 100644 index 0000000000..e94fec945e --- /dev/null +++ b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/simple_nbc.cc @@ -0,0 +1,122 @@ +/** + * @author Parikshit Ram (pram@cc.gatech.edu) + * @file simple_nbc.h + * + * A Naive Bayes Classifier which parametrically + * estimates the distribution of the features. + * It is assumed that the features have been + * sampled from a Gaussian PDF + * + */ + +#define ARMA_NO_DEBUG + +#include "fastlib/fastlib.h" +#include "simple_nbc.h" +#include "phi.h" + +void SimpleNaiveBayesClassifier::InitTrain(const arma::mat& data, datanode* nbc_module) { + + + index_t number_examples = data.n_cols; + index_t number_features = data.n_rows - 1; + nbc_module_ = nbc_module; + + arma::vec feature_sum(number_features), feature_sum_squared(number_features); + + // updating the variables, private and local, according to + // the number of features and classes present in the data + number_of_classes_ = fx_param_int_req(nbc_module_,"classes"); + class_probabilities_.set_size(number_of_classes_); + means_.set_size(number_features,number_of_classes_); + variances_.set_size(number_features,number_of_classes_); + + NOTIFY("%"LI"d examples with %"LI"d features each\n", + number_examples, number_features); + fx_result_int(nbc_module_, "features", number_features); + fx_result_int(nbc_module_, "examples", number_examples); + + // calculating the class probabilities as well as the + // sample mean and variance for each of the features + // with respect to each of the labels + for(index_t i = 0; i < number_of_classes_; i++ ) { + index_t number_of_occurrences = 0; + for (index_t j = 0; j < number_examples; j++) { + index_t flag = (index_t) data(number_features, j); + if(i == flag) { + ++number_of_occurrences; + for(index_t k = 0; k < number_features; k++) { + double tmp = data(k, j); + feature_sum(k) += tmp; + feature_sum_squared(k) += tmp*tmp; + } + } + } + class_probabilities_[i] = (double)number_of_occurrences + / (double)number_examples ; + for(index_t k = 0; k < number_features; k++) { + double fs = feature_sum(k), + fss = feature_sum_squared(k); + + means_(k, i) = (fs / number_of_occurrences); + variances_(k, i) = (fss + - (fs * fs / number_of_occurrences)) + /(number_of_occurrences - 1); + /* + means_(k, i) = (feature_sum(k) / number_of_occurrences); + variances_(k, i) = (feature_sum_squared(k) + - (feature_sum(k) * feature_sum(k) / number_of_occurrences)) + /(number_of_occurrences - 1); + */ + } + feature_sum.zeros(number_features); + feature_sum_squared.zeros(number_features); + } +} + +void SimpleNaiveBayesClassifier::Classify(const arma::mat& test_data, arma::vec& results){ + + // Checking that the number of features in the test data is same + // as in the training data + DEBUG_ASSERT(test_data.n_rows - 1 == means_.n_rows); + + arma::vec tmp_vals(number_of_classes_); + index_t number_features = test_data.n_rows - 1; + + results.zeros(test_data.n_cols); + + NOTIFY("%"LI"d test cases with %"LI"d features each\n", + test_data.n_cols, number_features); + + fx_result_int(nbc_module_,"tests", test_data.n_cols); + // Calculating the joint probability for each of the data points + // for each of the classes + + // looping over every test case + for (index_t n = 0; n < test_data.n_cols; n++) { + + //looping over every class + for (index_t i = 0; i < number_of_classes_; i++) { + // Using the log values to prevent floating point underflow + tmp_vals(i) = log(class_probabilities_(i)); + + //looping over every feature + for (index_t j = 0; j < number_features; j++) { + tmp_vals(i) += log(phi(test_data(j, n), + means_(j, i), + variances_(j, i)) + ); + } + } + + // Find the index of the maximum value in tmp_vals. + index_t max = 0; + for (index_t k = 0; k < number_of_classes_; k++) { + if(tmp_vals(max) < tmp_vals(k)) + max = k; + } + results(n) = max; + } + + return; +} diff --git a/fastlib/branches/fastlib-stl/mlpack/naive_bayes/simple_nbc.h b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/simple_nbc.h index b1d58e6c90..d011d0b5b8 100644 --- a/fastlib/branches/fastlib-stl/mlpack/naive_bayes/simple_nbc.h +++ b/fastlib/branches/fastlib-stl/mlpack/naive_bayes/simple_nbc.h @@ -83,7 +83,7 @@ class SimpleNaiveBayesClassifier { arma::mat means_, variances_; // The variable containing the class probabilities - std::vector class_probabilities_; + arma::vec class_probabilities_; // The variable keeping the information about the // number of classes present @@ -94,11 +94,6 @@ class SimpleNaiveBayesClassifier { public: SimpleNaiveBayesClassifier(){ - /* - means_.Init(0, 0); - variances_.Init(0, 0); - class_probabilities_.Init(0); - */ } ~SimpleNaiveBayesClassifier(){ @@ -116,58 +111,7 @@ class SimpleNaiveBayesClassifier { * nbc.InitTrain(training_data, nbc_module); * @endcode */ - void InitTrain(const arma::mat& data, datanode* nbc_module) { - - std::vector feature_sum, feature_sum_squared; - index_t number_examples = data.n_cols; - index_t number_features = data.n_rows - 1; - nbc_module_ = nbc_module; - - // updating the variables, private and local, according to - // the number of features and classes present in the data - number_of_classes_ = fx_param_int_req(nbc_module_,"classes"); - class_probabilities_.resize(number_of_classes_,0); - means_.zeros(number_features,number_of_classes_); - variances_.zeros(number_features,number_of_classes_); - feature_sum.resize(number_features,0); - feature_sum_squared.resize(number_features,0); - for(index_t k = 0; k < number_features; k++) { - feature_sum[k] = 0; - feature_sum_squared[k] = 0; - } - NOTIFY("%"LI"d examples with %"LI"d features each\n", - number_examples, number_features); - fx_result_int(nbc_module_, "features", number_features); - fx_result_int(nbc_module_, "examples", number_examples); - - // calculating the class probabilities as well as the - // sample mean and variance for each of the features - // with respect to each of the labels - for(index_t i = 0; i < number_of_classes_; i++ ) { - index_t number_of_occurrences = 0; - for (index_t j = 0; j < number_examples; j++) { - index_t flag = (index_t) data(number_features, j); - if(i == flag) { - ++number_of_occurrences; - for(index_t k = 0; k < number_features; k++) { - double tmp = data(k, j); - feature_sum[k] += tmp; - feature_sum_squared[k] += tmp*tmp; - } - } - } - class_probabilities_[i] = (double)number_of_occurrences - / (double)number_examples ; - for(index_t k = 0; k < number_features; k++) { - means_(k, i) = (feature_sum[k] / number_of_occurrences); - variances_(k, i) = (feature_sum_squared[k] - - (feature_sum[k] * feature_sum[k] / number_of_occurrences)) - /(number_of_occurrences - 1); - feature_sum[k] = 0; - feature_sum_squared[k] = 0; - } - } - } + void InitTrain(const arma::mat& data, datanode* nbc_module); /** * Given a bunch of data points, this function evaluates the class @@ -180,50 +124,6 @@ class SimpleNaiveBayesClassifier { * nbc.Classify(test_data, &results); * @endcode */ - void Classify(const arma::mat& test_data, arma::vec *results){ - - // Checking that the number of features in the test data is same - // as in the training data - DEBUG_ASSERT(test_data.n_rows - 1 == means_.n_rows); - - arma::vec tmp_vals(number_of_classes_); - index_t number_features = test_data.n_rows - 1; - - arma::vec evaluated_result(test_data.n_cols); - - NOTIFY("%"LI"d test cases with %"LI"d features each\n", - test_data.n_cols, number_features); - - fx_result_int(nbc_module_,"tests", test_data.n_cols); - // Calculating the joint probability for each of the data points - // for each of the classes - - // looping over every test case - for (index_t n = 0; n < test_data.n_cols; n++) { - - //looping over every class - for (index_t i = 0; i < number_of_classes_; i++) { - // Using the log values to prevent floating point underflow - tmp_vals[i] = log(class_probabilities_[i]); - for (index_t j = 0; j < number_features; j++) { - tmp_vals[i] += log(phi(test_data(j, n), - means_(j, i), - variances_(j, i)) - ); - } - } - - // Find the index of the maximum value in tmp_vals. - evaluated_result[n] = 0; - for (index_t k = 0; k < number_of_classes_; k++) { - if(tmp_vals[evaluated_result[n]] < tmp_vals[k]) - evaluated_result[n] = k; - } - } - // The result is being put in a vector - *results = evaluated_result; - - return; - } + void Classify(const arma::mat& test_data, arma::vec& results); }; #endif