diff --git a/fastlib/branches/fastlib-stl/fastlib/data/CMakeLists.txt b/fastlib/branches/fastlib-stl/fastlib/data/CMakeLists.txt index 516eabfe48..ccfdb37459 100644 --- a/fastlib/branches/fastlib-stl/fastlib/data/CMakeLists.txt +++ b/fastlib/branches/fastlib-stl/fastlib/data/CMakeLists.txt @@ -7,6 +7,10 @@ set(SOURCES crossvalidation.h dataset.h dataset.cc + dataset_info.h + dataset_info.cc + dataset_feature.h + dataset_feature.cc ) # add directory name to sources diff --git a/fastlib/branches/fastlib-stl/fastlib/data/crossvalidation.h b/fastlib/branches/fastlib-stl/fastlib/data/crossvalidation.h index 9b1e52428d..e7a848d210 100644 --- a/fastlib/branches/fastlib-stl/fastlib/data/crossvalidation.h +++ b/fastlib/branches/fastlib-stl/fastlib/data/crossvalidation.h @@ -38,12 +38,9 @@ #ifndef DATA_CROSSVALIDATION #define DATA_CROSSVALIDATION -#include -#include -#include +#include #include "dataset.h" -//#include "dataset.h" #include "../la/matrix.h" #include "../fx/fx.h" @@ -111,7 +108,7 @@ * // * void InitTrain(const Dataset& dataset, int n_classes, datanode *module); * // For a test datum, returns the class label 0 <= label < n_classes - * int Classify(const Vector& test_datum); + * int Classify(const vec& test_datum); * }; * @endcode */ @@ -139,7 +136,7 @@ class SimpleCrossValidator { /** Total number correct classified. */ index_t n_correct_; /** Confusion matrix. */ - Matrix confusion_matrix_; + arma::mat confusion_matrix_; public: SimpleCrossValidator() {} @@ -198,7 +195,7 @@ class SimpleCrossValidator { * The element at row i column j is the number of training samples where * the actual classification is i but the predicted classification is j. */ - const Matrix& confusion_matrix() const { + const arma::mat& confusion_matrix() const { return confusion_matrix_; } @@ -221,12 +218,12 @@ void SimpleCrossValidator::SaveTrainTest_( std::ostringstream o; if( !(o << i_fold) ) - abort(); //? + abort(); // bizarre error, should we really check? train_name = "train_" + o.str() + ".csv"; test_name = "test_" + o.str() + ".csv"; - train.WriteCsv(train_name.c_str()); - test.WriteCsv(test_name.c_str()); + train.WriteCsv(train_name); + test.WriteCsv(test_name); } @@ -258,8 +255,7 @@ void SimpleCrossValidator::Init( DEBUG_ONLY(n_correct_ = BIG_BAD_NUMBER); - confusion_matrix_.Init(n_classes_, n_classes_); - confusion_matrix_.SetZero(); + confusion_matrix_.zeros(n_classes_, n_classes_); } template @@ -286,7 +282,7 @@ void SimpleCrossValidator::Run(bool randomized) { kfold_module_->key, i_fold, classifier_fx_name_); datanode *foldmodule = fx_submodule(classifier_module, ".."); - data_->SplitTrainTest(n_folds_, i_fold, permutation, &train, &test); + data_->SplitTrainTest(n_folds_, i_fold, permutation, train, test); if (fx_param_bool(kfold_module_, "save", 0)) { SaveTrainTest_(i_fold, train, test); @@ -300,15 +296,12 @@ void SimpleCrossValidator::Run(bool randomized) { fx_timer_start(foldmodule, "test"); VERBOSE_MSG(1, "cross: Testing fold %d", i_fold); for (index_t i = 0; i < test.n_points(); i++) { - Vector test_vector_with_label; - Vector test_vector; - - test.matrix().MakeColumnVector(i, &test_vector_with_label); - test_vector_with_label.MakeSubvector( - 0, test.n_features()-1, &test_vector); + arma::vec test_vector(test.n_features() - 1); + for(int j = 0; j < test.n_features() - 1; j++) + test_vector[j] = test.matrix()(i, j); int label_predict = classifier.Classify(test_vector); - double label_expect_dbl = test_vector_with_label[test.n_features()-1]; + double label_expect_dbl = test.matrix()(i, test.n_features() - 1); int label_expect = int(label_expect_dbl); DEBUG_ASSERT(double(label_expect) == label_expect_dbl); @@ -321,7 +314,7 @@ void SimpleCrossValidator::Run(bool randomized) { local_n_correct++; } - confusion_matrix_.ref(label_expect, label_predict) += 1; + confusion_matrix_(label_expect, label_predict) += 1; } fx_timer_stop(foldmodule, "test"); @@ -405,7 +398,7 @@ class GeneralCrossValidator { /** Total number correct classified */ index_t clsf_n_correct_; /** Confusion matrix */ - Matrix clsf_confusion_matrix_; + arma::mat clsf_confusion_matrix_; /** variables for type 1/2: regression, density estimation, etc. */ /** mean squared error over all folds*/ @@ -468,7 +461,7 @@ class GeneralCrossValidator { * The element at row i column j is the number of training samples where * the actual classification is i but the predicted classification is j. */ - const Matrix& clsf_confusion_matrix() const { + const arma::mat& clsf_confusion_matrix() const { return clsf_confusion_matrix_; } @@ -489,9 +482,9 @@ class GeneralCrossValidator { // detemine the number of data samples for training and validation according to i_fold index_t n_cv_validation, i_validation, i_train; n_cv_validation = 0; - for (index_t i_classes=0; i_classesInitBlank(); train->info().InitContinuous(n_cv_features); - train->matrix().Init(n_cv_features, n_cv_train); + train->matrix().set_size(n_cv_features, n_cv_train); validation->InitBlank(); validation->info().InitContinuous(n_cv_features); - validation->matrix().Init(n_cv_features, n_cv_validation); + validation->matrix().set_size(n_cv_features, n_cv_validation); // make training set and vaidation set by concatenation i_train = 0; i_validation = 0; - for (index_t i_classes=0; i_classesmatrix().MakeColumnVector(i_train, &dest); + dest = train->matrix().colptr(i_train); i_train++; } else { // add to validation set - validation->matrix().MakeColumnVector(i_validation, &dest); + dest = validation->matrix().colptr(i_validation); i_validation++; } - data_->matrix().MakeColumnVector(cv_labels_startpos[i_classes]+j, &source); - dest.CopyValues(source); + memcpy(dest, data_->matrix().colptr(cv_labels_startpos[i_classes] + j), + sizeof(double) * data_->matrix().n_rows); } } } @@ -570,11 +563,10 @@ void GeneralCrossValidator::Init( clsf_n_classes_ = data_->n_labels(); clsf_n_correct_ = 0; // initialize confusion matrix - clsf_confusion_matrix_.Init(clsf_n_classes_, clsf_n_classes_); - clsf_confusion_matrix_.SetZero(); + clsf_confusion_matrix_.zeros(clsf_n_classes_, clsf_n_classes_); } else if (learner_typeid_ == 1 || learner_typeid_ == 2) { - clsf_confusion_matrix_.Init(1,1); + clsf_confusion_matrix_ = 0.0; /* 1x1 matrix */ // initialize mean squared error over all folds msq_err_all_folds_ = 0.0; } @@ -619,11 +611,6 @@ void GeneralCrossValidator::Run(bool randomized) { for (index_t j=0; j empty; - sub_permutation.swap(empty); - } } } // e.g. [10,13,5,17,0,6,7,,4,9,8,1,2,,...] else { @@ -659,14 +646,16 @@ void GeneralCrossValidator::Run(bool randomized) { VERBOSE_MSG(1, "cross: Validation fold %d", i_fold); for (index_t i = 0; i < validation.n_points(); i++) { - Vector validation_vector_with_label; - Vector validation_vector; + arma::vec validation_vector(validation.n_features() - 1); + + memcpy(validation_vector.memptr(), validation.matrix().colptr(i), + sizeof(double) * (validation_vector.n_elem)); validation.matrix().MakeColumnVector(i, &validation_vector_with_label); validation_vector_with_label.MakeSubvector(0, validation.n_features()-1, &validation_vector); // testing (classification) int label_predict = int(classifier.Predict(learner_typeid_, validation_vector)); - double label_expect_dbl = validation_vector_with_label[validation.n_features()-1]; + double label_expect_dbl = validation.matrix()(i, validation_vector.n_elem); int label_expect = int(label_expect_dbl); DEBUG_ASSERT(double(label_expect) == label_expect_dbl); @@ -678,7 +667,7 @@ void GeneralCrossValidator::Run(bool randomized) { if (label_expect == label_predict) { local_n_correct++; } - clsf_confusion_matrix_.ref(label_expect, label_predict) += 1; + clsf_confusion_matrix_(label_expect, label_predict) += 1; } fx_timer_stop(foldmodule, "validation"); @@ -720,7 +709,7 @@ void GeneralCrossValidator::Run(bool randomized) { datanode *foldmodule = fx_submodule(learner_module, ".."); // Split general data sets according to i_fold - data_->SplitTrainTest(n_folds_, i_fold, permutation, &train, &validation); + data_->SplitTrainTest(n_folds_, i_fold, permutation, train, validation); if (fx_param_bool(kfold_module_, "save", 0)) { SaveTrainValidationSet_(i_fold, train, validation); @@ -736,15 +725,14 @@ void GeneralCrossValidator::Run(bool randomized) { fx_timer_start(foldmodule, "validation"); VERBOSE_MSG(1, "cross: Validation fold %d", i_fold); for (index_t i = 0; i < validation.n_points(); i++) { - Vector validation_vector_with_label; - Vector validation_vector; + arma::vec validation_vector(validation.n_features() - 1); + + memcpy(validation_vector.memptr(), validation.matrix().colptr(i), + sizeof(double) * (validation_vector.n_elem)); - validation.matrix().MakeColumnVector(i, &validation_vector_with_label); - validation_vector_with_label.MakeSubvector( - 0, validation.n_features()-1, &validation_vector); - // testing + // testing double value_predict = learner.Predict(learner_typeid_, validation_vector); - double value_true = validation_vector_with_label[validation.n_features()-1]; + double value_true = validation.matrix()(i, validation_vector.n_elem); double value_err = value_predict - value_true; // Calculate squared error: sublevel @@ -769,5 +757,4 @@ void GeneralCrossValidator::Run(bool randomized) { } } - #endif diff --git a/fastlib/branches/fastlib-stl/fastlib/data/dataset.cc b/fastlib/branches/fastlib-stl/fastlib/data/dataset.cc index cd51c1f11e..53be18cb35 100644 --- a/fastlib/branches/fastlib-stl/fastlib/data/dataset.cc +++ b/fastlib/branches/fastlib-stl/fastlib/data/dataset.cc @@ -47,290 +47,26 @@ #include #include - -void DatasetFeature::Format(double value, std::string& result) const { - if (unlikely(isnan(value))) { - result = "?"; - return; - } - std::ostringstream o; - switch (type_) { - case CONTINUOUS: - if (floor(value) != value) { - // non-integer - o.setf( std::ios::scientific ); - } else { - // value is actually an integer - o.precision(17); - } - break; - case INTEGER: - case NOMINAL: - break; - #ifdef DEBUG - default: abort(); - #endif - } - if( !(o << value ) ) - abort(); - result = o.str(); -} - -success_t DatasetFeature::Parse(const std::string& str, double *d) const { - if (unlikely(str[0] == '?') && unlikely(str[1] == '\0')) { - *d = DBL_NAN; - return SUCCESS_PASS; - } - switch (type_) { - case CONTINUOUS: { -// *d = strtod(str, &end); - std::istringstream is(str); - if( !(is >> *d) ) - return SUCCESS_FAIL; - return SUCCESS_PASS; - } - case INTEGER: { - int i; - std::istringstream is(str); - if( !(is >> i) ) - return SUCCESS_FAIL; - *d = i; - return SUCCESS_PASS; - } - case NOMINAL: { - index_t i; - for (i = 0; i < value_names_.size(); i++) { - if (value_names_[i] == str) { - *d = i; - return SUCCESS_PASS; - } - } - *d = DBL_NAN; - return SUCCESS_FAIL; - } - default: abort(); - } -} - -// DatasetInfo ------------------------------------------------------ - - -void DatasetInfo::InitContinuous(index_t n_features, - const char *name_in) { - features_.reserve(n_features); - - name_ = name_in; - - for (index_t i = 0; i < n_features; i++) { - std::ostringstream o; - if(!(o << i)) - abort(); - DatasetFeature f; - f.InitContinuous(o.str()); - features_.push_back(f); - } -} - -void DatasetInfo::Init(const char *name_in) { - name_ = name_in; -} - -index_t DatasetInfo::SkipSpace_(std::string& s) { - int i = 0; - while (isspace(s[i])) { - ++i; - } - - if (unlikely(s[i] == '%') || unlikely(s[i] == '\0')) { - return s.length(); - } - - return i; -} - -char *DatasetInfo::SkipNonspace_(char *s) { - while (likely(*s != '\0') - && likely(*s != '%') - && likely(*s != ' ') - && likely(*s != '\t')) { - s++; - } - - return s; -} - -void DatasetInfo::SkipBlanks_(TextLineReader *reader) { - while (reader->MoreLines() && reader->Peek()[SkipSpace_(reader->Peek())] == '\0') { - reader->Gobble(); - } -} - -success_t DatasetInfo::InitFromArff(TextLineReader *reader, - const char *filename) { - success_t result = SUCCESS_PASS; - - Init(filename); - - while (1) { - SkipBlanks_(reader); - - std::string *peeked = &reader->Peek(); - std::vector portions; - - tokenizeString(*peeked, ", \t", portions, 0, "%{", 3, true ); - - if (portions.size() == 0) { - /* empty line */ - } else if (portions[0][0] != '@') { - reader->Error("ARFF: Unexpected @command. Did you forget @data?"); - result = SUCCESS_FAIL; - break; - } else { -// if (portions[0].EqualsNoCase("@relation")) { - if(strcasecmp(portions[0].c_str(), "@relation") == 0) { - if (portions.size() < 2) { - reader->Error("ARFF: @relation requires name"); - result = SUCCESS_FAIL; - } else { - set_name(portions[1]); - } -// } else if (portions[0].EqualsNoCase("@attribute")) { - } else if(strcasecmp(portions[0].c_str(), "@attribute") == 0) { - if (portions.size() < 3) { - reader->Error("ARFF: @attribute requires name and type."); - result = SUCCESS_FAIL; - } else { - DatasetFeature feature; - if (portions[2][0] == '{') { //} - feature.InitNominal(portions[1]); - // TODO: Doesn't support values with spaces { - tokenizeString(portions[2], ", \t", feature.value_names(), 1, "}%", 0); - features_.push_back(feature); - } else { - std::string type(portions[2]); - //portions[2].Trim(" \t", &type); -// if (type.EqualsNoCase("numeric") -// || type.EqualsNoCase("real")) { - if(strcasecmp(type.c_str(), "numeric") == 0 - || strcasecmp(type.c_str(), "real") == 0) { - feature.InitContinuous(portions[1]); - features_.push_back(feature); -// } else if (type.EqualsNoCase("integer")) { - } else if(strcasecmp( type.c_str(), "integer") == 0) { - feature.InitContinuous(portions[1]); - features_.push_back(feature); - } else { - reader->Error( - "ARFF: Only support 'numeric', 'real', and {nominal}."); - result = SUCCESS_FAIL; - } - } - } -// } else if (portions[0].EqualsNoCase("@data")) { - } else if(strcasecmp(portions[0].c_str(), "@data") == 0) { - /* Done! */ - reader->Gobble(); - break; - } else { - reader->Error("ARFF: Expected @relation, @attribute, or @data."); - result = SUCCESS_FAIL; - break; - } - } - - reader->Gobble(); - } - - return result; -} - -success_t DatasetInfo::InitFromCsv(TextLineReader *reader, - const char *filename) { - std::vector headers; - bool nonnumeric = false; - - Init(filename); - - tokenizeString(reader->Peek(), ", \t", headers); - - if (headers.size() == 0) { - reader->Error("Trying to parse empty file as CSV."); - return SUCCESS_FAIL; - } - - // Try to auto-detect if there is a header row - for (index_t i = 0; i < headers.size(); i++) { - char *end; - - (void) strtod(headers[i].c_str(), &end); - - if (end == headers[i].c_str()) { - nonnumeric = true; - break; - } - } - - if (nonnumeric) { - for (index_t i = 0; i < headers.size(); i++) { - DatasetFeature feature; - feature.InitContinuous(headers[i]); - features_.push_back(feature); - } - reader->Gobble(); - } else { - for (index_t i = 0; i < headers.size(); i++) { - DatasetFeature feature; - std::ostringstream o; - if(!(o << i)) - abort(); - feature.InitContinuous(o.str()); - features_.push_back(feature); - } - } - - return SUCCESS_PASS; -} - -success_t DatasetInfo::InitFromFile(TextLineReader *reader, - const char *filename) { - SkipBlanks_(reader); - - // WARNING: Safe? - char first_char = reader->Peek()[SkipSpace_(reader->Peek())]; - - if (first_char == '\0') { - Init(); - reader->Error("Could not parse the first line."); - return SUCCESS_FAIL; - } else if (first_char == '@') { - /* Okay, it's ARFF. */ - return InitFromArff(reader, filename); - } else { - /* It's CSV. We'll try to see if there are headers. */ - return InitFromCsv(reader, filename); - } -} - index_t Dataset::n_labels() const { index_t i = 0; - index_t label_row_idx = matrix_.n_rows() - 1; // the last row is for labels + index_t label_row_idx = matrix_.n_rows - 1; // the last row is for labels index_t n_labels = 0; double current_label; std::vector labels_list; - labels_list.push_back(matrix_.get(label_row_idx,0)); + labels_list.push_back(matrix_[label_row_idx,0]); n_labels++; - for (i = 1; i < matrix_.n_cols(); i++) { - current_label = matrix_.get(label_row_idx,i); + for(i = 1; i < matrix_.n_cols; i++) { + current_label = matrix_(label_row_idx, i); index_t j = 0; for (j = 0; j < n_labels; j++) { if (current_label == labels_list[j]) { break; } } - if (j == n_labels) { // new label + if(j == n_labels) { // new label labels_list.push_back(current_label); n_labels++; } @@ -344,17 +80,16 @@ void Dataset::GetLabels(std::vector &labels_list, std::vector &labels_ct, std::vector &labels_startpos) const { index_t i = 0; - index_t label_row_idx = matrix_.n_rows() - 1; // the last row is for labels - index_t n_points = matrix_.n_cols(); + index_t label_row_idx = matrix_.n_rows - 1; // the last row is for labels + index_t n_points = matrix_.n_cols; index_t n_labels = 0; double current_label; // these Arraylists need initialization before-hand - /* This faithfully replicates the effect of ArrayList.Renew(). - Is this necessary? If all we care about is initialization, - it shouldn't be. - */ + // This faithfully replicates the effect of ArrayList.Renew(). + // Is this necessary? If all we care about is initialization, + // it shouldn't be. { std::vector y; std::vector x[3]; @@ -368,19 +103,19 @@ void Dataset::GetLabels(std::vector &labels_list, std::vector labels_temp; labels_temp.reserve(n_points); - labels_temp[0] = 0; + labels_temp.push_back(0); - labels_list.push_back(matrix_.get(label_row_idx,0)); + labels_list.push_back(matrix_(label_row_idx,0)); labels_ct.push_back(1); n_labels++; for (i = 1; i < n_points; i++) { - current_label = matrix_.get(label_row_idx, i); + current_label = matrix_(label_row_idx, i); index_t j = 0; for (j = 0; j < n_labels; j++) { if (current_label == labels_list[j]) { labels_ct[j]++; - break; + break; } } labels_temp[i] = j; @@ -392,9 +127,8 @@ void Dataset::GetLabels(std::vector &labels_list, } labels_startpos.push_back(0); - for(i = 1; i < n_labels; i++){ - labels_startpos.push_back( labels_startpos[i-1] + labels_ct[i-1] ); - } + for(i = 1; i < n_labels; i++) + labels_startpos.push_back(labels_startpos[i - 1] + labels_ct[i - 1]); for(i = 0; i < n_points; i++) { labels_index[labels_startpos[labels_temp[i]]] = i; @@ -403,233 +137,34 @@ void Dataset::GetLabels(std::vector &labels_list, labels_startpos[0] = 0; for(i = 1; i < n_labels; i++) - labels_startpos[i] = labels_startpos[i-1] + labels_ct[i-1]; + labels_startpos[i] = labels_startpos[i - 1] + labels_ct[i - 1]; labels_temp.clear(); } -bool DatasetInfo::is_all_continuous() const { - for (index_t i = 0; i < features_.size(); i++) { - if (features_[i].type() != DatasetFeature::CONTINUOUS) { - return false; - } - } - - return true; -} - -success_t DatasetInfo::ReadMatrix(TextLineReader *reader, Matrix *matrix) const { - std::vector linearized; - index_t n_features = this->n_features(); - index_t n_points = 0; - success_t retval = SUCCESS_PASS; - bool is_done; - - // read through our file to find out how long it is - index_t cur_line = reader->line_num(); - while(reader->Gobble()) { } - matrix->Init(n_features, reader->line_num() - cur_line + 1); - - char *fname = strncpy(new char[strlen(reader->filename()) + 1], - reader->filename(), - strlen(reader->filename()) + 1); - - reader->Close(); - reader->Open(fname); - delete[] fname; - - // make sure we are in the same place in our file, just in case it was passed - // to us and we had already been some of the way through it (this is why we - // saved the line number - while(reader->line_num() < cur_line) { - reader->Gobble(); - } - - while((n_points < matrix->n_cols()) && !is_done && !FAILED(retval)) { - retval = ReadPoint(reader, matrix->GetColumnPtr(n_points), &is_done); - n_points++; - } - - if (!FAILED(retval)) { - n_points--; // last increment was the failure, so subtract that - DEBUG_ASSERT(n_points == matrix->n_rows()); - } - - return retval; -} - -success_t DatasetInfo::ReadPoint(TextLineReader *reader, double *point, - bool *is_done) const { - index_t n_features = this->n_features(); - std::string str; - std::string::iterator pos; - - *is_done = false; - - for (;;) { - if (!reader->MoreLines()) { - *is_done = true; - return SUCCESS_PASS; - } - - str = reader->Peek(); - pos = str.begin(); - - while (*pos == ' ' || *pos == '\t' || *pos == ',') { - pos++; - } - - if (unlikely(*pos == '\0' || *pos == '%')) { - reader->Gobble(); - } else { - break; - } - } - - for (index_t i = 0; i < n_features; i++) { - std::string::iterator next; - - while (*pos == ' ' || *pos == '\t' || *pos == ',') { - pos++; - } - - if (unlikely(*pos == '\0')) { - for (std::string::iterator s = reader->Peek().begin(); s < pos; s++) { // UNDEFINED - if (!*s) { - *s = ','; - } - } - reader->Error("I am expecting %"LI"d entries per row, " - "but this line has only %"LI"d.", - n_features, i); - return SUCCESS_FAIL; - } - - next = pos; - while (*next != '\0' && *next != ' ' && *next != '\t' && *next != ',' - && *next != '%') { - next++; - } - - if (*next != '\0') { - char c = *next; - *next = '\0'; - if (c != '%') { - next++; - } - } - - size_t len = str.end() - pos; - size_t cpos = pos - str.begin(); - if (!PASSED(features_[i].Parse(str.substr(cpos,len), &point[i]))) { - std::string::iterator end = reader->Peek().end(); - std::string tmp; - tmp.assign(pos,str.end()); - for (std::string::iterator s = reader->Peek().begin(); - s < next && s < end; s++) { - if (*s == '\0') { - *s = ','; - } - } - reader->Error("Invalid parse: [%s]", tmp.c_str()); - return SUCCESS_FAIL; - } - - pos = next; - } - - while (*pos == ' ' || *pos == '\t' || *pos == ',') { - pos++; - } - - if (*pos != '\0') { - for (std::string::iterator s = reader->Peek().begin(); s < pos; s++) { - if (*s == '\0') { - *s = ','; - } - } - reader->Error("Extra junk on line."); - return SUCCESS_FAIL; - } - - reader->Gobble(); - - return SUCCESS_PASS; -} - - -void DatasetInfo::WriteArffHeader(TextWriter *writer) const { - writer->Printf("@relation %s\n", name_.c_str()); - - for (index_t i = 0; i < features_.size(); i++) { - const DatasetFeature *feature = &features_[i]; - writer->Printf("@attribute %s ", feature->name().c_str()); - if (feature->type() == DatasetFeature::NOMINAL) { - writer->Printf("{"); - for (index_t v = 0; v < feature->n_values(); v++) { - if (v != 0) { - writer->Write(","); - } - writer->Write(feature->value_name(v).c_str()); - } - writer->Printf("}"); - } else { - writer->Write("real"); - } - writer->Write("\n"); - } - writer->Printf("@data\n"); -} - -void DatasetInfo::WriteCsvHeader(const char *sep, TextWriter *writer) const { - for (index_t i = 0; i < features_.size(); i++) { - if (i != 0) { - writer->Write(sep); - } - writer->Write(features_[i].name().c_str()); - } - writer->Write("\n"); -} - -void DatasetInfo::WriteMatrix(const Matrix& matrix, const char *sep, - TextWriter *writer) const { - for (index_t i = 0; i < matrix.n_cols(); i++) { - for (index_t f = 0; f < features_.size(); f++) { - if (f != 0) { - writer->Write(sep); - } - std::string str; - features_[f].Format(matrix.get(f, i), str); - writer->Write(str.c_str()); - } - writer->Write("\n"); - } -} - -// Dataset ------------------------------------------------------------------ success_t Dataset::InitFromFile(const char *fname) { TextLineReader reader; if (PASSED(reader.Open(fname))) { - return InitFromFile(&reader, fname); + return InitFromFile(reader, fname); } else { - matrix_.Init(0, 0); + matrix_ = 0.0; // 0x0 matrix info_.Init(); NONFATAL("Could not open file '%s' for reading.", fname); return SUCCESS_FAIL; } } -success_t Dataset::InitFromFile(TextLineReader *reader, +success_t Dataset::InitFromFile(TextLineReader& reader, const char *filename) { success_t result; result = info_.InitFromFile(reader, filename); if (PASSED(result)) { - result = info_.ReadMatrix(reader, &matrix_); + result = info_.ReadMatrix(reader, matrix_); } else { - matrix_.Init(0, 0); + matrix_ = 0.0; // 0x0 matrix } return result; @@ -644,9 +179,9 @@ success_t Dataset::WriteCsv(const char *fname, bool header) const { return SUCCESS_FAIL; } else { if (header) { - info_.WriteCsvHeader(",\t", &writer); + info_.WriteCsvHeader(",\t", writer); } - info_.WriteMatrix(matrix_, ",\t", &writer); + info_.WriteMatrix(matrix_, ",\t", writer); return writer.Close(); } } @@ -658,26 +193,30 @@ success_t Dataset::WriteArff(const char *fname) const { NONFATAL("Couldn't open '%s' for writing.", fname); return SUCCESS_FAIL; } else { - info_.WriteArffHeader(&writer); - info_.WriteMatrix(matrix_, ",", &writer); + info_.WriteArffHeader(writer); + info_.WriteMatrix(matrix_, ",", writer); return writer.Close(); } } void Dataset::SplitTrainTest(int folds, int fold_number, const std::vector& permutation, - Dataset *train, Dataset *test) const { + Dataset& train, Dataset& test) const { + // determine number of points in test and training sets index_t n_test = (n_points() + folds - fold_number - 1) / folds; index_t n_train = n_points() - n_test; - train->InitBlank(); - train->info().InitCopy(info()); + // initialize blank training data set + train.InitBlank(); + train.info().InitCopy(info()); - test->InitBlank(); - test->info().InitCopy(info()); + // initialize blank testing data set + test.InitBlank(); + test.info().InitCopy(info()); - train->matrix().Init(n_features(), n_train); - test->matrix().Init(n_features(), n_test); + // set sizes of training and test datasets + train.matrix().set_size(n_features(), n_train); + test.matrix().set_size(n_features(), n_test); index_t i_train = 0; index_t i_test = 0; @@ -687,32 +226,59 @@ void Dataset::SplitTrainTest(int folds, int fold_number, double *dest; if (unlikely((i_orig - fold_number) % folds == 0)) { - dest = test->matrix().GetColumnPtr(i_test); + // put this column into the test set + dest = test.matrix().colptr(i_test); i_test++; } else { - dest = train->matrix().GetColumnPtr(i_train); + // put this column into the training set + dest = train.matrix().colptr(i_train); i_train++; } - mem::Copy(dest, - this->matrix().GetColumnPtr(permutation[i_orig]), - n_features()); + // copy the column over in memory + memcpy(dest, + this->matrix().colptr(permutation[i_orig]), + sizeof(double) * n_features()); } - DEBUG_ASSERT(i_train == train->n_points()); - DEBUG_ASSERT(i_test == test->n_points()); + DEBUG_ASSERT(i_train == train.n_points()); + DEBUG_ASSERT(i_test == test.n_points()); } -success_t data::Load(const char *fname, Matrix *matrix) { - Dataset dataset; - success_t result = dataset.InitFromFile(fname); - matrix->Own(&dataset.matrix()); +success_t data::Load(const char *fname, arma::mat& matrix) { + TextLineReader reader; + DatasetInfo info; // we will ignore this, but it reads our matrix + success_t result; + + // clear our matrix + matrix.reset(); + + if (PASSED(reader.Open(fname))) { + // read our file, since it has successfully opened + result = info.InitFromFile(reader, fname); + if (PASSED(result)) { + result = info.ReadMatrix(reader, matrix); + } + } else { + NONFATAL("Could not open file '%s' for reading.", fname); + return SUCCESS_FAIL; + } + return result; } -success_t data::Save(const char *fname, const Matrix& matrix) { - Dataset dataset; - dataset.AliasMatrix(matrix); - return dataset.WriteCsv(fname); -} +success_t data::Save(const char *fname, const arma::mat& matrix) { + TextWriter writer; + // temporary info object that will help write our CSV + DatasetInfo info; + info.InitContinuous(matrix.n_rows); + + if (!PASSED(writer.Open(fname))) { + NONFATAL("Couldn't open '%s' for writing.", fname); + return SUCCESS_FAIL; + } + + info.WriteMatrix(matrix, ",\t", writer); + return writer.Close(); +} diff --git a/fastlib/branches/fastlib-stl/fastlib/data/dataset.h b/fastlib/branches/fastlib-stl/fastlib/data/dataset.h index c785dd6ed8..97cfcb235b 100644 --- a/fastlib/branches/fastlib-stl/fastlib/data/dataset.h +++ b/fastlib/branches/fastlib-stl/fastlib/data/dataset.h @@ -43,6 +43,7 @@ #ifndef DATA_DATASET_H #define DATA_DATASET_H +#include #include #include @@ -51,316 +52,12 @@ #include "../file/textfile.h" #include "../col/tokenizer.h" +#include "dataset_feature.h" +#include "dataset_info.h" + class TextLineReader; class TextWriter; -/** - * Metadata about a particular dataset feature (attribute). - * - * Supports nominal, continuous, and integer values. - */ -class DatasetFeature { - public: - /** - * Feature types supported. - */ - enum Type { - /** Real-valued data. */ - CONTINUOUS, - /** Integer valued data. */ - INTEGER, - /** Discrete data, each of which has a "name". */ - NOMINAL - }; - - private: - /** Name of the feature. */ - std::string name_; - /** Type of data this feature represents. */ - Type type_; - /** If nominal, the names of each numbered value. */ - std::vector value_names_; - - /** - * Initialization common to all features. - * - * @param name_in the name of the feature - */ - void InitGeneral(const std::string& name_in) { - name_ = name_in; - } - - public: - /** - * Initialize to be a continuous feature. - * - * @param name_in the name of the feature - */ - void InitContinuous(const std::string& name_in) { - InitGeneral(name_in); - type_ = CONTINUOUS; - } - - /** - * Initializes to an integer type. - * - * @param name_in the name of the feature - */ - void InitInteger(const std::string& name_in) { - InitGeneral(name_in); - type_ = INTEGER; - } - - /** - * Initializes to a nominal type. - * - * The value_names list starts empty, so you need to add the name of - * each feature to this. (The dataset reading functions will do this - * for you). - * - * @param name_in the name of the feature - */ - void InitNominal(const std::string& name_in) { - InitGeneral(name_in); - type_ = NOMINAL; - } - - /** - * Creates a text version of the value based on the type. - * - * Continuous parameters are printed in floating point, and integers - * are shown as integers. For nominal, the value_name(int(value)) is - * shown. NaN (missing data) is always shown as '?'. - * - * @param value the value to format - * @param result this will be initialized to the formatted text - */ - void Format(double value, std::string& result) const; - - /** - * Parses a string into the particular value. - * - * Integers and continuous are parsed using the normal functions. - * For nominal, the entry - * - * If an invalid parse occurs, such as a mal-formatted number or - * a nominal value not in the list, SUCCESS_FAIL will be returned. - * - * @param str the string to parse - * @param d where to store the result - */ - success_t Parse(const std::string& str, double *d) const; - - /** - * Gets what the feature is named. - * - * @return the name of the feature; for point, "Age" or "X Position" - */ - const std::string& name() const { - return name_; - } - - /** - * Identifies the type of feature. - * - * @return whether this is DatasetFeature::CONTINUOUS, INTEGER, or NOMINAL - */ - Type type() const { - return type_; - } - - /** - * Returns the name of a particular nominal value, given its index. - * - * The first nominal value is 0, the second is 1, etc. - * - * @param value the number of the value - */ - const std::string& value_name(int value) const { - DEBUG_ASSERT(type_ == NOMINAL); - return value_names_[value]; - } - - /** - * The number of nominal values. - * - * The values 0 to n_values() - 1 are valid. - * This will return zero for CONTINUOUS and INTEGER types. - * - * @return the number of nominal values - */ - index_t n_values() const { - return value_names_.size(); - } - - /** - * Gets the array of value names. - * - * Useful for creating a nominal feature yourself. - * - * @return a mutable array of value names - */ - std::vector& value_names() { - return value_names_; - } -}; - -/** - * Information describing a dataset and its features. - */ -class DatasetInfo { - private: - std::string name_; - std::vector features_; - - public: - /** Gets a mutable list of all features. */ - std::vector& features() { - return features_; - } - - /** Gets information about a particular feature. */ - const DatasetFeature& feature(index_t attrib_num) const { - return features_[attrib_num]; - } - - /** Gets the number of features. */ - index_t n_features() const { - return features_.size(); - } - - /** Gets the title of the data set. */ - const char *name() const { - return name_.c_str(); - } - - /** Sets the title of the data set. */ - void set_name(const std::string& name_in) { - name_ = name_in; - } - - /** - * Checks if all parameters are continuous. - */ - bool is_all_continuous() const; - - /** - * Initialize an all-continuous dataset; - * - * @param n_features the number of continuous features - * @param name_in the dataset title - */ - void InitContinuous(index_t n_features, - const char *name_in = "dataset"); - - /** - * Initialize a custom dataset. - * - * This assumes you will eventually use the features() to add features. - * - * @param name_in the dataset title - */ - void Init(const char *name_in = "dataset"); - - /*** - * Copy constructor, written manually to avoid use of OBJECT_TRAVERSAL functions. - * TODO: Make this Init() crap go away. - */ - void InitCopy(const DatasetInfo &info) { - name_ = info.name(); - features_ = info.features_; - } - - /** - * Writes the header for an ARFF file. - */ - void WriteArffHeader(TextWriter *writer) const; - - /** - * Writes header for CSV file. - * - * @param sep the value separator (use ",\t" for CSV) - * @param writer the text writer to write the header line to - */ - void WriteCsvHeader(const char *sep, TextWriter *writer) const; - - /** - * Writes the contents of a matrix to a file. - * - * @param matrix the matrix - * @param sep the separator (use ",\t" for CSV) - * @param writer the writer to write to - */ - void WriteMatrix(const Matrix& matrix, const char *sep, - TextWriter *writer) const; - - /** - * Initialize explicitly from an ARFF file. - * - * ARFF LIMITATIONS: Values cannot have spaces or commas, even with quotes; - * 'string' data type not supported (nominal is supported). - * - * You might just use InitFromFile, which will guess the type for you. - * - * This will read only the header information and leave the reader at the - * first line of data. - */ - success_t InitFromArff(TextLineReader *reader, - const char *filename = "dataset"); - - /** - * Initialize from a CSV-like file with numbers only, inferring - * automatically that if the first row has non-numeric characters, it is - * a header. - * - * InitFromFile will automatically detect this. - */ - success_t InitFromCsv(TextLineReader *reader, - const char *filename = "dataset"); - - /** - * Initializes the header from a file, either CSV or ARFF. - * - * All header lines will be gobbled, so the reader's position will be - * left at the first line of actual data. - * You can then read the data with matrix. - */ - success_t InitFromFile(TextLineReader *reader, - const char *filename = "dataset"); - /** - * Populates a matrix from a file, given the internal data model. - * - * ARFF LIMITATIONS: Values cannot have spaces or commas, even with quotes; - * 'string' data type not supported (nominal is supported). - * - * @param reader the reader to get lines from - * @param matrix the matrix to store text into - */ - success_t ReadMatrix(TextLineReader *reader, Matrix *matrix) const; - - /** - * Reads a single vector. - * - * @param reader the line reader being used - * @param point an array of length n_features() - * @param is_done set to true if we have finished reading the file - * successfully -- the value of is_done is undefined if the - * function returns failure! - * @return whether reading the line was successful - */ - success_t ReadPoint(TextLineReader *reader, double *point, - bool *is_done) const; - - private: - index_t SkipSpace_(std::string& s); - - char *SkipNonspace_(char *s); - - void SkipBlanks_(TextLineReader *reader); - -}; - - /** * Most generic dataset type. * @@ -374,7 +71,7 @@ class DatasetInfo { */ class Dataset { private: - Matrix matrix_; + arma::mat matrix_; DatasetInfo info_; public: @@ -407,9 +104,9 @@ class Dataset { * @return the number of features, or variables, in the dataset */ index_t n_features() const { - return matrix_.n_rows(); + return matrix_.n_rows; } - + /** * Gets the number of points/instances in the dataset. * @@ -418,7 +115,7 @@ class Dataset { * @return the number of points in the dataset */ index_t n_points() const { - return matrix_.n_cols(); + return matrix_.n_cols; } /** @@ -449,13 +146,13 @@ class Dataset { * @param labels_ct numbers of point in each label class. e.g. * [7,5,8] * @param labels_startpos start positions of each label class in - * labels_index. e.g. [0,7,12] + * labels_index. e.g. [0,7,12] */ void GetLabels(std::vector &labels_list, - std::vector &labels_index, - std::vector &labels_ct, - std::vector &labels_startpos) const; - + std::vector &labels_index, + std::vector &labels_ct, + std::vector &labels_startpos) const; + /** * Gets the numeric value of a particular feature and point. * @@ -463,9 +160,9 @@ class Dataset { * @param point the point index */ double get(index_t feature, index_t point) const { - return matrix_.get(feature, point); + return matrix_(feature, point); } - + /** * Gets the integer value of a particular feature and point. */ @@ -475,7 +172,7 @@ class Dataset { DEBUG_ASSERT(d == double(i)); return i; } - + /** * Modifies a value in the dataset. * @@ -484,40 +181,40 @@ class Dataset { * @param d the numeric value to set it to */ void set(index_t feature, index_t point, double d) { - matrix_.set(feature, point, d); + matrix_(feature, point) = d; } - + /** * Gets the "raw" form of a particular point. * * @return a C-like array of the values of a particular point */ - const double *point(index_t point) const { - return matrix_.GetColumnPtr(point); + const arma::rowvec point(index_t point) const { + return matrix_.row(point); } /** * Gets the "raw" form of a particular point. * * @return a C-like array of the values of a particular point */ - double *point(index_t point) { - return matrix_.GetColumnPtr(point); + arma::rowvec point(index_t point) { + return matrix_.row(point); } - + /** * Returns the matrix that stores all the data. */ - const Matrix& matrix() const { + const arma::mat& matrix() const { return matrix_; } /** * Returns the matrix that stores all the data * (can be used for modification). */ - Matrix& matrix() { + arma::mat& matrix() { return matrix_; } - + /** * Formats as text a particular location of the data set. * @@ -526,9 +223,9 @@ class Dataset { * @param result string that will be initialized to the formatted text */ void Format(index_t feature, index_t point, std::string& result) const { - info_.feature(feature).Format(get(feature, point), result); + info_.feature(feature).Format(matrix_(feature, point), result); } - + /** * Initializer that omits the matrix and info - you must initialize * these yourself. @@ -536,9 +233,8 @@ class Dataset { * (Although this currently does nothing, this is to future-proof your code * against possible changes.) */ - void InitBlank() { - } - + void InitBlank() { } + /** * Reads in an ARFF or CSV/WSV file. * @@ -548,7 +244,7 @@ class Dataset { * @param fname the name of an ARFF, CSV, or whitespace-separated */ success_t InitFromFile(const char *fname); - + /** * Reads in an ARFF or CSV/WSV file. * @@ -559,9 +255,9 @@ class Dataset { * @param filename a title given to this data set, doesn't necessarily * need to be anything significant */ - success_t InitFromFile(TextLineReader *reader, + success_t InitFromFile(TextLineReader& reader, const char *filename = "dataset"); - + /** * Writes to a CSV file. * @@ -584,12 +280,12 @@ class Dataset { * * @param matrix_in data where rows are features, columns are points */ - void CopyMatrix(const Matrix& matrix_in) { + void CopyMatrix(const arma::mat& matrix_in) { InitBlank(); - matrix_.Copy(matrix_in); - info_.InitContinuous(matrix_.n_rows()); + matrix_ = matrix_in; + info_.InitContinuous(matrix_.n_rows); } - + /** * Initializes by becoming the owner of an existing matrix, assuming * all features are continuous. @@ -600,12 +296,12 @@ class Dataset { * * @param matrix_in data where rows are features, columns are points */ - void OwnMatrix(Matrix* matrix_in) { - InitBlank(); - matrix_.Own(matrix_in); - info_.InitContinuous(matrix_.n_rows()); - } - +// void OwnMatrix(Matrix* matrix_in) { +// InitBlank(); +// matrix_.Own(matrix_in); +// info_.InitContinuous(matrix_.n_rows()); +// } + /** * Initializes as an alias or mirror of an existing matrix, assuming * all features are continuous. @@ -616,15 +312,17 @@ class Dataset { * * @param matrix_in data where rows are features, columns are points */ - void AliasMatrix(const Matrix& matrix_in) { - InitBlank(); - matrix_.Alias(matrix_in); - info_.InitContinuous(matrix_.n_rows()); - } - +// void AliasMatrix(const arma::mat& matrix_in) { +// InitBlank(); + // use the memory pointer of the other matrix directly; do not free it when + // we are done +// matrix_(matrix_in.memptr(), matrix_in.n_rows, matrix_in.n_cols, false); +// info_.InitContinuous(matrix_.n_rows); +// } + //--- Cross-validation features --- - /** + /* * Creates a training and test dataset for k-fold cross validation. * * The test set will be approximately n_points() / folds, and the @@ -635,13 +333,13 @@ class Dataset { * @param folds the number of folds being used * @param fold_number the fold number, 0 to folds - 1 * @param permutation the permutation to use, the same size as n_points() - * (use math::MakeIdentiyPermutation or math::MakeRandomPermutation) + * (use math::MakeIdentityPermutation or math::MakeRandomPermutation) * @param train the training set * @param test the test set */ void SplitTrainTest(int folds, int fold_number, const std::vector& permutation, - Dataset *train, Dataset *test) const; + Dataset& train, Dataset& test) const; }; /** @@ -662,12 +360,13 @@ namespace data { * @param fname the file name to load * @param matrix a pointer to an uninitialized matrix to load */ - success_t Load(const char *fname, Matrix *matrix); - /** + success_t Load(const char *fname, arma::mat& matrix); + + /** * Loads a matrix from a file. - * The Matrix is Statically Initialized from a memory mapped file + * The matrix is statically initialized from a memory mapped file. * - * This supports only CSV Datasets + * This supports only CSV datasets. * * @code * Matrix A; @@ -678,43 +377,50 @@ namespace data { * @param matrix a pointer to an uninitialized matrix to load */ template - success_t LargeLoad(const char *fname, GenMatrix *matrix) { + success_t LargeLoad(const char *fname, arma::Mat& matrix) { + // open our file TextLineReader *reader = new TextLineReader(); - if (reader->Open(fname)==SUCCESS_FAIL) { + if (reader->Open(fname) == SUCCESS_FAIL) { reader->Error("Couldn't open %s", fname); return SUCCESS_FAIL; - } - index_t dimension=0; - std::string line=reader->Peek(); - std::vector result; -// line.Split(",", &result); - tokenizeString( line, ",", result ); - dimension=result.size(); - while (reader->Gobble()) { } - matrix->StaticInit(dimension, reader->line_num()); - matrix->SetAll(0.0); + + // find dimensionality + index_t dimension = 0; + std::string line = reader->Peek(); + std::vector result; + tokenizeString(line, ",", result); + + dimension = result.size(); + + // count lines in file + while(reader->Gobble()) { } + + // resize matrix to correct size and set all to 0 + matrix.zeros(dimension, reader->line_num()); + delete reader; + + // second pass through file: fill matrix reader = new TextLineReader(); reader->Open(fname); - while (true) { - std::string line=reader->Peek(); + + do { + // parse this line + std::string line = reader->Peek(); std::vector result; -// line.Split(",", &result); - tokenizeString( line, ",", result ); - for(index_t i=0; iset(i, reader->line_num()-1, (Precision)num); + matrix(i, reader->line_num() - 1) = (Precision) num; } - if (reader->Gobble()==false) { - break; - } - } + } while(reader->Gobble()); // break when we can't read more + return SUCCESS_PASS; } - /** * Saves a matrix to a file. * @@ -729,7 +435,7 @@ namespace data { * @param fname the file name to load * @param matrix a pointer to an uninitialized matrix to load */ - success_t Save(const char *fname, const Matrix& matrix); + success_t Save(const char *fname, const arma::mat& matrix); }; #endif diff --git a/fastlib/branches/fastlib-stl/fastlib/data/dataset_feature.cc b/fastlib/branches/fastlib-stl/fastlib/data/dataset_feature.cc new file mode 100644 index 0000000000..5a30fbc747 --- /dev/null +++ b/fastlib/branches/fastlib-stl/fastlib/data/dataset_feature.cc @@ -0,0 +1,115 @@ +/* MLPACK 0.2 + * + * Copyright (c) 2008, 2009 Alexander Gray, + * Garry Boyer, + * Ryan Riegel, + * Nikolaos Vasiloglou, + * Dongryeol Lee, + * Chip Mappus, + * Nishant Mehta, + * Hua Ouyang, + * Parikshit Ram, + * Long Tran, + * Wee Chin Wong + * + * Copyright (c) 2008, 2009 Georgia Institute of Technology + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ +/** + * @file dataset_feature.cc + * + * Implementations for the DatasetFeature class. + * + * @bug These routines fail when trying to read files linewise that use the Mac + * eol '\r'. Both Windows and Unix eol ("\r\n" and '\n') work. Use the + * programs 'dos2unix' or 'tr' to convert the '\r's to '\n's. + * + */ + +#include "../base/base.h" + +#include "dataset_feature.h" + +#include +#include + +using namespace std; + +void DatasetFeature::Format(double value, string& result) const { + if (unlikely(isnan(value))) { + result = "?"; + return; + } + ostringstream o; + switch (type_) { + case CONTINUOUS: + if (floor(value) != value) { + // non-integer + o.setf(ios::scientific); + } else { + // value is actually an integer + o.precision(17); + } + break; + case INTEGER: + case NOMINAL: + break; + #ifdef DEBUG + default: abort(); + #endif + } + if(!(o << value)) + abort(); + result = o.str(); +} + +success_t DatasetFeature::Parse(const std::string& str, double& d) const { + if (unlikely(str[0] == '?') && unlikely(str[1] == '\0')) { + d = DBL_NAN; + return SUCCESS_PASS; + } + switch (type_) { + case CONTINUOUS: + { + istringstream is(str); + if(!(is >> d)) + return SUCCESS_FAIL; + return SUCCESS_PASS; + } + case INTEGER: + { + int i; + istringstream is(str); + if(!(is >> i)) + return SUCCESS_FAIL; + d = i; + return SUCCESS_PASS; + } + case NOMINAL: { + index_t i; + for (i = 0; i < value_names_.size(); i++) { + if (value_names_[i] == str) { + d = i; + return SUCCESS_PASS; + } + } + d = DBL_NAN; + return SUCCESS_FAIL; + } + default: abort(); + } +} diff --git a/fastlib/branches/fastlib-stl/fastlib/data/dataset_feature.h b/fastlib/branches/fastlib-stl/fastlib/data/dataset_feature.h new file mode 100644 index 0000000000..a734c7d3ae --- /dev/null +++ b/fastlib/branches/fastlib-stl/fastlib/data/dataset_feature.h @@ -0,0 +1,198 @@ +/* MLPACK 0.2 + * + * Copyright (c) 2008, 2009 Alexander Gray, + * Garry Boyer, + * Ryan Riegel, + * Nikolaos Vasiloglou, + * Dongryeol Lee, + * Chip Mappus, + * Nishant Mehta, + * Hua Ouyang, + * Parikshit Ram, + * Long Tran, + * Wee Chin Wong + * + * Copyright (c) 2008, 2009 Georgia Institute of Technology + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ +/** + * @file dataset_feature.h + * + * The DatasetFeature class, used by the Dataset class. + * + * @bug These routines fail when trying to read files linewise that use the Mac + * eol '\r'. Both Windows and Unix eol ("\r\n" and '\n') work. Use the + * programs 'dos2unix' or 'tr' to convert the '\r's to '\n's. + * + */ + +#ifndef DATA_DATASET_FEATURE_H +#define DATA_DATASET_FEATURE_H + +#include +#include + +#include "../la/matrix.h" +#include "../math/discrete.h" +#include "../file/textfile.h" + +/** + * Metadata about a particular dataset feature (attribute). + * + * Supports nominal, continuous, and integer values. + */ +class DatasetFeature { + public: + /** + * Supported feature types. + */ + enum Type { + CONTINUOUS, /** Real-valued data. */ + INTEGER, /** Integer valued data. */ + NOMINAL /** Discrete data, each of which has a "name". */ + }; + + private: + std::string name_; /** Name of the feature. */ + Type type_; /** Type of data this feature represents. */ + std::vector value_names_; /** If nominal, the names of each numbered value. */ + + /** + * Initialization common to all features. + * + * @param name_in the name of the feature + */ + void InitGeneral(const std::string& name_in) { + name_ = name_in; + } + + public: + /** + * Initialize to be a continuous feature. + * + * @param name_in the name of the feature + */ + void InitContinuous(const std::string& name_in) { + InitGeneral(name_in); + type_ = CONTINUOUS; + } + + /** + * Initializes to an integer type. + * + * @param name_in the name of the feature + */ + void InitInteger(const std::string& name_in) { + InitGeneral(name_in); + type_ = INTEGER; + } + + /** + * Initializes to a nominal type. + * + * The value_names list starts empty, so you need to add the name of + * each feature to this. (The dataset reading functions will do this + * for you). + * + * @param name_in the name of the feature + */ + void InitNominal(const std::string& name_in) { + InitGeneral(name_in); + type_ = NOMINAL; + } + + /** + * Creates a text version of the value based on the type. + * + * Continuous parameters are printed in floating point, and integers + * are shown as integers. For nominal, the value_name(int(value)) is + * shown. NaN (missing data) is always shown as '?'. + * + * @param value the value to format + * @param result this will be initialized to the formatted text + */ + void Format(double value, std::string& result) const; + + /** + * Parses a string into the particular value. + * + * Integers and continuous are parsed using the normal functions. + * For nominal, the entry + * + * If an invalid parse occurs, such as a mal-formatted number or + * a nominal value not in the list, SUCCESS_FAIL will be returned. + * + * @param str the string to parse + * @param d where to store the result + */ + success_t Parse(const std::string& str, double& d) const; + + /** + * Gets what the feature is named. + * + * @return the name of the feature; for point, "Age" or "X Position" + */ + const std::string& name() const { + return name_; + } + + /** + * Identifies the type of feature. + * + * @return whether this is DatasetFeature::CONTINUOUS, INTEGER, or NOMINAL + */ + Type type() const { + return type_; + } + + /** + * Returns the name of a particular nominal value, given its index. + * + * The first nominal value is 0, the second is 1, etc. + * + * @param value the number of the value + */ + const std::string& value_name(int value) const { + DEBUG_ASSERT(type_ == NOMINAL); + return value_names_[value]; + } + + /** + * The number of nominal values. + * + * The values 0 to n_values() - 1 are valid. + * This will return zero for CONTINUOUS and INTEGER types. + * + * @return the number of nominal values + */ + index_t n_values() const { + return value_names_.size(); + } + + /** + * Gets the array of value names. + * + * Useful for creating a nominal feature yourself. + * + * @return a mutable array of value names + */ + std::vector& value_names() { + return value_names_; + } +}; + +#endif diff --git a/fastlib/branches/fastlib-stl/fastlib/data/dataset_info.cc b/fastlib/branches/fastlib-stl/fastlib/data/dataset_info.cc new file mode 100644 index 0000000000..0fe7ba0e4a --- /dev/null +++ b/fastlib/branches/fastlib-stl/fastlib/data/dataset_info.cc @@ -0,0 +1,429 @@ +/* MLPACK 0.2 + * + * Copyright (c) 2008, 2009 Alexander Gray, + * Garry Boyer, + * Ryan Riegel, + * Nikolaos Vasiloglou, + * Dongryeol Lee, + * Chip Mappus, + * Nishant Mehta, + * Hua Ouyang, + * Parikshit Ram, + * Long Tran, + * Wee Chin Wong + * + * Copyright (c) 2008, 2009 Georgia Institute of Technology + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ +/** + * @file dataset_info.cc + * + * Implementations for DatasetInfo. + * + * @bug These routines fail when trying to read files linewise that use the Mac + * eol '\r'. Both Windows and Unix eol ("\r\n" and '\n') work. Use the + * programs 'dos2unix' or 'tr' to convert the '\r's to '\n's. + * + */ + +#include "../base/base.h" +#include "../col/tokenizer.h" + +#include "dataset_info.h" + +#include +#include + +using namespace std; + +void DatasetInfo::InitContinuous(index_t n_features, const std::string& name_in) { + features_.reserve(n_features); + + name_ = name_in; + + for (index_t i = 0; i < n_features; i++) { + ostringstream o; + if(!(o << i)) + abort(); + DatasetFeature f; + f.InitContinuous(o.str()); + features_.push_back(f); + } +} + +void DatasetInfo::Init(const string& name_in) { + name_ = name_in; +} + +index_t DatasetInfo::SkipSpace_(string& s) { + int i = 0; + while (isspace(s[i])) { + i++; + } + + if (unlikely(s[i] == '%') || unlikely(s[i] == '\0')) { + return s.length(); + } + + return i; +} + +char *DatasetInfo::SkipNonspace_(char *s) { + while (likely(*s != '\0') + && likely(*s != '%') + && likely(*s != ' ') + && likely(*s != '\t')) { + s++; + } + + return s; +} + +void DatasetInfo::SkipBlanks_(TextLineReader& reader) { + while (reader.MoreLines() && reader.Peek()[SkipSpace_(reader.Peek())] == '\0') { + reader.Gobble(); + } +} + +success_t DatasetInfo::InitFromArff(TextLineReader& reader, const string& filename) { + success_t result = SUCCESS_PASS; + + Init(filename); + + while (1) { + SkipBlanks_(reader); + + string *peeked = &reader.Peek(); + vector portions; + + tokenizeString(*peeked, ", \t", portions, 0, "%{", 3, true); + + if (portions.size() == 0) { + /* empty line */ + } else if (portions[0][0] != '@') { + reader.Error("ARFF: Unexpected @command. Did you forget @data?"); + result = SUCCESS_FAIL; + break; + } else { + if (strcasecmp(portions[0].c_str(), "@relation") == 0) { + if (portions.size() < 2) { + reader.Error("ARFF: @relation requires name"); + result = SUCCESS_FAIL; + } else { + set_name(portions[1]); + } + } else if (strcasecmp(portions[0].c_str(), "@relation") == 0) { + if (portions.size() < 3) { + reader.Error("ARFF: @attribute requires name and type."); + result = SUCCESS_FAIL; + } else { + DatasetFeature feature; + if (portions[2][0] == '{') { //} + feature.InitNominal(portions[1]); + // TODO: Doesn't support values with spaces { + tokenizeString(portions[2], ", \t", feature.value_names(), 1, "}%", 0); + } else { + string type(portions[2]); + //portions[2].Trim(" \t", &type); + if (strcasecmp(type.c_str(), "numeric") == 0 + || strcasecmp(type.c_str(), "real") == 0) { + feature.InitContinuous(portions[1]); + features_.push_back(feature); + } else if (strcasecmp(type.c_str(), "integer") == 0) { + feature.InitInteger(portions[1]); + features_.push_back(feature); + } else { + reader.Error( + "ARFF: Only supports 'numeric', 'real', and {nominal}."); + result = SUCCESS_FAIL; + } + } + } + } else if (strcasecmp(portions[0].c_str(), "@data") == 0) { + /* Done! */ + reader.Gobble(); + break; + } else { + reader.Error("ARFF: Expected @relation, @attribute, or @data."); + result = SUCCESS_FAIL; + break; + } + } + + reader.Gobble(); + } + + return result; +} + +success_t DatasetInfo::InitFromCsv(TextLineReader& reader, const std::string& filename) { + vector headers; + bool nonnumeric = false; + + Init(filename); + + tokenizeString(reader.Peek(), ", \t", headers); + + if (headers.size() == 0) { + reader.Error("Trying to parse empty file as CSV."); + return SUCCESS_FAIL; + } + + // Try to auto-detect if there is a header row + for (index_t i = 0; i < headers.size(); i++) { + char *end; + + (void) strtod(headers[i].c_str(), &end); + + if (end == headers[i].c_str()) { + nonnumeric = true; + break; + } + } + + if (nonnumeric) { + for (index_t i = 0; i < headers.size(); i++) { + DatasetFeature feature; + feature.InitContinuous(headers[i]); + features_.push_back(feature); + } + reader.Gobble(); + } else { + for (index_t i = 0; i < headers.size(); i++) { + DatasetFeature feature; + ostringstream o; + if(!(o << i)) + abort(); + feature.InitContinuous(o.str()); + features_.push_back(feature); + } + } + + return SUCCESS_PASS; +} + +success_t DatasetInfo::InitFromFile(TextLineReader& reader, const std::string& filename) { + SkipBlanks_(reader); + + // WARNING: Safe? + char first_char = reader.Peek()[SkipSpace_(reader.Peek())]; + + if (!first_char) { + Init(); + reader.Error("Could not parse the first line."); + return SUCCESS_FAIL; + } else if (first_char == '@') { + /* Okay, it's ARFF. */ + return InitFromArff(reader, filename); + } else { + /* It's CSV. We'll try to see if there are headers. */ + return InitFromCsv(reader, filename); + } +} + +success_t DatasetInfo::ReadMatrix(TextLineReader& reader, arma::mat &matrix) const { + vector linearized; + index_t n_features = this->n_features(); + index_t n_points = 0; + success_t retval = SUCCESS_PASS; + bool is_done; + + // read through our file to find out how long it is + index_t cur_line = reader.line_num(); + while(reader.Gobble()) { } + matrix.set_size(n_features, reader.line_num() - cur_line + 1); + + string fname = reader.filename(); + reader.Close(); + reader.Open(fname.c_str()); + + // make sure we are in the same place in our file, just in case it was passed + // to us and we had already been some of the way through it (this is why we + // saved the line number + while(reader.line_num() < cur_line) { + reader.Gobble(); + } + + while((n_points < matrix.n_cols) && !is_done && !FAILED(retval)) { + retval = ReadPoint(reader, matrix.begin_col(n_points), is_done); + n_points++; + } + + if (!FAILED(retval)) { + n_points--; // last increment was the failure, so subtract that + DEBUG_ASSERT(n_points == matrix.n_rows); + } + + return retval; +} + +success_t DatasetInfo::ReadPoint(TextLineReader& reader, arma::mat::col_iterator col, + bool &is_done) const { + index_t n_features = this->n_features(); + string str; + string::iterator pos; + + is_done = false; + + for (;;) { + if (!reader.MoreLines()) { + is_done = true; + return SUCCESS_PASS; + } + +// str = reader->Peek(); + pos = reader.Peek().begin(); + + while (*pos == ' ' || *pos == '\t' || *pos == ',') { + pos++; + } + + if (unlikely(*pos == '\0' || *pos == '%')) { + reader.Gobble(); + } else { + break; + } + } + + for (index_t i = 0; i < n_features; i++) { + string::iterator next; + + while (*pos == ' ' || *pos == '\t' || *pos == ',') { + pos++; + } + + if (unlikely(*pos == '\0')) { + for (string::iterator s = reader.Peek().begin(); s < pos; s++) { // UNDEFINED + if (!*s) { + *s = ','; + } + } + reader.Error("I am expecting %"LI"d entries per row, " + "but this line has only %"LI"d.", + n_features, i); + return SUCCESS_FAIL; + } + + next = pos; + while (*next != '\0' && *next != ' ' && *next != '\t' && *next != ',' + && *next != '%') { + next++; + } + + if (*next != '\0') { + char c = *next; + *next = '\0'; + if (c != '%') { + next++; + } + } + + size_t len = reader.Peek().end() - pos; + size_t cpos = pos - reader.Peek().begin(); + if (!PASSED(features_[i].Parse(reader.Peek().substr(cpos, len), col[i]))) { + string::iterator end = reader.Peek().end(); + string tmp; + tmp.assign(pos, reader.Peek().end()); + for (string::iterator s = reader.Peek().begin(); s < next && s < end; s++) { + if (*s == '\0') { + *s = ','; + } + } + reader.Error("Invalid parse: [%s]", tmp.c_str()); + return SUCCESS_FAIL; + } + + // increment the value we are looking at + pos = next; + } + + while (*pos == ' ' || *pos == '\t' || *pos == ',') { + pos++; + } + + if (*pos != '\0') { + for (string::iterator s = reader.Peek().begin(); s < pos; s++) { + if (*s == '\0') { + *s = ','; + } + } + reader.Error("Extra junk on line."); + return SUCCESS_FAIL; + } + + reader.Gobble(); + + return SUCCESS_PASS; +} + +bool DatasetInfo::is_all_continuous() const { + for (index_t i = 0; i < features_.size(); i++) { + if (features_[i].type() != DatasetFeature::CONTINUOUS) { + return false; + } + } + + return true; +} + +void DatasetInfo::WriteArffHeader(TextWriter& writer) const { + writer.Printf("@relation %s\n", name_.c_str()); + + for (index_t i = 0; i < features_.size(); i++) { + const DatasetFeature *feature = &features_[i]; + writer.Printf("@attribute %s ", feature->name().c_str()); + if (feature->type() == DatasetFeature::NOMINAL) { + writer.Printf("{"); + for (index_t v = 0; v < feature->n_values(); v++) { + if (v != 0) { + writer.Write(","); + } + writer.Write(feature->value_name(v).c_str()); + } + writer.Printf("}"); + } else { + writer.Write("real"); + } + writer.Write("\n"); + } + writer.Printf("@data\n"); +} + +void DatasetInfo::WriteCsvHeader(const char *sep, TextWriter& writer) const { + for (index_t i = 0; i < features_.size(); i++) { + if (i != 0) { + writer.Write(sep); + } + writer.Write(features_[i].name().c_str()); + } + writer.Write("\n"); +} + +void DatasetInfo::WriteMatrix(const arma::mat& matrix, const char *sep, + TextWriter& writer) const { + for (index_t i = 0; i < matrix.n_cols; i++) { + for (index_t f = 0; f < features_.size(); f++) { + if (f != 0) { + writer.Write(sep); + } + string str; + features_[f].Format(matrix(f, i), str); + writer.Write(str.c_str()); + } + writer.Write("\n"); + } +} diff --git a/fastlib/branches/fastlib-stl/fastlib/data/dataset_info.h b/fastlib/branches/fastlib-stl/fastlib/data/dataset_info.h new file mode 100644 index 0000000000..730eb12d32 --- /dev/null +++ b/fastlib/branches/fastlib-stl/fastlib/data/dataset_info.h @@ -0,0 +1,213 @@ +/* MLPACK 0.2 + * + * Copyright (c) 2008, 2009 Alexander Gray, + * Garry Boyer, + * Ryan Riegel, + * Nikolaos Vasiloglou, + * Dongryeol Lee, + * Chip Mappus, + * Nishant Mehta, + * Hua Ouyang, + * Parikshit Ram, + * Long Tran, + * Wee Chin Wong + * + * Copyright (c) 2008, 2009 Georgia Institute of Technology + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ +/** + * @file dataset_info.h + * + * Declaration of DatasetInfo class, which is a helper class for Dataset. + * + * @bug These routines fail when trying to read files linewise that use the Mac + * eol '\r'. Both Windows and Unix eol ("\r\n" and '\n') work. Use the + * programs 'dos2unix' or 'tr' to convert the '\r's to '\n's. + * + */ + +#ifndef DATA_DATASET_INFO_H +#define DATA_DATASET_INFO_H + +#include +#include +#include + +#include "../la/matrix.h" +#include "../math/discrete.h" +#include "../file/textfile.h" + +#include "dataset_feature.h" + +class TextLineReader; +class TextWriter; + +/** + * Information describing a dataset and its features. + */ +class DatasetInfo { + private: + std::string name_; + std::vector features_; + + public: + /** Gets a mutable list of all features. */ + std::vector& features() { + return features_; + } + + /** Gets information about a particular feature. */ + const DatasetFeature& feature(index_t attrib_num) const { + return features_[attrib_num]; + } + + /** Gets the number of features. */ + index_t n_features() const { + return features_.size(); + } + + /** Gets the title of the data set. */ + const std::string& name() const { + return name_; + } + + /** Sets the title of the data set. */ + void set_name(const std::string& name_in) { + name_ = name_in; + } + + /** + * Checks if all parameters are continuous. + */ + bool is_all_continuous() const; + + /** + * Initialize an all-continuous dataset; + * + * @param n_features the number of continuous features + * @param name_in the dataset title + */ + void InitContinuous(index_t n_features, + const std::string& name_in = "dataset"); + + /** + * Initialize a custom dataset. + * + * This assumes you will eventually use the features() to add features. + * + * @param name_in the dataset title + */ + void Init(const std::string& name_in = "dataset"); + + /** + * Copy constructor, written manually to avoid use of OBJECT_TRAVERSAL functions. + * TODO: Make this Init() crap go away. + */ + void InitCopy(const DatasetInfo &info) { + name_ = info.name(); + features_ = info.features_; + } + + /** + * Writes the header for an ARFF file. + */ + void WriteArffHeader(TextWriter& writer) const; + + /** + * Writes header for CSV file. + * + * @param sep the value separator (use ",\t" for CSV) + * @param writer the text writer to write the header line to + */ + void WriteCsvHeader(const char *sep, TextWriter& writer) const; + + /** + * Writes the contents of a matrix to a file. + * + * @param matrix the matrix + * @param sep the separator (use ",\t" for CSV) + * @param writer the writer to write to + */ + void WriteMatrix(const arma::mat& matrix, const char *sep, + TextWriter& writer) const; + + /** + * Initialize explicitly from an ARFF file. + * + * ARFF LIMITATIONS: Values cannot have spaces or commas, even with quotes; + * 'string' data type not supported (nominal is supported). + * + * You might just use InitFromFile, which will guess the type for you. + * + * This will read only the header information and leave the reader at the + * first line of data. + */ + success_t InitFromArff(TextLineReader& reader, + const std::string& filename = "dataset"); + + /** + * Initialize from a CSV-like file with numbers only, inferring + * automatically that if the first row has non-numeric characters, it is + * a header. + * + * InitFromFile will automatically detect this. + */ + success_t InitFromCsv(TextLineReader& reader, + const std::string& filename = "dataset"); + + /** + * Initializes the header from a file, either CSV or ARFF. + * + * All header lines will be gobbled, so the reader's position will be + * left at the first line of actual data. + * You can then read the data with matrix. + */ + success_t InitFromFile(TextLineReader& reader, + const std::string& filename = "dataset"); + + /** + * Populates a matrix from a file, given the internal data model. + * + * ARFF LIMITATIONS: Values cannot have spaces or commas, even with quotes; + * 'string' data type not supported (nominal is supported). + * + * @param reader the reader to get lines from + * @param matrix the matrix to store text into + */ + success_t ReadMatrix(TextLineReader& reader, arma::mat& matrix) const; + + /** + * Reads a single vector. + * + * @param reader the line reader being used + * @param col a mat::col_iterator referencing the beginning of the + * column that we are reading into + * @param is_done set to true if we have finished reading the file + * successfully -- the value of is_done is undefined if the + * function returns failure! + * @return whether reading the line was successful + */ + success_t ReadPoint(TextLineReader& reader, arma::mat::col_iterator col, + bool& is_done) const; + + private: + index_t SkipSpace_(std::string& s); + char *SkipNonspace_(char *s); + void SkipBlanks_(TextLineReader& reader); +}; + +#endif diff --git a/fastlib/branches/fastlib-stl/fastlib/data/dataset_test.cc b/fastlib/branches/fastlib-stl/fastlib/data/dataset_test.cc index 31ee124a17..6f17559fe4 100644 --- a/fastlib/branches/fastlib-stl/fastlib/data/dataset_test.cc +++ b/fastlib/branches/fastlib-stl/fastlib/data/dataset_test.cc @@ -29,23 +29,26 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA * 02110-1301, USA. */ +#include + #include "dataset.h" -//#include "dataset.h" #include "../math/discrete.h" #include "../base/test.h" +using arma::mat; + TEST_SUITE_BEGIN(dataset) void TestSplitTrainTest() { Dataset orig; orig.InitBlank(); - orig.matrix().Init(1, 12); + orig.matrix().set_size(1, 12); orig.info().InitContinuous(1); for (int i = 0; i < 12; i++) { - orig.matrix().set(0, i, i); + orig.matrix()(0, i) = i; } std::vector found; @@ -61,7 +64,7 @@ void TestSplitTrainTest() { permutation.push_back(i); orig.SplitTrainTest(5, 1, permutation, - &train, &test); + train, test); DEBUG_ASSERT(test.n_points() == 3); DEBUG_ASSERT(train.n_points() == 9); @@ -80,17 +83,17 @@ void TestSplitTrainTest() { DEBUG_ASSERT_MSG(train.get(0, 8) == 10, "%f", (train.get(0, 8))); } -void AssertSameMatrix(const Matrix& a, const Matrix& b) { - index_t r = a.n_rows(); - index_t c = a.n_cols(); +void AssertSameMatrix(const mat& a, const mat& b) { + index_t r = a.n_rows; + index_t c = a.n_cols; - TEST_ASSERT(a.n_rows() == b.n_rows()); - TEST_ASSERT(a.n_cols() == b.n_cols()); + TEST_ASSERT(a.n_rows == b.n_rows); + TEST_ASSERT(a.n_cols == b.n_cols); for (index_t ri = 0; ri < r; ri++) { for (index_t ci = 0; ci < c; ci++) { - DEBUG_ASSERT_MSG(a.get(ri, ci) == b.get(ri, ci), "(%d, %d): %f != %f", - ri, ci, a.get(ri, ci), b.get(ri, ci)); + DEBUG_ASSERT_MSG(a(ri, ci) == b(ri, ci), "(%d, %d): %f != %f", + ri, ci, a(ri, ci), b(ri, ci)); } } } @@ -129,8 +132,8 @@ void TestStoreLoad() { AssertSameMatrix(d1.matrix(), d2.matrix()); AssertSameMatrix(d1.matrix(), d3.matrix()); - DEBUG_ASSERT_MSG(strcmp(d1.info().name(), d2.info().name()) == 0, - "%s != %s", d1.info().name(), d2.info().name()); + DEBUG_ASSERT_MSG(strcmp(d1.info().name().c_str(), d2.info().name().c_str()) == 0, + "%s != %s", d1.info().name().c_str(), d2.info().name().c_str()); for (index_t i = 0; i < d1.info().n_features(); i++) { DEBUG_ASSERT( d1.info().feature(i).name() == d2.info().feature(i).name() ); DEBUG_ASSERT(d1.info().feature(i).type() == d2.info().feature(i).type());