diff --git a/fastlib/u/houyang/svm/README b/fastlib/u/houyang/svm/README index fa5d563bc7..f2f91611b6 100644 --- a/fastlib/u/houyang/svm/README +++ b/fastlib/u/houyang/svm/README @@ -10,18 +10,23 @@ build.py: build scripts svm_main.cc: contains main function of SVM svm.h: contains classification/regression/density estimation main routines smo.h: sequential minimal optimization + # Unit test data: -traindata2.csv: training data for a 2 classes toy problem. 10 samples in each class. -testdata2.csv: testing data for a 2 classes toy problem. 10 samples in each class. -traindata3.csv: training data for a 3 classes toy problem. 10 samples in each class. -testdata3.csv: testing data for a 3 classes toy problem. 10 samples in each class. +c_traindata2.csv: training data for a 2 classes toy problem. 10 samples in each class. +c_testdata2.csv: testing data for a 2 classes toy problem. 10 samples in each class. +c_traindata3.csv: training data for a 3 classes toy problem. 10 samples in each class. +c_testdata3.csv: testing data for a 3 classes toy problem. 10 samples in each class. + +r_traindata.csv: training data for regression, 100 samples of a 1-dim sinc function. +r_testdata.csv: testing data for regression, 100 samples of a 1-dim sinc function. + # Generated files: artificialdata.csv: artificial data if no training/testing data file provided svm_model: SVM model file -testlabels: classified labels +predicted_values: predicted result Explanation of arguments: -0. learner_name (required): the name of the support vecotr learner, values: "svc" for classification, "svr" for regression, "svde" for density estimation +0. learner_name (required): the name of the support vecotr learner, values: "svm_c" for classification, "svm_r" for regression, "svm_de" for density estimation 1. mode (required) : the mode of svm_main, values: "cv", "train", "train_test", "test". 2. k_cv (optional): the number of folds for cross validation, only required under "cv" mode. 3. cv_data (optional): file name for cross validation data, only required under "cv" mode. @@ -29,40 +34,47 @@ Explanation of arguments: 5. test_data (optional): file name for testing data, only required under "test" or "train_test" mode. 6. kernel (required): kernel name, values:"linear", "gaussian". 7. sigma (optional): sigma in the gaussian kernel k(x1,x2)=exp(-(x1-x2)^2/(2sigma^2)), only required when using "guassian" kernel. -8. c (required): the weight that controls compromise between large margins and small margin violations. -9. normalize (optional): where need to do data normalization before training/testing, values: "0" for no normalize, "1" for normalize. +8. c (for SVM_C,optional): the weight (0~1) that controls compromise between large margins and small margin violations. Default value: 10.0. +9. c_p (for SVM_C,optional): the weight (0~1) for the positive class (y==1). Default value: c. +10. c_n (for SVM_C,optional): the weight (0~1) for the negative class (y==-1). Default value: c. +11. epsilon (for SVM_R,optional): the epsilon in SVM regression. Default value: 0.1. +12. wss (optional): working set selection scheme. 1 for 1st order expansion; 2 for 2nd order expansion. Default value: 1. +13. normalize (optional): where need to do data normalization before training/testing, values: "0" for no normalize, "1" for normalize. Examples: ---------------------------------------------------------------------------------------------------- +--------------------------------------------------- SVM Classificaation ------------------------------------------------ a1.Support Vector Classification, Stratified cross validation mode -svm_main --learner_name=svc --mode=cv --k_cv=4 --cv_data=traindata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0 -svm_main --learner_name=svc --mode=cv --k_cv=4 --cv_data=traindata3.csv --kernel=linear --c=10 --normalize=0 +svm_main --learner_name=svm_c --mode=cv --k_cv=4 --cv_data=c_traindata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0 +svm_main --learner_name=svm_c --mode=cv --k_cv=4 --cv_data=c_traindata3.csv --kernel=linear --c=10 --normalize=0 a2.Support Vector Classification, Training mode (model will be saved as "svm_model") -svm_main --learner_name=svc --mode=train --train_data=traindata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0 -svm_main --learner_name=svc --mode=train --train_data=traindata3.csv --kernel=linear --c=10 --normalize=0 +svm_main --learner_name=svm_c --mode=train --train_data=c_traindata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0 +svm_main --learner_name=svm_c --mode=train --train_data=c_traindata3.csv --kernel=linear --c=10 --normalize=0 a3.Support Vector Classification, Training+testing mode -svm_main --learner_name=svc --mode=train_test --train_data=traindata3.csv --test_data=testdata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0 -svm_main --learner_name=svc --mode=train_test --train_data=traindata3.csv --test_data=testdata3.csv --kernel=linear --c=10 --normalize=0 +svm_main --learner_name=svm_c --mode=train_test --train_data=c_traindata3.csv --test_data=c_testdata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0 +svm_main --learner_name=svm_c --mode=train_test --train_data=c_traindata3.csv --test_data=c_testdata3.csv --kernel=linear --c=10 --normalize=0 a4.Support Vector Classification, Testing mode (the model file "svm_model" should exist) -svm_main --learner_name=svc --mode=test --test_data=testdata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0 -svm_main --learner_name=svc --mode=test --test_data=testdata3.csv --kernel=linear --c=10 --normalize=0 +svm_main --learner_name=svm_c --mode=test --test_data=c_testdata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0 +svm_main --learner_name=svm_c --mode=test --test_data=c_testdata3.csv --kernel=linear --c=10 --normalize=0 ---------------------------------------------------------------------------------------------------- +--------------------------------------------------- SVM Regression ------------------------------------------------------- b1.Support Vector Regression, Cross validation mode -svm_main --learner_name=svc --mode=cv --k_cv=4 --cv_data=traindata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0 -svm_main --learner_name=svc --mode=cv --k_cv=4 --cv_data=traindata3.csv --kernel=linear --c=10 --normalize=0 +svm_main --learner_name=svm_r --mode=cv --k_cv=4 --cv_data=r_traindata.csv --kernel=gaussian --sigma=0.1 --epsilon=0.1 --c=10 --normalize=0 +svm_main --learner_name=svm_r --mode=cv --k_cv=4 --cv_data=r_traindata.csv --kernel=linear --epsilon=0.1 --c=10 --normalize=0 b2.Support Vector Regression, Training mode (model will be saved as "svm_model") -svm_main --learner_name=svc --mode=train --train_data=traindata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0 -svm_main --learner_name=svc --mode=train --train_data=traindata3.csv --kernel=linear --c=10 --normalize=0 +svm_main --learner_name=svm_r --mode=train --train_data=r_traindata.csv --kernel=gaussian --sigma=0.1 --epsilon=0.1 --c=10 --normalize=0 +svm_main --learner_name=svm_r --mode=train --train_data=r_traindata.csv --kernel=linear --epsilon=0.1 --c=10 --normalize=0 b3.Support Vector Regression, Training+testing mode -svm_main --learner_name=svc --mode=train_test --train_data=traindata3.csv --test_data=testdata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0 -svm_main --learner_name=svc --mode=train_test --train_data=traindata3.csv --test_data=testdata3.csv --kernel=linear --c=10 --normalize=0 +svm_main --learner_name=svm_r --mode=train_test --train_data=r_traindata.csv --test_data=r_testdata.csv --kernel=gaussian --sigma=0.1 --epsilon=0.1 --c=10 --normalize=0 +svm_main --learner_name=svm_r --mode=train_test --train_data=r_traindata.csv --test_data=r_testdata.csv --kernel=linear --epsilon=0.1 --c=10 --normalize=0 b4.Support Vector Regression, Testing mode (the model file "svm_model" should exist) -svm_main --learner_name=svc --mode=test --test_data=testdata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0 -svm_main --learner_name=svc --mode=test --test_data=testdata3.csv --kernel=linear --c=10 --normalize=0 +svm_main --learner_name=svm_r --mode=test --test_data=r_testdata.csv --kernel=gaussian --sigma=0.1 --epsilon=0.1 --c=10 --normalize=0 +svm_main --learner_name=svm_r --mode=test --test_data=r_testdata.csv --kernel=linear --epsilon=0.1 --c=10 --normalize=0 + +--------------------------------------------------- SVM Density Estimation -------------------------------------------------- +c1... TODO diff --git a/fastlib/u/houyang/svm/testdata2.csv b/fastlib/u/houyang/svm/c_testdata2.csv similarity index 100% rename from fastlib/u/houyang/svm/testdata2.csv rename to fastlib/u/houyang/svm/c_testdata2.csv diff --git a/fastlib/u/houyang/svm/testdata3.csv b/fastlib/u/houyang/svm/c_testdata3.csv similarity index 100% rename from fastlib/u/houyang/svm/testdata3.csv rename to fastlib/u/houyang/svm/c_testdata3.csv diff --git a/fastlib/u/houyang/svm/traindata2.csv b/fastlib/u/houyang/svm/c_traindata2.csv similarity index 100% rename from fastlib/u/houyang/svm/traindata2.csv rename to fastlib/u/houyang/svm/c_traindata2.csv diff --git a/fastlib/u/houyang/svm/traindata3.csv b/fastlib/u/houyang/svm/c_traindata3.csv similarity index 100% rename from fastlib/u/houyang/svm/traindata3.csv rename to fastlib/u/houyang/svm/c_traindata3.csv diff --git a/fastlib/u/houyang/svm/r_testdata.csv b/fastlib/u/houyang/svm/r_testdata.csv new file mode 100644 index 0000000000..b2f739d224 --- /dev/null +++ b/fastlib/u/houyang/svm/r_testdata.csv @@ -0,0 +1,100 @@ + -10.0000 -0.0544 + -9.7980 -0.0372 + -9.5960 -0.0178 + -9.3939 0.0033 + -9.1919 0.0251 + -8.9899 0.0469 + -8.7879 0.0677 + -8.5859 0.0866 + -8.3838 0.1029 + -8.1818 0.1157 + -7.9798 0.1243 + -7.7778 0.1282 + -7.5758 0.1269 + -7.3737 0.1203 + -7.1717 0.1082 + -6.9697 0.0909 + -6.7677 0.0688 + -6.5657 0.0425 + -6.3636 0.0126 + -6.1616 -0.0197 + -5.9596 -0.0534 + -5.7576 -0.0871 + -5.5556 -0.1197 + -5.3535 -0.1497 + -5.1515 -0.1757 + -4.9495 -0.1964 + -4.7475 -0.2105 + -4.5455 -0.2169 + -4.3434 -0.2147 + -4.1414 -0.2032 + -3.9394 -0.1817 + -3.7374 -0.1501 + -3.5354 -0.1085 + -3.3333 -0.0572 + -3.1313 0.0033 + -2.9293 0.0719 + -2.7273 0.1476 + -2.5253 0.2289 + -2.3232 0.3142 + -2.1212 0.4018 + -1.9192 0.4897 + -1.7172 0.5761 + -1.5152 0.6590 + -1.3131 0.7364 + -1.1111 0.8066 + -0.9091 0.8678 + -0.7071 0.9187 + -0.5051 0.9580 + -0.3030 0.9848 + -0.1010 0.9983 + 0.1010 0.9983 + 0.3030 0.9848 + 0.5051 0.9580 + 0.7071 0.9187 + 0.9091 0.8678 + 1.1111 0.8066 + 1.3131 0.7364 + 1.5152 0.6590 + 1.7172 0.5761 + 1.9192 0.4897 + 2.1212 0.4018 + 2.3232 0.3142 + 2.5253 0.2289 + 2.7273 0.1476 + 2.9293 0.0719 + 3.1313 0.0033 + 3.3333 -0.0572 + 3.5354 -0.1085 + 3.7374 -0.1501 + 3.9394 -0.1817 + 4.1414 -0.2032 + 4.3434 -0.2147 + 4.5455 -0.2169 + 4.7475 -0.2105 + 4.9495 -0.1964 + 5.1515 -0.1757 + 5.3535 -0.1497 + 5.5556 -0.1197 + 5.7576 -0.0871 + 5.9596 -0.0534 + 6.1616 -0.0197 + 6.3636 0.0126 + 6.5657 0.0425 + 6.7677 0.0688 + 6.9697 0.0909 + 7.1717 0.1082 + 7.3737 0.1203 + 7.5758 0.1269 + 7.7778 0.1282 + 7.9798 0.1243 + 8.1818 0.1157 + 8.3838 0.1029 + 8.5859 0.0866 + 8.7879 0.0677 + 8.9899 0.0469 + 9.1919 0.0251 + 9.3939 0.0033 + 9.5960 -0.0178 + 9.7980 -0.0372 + 10.0000 -0.0544 \ No newline at end of file diff --git a/fastlib/u/houyang/svm/r_traindata.csv b/fastlib/u/houyang/svm/r_traindata.csv new file mode 100644 index 0000000000..ab4023730a --- /dev/null +++ b/fastlib/u/houyang/svm/r_traindata.csv @@ -0,0 +1,100 @@ + -10.0000, 0.0320 + -9.7980, -0.0278 + -9.5960, -0.1029 + -9.3939, 0.0906 + -9.1919, -0.0187 + -8.9899, 0.0039 + -8.7879, -0.0426 + -8.5859, 0.1263 + -8.3838, 0.0064 + -8.1818, 0.1326 + -7.9798, -0.0722 + -7.7778, 0.0538 + -7.5758, 0.0717 + -7.3737, 0.0383 + -7.1717, 0.2191 + -6.9697, 0.0294 + -6.7677, 0.0434 + -6.5657, 0.0155 + -6.3636, -0.1546 + -6.1616, -0.2073 + -5.9596, 0.0041 + -5.7576, -0.1738 + -5.5556, -0.3314 + -5.3535, -0.2461 + -5.1515, -0.1544 + -4.9495, -0.1486 + -4.7475, -0.2004 + -4.5455, -0.1872 + -4.3434, -0.1577 + -4.1414, -0.3656 + -3.9394, -0.1174 + -3.7374, -0.0820 + -3.5354, -0.1071 + -3.3333, -0.1873 + -3.1313, -0.1252 + -2.9293, 0.1532 + -2.7273, 0.2315 + -2.5253, 0.3709 + -2.3232, 0.2153 + -2.1212, 0.2835 + -1.9192, 0.4431 + -1.7172, 0.5395 + -1.5152, 0.7708 + -1.3131, 0.6898 + -1.1111, 0.6505 + -0.9091, 0.8395 + -0.7071, 0.7864 + -0.5051, 0.9384 + -0.3030, 1.0267 + -0.1010, 1.0725 + 0.1010, 0.9840 + 0.3030, 0.7686 + 0.5051, 0.8936 + 0.7071, 1.0627 + 0.9091, 0.7831 + 1.1111, 0.8123 + 1.3131, 0.8007 + 1.5152, 0.5919 + 1.7172, 0.5758 + 1.9192, 0.5250 + 2.1212, 0.5198 + 2.3232, 0.2456 + 2.5253, 0.3966 + 2.7273, 0.1221 + 2.9293, 0.0072 + 3.1313, -0.0149 + 3.3333, 0.0280 + 3.5354, -0.1392 + 3.7374, -0.1942 + 3.9394, -0.2429 + 4.1414, -0.2517 + 4.3434, -0.0950 + 4.5455, -0.0775 + 4.7475, -0.1940 + 4.9495, -0.2474 + 5.1515, -0.0379 + 5.3535, -0.0198 + 5.5556, -0.1327 + 5.7576, -0.0131 + 5.9596, 0.0798 + 6.1616, -0.0475 + 6.3636, -0.0202 + 6.5657, 0.0412 + 6.7677, 0.1591 + 6.9697, -0.0203 + 7.1717, 0.0243 + 7.3737, 0.1238 + 7.5758, 0.0023 + 7.7778, 0.2166 + 7.9798, 0.3782 + 8.1818, 0.2474 + 8.3838, 0.2471 + 8.5859, 0.2333 + 8.7879, -0.0430 + 8.9899, 0.0008 + 9.1919, 0.0231 + 9.3939, -0.0013 + 9.5960, -0.0722 + 9.7980, 0.0545 + 10.0000, -0.0563 \ No newline at end of file diff --git a/fastlib/u/houyang/svm/smo.h b/fastlib/u/houyang/svm/smo.h index 469771dafc..6a2374b4a4 100644 --- a/fastlib/u/houyang/svm/smo.h +++ b/fastlib/u/houyang/svm/smo.h @@ -1,22 +1,38 @@ /** - * @author Garry Boyer, Hua Ouyang + * @author Hua Ouyang * * @file smo.h * * This head file contains functions for performing Budget Sequential Minimal Optimization (SMO) - * for a bi-class (labels 1 and -1) classification problem. * * The algorithms in the following papers are implemented: * - * Multiclass Support Vector Machine Classification + * 1. SMO and Working set selecting using 1st order expansion + * @ARTICLE{Platt_SMO, + * author = "J. C. Platt", + * title = "{Fast Training of Support Vector Machines using Sequential Minimal Optimization}", + * booktitle = "{Advances in Kernel Methods - Support Vector Learning}", + * year = 1999, + * publisher = "MIT Press" + * } + * + * 2. Shrinkng for SMO * @ARTICLE{Joachims_SVMLIGHT, * author = "T. Joachims", * title = "{Making large-Scale SVM Learning Practical}", - * journal = "{Advances in Kernel Methods - Support Vector Learning}", + * booktitle = "{Advances in Kernel Methods - Support Vector Learning}", * year = 1999, * publisher = "MIT Press" * } * + * 3. Working set selecting using 2nd order expansion + * @ARTICLE{Fan_JMLR, + * author = "R. Fan, P. Chen, C. Lin", + * title = "{Working Set Selection using Second Order Information for Training Support Vector Machines}", + * journal = "{Jornal of Machine Learning Research}", + * year = 2005 + * } + * * @INPROCEEDINGS{Budge_SVM, * author = "O. Dekel and Y. Singer", * title = "{Support Vector Machines on a Budget}", @@ -33,11 +49,17 @@ #include "fastlib/fastlib.h" -/* TODO: I don't actually want these to be public */ -/* but sometimes we should provide freedoms for our advanced users */ -const double SMO_ZERO = 1.0e-8; -const double SMO_EPS = 1.0e-4; -const double SMO_TOLERANCE = 1.0e-4; +const double SMO_OPT_TOLERANCE = 1.0e-4; +const index_t MAX_NUM_ITER = 10000; +const index_t NUM_FOR_SHRINKING = 1000; + +const double ID_LOWER_BOUNDED = -1; +const double ID_UPPER_BOUNDED = 1; +const double ID_FREE = 0; + +const double TAU = 1e-12; + +const double SMO_ALPHA_ZERO = 1.0e-4; template class SMO { @@ -47,428 +69,558 @@ class SMO { typedef TKernel Kernel; private: - Matrix kernel_cache_sign_; + int learner_typeid_; + index_t ct_iter_; /* counter for the number of iterations */ + index_t ct_shrinking_; /* counter for doing shrinking */ + Kernel kernel_; const Dataset *dataset_; index_t n_data_; /* number of data samples */ - Matrix matrix_; /* alias for the data matrix */ + Matrix datamatrix_; /* alias for the data matrix */ + Vector alpha_; /* the alphas, to be optimized */ + Vector alpha_status_; /* ID_LOWER_BOUND (-1), ID_UPPER_BOUND (1), ID_FREE (0) */ + + index_t n_alpha_; /* number of variables to be optimized */ + index_t n_active_; /* number of samples in the active set */ + ArrayList active_set_; + + ArrayList y_; + + double bias_; index_t n_sv_; /* number of support vectors */ - Vector error_; - double thresh_; - double c_; + + Vector grad_; /* gradient value */ + Vector grad_bar_; /* gradient value when treat free variables as 0 */ + + // parameters int budget_; - double sum_alpha_; + double Cp_; // C_+, for SVM_C, y==1 + double Cn_; // C_-, for SVM_C, y==-1 + double epsilon_; // for SVM_R + int wss_; // working set selection scheme, 1 for 1st order expansion; 2 for 2nd order expansion public: SMO() {} ~SMO() {} /** - * Initialization + * Initialization for parameters */ - void Init(double c_in, int budget_in) { - c_ = c_in; - budget_ = budget_in; - - thresh_ = 0.0; - n_sv_ = 0; + void InitPara(int learner_typeid, ArrayList ¶m_) { + // init parameters + budget_ = (int)param_[0]; + if (learner_typeid == 0) { // SVM_C + Cp_ = param_[1]; + Cn_ = param_[2]; + wss_ = (int) param_[3]; + } + else if (learner_typeid == 1) { // SVM_R + Cp_ = param_[1]; + Cn_ = Cp_; + epsilon_ = param_[2]; + wss_ = (int) param_[3]; + } } - void Train(const Dataset* dataset_in); + void Train(int learner_typeid, const Dataset* dataset_in); Kernel& kernel() { return kernel_; } - double threshold() const { - return thresh_; + double Bias() const { + return bias_; } - //index_t num_sv() const { - // return n_sv_; - //} - void GetSVM(ArrayList &dataset_index, ArrayList &coef, ArrayList &sv_indicator); private: - index_t TrainIteration_(bool examine_all); + void LearnersInit_(int learner_typeid); + + int TrainIteration_(); + + void ReconstructGradient_(int learner_typeid); - bool TryChange_(index_t j); + void Shrinking_(); + + bool WorkingSetSelection_(index_t &i, index_t &j); - bool TakeStep_(index_t i, index_t j, double error_j); + void UpdatingGradientAlpha_(index_t i, index_t j); - double FixAlpha_(double alpha) const { - if (alpha < SMO_ZERO) { - alpha = 0; - } else if (alpha > c_ - SMO_ZERO) { - alpha = c_; - } - return alpha; - } - - bool IsBound_(double alpha) const { - return alpha <= 0 || alpha >= c_; - } - - /* labels: 1 and -1 */ - int GetLabelSign_(index_t i) const { - return matrix_.get(matrix_.n_rows()-1, i) == 1 ? 1 : -1; - } + void CalcBias_(); void GetVector_(index_t i, Vector *v) const { - matrix_.MakeColumnSubvector(i, 0, matrix_.n_rows()-1, v); + datamatrix_.MakeColumnSubvector(i, 0, datamatrix_.n_rows()-1, v); } - double Error_(index_t i) const { - double val; - if (!IsBound_(alpha_[i])) { - val = error_[i]; - VERBOSE_MSG(0, "error values %f and %f", error_[i], Evaluate_(i) - GetLabelSign_(i)); - } else { - val = CalculateError_(i); + /** + * Instead of C, we use C_+ and C_- to handle unbalanced data + */ + double GetC_(index_t i) { + return (y_[i] > 0 ? Cp_ : Cn_); + } + + void UpdateAlphaStatus_(index_t i) { + if (alpha_[i] >= GetC_(i)) { + alpha_status_[i] = ID_UPPER_BOUNDED; + } + else if (alpha_[i] <= 0) { + alpha_status_[i] = ID_LOWER_BOUNDED; + } + else { // 0 < alpha_[i] < C + alpha_status_[i] = ID_FREE; } - return val; } - double CalculateError_(index_t i) const { - return Evaluate_(i) - GetLabelSign_(i); + bool IsUpperBounded(index_t i) { + return alpha_status_[i] == ID_UPPER_BOUNDED; + } + bool IsLowerBounded(index_t i) { + return alpha_status_[i] == ID_LOWER_BOUNDED; } - - double Evaluate_(index_t i) const; - double EvalKernel_(index_t i, index_t j) const { - return kernel_cache_sign_.get(i, j) * (GetLabelSign_(i) * GetLabelSign_(j)); - } - /** * Calculate kernel values */ - void CalcKernels_() { - kernel_cache_sign_.Init(n_data_, n_data_); - fprintf(stderr, "Kernel Start\n"); - for (index_t i = 0; i < n_data_; i++) { - for (index_t j = 0; j < n_data_; j++) { - Vector v_i; - GetVector_(i, &v_i); - Vector v_j; - GetVector_(j, &v_j); - double k = kernel_.Eval(v_i, v_j); - kernel_cache_sign_.set(j, i, k * GetLabelSign_(i) * GetLabelSign_(j)); - } - } - fprintf(stderr, "Kernel Stop\n"); + double CalcKernelValue_(index_t i, index_t j) { + i = i >= n_data_ ? (i-n_data_) : i; // for SVM_R + j = j >= n_data_ ? (j-n_data_) : j; + Vector v_i; + GetVector_(i, &v_i); + Vector v_j; + GetVector_(j, &v_j); + return kernel_.Eval(v_i, v_j); } }; + +/** + * Initialization for different SVM learners + * + * @param: learner type id + */ +template +void SMO::LearnersInit_(int learner_typeid) { + index_t i; + learner_typeid_ = learner_typeid; + + if (learner_typeid_ == 0) { // SVM_C + n_alpha_ = n_data_; + + alpha_.Init(n_alpha_); + alpha_.SetZero(); + + // initialize gradient + grad_.Init(n_alpha_); + for (i=0; i 0 ? 1 : -1; + } + } + else if (learner_typeid_ == 1) { // SVM_R + n_alpha_ = 2 * n_data_; + + alpha_.Init(2 * n_alpha_); + alpha_.SetZero(); + + // initialize gradient + grad_.Init(n_alpha_); + y_.Init(n_alpha_); + for (i = 0; i < n_data_; i++) { + y_[i] = 1; // -> alpha_i + y_[i + n_data_] = -1; // -> alpha_i^* + grad_[i] = epsilon_ - datamatrix_.get(datamatrix_.n_rows()-1, i); + grad_[i + n_data_] = epsilon_ + datamatrix_.get(datamatrix_.n_rows()-1, i); + } + } + else if (learner_typeid_ == 2) { // SVM_DE + } + + // initialize active set + n_active_ = n_alpha_; + active_set_.Init(n_active_); + for (i=0; i +void SMO::ReconstructGradient_(int learner_typeid) { + index_t i, j; + if (n_active_ == n_alpha_) + return; + if (learner_typeid == 0) { // SVM_C + for (i=n_active_; i +void SMO::Shrinking_() { +} + + /** * Budget SMO training for 2-classes * * @param: input 2-classes data matrix with labels (1,-1) in the last row */ template -void SMO::Train(const Dataset* dataset_in) { - bool examine_all = true; - index_t num_changed = 0; - int n_iter = 0; - - /* data-dependent initialization */ +void SMO::Train(int learner_typeid, const Dataset* dataset_in) { + index_t i,j; + /* General initializations */ dataset_ = dataset_in; - matrix_.Alias(dataset_->matrix()); - - n_data_ = matrix_.n_cols(); + datamatrix_.Alias(dataset_->matrix()); + n_data_ = datamatrix_.n_cols(); + budget_ = min(budget_, n_data_); - alpha_.Init(n_data_); - - alpha_.SetZero(); - sum_alpha_ = 0; - - error_.Init(n_data_); - error_.SetZero(); + bias_ = 0.0; + n_sv_ = 0; - /* calculate kernel_cache_sign_: [k_ij* y_i* y_j] */ - CalcKernels_(); - while (num_changed > 0 || examine_all) { // TODO: other stopping criteria + /* learners initialization */ + LearnersInit_(learner_typeid); + + alpha_status_.Init(n_alpha_); + for (i=0; iterminate + if (stop_condition == 1) // optimality reached + break; + else if (stop_condition == 2) {// max num of iterations exceeded + fprintf(stderr, "Max iterations (%d) exceeded !!!\n", MAX_NUM_ITER); break; } - - /* for every 100 iterations, do budget shrink */ - if (n_iter % 100 == 0 && budget_ < n_data_) { - MinHeap alphas; - - alphas.Init(); - for (index_t i = 0; i < n_data_; i++) { - alphas.Put(-alpha_[i], i); - } - - for (index_t i = 0; i < budget_; i++) { - alphas.Pop(); - } - - while (alphas.size() > 0) { - alpha_[alphas.Pop()] = 0; - } - - sum_alpha_ = 0; - - for (index_t i = 0; i < n_data_; i++) { - if (!IsBound_(alpha_[i])) { - error_[i] = CalculateError_(i); - } else { - error_[i] = 0; - } - sum_alpha_ += alpha_[i]; - } - } - } } /** * SMO training iterations -* -* @param: indicator: whether all the +* +* @return: stopping condition id */ template -index_t SMO::TrainIteration_(bool examine_all) { - index_t num_changed = 0; - - for (index_t i = 0; i < n_data_; i++) { - if ((examine_all || !IsBound_(alpha_[i])) && TryChange_(i)) { - num_changed++; +int SMO::TrainIteration_() { + ct_iter_ ++; + index_t i,j; + if (WorkingSetSelection_(i,j) == true) { + ReconstructGradient_(learner_typeid_); // reconstruct the whole gradient + n_active_ = 1; + if (WorkingSetSelection_(i,j) == true) { // optimality reached + return 1; + } + else { + ct_shrinking_ = 1; // do shrinking in the next iteration + return 0; } } - - return num_changed; + else if (ct_iter_ >= MAX_NUM_ITER) { // max num of iterations exceeded + return 2; + } + else{ // update gradient and alphas, and continue iterations + UpdatingGradientAlpha_(i, j); + return 0; + } } -/* Try to find working set (maximal violating pair) */ +/** +* Try to find a working set (i,j) +* +* @param: working set (i, j) +* +* @return: indicator of whether the optimal solution is reached (true:reached) +*/ template -bool SMO::TryChange_(index_t j) { - double error_j = Error_(j); /* -y_j g_j^* -thresh */ - double rj = error_j * GetLabelSign_(j); - - VERBOSE_GOT_HERE(0); - - if (!( (rj < -SMO_TOLERANCE && alpha_[j] < c_) - ||(rj > SMO_TOLERANCE && alpha_[j] > 0) )) { - return false; /* nothing to change */ +bool SMO::WorkingSetSelection_(index_t &out_i, index_t &out_j) { + double grad_max = -INFINITY; + double grad_min = INFINITY; + int idx_i = -1; + int idx_j = -1; + + // Find i using maximal violating pair scheme + index_t t; + for (t=0; t= grad_max) { // y==1 + grad_max = grad_[t]; + idx_i = t; + } + } + else { // y[t] == -1 + if (!IsLowerBounded(t)) // t\in I_up, y==-1: y[t]alpha[t] <= 0 + if (-grad_[t] >= grad_max) { // y==-1 + grad_max = -grad_[t]; + idx_i = t; + } + } } + out_i = idx_i; // i found - /* first try the one we suspect to have the largest yield */ - - if (error_j != 0) { - index_t i = -1; - - double diff_max = 0; - - /* find the max(abs(y_i g_i^*)) */ - for (index_t k = 0; k < n_data_; k++) { - if (!IsBound_(alpha_[k])) { /* if 0 < alpha_[k] < c_ */ - double error_k = error_[k]; - double diff_k = fabs(error_k - error_j); /* abs(y_j g_j^* - y_k g_k^*) */ - if (unlikely(diff_k > diff_max)) { - diff_max = diff_k; - i = k; - } + /* Find j using maximal violating pair scheme (1st order approximation of obj func) */ + if (wss_ == 1) { + for (t=0; t= 0 + if (grad_[t] <= grad_min) { // y==1 + grad_min = grad_[t]; + idx_j = t; + } + } + else { // y[t] == -1 + if (!IsUpperBounded(t)) // t\in I_down, y==-1: y[t]alpha[t] >= -C + if (-grad_[t] <= grad_min) { // y==-1 + grad_min = -grad_[t]; + idx_j = t; + } } } - if (i != -1 && TakeStep_(i, j, error_j)) { - return true; + out_j = idx_j; // i found + } + /* Find j using 2nd order working set selection scheme; need to calc kernels, but faster convergence */ + else if (wss_ == 2) { + double K_ii = CalcKernelValue_(out_i, out_i); + double opt_gain_max = -INFINITY; + double grad_diff; + double quad_kernel; + double opt_gain; + for (t=0; t= 0 + // calculate grad_min for Stopping Criterion + if (grad_[t] <= grad_min) // y==1 + grad_min = grad_[t]; + // find j + grad_diff = grad_max - grad_[t]; // max(y_i*grad_i) - y_t*grad_t + if (grad_diff > 0) { + quad_kernel = K_ii + K_tt - 2 * K_it; + if (quad_kernel > 0) // for positive definite kernels + opt_gain = ( grad_diff * grad_diff ) / quad_kernel; // actually ../2*quad_kernel + else // handle non-positive definite kernels + opt_gain = ( grad_diff * grad_diff ) / TAU; + } + if (opt_gain > opt_gain_max) { + idx_j = t; + opt_gain_max = opt_gain; + } + } + } + else { // y[t] == -1 + if (!IsUpperBounded(t)) {// t\in I_down, y==-1: y[t]alpha[t] >= -C + // calculate grad_min for Stopping Criterion + if (-grad_[t] <= grad_min) // y==-1 + grad_min = -grad_[t]; + // find j + grad_diff = grad_max + grad_[t]; // max(y_i*grad_i) - y_t*grad_t + if (grad_diff > 0) { + quad_kernel = K_ii + K_tt - 2 * K_it; + if (quad_kernel > 0) // for positive definite kernels + opt_gain = ( grad_diff * grad_diff ) / quad_kernel; // actually ../2*quad_kernel + else // handle non-positive definite kernels + opt_gain = ( grad_diff * grad_diff ) / TAU; + } + if (opt_gain > opt_gain_max) { + idx_j = t; + opt_gain_max = opt_gain; + } + } + } } } - - VERBOSE_GOT_HERE(0); - /* try searching through non-bound examples */ - index_t start_i = rand() % n_data_; - index_t i = start_i; - - do { - //if (alpha_[i] != 0 && TakeStep_(i, j, error_j)) { - if (!IsBound_(alpha_[i]) && TakeStep_(i, j, error_j)) { - return true; - } - i = (i + 1) % n_data_; - } while (i != start_i); - - VERBOSE_GOT_HERE(0); - /* try searching through all examples */ - start_i = rand() % n_data_; - i = start_i; - - do { - if (IsBound_(alpha_[i]) && TakeStep_(i, j, error_j)) { - return true; - } - i = (i + 1) % n_data_; - } while (i != start_i); + out_j = idx_j; // j found + + // Stopping Criterion check + if (grad_max - grad_min <= SMO_OPT_TOLERANCE) + return true; // optimality reached return false; } -/* search direction, update gradient */ +/** +* Search direction; Update gradient and alphas +* +* @param: a working set (i,j) found by working set selection +* +*/ template -bool SMO::TakeStep_(index_t i, index_t j, double error_j) { - if (i == j) { - VERBOSE_GOT_HERE(0); - return false; - } +void SMO::UpdatingGradientAlpha_(index_t i, index_t j) { + index_t t; - int yi = GetLabelSign_(i); - int yj = GetLabelSign_(j); - double alpha_i = alpha_[i]; - double alpha_j = alpha_[j]; - double delta_thresh; - double l; - double u; - int s = (yi == yj) ? 1 : -1; - double error_i = Error_(i); - double r; - double budget_upper_bound; - - if (s < 0) { - DEBUG_ASSERT(s == -1); - r = alpha_j - alpha_i; /* target values are not equal */ - //budget_upper_bound = (budget_*c_ - sum_alpha_ + 2*alpha_j) / 2; - double gamma = alpha_i - alpha_j; - budget_upper_bound = (gamma - budget_*c_ + sum_alpha_ - alpha_i - alpha_j) / (-2); - } else { - r = alpha_j + alpha_i - c_; // target values are equal - budget_upper_bound = DBL_MAX; - } - - l = math::ClampNonNegative(r); - u = min(c_ + math::ClampNonPositive(r), budget_upper_bound); - - if (l >= u - SMO_TOLERANCE) { - // TODO: might put in some tolerance - VERBOSE_MSG(0, "l=%f, u=%f, r=%f, c_=%f, s=%f", l, u, r, c_, s); - VERBOSE_GOT_HERE(0); - return false; - } + double a_i = alpha_[i]; + double a_j = alpha_[j]; + int y_i = y_[i]; + int y_j = y_[j]; + double C_i = GetC_(i); // can be Cp (for y==1) or Cn (for y==-1) + double C_j = GetC_(j); /* cached kernel values */ - double kii = EvalKernel_(i, i); - double kij = EvalKernel_(i, j); - double kjj = EvalKernel_(j, j); - // second derivative of the objective function - double eta = +2*kij - kii - kjj; + double K_ii, K_ij, K_jj; + K_ii = CalcKernelValue_(i, i); + K_ij = CalcKernelValue_(i, j); + K_jj = CalcKernelValue_(j, j); + - VERBOSE_MSG(0, "kij=%f, kii=%f, kjj=%f", kij, kii, kjj); + double first_order_diff = y_i * grad_[i] - y_j * grad_[j]; + double second_order_diff = K_ii + K_jj - 2 * K_ij; + if (second_order_diff < 0) + second_order_diff = TAU; + double newton_step = first_order_diff / second_order_diff; - /* update alpha_j */ - if (likely(eta < 0)) { - VERBOSE_MSG(0, "Common case"); - alpha_j = alpha_[j] - yj * (error_i - error_j) / eta; - alpha_j = FixAlpha_(math::ClampRange(alpha_j, l, u)); - } else { - VERBOSE_MSG(0, "Uncommon case"); - //abort(); - double c1 = eta/2; - double c2 = yj * (error_i - error_j) - eta * alpha_j; - double objlower = c1*l*l + c2*l; - double objupper = c1*u*u + c2*u; - - if (objlower > objupper + SMO_EPS) { - alpha_j = l; - } else if (objlower < objupper - SMO_EPS) { - alpha_j = u; - } else { - alpha_j = alpha_[j]; - } - DEBUG_ASSERT(alpha_j == FixAlpha_(alpha_j)); + double step_B, step_A; + if (y_i == 1) { + step_B = C_i - a_i; + } + else { // y_i == -1 + step_B = a_i; + } + if (y_j == 1) { + step_A = a_j; + } + else { // y_j == -1 + step_A = C_j - a_j; + } + double min_step_temp = min(step_B, step_A); + double min_step = min(min_step_temp, newton_step); + + // Update alphas + alpha_[i] = a_i + y_i * min_step; + alpha_[j] = a_j - y_j * min_step; + + // Update gradient + for (t=0; t c_ - SMO_ZERO) { - double t = alpha_i - c_; - alpha_j += s * t; - alpha_i = c_; + // Update gradient_bar + bool ub_i = IsUpperBounded(i); + bool ub_j = IsUpperBounded(j); + if( ub_i != IsUpperBounded(i) ) { + if(ub_i) + for(t=0; t -double SMO::Evaluate_(index_t i) const { - // TODO: This only handles linear - Vector kernel_values; - double summation = 0; +void SMO::CalcBias_() { + double b; + index_t n_free = 0; + double ub = INFINITY, lb = -INFINITY, sum_free = 0; + for (index_t i=0; i0) + b = - sum_free / n_free; + else + b = - (ub + lb) / 2; - return (summation - thresh_); + bias_ = b; } /* Get SVM results:coefficients, number and indecies of SVs @@ -476,19 +628,36 @@ double SMO::Evaluate_(index_t i) const { * @param: sample indices of the training (sub)set in the total training set * @param: support vector coefficients: alpha*y * @param: bool indicators FOR THE TRAINING SET: is/isn't a support vector +* */ template void SMO::GetSVM(ArrayList &dataset_index, ArrayList &coef, ArrayList &sv_indicator) { - for (index_t i = 0; i < n_data_; i++) { - if (alpha_[i] != 0) { /* support vectors */ - *coef.AddBack() = alpha_[i] * GetLabelSign_(i); - sv_indicator[dataset_index[i]] = true; - n_sv_++; + if (learner_typeid_ == 0) {// SVM_C + for (index_t i = 0; i < n_data_; i++) { + if (alpha_[i] >= SMO_ALPHA_ZERO) { // support vectors found + coef.AddBackItem( alpha_[i] * y_[i] ); + sv_indicator[dataset_index[i]] = true; + n_sv_++; + } + else { + coef.AddBackItem(0); + } } - else { - *coef.AddBack() = 0; + } + else if (learner_typeid_ == 1) {// SVM_R + for (index_t i = 0; i < n_data_; i++) { + double alpha_diff = -alpha_[i] + alpha_[i+n_data_]; // alpha_i^* - alpha_i + if (fabs(alpha_diff) >= SMO_ALPHA_ZERO) { // support vectors found + coef.AddBackItem(alpha_diff); + sv_indicator[dataset_index[i]] = true; + n_sv_++; + } + else { + coef.AddBackItem(0); + } } } } #endif + diff --git a/fastlib/u/houyang/svm/svm.h b/fastlib/u/houyang/svm/svm.h index 4a52e751fd..02c98b2d5e 100644 --- a/fastlib/u/houyang/svm/svm.h +++ b/fastlib/u/houyang/svm/svm.h @@ -3,8 +3,8 @@ * * @file svm.h * - * This head file contains functions for performing multiclass SVM - * classification. One-vs-One method is employed. + * This head file contains functions for performing SVM training and prediction + * Supported SVM learner type:SVM_C, SVM_R, SVM_DE * * @see smo.h */ @@ -86,12 +86,13 @@ class SVMRBFKernel { */ template class SVM { + private: /** - * Type id of the learner: - * 0:SVM Classification (svc); - * 1:SVM Regression (svr); - * 2:SVM density estimation (svde); + * Type id of the SVM learner: + * 0:SVM Classification (svm_c); + * 1:SVM Regression (svm_r); + * 2:SVM density estimation (svm_de); * Developers may add more learner types if necessary */ int learner_typeid_; @@ -99,19 +100,26 @@ class SVM { Need to train num_classes_*(num_classes_-1)/2 binary models */ struct SVM_MODELS { /* bias term in each binary model */ - double thresh_; + double bias_; /* all coefficients of the binary dataset, not necessarily thoes of SVs */ - ArrayList bi_coef_; + ArrayList coef_; }; ArrayList models_; /* list of labels, double type, but may be converted to integers. e.g. [0.0,1.0,2.0] for a 3-class dataset */ ArrayList train_labels_list_; + /* array of label indices, after grouping. e.g. [c1[0,5,6,7,10,13,17],c2[1,2,4,8,9],c3[...]]*/ + ArrayList train_labels_index_; + /* counted number of label for each class. e.g. [7,5,8]*/ + ArrayList train_labels_ct_; + /* start positions of each classes in the training label list. e.g. [0,7,12] */ + ArrayList train_labels_startpos_; /* total set of support vectors and their coefficients */ Matrix sv_; Matrix sv_coef_; + ArrayList trainset_sv_indicator_; /* total number of support vectors */ index_t total_num_sv_; @@ -122,16 +130,25 @@ class SVM { /* counted number of support vectors for each class */ ArrayList sv_list_ct_; - /* SVM parameters, same for every binary model */ - struct SVM_PARAMETERS { + /* SVM parameters */ + struct PARAMETERS { TKernel kernel_; String kernelname_; int kerneltypeid_; - double c_; int b_; + double C_; + // for SVM_C of unbalanced data + double Cp_; // C for y==1 + double Cn_; // C for y==-1 + // for SVM_R + double epsilon_; + // working set selection scheme, 1 for 1st order expansion; 2 for 2nd order expansion + double wss_; }; - SVM_PARAMETERS param_; + PARAMETERS param_; + /* number of data samples */ + index_t n_data_; /* number of classes in the training set */ int num_classes_; /* number of binary models to be trained, i.e. num_classes_*(num_classes_-1)/2 */ @@ -140,14 +157,25 @@ class SVM { public: typedef TKernel Kernel; +friend class SMO; - void Init(int learner_typeid, const Dataset& dataset, int n_classes, datanode *module); - void InitTrain(int learner_typeid, const Dataset& dataset, int n_classes, datanode *module); - void SaveModel(String modelfilename); - void LoadModel(Dataset* testset, String modelfilename); - double Predict(const Vector& vector); - void BatchPredict(Dataset* testset, String testlabelfilename); - void LoadModelBatchPredict(Dataset* testset, String modelfilename, String testlabelfilename); + void Init(int learner_typeid, const Dataset& dataset, datanode *module); + void InitTrain(int learner_typeid, const Dataset& dataset, datanode *module); + + double Predict(int learner_typeid, const Vector& vector); + void BatchPredict(int learner_typeid, Dataset& testset, String predictedvalue_filename); + void LoadModelBatchPredict(int learner_typeid, Dataset& testset, String model_filename, String predictedvalue_filename); + + private: + void SVM_C_Train_(int learner_typeid, const Dataset& dataset, datanode *module); + void SVM_R_Train_(int learner_typeid, const Dataset& dataset, datanode *module); + void SVM_DE_Train_(int learner_typeid, const Dataset& dataset, datanode *module); + double SVM_C_Predict_(const Vector& vector); + double SVM_R_Predict_(const Vector& vector); + double SVM_DE_Predict_(const Vector& vector); + + void SaveModel_(int learner_typeid, String model_filename); + void LoadModel_(int learner_typeid, String model_filename); }; /** @@ -158,104 +186,157 @@ class SVM { * @param: module name */ template -void SVM::Init(int learner_typeid, const Dataset& dataset, int n_classes, datanode *module){ +void SVM::Init(int learner_typeid, const Dataset& dataset, datanode *module){ learner_typeid_ = learner_typeid; + + n_data_ = dataset.n_points(); + // # of features == # of row - 1, exclude the last row (for labels) + num_features_ = dataset.n_features()-1; + // # of classes of the training set + num_classes_ = dataset.n_labels(); + + train_labels_list_.Init(); + train_labels_index_.Init(); + train_labels_ct_.Init(); + train_labels_startpos_.Init(); + + if (learner_typeid == 0) { /* for multiclass SVM classificatioin*/ + num_models_ = num_classes_ * (num_classes_-1) / 2; + sv_list_startpos_.Init(num_classes_); + sv_list_ct_.Init(num_classes_); + } + else { /* for other SVM learners */ + num_classes_ = 2; // dummy #, only meaningful in SaveModel and LoadModel + + num_models_ = 1; + sv_list_startpos_.Init(); + sv_list_ct_.Init(); + } + models_.Init(); sv_index_.Init(); total_num_sv_ = 0; + /* bool indicators FOR THE TRAINING SET: is/isn't a support vector */ + /* Note: it has the same index as the training !!! */ + trainset_sv_indicator_.Init(n_data_); + for (index_t i=0; i -void SVM::InitTrain(int learner_typeid, const Dataset& dataset, int n_classes, datanode *module) { - Init(learner_typeid, dataset, n_classes, module); - /* # of features == # of rows in data matrix, since last row in dataset is for discrete labels or continuous values */ - num_features_ = dataset.n_features()-1; - - /* bool indicators FOR THE TRAINING SET: is/isn't a support vector */ - /* Note: it has the same index as the training !!! */ - ArrayList trainset_sv_indicator; - trainset_sv_indicator.Init(dataset.n_points()); - for (index_t i=0; i::InitTrain(int learner_typeid, const Dataset& dataset, datanode *module) { + Init(learner_typeid, dataset, module); - /* array of label indices, after grouping. e.g. [c1[0,5,6,7,10,13,17],c2[1,2,4,8,9],c3[...]]*/ - ArrayList train_labels_index; - /* counted number of label for each class. e.g. [7,5,8]*/ - ArrayList train_labels_ct; - /* start positions of each classes in the training label list. e.g. [0,7,12] */ - ArrayList train_labels_startpos; - /* Group labels, split the training dataset for training bi-class SVM classifiers */ - dataset.GetLabels(train_labels_list_, train_labels_index, train_labels_ct, train_labels_startpos); + if (learner_typeid == 0) { // Multiclass SVM Clssification + SVM_C_Train_(learner_typeid, dataset, module); + } + else if (learner_typeid == 1) { // SVM Regression + SVM_R_Train_(learner_typeid, dataset, module); + } + else if (learner_typeid == 2) { // SVM Density Estimation + SVM_DE_Train_(learner_typeid, dataset, module); + } + + /* Save models to file "svm_model" */ + SaveModel_(learner_typeid, "svm_model"); // TODO: param_req, and for CV mode + // TODO: calculate training error +} - /* Train n_classes*(n_classes-1)/2 binary class(labels:-1, 1) models using SMO */ + +/** +* Training for Multiclass SVM Clssification, using One-vs-One method +* +* @param: type id of the learner +* @param: training set +* @param: number of classes of the training set +* @param: module name +*/ +template +void SVM::SVM_C_Train_(int learner_typeid, const Dataset& dataset, datanode *module) { + num_classes_ = dataset.n_labels(); + /* Group labels, split the training dataset for training bi-class SVM classifiers */ + dataset.GetLabels(train_labels_list_, train_labels_index_, train_labels_ct_, train_labels_startpos_); + + /* Train num_classes*(num_classes-1)/2 binary class(labels:-1, 1) models using SMO */ index_t ct = 0; - index_t i; index_t j; - for (i = 0; i < n_classes; i++) { - for (j = i+1; j < n_classes; j++) { + index_t i, j; + for (i = 0; i < num_classes_; i++) { + for (j = i+1; j < num_classes_; j++) { models_.AddBack(); SMO smo; - /* Initialize parameters c_, budget_, alpha_, error_, thresh_ */ - smo.Init(param_.c_, param_.b_); + /* Initialize SMO parameters */ + ArrayList param_feed_db; + param_feed_db.Init(); + param_feed_db.AddBackItem(param_.b_); + param_feed_db.AddBackItem(param_.Cp_); + param_feed_db.AddBackItem(param_.Cn_); + param_feed_db.AddBackItem(param_.wss_); + smo.InitPara(learner_typeid, param_feed_db); + /* Initialize kernel */ smo.kernel().Init(fx_submodule(module, "kernel", "kernel")); /* Construct dataset consists of two classes i and j (reassign labels 1 and -1) */ Dataset dataset_bi; dataset_bi.InitBlank(); dataset_bi.info().Init(); - dataset_bi.matrix().Init(num_features_+1, train_labels_ct[i]+train_labels_ct[j]); + dataset_bi.matrix().Init(num_features_+1, train_labels_ct_[i]+train_labels_ct_[j]); ArrayList dataset_bi_index; - dataset_bi_index.Init(train_labels_ct[i]+train_labels_ct[j]); - for (index_t m = 0; m < train_labels_ct[i]; m++) { + dataset_bi_index.Init(train_labels_ct_[i]+train_labels_ct_[j]); + for (index_t m = 0; m < train_labels_ct_[i]; m++) { Vector source, dest; dataset_bi.matrix().MakeColumnVector(m, &dest); - dataset.matrix().MakeColumnVector(train_labels_index[train_labels_startpos[i]+m], &source); + dataset.matrix().MakeColumnVector(train_labels_index_[train_labels_startpos_[i]+m], &source); dest.CopyValues(source); /* last row for labels 1 */ dataset_bi.matrix().set(num_features_, m, 1); - dataset_bi_index[m] = train_labels_index[train_labels_startpos[i]+m]; + dataset_bi_index[m] = train_labels_index_[train_labels_startpos_[i]+m]; } - for (index_t n = 0; n < train_labels_ct[j]; n++) { + for (index_t n = 0; n < train_labels_ct_[j]; n++) { Vector source, dest; - dataset_bi.matrix().MakeColumnVector(n+train_labels_ct[i], &dest); - dataset.matrix().MakeColumnVector(train_labels_index[train_labels_startpos[j]+n], &source); + dataset_bi.matrix().MakeColumnVector(n+train_labels_ct_[i], &dest); + dataset.matrix().MakeColumnVector(train_labels_index_[train_labels_startpos_[j]+n], &source); dest.CopyValues(source); /* last row for labels -1 */ - dataset_bi.matrix().set(num_features_, n+train_labels_ct[i], -1); - dataset_bi_index[n+train_labels_ct[i]] = train_labels_index[train_labels_startpos[j]+n]; + dataset_bi.matrix().set(num_features_, n+train_labels_ct_[i], -1); + dataset_bi_index[n+train_labels_ct_[i]] = train_labels_index_[train_labels_startpos_[j]+n]; } - /* 2-classes SVM training using SMO */ - smo.Train(&dataset_bi); + /* 2-classes SVM training */ + smo.Train(learner_typeid, &dataset_bi); /* Get the trained bi-class model */ - models_[ct].thresh_ = smo.threshold(); - models_[ct].bi_coef_.Init(); - smo.GetSVM(dataset_bi_index, models_[ct].bi_coef_, trainset_sv_indicator); + models_[ct].bias_ = smo.Bias(); + models_[ct].coef_.Init(); + smo.GetSVM(dataset_bi_index, models_[ct].coef_, trainset_sv_indicator_); ct++; } @@ -264,12 +345,13 @@ void SVM::InitTrain(int learner_typeid, const Dataset& dataset, int n_c /* Get total set of SVs from all the binary models */ index_t k; sv_list_startpos_[0] = 0; - total_num_sv_ = 0; - for (i = 0; i < n_classes; i++) { + + for (i = 0; i < num_classes_; i++) { ct = 0; - for (j = 0; j < train_labels_ct[i]; j++) { - if (trainset_sv_indicator[ train_labels_index[train_labels_startpos[i]+j] ]) { - *sv_index_.AddBack() = train_labels_index[train_labels_startpos[i]+j]; + for (j = 0; j < train_labels_ct_[i]; j++) { + if (trainset_sv_indicator_[ train_labels_index_[train_labels_startpos_[i]+j] ]) { + //*sv_index_.AddBack() = train_labels_index_[train_labels_startpos_[i]+j]; + sv_index_.AddBackItem( train_labels_index_[train_labels_startpos_[i]+j] ); total_num_sv_++; ct++; } @@ -287,77 +369,339 @@ void SVM::InitTrain(int learner_typeid, const Dataset& dataset, int n_c dest.CopyValues(source); } /* Get the matrix sv_coef_ which stores the coefficients of all sets of SVs */ - /* i.e. models_[x].bi_coef_ -> sv_coef_ */ + /* i.e. models_[x].coef_ -> sv_coef_ */ index_t ct_model = 0; index_t ct_bi_cv; index_t p; - sv_coef_.Init(n_classes-1, total_num_sv_); + sv_coef_.Init(num_classes_-1, total_num_sv_); sv_coef_.SetZero(); - for (i = 0; i < n_classes; i++) { - for (j = i+1; j < n_classes; j++) { + for (i = 0; i < num_classes_; i++) { + for (j = i+1; j < num_classes_; j++) { ct_bi_cv = 0; p = sv_list_startpos_[i]; - for (k = 0; k < train_labels_ct[i]; k++) { - if (trainset_sv_indicator[ train_labels_index[train_labels_startpos[i]+k] ]) { - sv_coef_.set(j-1, p++, models_[ct_model].bi_coef_[ct_bi_cv]); + for (k = 0; k < train_labels_ct_[i]; k++) { + if (trainset_sv_indicator_[ train_labels_index_[train_labels_startpos_[i]+k] ]) { + sv_coef_.set(j-1, p++, models_[ct_model].coef_[ct_bi_cv]); ct_bi_cv ++; } } p = sv_list_startpos_[j]; - for (k = 0; k < train_labels_ct[j]; k++) { - if (trainset_sv_indicator[ train_labels_index[train_labels_startpos[j]+k] ]) { - sv_coef_.set(i, p++, models_[ct_model].bi_coef_[ct_bi_cv]); + for (k = 0; k < train_labels_ct_[j]; k++) { + if (trainset_sv_indicator_[ train_labels_index_[train_labels_startpos_[j]+k] ]) { + sv_coef_.set(i, p++, models_[ct_model].coef_[ct_bi_cv]); ct_bi_cv ++; } } ct_model++; } } - /* Save models to file "svm_model" */ - SaveModel("svm_model"); // TODO: param_req, and for CV mode - // TODO: calculate training error } /** -* Save multiclass SVM model to a text file +* Training for SVM Regression * +* @param: type id of the learner +* @param: training set +* @param: module name +*/ +template +void SVM::SVM_R_Train_(int learner_typeid, const Dataset& dataset, datanode *module) { + index_t i; + ArrayList dataset_index; + dataset_index.Init(n_data_); + for (i=0; i smo; + /* Initialize SMO parameters */ + ArrayList param_feed_db; + param_feed_db.Init(); + param_feed_db.AddBackItem(param_.b_); + param_feed_db.AddBackItem(param_.C_); + param_feed_db.AddBackItem(param_.epsilon_); + param_feed_db.AddBackItem(param_.wss_); + smo.InitPara(learner_typeid, param_feed_db); + /* Initialize kernel */ + smo.kernel().Init(fx_submodule(module, "kernel", "kernel")); + /* SVM_R Training */ + smo.Train(learner_typeid, &dataset); + + /* Get the trained model */ + models_[0].bias_ = smo.Bias(); + models_[0].coef_.Init(); + smo.GetSVM(dataset_index, models_[0].coef_, trainset_sv_indicator_); + + /* Get index list of support vectors */ + for (i = 0; i < n_data_; i++) { + if (trainset_sv_indicator_[i]) { + sv_index_.AddBackItem(i); + total_num_sv_++; + } + } + + /* Get support vecotors and coefficients */ + sv_.Init(num_features_, total_num_sv_); + for (i = 0; i < total_num_sv_; i++) { + Vector source, dest; + sv_.MakeColumnVector(i, &dest); + /* last row of dataset is for labels */ + dataset.matrix().MakeColumnSubvector(sv_index_[i], 0, num_features_, &source); + dest.CopyValues(source); + } + sv_coef_.Init(1, total_num_sv_); + for (i = 0; i < total_num_sv_; i++) { + sv_coef_.set(0, i, models_[0].coef_[i]); + } + +} + +/** +* Training for SVM Density Estimation +* +* @param: type id of the learner +* @param: training set +* @param: module name +*/ +template +void SVM::SVM_DE_Train_(int learner_typeid, const Dataset& dataset, datanode *module) { + +} + + +/** +* SVM prediction for one testing vector +* +* @param: type id of the learner +* @param: testing vector +* +* @return: predited value +*/ +template +double SVM::Predict(int learner_typeid, const Vector& datum) { + double predicted_value; + if (learner_typeid == 0) { // Multiclass SVM Clssification + predicted_value = SVM_C_Predict_(datum); + } + else if (learner_typeid == 1) { // SVM Regression + predicted_value = SVM_R_Predict_(datum); + } + else if (learner_typeid == 2) { // SVM Density Estimation + predicted_value = SVM_DE_Predict_(datum); + } + return predicted_value; +} + +/** +* Multiclass SVM classification for one testing vector +* +* @param: testing vector +* +* @return: a label (double-type-integer, e.g. 1.0, 2.0, 3.0) +*/ +template +double SVM::SVM_C_Predict_(const Vector& datum) { + index_t i, j, k; + ArrayList keval; + keval.Init(total_num_sv_); + for (i = 0; i < total_num_sv_; i++) { + Vector support_vector_i; + sv_.MakeColumnVector(i, &support_vector_i); + keval[i] = param_.kernel_.Eval(datum, support_vector_i); + } + ArrayList values; + values.Init(num_models_); + index_t ct = 0; + for (i = 0; i < num_classes_; i++) { + for (j = i+1; j < num_classes_; j++) { + double sum = 0; + for(k = 0; k < sv_list_ct_[i]; k++) { + sum += sv_coef_.get(j-1, sv_list_startpos_[i]+k) * keval[sv_list_startpos_[i]+k]; + } + for(k = 0; k < sv_list_ct_[j]; k++) { + sum += sv_coef_.get(i, sv_list_startpos_[j]+k) * keval[sv_list_startpos_[j]+k]; + } + sum += models_[ct].bias_; + values[ct] = sum; + ct++; + } + } + + ArrayList vote; + vote.Init(num_classes_); + for (i = 0; i < num_classes_; i++) { + vote[i] = 0; + } + ct = 0; + for (i = 0; i < num_classes_; i++) { + for (j = i+1; j < num_classes_; j++) { + if(values[ct] > 0.0) { // label 1 in bi-classifiers (for i=...) + vote[i] = vote[i] + 1; + } + else { // label -1 in bi-classifiers (for j=...) + vote[j] = vote[j] + 1; + } + ct++; + } + } + index_t vote_max_idx = 0; + for (i = 1; i < num_classes_; i++) { + if (vote[i] >= vote[vote_max_idx]) { + vote_max_idx = i; + } + } + return train_labels_list_[vote_max_idx]; +} + +/** +* SVM Regression Prediction for one testing vector +* +* @param: testing vector +* +* @return: predicted regression value +*/ +template +double SVM::SVM_R_Predict_(const Vector& datum) { + index_t i; + double sum = 0.0; + for (i = 0; i < total_num_sv_; i++) { + Vector support_vector_i; + sv_.MakeColumnVector(i, &support_vector_i); + sum += sv_coef_.get(0, i) * param_.kernel_.Eval(datum, support_vector_i); + } + sum += models_[0].bias_; + return sum; +} + +/** +* SVM Density Estimation Prediction for one testing vector +* +* @param: testing vector +* +* @return: estimated density value +*/ +template +double SVM::SVM_DE_Predict_(const Vector& datum) { + return 0.0; +} + + + +/** +* Online batch classification for multiple testing vectors. No need to load model file, +* since models are already in RAM. +* +* Note: for test set, if no true test labels provided, just put some dummy labels +* (e.g. all -1) in the last row of testset +* +* @param: type id of the learner +* @param: testing set +* @param: file name of the testing data +*/ +template +void SVM::BatchPredict(int learner_typeid, Dataset& testset, String predictedvalue_filename) { + FILE *fp = fopen(predictedvalue_filename, "w"); + if (fp == NULL) { + fprintf(stderr, "Cannot save predicted values to file!"); + return; + } + //index_t err_ct = 0; + num_features_ = testset.n_features()-1; + for (index_t i = 0; i < testset.n_points(); i++) { + Vector testvec; + testset.matrix().MakeColumnSubvector(i, 0, num_features_, &testvec); + double predictedvalue = Predict(learner_typeid, testvec); + //if (testlabel != testset.matrix().get(num_features_, i)) + // err_ct++; + /* save predicted values to file*/ + fprintf(fp, "%f\n", predictedvalue); + } + fclose(fp); + /* calculate testing error */ + //fprintf( stderr, "\n*** %d out of %d misclassified ***\n", err_ct, testset.n_points() ); + //fprintf( stderr, "*** Testing error is %f ***\n", double(err_ct)/double(testset.n_points()) ); + //fprintf( stderr, "*** Results are save in \"%s\" ***\n\n", predictedvalue_filename.c_str()); +} + +/** +* Load models from a file, and perform offline batch classification for multiple testing vectors +* +* @param: type id of the learner +* @param: testing set +* @param: name of the model file +* @param: name of the file to store classified labels +*/ +template +void SVM::LoadModelBatchPredict(int learner_typeid, Dataset& testset, String model_filename, String predictedvalue_filename) { + LoadModel_(learner_typeid, model_filename); + BatchPredict(learner_typeid, testset, predictedvalue_filename); +} + + +/** +* Save SVM model to a text file +* +* @param: type id of the learner * @param: name of the model file */ // TODO: use XML template -void SVM::SaveModel(String modelfilename) { - FILE *fp = fopen(modelfilename, "w"); +void SVM::SaveModel_(int learner_typeid, String model_filename) { + FILE *fp = fopen(model_filename, "w"); if (fp == NULL) { fprintf(stderr, "Cannot save trained model to file!"); return; } index_t i, j; - fprintf(fp, "svm_type svm_c\n"); // TODO: svm-mu, svm-regression... - fprintf(fp, "num_classes %d\n", num_classes_); // TODO: only for svm_c + if (learner_typeid == 0) { // for SVM_C + fprintf(fp, "svm_type SVM_C\n"); + fprintf(fp, "total_num_sv %d\n", total_num_sv_); + fprintf(fp, "num_classes %d\n", num_classes_); + // save labels + fprintf(fp, "labels "); + for (i = 0; i < num_classes_; i++) + fprintf(fp, "%f ", train_labels_list_[i]); + fprintf(fp, "\n"); + // save support vector info + fprintf(fp, "sv_list_startpos "); + for (i =0; i < num_classes_; i++) + fprintf(fp, "%d ", sv_list_startpos_[i]); + fprintf(fp, "\n"); + fprintf(fp, "sv_list_ct "); + for (i =0; i < num_classes_; i++) + fprintf(fp, "%d ", sv_list_ct_[i]); + fprintf(fp, "\n"); + } + else if (learner_typeid == 1) { // for SVM_R + fprintf(fp, "svm_type SVM_R\n"); + fprintf(fp, "total_num_sv %d\n", total_num_sv_); + fprintf(fp, "sv_index "); + for (i = 0; i < total_num_sv_; i++) + fprintf(fp, "%d ", sv_index_[i]); + fprintf(fp, "\n"); + } + else if (learner_typeid == 2) { // for SVM_DE + fprintf(fp, "svm_type SVM_DE\n"); + fprintf(fp, "total_num_sv %d\n", total_num_sv_); + fprintf(fp, "sv_index "); + for (i = 0; i < total_num_sv_; i++) + fprintf(fp, "%d ", sv_index_[i]); + fprintf(fp, "\n"); + } + + // save kernel parameters fprintf(fp, "kernel_name %s\n", param_.kernelname_.c_str()); fprintf(fp, "kernel_typeid %d\n", param_.kerneltypeid_); - /* save kernel parameters */ param_.kernel_.SaveParam(fp); - fprintf(fp, "total_num_sv %d\n", total_num_sv_); - fprintf(fp, "labels "); - for (i = 0; i < num_classes_; i++) - fprintf(fp, "%f ", train_labels_list_[i]); - fprintf(fp, "\n"); - /* save models */ - fprintf(fp, "thresholds "); + + // save models: bias, coefficients and support vectors + fprintf(fp, "bias "); for (i = 0; i < num_models_; i++) - fprintf(fp, "%f ", models_[i].thresh_); + fprintf(fp, "%f ", models_[i].bias_); fprintf(fp, "\n"); - fprintf(fp, "sv_list_startpos "); - for (i =0; i < num_classes_; i++) - fprintf(fp, "%d ", sv_list_startpos_[i]); - fprintf(fp, "\n"); - fprintf(fp, "sv_list_ct "); - for (i =0; i < num_classes_; i++) - fprintf(fp, "%d ", sv_list_ct_[i]); - fprintf(fp, "\n"); - /* save coefficients and support vectors */ + fprintf(fp, "SV_coefs\n"); for (i = 0; i < total_num_sv_; i++) { for (j = 0; j < num_classes_-1; j++) { @@ -365,6 +709,7 @@ void SVM::SaveModel(String modelfilename) { } fprintf(fp, "\n"); } + fprintf(fp, "SVs\n"); for (i = 0; i < total_num_sv_; i++) { for (j = 0; j < num_features_; j++) { // n_rows-1 @@ -374,20 +719,23 @@ void SVM::SaveModel(String modelfilename) { } fclose(fp); } + /** * Load SVM model file * +* @param: type id of the learner * @param: name of the model file */ // TODO: use XML template -void SVM::LoadModel(Dataset* testset, String modelfilename) { - /* Init */ - train_labels_list_.Init(num_classes_); - num_features_ = testset->n_features() - 1; +void SVM::LoadModel_(int learner_typeid, String model_filename) { + if (learner_typeid == 0) {// SVM_C + train_labels_list_.Destruct(); + train_labels_list_.Init(num_classes_); // get labels list from the model file + } /* load model file */ - FILE *fp = fopen(modelfilename, "r"); + FILE *fp = fopen(model_filename, "r"); if (fp == NULL) { fprintf(stderr, "Cannot open SVM model file!"); return; @@ -396,44 +744,30 @@ void SVM::LoadModel(Dataset* testset, String modelfilename) { int i, j; int temp_d; double temp_f; for (i = 0; i < num_models_; i++) { models_.AddBack(); - models_[i].bi_coef_.Init(); + models_[i].coef_.Init(); } while (1) { fscanf(fp,"%80s",cmd); if(strcmp(cmd,"svm_type")==0) { - fscanf(fp,"%80s",cmd); - if(strcmp(cmd,"svm_c")==0) { - fprintf(stderr, "SVM_C\n"); - } - } - else if (strcmp(cmd, "num_classes")==0) { - fscanf(fp,"%d",&num_classes_); - } - else if (strcmp(cmd, "kernel_name")==0) { - fscanf(fp,"%80s",param_.kernelname_.c_str()); - } - else if (strcmp(cmd, "kernel_typeid")==0) { - fscanf(fp,"%d",¶m_.kerneltypeid_); - } - else if (strcmp(cmd, "sigma")==0) { - fscanf(fp,"%lf",¶m_.kernel_.kpara_[0]); /* for gaussian kernels only */ - } - else if (strcmp(cmd, "gamma")==0) { - fscanf(fp,"%lf",¶m_.kernel_.kpara_[1]); /* for gaussian kernels only */ + fscanf(fp,"%80s", cmd); + if (strcmp(cmd,"SVM_C")==0) + learner_typeid_ = 0; + else if (strcmp(cmd,"SVM_R")==0) + learner_typeid_ = 1; + else if (strcmp(cmd,"SVM_DE")==0) + learner_typeid_ = 2; } else if (strcmp(cmd, "total_num_sv")==0) { fscanf(fp,"%d",&total_num_sv_); } + // for SVM_C + else if (strcmp(cmd, "num_classes")==0) { + fscanf(fp,"%d",&num_classes_); + } else if (strcmp(cmd, "labels")==0) { for (i=0; i::LoadModel(Dataset* testset, String modelfilename) { fscanf(fp,"%d",&temp_d); sv_list_ct_[i]= temp_d; } + } + // for SVM_R + else if (strcmp(cmd, "sv_index")==0) { + for ( i= 0; i < total_num_sv_; i++) { + fscanf(fp,"%d",&temp_d); + sv_index_.AddBackItem(temp_d); + } + } + // load kernel info + else if (strcmp(cmd, "kernel_name")==0) { + fscanf(fp,"%80s",param_.kernelname_.c_str()); + } + else if (strcmp(cmd, "kernel_typeid")==0) { + fscanf(fp,"%d",¶m_.kerneltypeid_); + } + else if (strcmp(cmd, "sigma")==0) { + fscanf(fp,"%lf",¶m_.kernel_.kpara_[0]); /* for gaussian kernels only */ + } + else if (strcmp(cmd, "gamma")==0) { + fscanf(fp,"%lf",¶m_.kernel_.kpara_[1]); /* for gaussian kernels only */ + } + // load bias + else if (strcmp(cmd, "bias")==0) { + for ( i= 0; i < num_models_; i++) { + fscanf(fp,"%lf",&temp_f); + models_[i].bias_= temp_f; + } break; } } + + // load coefficients and support vectors sv_coef_.Init(num_classes_-1, total_num_sv_); sv_coef_.SetZero(); sv_.Init(num_features_, total_num_sv_); - while (1) { fscanf(fp,"%80s",cmd); if (strcmp(cmd, "SV_coefs")==0) { @@ -477,113 +839,4 @@ void SVM::LoadModel(Dataset* testset, String modelfilename) { fclose(fp); } -/** -* Multiclass SVM classification for one testing vector -* -* @param: testing vector -* -* @return: a label (double-type-integer, e.g. 1.0, 2.0, 3.0) -*/ -template -double SVM::Predict(const Vector& datum) { - index_t i, j, k; - ArrayList keval; - keval.Init(total_num_sv_); - for (i = 0; i < total_num_sv_; i++) { - Vector support_vector_i; - sv_.MakeColumnVector(i, &support_vector_i); - keval[i] = param_.kernel_.Eval(datum, support_vector_i); - } - ArrayList values; - values.Init(num_models_); - index_t ct = 0; - for (i = 0; i < num_classes_; i++) { - for (j = i+1; j < num_classes_; j++) { - double sum = 0; - for(k = 0; k < sv_list_ct_[i]; k++) { - sum += sv_coef_.get(j-1, sv_list_startpos_[i]+k) * keval[sv_list_startpos_[i]+k]; - } - for(k = 0; k < sv_list_ct_[j]; k++) { - sum += sv_coef_.get(i, sv_list_startpos_[j]+k) * keval[sv_list_startpos_[j]+k]; - } - sum -= models_[ct].thresh_; - values[ct] = sum; - ct++; - } - } - - ArrayList vote; - vote.Init(num_classes_); - for (i = 0; i < num_classes_; i++) { - vote[i] = 0; - } - ct = 0; - for (i = 0; i < num_classes_; i++) { - for (j = i+1; j < num_classes_; j++) { - if(values[ct] > 0.0) { // label 1 in bi-classifiers (for i=...) - ++vote[i]; - } - else { // label -1 in bi-classifiers (for j=...) - ++vote[j]; - } - ct++; - } - } - index_t vote_max_idx = 0; - for (i = 1; i < num_classes_; i++) { - if (vote[i] >= vote[vote_max_idx]) { - vote_max_idx = i; - } - } - return train_labels_list_[vote_max_idx]; -} - -/** -* Online batch classification for multiple testing vectors. No need to load model file, -* since models are already in RAM. -* -* Note: for test set, if no true test labels provided, just put some dummy labels -* (e.g. all -1) in the last row of testset -* -* @param: testing set -* @param: file name of the testing data -*/ -template -void SVM::BatchPredict(Dataset* testset, String testlablefilename) { - FILE *fp = fopen(testlablefilename, "w"); - if (fp == NULL) { - fprintf(stderr, "Cannot save test labels to file!"); - return; - } - index_t err_ct = 0; - num_features_ = testset->n_features()-1; - for (index_t i = 0; i < testset->n_points(); i++) { - Vector testvec; - testset->matrix().MakeColumnSubvector(i, 0, num_features_, &testvec); - int testlabel = int(Predict(testvec)); - if (testlabel != testset->matrix().get(num_features_, i)) - err_ct++; - /* save classified labels to file*/ - fprintf(fp, "%d\n", testlabel); - } - fclose(fp); - /* calculate testing error */ - fprintf( stderr, "\n*** %d out of %d misclassified ***\n", err_ct, testset->n_points() ); - fprintf( stderr, "*** Testing error is %f ***\n", double(err_ct)/double(testset->n_points()) ); - fprintf( stderr, "*** Results are save in \"%s\" ***\n\n", testlablefilename.c_str()); -} - -/** -* Load models from a file, and perform offline batch classification for multiple testing vectors -* -* @param: testing set -* @param: name of the model file -* @param: name of the file to store classified labels -*/ -template -void SVM::LoadModelBatchPredict(Dataset* testset, String modelfilename, String testlabelfilename) { - LoadModel(testset, modelfilename); - BatchPredict(testset, testlabelfilename); -} - #endif diff --git a/fastlib/u/houyang/svm/svm_main.cc b/fastlib/u/houyang/svm/svm_main.cc index 73a9e8d1ef..3d3c1819d7 100644 --- a/fastlib/u/houyang/svm/svm_main.cc +++ b/fastlib/u/houyang/svm/svm_main.cc @@ -3,10 +3,10 @@ * * @file svm_main.cc * - * This file contains main routines for performing multiclass - * SVM classification (SVC, one-vs-one method is employed) and - * SVM regression (epsilon-insensitive loss i.e. epsilon-SVR; and - * automatic adjustment of epsilon, i.e. mu-SVR). + * This file contains main routines for performing + * 0. multiclass SVM classification (one-vs-one method is employed). + * 1. SVM regression (epsilon-insensitive loss i.e. epsilon-SVR). + * 2. SVM density estimation (one-class SVM) * * It provides four modes: * "cv": cross validation; @@ -155,10 +155,10 @@ int LoadData(Dataset* dataset, String datafilename){ } if (fx_param_bool(NULL, "normalize", 1)) { - fprintf(stderr, "Normalizing\n"); + fprintf(stderr, "Normalizing...\n"); DoSvmNormalize(dataset); } else { - fprintf(stderr, "Skipping normalization\n"); + fprintf(stderr, "Skipping normalization...\n"); } return 1; } @@ -178,13 +178,13 @@ int main(int argc, char *argv[]) { String learner_name = fx_param_str_req(NULL,"learner_name"); int learner_typeid; - if (learner_name == "svc") { // Support Vector Classfication + if (learner_name == "svm_c") { // Support Vector Classfication learner_typeid = 0; } - else if (learner_name == "svr") { // Support Vector Regression + else if (learner_name == "svm_r") { // Support Vector Regression learner_typeid = 1; } - else if (learner_name == "svde") { // Support Vector Density Estimation + else if (learner_name == "svm_de") { // Support Vector Density Estimation learner_typeid = 2; } else { @@ -234,7 +234,7 @@ int main(int argc, char *argv[]) { if (kernel == "linear") { SVM svm; - svm.InitTrain(learner_typeid, trainset, trainset.n_labels(), svm_module); + svm.InitTrain(learner_typeid, trainset, svm_module); /* training and testing, thus no need to load model from file */ if (mode=="train_test"){ fprintf(stderr, "SVM Predicting... \n"); @@ -242,12 +242,12 @@ int main(int argc, char *argv[]) { Dataset testset; if (LoadData(&testset, "test_data") == 0) // TODO:param_req return 1; - svm.BatchPredict(&testset, "testlabels"); + svm.BatchPredict(learner_typeid, testset, "predicted_values"); } } else if (kernel == "gaussian") { SVM svm; - svm.InitTrain(learner_typeid, trainset, trainset.n_labels(), svm_module); + svm.InitTrain(learner_typeid, trainset, svm_module); /* training and testing, thus no need to load model from file */ if (mode=="train_test"){ fprintf(stderr, "SVM Predicting... \n"); @@ -255,7 +255,7 @@ int main(int argc, char *argv[]) { Dataset testset; if (LoadData(&testset, "test_data") == 0) // TODO:param_req return 1; - svm.BatchPredict(&testset, "testlabels"); // TODO:param_req + svm.BatchPredict(learner_typeid, testset, "predicted_values"); // TODO:param_req } } } @@ -268,18 +268,18 @@ int main(int argc, char *argv[]) { if (LoadData(&testset, "test_data") == 0) // TODO:param_req return 1; - /* Begin Classification */ + /* Begin Prediction */ datanode *svm_module = fx_submodule(fx_root, NULL, "svm"); if (kernel == "linear") { SVM svm; - svm.Init(learner_typeid, testset, testset.n_labels(), svm_module); // TODO:n_labels() -> num_classes_ - svm.LoadModelBatchPredict(&testset, "svm_model", "testlabels"); // TODO:param_req + svm.Init(learner_typeid, testset, svm_module); + svm.LoadModelBatchPredict(learner_typeid, testset, "svm_model", "predicted_values"); // TODO:param_req } else if (kernel == "gaussian") { SVM svm; - svm.Init(learner_typeid, testset, testset.n_labels(), svm_module); // TODO:n_labels() -> num_classes_ - svm.LoadModelBatchPredict(&testset, "svm_model", "testlabels"); // TODO:param_req + svm.Init(learner_typeid, testset, svm_module); + svm.LoadModelBatchPredict(learner_typeid, testset, "svm_model", "predicted_values"); // TODO:param_req } } fx_done();