apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New SMO implementation. Support vector machine for regression now functional.

Browse Source
This commit is contained in:
houyang
2008-04-22 14:40:21 +00:00
parent 610e76341b
commit c030200b69
10 changed files with 1276 additions and 642 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastlib/u/houyang/svm/README
+38 -26
View File
@@ -10,18 +10,23 @@ build.py: build scripts
svm_main.cc: contains main function of SVM
svm.h: contains classification/regression/density estimation main routines
smo.h: sequential minimal optimization
# Unit test data:
traindata2.csv: training data for a 2 classes toy problem. 10 samples in each class.
testdata2.csv: testing data for a 2 classes toy problem. 10 samples in each class.
traindata3.csv: training data for a 3 classes toy problem. 10 samples in each class.
testdata3.csv: testing data for a 3 classes toy problem. 10 samples in each class.
c_traindata2.csv: training data for a 2 classes toy problem. 10 samples in each class.
c_testdata2.csv: testing data for a 2 classes toy problem. 10 samples in each class.
c_traindata3.csv: training data for a 3 classes toy problem. 10 samples in each class.
c_testdata3.csv: testing data for a 3 classes toy problem. 10 samples in each class.
r_traindata.csv: training data for regression, 100 samples of a 1-dim sinc function.
r_testdata.csv: testing data for regression, 100 samples of a 1-dim sinc function.
# Generated files:
artificialdata.csv: artificial data if no training/testing data file provided
svm_model: SVM model file
testlabels: classified labels
predicted_values: predicted result
Explanation of arguments:
0. learner_name (required): the name of the support vecotr learner, values: "svc" for classification, "svr" for regression, "svde" for density estimation
0. learner_name (required): the name of the support vecotr learner, values: "svm_c" for classification, "svm_r" for regression, "svm_de" for density estimation
1. mode (required) : the mode of svm_main, values: "cv", "train", "train_test", "test".
2. k_cv (optional): the number of folds for cross validation, only required under "cv" mode.
3. cv_data (optional): file name for cross validation data, only required under "cv" mode.
@@ -29,40 +34,47 @@ Explanation of arguments:
5. test_data (optional): file name for testing data, only required under "test" or "train_test" mode.
6. kernel (required): kernel name, values:"linear", "gaussian".
7. sigma (optional): sigma in the gaussian kernel k(x1,x2)=exp(-(x1-x2)^2/(2sigma^2)), only required when using "guassian" kernel.
8. c (required): the weight that controls compromise between large margins and small margin violations.
9. normalize (optional): where need to do data normalization before training/testing, values: "0" for no normalize, "1" for normalize.
8. c (for SVM_C,optional): the weight (0~1) that controls compromise between large margins and small margin violations. Default value: 10.0.
9. c_p (for SVM_C,optional): the weight (0~1) for the positive class (y==1). Default value: c.
10. c_n (for SVM_C,optional): the weight (0~1) for the negative class (y==-1). Default value: c.
11. epsilon (for SVM_R,optional): the epsilon in SVM regression. Default value: 0.1.
12. wss (optional): working set selection scheme. 1 for 1st order expansion; 2 for 2nd order expansion. Default value: 1.
13. normalize (optional): where need to do data normalization before training/testing, values: "0" for no normalize, "1" for normalize.
Examples:
---------------------------------------------------------------------------------------------------
--------------------------------------------------- SVM Classificaation ------------------------------------------------
a1.Support Vector Classification, Stratified cross validation mode
svm_main --learner_name=svc --mode=cv --k_cv=4 --cv_data=traindata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0
svm_main --learner_name=svc --mode=cv --k_cv=4 --cv_data=traindata3.csv --kernel=linear --c=10 --normalize=0
svm_main --learner_name=svm_c --mode=cv --k_cv=4 --cv_data=c_traindata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0
svm_main --learner_name=svm_c --mode=cv --k_cv=4 --cv_data=c_traindata3.csv --kernel=linear --c=10 --normalize=0
a2.Support Vector Classification, Training mode (model will be saved as "svm_model")
svm_main --learner_name=svc --mode=train --train_data=traindata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0
svm_main --learner_name=svc --mode=train --train_data=traindata3.csv --kernel=linear --c=10 --normalize=0
svm_main --learner_name=svm_c --mode=train --train_data=c_traindata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0
svm_main --learner_name=svm_c --mode=train --train_data=c_traindata3.csv --kernel=linear --c=10 --normalize=0
a3.Support Vector Classification, Training+testing mode
svm_main --learner_name=svc --mode=train_test --train_data=traindata3.csv --test_data=testdata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0
svm_main --learner_name=svc --mode=train_test --train_data=traindata3.csv --test_data=testdata3.csv --kernel=linear --c=10 --normalize=0
svm_main --learner_name=svm_c --mode=train_test --train_data=c_traindata3.csv --test_data=c_testdata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0
svm_main --learner_name=svm_c --mode=train_test --train_data=c_traindata3.csv --test_data=c_testdata3.csv --kernel=linear --c=10 --normalize=0
a4.Support Vector Classification, Testing mode (the model file "svm_model" should exist)
svm_main --learner_name=svc --mode=test --test_data=testdata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0
svm_main --learner_name=svc --mode=test --test_data=testdata3.csv --kernel=linear --c=10 --normalize=0
svm_main --learner_name=svm_c --mode=test --test_data=c_testdata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0
svm_main --learner_name=svm_c --mode=test --test_data=c_testdata3.csv --kernel=linear --c=10 --normalize=0
---------------------------------------------------------------------------------------------------
--------------------------------------------------- SVM Regression -------------------------------------------------------
b1.Support Vector Regression, Cross validation mode
svm_main --learner_name=svc --mode=cv --k_cv=4 --cv_data=traindata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0
svm_main --learner_name=svc --mode=cv --k_cv=4 --cv_data=traindata3.csv --kernel=linear --c=10 --normalize=0
svm_main --learner_name=svm_r --mode=cv --k_cv=4 --cv_data=r_traindata.csv --kernel=gaussian --sigma=0.1 --epsilon=0.1 --c=10 --normalize=0
svm_main --learner_name=svm_r --mode=cv --k_cv=4 --cv_data=r_traindata.csv --kernel=linear --epsilon=0.1 --c=10 --normalize=0
b2.Support Vector Regression, Training mode (model will be saved as "svm_model")
svm_main --learner_name=svc --mode=train --train_data=traindata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0
svm_main --learner_name=svc --mode=train --train_data=traindata3.csv --kernel=linear --c=10 --normalize=0
svm_main --learner_name=svm_r --mode=train --train_data=r_traindata.csv --kernel=gaussian --sigma=0.1 --epsilon=0.1 --c=10 --normalize=0
svm_main --learner_name=svm_r --mode=train --train_data=r_traindata.csv --kernel=linear --epsilon=0.1 --c=10 --normalize=0
b3.Support Vector Regression, Training+testing mode
svm_main --learner_name=svc --mode=train_test --train_data=traindata3.csv --test_data=testdata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0
svm_main --learner_name=svc --mode=train_test --train_data=traindata3.csv --test_data=testdata3.csv --kernel=linear --c=10 --normalize=0
svm_main --learner_name=svm_r --mode=train_test --train_data=r_traindata.csv --test_data=r_testdata.csv --kernel=gaussian --sigma=0.1 --epsilon=0.1 --c=10 --normalize=0
svm_main --learner_name=svm_r --mode=train_test --train_data=r_traindata.csv --test_data=r_testdata.csv --kernel=linear --epsilon=0.1 --c=10 --normalize=0
b4.Support Vector Regression, Testing mode (the model file "svm_model" should exist)
svm_main --learner_name=svc --mode=test --test_data=testdata3.csv --kernel=gaussian --sigma=0.1 --c=10 --normalize=0
svm_main --learner_name=svc --mode=test --test_data=testdata3.csv --kernel=linear --c=10 --normalize=0
svm_main --learner_name=svm_r --mode=test --test_data=r_testdata.csv --kernel=gaussian --sigma=0.1 --epsilon=0.1 --c=10 --normalize=0
svm_main --learner_name=svm_r --mode=test --test_data=r_testdata.csv --kernel=linear --epsilon=0.1 --c=10 --normalize=0
--------------------------------------------------- SVM Density Estimation --------------------------------------------------
c1... TODO
fastlib/u/houyang/svm/testdata2.csv → fastlib/u/houyang/svm/c_testdata2.csv
View File
fastlib/u/houyang/svm/testdata3.csv → fastlib/u/houyang/svm/c_testdata3.csv
View File
fastlib/u/houyang/svm/traindata2.csv → fastlib/u/houyang/svm/c_traindata2.csv
View File
fastlib/u/houyang/svm/traindata3.csv → fastlib/u/houyang/svm/c_traindata3.csv
View File
fastlib/u/houyang/svm/r_testdata.csv
+100
View File
@@ -0,0 +1,100 @@
-10.0000 -0.0544
-9.7980 -0.0372
-9.5960 -0.0178
-9.3939 0.0033
-9.1919 0.0251
-8.9899 0.0469
-8.7879 0.0677
-8.5859 0.0866
-8.3838 0.1029
-8.1818 0.1157
-7.9798 0.1243
-7.7778 0.1282
-7.5758 0.1269
-7.3737 0.1203
-7.1717 0.1082
-6.9697 0.0909
-6.7677 0.0688
-6.5657 0.0425
-6.3636 0.0126
-6.1616 -0.0197
-5.9596 -0.0534
-5.7576 -0.0871
-5.5556 -0.1197
-5.3535 -0.1497
-5.1515 -0.1757
-4.9495 -0.1964
-4.7475 -0.2105
-4.5455 -0.2169
-4.3434 -0.2147
-4.1414 -0.2032
-3.9394 -0.1817
-3.7374 -0.1501
-3.5354 -0.1085
-3.3333 -0.0572
-3.1313 0.0033
-2.9293 0.0719
-2.7273 0.1476
-2.5253 0.2289
-2.3232 0.3142
-2.1212 0.4018
-1.9192 0.4897
-1.7172 0.5761
-1.5152 0.6590
-1.3131 0.7364
-1.1111 0.8066
-0.9091 0.8678
-0.7071 0.9187
-0.5051 0.9580
-0.3030 0.9848
-0.1010 0.9983
0.1010 0.9983
0.3030 0.9848
0.5051 0.9580
0.7071 0.9187
0.9091 0.8678
1.1111 0.8066
1.3131 0.7364
1.5152 0.6590
1.7172 0.5761
1.9192 0.4897
2.1212 0.4018
2.3232 0.3142
2.5253 0.2289
2.7273 0.1476
2.9293 0.0719
3.1313 0.0033
3.3333 -0.0572
3.5354 -0.1085
3.7374 -0.1501
3.9394 -0.1817
4.1414 -0.2032
4.3434 -0.2147
4.5455 -0.2169
4.7475 -0.2105
4.9495 -0.1964
5.1515 -0.1757
5.3535 -0.1497
5.5556 -0.1197
5.7576 -0.0871
5.9596 -0.0534
6.1616 -0.0197
6.3636 0.0126
6.5657 0.0425
6.7677 0.0688
6.9697 0.0909
7.1717 0.1082
7.3737 0.1203
7.5758 0.1269
7.7778 0.1282
7.9798 0.1243
8.1818 0.1157
8.3838 0.1029
8.5859 0.0866
8.7879 0.0677
8.9899 0.0469
9.1919 0.0251
9.3939 0.0033
9.5960 -0.0178
9.7980 -0.0372
10.0000 -0.0544
1 -10.0000 -0.0544
2 -9.7980 -0.0372
3 -9.5960 -0.0178
4 -9.3939 0.0033
5 -9.1919 0.0251
6 -8.9899 0.0469
7 -8.7879 0.0677
8 -8.5859 0.0866
9 -8.3838 0.1029
10 -8.1818 0.1157
11 -7.9798 0.1243
12 -7.7778 0.1282
13 -7.5758 0.1269
14 -7.3737 0.1203
15 -7.1717 0.1082
16 -6.9697 0.0909
17 -6.7677 0.0688
18 -6.5657 0.0425
19 -6.3636 0.0126
20 -6.1616 -0.0197
21 -5.9596 -0.0534
22 -5.7576 -0.0871
23 -5.5556 -0.1197
24 -5.3535 -0.1497
25 -5.1515 -0.1757
26 -4.9495 -0.1964
27 -4.7475 -0.2105
28 -4.5455 -0.2169
29 -4.3434 -0.2147
30 -4.1414 -0.2032
31 -3.9394 -0.1817
32 -3.7374 -0.1501
33 -3.5354 -0.1085
34 -3.3333 -0.0572
35 -3.1313 0.0033
36 -2.9293 0.0719
37 -2.7273 0.1476
38 -2.5253 0.2289
39 -2.3232 0.3142
40 -2.1212 0.4018
41 -1.9192 0.4897
42 -1.7172 0.5761
43 -1.5152 0.6590
44 -1.3131 0.7364
45 -1.1111 0.8066
46 -0.9091 0.8678
47 -0.7071 0.9187
48 -0.5051 0.9580
49 -0.3030 0.9848
50 -0.1010 0.9983
51 0.1010 0.9983
52 0.3030 0.9848
53 0.5051 0.9580
54 0.7071 0.9187
55 0.9091 0.8678
56 1.1111 0.8066
57 1.3131 0.7364
58 1.5152 0.6590
59 1.7172 0.5761
60 1.9192 0.4897
61 2.1212 0.4018
62 2.3232 0.3142
63 2.5253 0.2289
64 2.7273 0.1476
65 2.9293 0.0719
66 3.1313 0.0033
67 3.3333 -0.0572
68 3.5354 -0.1085
69 3.7374 -0.1501
70 3.9394 -0.1817
71 4.1414 -0.2032
72 4.3434 -0.2147
73 4.5455 -0.2169
74 4.7475 -0.2105
75 4.9495 -0.1964
76 5.1515 -0.1757
77 5.3535 -0.1497
78 5.5556 -0.1197
79 5.7576 -0.0871
80 5.9596 -0.0534
81 6.1616 -0.0197
82 6.3636 0.0126
83 6.5657 0.0425
84 6.7677 0.0688
85 6.9697 0.0909
86 7.1717 0.1082
87 7.3737 0.1203
88 7.5758 0.1269
89 7.7778 0.1282
90 7.9798 0.1243
91 8.1818 0.1157
92 8.3838 0.1029
93 8.5859 0.0866
94 8.7879 0.0677
95 8.9899 0.0469
96 9.1919 0.0251
97 9.3939 0.0033
98 9.5960 -0.0178
99 9.7980 -0.0372
100 10.0000 -0.0544
fastlib/u/houyang/svm/r_traindata.csv
+100
View File
@@ -0,0 +1,100 @@
-10.0000, 0.0320
-9.7980, -0.0278
-9.5960, -0.1029
-9.3939, 0.0906
-9.1919, -0.0187
-8.9899, 0.0039
-8.7879, -0.0426
-8.5859, 0.1263
-8.3838, 0.0064
-8.1818, 0.1326
-7.9798, -0.0722
-7.7778, 0.0538
-7.5758, 0.0717
-7.3737, 0.0383
-7.1717, 0.2191
-6.9697, 0.0294
-6.7677, 0.0434
-6.5657, 0.0155
-6.3636, -0.1546
-6.1616, -0.2073
-5.9596, 0.0041
-5.7576, -0.1738
-5.5556, -0.3314
-5.3535, -0.2461
-5.1515, -0.1544
-4.9495, -0.1486
-4.7475, -0.2004
-4.5455, -0.1872
-4.3434, -0.1577
-4.1414, -0.3656
-3.9394, -0.1174
-3.7374, -0.0820
-3.5354, -0.1071
-3.3333, -0.1873
-3.1313, -0.1252
-2.9293, 0.1532
-2.7273, 0.2315
-2.5253, 0.3709
-2.3232, 0.2153
-2.1212, 0.2835
-1.9192, 0.4431
-1.7172, 0.5395
-1.5152, 0.7708
-1.3131, 0.6898
-1.1111, 0.6505
-0.9091, 0.8395
-0.7071, 0.7864
-0.5051, 0.9384
-0.3030, 1.0267
-0.1010, 1.0725
0.1010, 0.9840
0.3030, 0.7686
0.5051, 0.8936
0.7071, 1.0627
0.9091, 0.7831
1.1111, 0.8123
1.3131, 0.8007
1.5152, 0.5919
1.7172, 0.5758
1.9192, 0.5250
2.1212, 0.5198
2.3232, 0.2456
2.5253, 0.3966
2.7273, 0.1221
2.9293, 0.0072
3.1313, -0.0149
3.3333, 0.0280
3.5354, -0.1392
3.7374, -0.1942
3.9394, -0.2429
4.1414, -0.2517
4.3434, -0.0950
4.5455, -0.0775
4.7475, -0.1940
4.9495, -0.2474
5.1515, -0.0379
5.3535, -0.0198
5.5556, -0.1327
5.7576, -0.0131
5.9596, 0.0798
6.1616, -0.0475
6.3636, -0.0202
6.5657, 0.0412
6.7677, 0.1591
6.9697, -0.0203
7.1717, 0.0243
7.3737, 0.1238
7.5758, 0.0023
7.7778, 0.2166
7.9798, 0.3782
8.1818, 0.2474
8.3838, 0.2471
8.5859, 0.2333
8.7879, -0.0430
8.9899, 0.0008
9.1919, 0.0231
9.3939, -0.0013
9.5960, -0.0722
9.7980, 0.0545
10.0000, -0.0563
1 -10.0000 0.0320
2 -9.7980 -0.0278
3 -9.5960 -0.1029
4 -9.3939 0.0906
5 -9.1919 -0.0187
6 -8.9899 0.0039
7 -8.7879 -0.0426
8 -8.5859 0.1263
9 -8.3838 0.0064
10 -8.1818 0.1326
11 -7.9798 -0.0722
12 -7.7778 0.0538
13 -7.5758 0.0717
14 -7.3737 0.0383
15 -7.1717 0.2191
16 -6.9697 0.0294
17 -6.7677 0.0434
18 -6.5657 0.0155
19 -6.3636 -0.1546
20 -6.1616 -0.2073
21 -5.9596 0.0041
22 -5.7576 -0.1738
23 -5.5556 -0.3314
24 -5.3535 -0.2461
25 -5.1515 -0.1544
26 -4.9495 -0.1486
27 -4.7475 -0.2004
28 -4.5455 -0.1872
29 -4.3434 -0.1577
30 -4.1414 -0.3656
31 -3.9394 -0.1174
32 -3.7374 -0.0820
33 -3.5354 -0.1071
34 -3.3333 -0.1873
35 -3.1313 -0.1252
36 -2.9293 0.1532
37 -2.7273 0.2315
38 -2.5253 0.3709
39 -2.3232 0.2153
40 -2.1212 0.2835
41 -1.9192 0.4431
42 -1.7172 0.5395
43 -1.5152 0.7708
44 -1.3131 0.6898
45 -1.1111 0.6505
46 -0.9091 0.8395
47 -0.7071 0.7864
48 -0.5051 0.9384
49 -0.3030 1.0267
50 -0.1010 1.0725
51 0.1010 0.9840
52 0.3030 0.7686
53 0.5051 0.8936
54 0.7071 1.0627
55 0.9091 0.7831
56 1.1111 0.8123
57 1.3131 0.8007
58 1.5152 0.5919
59 1.7172 0.5758
60 1.9192 0.5250
61 2.1212 0.5198
62 2.3232 0.2456
63 2.5253 0.3966
64 2.7273 0.1221
65 2.9293 0.0072
66 3.1313 -0.0149
67 3.3333 0.0280
68 3.5354 -0.1392
69 3.7374 -0.1942
70 3.9394 -0.2429
71 4.1414 -0.2517
72 4.3434 -0.0950
73 4.5455 -0.0775
74 4.7475 -0.1940
75 4.9495 -0.2474
76 5.1515 -0.0379
77 5.3535 -0.0198
78 5.5556 -0.1327
79 5.7576 -0.0131
80 5.9596 0.0798
81 6.1616 -0.0475
82 6.3636 -0.0202
83 6.5657 0.0412
84 6.7677 0.1591
85 6.9697 -0.0203
86 7.1717 0.0243
87 7.3737 0.1238
88 7.5758 0.0023
89 7.7778 0.2166
90 7.9798 0.3782
91 8.1818 0.2474
92 8.3838 0.2471
93 8.5859 0.2333
94 8.7879 -0.0430
95 8.9899 0.0008
96 9.1919 0.0231
97 9.3939 -0.0013
98 9.5960 -0.0722
99 9.7980 0.0545
100 10.0000 -0.0563
fastlib/u/houyang/svm/smo.h
+510 -341
View File
@@ -1,22 +1,38 @@
/**
* @author Garry Boyer, Hua Ouyang
* @author Hua Ouyang
*
* @file smo.h
*
* This head file contains functions for performing Budget Sequential Minimal Optimization (SMO)
* for a bi-class (labels 1 and -1) classification problem.
*
* The algorithms in the following papers are implemented:
*
* Multiclass Support Vector Machine Classification
* 1. SMO and Working set selecting using 1st order expansion
* @ARTICLE{Platt_SMO,
* author = "J. C. Platt",
* title = "{Fast Training of Support Vector Machines using Sequential Minimal Optimization}",
* booktitle = "{Advances in Kernel Methods - Support Vector Learning}",
* year = 1999,
* publisher = "MIT Press"
* }
*
* 2. Shrinkng for SMO
* @ARTICLE{Joachims_SVMLIGHT,
* author = "T. Joachims",
* title = "{Making large-Scale SVM Learning Practical}",
* journal = "{Advances in Kernel Methods - Support Vector Learning}",
* booktitle = "{Advances in Kernel Methods - Support Vector Learning}",
* year = 1999,
* publisher = "MIT Press"
* }
*
* 3. Working set selecting using 2nd order expansion
* @ARTICLE{Fan_JMLR,
* author = "R. Fan, P. Chen, C. Lin",
* title = "{Working Set Selection using Second Order Information for Training Support Vector Machines}",
* journal = "{Jornal of Machine Learning Research}",
* year = 2005
* }
*
* @INPROCEEDINGS{Budge_SVM,
* author = "O. Dekel and Y. Singer",
* title = "{Support Vector Machines on a Budget}",
@@ -33,11 +49,17 @@
#include "fastlib/fastlib.h"
/* TODO: I don't actually want these to be public */
/* but sometimes we should provide freedoms for our advanced users */
const double SMO_ZERO = 1.0e-8;
const double SMO_EPS = 1.0e-4;
const double SMO_TOLERANCE = 1.0e-4;
const double SMO_OPT_TOLERANCE = 1.0e-4;
const index_t MAX_NUM_ITER = 10000;
const index_t NUM_FOR_SHRINKING = 1000;
const double ID_LOWER_BOUNDED = -1;
const double ID_UPPER_BOUNDED = 1;
const double ID_FREE = 0;
const double TAU = 1e-12;
const double SMO_ALPHA_ZERO = 1.0e-4;
template<typename TKernel>
class SMO {
@@ -47,428 +69,558 @@ class SMO {
typedef TKernel Kernel;
private:
Matrix kernel_cache_sign_;
int learner_typeid_;
index_t ct_iter_; /* counter for the number of iterations */
index_t ct_shrinking_; /* counter for doing shrinking */
Kernel kernel_;
const Dataset *dataset_;
index_t n_data_; /* number of data samples */
Matrix matrix_; /* alias for the data matrix */
Matrix datamatrix_; /* alias for the data matrix */
Vector alpha_; /* the alphas, to be optimized */
Vector alpha_status_; /* ID_LOWER_BOUND (-1), ID_UPPER_BOUND (1), ID_FREE (0) */
index_t n_alpha_; /* number of variables to be optimized */
index_t n_active_; /* number of samples in the active set */
ArrayList<int> active_set_;
ArrayList<int> y_;
double bias_;
index_t n_sv_; /* number of support vectors */
Vector error_;
double thresh_;
double c_;
Vector grad_; /* gradient value */
Vector grad_bar_; /* gradient value when treat free variables as 0 */
// parameters
int budget_;
double sum_alpha_;
double Cp_; // C_+, for SVM_C, y==1
double Cn_; // C_-, for SVM_C, y==-1
double epsilon_; // for SVM_R
int wss_; // working set selection scheme, 1 for 1st order expansion; 2 for 2nd order expansion
public:
SMO() {}
~SMO() {}
/**
* Initialization
* Initialization for parameters
*/
void Init(double c_in, int budget_in) {
c_ = c_in;
budget_ = budget_in;
thresh_ = 0.0;
n_sv_ = 0;
void InitPara(int learner_typeid, ArrayList<double> &param_) {
// init parameters
budget_ = (int)param_[0];
if (learner_typeid == 0) { // SVM_C
Cp_ = param_[1];
Cn_ = param_[2];
wss_ = (int) param_[3];
}
else if (learner_typeid == 1) { // SVM_R
Cp_ = param_[1];
Cn_ = Cp_;
epsilon_ = param_[2];
wss_ = (int) param_[3];
}
}
void Train(const Dataset* dataset_in);
void Train(int learner_typeid, const Dataset* dataset_in);
Kernel& kernel() {
return kernel_;
}
double threshold() const {
return thresh_;
double Bias() const {
return bias_;
}
//index_t num_sv() const {
// return n_sv_;
//}
void GetSVM(ArrayList<index_t> &dataset_index, ArrayList<double> &coef, ArrayList<bool> &sv_indicator);
private:
index_t TrainIteration_(bool examine_all);
void LearnersInit_(int learner_typeid);
int TrainIteration_();
void ReconstructGradient_(int learner_typeid);
bool TryChange_(index_t j);
void Shrinking_();
bool WorkingSetSelection_(index_t &i, index_t &j);
bool TakeStep_(index_t i, index_t j, double error_j);
void UpdatingGradientAlpha_(index_t i, index_t j);
double FixAlpha_(double alpha) const {
if (alpha < SMO_ZERO) {
alpha = 0;
} else if (alpha > c_ - SMO_ZERO) {
alpha = c_;
}
return alpha;
}
bool IsBound_(double alpha) const {
return alpha <= 0 || alpha >= c_;
}
/* labels: 1 and -1 */
int GetLabelSign_(index_t i) const {
return matrix_.get(matrix_.n_rows()-1, i) == 1 ? 1 : -1;
}
void CalcBias_();
void GetVector_(index_t i, Vector *v) const {
matrix_.MakeColumnSubvector(i, 0, matrix_.n_rows()-1, v);
datamatrix_.MakeColumnSubvector(i, 0, datamatrix_.n_rows()-1, v);
}
double Error_(index_t i) const {
double val;
if (!IsBound_(alpha_[i])) {
val = error_[i];
VERBOSE_MSG(0, "error values %f and %f", error_[i], Evaluate_(i) - GetLabelSign_(i));
} else {
val = CalculateError_(i);
/**
* Instead of C, we use C_+ and C_- to handle unbalanced data
*/
double GetC_(index_t i) {
return (y_[i] > 0 ? Cp_ : Cn_);
}
void UpdateAlphaStatus_(index_t i) {
if (alpha_[i] >= GetC_(i)) {
alpha_status_[i] = ID_UPPER_BOUNDED;
}
else if (alpha_[i] <= 0) {
alpha_status_[i] = ID_LOWER_BOUNDED;
}
else { // 0 < alpha_[i] < C
alpha_status_[i] = ID_FREE;
}
return val;
}
double CalculateError_(index_t i) const {
return Evaluate_(i) - GetLabelSign_(i);
bool IsUpperBounded(index_t i) {
return alpha_status_[i] == ID_UPPER_BOUNDED;
}
bool IsLowerBounded(index_t i) {
return alpha_status_[i] == ID_LOWER_BOUNDED;
}
double Evaluate_(index_t i) const;
double EvalKernel_(index_t i, index_t j) const {
return kernel_cache_sign_.get(i, j) * (GetLabelSign_(i) * GetLabelSign_(j));
}
/**
* Calculate kernel values
*/
void CalcKernels_() {
kernel_cache_sign_.Init(n_data_, n_data_);
fprintf(stderr, "Kernel Start\n");
for (index_t i = 0; i < n_data_; i++) {
for (index_t j = 0; j < n_data_; j++) {
Vector v_i;
GetVector_(i, &v_i);
Vector v_j;
GetVector_(j, &v_j);
double k = kernel_.Eval(v_i, v_j);
kernel_cache_sign_.set(j, i, k * GetLabelSign_(i) * GetLabelSign_(j));
}
}
fprintf(stderr, "Kernel Stop\n");
double CalcKernelValue_(index_t i, index_t j) {
i = i >= n_data_ ? (i-n_data_) : i; // for SVM_R
j = j >= n_data_ ? (j-n_data_) : j;
Vector v_i;
GetVector_(i, &v_i);
Vector v_j;
GetVector_(j, &v_j);
return kernel_.Eval(v_i, v_j);
}
};
/**
* Initialization for different SVM learners
*
* @param: learner type id
*/
template<typename TKernel>
void SMO<TKernel>::LearnersInit_(int learner_typeid) {
index_t i;
learner_typeid_ = learner_typeid;
if (learner_typeid_ == 0) { // SVM_C
n_alpha_ = n_data_;
alpha_.Init(n_alpha_);
alpha_.SetZero();
// initialize gradient
grad_.Init(n_alpha_);
for (i=0; i<n_alpha_; i++) {
grad_[i] = 1;
}
y_.Init(n_alpha_);
for (i = 0; i < n_alpha_; i++) {
y_[i] = datamatrix_.get(datamatrix_.n_rows()-1, i) > 0 ? 1 : -1;
}
}
else if (learner_typeid_ == 1) { // SVM_R
n_alpha_ = 2 * n_data_;
alpha_.Init(2 * n_alpha_);
alpha_.SetZero();
// initialize gradient
grad_.Init(n_alpha_);
y_.Init(n_alpha_);
for (i = 0; i < n_data_; i++) {
y_[i] = 1; // -> alpha_i
y_[i + n_data_] = -1; // -> alpha_i^*
grad_[i] = epsilon_ - datamatrix_.get(datamatrix_.n_rows()-1, i);
grad_[i + n_data_] = epsilon_ + datamatrix_.get(datamatrix_.n_rows()-1, i);
}
}
else if (learner_typeid_ == 2) { // SVM_DE
}
// initialize active set
n_active_ = n_alpha_;
active_set_.Init(n_active_);
for (i=0; i<n_active_; i++) {
active_set_[i] = i;
}
}
/**
* Reconstruct inactive elements of G from G_bar and free variables
*
* @param: learner type id
*/
template<typename TKernel>
void SMO<TKernel>::ReconstructGradient_(int learner_typeid) {
index_t i, j;
if (n_active_ == n_alpha_)
return;
if (learner_typeid == 0) { // SVM_C
for (i=n_active_; i<n_alpha_; i++) {
grad_[i] = grad_bar_[i] + 1;
}
}
else if (learner_typeid == 1) { // SVM_R
for (i=n_active_; i<n_alpha_; i++) {
grad_[i] = grad_bar_[i] + datamatrix_.get(datamatrix_.n_rows()-1, i) - epsilon_;
}
}
for (i=0; i<n_active_; i++) {
if (alpha_status_[i] == ID_FREE) {
for (j=n_active_; j<n_alpha_; j++) {
grad_[j] += alpha_[i] * CalcKernelValue_(i,j);
}
}
}
}
/**
* Do Shrinking
*
*/
template<typename TKernel>
void SMO<TKernel>::Shrinking_() {
}
/**
* Budget SMO training for 2-classes
*
* @param: input 2-classes data matrix with labels (1,-1) in the last row
*/
template<typename TKernel>
void SMO<TKernel>::Train(const Dataset* dataset_in) {
bool examine_all = true;
index_t num_changed = 0;
int n_iter = 0;
/* data-dependent initialization */
void SMO<TKernel>::Train(int learner_typeid, const Dataset* dataset_in) {
index_t i,j;
/* General initializations */
dataset_ = dataset_in;
matrix_.Alias(dataset_->matrix());
n_data_ = matrix_.n_cols();
datamatrix_.Alias(dataset_->matrix());
n_data_ = datamatrix_.n_cols();
budget_ = min(budget_, n_data_);
alpha_.Init(n_data_);
alpha_.SetZero();
sum_alpha_ = 0;
error_.Init(n_data_);
error_.SetZero();
bias_ = 0.0;
n_sv_ = 0;
/* calculate kernel_cache_sign_: [k_ij* y_i* y_j] */
CalcKernels_();
while (num_changed > 0 || examine_all) { // TODO: other stopping criteria
/* learners initialization */
LearnersInit_(learner_typeid);
alpha_status_.Init(n_alpha_);
for (i=0; i<n_alpha_; i++)
UpdateAlphaStatus_(i);
// initialize gradient_bar
grad_bar_.Init(n_alpha_);
grad_bar_.SetZero();
for(i=0; i<n_alpha_; i++) {
if(!IsLowerBounded(i))
{
for(j=0; j<n_alpha_; j++)
grad_[j] += alpha_[i] * CalcKernelValue_(i,j);
if(IsUpperBounded(i))
for(j=0; j<n_alpha_; j++)
grad_bar_[j] += GetC_(i) * CalcKernelValue_(i,j);
}
}
ct_iter_ = 0;
ct_shrinking_ = min(n_data_, NUM_FOR_SHRINKING) + 1;
/* Begin SMO iterations */
int stop_condition = 0;
while (1) {
VERBOSE_GOT_HERE(0);
/* SMO iterations */
num_changed = TrainIteration_(examine_all);
if (examine_all) {
examine_all = false;
} else if (num_changed == 0) {
examine_all = true;
}
/* if exceed the maximum number of iterations, finished */
// TODO: 200....
if (++n_iter == 200) {
fprintf(stderr, "Max iterations %f!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
sum_alpha_);
/* for every min(n_data_, 1000) iterations, do shrinking */
if (--ct_shrinking_ == 0) {
Shrinking_();
ct_shrinking_ = min(n_data_, NUM_FOR_SHRINKING);
}
// Find working set, check stopping criterion, update gradient and alphas
stop_condition = TrainIteration_();
// termination check, stop_condition==1 or 2->terminate
if (stop_condition == 1) // optimality reached
break;
else if (stop_condition == 2) {// max num of iterations exceeded
fprintf(stderr, "Max iterations (%d) exceeded !!!\n", MAX_NUM_ITER);
break;
}
/* for every 100 iterations, do budget shrink */
if (n_iter % 100 == 0 && budget_ < n_data_) {
MinHeap<double, int> alphas;
alphas.Init();
for (index_t i = 0; i < n_data_; i++) {
alphas.Put(-alpha_[i], i);
}
for (index_t i = 0; i < budget_; i++) {
alphas.Pop();
}
while (alphas.size() > 0) {
alpha_[alphas.Pop()] = 0;
}
sum_alpha_ = 0;
for (index_t i = 0; i < n_data_; i++) {
if (!IsBound_(alpha_[i])) {
error_[i] = CalculateError_(i);
} else {
error_[i] = 0;
}
sum_alpha_ += alpha_[i];
}
}
}
}
/**
* SMO training iterations
*
* @param: indicator: whether all the
*
* @return: stopping condition id
*/
template<typename TKernel>
index_t SMO<TKernel>::TrainIteration_(bool examine_all) {
index_t num_changed = 0;
for (index_t i = 0; i < n_data_; i++) {
if ((examine_all || !IsBound_(alpha_[i])) && TryChange_(i)) {
num_changed++;
int SMO<TKernel>::TrainIteration_() {
ct_iter_ ++;
index_t i,j;
if (WorkingSetSelection_(i,j) == true) {
ReconstructGradient_(learner_typeid_); // reconstruct the whole gradient
n_active_ = 1;
if (WorkingSetSelection_(i,j) == true) { // optimality reached
return 1;
}
else {
ct_shrinking_ = 1; // do shrinking in the next iteration
return 0;
}
}
return num_changed;
else if (ct_iter_ >= MAX_NUM_ITER) { // max num of iterations exceeded
return 2;
}
else{ // update gradient and alphas, and continue iterations
UpdatingGradientAlpha_(i, j);
return 0;
}
}
/* Try to find working set (maximal violating pair) */
/**
* Try to find a working set (i,j)
*
* @param: working set (i, j)
*
* @return: indicator of whether the optimal solution is reached (true:reached)
*/
template<typename TKernel>
bool SMO<TKernel>::TryChange_(index_t j) {
double error_j = Error_(j); /* -y_j g_j^* -thresh */
double rj = error_j * GetLabelSign_(j);
VERBOSE_GOT_HERE(0);
if (!( (rj < -SMO_TOLERANCE && alpha_[j] < c_)
||(rj > SMO_TOLERANCE && alpha_[j] > 0) )) {
return false; /* nothing to change */
bool SMO<TKernel>::WorkingSetSelection_(index_t &out_i, index_t &out_j) {
double grad_max = -INFINITY;
double grad_min = INFINITY;
int idx_i = -1;
int idx_j = -1;
// Find i using maximal violating pair scheme
index_t t;
for (t=0; t<n_active_; t++) { // find argmax(y*grad), t\in I_up
if (y_[t] == 1) {
if (!IsUpperBounded(t)) // t\in I_up, y==1: y[t]alpha[t] <= C
if (grad_[t] >= grad_max) { // y==1
grad_max = grad_[t];
idx_i = t;
}
}
else { // y[t] == -1
if (!IsLowerBounded(t)) // t\in I_up, y==-1: y[t]alpha[t] <= 0
if (-grad_[t] >= grad_max) { // y==-1
grad_max = -grad_[t];
idx_i = t;
}
}
}
out_i = idx_i; // i found
/* first try the one we suspect to have the largest yield */
if (error_j != 0) {
index_t i = -1;
double diff_max = 0;
/* find the max(abs(y_i g_i^*)) */
for (index_t k = 0; k < n_data_; k++) {
if (!IsBound_(alpha_[k])) { /* if 0 < alpha_[k] < c_ */
double error_k = error_[k];
double diff_k = fabs(error_k - error_j); /* abs(y_j g_j^* - y_k g_k^*) */
if (unlikely(diff_k > diff_max)) {
diff_max = diff_k;
i = k;
}
/* Find j using maximal violating pair scheme (1st order approximation of obj func) */
if (wss_ == 1) {
for (t=0; t<n_active_; t++) { // find argmin(y*grad), t\in I_down
if (y_[t] == 1) {
if (!IsLowerBounded(t)) // t\in I_down, y==1: y[t]alpha[t] >= 0
if (grad_[t] <= grad_min) { // y==1
grad_min = grad_[t];
idx_j = t;
}
}
else { // y[t] == -1
if (!IsUpperBounded(t)) // t\in I_down, y==-1: y[t]alpha[t] >= -C
if (-grad_[t] <= grad_min) { // y==-1
grad_min = -grad_[t];
idx_j = t;
}
}
}
if (i != -1 && TakeStep_(i, j, error_j)) {
return true;
out_j = idx_j; // i found
}
/* Find j using 2nd order working set selection scheme; need to calc kernels, but faster convergence */
else if (wss_ == 2) {
double K_ii = CalcKernelValue_(out_i, out_i);
double opt_gain_max = -INFINITY;
double grad_diff;
double quad_kernel;
double opt_gain;
for (t=0; t<n_active_; t++) {
double K_it = CalcKernelValue_(out_i, t);
double K_tt = CalcKernelValue_(t, t);
if (y_[t] == 1) {
if (!IsLowerBounded(t)) { // t\in I_down, y==1: y[t]alpha[t] >= 0
// calculate grad_min for Stopping Criterion
if (grad_[t] <= grad_min) // y==1
grad_min = grad_[t];
// find j
grad_diff = grad_max - grad_[t]; // max(y_i*grad_i) - y_t*grad_t
if (grad_diff > 0) {
quad_kernel = K_ii + K_tt - 2 * K_it;
if (quad_kernel > 0) // for positive definite kernels
opt_gain = ( grad_diff * grad_diff ) / quad_kernel; // actually ../2*quad_kernel
else // handle non-positive definite kernels
opt_gain = ( grad_diff * grad_diff ) / TAU;
}
if (opt_gain > opt_gain_max) {
idx_j = t;
opt_gain_max = opt_gain;
}
}
}
else { // y[t] == -1
if (!IsUpperBounded(t)) {// t\in I_down, y==-1: y[t]alpha[t] >= -C
// calculate grad_min for Stopping Criterion
if (-grad_[t] <= grad_min) // y==-1
grad_min = -grad_[t];
// find j
grad_diff = grad_max + grad_[t]; // max(y_i*grad_i) - y_t*grad_t
if (grad_diff > 0) {
quad_kernel = K_ii + K_tt - 2 * K_it;
if (quad_kernel > 0) // for positive definite kernels
opt_gain = ( grad_diff * grad_diff ) / quad_kernel; // actually ../2*quad_kernel
else // handle non-positive definite kernels
opt_gain = ( grad_diff * grad_diff ) / TAU;
}
if (opt_gain > opt_gain_max) {
idx_j = t;
opt_gain_max = opt_gain;
}
}
}
}
}
VERBOSE_GOT_HERE(0);
/* try searching through non-bound examples */
index_t start_i = rand() % n_data_;
index_t i = start_i;
do {
//if (alpha_[i] != 0 && TakeStep_(i, j, error_j)) {
if (!IsBound_(alpha_[i]) && TakeStep_(i, j, error_j)) {
return true;
}
i = (i + 1) % n_data_;
} while (i != start_i);
VERBOSE_GOT_HERE(0);
/* try searching through all examples */
start_i = rand() % n_data_;
i = start_i;
do {
if (IsBound_(alpha_[i]) && TakeStep_(i, j, error_j)) {
return true;
}
i = (i + 1) % n_data_;
} while (i != start_i);
out_j = idx_j; // j found
// Stopping Criterion check
if (grad_max - grad_min <= SMO_OPT_TOLERANCE)
return true; // optimality reached
return false;
}
/* search direction, update gradient */
/**
* Search direction; Update gradient and alphas
*
* @param: a working set (i,j) found by working set selection
*
*/
template<typename TKernel>
bool SMO<TKernel>::TakeStep_(index_t i, index_t j, double error_j) {
if (i == j) {
VERBOSE_GOT_HERE(0);
return false;
}
void SMO<TKernel>::UpdatingGradientAlpha_(index_t i, index_t j) {
index_t t;
int yi = GetLabelSign_(i);
int yj = GetLabelSign_(j);
double alpha_i = alpha_[i];
double alpha_j = alpha_[j];
double delta_thresh;
double l;
double u;
int s = (yi == yj) ? 1 : -1;
double error_i = Error_(i);
double r;
double budget_upper_bound;
if (s < 0) {
DEBUG_ASSERT(s == -1);
r = alpha_j - alpha_i; /* target values are not equal */
//budget_upper_bound = (budget_*c_ - sum_alpha_ + 2*alpha_j) / 2;
double gamma = alpha_i - alpha_j;
budget_upper_bound = (gamma - budget_*c_ + sum_alpha_ - alpha_i - alpha_j) / (-2);
} else {
r = alpha_j + alpha_i - c_; // target values are equal
budget_upper_bound = DBL_MAX;
}
l = math::ClampNonNegative(r);
u = min(c_ + math::ClampNonPositive(r), budget_upper_bound);
if (l >= u - SMO_TOLERANCE) {
// TODO: might put in some tolerance
VERBOSE_MSG(0, "l=%f, u=%f, r=%f, c_=%f, s=%f", l, u, r, c_, s);
VERBOSE_GOT_HERE(0);
return false;
}
double a_i = alpha_[i];
double a_j = alpha_[j];
int y_i = y_[i];
int y_j = y_[j];
double C_i = GetC_(i); // can be Cp (for y==1) or Cn (for y==-1)
double C_j = GetC_(j);
/* cached kernel values */
double kii = EvalKernel_(i, i);
double kij = EvalKernel_(i, j);
double kjj = EvalKernel_(j, j);
// second derivative of the objective function
double eta = +2*kij - kii - kjj;
double K_ii, K_ij, K_jj;
K_ii = CalcKernelValue_(i, i);
K_ij = CalcKernelValue_(i, j);
K_jj = CalcKernelValue_(j, j);
VERBOSE_MSG(0, "kij=%f, kii=%f, kjj=%f", kij, kii, kjj);
double first_order_diff = y_i * grad_[i] - y_j * grad_[j];
double second_order_diff = K_ii + K_jj - 2 * K_ij;
if (second_order_diff < 0)
second_order_diff = TAU;
double newton_step = first_order_diff / second_order_diff;
/* update alpha_j */
if (likely(eta < 0)) {
VERBOSE_MSG(0, "Common case");
alpha_j = alpha_[j] - yj * (error_i - error_j) / eta;
alpha_j = FixAlpha_(math::ClampRange(alpha_j, l, u));
} else {
VERBOSE_MSG(0, "Uncommon case");
//abort();
double c1 = eta/2;
double c2 = yj * (error_i - error_j) - eta * alpha_j;
double objlower = c1*l*l + c2*l;
double objupper = c1*u*u + c2*u;
if (objlower > objupper + SMO_EPS) {
alpha_j = l;
} else if (objlower < objupper - SMO_EPS) {
alpha_j = u;
} else {
alpha_j = alpha_[j];
}
DEBUG_ASSERT(alpha_j == FixAlpha_(alpha_j));
double step_B, step_A;
if (y_i == 1) {
step_B = C_i - a_i;
}
else { // y_i == -1
step_B = a_i;
}
if (y_j == 1) {
step_A = a_j;
}
else { // y_j == -1
step_A = C_j - a_j;
}
double min_step_temp = min(step_B, step_A);
double min_step = min(min_step_temp, newton_step);
// Update alphas
alpha_[i] = a_i + y_i * min_step;
alpha_[j] = a_j - y_j * min_step;
// Update gradient
for (t=0; t<n_active_; t++) {
grad_[t] = grad_[t] + min_step * y_[t] *( CalcKernelValue_(j, t) - CalcKernelValue_(i, t) );
}
alpha_j = FixAlpha_(alpha_j);
// Update alpha active status
UpdateAlphaStatus_(i);
UpdateAlphaStatus_(j);
double delta_alpha_j = alpha_j - alpha_[j];
/* check if there is progress */
if (fabs(delta_alpha_j) < SMO_EPS*(alpha_j + alpha_[j] + SMO_EPS)) {
VERBOSE_GOT_HERE(0);
return false;
}
/* update alpha_i */
alpha_i = alpha_i - (s)*(delta_alpha_j);
if (alpha_i < SMO_ZERO) {
alpha_j += s * alpha_i;
alpha_i = 0;
} else if (alpha_i > c_ - SMO_ZERO) {
double t = alpha_i - c_;
alpha_j += s * t;
alpha_i = c_;
// Update gradient_bar
bool ub_i = IsUpperBounded(i);
bool ub_j = IsUpperBounded(j);
if( ub_i != IsUpperBounded(i) ) {
if(ub_i)
for(t=0; t<n_alpha_; t++)
grad_bar_[t] -= C_i * CalcKernelValue_(i, t);
else
for(t=0; t<n_alpha_; t++)
grad_bar_[t] += C_i * CalcKernelValue_(i, t);
}
double delta_alpha_i = alpha_i - alpha_[i];
/* calculate threshold */
double delta_thresh_i = error_i + yi*delta_alpha_i*kii + yj*delta_alpha_j*kij;
double delta_thresh_j = error_j + yi*delta_alpha_i*kij + yj*delta_alpha_j*kjj;
if (!IsBound_(alpha_i)) {
delta_thresh = delta_thresh_i;
} else if (!IsBound_(alpha_j)) {
delta_thresh = delta_thresh_j;
} else {
delta_thresh = (delta_thresh_i + delta_thresh_j) / 2.0;
if( ub_j != IsUpperBounded(j) ) {
if(ub_j)
for(t=0; t<n_alpha_; t++)
grad_bar_[t] -= C_j * CalcKernelValue_(j, t);
else
for(t=0; t<n_alpha_; t++)
grad_bar_[t] += C_j * CalcKernelValue_(j, t);
}
Vector kernel_i;
Vector kernel_j;
kernel_cache_sign_.MakeColumnVector(i, &kernel_i);
kernel_cache_sign_.MakeColumnVector(j, &kernel_j);
/* update gradient */
for (index_t k = 0; k < n_data_; k++) {
if (likely(k != i) && likely(k != j) && !IsBound_(alpha_[k])) {
error_[k] += (delta_alpha_i*kernel_i[k] + delta_alpha_j*kernel_j[k]) * GetLabelSign_(k) - delta_thresh;
}
}
thresh_ += delta_thresh;
alpha_[i] = alpha_i;
alpha_[j] = alpha_j;
sum_alpha_ += delta_alpha_i + delta_alpha_j;
/* this is only necessary when i or j are not bound, but there is nothing
* wrong with doing this all the time */
error_[i] = 0;
error_[j] = 0;
// Calculate the bias term
CalcBias_();
VERBOSE_GOT_HERE(0);
return true;
}
/**
* Calcualte bias term
*
* @return: the bias
*
*/
template<typename TKernel>
double SMO<TKernel>::Evaluate_(index_t i) const {
// TODO: This only handles linear
Vector kernel_values;
double summation = 0;
void SMO<TKernel>::CalcBias_() {
double b;
index_t n_free = 0;
double ub = INFINITY, lb = -INFINITY, sum_free = 0;
for (index_t i=0; i<n_active_; i++){
double yg = y_[i] * grad_[i];
if (IsUpperBounded(i)) {
if(y_[i] == -1)
ub = min(ub, yg);
else
lb = max(lb, yg);
}
else if (IsLowerBounded(i)) {
if(y_[i] == +1)
ub = min(ub, yg);
else
lb = max(lb, yg);
}
else {
n_free++;
sum_free += yg;
}
}
kernel_cache_sign_.MakeColumnVector(i, &kernel_values);
summation = la::Dot(alpha_, kernel_values) * GetLabelSign_(i);
if(n_free>0)
b = - sum_free / n_free;
else
b = - (ub + lb) / 2;
return (summation - thresh_);
bias_ = b;
}
/* Get SVM results:coefficients, number and indecies of SVs
@@ -476,19 +628,36 @@ double SMO<TKernel>::Evaluate_(index_t i) const {
* @param: sample indices of the training (sub)set in the total training set
* @param: support vector coefficients: alpha*y
* @param: bool indicators FOR THE TRAINING SET: is/isn't a support vector
*
*/
template<typename TKernel>
void SMO<TKernel>::GetSVM(ArrayList<index_t> &dataset_index, ArrayList<double> &coef, ArrayList<bool> &sv_indicator) {
for (index_t i = 0; i < n_data_; i++) {
if (alpha_[i] != 0) { /* support vectors */
*coef.AddBack() = alpha_[i] * GetLabelSign_(i);
sv_indicator[dataset_index[i]] = true;
n_sv_++;
if (learner_typeid_ == 0) {// SVM_C
for (index_t i = 0; i < n_data_; i++) {
if (alpha_[i] >= SMO_ALPHA_ZERO) { // support vectors found
coef.AddBackItem( alpha_[i] * y_[i] );
sv_indicator[dataset_index[i]] = true;
n_sv_++;
}
else {
coef.AddBackItem(0);
}
}
else {
*coef.AddBack() = 0;
}
else if (learner_typeid_ == 1) {// SVM_R
for (index_t i = 0; i < n_data_; i++) {
double alpha_diff = -alpha_[i] + alpha_[i+n_data_]; // alpha_i^* - alpha_i
if (fabs(alpha_diff) >= SMO_ALPHA_ZERO) { // support vectors found
coef.AddBackItem(alpha_diff);
sv_indicator[dataset_index[i]] = true;
n_sv_++;
}
else {
coef.AddBackItem(0);
}
}
}
}
#endif
fastlib/u/houyang/svm/svm.h
+510 -257
View File
@@ -3,8 +3,8 @@
*
* @file svm.h
*
* This head file contains functions for performing multiclass SVM
* classification. One-vs-One method is employed.
* This head file contains functions for performing SVM training and prediction
* Supported SVM learner type:SVM_C, SVM_R, SVM_DE
*
* @see smo.h
*/
@@ -86,12 +86,13 @@ class SVMRBFKernel {
*/
template<typename TKernel>
class SVM {
private:
/**
* Type id of the learner:
* 0:SVM Classification (svc);
* 1:SVM Regression (svr);
* 2:SVM density estimation (svde);
* Type id of the SVM learner:
* 0:SVM Classification (svm_c);
* 1:SVM Regression (svm_r);
* 2:SVM density estimation (svm_de);
* Developers may add more learner types if necessary
*/
int learner_typeid_;
@@ -99,19 +100,26 @@ class SVM {
Need to train num_classes_*(num_classes_-1)/2 binary models */
struct SVM_MODELS {
/* bias term in each binary model */
double thresh_;
double bias_;
/* all coefficients of the binary dataset, not necessarily thoes of SVs */
ArrayList<double> bi_coef_;
ArrayList<double> coef_;
};
ArrayList<SVM_MODELS> models_;
/* list of labels, double type, but may be converted to integers.
e.g. [0.0,1.0,2.0] for a 3-class dataset */
ArrayList<double> train_labels_list_;
/* array of label indices, after grouping. e.g. [c1[0,5,6,7,10,13,17],c2[1,2,4,8,9],c3[...]]*/
ArrayList<index_t> train_labels_index_;
/* counted number of label for each class. e.g. [7,5,8]*/
ArrayList<index_t> train_labels_ct_;
/* start positions of each classes in the training label list. e.g. [0,7,12] */
ArrayList<index_t> train_labels_startpos_;
/* total set of support vectors and their coefficients */
Matrix sv_;
Matrix sv_coef_;
ArrayList<bool> trainset_sv_indicator_;
/* total number of support vectors */
index_t total_num_sv_;
@@ -122,16 +130,25 @@ class SVM {
/* counted number of support vectors for each class */
ArrayList<index_t> sv_list_ct_;
/* SVM parameters, same for every binary model */
struct SVM_PARAMETERS {
/* SVM parameters */
struct PARAMETERS {
TKernel kernel_;
String kernelname_;
int kerneltypeid_;
double c_;
int b_;
double C_;
// for SVM_C of unbalanced data
double Cp_; // C for y==1
double Cn_; // C for y==-1
// for SVM_R
double epsilon_;
// working set selection scheme, 1 for 1st order expansion; 2 for 2nd order expansion
double wss_;
};
SVM_PARAMETERS param_;
PARAMETERS param_;
/* number of data samples */
index_t n_data_;
/* number of classes in the training set */
int num_classes_;
/* number of binary models to be trained, i.e. num_classes_*(num_classes_-1)/2 */
@@ -140,14 +157,25 @@ class SVM {
public:
typedef TKernel Kernel;
friend class SMO<Kernel>;
void Init(int learner_typeid, const Dataset& dataset, int n_classes, datanode *module);
void InitTrain(int learner_typeid, const Dataset& dataset, int n_classes, datanode *module);
void SaveModel(String modelfilename);
void LoadModel(Dataset* testset, String modelfilename);
double Predict(const Vector& vector);
void BatchPredict(Dataset* testset, String testlabelfilename);
void LoadModelBatchPredict(Dataset* testset, String modelfilename, String testlabelfilename);
void Init(int learner_typeid, const Dataset& dataset, datanode *module);
void InitTrain(int learner_typeid, const Dataset& dataset, datanode *module);
double Predict(int learner_typeid, const Vector& vector);
void BatchPredict(int learner_typeid, Dataset& testset, String predictedvalue_filename);
void LoadModelBatchPredict(int learner_typeid, Dataset& testset, String model_filename, String predictedvalue_filename);
private:
void SVM_C_Train_(int learner_typeid, const Dataset& dataset, datanode *module);
void SVM_R_Train_(int learner_typeid, const Dataset& dataset, datanode *module);
void SVM_DE_Train_(int learner_typeid, const Dataset& dataset, datanode *module);
double SVM_C_Predict_(const Vector& vector);
double SVM_R_Predict_(const Vector& vector);
double SVM_DE_Predict_(const Vector& vector);
void SaveModel_(int learner_typeid, String model_filename);
void LoadModel_(int learner_typeid, String model_filename);
};
/**
@@ -158,104 +186,157 @@ class SVM {
* @param: module name
*/
template<typename TKernel>
void SVM<TKernel>::Init(int learner_typeid, const Dataset& dataset, int n_classes, datanode *module){
void SVM<TKernel>::Init(int learner_typeid, const Dataset& dataset, datanode *module){
learner_typeid_ = learner_typeid;
n_data_ = dataset.n_points();
// # of features == # of row - 1, exclude the last row (for labels)
num_features_ = dataset.n_features()-1;
// # of classes of the training set
num_classes_ = dataset.n_labels();
train_labels_list_.Init();
train_labels_index_.Init();
train_labels_ct_.Init();
train_labels_startpos_.Init();
if (learner_typeid == 0) { /* for multiclass SVM classificatioin*/
num_models_ = num_classes_ * (num_classes_-1) / 2;
sv_list_startpos_.Init(num_classes_);
sv_list_ct_.Init(num_classes_);
}
else { /* for other SVM learners */
num_classes_ = 2; // dummy #, only meaningful in SaveModel and LoadModel
num_models_ = 1;
sv_list_startpos_.Init();
sv_list_ct_.Init();
}
models_.Init();
sv_index_.Init();
total_num_sv_ = 0;
/* bool indicators FOR THE TRAINING SET: is/isn't a support vector */
/* Note: it has the same index as the training !!! */
trainset_sv_indicator_.Init(n_data_);
for (index_t i=0; i<n_data_; i++)
trainset_sv_indicator_[i] = false;
param_.kernel_.Init(fx_submodule(module, "kernel", "kernel"));
param_.kernel_.GetName(&param_.kernelname_);
param_.kerneltypeid_ = param_.kernel_.GetTypeId();
// budget parameter, contorls # of support vectors; default: # of data samples (use all)
param_.b_ = fx_param_int(NULL, "b", dataset.n_points());
// working set selection scheme. default: 1st order expansion
param_.wss_ = fx_param_int(NULL, "wss", 1);
param_.c_ = fx_param_double_req(NULL, "c");
/* budget parameter, contorls # of support vectors; default: # of data samples (use all) */
param_.b_ = fx_param_int(module, "b", dataset.n_points()); // TODO: param_req
// the tradeoff parameter "C", default: 10.0
param_.C_ = fx_param_double(NULL, "c", 10.0);
param_.Cp_ = fx_param_double(NULL, "c_p", param_.C_);
param_.Cn_ = fx_param_double(NULL, "c_n", param_.C_);
num_features_ = 0;
sv_list_startpos_.Init(n_classes);
sv_list_ct_.Init(n_classes);
/* for SVM classificatioin*/
num_classes_ = n_classes;
num_models_ = num_classes_ * (num_classes_-1) / 2;
if (learner_typeid == 1) { // for SVM_R only
// the "epsilon", default: 0.1
param_.epsilon_ = fx_param_double(NULL, "epsilon", 0.1);
}
else if (learner_typeid == 2) { // SVM_DE
}
}
/**
* Initialization(data dependent) and training for Multiclass SVM Classifier
* Use One-vs-One, or called All-vs-All method
* Initialization(data dependent) and training for SVM learners
*
* @param: labeled training set
* @param: typeid of the learner
* @param: number of classes (different labels) in the training set
* @param: module name
*/
template<typename TKernel>
void SVM<TKernel>::InitTrain(int learner_typeid, const Dataset& dataset, int n_classes, datanode *module) {
Init(learner_typeid, dataset, n_classes, module);
/* # of features == # of rows in data matrix, since last row in dataset is for discrete labels or continuous values */
num_features_ = dataset.n_features()-1;
/* bool indicators FOR THE TRAINING SET: is/isn't a support vector */
/* Note: it has the same index as the training !!! */
ArrayList<bool> trainset_sv_indicator;
trainset_sv_indicator.Init(dataset.n_points());
for (index_t i=0; i<dataset.n_points(); i++)
trainset_sv_indicator[i] = false;
void SVM<TKernel>::InitTrain(int learner_typeid, const Dataset& dataset, datanode *module) {
Init(learner_typeid, dataset, module);
/* array of label indices, after grouping. e.g. [c1[0,5,6,7,10,13,17],c2[1,2,4,8,9],c3[...]]*/
ArrayList<index_t> train_labels_index;
/* counted number of label for each class. e.g. [7,5,8]*/
ArrayList<index_t> train_labels_ct;
/* start positions of each classes in the training label list. e.g. [0,7,12] */
ArrayList<index_t> train_labels_startpos;
/* Group labels, split the training dataset for training bi-class SVM classifiers */
dataset.GetLabels(train_labels_list_, train_labels_index, train_labels_ct, train_labels_startpos);
if (learner_typeid == 0) { // Multiclass SVM Clssification
SVM_C_Train_(learner_typeid, dataset, module);
}
else if (learner_typeid == 1) { // SVM Regression
SVM_R_Train_(learner_typeid, dataset, module);
}
else if (learner_typeid == 2) { // SVM Density Estimation
SVM_DE_Train_(learner_typeid, dataset, module);
}
/* Save models to file "svm_model" */
SaveModel_(learner_typeid, "svm_model"); // TODO: param_req, and for CV mode
// TODO: calculate training error
}
/* Train n_classes*(n_classes-1)/2 binary class(labels:-1, 1) models using SMO */
/**
* Training for Multiclass SVM Clssification, using One-vs-One method
*
* @param: type id of the learner
* @param: training set
* @param: number of classes of the training set
* @param: module name
*/
template<typename TKernel>
void SVM<TKernel>::SVM_C_Train_(int learner_typeid, const Dataset& dataset, datanode *module) {
num_classes_ = dataset.n_labels();
/* Group labels, split the training dataset for training bi-class SVM classifiers */
dataset.GetLabels(train_labels_list_, train_labels_index_, train_labels_ct_, train_labels_startpos_);
/* Train num_classes*(num_classes-1)/2 binary class(labels:-1, 1) models using SMO */
index_t ct = 0;
index_t i; index_t j;
for (i = 0; i < n_classes; i++) {
for (j = i+1; j < n_classes; j++) {
index_t i, j;
for (i = 0; i < num_classes_; i++) {
for (j = i+1; j < num_classes_; j++) {
models_.AddBack();
SMO<Kernel> smo;
/* Initialize parameters c_, budget_, alpha_, error_, thresh_ */
smo.Init(param_.c_, param_.b_);
/* Initialize SMO parameters */
ArrayList<double> param_feed_db;
param_feed_db.Init();
param_feed_db.AddBackItem(param_.b_);
param_feed_db.AddBackItem(param_.Cp_);
param_feed_db.AddBackItem(param_.Cn_);
param_feed_db.AddBackItem(param_.wss_);
smo.InitPara(learner_typeid, param_feed_db);
/* Initialize kernel */
smo.kernel().Init(fx_submodule(module, "kernel", "kernel"));
/* Construct dataset consists of two classes i and j (reassign labels 1 and -1) */
Dataset dataset_bi;
dataset_bi.InitBlank();
dataset_bi.info().Init();
dataset_bi.matrix().Init(num_features_+1, train_labels_ct[i]+train_labels_ct[j]);
dataset_bi.matrix().Init(num_features_+1, train_labels_ct_[i]+train_labels_ct_[j]);
ArrayList<index_t> dataset_bi_index;
dataset_bi_index.Init(train_labels_ct[i]+train_labels_ct[j]);
for (index_t m = 0; m < train_labels_ct[i]; m++) {
dataset_bi_index.Init(train_labels_ct_[i]+train_labels_ct_[j]);
for (index_t m = 0; m < train_labels_ct_[i]; m++) {
Vector source, dest;
dataset_bi.matrix().MakeColumnVector(m, &dest);
dataset.matrix().MakeColumnVector(train_labels_index[train_labels_startpos[i]+m], &source);
dataset.matrix().MakeColumnVector(train_labels_index_[train_labels_startpos_[i]+m], &source);
dest.CopyValues(source);
/* last row for labels 1 */
dataset_bi.matrix().set(num_features_, m, 1);
dataset_bi_index[m] = train_labels_index[train_labels_startpos[i]+m];
dataset_bi_index[m] = train_labels_index_[train_labels_startpos_[i]+m];
}
for (index_t n = 0; n < train_labels_ct[j]; n++) {
for (index_t n = 0; n < train_labels_ct_[j]; n++) {
Vector source, dest;
dataset_bi.matrix().MakeColumnVector(n+train_labels_ct[i], &dest);
dataset.matrix().MakeColumnVector(train_labels_index[train_labels_startpos[j]+n], &source);
dataset_bi.matrix().MakeColumnVector(n+train_labels_ct_[i], &dest);
dataset.matrix().MakeColumnVector(train_labels_index_[train_labels_startpos_[j]+n], &source);
dest.CopyValues(source);
/* last row for labels -1 */
dataset_bi.matrix().set(num_features_, n+train_labels_ct[i], -1);
dataset_bi_index[n+train_labels_ct[i]] = train_labels_index[train_labels_startpos[j]+n];
dataset_bi.matrix().set(num_features_, n+train_labels_ct_[i], -1);
dataset_bi_index[n+train_labels_ct_[i]] = train_labels_index_[train_labels_startpos_[j]+n];
}
/* 2-classes SVM training using SMO */
smo.Train(&dataset_bi);
/* 2-classes SVM training */
smo.Train(learner_typeid, &dataset_bi);
/* Get the trained bi-class model */
models_[ct].thresh_ = smo.threshold();
models_[ct].bi_coef_.Init();
smo.GetSVM(dataset_bi_index, models_[ct].bi_coef_, trainset_sv_indicator);
models_[ct].bias_ = smo.Bias();
models_[ct].coef_.Init();
smo.GetSVM(dataset_bi_index, models_[ct].coef_, trainset_sv_indicator_);
ct++;
}
@@ -264,12 +345,13 @@ void SVM<TKernel>::InitTrain(int learner_typeid, const Dataset& dataset, int n_c
/* Get total set of SVs from all the binary models */
index_t k;
sv_list_startpos_[0] = 0;
total_num_sv_ = 0;
for (i = 0; i < n_classes; i++) {
for (i = 0; i < num_classes_; i++) {
ct = 0;
for (j = 0; j < train_labels_ct[i]; j++) {
if (trainset_sv_indicator[ train_labels_index[train_labels_startpos[i]+j] ]) {
*sv_index_.AddBack() = train_labels_index[train_labels_startpos[i]+j];
for (j = 0; j < train_labels_ct_[i]; j++) {
if (trainset_sv_indicator_[ train_labels_index_[train_labels_startpos_[i]+j] ]) {
//*sv_index_.AddBack() = train_labels_index_[train_labels_startpos_[i]+j];
sv_index_.AddBackItem( train_labels_index_[train_labels_startpos_[i]+j] );
total_num_sv_++;
ct++;
}
@@ -287,77 +369,339 @@ void SVM<TKernel>::InitTrain(int learner_typeid, const Dataset& dataset, int n_c
dest.CopyValues(source);
}
/* Get the matrix sv_coef_ which stores the coefficients of all sets of SVs */
/* i.e. models_[x].bi_coef_ -> sv_coef_ */
/* i.e. models_[x].coef_ -> sv_coef_ */
index_t ct_model = 0;
index_t ct_bi_cv;
index_t p;
sv_coef_.Init(n_classes-1, total_num_sv_);
sv_coef_.Init(num_classes_-1, total_num_sv_);
sv_coef_.SetZero();
for (i = 0; i < n_classes; i++) {
for (j = i+1; j < n_classes; j++) {
for (i = 0; i < num_classes_; i++) {
for (j = i+1; j < num_classes_; j++) {
ct_bi_cv = 0;
p = sv_list_startpos_[i];
for (k = 0; k < train_labels_ct[i]; k++) {
if (trainset_sv_indicator[ train_labels_index[train_labels_startpos[i]+k] ]) {
sv_coef_.set(j-1, p++, models_[ct_model].bi_coef_[ct_bi_cv]);
for (k = 0; k < train_labels_ct_[i]; k++) {
if (trainset_sv_indicator_[ train_labels_index_[train_labels_startpos_[i]+k] ]) {
sv_coef_.set(j-1, p++, models_[ct_model].coef_[ct_bi_cv]);
ct_bi_cv ++;
}
}
p = sv_list_startpos_[j];
for (k = 0; k < train_labels_ct[j]; k++) {
if (trainset_sv_indicator[ train_labels_index[train_labels_startpos[j]+k] ]) {
sv_coef_.set(i, p++, models_[ct_model].bi_coef_[ct_bi_cv]);
for (k = 0; k < train_labels_ct_[j]; k++) {
if (trainset_sv_indicator_[ train_labels_index_[train_labels_startpos_[j]+k] ]) {
sv_coef_.set(i, p++, models_[ct_model].coef_[ct_bi_cv]);
ct_bi_cv ++;
}
}
ct_model++;
}
}
/* Save models to file "svm_model" */
SaveModel("svm_model"); // TODO: param_req, and for CV mode
// TODO: calculate training error
}
/**
* Save multiclass SVM model to a text file
* Training for SVM Regression
*
* @param: type id of the learner
* @param: training set
* @param: module name
*/
template<typename TKernel>
void SVM<TKernel>::SVM_R_Train_(int learner_typeid, const Dataset& dataset, datanode *module) {
index_t i;
ArrayList<index_t> dataset_index;
dataset_index.Init(n_data_);
for (i=0; i<n_data_; i++)
dataset_index[i] = i;
models_.AddBack();
SMO<Kernel> smo;
/* Initialize SMO parameters */
ArrayList<double> param_feed_db;
param_feed_db.Init();
param_feed_db.AddBackItem(param_.b_);
param_feed_db.AddBackItem(param_.C_);
param_feed_db.AddBackItem(param_.epsilon_);
param_feed_db.AddBackItem(param_.wss_);
smo.InitPara(learner_typeid, param_feed_db);
/* Initialize kernel */
smo.kernel().Init(fx_submodule(module, "kernel", "kernel"));
/* SVM_R Training */
smo.Train(learner_typeid, &dataset);
/* Get the trained model */
models_[0].bias_ = smo.Bias();
models_[0].coef_.Init();
smo.GetSVM(dataset_index, models_[0].coef_, trainset_sv_indicator_);
/* Get index list of support vectors */
for (i = 0; i < n_data_; i++) {
if (trainset_sv_indicator_[i]) {
sv_index_.AddBackItem(i);
total_num_sv_++;
}
}
/* Get support vecotors and coefficients */
sv_.Init(num_features_, total_num_sv_);
for (i = 0; i < total_num_sv_; i++) {
Vector source, dest;
sv_.MakeColumnVector(i, &dest);
/* last row of dataset is for labels */
dataset.matrix().MakeColumnSubvector(sv_index_[i], 0, num_features_, &source);
dest.CopyValues(source);
}
sv_coef_.Init(1, total_num_sv_);
for (i = 0; i < total_num_sv_; i++) {
sv_coef_.set(0, i, models_[0].coef_[i]);
}
}
/**
* Training for SVM Density Estimation
*
* @param: type id of the learner
* @param: training set
* @param: module name
*/
template<typename TKernel>
void SVM<TKernel>::SVM_DE_Train_(int learner_typeid, const Dataset& dataset, datanode *module) {
}
/**
* SVM prediction for one testing vector
*
* @param: type id of the learner
* @param: testing vector
*
* @return: predited value
*/
template<typename TKernel>
double SVM<TKernel>::Predict(int learner_typeid, const Vector& datum) {
double predicted_value;
if (learner_typeid == 0) { // Multiclass SVM Clssification
predicted_value = SVM_C_Predict_(datum);
}
else if (learner_typeid == 1) { // SVM Regression
predicted_value = SVM_R_Predict_(datum);
}
else if (learner_typeid == 2) { // SVM Density Estimation
predicted_value = SVM_DE_Predict_(datum);
}
return predicted_value;
}
/**
* Multiclass SVM classification for one testing vector
*
* @param: testing vector
*
* @return: a label (double-type-integer, e.g. 1.0, 2.0, 3.0)
*/
template<typename TKernel>
double SVM<TKernel>::SVM_C_Predict_(const Vector& datum) {
index_t i, j, k;
ArrayList<double> keval;
keval.Init(total_num_sv_);
for (i = 0; i < total_num_sv_; i++) {
Vector support_vector_i;
sv_.MakeColumnVector(i, &support_vector_i);
keval[i] = param_.kernel_.Eval(datum, support_vector_i);
}
ArrayList<double> values;
values.Init(num_models_);
index_t ct = 0;
for (i = 0; i < num_classes_; i++) {
for (j = i+1; j < num_classes_; j++) {
double sum = 0;
for(k = 0; k < sv_list_ct_[i]; k++) {
sum += sv_coef_.get(j-1, sv_list_startpos_[i]+k) * keval[sv_list_startpos_[i]+k];
}
for(k = 0; k < sv_list_ct_[j]; k++) {
sum += sv_coef_.get(i, sv_list_startpos_[j]+k) * keval[sv_list_startpos_[j]+k];
}
sum += models_[ct].bias_;
values[ct] = sum;
ct++;
}
}
ArrayList<index_t> vote;
vote.Init(num_classes_);
for (i = 0; i < num_classes_; i++) {
vote[i] = 0;
}
ct = 0;
for (i = 0; i < num_classes_; i++) {
for (j = i+1; j < num_classes_; j++) {
if(values[ct] > 0.0) { // label 1 in bi-classifiers (for i=...)
vote[i] = vote[i] + 1;
}
else { // label -1 in bi-classifiers (for j=...)
vote[j] = vote[j] + 1;
}
ct++;
}
}
index_t vote_max_idx = 0;
for (i = 1; i < num_classes_; i++) {
if (vote[i] >= vote[vote_max_idx]) {
vote_max_idx = i;
}
}
return train_labels_list_[vote_max_idx];
}
/**
* SVM Regression Prediction for one testing vector
*
* @param: testing vector
*
* @return: predicted regression value
*/
template<typename TKernel>
double SVM<TKernel>::SVM_R_Predict_(const Vector& datum) {
index_t i;
double sum = 0.0;
for (i = 0; i < total_num_sv_; i++) {
Vector support_vector_i;
sv_.MakeColumnVector(i, &support_vector_i);
sum += sv_coef_.get(0, i) * param_.kernel_.Eval(datum, support_vector_i);
}
sum += models_[0].bias_;
return sum;
}
/**
* SVM Density Estimation Prediction for one testing vector
*
* @param: testing vector
*
* @return: estimated density value
*/
template<typename TKernel>
double SVM<TKernel>::SVM_DE_Predict_(const Vector& datum) {
return 0.0;
}
/**
* Online batch classification for multiple testing vectors. No need to load model file,
* since models are already in RAM.
*
* Note: for test set, if no true test labels provided, just put some dummy labels
* (e.g. all -1) in the last row of testset
*
* @param: type id of the learner
* @param: testing set
* @param: file name of the testing data
*/
template<typename TKernel>
void SVM<TKernel>::BatchPredict(int learner_typeid, Dataset& testset, String predictedvalue_filename) {
FILE *fp = fopen(predictedvalue_filename, "w");
if (fp == NULL) {
fprintf(stderr, "Cannot save predicted values to file!");
return;
}
//index_t err_ct = 0;
num_features_ = testset.n_features()-1;
for (index_t i = 0; i < testset.n_points(); i++) {
Vector testvec;
testset.matrix().MakeColumnSubvector(i, 0, num_features_, &testvec);
double predictedvalue = Predict(learner_typeid, testvec);
//if (testlabel != testset.matrix().get(num_features_, i))
// err_ct++;
/* save predicted values to file*/
fprintf(fp, "%f\n", predictedvalue);
}
fclose(fp);
/* calculate testing error */
//fprintf( stderr, "\n*** %d out of %d misclassified ***\n", err_ct, testset.n_points() );
//fprintf( stderr, "*** Testing error is %f ***\n", double(err_ct)/double(testset.n_points()) );
//fprintf( stderr, "*** Results are save in \"%s\" ***\n\n", predictedvalue_filename.c_str());
}
/**
* Load models from a file, and perform offline batch classification for multiple testing vectors
*
* @param: type id of the learner
* @param: testing set
* @param: name of the model file
* @param: name of the file to store classified labels
*/
template<typename TKernel>
void SVM<TKernel>::LoadModelBatchPredict(int learner_typeid, Dataset& testset, String model_filename, String predictedvalue_filename) {
LoadModel_(learner_typeid, model_filename);
BatchPredict(learner_typeid, testset, predictedvalue_filename);
}
/**
* Save SVM model to a text file
*
* @param: type id of the learner
* @param: name of the model file
*/
// TODO: use XML
template<typename TKernel>
void SVM<TKernel>::SaveModel(String modelfilename) {
FILE *fp = fopen(modelfilename, "w");
void SVM<TKernel>::SaveModel_(int learner_typeid, String model_filename) {
FILE *fp = fopen(model_filename, "w");
if (fp == NULL) {
fprintf(stderr, "Cannot save trained model to file!");
return;
}
index_t i, j;
fprintf(fp, "svm_type svm_c\n"); // TODO: svm-mu, svm-regression...
fprintf(fp, "num_classes %d\n", num_classes_); // TODO: only for svm_c
if (learner_typeid == 0) { // for SVM_C
fprintf(fp, "svm_type SVM_C\n");
fprintf(fp, "total_num_sv %d\n", total_num_sv_);
fprintf(fp, "num_classes %d\n", num_classes_);
// save labels
fprintf(fp, "labels ");
for (i = 0; i < num_classes_; i++)
fprintf(fp, "%f ", train_labels_list_[i]);
fprintf(fp, "\n");
// save support vector info
fprintf(fp, "sv_list_startpos ");
for (i =0; i < num_classes_; i++)
fprintf(fp, "%d ", sv_list_startpos_[i]);
fprintf(fp, "\n");
fprintf(fp, "sv_list_ct ");
for (i =0; i < num_classes_; i++)
fprintf(fp, "%d ", sv_list_ct_[i]);
fprintf(fp, "\n");
}
else if (learner_typeid == 1) { // for SVM_R
fprintf(fp, "svm_type SVM_R\n");
fprintf(fp, "total_num_sv %d\n", total_num_sv_);
fprintf(fp, "sv_index ");
for (i = 0; i < total_num_sv_; i++)
fprintf(fp, "%d ", sv_index_[i]);
fprintf(fp, "\n");
}
else if (learner_typeid == 2) { // for SVM_DE
fprintf(fp, "svm_type SVM_DE\n");
fprintf(fp, "total_num_sv %d\n", total_num_sv_);
fprintf(fp, "sv_index ");
for (i = 0; i < total_num_sv_; i++)
fprintf(fp, "%d ", sv_index_[i]);
fprintf(fp, "\n");
}
// save kernel parameters
fprintf(fp, "kernel_name %s\n", param_.kernelname_.c_str());
fprintf(fp, "kernel_typeid %d\n", param_.kerneltypeid_);
/* save kernel parameters */
param_.kernel_.SaveParam(fp);
fprintf(fp, "total_num_sv %d\n", total_num_sv_);
fprintf(fp, "labels ");
for (i = 0; i < num_classes_; i++)
fprintf(fp, "%f ", train_labels_list_[i]);
fprintf(fp, "\n");
/* save models */
fprintf(fp, "thresholds ");
// save models: bias, coefficients and support vectors
fprintf(fp, "bias ");
for (i = 0; i < num_models_; i++)
fprintf(fp, "%f ", models_[i].thresh_);
fprintf(fp, "%f ", models_[i].bias_);
fprintf(fp, "\n");
fprintf(fp, "sv_list_startpos ");
for (i =0; i < num_classes_; i++)
fprintf(fp, "%d ", sv_list_startpos_[i]);
fprintf(fp, "\n");
fprintf(fp, "sv_list_ct ");
for (i =0; i < num_classes_; i++)
fprintf(fp, "%d ", sv_list_ct_[i]);
fprintf(fp, "\n");
/* save coefficients and support vectors */
fprintf(fp, "SV_coefs\n");
for (i = 0; i < total_num_sv_; i++) {
for (j = 0; j < num_classes_-1; j++) {
@@ -365,6 +709,7 @@ void SVM<TKernel>::SaveModel(String modelfilename) {
}
fprintf(fp, "\n");
}
fprintf(fp, "SVs\n");
for (i = 0; i < total_num_sv_; i++) {
for (j = 0; j < num_features_; j++) { // n_rows-1
@@ -374,20 +719,23 @@ void SVM<TKernel>::SaveModel(String modelfilename) {
}
fclose(fp);
}
/**
* Load SVM model file
*
* @param: type id of the learner
* @param: name of the model file
*/
// TODO: use XML
template<typename TKernel>
void SVM<TKernel>::LoadModel(Dataset* testset, String modelfilename) {
/* Init */
train_labels_list_.Init(num_classes_);
num_features_ = testset->n_features() - 1;
void SVM<TKernel>::LoadModel_(int learner_typeid, String model_filename) {
if (learner_typeid == 0) {// SVM_C
train_labels_list_.Destruct();
train_labels_list_.Init(num_classes_); // get labels list from the model file
}
/* load model file */
FILE *fp = fopen(modelfilename, "r");
FILE *fp = fopen(model_filename, "r");
if (fp == NULL) {
fprintf(stderr, "Cannot open SVM model file!");
return;
@@ -396,44 +744,30 @@ void SVM<TKernel>::LoadModel(Dataset* testset, String modelfilename) {
int i, j; int temp_d; double temp_f;
for (i = 0; i < num_models_; i++) {
models_.AddBack();
models_[i].bi_coef_.Init();
models_[i].coef_.Init();
}
while (1) {
fscanf(fp,"%80s",cmd);
if(strcmp(cmd,"svm_type")==0) {
fscanf(fp,"%80s",cmd);
if(strcmp(cmd,"svm_c")==0) {
fprintf(stderr, "SVM_C\n");
}
}
else if (strcmp(cmd, "num_classes")==0) {
fscanf(fp,"%d",&num_classes_);
}
else if (strcmp(cmd, "kernel_name")==0) {
fscanf(fp,"%80s",param_.kernelname_.c_str());
}
else if (strcmp(cmd, "kernel_typeid")==0) {
fscanf(fp,"%d",&param_.kerneltypeid_);
}
else if (strcmp(cmd, "sigma")==0) {
fscanf(fp,"%lf",&param_.kernel_.kpara_[0]); /* for gaussian kernels only */
}
else if (strcmp(cmd, "gamma")==0) {
fscanf(fp,"%lf",&param_.kernel_.kpara_[1]); /* for gaussian kernels only */
fscanf(fp,"%80s", cmd);
if (strcmp(cmd,"SVM_C")==0)
learner_typeid_ = 0;
else if (strcmp(cmd,"SVM_R")==0)
learner_typeid_ = 1;
else if (strcmp(cmd,"SVM_DE")==0)
learner_typeid_ = 2;
}
else if (strcmp(cmd, "total_num_sv")==0) {
fscanf(fp,"%d",&total_num_sv_);
}
// for SVM_C
else if (strcmp(cmd, "num_classes")==0) {
fscanf(fp,"%d",&num_classes_);
}
else if (strcmp(cmd, "labels")==0) {
for (i=0; i<num_classes_; i++) {
fscanf(fp,"%f",&temp_d);
train_labels_list_[i] = temp_d;
}
}
else if (strcmp(cmd, "thresholds")==0) {
for ( i= 0; i < num_models_; i++) {
fscanf(fp,"%lf",&temp_f);
models_[i].thresh_= temp_f;
fscanf(fp,"%lf",&temp_f);
train_labels_list_[i] = temp_f;
}
}
else if (strcmp(cmd, "sv_list_startpos")==0) {
@@ -447,13 +781,41 @@ void SVM<TKernel>::LoadModel(Dataset* testset, String modelfilename) {
fscanf(fp,"%d",&temp_d);
sv_list_ct_[i]= temp_d;
}
}
// for SVM_R
else if (strcmp(cmd, "sv_index")==0) {
for ( i= 0; i < total_num_sv_; i++) {
fscanf(fp,"%d",&temp_d);
sv_index_.AddBackItem(temp_d);
}
}
// load kernel info
else if (strcmp(cmd, "kernel_name")==0) {
fscanf(fp,"%80s",param_.kernelname_.c_str());
}
else if (strcmp(cmd, "kernel_typeid")==0) {
fscanf(fp,"%d",&param_.kerneltypeid_);
}
else if (strcmp(cmd, "sigma")==0) {
fscanf(fp,"%lf",&param_.kernel_.kpara_[0]); /* for gaussian kernels only */
}
else if (strcmp(cmd, "gamma")==0) {
fscanf(fp,"%lf",&param_.kernel_.kpara_[1]); /* for gaussian kernels only */
}
// load bias
else if (strcmp(cmd, "bias")==0) {
for ( i= 0; i < num_models_; i++) {
fscanf(fp,"%lf",&temp_f);
models_[i].bias_= temp_f;
}
break;
}
}
// load coefficients and support vectors
sv_coef_.Init(num_classes_-1, total_num_sv_);
sv_coef_.SetZero();
sv_.Init(num_features_, total_num_sv_);
while (1) {
fscanf(fp,"%80s",cmd);
if (strcmp(cmd, "SV_coefs")==0) {
@@ -477,113 +839,4 @@ void SVM<TKernel>::LoadModel(Dataset* testset, String modelfilename) {
fclose(fp);
}
/**
* Multiclass SVM classification for one testing vector
*
* @param: testing vector
*
* @return: a label (double-type-integer, e.g. 1.0, 2.0, 3.0)
*/
template<typename TKernel>
double SVM<TKernel>::Predict(const Vector& datum) {
index_t i, j, k;
ArrayList<double> keval;
keval.Init(total_num_sv_);
for (i = 0; i < total_num_sv_; i++) {
Vector support_vector_i;
sv_.MakeColumnVector(i, &support_vector_i);
keval[i] = param_.kernel_.Eval(datum, support_vector_i);
}
ArrayList<double> values;
values.Init(num_models_);
index_t ct = 0;
for (i = 0; i < num_classes_; i++) {
for (j = i+1; j < num_classes_; j++) {
double sum = 0;
for(k = 0; k < sv_list_ct_[i]; k++) {
sum += sv_coef_.get(j-1, sv_list_startpos_[i]+k) * keval[sv_list_startpos_[i]+k];
}
for(k = 0; k < sv_list_ct_[j]; k++) {
sum += sv_coef_.get(i, sv_list_startpos_[j]+k) * keval[sv_list_startpos_[j]+k];
}
sum -= models_[ct].thresh_;
values[ct] = sum;
ct++;
}
}
ArrayList<index_t> vote;
vote.Init(num_classes_);
for (i = 0; i < num_classes_; i++) {
vote[i] = 0;
}
ct = 0;
for (i = 0; i < num_classes_; i++) {
for (j = i+1; j < num_classes_; j++) {
if(values[ct] > 0.0) { // label 1 in bi-classifiers (for i=...)
++vote[i];
}
else { // label -1 in bi-classifiers (for j=...)
++vote[j];
}
ct++;
}
}
index_t vote_max_idx = 0;
for (i = 1; i < num_classes_; i++) {
if (vote[i] >= vote[vote_max_idx]) {
vote_max_idx = i;
}
}
return train_labels_list_[vote_max_idx];
}
/**
* Online batch classification for multiple testing vectors. No need to load model file,
* since models are already in RAM.
*
* Note: for test set, if no true test labels provided, just put some dummy labels
* (e.g. all -1) in the last row of testset
*
* @param: testing set
* @param: file name of the testing data
*/
template<typename TKernel>
void SVM<TKernel>::BatchPredict(Dataset* testset, String testlablefilename) {
FILE *fp = fopen(testlablefilename, "w");
if (fp == NULL) {
fprintf(stderr, "Cannot save test labels to file!");
return;
}
index_t err_ct = 0;
num_features_ = testset->n_features()-1;
for (index_t i = 0; i < testset->n_points(); i++) {
Vector testvec;
testset->matrix().MakeColumnSubvector(i, 0, num_features_, &testvec);
int testlabel = int(Predict(testvec));
if (testlabel != testset->matrix().get(num_features_, i))
err_ct++;
/* save classified labels to file*/
fprintf(fp, "%d\n", testlabel);
}
fclose(fp);
/* calculate testing error */
fprintf( stderr, "\n*** %d out of %d misclassified ***\n", err_ct, testset->n_points() );
fprintf( stderr, "*** Testing error is %f ***\n", double(err_ct)/double(testset->n_points()) );
fprintf( stderr, "*** Results are save in \"%s\" ***\n\n", testlablefilename.c_str());
}
/**
* Load models from a file, and perform offline batch classification for multiple testing vectors
*
* @param: testing set
* @param: name of the model file
* @param: name of the file to store classified labels
*/
template<typename TKernel>
void SVM<TKernel>::LoadModelBatchPredict(Dataset* testset, String modelfilename, String testlabelfilename) {
LoadModel(testset, modelfilename);
BatchPredict(testset, testlabelfilename);
}
#endif
fastlib/u/houyang/svm/svm_main.cc
+18 -18
View File
@@ -3,10 +3,10 @@
*
* @file svm_main.cc
*
* This file contains main routines for performing multiclass
* SVM classification (SVC, one-vs-one method is employed) and
* SVM regression (epsilon-insensitive loss i.e. epsilon-SVR; and
* automatic adjustment of epsilon, i.e. mu-SVR).
* This file contains main routines for performing
* 0. multiclass SVM classification (one-vs-one method is employed).
* 1. SVM regression (epsilon-insensitive loss i.e. epsilon-SVR).
* 2. SVM density estimation (one-class SVM)
*
* It provides four modes:
* "cv": cross validation;
@@ -155,10 +155,10 @@ int LoadData(Dataset* dataset, String datafilename){
}
if (fx_param_bool(NULL, "normalize", 1)) {
fprintf(stderr, "Normalizing\n");
fprintf(stderr, "Normalizing...\n");
DoSvmNormalize(dataset);
} else {
fprintf(stderr, "Skipping normalization\n");
fprintf(stderr, "Skipping normalization...\n");
}
return 1;
}
@@ -178,13 +178,13 @@ int main(int argc, char *argv[]) {
String learner_name = fx_param_str_req(NULL,"learner_name");
int learner_typeid;
if (learner_name == "svc") { // Support Vector Classfication
if (learner_name == "svm_c") { // Support Vector Classfication
learner_typeid = 0;
}
else if (learner_name == "svr") { // Support Vector Regression
else if (learner_name == "svm_r") { // Support Vector Regression
learner_typeid = 1;
}
else if (learner_name == "svde") { // Support Vector Density Estimation
else if (learner_name == "svm_de") { // Support Vector Density Estimation
learner_typeid = 2;
}
else {
@@ -234,7 +234,7 @@ int main(int argc, char *argv[]) {
if (kernel == "linear") {
SVM<SVMLinearKernel> svm;
svm.InitTrain(learner_typeid, trainset, trainset.n_labels(), svm_module);
svm.InitTrain(learner_typeid, trainset, svm_module);
/* training and testing, thus no need to load model from file */
if (mode=="train_test"){
fprintf(stderr, "SVM Predicting... \n");
@@ -242,12 +242,12 @@ int main(int argc, char *argv[]) {
Dataset testset;
if (LoadData(&testset, "test_data") == 0) // TODO:param_req
return 1;
svm.BatchPredict(&testset, "testlabels");
svm.BatchPredict(learner_typeid, testset, "predicted_values");
}
}
else if (kernel == "gaussian") {
SVM<SVMRBFKernel> svm;
svm.InitTrain(learner_typeid, trainset, trainset.n_labels(), svm_module);
svm.InitTrain(learner_typeid, trainset, svm_module);
/* training and testing, thus no need to load model from file */
if (mode=="train_test"){
fprintf(stderr, "SVM Predicting... \n");
@@ -255,7 +255,7 @@ int main(int argc, char *argv[]) {
Dataset testset;
if (LoadData(&testset, "test_data") == 0) // TODO:param_req
return 1;
svm.BatchPredict(&testset, "testlabels"); // TODO:param_req
svm.BatchPredict(learner_typeid, testset, "predicted_values"); // TODO:param_req
}
}
}
@@ -268,18 +268,18 @@ int main(int argc, char *argv[]) {
if (LoadData(&testset, "test_data") == 0) // TODO:param_req
return 1;
/* Begin Classification */
/* Begin Prediction */
datanode *svm_module = fx_submodule(fx_root, NULL, "svm");
if (kernel == "linear") {
SVM<SVMLinearKernel> svm;
svm.Init(learner_typeid, testset, testset.n_labels(), svm_module); // TODO:n_labels() -> num_classes_
svm.LoadModelBatchPredict(&testset, "svm_model", "testlabels"); // TODO:param_req
svm.Init(learner_typeid, testset, svm_module);
svm.LoadModelBatchPredict(learner_typeid, testset, "svm_model", "predicted_values"); // TODO:param_req
}
else if (kernel == "gaussian") {
SVM<SVMRBFKernel> svm;
svm.Init(learner_typeid, testset, testset.n_labels(), svm_module); // TODO:n_labels() -> num_classes_
svm.LoadModelBatchPredict(&testset, "svm_model", "testlabels"); // TODO:param_req
svm.Init(learner_typeid, testset, svm_module);
svm.LoadModelBatchPredict(learner_typeid, testset, "svm_model", "predicted_values"); // TODO:param_req
}
}
fx_done();