From 31c7b435540d14b4a22dfdcc783228b767a47fe6 Mon Sep 17 00:00:00 2001 From: houyang Date: Sun, 23 Oct 2011 16:43:15 +0000 Subject: [PATCH] --- .../trunk/contrib/houyang/pole/exp_nasa.txt | 2 +- .../trunk/contrib/houyang/pole/exp_ogd.txt | 4 +++ fastlib/trunk/contrib/houyang/pole/learner.cc | 1 + fastlib/trunk/contrib/houyang/pole/learner.h | 1 + fastlib/trunk/contrib/houyang/pole/opt_ogd.cc | 34 ++++++++++++------- fastlib/trunk/contrib/houyang/pole/pole.cc | 1 + 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/fastlib/trunk/contrib/houyang/pole/exp_nasa.txt b/fastlib/trunk/contrib/houyang/pole/exp_nasa.txt index b6ff8f2d19..506fcd1a49 100644 --- a/fastlib/trunk/contrib/houyang/pole/exp_nasa.txt +++ b/fastlib/trunk/contrib/houyang/pole/exp_nasa.txt @@ -52,7 +52,7 @@ Examples: ================================ Batch learning ================================ ./pole_pt --batch 1 -d svmguide1 -t svmguide1 --random 1 -m nasa --type classification -l hinge --reg 2 --bias 1 --lambda 0.01 --threads_test 2 -e 600 --dbound 1.5 -472.062 0.94788 2.45 +461.375 0.950146 2.77 ./pole_pt --batch 1 -d rcv1_train.binary -t rcv1_test.binary --random 1 -m nasa --type classification -l hinge --reg 2 --bias 1 --lambda 0.00001 --threads_test 4 -e 1 --dbound 16 83341.9 0.95506 2:26.56 diff --git a/fastlib/trunk/contrib/houyang/pole/exp_ogd.txt b/fastlib/trunk/contrib/houyang/pole/exp_ogd.txt index ff67ef58c8..5534f994a4 100644 --- a/fastlib/trunk/contrib/houyang/pole/exp_ogd.txt +++ b/fastlib/trunk/contrib/houyang/pole/exp_ogd.txt @@ -51,6 +51,8 @@ Examples: ================================ Batch learning ================================ ./pole_pt --batch 1 -d svmguide1 -t svmguide1 --random 1 -m ogd --type classification -l hinge --reg 2 --bias 1 --lambda 0.01 --threads_test 2 -e 600 --dbound 0.025 485.553 0.939786 1.72 +./pole_pt --batch 1 -d svmguide1 -t svmguide1 --random 1 -m ogd_a --type classification -l hinge --reg 2 --bias 1 --lambda 0.01 --threads_test 2 -e 600 --dbound 5500 --strongness 0.01 +487.144 0.942052 2.25 ./pole_pt --batch 1 -d rcv1_train.binary -t rcv1_test.binary --random 1 -m ogd --type classification -l hinge --reg 2 --bias 1 --lambda 0.00001 --threads_test 4 -e 1 --dbound 17 92298.2 0.95138 2:19.310817 @@ -64,3 +66,5 @@ Examples: 53690 0.77541 2.50 ./pole_pt --batch 1 -d zeta_train --td_ratio 0.2 --random 1 -m ogd --type classification -l hinge --reg 2 --bias 1 --lambda 0.00001 --threads_test 4 -e 1 --dbound 65 +98849.5 0.50529 10.31 + diff --git a/fastlib/trunk/contrib/houyang/pole/learner.cc b/fastlib/trunk/contrib/houyang/pole/learner.cc index 981aa5c2e0..76424da456 100644 --- a/fastlib/trunk/contrib/houyang/pole/learner.cc +++ b/fastlib/trunk/contrib/houyang/pole/learner.cc @@ -11,6 +11,7 @@ Learner::Learner() { TE_ = NULL; LF_ = NULL; LOG_ = NULL; + batch_ = false; // default: online learning } /////////////// diff --git a/fastlib/trunk/contrib/houyang/pole/learner.h b/fastlib/trunk/contrib/houyang/pole/learner.h index 3c5137e542..fa364ce368 100644 --- a/fastlib/trunk/contrib/houyang/pole/learner.h +++ b/fastlib/trunk/contrib/houyang/pole/learner.h @@ -24,6 +24,7 @@ class Learner { Data *VA_; // validation set Data *TE_; // testing set + bool batch_; // batch learning or not bool random_data_; T_IDX n_source_, port_; bool read_port_; // read data from port or file diff --git a/fastlib/trunk/contrib/houyang/pole/opt_ogd.cc b/fastlib/trunk/contrib/houyang/pole/opt_ogd.cc index efb24349e2..625a34dbf4 100644 --- a/fastlib/trunk/contrib/houyang/pole/opt_ogd.cc +++ b/fastlib/trunk/contrib/houyang/pole/opt_ogd.cc @@ -61,14 +61,15 @@ void* OGD::LearnThread(void *in_par) { // Make prediction and get loss for (T_IDX b = 0; bmb_size_; b++) { - /* - // calculate w_avg and make logs - Lp->w_avg_pool_[tid] *= (Lp->t_n_it_[tid] - 1.0); - Lp->w_avg_pool_[tid] += Lp->w_pool_[tid]; - Lp->w_avg_pool_[tid] *= (1.0/Lp->t_n_it_[tid]); - */ - Lp->w_avg_pool_[tid] = Lp->w_pool_[tid]; - double pred_val = Lp->LinearPredictBias(Lp->w_avg_pool_[tid], + // for batch: calculate w_avg + if (Lp->batch_) { + Lp->w_avg_pool_[tid] *= (Lp->t_n_it_[tid] - 1.0); + Lp->w_avg_pool_[tid] += Lp->w_pool_[tid]; + Lp->w_avg_pool_[tid] *= (1.0/Lp->t_n_it_[tid]); + //Lp->w_avg_pool_[tid] = Lp->w_pool_[tid]; + } + // online prediction + double pred_val = Lp->LinearPredictBias(Lp->w_pool_[tid], *exs[b], Lp->b_pool_[tid]); Lp->MakeLearnLog(tid, exs[b], pred_val); update = Lp->LF_->GetUpdate(pred_val, (double)exs[b]->y_); @@ -83,6 +84,15 @@ void* OGD::LearnThread(void *in_par) { else if (Lp->opt_name_ == "ogd") { eta = Lp->dbound_ / sqrt(Lp->t_n_it_[tid]); } + else if (Lp->opt_name_ == "ogd_str_a") { // Xu's Averaged SGD + // 1/( M(1+t\lambda_0/M)^2/3 ) + eta = 1 / ( Lp->dbound_ * pow(1+Lp->strongness_*Lp->t_n_it_[tid]/Lp->dbound_, 2/3) ); + } + // Assuming general convexity: ogd + else if (Lp->opt_name_ == "ogd_a") { // Xu's Averaged SGD + // 1/( M(1+t\lambda_0/M)^3/4 ) + eta = 1 / ( Lp->dbound_ * pow(1+Lp->strongness_*Lp->t_n_it_[tid]/Lp->dbound_, 0.75) ); + } else { cout << "ERROR! Unkown OGD method."<< endl; exit(1); @@ -170,7 +180,7 @@ void* OGD::TestThread(void *in_par) { while (true) { if ( Lp->GetTestExample(Lp->TE_, exs, tid) ) { // new test example read - // testing using Thread[0]'s w & b. TODO... + // testing using Thread[0]'s running-averaged w & b. TODO... double pred_val = Lp->LinearPredictBias(Lp->w_avg_pool_[0], *exs[0], Lp->b_pool_[0]); Lp->MakeTestLog(tid, exs[0], pred_val); @@ -203,11 +213,11 @@ void OGD::MakeLearnLog(T_IDX tid, Example *x, double pred_val) { if (reg_type_ == 2 && reg_factor_ != 0) { //L + \lambda/2 \|w\|^2 <=> CL + 1/2 \|w\|^2 t_loss_[tid] = t_loss_[tid] + - 0.5 * reg_factor_ * w_avg_pool_[tid].SparseSqL2Norm(); + 0.5 * reg_factor_ * w_pool_[tid].SparseSqL2Norm(); } // for classification only: calc # of misclassifications if (type_ == "classification") { - T_LBL pred_lbl = LinearPredictBiasLabelBinary(w_avg_pool_[tid], *x, b_pool_[tid]); + T_LBL pred_lbl = LinearPredictBiasLabelBinary(w_pool_[tid], *x, b_pool_[tid]); //cout << x->y_ << " : " << pred_lbl << endl; if (pred_lbl != x->y_) { t_err_[tid] = t_err_[tid] + 1; @@ -292,7 +302,7 @@ void OGD::SaveLearnLog() { void OGD::MakeTestLog(T_IDX tid, Example *x, double pred_val) { //cout << "pred: " << pred_val <<", y: " << (double)x->y_ << endl; t_test_loss_[tid] = t_test_loss_[tid] + LF_->GetLoss(pred_val, (double)x->y_); - // testing using Thread[0]'s w. TODO... + // testing using Thread[0]'s running-averaged w. TODO... if (reg_type_ == 2 && reg_factor_ != 0) { //L + \lambda/2 \|w\|^2 <=> CL + 1/2 \|w\|^2 t_loss_[tid] = t_loss_[tid] + diff --git a/fastlib/trunk/contrib/houyang/pole/pole.cc b/fastlib/trunk/contrib/houyang/pole/pole.cc index 2472b96286..84b146ae06 100644 --- a/fastlib/trunk/contrib/houyang/pole/pole.cc +++ b/fastlib/trunk/contrib/houyang/pole/pole.cc @@ -305,6 +305,7 @@ void Pole::Run() { L_->OnlineLearn(); } else { // Batch Learning + L_->batch_ = true; L_->BatchLearn(); } }