From 0cec193f2dc9a7c17e08d2ae4ecdf093bbc591fa Mon Sep 17 00:00:00 2001 From: Garry Boyer Date: Sun, 20 May 2007 23:45:40 +0000 Subject: [PATCH] hi --- fastlib/u/garryb/nbr/dfs.h | 272 +++++++++++++++++++++------------- fastlib/u/garryb/nbr/kdtree.h | 4 +- fastlib/u/garryb/nbr/main.cc | 62 ++++++++ 3 files changed, 229 insertions(+), 109 deletions(-) create mode 100644 fastlib/u/garryb/nbr/main.cc diff --git a/fastlib/u/garryb/nbr/dfs.h b/fastlib/u/garryb/nbr/dfs.h index 20c3006cbc..52460ed663 100644 --- a/fastlib/u/garryb/nbr/dfs.h +++ b/fastlib/u/garryb/nbr/dfs.h @@ -12,14 +12,25 @@ class DualTreeDepthFirst { }; private: + /* KdTreeMidpointBuilder q_tree_; KdTreeMidpointBuilder r_tree_; + */ typename GNP::Param param_; typename GNP::GlobalResult global_result_; - ArrayList q_results_; - ArrayList q_mutables_; + + CacheArray q_points_; + CacheArray q_point_infos_; + CacheArray q_nodes_; + CacheArray q_results_; + CacheArray q_mutables_; + + CacheArray r_points_; + CacheArray r_point_infos_; + CacheArray r_nodes_; + bool do_naive_; datanode *datanode_; uint64 n_naive_; @@ -40,15 +51,46 @@ class DualTreeDepthFirst { const typename GNP::QMassResult& exclusive_unvisited, QMutableInfo *q_node_mut); void PushDown_(index_t q_node_i); - private: - typename GNP::QNode *qnode_(index_t i) { - return &q_tree_.nodes()[i]; - } - const typename GNP::RNode *rnode_(index_t i) { - return &r_tree_.nodes()[i]; - } }; +template +class LocalTreeManager { + private: + SmallCache point_cache; + SmallCache point_info_cache; + SmallCache node_cache; + + public: + TreeManager(); + ~TreeManager(); + + void BuildTree(struct datanode *datanode, + const Param& param, + CacheArray *points_out, + CacheArray *point_infos_out, + CacheArray *nodes_out) { + const char *fname = fx_param_str_req(datanode, ""); + Matrix matrix; + data::Load(fname, &matrix); + + Vector first_data; + matrix.MakeColumnVector(0, &first_data); + + PointInfo blank_info; // WALDO + + BlockActionHandler *point_handler = + new CacheArrayBlockActionHandler(first_data); + + KdTreeMidpointBuilder tree_builder; + tree_builder.InitBuild( + fx_submodule(datanode, "tree", "tree"), + ¶m, + points_out, + point_infos_out, + nodes_out); + } +} + template void DualTreeDepthFirst::Init(datanode *datanode) { datanode_ = datanode; @@ -158,7 +200,7 @@ void DualTreeDepthFirst::Begin() { template void DualTreeDepthFirst::PushDown_(index_t q_node_i) { - typename GNP::QNode *q_node = q_nodes_.StartWrite(q_node_i); + const typename GNP::QNode *q_node = q_nodes_.StartRead(q_node_i); QMutableInfo *q_node_mut = q_mutables_.StartWrite(q_node_i); if (q_node->is_leaf()) { @@ -187,7 +229,7 @@ void DualTreeDepthFirst::PushDown_(index_t q_node_i) { } } - q_nodes_.StopWrite(q_node_i); + q_nodes_.StopRead(q_node_i); q_mutables_.StopWrite(q_node_i); } @@ -195,9 +237,9 @@ template void DualTreeDepthFirst::Pair_(index_t q_node_i, index_t r_node_i, const typename GNP::Delta& delta, const typename GNP::QMassResult& exclusive_unvisited) { - typename GNP::QNode *q_node = qnode_(q_node_i); - const typename GNP::RNode *r_node = rnode_(r_node_i); - QMutableInfo *q_node_mut = &q_mutables_[q_node_i]; + const typename GNP::RNode *r_node = r_nodes_.StartRead(r_node_i); + const typename GNP::QNode *q_node = q_nodes_.StartRead(q_node_i); + QMutableInfo *q_node_mut = q_mutables_.StartWrite(q_node_i); DEBUG_MSG(1.0, "Checking (%d,%d) x (%d,%d)", q_node->begin(), q_node->end(), @@ -214,99 +256,103 @@ void DualTreeDepthFirst::Pair_(index_t q_node_i, index_t r_node_i, param_, *q_node, mu, global_result_, &q_node_mut->postponed)) { q_node_mut->mass_result.ApplyDelta(param_, delta); DEBUG_MSG(1.0, "Termination prune"); - return; - } - - if (!GNP::Algorithm::ConsiderPairExtrinsic( + } else if (!GNP::Algorithm::ConsiderPairExtrinsic( param_, *q_node, *r_node, delta, mu, global_result_, &q_node_mut->postponed)) { DEBUG_MSG(1.0, "Extrinsic prune"); - return; - } - /* end prune checks */ - - global_result_.UndoDelta(param_, delta); - - if (q_node->is_leaf() && r_node->is_leaf()) { - DEBUG_MSG(1.0, "Base case"); - BaseCase_(q_node, r_node, exclusive_unvisited, q_node_mut); - } else if (r_node->is_leaf() - || (q_node->count() >= r_node->count() && !q_node->is_leaf())) { - DEBUG_MSG(1.0, "Splitting Q"); - // Phase 2: Explore children, and reincorporate their results. - q_node_mut->mass_result.StartReaccumulate(param_, *q_node); - - for (index_t k = 0; k < 2; k++) { - typename GNP::Delta child_delta; - index_t q_child_i = q_node->child(k); - typename GNP::QNode *q_child = qnode_(q_child_i); - QMutableInfo *q_child_mut = &q_mutables_[q_child_i]; - - q_child_mut->postponed.ApplyPostponed( - param_, q_node_mut->postponed); - child_delta.Init(param_); - - if (GNP::Algorithm::ConsiderPairIntrinsic( - param_, *q_child, *r_node, &child_delta, - &global_result_, &q_child_mut->postponed)) { - Pair_(q_child_i, r_node_i, delta, exclusive_unvisited); - } - - // We must VERY carefully apply both the horizontal and vertical join - // operators here for postponed results. - typename GNP::QMassResult tmp_result(q_child_mut->mass_result); - tmp_result.ApplyPostponed(param_, q_child_mut->postponed, *q_child); - q_node_mut->mass_result.Accumulate(param_, tmp_result, q_node->count()); - } - - q_node_mut->mass_result.FinishReaccumulate(param_, *q_node); - q_node_mut->postponed.Reset(param_); } else { - DEBUG_MSG(1.0, "Splitting R"); - index_t r_child1_i = r_node->child(0); - index_t r_child2_i = r_node->child(1); - const typename GNP::RNode *r_child1 = rnode_(r_child1_i); - const typename GNP::RNode *r_child2 = rnode_(r_child2_i); - - double r_child1_h = GNP::Algorithm::Heuristic( - param_, *q_node, *rnode_(r_child1_i)); - double r_child2_h = GNP::Algorithm::Heuristic( - param_, *q_node, *rnode_(r_child2_i)); + global_result_.UndoDelta(param_, delta); - if (unlikely(r_child2_h < r_child1_h)) { - const typename GNP::RNode *r_child_t = r_child1; - r_child1 = r_child2; - r_child2 = r_child_t; - - index_t r_child_t_i = r_child1_i; - r_child1_i = r_child2_i; - r_child2_i = r_child_t_i; - } + if (q_node->is_leaf() && r_node->is_leaf()) { + DEBUG_MSG(1.0, "Base case"); + BaseCase_(q_node, r_node, exclusive_unvisited, q_node_mut); + } else if (r_node->is_leaf() + || (q_node->count() >= r_node->count() && !q_node->is_leaf())) { + DEBUG_MSG(1.0, "Splitting Q"); + // Phase 2: Explore children, and reincorporate their results. + q_node_mut->mass_result.StartReaccumulate(param_, *q_node); - typename GNP::Delta delta1; - typename GNP::Delta delta2; + for (index_t k = 0; k < 2; k++) { + typename GNP::Delta child_delta; + index_t q_child_i = q_node->child(k); + const typename GNP::QNode *q_child = q_nodes_.StartRead(q_child_i); + QMutableInfo *q_child_mut = q_mutables_.StartWrite(q_child_i); + q_child_mut->postponed.ApplyPostponed( + param_, q_node_mut->postponed); + child_delta.Init(param_); - delta1.Init(param_); - delta2.Init(param_); + if (GNP::Algorithm::ConsiderPairIntrinsic( + param_, *q_child, *r_node, &child_delta, + &global_result_, &q_child_mut->postponed)) { + Pair_(q_child_i, r_node_i, delta, exclusive_unvisited); + } - bool do_r2 = GNP::Algorithm::ConsiderPairIntrinsic( - param_, *q_node, *r_child2, &delta2, - &global_result_, &q_node_mut->postponed); - - if (GNP::Algorithm::ConsiderPairIntrinsic( - param_, *q_node, *r_child1, &delta1, - &global_result_, &q_node_mut->postponed)) { - typename GNP::QMassResult exclusive_unvisited_for_r1( - exclusive_unvisited); - if (do_r2) { - exclusive_unvisited_for_r1.ApplyDelta(param_, delta2); + // We must VERY carefully apply both the horizontal and vertical join + // operators here for postponed results. + typename GNP::QMassResult tmp_result(q_child_mut->mass_result); + tmp_result.ApplyPostponed(param_, q_child_mut->postponed, *q_child); + q_node_mut->mass_result.Accumulate(param_, tmp_result, q_node->count()); + + q_mutables_.StopWrite(q_child_i); + q_nodes_.StopRead(q_child_i); } - Pair_(q_node_i, r_child1_i, delta1, exclusive_unvisited_for_r1); - } - if (do_r2) { - Pair_(q_node_i, r_child2_i, delta2, exclusive_unvisited); + + q_node_mut->mass_result.FinishReaccumulate(param_, *q_node); + q_node_mut->postponed.Reset(param_); + } else { + DEBUG_MSG(1.0, "Splitting R"); + index_t r_child1_i = r_node->child(0); + index_t r_child2_i = r_node->child(1); + const typename GNP::RNode *r_child1 = r_nodes_.StartRead(r_child1_i); + const typename GNP::RNode *r_child2 = r_nodes_.StartRead(r_child2_i); + + double r_child1_h = GNP::Algorithm::Heuristic( + param_, *q_node, *rnode_(r_child1_i)); + double r_child2_h = GNP::Algorithm::Heuristic( + param_, *q_node, *rnode_(r_child2_i)); + + if (unlikely(r_child2_h < r_child1_h)) { + const typename GNP::RNode *r_child_t = r_child1; + r_child1 = r_child2; + r_child2 = r_child_t; + + index_t r_child_t_i = r_child1_i; + r_child1_i = r_child2_i; + r_child2_i = r_child_t_i; + } + + typename GNP::Delta delta1; + typename GNP::Delta delta2; + + delta1.Init(param_); + delta2.Init(param_); + + bool do_r2 = GNP::Algorithm::ConsiderPairIntrinsic( + param_, *q_node, *r_child2, &delta2, + &global_result_, &q_node_mut->postponed); + + if (GNP::Algorithm::ConsiderPairIntrinsic( + param_, *q_node, *r_child1, &delta1, + &global_result_, &q_node_mut->postponed)) { + typename GNP::QMassResult exclusive_unvisited_for_r1( + exclusive_unvisited); + if (do_r2) { + exclusive_unvisited_for_r1.ApplyDelta(param_, delta2); + } + Pair_(q_node_i, r_child1_i, delta1, exclusive_unvisited_for_r1); + } + if (do_r2) { + Pair_(q_node_i, r_child2_i, delta2, exclusive_unvisited); + } + + r_nodes_.StopRead(r_child1_i); + r_nodes_.StopRead(r_child2_i); } } + + r_nodes_.StopRead(r_node_i); + q_nodes_.StopRead(q_node_i); + q_mutables_.StopWrite(q_node_i); } template @@ -316,9 +362,16 @@ void DualTreeDepthFirst::BaseCase_( const typename GNP::QMassResult& exclusive_unvisited, QMutableInfo *q_node_mut) { typename GNP::PairVisitor visitor; + const typename GNP::Point *r_points[r_node->count()]; + const typename GNP::RPointInfo *r_infos[r_node->count()]; + + for (index_t r_i_rel = 0; r_i_rel < r_node->count(); ++r_i_rel) { + r_points[r_i_rel] = r_points_.StartRead(r_i_rel + r_node->begin()); + r_infos[r_i_rel] = r_point_infos_.StartRead(r_i_rel + r_node->begin()); + } DEBUG_ONLY(n_pre_naive_ += q_node->count() * r_node->count()); - + visitor.Init(param_); q_node_mut->mass_result.StartReaccumulate(param_, *q_node); @@ -326,22 +379,19 @@ void DualTreeDepthFirst::BaseCase_( int(q_node - qnode_(0)), q_node->begin(), q_node->count()); for (index_t q_i = q_node->begin(); q_i < q_node->end(); ++q_i) { - typename GNP::Point *q_point = &q_tree_.points()[q_i]; - typename GNP::QPointInfo *q_info = &q_tree_.point_info()[q_i]; - typename GNP::QResult *q_result = &q_results_[q_i]; + const typename GNP::Point *q_point = q_points_.StartRead(q_i); + const typename GNP::QPointInfo *q_info = q_point_infos_.StopRead(q_i); + typename GNP::QResult *q_result = q_results_.StartWrite(q_i); q_result->ApplyPostponed(param_, q_node_mut->postponed, *q_point); if (visitor.StartVisitingQueryPoint(param_, *q_point, *q_info, *r_node, exclusive_unvisited, q_result, &global_result_)) { - index_t r_end = r_node->end(); - - for (index_t r_i = r_node->begin(); r_i < r_end; ++r_i) { - typename GNP::Point *r_point = &r_tree_.points()[r_i]; - typename GNP::RPointInfo *r_info = &r_tree_.point_info()[r_i]; + index_t r_count = r_node->count(); + for (index_t r_i_rel = 0; r_i_rel < r_count; ++r_i_rel) { visitor.VisitPair(param_, *q_point, *q_info, q_i, - *r_point, *r_info, r_i); + *r_point, *r_info, r_i_rel + r_node->begin()); } visitor.FinishVisitingQueryPoint(param_, *q_point, *q_info, *r_node, @@ -351,6 +401,15 @@ void DualTreeDepthFirst::BaseCase_( } q_node_mut->mass_result.Accumulate(param_, *q_result); + + q_points_.StopRead(q_i); + q_point_infos_.StopRead(q_i); + q_results_.StartRead(q_i); + } + + for (index_t r_i_rel = 0; r_i_rel < r_node->count(); ++r_i_rel) { + r_points_.StopRead(r_i_rel + r_node->begin()); + r_point_infos_.StopRead(r_i_rel + r_node->begin()); } q_node_mut->mass_result.FinishReaccumulate(param_, *q_node); @@ -358,4 +417,3 @@ void DualTreeDepthFirst::BaseCase_( } #endif - diff --git a/fastlib/u/garryb/nbr/kdtree.h b/fastlib/u/garryb/nbr/kdtree.h index 8edce2e29c..f565c0c756 100644 --- a/fastlib/u/garryb/nbr/kdtree.h +++ b/fastlib/u/garryb/nbr/kdtree.h @@ -44,12 +44,12 @@ class KdTreeMidpointBuilder { const Param* param_in_, CacheArray *points_in, CacheArray *point_infos_in, - CacheArray *nodes_inout) { + CacheArray *nodes_out) { param_ = param_in_; points_.Init(points_in, BlockDevice::MODIFY); point_infos_.Init(nodes_, BlockDevice::MODIFY); - nodes_.Init(nodes_inout_, BlockDevice::CREATE); + nodes_.Init(nodes_out_, BlockDevice::CREATE); allocator_.Init(&nodes_); const Vector *first_point = points_.StartRead(points_.begin_index()); diff --git a/fastlib/u/garryb/nbr/main.cc b/fastlib/u/garryb/nbr/main.cc new file mode 100644 index 0000000000..4813d8313c --- /dev/null +++ b/fastlib/u/garryb/nbr/main.cc @@ -0,0 +1,62 @@ + +template +class LocalTreeManager { + private: + SmallCache point_cache; + SmallCache point_info_cache; + SmallCache node_cache; + + public: + TreeManager(); + ~TreeManager(); + + void BuildTree(struct datanode *datanode, + const Param& param, + CacheArray *points_out, + CacheArray *point_infos_out, + CacheArray *nodes_out) { + const char *fname = fx_param_str_req(datanode, ""); + Matrix matrix; + data::Load(fname, &matrix); + + Vector first_data; + matrix.MakeColumnVector(0, &first_data); + + PointInfo blank_info; // WALDO + + BlockActionHandler *point_handler = + new CacheArrayBlockActionHandler(first_data); + + KdTreeMidpointBuilder tree_builder; + tree_builder.InitBuild( + fx_submodule(datanode, "tree", "tree"), + ¶m, + points_out, + point_infos_out, + nodes_out); + } +} + + +outline + +template +class DualTreeRunner { + private: + LocalTreeManager q_; + LocalTreeManager r_; + SmallCache WALDO; + + public: + void InitRun(struct datanode *datanode) { + param_.Init(fx_submodule(datanode, "param")); + + + } +}; + +int main(void) { + +} + +