Another checkpoint.

This commit is contained in:
Dongryeol Lee
2010-12-28 16:16:17 +00:00
parent 44ad591849
commit c4e7fbf279
13 changed files with 157 additions and 137 deletions
@@ -9,6 +9,7 @@ add_subdirectory(math)
add_subdirectory(metric_kernels)
add_subdirectory(monte_carlo)
add_subdirectory(optimization)
add_subdirectory(parallel)
add_subdirectory(table)
add_subdirectory(tree)
add_subdirectory(util)
@@ -83,10 +83,8 @@ void core::gnp::DualtreeDfs<ProblemType>::Compute(
// Call the algorithm computation.
core::math::Range squared_distance_range =
(query_table_->get_node_bound(query_table_->get_tree())).RangeDistanceSq(
metric,
reference_table_->get_node_bound
(reference_table_->get_tree()));
(query_table_->get_tree()->bound()).RangeDistanceSq(
metric, reference_table_->get_tree()->bound());
if(do_initializations) {
PreProcess_(query_table_->get_tree());
@@ -106,10 +104,10 @@ template<typename ProblemType>
void core::gnp::DualtreeDfs<ProblemType>::ResetStatisticRecursion_(
typename ProblemType::TableType::TreeType *node,
typename ProblemType::TableType * table) {
table->get_node_stat(node).SetZero();
if(table->node_is_leaf(node) == false) {
ResetStatisticRecursion_(table->get_node_left_child(node), table);
ResetStatisticRecursion_(table->get_node_right_child(node), table);
node->stat().SetZero();
if(node->is_leaf() == false) {
ResetStatisticRecursion_(node->left(), table);
ResetStatisticRecursion_(node->right(), table);
}
}
@@ -117,21 +115,19 @@ template<typename ProblemType>
void core::gnp::DualtreeDfs<ProblemType>::PreProcessReferenceTree_(
typename ProblemType::TableType::TreeType *rnode) {
typename ProblemType::StatisticType &rnode_stat =
reference_table_->get_node_stat(rnode);
typename ProblemType::StatisticType &rnode_stat = rnode->stat();
typename ProblemType::TableType::TreeIterator rnode_it =
reference_table_->get_node_iterator(rnode);
if(reference_table_->node_is_leaf(rnode)) {
if(rnode->is_leaf()) {
rnode_stat.Init(rnode_it);
}
else {
// Get the left and the right children.
typename ProblemType::TableType::TreeType *rnode_left_child =
reference_table_->get_node_left_child(rnode);
typename ProblemType::TableType::TreeType *rnode_left_child = rnode->left();
typename ProblemType::TableType::TreeType *rnode_right_child =
reference_table_->get_node_right_child(rnode);
rnode->right();
// Recurse to the left and the right.
PreProcessReferenceTree_(rnode_left_child);
@@ -139,9 +135,9 @@ void core::gnp::DualtreeDfs<ProblemType>::PreProcessReferenceTree_(
// Build the node stat by combining those owned by the children.
typename ProblemType::StatisticType &rnode_left_child_stat =
reference_table_->get_node_stat(rnode_left_child) ;
rnode_left_child->stat();
typename ProblemType::StatisticType &rnode_right_child_stat =
reference_table_->get_node_stat(rnode_right_child) ;
rnode_right_child->stat();
rnode_stat.Init(
rnode_it, rnode_left_child_stat, rnode_right_child_stat);
}
@@ -151,13 +147,12 @@ template<typename ProblemType>
void core::gnp::DualtreeDfs<ProblemType>::PreProcess_(
typename ProblemType::TableType::TreeType *qnode) {
typename ProblemType::StatisticType &qnode_stat =
query_table_->get_node_stat(qnode);
typename ProblemType::StatisticType &qnode_stat = qnode->stat();
qnode_stat.SetZero();
if(!query_table_->node_is_leaf(qnode)) {
PreProcess_(query_table_->get_node_left_child(qnode));
PreProcess_(query_table_->get_node_right_child(qnode));
if(! qnode->is_leaf()) {
PreProcess_(qnode->left());
PreProcess_(qnode->right());
}
}
@@ -170,8 +165,7 @@ void core::gnp::DualtreeDfs<ProblemType>::DualtreeBase_(
// Clear the summary statistics of the current query node so that we
// can refine it to better bounds.
typename ProblemType::StatisticType &qnode_stat =
query_table_->get_node_stat(qnode);
typename ProblemType::StatisticType &qnode_stat = qnode->stat();
qnode_stat.summary_.StartReaccumulate();
// Postponed object to hold each query contribution.
@@ -237,8 +231,7 @@ bool core::gnp::DualtreeDfs<ProblemType>::CanProbabilisticSummarize_(
typename ProblemType::DeltaType &delta,
typename ProblemType::ResultType *query_results) {
typename ProblemType::StatisticType &qnode_stat =
query_table_->get_node_stat(qnode);
typename ProblemType::StatisticType &qnode_stat = qnode->stat();
typename ProblemType::SummaryType new_summary(qnode_stat.summary_);
new_summary.ApplyPostponed(qnode_stat.postponed_);
new_summary.ApplyDelta(delta);
@@ -267,8 +260,7 @@ bool core::gnp::DualtreeDfs<ProblemType>::CanSummarize_(
const typename ProblemType::DeltaType &delta,
typename ProblemType::ResultType *query_results) {
typename ProblemType::StatisticType &qnode_stat =
query_table_->get_node_stat(qnode);
typename ProblemType::StatisticType &qnode_stat = qnode->stat();
typename ProblemType::SummaryType new_summary(qnode_stat.summary_);
new_summary.ApplyPostponed(qnode_stat.postponed_);
new_summary.ApplyDelta(delta);
@@ -283,8 +275,7 @@ void core::gnp::DualtreeDfs<ProblemType>::Summarize_(
const typename ProblemType::DeltaType &delta,
typename ProblemType::ResultType *query_results) {
typename ProblemType::StatisticType &qnode_stat =
query_table_->get_node_stat(qnode);
typename ProblemType::StatisticType &qnode_stat = qnode->stat();
qnode_stat.postponed_.ApplyDelta(delta, query_results);
}
@@ -302,13 +293,11 @@ void core::gnp::DualtreeDfs<ProblemType>::Heuristic_(
core::math::Range &second_squared_distance_range) {
core::math::Range tmp_first_squared_distance_range =
node_table->get_node_bound(node).RangeDistanceSq(
metric,
candidate_table->get_node_bound(first_candidate));
(node->bound()).RangeDistanceSq(
metric, first_candidate->bound());
core::math::Range tmp_second_squared_distance_range =
node_table->get_node_bound(node).RangeDistanceSq(
metric,
candidate_table->get_node_bound(second_candidate));
(node->bound()).RangeDistanceSq(
metric, second_candidate->bound());
if(tmp_first_squared_distance_range.lo <=
tmp_second_squared_distance_range.lo) {
@@ -357,10 +346,10 @@ bool core::gnp::DualtreeDfs<ProblemType>::DualtreeCanonical_(
}
// If it is not prunable and the query node is a leaf,
if(query_table_->node_is_leaf(qnode)) {
if(qnode->is_leaf()) {
bool exact_compute = true;
if(reference_table_->node_is_leaf(rnode)) {
if(rnode->is_leaf()) {
if(do_base_case_) {
@@ -383,11 +372,8 @@ bool core::gnp::DualtreeDfs<ProblemType>::DualtreeCanonical_(
squared_distance_range_second;
typename ProblemType::TableType::TreeType *rnode_second;
Heuristic_(
metric, qnode, query_table_,
reference_table_->get_node_left_child(rnode),
reference_table_->get_node_right_child(rnode),
reference_table_,
&rnode_first, squared_distance_range_first,
metric, qnode, query_table_, rnode->left(), rnode->right(),
reference_table_, &rnode_first, squared_distance_range_first,
&rnode_second, squared_distance_range_second);
// Recurse.
@@ -410,25 +396,20 @@ bool core::gnp::DualtreeDfs<ProblemType>::DualtreeCanonical_(
bool exact_compute_nonleaf_qnode = true;
// Get the current query node statistic.
typename ProblemType::StatisticType &qnode_stat =
query_table_->get_node_stat(qnode);
typename ProblemType::StatisticType &qnode_stat = qnode->stat();
// Left and right nodes of the query node and their statistic.
typename ProblemType::TableType::TreeType *qnode_left =
query_table_->get_node_left_child(qnode);
typename ProblemType::TableType::TreeType *qnode_right =
query_table_->get_node_right_child(qnode);
typename ProblemType::StatisticType &qnode_left_stat =
query_table_->get_node_stat(qnode_left);
typename ProblemType::StatisticType &qnode_right_stat =
query_table_->get_node_stat(qnode_right);
typename ProblemType::TableType::TreeType *qnode_left = qnode->left();
typename ProblemType::TableType::TreeType *qnode_right = qnode->right();
typename ProblemType::StatisticType &qnode_left_stat = qnode_left->stat();
typename ProblemType::StatisticType &qnode_right_stat = qnode_right->stat();
// Push down postponed and clear.
qnode_left_stat.postponed_.ApplyPostponed(qnode_stat.postponed_);
qnode_right_stat.postponed_.ApplyPostponed(qnode_stat.postponed_);
qnode_stat.postponed_.SetZero();
if(reference_table_->node_is_leaf(rnode)) {
if(rnode->is_leaf()) {
typename ProblemType::TableType::TreeType *qnode_first;
core::math::Range
squared_distance_range_first, squared_distance_range_second;
@@ -457,8 +438,7 @@ bool core::gnp::DualtreeDfs<ProblemType>::DualtreeCanonical_(
typename ProblemType::TableType::TreeType *rnode_second;
Heuristic_(
metric, qnode_left, query_table_,
reference_table_->get_node_left_child(rnode),
reference_table_->get_node_right_child(rnode),
rnode->left(), rnode->right(),
reference_table_, &rnode_first, squared_distance_range_first,
&rnode_second, squared_distance_range_second);
@@ -476,8 +456,7 @@ bool core::gnp::DualtreeDfs<ProblemType>::DualtreeCanonical_(
Heuristic_(
metric, qnode_right, query_table_,
reference_table_->get_node_left_child(rnode),
reference_table_->get_node_right_child(rnode),
rnode->left(), rnode->right(),
reference_table_, &rnode_first, squared_distance_range_first,
&rnode_second, squared_distance_range_second);
@@ -517,10 +496,9 @@ void core::gnp::DualtreeDfs<ProblemType>::PostProcess_(
typename ProblemType::TableType::TreeType *qnode,
typename ProblemType::ResultType *query_results) {
typename ProblemType::StatisticType &qnode_stat =
query_table_->get_node_stat(qnode);
typename ProblemType::StatisticType &qnode_stat = qnode->stat();
if(query_table_->node_is_leaf(qnode)) {
if(qnode->is_leaf()) {
typename ProblemType::TableType::TreeIterator qnode_iterator =
query_table_->get_node_iterator(qnode);
@@ -543,14 +521,10 @@ void core::gnp::DualtreeDfs<ProblemType>::PostProcess_(
qnode_stat.postponed_.SetZero();
}
else {
typename ProblemType::TableType::TreeType *qnode_left =
query_table_->get_node_left_child(qnode);
typename ProblemType::TableType::TreeType *qnode_right =
query_table_->get_node_right_child(qnode);
typename ProblemType::StatisticType &qnode_left_stat =
query_table_->get_node_stat(qnode_left);
typename ProblemType::StatisticType &qnode_right_stat =
query_table_->get_node_stat(qnode_right);
typename ProblemType::TableType::TreeType *qnode_left = qnode->left();
typename ProblemType::TableType::TreeType *qnode_right = qnode->right();
typename ProblemType::StatisticType &qnode_left_stat = qnode_left->stat();
typename ProblemType::StatisticType &qnode_right_stat = qnode_right->stat();
qnode_left_stat.postponed_.ApplyPostponed(qnode_stat.postponed_);
qnode_right_stat.postponed_.ApplyPostponed(qnode_stat.postponed_);
@@ -111,10 +111,10 @@ void core::gnp::TripletreeDfs<ProblemType>::Compute(
template<typename ProblemType>
void core::gnp::TripletreeDfs<ProblemType>::ResetStatisticRecursion_(
typename ProblemType::TableType::TreeType *node) {
table_->get_node_stat(node).SetZero();
if(table_->node_is_leaf(node) == false) {
ResetStatisticRecursion_(table_->get_node_left_child(node));
ResetStatisticRecursion_(table_->get_node_right_child(node));
node->stat().SetZero();
if(node->is_leaf() == false) {
ResetStatisticRecursion_(node->left());
ResetStatisticRecursion_(node->right());
}
}
@@ -122,17 +122,15 @@ template<typename ProblemType>
void core::gnp::TripletreeDfs<ProblemType>::PreProcess_(
typename ProblemType::TableType::TreeType *qnode) {
typename ProblemType::StatisticType &qnode_stat =
table_->get_node_stat(qnode);
typename ProblemType::StatisticType &qnode_stat = qnode->stat();
typename ProblemType::TableType::TreeIterator qnode_it =
table_->get_node_iterator(qnode);
if(! table_->node_is_leaf(qnode)) {
PreProcess_(table_->get_node_left_child(qnode));
PreProcess_(table_->get_node_right_child(qnode));
if(! qnode->is_leaf()) {
PreProcess_(qnode->left());
PreProcess_(qnode->right());
qnode_stat.Init(
qnode_it, table_->get_node_left_child(qnode)->stat(),
table_->get_node_right_child(qnode)->stat());
qnode_it, qnode->left()->stat(), qnode->right()->stat());
}
else {
qnode_stat.Init(qnode_it);
@@ -228,8 +226,7 @@ void core::gnp::TripletreeDfs<ProblemType>::TripletreeBase_(
// can refine it to better bounds.
typename ProblemType::TableType::TreeType *node =
range_sq_in.node(node_index);
typename ProblemType::StatisticType &node_stat =
problem_->table()->get_node_stat(node);
typename ProblemType::StatisticType &node_stat = node->stat();
node_stat.summary_.StartReaccumulate(problem_->global());
// Get the query node iterator and the reference node iterator.
@@ -306,8 +303,7 @@ bool core::gnp::TripletreeDfs<ProblemType>::CanProbabilisticSummarize_(
typename core::gnp::TripletreeDfs<ProblemType>::TreeType *node =
range_in.node(i);
if(i == 0 || node != range_in.node(i - 1)) {
typename ProblemType::StatisticType &node_stat =
table_->get_node_stat(node);
typename ProblemType::StatisticType &node_stat = node->stat();
// Loop over each point on this node.
typename TableType::TreeIterator node_it =
@@ -405,8 +401,7 @@ bool core::gnp::TripletreeDfs<ProblemType>::CanSummarize_(
typename core::gnp::TripletreeDfs<ProblemType>::TreeType *node =
triple_range_distance_sq_in.node(i);
if(i == 0 || node != triple_range_distance_sq_in.node(i - 1)) {
typename ProblemType::StatisticType &node_stat =
table_->get_node_stat(node);
typename ProblemType::StatisticType &node_stat = node->stat();
new_summaries[i] = node_stat.summary_;
new_summaries[i].ApplyPostponed(node_stat.postponed_);
new_summaries[i].ApplyDelta(delta, i);
@@ -458,8 +453,7 @@ void core::gnp::TripletreeDfs<ProblemType>::Summarize_(
typename core::gnp::TripletreeDfs<ProblemType>::TreeType *node =
triple_range_distance_sq.node(i);
if(i == 0 || node != triple_range_distance_sq.node(i - 1)) {
typename ProblemType::StatisticType &node_stat =
table_->get_node_stat(node);
typename ProblemType::StatisticType &node_stat = node->stat();
node_stat.postponed_.ApplyDelta(delta, i, query_results);
}
}
@@ -567,17 +561,17 @@ void core::gnp::TripletreeDfs<ProblemType>::RecursionHelper_(
// Get the current query node statistic.
typename ProblemType::StatisticType &current_node_stat =
table_->get_node_stat(current_node);
current_node->stat();
// Left and right nodes of the query node and their statistic.
typename ProblemType::TableType::TreeType *current_node_left =
table_->get_node_left_child(current_node);
current_node->left();
typename ProblemType::TableType::TreeType *current_node_right =
table_->get_node_right_child(current_node);
current_node->right();
typename ProblemType::StatisticType &current_node_left_stat =
table_->get_node_stat(current_node_left);
current_node_left->stat();
typename ProblemType::StatisticType &current_node_right_stat =
table_->get_node_stat(current_node_right);
current_node_right->stat();
// Push down postponed and clear.
current_node_left_stat.postponed_.ApplyPostponed(
@@ -705,10 +699,9 @@ void core::gnp::TripletreeDfs<ProblemType>::PostProcess_(
typename ProblemType::ResultType *query_results,
bool do_query_results_postprocess) {
typename ProblemType::StatisticType &qnode_stat =
table_->get_node_stat(qnode);
typename ProblemType::StatisticType &qnode_stat = qnode->stat();
if(table_->node_is_leaf(qnode)) {
if(qnode->is_leaf()) {
typename ProblemType::TableType::TreeIterator qnode_iterator =
table_->get_node_iterator(qnode);
@@ -738,14 +731,10 @@ void core::gnp::TripletreeDfs<ProblemType>::PostProcess_(
qnode_stat.postponed_.SetZero();
}
else {
typename ProblemType::TableType::TreeType *qnode_left =
table_->get_node_left_child(qnode);
typename ProblemType::TableType::TreeType *qnode_right =
table_->get_node_right_child(qnode);
typename ProblemType::StatisticType &qnode_left_stat =
table_->get_node_stat(qnode_left);
typename ProblemType::StatisticType &qnode_right_stat =
table_->get_node_stat(qnode_right);
typename ProblemType::TableType::TreeType *qnode_left = qnode->left();
typename ProblemType::TableType::TreeType *qnode_right = qnode->right();
typename ProblemType::StatisticType &qnode_left_stat = qnode_left->stat();
typename ProblemType::StatisticType &qnode_right_stat = qnode_right->stat();
qnode_left_stat.postponed_.ApplyPostponed(qnode_stat.postponed_);
qnode_right_stat.postponed_.ApplyPostponed(qnode_stat.postponed_);
@@ -0,0 +1,25 @@
cmake_minimum_required(VERSION 2.8)
set(SOURCES
distributed_auction.h
distributed_local_kmeans.h
tree_exchange.h
)
set(DIR_SRCS)
foreach(file ${SOURCES})
set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
endforeach()
set(CORE_SRCS ${CORE_SRCS} ${DIR_SRCS} PARENT_SCOPE)
# test executable
add_executable(distributed_auction-test
EXCLUDE_FROM_ALL
distributed_auction.test.cc
)
# link dependencies of test executable
target_link_libraries(distributed_auction-test
core
${Boost_LIBRARIES}
)
@@ -3,14 +3,14 @@
* @author Dongryeol Lee (dongryel@cc.gatech.edu)
*/
#ifndef CORE_TABLE_DISTRIBUTED_AUCTION_H
#define CORE_TABLE_DISTRIBUTED_AUCTION_H
#ifndef CORE_PARALLEL_DISTRIBUTED_AUCTION_H
#define CORE_PARALLEL_DISTRIBUTED_AUCTION_H
#include <boost/mpi.hpp>
#include <boost/serialization/string.hpp>
namespace core {
namespace table {
namespace parallel {
class DistributedAuction {
private:
@@ -1,9 +1,11 @@
/** @file distributed_auction.test.cc
*
* A simple test for distributed auction algorithm.
*
* @author Dongryeol Lee (dongryel@cc.gatech.edu)
*/
#include "core/table/distributed_auction.h"
#include "core/parallel/distributed_auction.h"
#include "core/math/math_lib.h"
int main(int argc, char *argv[]) {
@@ -11,7 +13,7 @@ int main(int argc, char *argv[]) {
// Initialize boost MPI.
boost::mpi::environment env(argc, argv);
boost::mpi::communicator world;
core::table::DistributedAuction auction;
core::parallel::DistributedAuction auction;
// Seed the random number generator.
srand(time(NULL) + world.rank());
@@ -0,0 +1,51 @@
/** @file table_exchange.h
*
* A class to do a set of all-to-all table exchanges.
*
* @author Dongryeol Lee (dongryel@cc.gatech.edu)
*/
#ifndef CORE_PARALLEL_TABLE_EXCHANGE_H
#define CORE_PARALLEL_TABLE_EXCHANGE_H
#include <boost/mpi.hpp>
namespace core {
namespace parallel {
template<typename SubTableType>
class TableExchange {
public:
template<typename TableType>
void AllToAll(
boost::mpi::communicator &world,
int max_num_levels_to_serialize,
TableType &local_table,
const std::vector< std::vector< std::pair<int, int> > > &receive_requests,
std::vector< std::vector<SubTableType> > *received_subtables) {
// The gathered request lists to send to each process.
std::vector< std::vector< std::pair<int, int> > > send_requests;
// Each process gathers the list of requests: (node
// begin, node end index) pairs.
boost::mpi::all_to_all(
world, receive_requests, send_requests);
// Prepare the list of subtables, and do another all_to_all.
std::vector< std::vector<SubTableType> > send_subtables;
send_subtables.resize(send_requests.size());
for(unsigned int j = 0; j < send_requests.size(); j++) {
send_subtables[j].resize(send_requests[j].size());
for(unsigned int i = 0; i < send_requests[j].size(); i++) {
send_subtables[j][i].Init();
}
}
boost::mpi::all_to_all(world, send_subtables, *received_subtables);
}
};
};
};
#endif
@@ -4,7 +4,6 @@ project(FASTLIB C CXX Fortran)
set(SOURCES
dense_matrix.h
dense_point.h
distributed_auction.h
distributed_table.h
global.cc
index_util.h
@@ -22,14 +21,3 @@ foreach(file ${SOURCES})
endforeach()
set(CORE_SRCS ${CORE_SRCS} ${DIR_SRCS} PARENT_SCOPE)
# test executable
add_executable(distributed_auction-test
EXCLUDE_FROM_ALL
distributed_auction.test.cc
)
# link dependencies of test executable
target_link_libraries(distributed_auction-test
core
${Boost_LIBRARIES}
)
@@ -16,9 +16,9 @@
#include "core/table/table.h"
#include "core/table/memory_mapped_file.h"
#include "core/tree/gen_metric_tree.h"
#include "core/table/distributed_auction.h"
#include "core/parallel/distributed_auction.h"
#include "core/table/offset_dense_matrix.h"
#include "core/tree/distributed_local_kmeans.h"
#include "core/parallel/distributed_local_kmeans.h"
#include "core/table/index_util.h"
namespace core {
@@ -303,7 +303,7 @@ class DistributedTable: public boost::noncopyable {
const std::vector<double> &num_points_assigned_to_leaf_nodes) {
if(table_outbox_group_comm.size() > 1) {
core::table::DistributedAuction auction;
core::parallel::DistributedAuction auction;
return auction.Assign(
table_outbox_group_comm, num_points_assigned_to_leaf_nodes,
1.0 / static_cast<double>(table_outbox_group_comm.size()));
@@ -1,4 +1,7 @@
/** @file table.h
*
* An abstract organization of a multidimensional dataset with its
* indexing structure.
*
* @author Dongryeol Lee (dongryel@cc.gatech.edu)
*/
@@ -276,18 +279,6 @@ class Table {
return node->bound();
}
TreeType *get_node_left_child(TreeType *node) {
return node->left();
}
TreeType *get_node_right_child(TreeType *node) {
return node->right();
}
bool node_is_leaf(TreeType *node) const {
return node->is_leaf();
}
StatisticType &get_node_stat(TreeType *node) {
return node->stat();
}
@@ -2,7 +2,6 @@ cmake_minimum_required(VERSION 2.8)
set(SOURCES
ball_bound.h
distributed_local_kmeans.h
general_spacetree.h
gen_kdtree.h
gen_metric_tree.h
@@ -42,7 +42,7 @@ class TestTree {
}
while(node_it.HasNext());
if(table.node_is_leaf(node) == false) {
if(node->is_leaf() == false) {
return TestTreeIterator_(
table.get_node_left_child(node), table) &&
TestTreeIterator_(