diff --git a/fastlib/u/garryb/superpar/notes.txt b/fastlib/u/garryb/superpar/notes.txt index 08a6cad30e..6946a33262 100644 --- a/fastlib/u/garryb/superpar/notes.txt +++ b/fastlib/u/garryb/superpar/notes.txt @@ -42,3 +42,74 @@ Array - blocks have no alignment - read blocks on first use - explicit ranged writebacks + + +how to build tree + +- build first (log p) levels in parallel + - first: assume block size >= p + - first block is not completely filled + - next: if block size < p + - second layer of blocks not completely filled + - this may be space-hungry if B^2 < N, but in that case, you're kind of + silly for using that many processors + - so overall + - block wastage: P blocks max (really B^(floor((log P)/(log B)))), + (N log P)/(B log B) percent +- fits in memory + - arrays are synced by region + - trees are private-write +- doesn't fit in memory + - first: + - ensure writable portion fits in memory, lock it, and private writes + - allocate cache that can exactly fit it? + - non-writable parts of tree are not locked + - DFS: fit entire query into RAM, fit useful part of reference into RAM + - RBFS: only one or logarithmic query nodes even needed in RAM at a time + - for best performance, all relevant references must fit + - okay for nearest neighbors (linear search space), bad for larger + search spaces + - generalized: okay for *any* algorithm that performs well + single-tree!!!!!!!!! + - later: any point in *not* making this assumption? + - breadth first? + - query-reference problems: use recursive breadth first + - global problems: no writes to trees necessary + +how to execute + +- operation by pattern + - in best conditions these are all O((Q+R)/B) + - DFS + - (QR)/(MB) + - tree + - query: lock into RAM, private writes + - reference: load via cache + - I/O worst-case: (4QR/(MB)) + - data: + - make sure query fits in RAM + - reference: ranged + - RBFS + - (QR log B)/(B) without block-chunking + - tree + - query: one block at a time + - reference: via cache + - if reference set stays large, cache performance will be abysmal + - make sure references are sorted based on block order + - if it was N^2 time before I/O efficiency would be + - I/O rate: O(Q(R/B)(log B)), but it could have been + O((Q/M)(R/B)) + - *sigh* still better than single-tree + - data: cached with ranges + - Block breadth-first + - (QR log B)/(MB) + - looks at pairs of blocks + - tree: only one query and one references needed at any time + - queue size: QR/B^2 + - I/O worst-case: either QR/B^2 or (4QR/(MB))(log B) + - towards the bottom, we end up working on pairs of half-memory-sized + chunks + - we may choose number of points per leaf to be comparable to log B + - for simplicity, my first version simply won't support mu + (mu will just equal the root's delta); we will just target + gamma-based problems diff --git a/fastlib/u/garryb/superpar/rbfs.h b/fastlib/u/garryb/superpar/rbfs.h index 153d6a946d..562cb0e2ba 100644 --- a/fastlib/u/garryb/superpar/rbfs.h +++ b/fastlib/u/garryb/superpar/rbfs.h @@ -168,8 +168,10 @@ class RecursiveBreadthFirstDualTreeRunner { Param param_; ArrayList q_info_; ArrayList r_info_; - QNode *q_root_; - RNode *r_root_; + const QNode *q_root_; + const RNode *r_root_; + ArrayList q_result_; + GlobalResult global_result_; public: void Init(struct datanode *module, @@ -187,11 +189,15 @@ class RecursiveBreadthFirstDualTreeRunner { } ArrayList& q_info() { - return q_info; + return q_info_; } ArrayList& r_info() { - return r_info; + return r_info_; + } + + ArrayList& q_result() { + return q_result_; } private: @@ -203,20 +209,20 @@ class RecursiveBreadthFirstDualTreeRunner { DEBUG_POISON_PTR(r_node); } - RNode *r_node; + const RNode *r_node; Delta delta; }; private: const Param *param_; - QNode *q_node_; + const QNode *q_node_; ArrayList list_; - MassResult q_mass_result_; - PostponedResult q_postponed_; + QMassResult q_mass_result_; + QPostponedResult q_postponed_; GlobalResult *global_result_; public: - void Init(QNode* q_node_in, const Param* param, + void Init(const QNode* q_node_in, const Param* param, GlobalResult* global_result_in) { param_ = param; q_node_ = q_node_in; @@ -230,7 +236,7 @@ class RecursiveBreadthFirstDualTreeRunner { global_result_ = global_result_in; } - void Init(QNode* q_node_in, const Queue& parent) { + void Init(const QNode* q_node_in, const Queue& parent) { param_ = parent.param; q_node_ = q_node_in; @@ -244,14 +250,14 @@ class RecursiveBreadthFirstDualTreeRunner { global_result_ = parent.global_result_; } - void Add(RNode *r_node) { + void Add(const RNode *r_node) { Entry *entry = list_.AddBack(); bool try_explore = Algorithm::ConsiderPairIntrinsic( *param_, *q_node_, *r_node, - &entry->delta, &q_mass_result_, global_result_, &q_postponed_); - + &entry->delta, global_result_, &q_postponed_); if (try_explore) { entry->r_node = r_node; + q_mass_result_.ApplyDelta(*param_, entry->delta); } else { list_.PopBack(); } @@ -266,7 +272,7 @@ class RecursiveBreadthFirstDualTreeRunner { return list_.size(); } - RNode* rnode(index_t i) const { + const RNode* rnode(index_t i) const { return ; } @@ -275,92 +281,195 @@ class RecursiveBreadthFirstDualTreeRunner { return list_[i].delta; } - const MassResult& q_mass_result() const { return q_mass_result_; } + const QMassResult& q_mass_result() const { return q_mass_result_; } - const PostponedResult& q_postponed() const { return q_postponed_; } + const QPostponedResult& q_postponed() const { return q_postponed_; } - PostponedResult& q_postponed() { return q_postponed_; } + QPostponedResult& q_postponed() { return q_postponed_; } }; - - void SplitQ(QNode *q_node, const Queue& list_old) { - if (q_node->is_leaf()) { - DoQLeafStuffWALDO_TODO(q_node, list_old); - return; - } - - if (Algorithm::ConsiderQueryTermination(¶m_, *q_node, + void RecursivelyApplyPostponed( + const QNode* q_node, const QPostponedResult& postponed_result); + + bool TryTerminate(const QNode *q_node, const Queue& list_old) { + if (Algorithm::ConsiderQueryTermination(param_, *q_node, list_old.q_mass_result(), *global_result_, &list_old.q_postponed())) { RecursivelyApplyPostponed(q_node, list_old.q_postponed()); return; } - - Queue list_new[cardinality]; - - /* TODO: termination prunes can be checked here */ - - for (int c = 0; c < cardinality; c++) { - list_new[c].Init(q_node->child(i), list_old); - } - - QPostponedResults postponed; - - postponed.Init(*param_); - // We haven't done any exhaustive comparisons, we start with an empty. - - for (index_t i = 0; i < list_old.size(); i++) { - RNode *r_node = list_old.rnode(i); - const Delta* delta = &list_old.delta(i); - - if (likely(Algorithm::ConsiderPairExtrinsic(param_, *q_node, *r_node, - *delta, list_old.q_mass_result(), *global_result_, &postponed))) { - global_result_.UndoDelta(param_, *delta); - for (int c_q = 0; c_q < cardinality; c_q++) { - global_result_.ApplyDelta(param_, *delta); - list_new[c_q].Add(r_node); - } - } - } - - /* recurse over query children */ - - for (int c = 0; c < cardinality; c++) { - list_new[c].q_postponed().ApplyPostponed(*param_, postponed); - list_new[c].Finish(param_); - SplitR(q_node->child(c), list_new[c]); - } } - void SplitR(QNode *q_node, const ArrayList& list_old) { - Queue list_new; + void FinishQ(const QNode *q_node, const Queue& list_old); + + void SplitQ(const QNode *q_node, const Queue& list_old); + + void SplitR(const QNode *q_node, const ArrayList& list_old); +}; + +template +void RecursiveBreadthFirstDualTreeRunner::RecursivelyApplyPostponed( + const QNode* q_node, const QPostponedResult& postponed_result) { + for (index_t i = q_node->begin(); i < q_node->end(); i++) { + q_results_[i].ApplyPostponedResult(param_, postponed_result, *q_node); + } +} + +template +void RecursiveBreadthFirstDualTreeRunner::SplitQ( + const QNode *q_node, const Queue& list_old) { + if (q_node->is_leaf()) { + RecurseOnLeaves(q_node, list_old); + return; + } + + if (TryTerminate(q_node, list_old)) { + return; + } + + Queue list_new[cardinality]; + + /* TODO: termination prunes can be checked here */ + + for (int c = 0; c < cardinality; c++) { + list_new[c].Init(q_node->child(i), list_old); + } + + QPostponedResults postponed; + + postponed.Init(param_); + // We haven't done any exhaustive comparisons, we start with an empty. + + for (index_t i = 0; i < list_old.size(); i++) { + const RNode *r_node = list_old.rnode(i); + const Delta* delta = &list_old.delta(i); - if (Algorithm::ConsiderQueryTermination(¶m_, *q_node, - list_old.q_mass_result(), *global_result_, &list_old.q_postponed())) { - RecursivelyApplyPostponed(q_node, list_old.q_postponed()); - return; + if (likely(Algorithm::ConsiderPairExtrinsic(param_, *q_node, *r_node, + *delta, list_old.q_mass_result(), *global_result_, &postponed))) { + global_result_.UndoDelta(param_, *delta); + for (int c_q = 0; c_q < cardinality; c_q++) { + global_result_.ApplyDelta(param_, *delta); + list_new[c_q].Add(r_node); + } } + } + + /* recurse over query children */ + + for (int c = 0; c < cardinality; c++) { + list_new[c].q_postponed().ApplyPostponed(param_, postponed); + list_new[c].Finish(param_); + SplitR(q_node->child(c), list_new[c]); + } +} + +template +void RecursiveBreadthFirstDualTreeRunner::SplitR( + const QNode *q_node, const ArrayList& list_old) { + Queue list_new; + + if (TryTerminate(q_node, list_old)) { + return; + } + + list_new.Init(q_node, list_old); + + for (index_t i = 0; i < list_old.size(); i++) { + const RNode *r_node = list_old.rnode(i); + const Delta* delta = &list_old.delta(i); - list_new.Init(q_node, list_old); - - for (index_t i = 0; i < list_old.size(); i++) { - RNode *r_node = list_old.rnode(i); - const Delta* delta = &list_old.delta(i); - + if (likely(Algorithm::ConsiderPairExtrinsic(param_, *q_node, *r_node, + *delta, list_old.q_mass_result(), *global_result_, + &list_new.q_postponed()))) { if (entry_old->r_node->is_leaf()) { // TODO: We can collapse the mu's for these together list_new.Add(r_node); } else { - if (likely(Algorithm::ConsiderPairExtrinsic(param_, *q_node, *r_node, - *delta, list_old.q_mass_result(), *global_result_, - &list_new.q_postponed()))) { - global_result_.UndoDelta(*delta); - for (int c = 0; c < cardinality; c++) { - global_result_.ApplyDelta(*delta); - list_new.Add(node->child(i)); - } + global_result_.UndoDelta(*delta); + for (int c = 0; c < cardinality; c++) { + global_result_.ApplyDelta(*delta); + list_new.Add(node->child(i)); } } } } -}; + + SplitQ(q_node, list_new); +} + +template +void RecursiveBreadthFirstDualTreeRunner::EnumerateLeaves( + const QNode *q_node, const RNode *r_node, + const QMassResult& old_q_mass_result, + ) { + DEBUG_ASSERT(q_node->is_leaf()); + + if (likely(Algorithm::ConsiderPairExtrinsic(param_, *q_node, *r_node, + *delta, old_q_mass_result(), *global_result_, + &list_new.q_postponed()))) { + if (r_node->is_leaf()) { + void Add(const RNode *r_node) { + Entry *entry = list_.AddBack(); + bool try_explore = Algorithm::ConsiderPairIntrinsic( + *param_, *q_node_, *r_node, + &entry->delta, global_result_, &q_postponed_); + if (try_explore) { + entry->r_node = r_node; + q_mass_result_.ApplyDelta(*param_, entry->delta); + } else { + list_.PopBack(); + } + } + node_list->AddBack(r_node); + } else { + global_result_.UndoDelta(*delta); + for (int c = 0; c < cardinality; c++) { + global_result_.ApplyDelta(*delta); + EnumerateLeaves(q_node, node->child(i)); + } + } + } +} + +template +void RecursiveBreadthFirstDualTreeRunner::HandleQueryLeaf( + const QNode *q_node, const Queue& list_old) { + Queue list_new; + + DEBUG_ASSERT(q->is_leaf()); + + check for termination + + list_new.Init(q_node, list_old); + + for (index_t i = 0; i < list_old.size(); i++) { + const RNode *r_node = list_old.rnode(i); + const Delta* delta = &list_old.delta(i); + + EnumerateLeaves(q_node, r_node, list_old.q_mass_result(), &list_new); + } + + + MinHeap priority_queue; + + for (index_t i = 0; i < list_new.size(); i++) { + const Queue::Entry* entry = list_new.entry(i); + double heuristic = Algorithm::Heuristic(param_, *q_node, *entry->r_node, + entry->delta, list_old.mass_result()); + priority_queue.Put(-heuristic, entry); + } + + ArrayList entries; + ArrayList mass_results; + const QMassResult* cur_mass_result = ; + + mass_results.Init(new_list.size()); + entries.Init(new_list.size()); + + for (index_t i = new_list.size(); i--;) { + entries[i] = priority_queue.Pop(); + mass_results[i].Copy(*cur_mass_result); + cur_mass_result = &mass_results[i]; + } + +} diff --git a/fastlib/u/garryb/superpar/tkde.cc b/fastlib/u/garryb/superpar/tkde.cc index 08e93bd9a8..984c8f3d4c 100644 --- a/fastlib/u/garryb/superpar/tkde.cc +++ b/fastlib/u/garryb/superpar/tkde.cc @@ -47,17 +47,35 @@ class Tkde { } }; - struct QInfo {}; + struct BlankInfo { + template + void Serialize(Serializer *s) const {} + template + void Deserialize(Deserializer *s) {} + }; - struct RInfo {}; + typedef BlankInfo QInfo; + typedef BlankInfo RInfo; struct MomentInfo { ALLOW_COPY(MomentInfo); - public: Vector mass; double sumsq; index_t count; + + template + void Serialize(Serializer *s) const { + mass->Serialize(s); + s->Put(sumsq); + s->Put(count); + } + template + void Deserialize(Deserializer *s) { + mass->Deserialize(s); + s->Get(&sumsq); + s->Get(&count); + } void Init(const TkdeParam& param) { mass.Init(param.dim); @@ -110,6 +128,15 @@ class Tkde { struct TkdeStat { MomentInfo moment_info; + + template + void Serialize(Serializer *s) const { + moment_info->Serialize(s); + } + template + void Deserialize(Deserializer *s) { + moment_info->Deserialize(s); + } void InitZero(const TkdeParam& param) { moment_info.Init(param); @@ -147,6 +174,16 @@ class Tkde { /** We pruned an entire part of the tree with a particular label. */ Label label; + template + void Serialize(Serializer *s) const { + moment_info.Serialize(s); + s->Put(label); + } + template + void Deserialize(Deserializer *s) { + moment_info->Deserialize(s); + s->Get(&label); + } void Init(const TkdeParam& param) { moment_info.Init(param); @@ -169,6 +206,15 @@ class Tkde { struct TkdeDelta { /** Density update to apply to children's bound. */ DRange d_density; + + template + void Serialize(Serializer *s) const { + d_density.Serialize(s); + } + template + void Deserialize(Deserializer *s) { + d_density.Deserialize(s); + } void Init(const TkdeParam& param) { d_density.Init(0, 0); @@ -184,6 +230,17 @@ class Tkde { double density; Label label; + template + void Serialize(Serializer *s) const { + s->Put(density); + s->Put(label); + } + template + void Deserialize(Deserializer *s) { + s->Get(&density); + s->Get(&label); + } + void Init(const TkdeParam& param, const Vector& q_point, const QInfo& q_info, const RNode& r_root) { @@ -214,6 +271,10 @@ class Tkde { }; class TkdeGlobalResult { + template + void Serialize(Serializer *s) const {} + template + void Deserialize(Deserializer *s) {} void Init(const TkdeParam& param) {} void Accumulate(const TkdeParam& param, const TkdeGlobalResult& other_global_result) {} @@ -227,6 +288,17 @@ class Tkde { DRange density; Label label; + template + void Serialize(Serializer *s) const { + density.Serialize(s); + s->Put(label); + } + template + void Deserialize(Deserializer *s) { + density.Deserialize(s); + s->Get(&label); + } + void Copy(const TkdeMassResult& other) { density = other.density; label = other.label; diff --git a/fastlib/u/garryb/superpar/todo.txt b/fastlib/u/garryb/superpar/todo.txt new file mode 100644 index 0000000000..4442874992 --- /dev/null +++ b/fastlib/u/garryb/superpar/todo.txt @@ -0,0 +1,81 @@ +remember 143 + +/- serialization of tkde +- array char abstraction +- array char mem abstraction +- array char disk abstraction +- array char local cache abstraction +- array abstraction (kind of done already) +- dfs + + +- how to resolve write conflicts + - a: bit array + - b: ranges (requires carefulness) + - c: block exclusivity, enforced + - d: do it manually + - c: block exclusivity, assumed + + blocker.read_block(i).offset(i) + +- question: in DFS how do we get fault tolerance? + XXX- option 1: rely on mu + - we have to know the state of mu + - impossible to guarantee using a cache + - we never know the state of the data + - option 2: nuke + - we can always nuke a region of the query tree + - also required for work stealing + +final answer: + - bit-vector or pre-allocated dirty flags + + +- abstractions + - to make network/disk access easy we use block backends. but there + are two frontends to these: + - scanning frontend (cache size is single block) + - Scanner(blockdevice, begin_id, end_id) + - sequential access + - can we put lots of scanners in a single dude? + - we don't want to + - avoid having tiny files + - allow run-time decision of backends: use virtual functions + - Cache(blockdevice, begin_id, end_id) + - random-access read write + +void treebuild(array) { + makesample(array, &sample) + tree <- maketree(sample) + assign a number 1 to B to each leaf of the tree + + newarray <- memarray hosted at me + allocators <- new blockallocators[B] + links <- new int[B] + + for elem in array: + assign i to elem + int address = allocators[i].allocate() + newarray[address].content <- memcpy(elem) + if links[i]: + newarray[links[i]].link <- address + links[i] = address + + for i in range(B) in parallel: + treebuild(newarray, []) + finish postponed items (the work queue thing realizes this thread is + blocking and uses it to run one of the work + threads) +} + + +NetManager + + - simulates java RPC + manager.register(id, networklistener) + manager.unregister(id) + manager.send(id, datasize, data) + manager.workqueue_crap_crap + + +