hi

fastlib/u/garryb/superpar/notes.txt

@@ -42,3 +42,74 @@ Array
     - blocks have no alignment
     - read blocks on first use
     - explicit ranged writebacks
+
+
+how to build tree
+
+- build first (log p) levels in parallel
+  - first: assume block size >= p
+    - first block is not completely filled
+  - next: if block size < p
+    - second layer of blocks not completely filled
+    - this may be space-hungry if B^2 < N, but in that case, you're kind of
+    silly for using that many processors
+  - so overall
+    - block wastage: P blocks max (really B^(floor((log P)/(log B)))),
+      (N log P)/(B log B) percent
+- fits in memory
+  - arrays are synced by region
+  - trees are private-write
+- doesn't fit in memory
+  - first:
+    - ensure writable portion fits in memory, lock it, and private writes
+      - allocate cache that can exactly fit it?
+    - non-writable parts of tree are not locked
+    - DFS: fit entire query into RAM, fit useful part of reference into RAM
+    - RBFS: only one or logarithmic query nodes even needed in RAM at a time
+      - for best performance, all relevant references must fit
+        - okay for nearest neighbors (linear search space), bad for larger
+        search spaces
+        - generalized: okay for *any* algorithm that performs well
+        single-tree!!!!!!!!!
+  - later: any point in *not* making this assumption?
+    - breadth first?
+      - query-reference problems: use recursive breadth first
+      - global problems: no writes to trees necessary
+
+how to execute
+
+- operation by pattern
+  - in best conditions these are all O((Q+R)/B)
+  - DFS
+    - (QR)/(MB)
+    - tree
+      - query: lock into RAM, private writes
+      - reference: load via cache
+      - I/O worst-case: (4QR/(MB))
+    - data:
+      - make sure query fits in RAM
+      - reference: ranged
+  - RBFS
+    - (QR log B)/(B) without block-chunking
+    - tree
+      - query: one block at a time
+      - reference: via cache
+        - if reference set stays large, cache performance will be abysmal
+        - make sure references are sorted based on block order
+        - if it was N^2 time before I/O efficiency would be
+          - I/O rate: O(Q(R/B)(log B)), but it could have been
+            O((Q/M)(R/B))
+        - *sigh* still better than single-tree
+    - data: cached with ranges
+  - Block breadth-first
+    - (QR log B)/(MB)
+    - looks at pairs of blocks
+    - tree: only one query and one references needed at any time
+    - queue size: QR/B^2
+    - I/O worst-case: either QR/B^2 or (4QR/(MB))(log B)
+      - towards the bottom, we end up working on pairs of half-memory-sized
+      chunks
+      - we may choose number of points per leaf to be comparable to log B
+    - for simplicity, my first version simply won't support mu
+    (mu will just equal the root's delta); we will just target
+    gamma-based problems

fastlib/u/garryb/superpar/rbfs.h

@@ -168,8 +168,10 @@ class RecursiveBreadthFirstDualTreeRunner {
   Param param_;
   ArrayList<QInfo> q_info_;
   ArrayList<RInfo> r_info_;
-  QNode *q_root_;
-  RNode *r_root_;
+  const QNode *q_root_;
+  const RNode *r_root_;
+  ArrayList<QResult> q_result_;
+  GlobalResult global_result_;
   
  public:
   void Init(struct datanode *module,
@@ -187,11 +189,15 @@ class RecursiveBreadthFirstDualTreeRunner {
   }
   
   ArrayList<QInfo>& q_info() {
-    return q_info;
+    return q_info_;
   }
   
   ArrayList<RInfo>& r_info() {
-    return r_info;
+    return r_info_;
+  }
+  
+  ArrayList<QResult>& q_result() {
+    return q_result_;
   }
 
  private:
@@ -203,20 +209,20 @@ class RecursiveBreadthFirstDualTreeRunner {
         DEBUG_POISON_PTR(r_node);
       }
       
-      RNode *r_node;
+      const RNode *r_node;
       Delta delta;
     };
    
    private: 
     const Param *param_;
-    QNode *q_node_;
+    const QNode *q_node_;
     ArrayList<Entry> list_;
-    MassResult q_mass_result_;
-    PostponedResult q_postponed_;
+    QMassResult q_mass_result_;
+    QPostponedResult q_postponed_;
     GlobalResult *global_result_;
     
    public:
-    void Init(QNode* q_node_in, const Param* param,
+    void Init(const QNode* q_node_in, const Param* param,
         GlobalResult* global_result_in) {
       param_ = param;
       q_node_ = q_node_in;
@@ -230,7 +236,7 @@ class RecursiveBreadthFirstDualTreeRunner {
       global_result_ = global_result_in;
     }
     
-    void Init(QNode* q_node_in, const Queue& parent) {
+    void Init(const QNode* q_node_in, const Queue& parent) {
       param_ = parent.param;
       q_node_ = q_node_in;
       
@@ -244,14 +250,14 @@ class RecursiveBreadthFirstDualTreeRunner {
       global_result_ = parent.global_result_;
     }
     
-    void Add(RNode *r_node) {
+    void Add(const RNode *r_node) {
       Entry *entry = list_.AddBack();
       bool try_explore = Algorithm::ConsiderPairIntrinsic(
           *param_, *q_node_, *r_node,
-          &entry->delta, &q_mass_result_, global_result_, &q_postponed_);
-      
+          &entry->delta, global_result_, &q_postponed_);
       if (try_explore) {
         entry->r_node = r_node;
+        q_mass_result_.ApplyDelta(*param_, entry->delta);
       } else {
         list_.PopBack();
       }
@@ -266,7 +272,7 @@ class RecursiveBreadthFirstDualTreeRunner {
       return list_.size();
     }
     
-    RNode* rnode(index_t i) const {
+    const RNode* rnode(index_t i) const {
       return ;
     }
     
@@ -275,92 +281,195 @@ class RecursiveBreadthFirstDualTreeRunner {
       return list_[i].delta;
     }
 
-    const MassResult& q_mass_result() const { return q_mass_result_; }
+    const QMassResult& q_mass_result() const { return q_mass_result_; }
     
-    const PostponedResult& q_postponed() const { return q_postponed_; }
+    const QPostponedResult& q_postponed() const { return q_postponed_; }
     
-    PostponedResult& q_postponed() { return q_postponed_; }
+    QPostponedResult& q_postponed() { return q_postponed_; }
   };
 
-
-  void SplitQ(QNode *q_node, const Queue& list_old) {
-    if (q_node->is_leaf()) {
-      DoQLeafStuffWALDO_TODO(q_node, list_old);
-      return;
-    }
-    
-    if (Algorithm::ConsiderQueryTermination(&param_, *q_node,
+  void RecursivelyApplyPostponed(
+      const QNode* q_node, const QPostponedResult& postponed_result);
+  
+  bool TryTerminate(const QNode *q_node, const Queue& list_old) {
+    if (Algorithm::ConsiderQueryTermination(param_, *q_node,
         list_old.q_mass_result(), *global_result_, &list_old.q_postponed())) {
       RecursivelyApplyPostponed(q_node, list_old.q_postponed());
       return;
     }
-
-    Queue list_new[cardinality];
-    
-    /* TODO: termination prunes can be checked here */
-    
-    for (int c = 0; c < cardinality; c++) {
-      list_new[c].Init(q_node->child(i), list_old);
-    }
-    
-    QPostponedResults postponed;
-    
-    postponed.Init(*param_);
-    // We haven't done any exhaustive comparisons, we start with an empty.
-    
-    for (index_t i = 0; i < list_old.size(); i++) {
-      RNode *r_node = list_old.rnode(i);
-      const Delta* delta = &list_old.delta(i);
-      
-      if (likely(Algorithm::ConsiderPairExtrinsic(param_, *q_node, *r_node,
-          *delta, list_old.q_mass_result(), *global_result_, &postponed))) {
-        global_result_.UndoDelta(param_, *delta);
-        for (int c_q = 0; c_q < cardinality; c_q++) {
-          global_result_.ApplyDelta(param_, *delta);
-          list_new[c_q].Add(r_node);
-        }
-      }
-    }
-    
-    /* recurse over query children */
-    
-    for (int c = 0; c < cardinality; c++) {
-      list_new[c].q_postponed().ApplyPostponed(*param_, postponed);
-      list_new[c].Finish(param_);
-      SplitR(q_node->child(c), list_new[c]);
-    }
   }
 
-  void SplitR(QNode *q_node, const ArrayList<Entry>& list_old) {
-    Queue list_new;
+  void FinishQ(const QNode *q_node, const Queue& list_old);
+
+  void SplitQ(const QNode *q_node, const Queue& list_old);
+
+  void SplitR(const QNode *q_node, const ArrayList<Entry>& list_old);
+};
+
+template<class GNP>
+void RecursiveBreadthFirstDualTreeRunner<GNP>::RecursivelyApplyPostponed(
+    const QNode* q_node, const QPostponedResult& postponed_result) {
+  for (index_t i = q_node->begin(); i < q_node->end(); i++) {
+    q_results_[i].ApplyPostponedResult(param_, postponed_result, *q_node);
+  }
+}
+
+template<class GNP>
+void RecursiveBreadthFirstDualTreeRunner<GNP>::SplitQ(
+    const QNode *q_node, const Queue& list_old) {
+  if (q_node->is_leaf()) {
+    RecurseOnLeaves(q_node, list_old);
+    return;
+  }
+  
+  if (TryTerminate(q_node, list_old)) {
+    return;
+  }
+
+  Queue list_new[cardinality];
+  
+  /* TODO: termination prunes can be checked here */
+  
+  for (int c = 0; c < cardinality; c++) {
+    list_new[c].Init(q_node->child(i), list_old);
+  }
+  
+  QPostponedResults postponed;
+  
+  postponed.Init(param_);
+  // We haven't done any exhaustive comparisons, we start with an empty.
+  
+  for (index_t i = 0; i < list_old.size(); i++) {
+    const RNode *r_node = list_old.rnode(i);
+    const Delta* delta = &list_old.delta(i);
     
-    if (Algorithm::ConsiderQueryTermination(&param_, *q_node,
-        list_old.q_mass_result(), *global_result_, &list_old.q_postponed())) {
-      RecursivelyApplyPostponed(q_node, list_old.q_postponed());
-      return;
+    if (likely(Algorithm::ConsiderPairExtrinsic(param_, *q_node, *r_node,
+        *delta, list_old.q_mass_result(), *global_result_, &postponed))) {
+      global_result_.UndoDelta(param_, *delta);
+      for (int c_q = 0; c_q < cardinality; c_q++) {
+        global_result_.ApplyDelta(param_, *delta);
+        list_new[c_q].Add(r_node);
+      }
     }
+  }
+  
+  /* recurse over query children */
+  
+  for (int c = 0; c < cardinality; c++) {
+    list_new[c].q_postponed().ApplyPostponed(param_, postponed);
+    list_new[c].Finish(param_);
+    SplitR(q_node->child(c), list_new[c]);
+  }
+}
+
+template<class GNP>
+void RecursiveBreadthFirstDualTreeRunner<GNP>::SplitR(
+    const QNode *q_node, const ArrayList<Entry>& list_old) {
+  Queue list_new;
+  
+  if (TryTerminate(q_node, list_old)) {
+    return;
+  }
+  
+  list_new.Init(q_node, list_old);
+  
+  for (index_t i = 0; i < list_old.size(); i++) {
+    const RNode *r_node = list_old.rnode(i);
+    const Delta* delta = &list_old.delta(i);
     
-    list_new.Init(q_node, list_old);
-    
-    for (index_t i = 0; i < list_old.size(); i++) {
-      RNode *r_node = list_old.rnode(i);
-      const Delta* delta = &list_old.delta(i);
-      
+    if (likely(Algorithm::ConsiderPairExtrinsic(param_, *q_node, *r_node,
+        *delta, list_old.q_mass_result(), *global_result_,
+        &list_new.q_postponed()))) {
       if (entry_old->r_node->is_leaf()) {
         // TODO: We can collapse the mu's for these together
         list_new.Add(r_node);
       } else {
-        if (likely(Algorithm::ConsiderPairExtrinsic(param_, *q_node, *r_node,
-            *delta, list_old.q_mass_result(), *global_result_,
-            &list_new.q_postponed()))) {
-          global_result_.UndoDelta(*delta);
-          for (int c = 0; c < cardinality; c++) {
-            global_result_.ApplyDelta(*delta);
-            list_new.Add(node->child(i));
-          }
+        global_result_.UndoDelta(*delta);
+        for (int c = 0; c < cardinality; c++) {
+          global_result_.ApplyDelta(*delta);
+          list_new.Add(node->child(i));
         }
       }
     }
   }
-};
+  
+  SplitQ(q_node, list_new);
+}
+
+template<class GNP>
+void RecursiveBreadthFirstDualTreeRunner<GNP>::EnumerateLeaves(
+    const QNode *q_node, const RNode *r_node,
+    const QMassResult& old_q_mass_result,
+    ) {
+  DEBUG_ASSERT(q_node->is_leaf());
+
+  if (likely(Algorithm::ConsiderPairExtrinsic(param_, *q_node, *r_node,
+      *delta, old_q_mass_result(), *global_result_,
+      &list_new.q_postponed()))) {
+    if (r_node->is_leaf()) {
+      void Add(const RNode *r_node) {
+        Entry *entry = list_.AddBack();
+        bool try_explore = Algorithm::ConsiderPairIntrinsic(
+            *param_, *q_node_, *r_node,
+            &entry->delta, global_result_, &q_postponed_);
+        if (try_explore) {
+          entry->r_node = r_node;
+          q_mass_result_.ApplyDelta(*param_, entry->delta);
+        } else {
+          list_.PopBack();
+        }
+      }
+      node_list->AddBack(r_node);
+    } else {
+      global_result_.UndoDelta(*delta);
+      for (int c = 0; c < cardinality; c++) {
+        global_result_.ApplyDelta(*delta);
+        EnumerateLeaves(q_node, node->child(i));
+      }
+    }
+  }
+}
+
+template<class GNP>
+void RecursiveBreadthFirstDualTreeRunner<GNP>::HandleQueryLeaf(
+    const QNode *q_node, const Queue& list_old) {
+  Queue list_new;
+  
+  DEBUG_ASSERT(q->is_leaf());
+  
+  check for termination
+  
+  list_new.Init(q_node, list_old);
+  
+  for (index_t i = 0; i < list_old.size(); i++) {
+    const RNode *r_node = list_old.rnode(i);
+    const Delta* delta = &list_old.delta(i);
+    
+    EnumerateLeaves(q_node, r_node, list_old.q_mass_result(), &list_new);
+  }
+  
+  
+  MinHeap<double, Queue::Entry*> priority_queue;
+  
+  for (index_t i = 0; i < list_new.size(); i++) {
+    const Queue::Entry* entry = list_new.entry(i);
+    double heuristic = Algorithm::Heuristic(param_, *q_node, *entry->r_node,
+        entry->delta, list_old.mass_result());
+    priority_queue.Put(-heuristic, entry);
+  }
+  
+  ArrayList<const Queue::Entry*> entries;
+  ArrayList<QMassResult> mass_results;
+  const QMassResult* cur_mass_result = ;
+  
+  mass_results.Init(new_list.size());
+  entries.Init(new_list.size());
+  
+  for (index_t i = new_list.size(); i--;) {
+    entries[i] = priority_queue.Pop();
+    mass_results[i].Copy(*cur_mass_result);
+    cur_mass_result = &mass_results[i];
+  }
+  
+}
 

fastlib/u/garryb/superpar/tkde.cc

@@ -47,17 +47,35 @@ class Tkde {
     }
   };
 
-  struct QInfo {};
+  struct BlankInfo {
+    template<typename Serializer>
+    void Serialize(Serializer *s) const {}
+    template<typename Deserializer>
+    void Deserialize(Deserializer *s) {}
+  };
 
-  struct RInfo {};
+  typedef BlankInfo QInfo;
+  typedef BlankInfo RInfo;
 
   struct MomentInfo {
     ALLOW_COPY(MomentInfo);
     
-   public:
     Vector mass;
     double sumsq;
     index_t count;
+
+    template<typename Serializer>
+    void Serialize(Serializer *s) const {
+      mass->Serialize(s);
+      s->Put(sumsq);
+      s->Put(count);
+    }
+    template<typename Deserializer>
+    void Deserialize(Deserializer *s) {
+      mass->Deserialize(s);
+      s->Get(&sumsq);
+      s->Get(&count);
+    }
     
     void Init(const TkdeParam& param) {
       mass.Init(param.dim);
@@ -110,6 +128,15 @@ class Tkde {
 
   struct TkdeStat {
     MomentInfo moment_info;
+
+    template<typename Serializer>
+    void Serialize(Serializer *s) const {
+      moment_info->Serialize(s);
+    }
+    template<typename Deserializer>
+    void Deserialize(Deserializer *s) {
+      moment_info->Deserialize(s);
+    }
     
     void InitZero(const TkdeParam& param) {
       moment_info.Init(param);
@@ -147,6 +174,16 @@ class Tkde {
     /** We pruned an entire part of the tree with a particular label. */
     Label label;
     
+    template<typename Serializer>
+    void Serialize(Serializer *s) const {
+      moment_info.Serialize(s);
+      s->Put(label);
+    }
+    template<typename Deserializer>
+    void Deserialize(Deserializer *s) {
+      moment_info->Deserialize(s);
+      s->Get(&label);
+    }
 
     void Init(const TkdeParam& param) {
       moment_info.Init(param);
@@ -169,6 +206,15 @@ class Tkde {
   struct TkdeDelta {
     /** Density update to apply to children's bound. */
     DRange d_density;
+
+    template<typename Serializer>
+    void Serialize(Serializer *s) const {
+      d_density.Serialize(s);
+    }
+    template<typename Deserializer>
+    void Deserialize(Deserializer *s) {
+      d_density.Deserialize(s);
+    }
     
     void Init(const TkdeParam& param) {
       d_density.Init(0, 0);
@@ -184,6 +230,17 @@ class Tkde {
     double density;
     Label label;
 
+    template<typename Serializer>
+    void Serialize(Serializer *s) const {
+      s->Put(density);
+      s->Put(label);
+    }
+    template<typename Deserializer>
+    void Deserialize(Deserializer *s) {
+      s->Get(&density);
+      s->Get(&label);
+    }
+
     void Init(const TkdeParam& param,
         const Vector& q_point, const QInfo& q_info,
         const RNode& r_root) {
@@ -214,6 +271,10 @@ class Tkde {
   };
 
   class TkdeGlobalResult {
+    template<typename Serializer>
+    void Serialize(Serializer *s) const {}
+    template<typename Deserializer>
+    void Deserialize(Deserializer *s) {}
     void Init(const TkdeParam& param) {}
     void Accumulate(const TkdeParam& param,
         const TkdeGlobalResult& other_global_result) {}
@@ -227,6 +288,17 @@ class Tkde {
     DRange density;
     Label label;
 
+    template<typename Serializer>
+    void Serialize(Serializer *s) const {
+      density.Serialize(s);
+      s->Put(label);
+    }
+    template<typename Deserializer>
+    void Deserialize(Deserializer *s) {
+      density.Deserialize(s);
+      s->Get(&label);
+    }
+
     void Copy(const TkdeMassResult& other) {
       density = other.density;
       label = other.label;

fastlib/u/garryb/superpar/todo.txt

@@ -0,0 +1,81 @@
+remember 143
+
+/- serialization of tkde
+- array char abstraction
+- array char mem abstraction
+- array char disk abstraction
+- array char local cache abstraction
+- array abstraction (kind of done already)
+- dfs
+
+
+- how to resolve write conflicts
+  - a: bit array
+  - b: ranges (requires carefulness)
+  - c: block exclusivity, enforced
+  - d: do it manually
+  - c: block exclusivity, assumed
+
+  blocker.read_block(i).offset(i)
+
+- question: in DFS how do we get fault tolerance?
+  XXX- option 1: rely on mu
+    - we have to know the state of mu
+    - impossible to guarantee using a cache
+    - we never know the state of the data
+  - option 2: nuke
+    - we can always nuke a region of the query tree
+    - also required for work stealing
+
+final answer:
+  - bit-vector or pre-allocated dirty flags
+
+
+- abstractions
+  - to make network/disk access easy we use block backends.  but there
+  are two frontends to these:
+    - scanning frontend (cache size is single block)
+      - Scanner(blockdevice, begin_id, end_id)
+        - sequential access
+        - can we put lots of scanners in a single dude?
+          - we don't want to
+          - avoid having tiny files
+          - allow run-time decision of backends: use virtual functions
+      - Cache(blockdevice, begin_id, end_id)
+        - random-access read write
+
+void treebuild(array) {
+  makesample(array, &sample)
+  tree <- maketree(sample)
+  assign a number 1 to B to each leaf of the tree
+
+  newarray <- memarray hosted at me
+  allocators <- new blockallocators[B]
+  links <- new int[B]
+  
+  for elem in array:
+    assign i to elem
+    int address = allocators[i].allocate()
+    newarray[address].content <- memcpy(elem)
+    if links[i]:
+      newarray[links[i]].link <- address
+    links[i] = address
+
+  for i in range(B) in parallel:
+    treebuild(newarray, [])
+  finish postponed items (the work queue thing realizes this thread is
+                          blocking and uses it to run one of the work
+                          threads)
+}
+
+
+NetManager
+
+  - simulates java RPC
+  manager.register(id, networklistener)
+  manager.unregister(id)
+  manager.send(id, datasize, data)
+  manager.workqueue_crap_crap
+
+
+