From 3be6b5c72a795b2bacf5fd2d693ba4de4b05fe8d Mon Sep 17 00:00:00 2001 From: Garry Boyer Date: Tue, 10 Jul 2007 05:27:16 +0000 Subject: [PATCH] added intmap --- fastlib/base/common.h | 2 + fastlib/col/arraylist.h | 128 +++++++++++++++++++--------------- fastlib/col/build.py | 2 +- fastlib/col/col_test.cc | 19 ++++- fastlib/col/intmap.h | 62 ++++++++++++++++ fastlib/fastlib/fastlib.h | 3 + fastlib/fastlib/fastlib_int.h | 5 ++ fastlib/la/build.py | 4 +- fastlib/par/mpigrain.h | 100 +++++++++++++------------- fastlib/par/thread.h | 24 +++---- 10 files changed, 227 insertions(+), 122 deletions(-) create mode 100644 fastlib/col/intmap.h diff --git a/fastlib/base/common.h b/fastlib/base/common.h index 6412bd3cce..e39d9289fc 100644 --- a/fastlib/base/common.h +++ b/fastlib/base/common.h @@ -14,6 +14,8 @@ #ifndef BASE_COMMON_H #define BASE_COMMON_H +#define _REENTRANT + #ifdef __cplusplus #include "cc.h" #endif diff --git a/fastlib/col/arraylist.h b/fastlib/col/arraylist.h index 75010a1b25..5401c2764b 100644 --- a/fastlib/col/arraylist.h +++ b/fastlib/col/arraylist.h @@ -65,12 +65,12 @@ class ArrayList { Element* ptr_; index_t size_; index_t cap_; - + OT_DEF(ArrayList) { OT_MY_OBJECT(size_); OT_MALLOC_ARRAY_NULLABLE(ptr_, size_); } - + OT_FIX(ArrayList) { cap_ = size_; } @@ -89,7 +89,7 @@ class ArrayList { ~ArrayList() { Destruct(); } - + /** * Returns this to an invalid state so it can be re-initialized. * @@ -124,7 +124,7 @@ class ArrayList { size_ = 0; cap_ = 0; } - + /** * Initializes to a given size. * @@ -134,7 +134,7 @@ class ArrayList { void Init(index_t size_in) { Init(size_in, size_in); } - + /** * Initializes with a given size, but with a perhaps larger allocation. * @@ -144,16 +144,16 @@ class ArrayList { void Init(index_t size_in, index_t cap_in) { DEBUG_ASSERT_MSG(size_ == BIG_BAD_NUMBER, "reinitialization not allowed"); DEBUG_ASSERT(size_in <= cap_in); - + size_ = size_in; cap_ = cap_in; - + ptr_ = mem::Alloc(cap_); // TODO: Default integer constructor initializes to zero; is there // a way to avoid this? mem::ConstructAll(ptr_, size_); } - + /** * Copies from another ArrayList. * @@ -162,7 +162,7 @@ class ArrayList { void Copy(const ArrayList& other) { Copy(other.ptr_, other.size_); } - + /** * Copies bit-for-bit from another array. * @@ -171,14 +171,17 @@ class ArrayList { */ void Copy(const Element *ptr, index_t size) { DEBUG_ASSERT_MSG(size_ == BIG_BAD_NUMBER, "reinitialization not allowed"); - + ptr_ = mem::DupConstruct(ptr, size); cap_ = size; size_ = size; } - + /** - * Resets to zero size. + * Resets to zero size and frees RAM. + * + * This is slower than Resize(0), since Resize(0) will hold onto the RAM + * that was previously in use. */ void Clear() { mem::Free(ptr_); @@ -186,14 +189,14 @@ class ArrayList { size_ = 0; cap_ = 0; } - + /** * Steals the contents of another ArrayList, initializing this ArrayList and * making the other array list zero in size. */ void Steal(ArrayList* other) { DEBUG_ASSERT_MSG(size_ == BIG_BAD_NUMBER, "reinitialization not allowed"); - + ptr_ = other->ptr_; size_ = other->size_; cap_ = other->cap_; @@ -201,21 +204,21 @@ class ArrayList { other->size_ = 0; other->cap_ = 0; } - + /** * Steals the contents of another ArrayList, initializing this ArrayList and * destructing the other ArrayList. */ void StealDestruct(ArrayList* other) { DEBUG_ASSERT_MSG(size_ == BIG_BAD_NUMBER, "reinitialization not allowed"); - + ptr_ = other->ptr_; size_ = other->size_; cap_ = other->cap_; - + DEBUG_ONLY(other->Invalidate_()); } - + /** * Initializes this to a pointer allocated with mem::Alloc. * @@ -228,12 +231,12 @@ class ArrayList { */ void Steal(Element *ptr, index_t len, index_t capacity) { DEBUG_ASSERT_MSG(size_ == BIG_BAD_NUMBER, "reinitialization not allowed"); - + ptr_ = ptr; size_ = len; cap_ = capacity; } - + /** * Returns the pointer to the beginning of the array, and reinitializes this * list to empty. @@ -245,14 +248,24 @@ class ArrayList { cap_ = 0; return retval; } - + /** * Switches the arrays pointed to by each array. */ void Swap(ArrayList *other) { - mem::Swap(this, other); + Element *t_ptr = other->ptr_; + other->ptr_ = ptr_; + ptr_ = t_ptr; + + index_t t_size = other->size_; + other->size_ = size_; + size_ = t_size; + + index_t t_cap = other->cap_; + other->cap_ = cap_; + cap_ = t_cap; } - + /** * Explicitly sets the size of the list. * @@ -284,7 +297,7 @@ class ArrayList { GrowTo(size_min); } } - + /** * Serializes this arraylist. * @@ -296,7 +309,7 @@ class ArrayList { s->Put(size_); s->Put(ptr_, size_); } - + /** * Initializes this list, deserializing from the given source. */ @@ -307,7 +320,7 @@ class ArrayList { ptr_ = mem::Alloc(cap_); s->Get(ptr_, size_); } - + /** * Use this to shrink the ArrayList. */ @@ -315,7 +328,7 @@ class ArrayList { DecreaseSizeHelper_(size_in); size_ = size_in; } - + /** * Use this to grow the ArrayList. */ @@ -323,7 +336,7 @@ class ArrayList { IncreaseSizeHelper_(size_in); size_ = size_in; } - + /** * Use this to grow the size by a specified amount, returning a pointer * to the beginning of the chunk. @@ -332,16 +345,16 @@ class ArrayList { if (unlikely(size_ + size_increment > cap_)) { IncreaseCap_(cap_ * 2 + size_increment); } - + Element* chunk = ptr_ + size_; - + size_ += size_increment; - + mem::ConstructAll(chunk, size_increment); - + return chunk; } - + /** * Adds one new element to the back, and returns the pointer to it. * @@ -364,15 +377,15 @@ class ArrayList { if (unlikely(size_ == cap_)) { IncreaseCap_((cap_ + 1) * 2); } - + Element* elem = ptr_ + size_; - + ++size_; mem::Construct(elem); // call default constructor - + return elem; } - + /** * Removes the last element of the list. */ @@ -381,19 +394,22 @@ class ArrayList { --size_; mem::Destruct(ptr_ + size_); } - + /** * Returns a pointer to the last element, and decreases the size. * - * Note that it is *your* responsibility to call the destructor of this - * object if it is not a structable (mem::Destruct). + * Note that it is *your* responsibility to call the destructor (using + * mem::Destruct or the destructor explicitly) of this + * object if it is not a primitive. + * + * This will be invalidated if the ArrayList is subsequently trimmed. */ Element* PopBackPtr() { --size_; - + return ptr_ + size_; } - + /** * Reallocates to the minimum memory usage to hold the data in the array. * @@ -402,7 +418,7 @@ class ArrayList { void Trim() { DecreaseCap_(size_); } - + /** * Gets a constant element out of a constant ArrayList. */ @@ -418,7 +434,7 @@ class ArrayList { DEBUG_BOUNDS(i, size_); return ptr_[i]; } - + public: /** * Gets the number of elements. @@ -426,7 +442,7 @@ class ArrayList { index_t size() const { return size_; } - + /** * Gets the number of elements this can hold without performing any * additional reallocations. @@ -434,49 +450,49 @@ class ArrayList { index_t capacity() const { return cap_; } - + /** * Returns a pointer to the first element. */ const Element* begin() const { return ptr_; } - + /** * Returns a pointer one beyond the last element. */ const Element* end() const { return ptr_ + size_; } - + /** * Returns a pointer to the last element. */ const Element* last() const { return ptr_ + size_ - 1; } - + /** * Returns a pointer to the first element. */ Element* begin() { return ptr_; } - + /** * Returns a pointer one beyond the last element. */ Element* end() { return ptr_ + size_; } - + /** * Returns a pointer to the last element. */ Element* last() { return ptr_ + size_ - 1; } - + private: /** * Increases the size by reallocating intelligently. @@ -488,7 +504,7 @@ class ArrayList { } mem::ConstructAll(ptr_ + size_, size_in - size_); } - + /** * Decreases the size, but doesn't actually resize. */ @@ -496,14 +512,14 @@ class ArrayList { DEBUG_ASSERT(size_in <= size_); mem::DestructAll(ptr_ + size_in, size_ - size_in); } - + /** * Increases the array to be a larger size. * * This should not be inlined, because it is an unlikely case. */ void IncreaseCap_(index_t cap_in); - + /** * Reallocates the array to be smaller. */ @@ -511,7 +527,7 @@ class ArrayList { ptr_ = mem::Resize(ptr_, cap_in); cap_ = cap_in; } - + /** * Sets fields to invalid values to ensure earliest possible catching of * debugging problems. diff --git a/fastlib/col/build.py b/fastlib/col/build.py index f58f654797..50298f705a 100644 --- a/fastlib/col/build.py +++ b/fastlib/col/build.py @@ -1,7 +1,7 @@ librule( sources = ["col.cc"], - headers = ["arraylist.h", "heap.h", "string.h", "fastalloc.h"], + headers = ["arraylist.h", "heap.h", "string.h", "fastalloc.h", "intmap.h"], deplibs = ["base:base"] ) diff --git a/fastlib/col/col_test.cc b/fastlib/col/col_test.cc index deb3a945e5..a0fbbcf5a8 100644 --- a/fastlib/col/col_test.cc +++ b/fastlib/col/col_test.cc @@ -1,6 +1,7 @@ #include "arraylist.h" #include "heap.h" #include "fastalloc.h" +#include "intmap.h" #include "base/test.h" @@ -120,5 +121,21 @@ void TestFastAlloc() { fast_delete(b); } -TEST_SUITE_END(col, TestArrayListInt, TestMinHeap, TestFastAlloc) +void TestIntMap() { + DenseIntMap map; + map.Init(); + map.default_value() = 0.0; + map[31] = 31; + map[41] = 41; + map[59] = 59; + map[26] = 26; + TEST_DOUBLE_EXACT(map[0], 0); + TEST_DOUBLE_EXACT(map[499], 0); + TEST_DOUBLE_EXACT(map[31], 31); + TEST_DOUBLE_EXACT(map[41], 41); + TEST_DOUBLE_EXACT(map[59], 59); +} + +TEST_SUITE_END(col, TestArrayListInt, TestMinHeap, + TestFastAlloc, TestIntMap) diff --git a/fastlib/col/intmap.h b/fastlib/col/intmap.h new file mode 100644 index 0000000000..057f1ba272 --- /dev/null +++ b/fastlib/col/intmap.h @@ -0,0 +1,62 @@ +/** + * @file intmap.h + * + * Dense integer-to-value map. + */ + +template +class DenseIntMap { + FORBID_COPY(DenseIntMap); + + public: + typedef TValue Value; + + private: + Value *ptr_; + index_t size_; + Value default_value_; + + public: + DenseIntMap() { + DEBUG_POISON_PTR(ptr_); + size_ = BIG_BAD_NUMBER; + } + ~DenseIntMap() { + DEBUG_ASSERT(size_ != BIG_BAD_NUMBER); + mem::Free(ptr_); + } + + void Init() { + ptr_ = NULL; + size_ = 0; + } + + Value& default_value() { + return default_value_; + } + const Value& default_value() const { + return default_value_; + } + + Value& operator [] (index_t index) { + if (unlikely(index >= size_)) { + index_t old_size = size_; + size_ = max(size_ * 2, index + 1); + ptr_ = mem::Resize(ptr_, size_); + for (index_t i = old_size; i < size_; i++) { + new(ptr_+i)Value(default_value_); + } + } + return ptr_[index]; + } + const Value& operator [] (index_t index) const { + return get(index); + } + const Value& get(index_t index) const { + if (likely(index < size_)) { + return ptr_[index]; + } else { + return default_value_; + } + } +}; diff --git a/fastlib/fastlib/fastlib.h b/fastlib/fastlib/fastlib.h index ebb66c1a9a..d97f8a7443 100644 --- a/fastlib/fastlib/fastlib.h +++ b/fastlib/fastlib/fastlib.h @@ -6,6 +6,8 @@ * Includes all of fastlib. */ +#ifndef FASTLIB_FASTLIB_H +#define FASTLIB_FASTLIB_H #include "base/common.h" #include "base/cc.h" @@ -27,6 +29,7 @@ #include "tree/statistic.h" #include "tree/kdtree.h" +#endif /** @mainpage FASTlib Documentation * diff --git a/fastlib/fastlib/fastlib_int.h b/fastlib/fastlib/fastlib_int.h index a0ddf755d4..57cb31b951 100644 --- a/fastlib/fastlib/fastlib_int.h +++ b/fastlib/fastlib/fastlib_int.h @@ -6,6 +6,8 @@ * Internal FASTlib header file */ +#ifndef FASTLIB_FASTLIB_INT_H +#define FASTLIB_FASTLIB_INT_H #include "base/common.h" #include "base/cc.h" @@ -14,6 +16,7 @@ #include "col/arraylist.h" #include "col/heap.h" #include "col/string.h" +#include "col/intmap.h" #include "data/dataset.h" #include "data/crossvalidation.h" #include "math/math.h" @@ -29,3 +32,5 @@ #include "tree/kdtree.h" #include "base/otrav.h" + +#endif diff --git a/fastlib/la/build.py b/fastlib/la/build.py index 39f7b4af5b..dcab8fbe31 100644 --- a/fastlib/la/build.py +++ b/fastlib/la/build.py @@ -1,5 +1,5 @@ -# This first part is dedicated to compiling and installing LAPACK. -# Scroll down later... +# Scroll down to see the main part +# (This first part is dedicated to compiling and installing LAPACK.) wgetrule( name = "blaspack_tgz", diff --git a/fastlib/par/mpigrain.h b/fastlib/par/mpigrain.h index 81bd87a6b8..fb3a0119fa 100644 --- a/fastlib/par/mpigrain.h +++ b/fastlib/par/mpigrain.h @@ -37,7 +37,7 @@ template class MPIGrainRunner { FORBID_COPY(MPIGrainRunner); friend class MPIDispatcher; - + public: typedef TGrain Grain; typedef TContext Context; @@ -45,34 +45,34 @@ class MPIGrainRunner { class Dispatcher { FORBID_COPY(Dispatcher); friend class MPIGrainRunner; - + private: class MPIMasterTask : public Task { private: struct MPIGrainRunner *runner_; - + public: MPIMasterTask(MPIGrainRunner *runner_in) { runner_ = runner_in; } - + void Run() { int rank; int message; int n_slaves_alive = 0; int n_slaves_busy = 0; - + DEBUG_MSG(1.0, "DISPATCH: Firing up the cannons."); DEBUG_MSG(1.0, "DISPATCH: We will accomplish %u tasks.", unsigned(runner_->dispatcher_->queue_->size())); - + for (;;) { runner_->RecvInt_(&rank, &message); - + runner_->mutex_.Lock(); Grain* grain = runner_->dispatcher_->queue_->Pop(); runner_->mutex_.Unlock(); - + if (message == BIRTH) { DEBUG_MSG(1.0, "DISPATCH: Received birth message."); n_slaves_alive++; @@ -81,14 +81,14 @@ class MPIGrainRunner { DEBUG_ASSERT(message == GIVE_ME_WORK); DEBUG_MSG(1.0, "DISPATCH: Somebody wants more work."); } - + if (grain) { char buf[sizeof(Grain) + 1]; - + buf[0] = 1; // data is available mem::CopyBytes(buf + 1, grain, sizeof(*grain)); delete grain; - + DEBUG_MSG(1.0, "DISPATCH: Sending work on over."); MPI_Send(buf, sizeof(buf), MPI_CHAR, rank, runner_->tag_, MPI_COMM_WORLD); @@ -98,21 +98,21 @@ class MPIGrainRunner { break; } } - + DEBUG_MSG(1.0, "DISPATCH: Waiting for workers to die..."); - + /* Wait for all to die */ while (n_slaves_alive != 0) { // we received a message from the last loop - + if (message == GIVE_ME_WORK) { char buf[1]; - + buf[0] = 0; // tell them to die MPI_Send(buf, sizeof(buf), MPI_CHAR, rank, runner_->tag_, MPI_COMM_WORLD); - + DEBUG_MSG(1.0, "DISPATCH: I told a worker to die."); n_slaves_busy--; } else if (message == DEATH) { @@ -122,35 +122,35 @@ class MPIGrainRunner { DEBUG_ASSERT_MSG(0, "DISPATCHED: Message was %d??", message); } - + if (n_slaves_alive != 0) { runner_->RecvInt_(&rank, &message); } } - + DEBUG_MSG(1.0, "DISPATCH: All workers have died."); - + delete this; } }; - + private: MPIGrainRunner *runner_; GrainQueue *queue_; - + public: Dispatcher() {} ~Dispatcher() {} - + void Init(MPIGrainRunner *runner_in) { queue_ = NULL; runner_ = runner_in; } - + void set_queue(GrainQueue *queue_in) { queue_ = queue_in; } - + private: void MasterLoop_() { if (runner_->n_slaves_ > 0) { @@ -159,19 +159,19 @@ class MPIGrainRunner { } } }; - + private: class ConsumerTask : public Task { FORBID_COPY(ConsumerTask); - + private: struct MPIGrainRunner *runner_; - + public: ConsumerTask(MPIGrainRunner *runner_in) { runner_ = runner_in; } - + void Run() { int my_grains = 0; for (;;) { @@ -200,18 +200,18 @@ class MPIGrainRunner { int n_nodes_; int n_slaves_; Dispatcher *dispatcher_; - + WaitCondition need_work_cond_; volatile int need_work_; WaitCondition have_work_cond_; volatile int have_work_; Mutex mutex_; ArrayList slave_grains_; - + public: MPIGrainRunner() {} ~MPIGrainRunner() {} - + /** * Initialize this. * @@ -240,11 +240,11 @@ class MPIGrainRunner { slave_grains_.Init(); xrun_subparam_set(name, "n_nodes", "%d", n_nodes_); } - + Dispatcher *dispatcher() const { return dispatcher_; } - + Thread *SpawnThread() { ConsumerTask *task = new ConsumerTask(this); Thread *thread = new Thread(); @@ -252,7 +252,7 @@ class MPIGrainRunner { thread->Start(); return thread; } - + /** * Creates the specified number of threads, and uses those to execute * all grains of work. @@ -263,7 +263,7 @@ class MPIGrainRunner { DEBUG_MSG(1.0, "Rank %d is ready to roll.", my_rank_); ArrayList threads; int num_worker_threads = num_threads; - + if (num_worker_threads != 2) abort(); threads.Init(num_worker_threads); @@ -287,7 +287,7 @@ class MPIGrainRunner { } DEBUG_MSG(1.0, "Rank %d killed all threads.", my_rank_); } - + private: void SendInt_(int dest_rank, int num) { MPI_Send(&num, 1, @@ -295,7 +295,7 @@ class MPIGrainRunner { tag_ + 1, MPI_COMM_WORLD); } - + void RecvInt_(int *send_rank, int *num_ptr) { MPI_Status status; MPI_Recv(num_ptr, 1, @@ -306,10 +306,10 @@ class MPIGrainRunner { &status); *send_rank = status.MPI_SOURCE; } - + Grain *NextGrain_() { Grain *grain; - + if (dispatcher_) { DEBUG_ASSERT((my_rank_ == master_rank_)); mutex_.Lock(); @@ -323,7 +323,7 @@ class MPIGrainRunner { mutex_.Unlock(); need_work_cond_.Signal(); - + mutex_.Lock(); while (!have_work_) { have_work_cond_.Wait(&mutex_); @@ -337,16 +337,16 @@ class MPIGrainRunner { mutex_.Unlock(); DEBUG_MSG(2.0, "Slave gave me stuff!"); } - + return grain; } - + void SlaveLoop_() { bool done = false; - + DEBUG_MSG(1.0, "%d, WORKER: Announcing birth...", my_rank_); SendInt_(master_rank_, BIRTH); - + while (!done) { char buf[sizeof(Grain) + 1] = "q"; MPI_Status status; @@ -356,13 +356,13 @@ class MPIGrainRunner { MPI_CHAR, MPI_ANY_SOURCE, tag_, MPI_COMM_WORLD, &status); - + Grain *grain = NULL; - + if (buf[0] == 1) { DEBUG_MSG(1.0, "%d, WORKER: Got some work. There are %d waiting.", my_rank_, need_work_); - + grain = new Grain(); mem::CopyBytes(grain, buf+1, sizeof(Grain)); @@ -384,13 +384,13 @@ class MPIGrainRunner { done = true; } } - + DEBUG_MSG(1.0, "%d, WORKER: Duly dying !!!!!!!!!!!!!!!!!!!", my_rank_); - + mutex_.Lock(); have_work_ = -1; mutex_.Unlock(); - + have_work_cond_.Broadcast(); } }; diff --git a/fastlib/par/thread.h b/fastlib/par/thread.h index 3848d5bfcf..872f9653d9 100644 --- a/fastlib/par/thread.h +++ b/fastlib/par/thread.h @@ -25,7 +25,7 @@ class Thread { private: #ifdef DEBUG - enum {UNINIT, READY, ATTACHED, DETACHED, DONE} status; + enum {UNINIT, READY, ATTACHED, DETACHED, DONE} status_; #endif pthread_t thread_; Task *task_; @@ -44,30 +44,30 @@ class Thread { public: Thread() { - DEBUG_ONLY(status = UNINIT); + DEBUG_ONLY(status_ = UNINIT); } ~Thread() { - DEBUG_ASSERT(status == DETACHED || status == READY || status == DONE); - DEBUG_ONLY(status = UNINIT); + DEBUG_ASSERT(status_ == DETACHED || status_ == READY || status_ == DONE); + DEBUG_ONLY(status_ = UNINIT); } /** * Initializes, given a task to run. */ void Init(Task* task_in) { - DEBUG_ASSERT(status == UNINIT); + DEBUG_ASSERT(status_ == UNINIT); task_ = task_in; - DEBUG_ONLY(status = READY); + DEBUG_ONLY(status_ = READY); } /** * Starts the thread running. */ void Start() { - DEBUG_ASSERT(status == READY); + DEBUG_ASSERT(status_ == READY); pthread_create(&thread_, NULL, ThreadMain_, reinterpret_cast(this)); - DEBUG_ONLY(status = ATTACHED); + DEBUG_ONLY(status_ = ATTACHED); } /** @@ -75,9 +75,9 @@ class Thread { * completes. You may not call WaitStop on this thread afterwards. */ void Detach() { - DEBUG_ASSERT(status == ATTACHED); + DEBUG_ASSERT(status_ == ATTACHED); pthread_detach(thread_); - DEBUG_ONLY(status = DETACHED); + DEBUG_ONLY(status_ = DETACHED); } /** @@ -86,9 +86,9 @@ class Thread { * Failure to do this may cause your program to hang when it is done. */ void WaitStop() { - DEBUG_ASSERT(status == ATTACHED); + DEBUG_ASSERT(status_ == ATTACHED); pthread_join(thread_, NULL); - DEBUG_ONLY(status = DONE); + DEBUG_ONLY(status_ = DONE); } /**