/* Template implementations for cachearray.h. */ template void CacheArrayBlockHandler::Init(const T& default_obj) { default_elem_.Init(ot::FrozenSize(default_obj)); ot::Freeze(default_elem_.begin(), default_obj); } template void CacheArrayBlockHandler::Serialize(ArrayList* data) const { data->InitCopy(default_elem_); } template void CacheArrayBlockHandler::Deserialize(const ArrayList& data) { default_elem_.InitCopy(data); } template void CacheArrayBlockHandler::BlockInitFrozen(BlockDevice::blockid_t blockid, BlockDevice::offset_t begin, BlockDevice::offset_t bytes, char *block) { DEBUG_ASSERT((begin % default_elem_.size()) == 0); index_t elems = bytes / default_elem_.size(); for (index_t i = 0; i < elems; i++) { mem::CopyBytes(block, default_elem_.begin(), default_elem_.size()); block += default_elem_.size(); } } template void CacheArrayBlockHandler::BlockFreeze(BlockDevice::blockid_t blockid, BlockDevice::offset_t begin, BlockDevice::offset_t bytes, const char *old_location, char *block) { DEBUG_ASSERT(begin % default_elem_.size() == 0); index_t elems = bytes / default_elem_.size(); for (index_t i = 0; i < elems; i++) { ot::SemiFreeze(block, reinterpret_cast(old_location)); block += default_elem_.size(); old_location += default_elem_.size(); } } template void CacheArrayBlockHandler::BlockThaw(BlockDevice::blockid_t blockid, BlockDevice::offset_t begin, BlockDevice::offset_t bytes, char *block) { DEBUG_ASSERT(begin % default_elem_.size() == 0); index_t elems = bytes / default_elem_.size(); for (index_t i = 0; i < elems; i++) { ot::SemiThaw(block); block += default_elem_.size(); } } template void CacheArrayBlockHandler::GetDefaultElement(T *default_element_out) { ArrayList tmp; tmp.InitCopy(default_elem_); const T* source = ot::SemiThaw(tmp.begin()); ot::InitCopy(default_element_out, *source); } //-------------------------------------------------------------------------- template index_t CacheArray::ConvertBlockSize( const Element& element, int kilobytes) { size_t elem_size = ot::FrozenSize(element); size_t bytes = size_t(kilobytes) << 10; int i; for (i = 0; (size_t(1) << i) * elem_size <= bytes; i++) {} //fprintf(stderr, "%d %d %d %d\n", int(bytes), int(elem_size), int(1 << i), int(elem_size)); return index_t(1) << (i - 1); } template void CacheArray::CreateCacheMaster(int channel, index_t n_block_elems, const Element& default_elem, double megs, DistributedCache *cache) { CacheArrayBlockHandler *handler = new CacheArrayBlockHandler(); handler->Init(default_elem); cache->InitMaster(channel, n_block_elems * handler->n_elem_bytes(), math::RoundInt(megs * MEGABYTE), handler); } template void CacheArray::CreateCacheWorker(int channel, double megs, DistributedCache *cache) { CacheArrayBlockHandler *handler = new CacheArrayBlockHandler(); cache->InitWorker(channel, math::RoundInt(megs * MEGABYTE), handler); } template void CacheArray::Grow(index_t end_element) { DEBUG_ASSERT_MSG(end_element >= end_, "end_element [%"LI"d] >= end_ [%"LI"d]", end_element, end_); end_ = end_element; next_alloc_ = end_element; metadatas_.Resize(((end_ + n_block_elems_mask()) >> n_block_elems_log()) - skip_blocks_); adjusted_metadatas_ = metadatas_.begin() - skip_blocks_; MarkRanges_(); } template void CacheArray::Grow() { Grow(cache_->n_blocks() << n_block_elems_log()); } template void CacheArray::Swap(index_t index_a, index_t index_b) { DEBUG_ONLY(BoundsCheck_(index_a)); DEBUG_ONLY(BoundsCheck_(index_b)); DEBUG_ASSERT(BlockDevice::can_write(mode_)); char *a = reinterpret_cast(StartWrite(index_a)); char *b = reinterpret_cast(StartWrite(index_b)); mem::BitSwap(a, b, n_elem_bytes_); /* TODO: Fix potential memory leak here */ ot::SemiCopy(b, a); ot::SemiCopy(a, b); ReleaseElement(index_a); ReleaseElement(index_b); } template void CacheArray::Copy(index_t index_src, index_t index_dest) { DEBUG_ONLY(BoundsCheck_(index_src)); DEBUG_ONLY(BoundsCheck_(index_dest)); DEBUG_ASSERT(BlockDevice::can_write(mode_)); const char *src = reinterpret_cast(StartWrite(index_src)); char *dest = reinterpret_cast(StartWrite(index_dest)); mem::Copy(dest, src, n_elem_bytes_); /* TODO: Fix potential memory leak here */ ot::SemiCopy(dest, src); ReleaseElement(index_src); ReleaseElement(index_dest); } template index_t CacheArray::AllocD(int owner, index_t count) { DEBUG_ASSERT(BlockDevice::is_dynamic(mode_)); if (unlikely(next_alloc_ + count > end_)) { BlockDevice::blockid_t blocks_to_alloc = (count + n_block_elems_mask()) >> n_block_elems_log(); BlockDevice::blockid_t blockid = cache_->AllocBlocks( blocks_to_alloc, owner); metadatas_.Resize(blockid + blocks_to_alloc - skip_blocks_); adjusted_metadatas_ = metadatas_.begin() - skip_blocks_; next_alloc_ = blockid << n_block_elems_log(); end_ = next_alloc_ + (blocks_to_alloc << n_block_elems_log()); MarkRanges_(); } index_t ret_pos = next_alloc_; next_alloc_ += count; return ret_pos; } template index_t CacheArray::AllocD(int owner) { DEBUG_ASSERT(BlockDevice::is_dynamic(mode_)); if (unlikely(next_alloc_ >= end_)) { BlockDevice::blockid_t blockid = cache_->AllocBlocks(1, owner); metadatas_.Resize(blockid - skip_blocks_ + 1); adjusted_metadatas_ = metadatas_.begin() - skip_blocks_; next_alloc_ = blockid << n_block_elems_log(); end_ = next_alloc_ + n_block_elems(); MarkRanges_(); } index_t ret_pos = next_alloc_; next_alloc_++; return ret_pos; } template void CacheArray::MarkRanges_() { if (!BlockDevice::is_dynamic(mode_)) { cache_->AddPartialDirtyRange( Blockid(begin_), Offset(begin_), Blockid(end_), Offset(end_)); } } template void CacheArray::Init( DistributedCache *cache_in, BlockDevice::mode_t mode_in, index_t begin_index_in, index_t end_index_in) { CacheArrayBlockHandler* handler = static_cast*>( cache_in->block_handler()); DEBUG_ONLY(n_fifo_locks_ = 0); cache_ = cache_in; begin_ = begin_index_in; end_ = end_index_in; DEBUG_ASSERT(end_ >= begin_); DEBUG_ASSERT(begin_ >= 0); next_alloc_ = end_; mode_ = mode_in; n_elem_bytes_ = handler->n_elem_bytes(); fifo_size_ = 64; fifo_ = mem::Alloc(fifo_size_); mem::RepeatConstruct(fifo_, -1, fifo_size_); fifo_index_ = 0; unsigned n_block_elems_calc = cache_->n_block_bytes() / n_elem_bytes_; // Cache size must be a power of 2. n_block_elems_log_ = math::IntLog2(n_block_elems_calc); n_block_elems_mask_ = n_block_elems_calc - 1; skip_blocks_ = begin_ / n_block_elems_calc; DEBUG_ASSERT_MSG(cache_->n_block_bytes() % n_elem_bytes_ == 0, "Block size must be a multiple of element size."); MarkRanges_(); metadatas_.Init(((end_ + n_block_elems_mask()) >> n_block_elems_log()) - skip_blocks_); adjusted_metadatas_ = metadatas_.begin() - skip_blocks_; } template void CacheArray::Flush() { for (int i = 0; i < fifo_size_; i++) { BlockDevice::blockid_t blockid = fifo_[i]; if (blockid >= 0) { Metadata *metadata = adjusted_metadatas_ + blockid; if (BlockDevice::can_write(mode_)) { cache_->StopWrite(blockid); } else { cache_->StopRead(blockid); } DEBUG_SAME_SIZE(metadata->lock_count, 0); metadata->data = NULL; fifo_[i] = -1; } } DEBUG_ONLY(cache_->AddFifoLocks(n_fifo_locks_)); DEBUG_ONLY(n_fifo_locks_ = 0); } template typename CacheArray::Element* CacheArray::HandleCacheMiss_( index_t element_id) { BlockDevice::blockid_t victim; Metadata *victim_metadata; bool looped = false; // warning, this isn't very readable... basically, look for the first // unlocked item -- the most likely case is that the first item in the // fifo is non-negative (i.e. it exists) and it's most likely not locked for (;;) { if (unlikely(fifo_index_ == 0)) { if (unlikely(looped)) { int old_size = fifo_size_; fifo_size_ *= 2; fifo_ = mem::Realloc(fifo_, fifo_size_); for (int i = old_size; i < fifo_size_; i++) { fifo_[i] = -1; } } looped = true; fifo_index_ = fifo_size_; } fifo_index_--; victim = fifo_[fifo_index_]; if (unlikely(victim < 0)) { break; } victim_metadata = adjusted_metadatas_ + victim; if (unlikely(victim_metadata->lock_count != 0)) { // the block was locked continue; } DEBUG_ASSERT(victim_metadata->data != NULL); if (BlockDevice::can_write(mode_)) { cache_->StopWrite(victim); } else { cache_->StopRead(victim); } victim_metadata->data = NULL; break; } BlockDevice::blockid_t blockid = Blockid(element_id); Metadata *metadata = adjusted_metadatas_ + blockid; fifo_[fifo_index_] = blockid; if (BlockDevice::can_write(mode_)) { metadata->data = cache_->StartWrite(blockid, !BlockDevice::is_dynamic(mode_)); } else { metadata->data = cache_->StartRead(blockid); } BlockDevice::offset_t offset = uint(element_id & (n_block_elems_mask())) * n_elem_bytes_; return reinterpret_cast(metadata->data + offset); } //-------------------------------------------------------------------------- template class ZCacheIterImpl_ { FORBID_ACCIDENTAL_COPIES(ZCacheIterImpl_); private: Element *element_; uint stride_; uint left_; CacheArray *cache_; BlockDevice::blockid_t blockid_; public: ZCacheIterImpl_(CacheArray* cache_in, index_t begin_index) { cache_ = cache_in; blockid_ = cache_->Blockid(begin_index); element_ = Helperclass::MyStartAccess_(cache_, begin_index); stride_ = cache_->n_elem_bytes(); unsigned int mask = cache_->n_block_elems_mask(); // equivalent to: block_size - (begin_index % block_size) - 1 left_ = (begin_index ^ mask) & mask; } ~ZCacheIterImpl_() { if (likely(element_ != NULL)) { cache_->ReleaseBlock(blockid_); } } operator Element * () const { return element_; } Element * operator -> () const { return element_; } Element & operator * () const { return *element_; } void SetIndex(index_t begin_index) { cache_->ReleaseBlock(blockid_); blockid_ = cache_->Blockid(begin_index); element_ = Helperclass::MyStartAccess_(cache_, begin_index); unsigned int mask = cache_->n_block_elems_mask(); left_ = (begin_index ^ mask) & mask; } void Next() { DEBUG_BOUNDS(left_, cache_->n_block_elems() + 1); element_ = mem::PtrAddBytes(element_, stride_); if (unlikely(left_ == 0)) { NextBlock_(); return; } --left_; } private: COMPILER_NO_INLINE void NextBlock_(); }; template class ZCacheReadIterHelperclass_ { public: static const Element *MyStartAccess_(CacheArray* a, index_t i) { return a->StartRead(i); } }; template class ZCacheWriteIterHelperclass_ { public: static Element *MyStartAccess_(CacheArray* a, index_t i) { return a->StartWrite(i); } }; template void ZCacheIterImpl_::NextBlock_() { left_ = cache_->n_block_elems_mask(); cache_->ReleaseBlock(blockid_); ++blockid_; index_t elem_id = cache_->BlockElement(blockid_); if (likely(elem_id < cache_->end_index())) { element_ = Helperclass::MyStartAccess_(cache_, elem_id); } else { element_ = NULL; } } //-------------------------------------------------------------------------- template void SubsetArray::Init(const Element& default_elem, index_t begin, index_t end) { n_elem_bytes_ = ot::FrozenSize(default_elem); begin_ = begin; end_ = end; adjusted_ = NULL; if (begin_ < end_) { char *base = mem::Alloc(n_elem_bytes_ * (end - begin)); char *adjusted = base - (begin * n_elem_bytes_); ot::Freeze(base, default_elem); for (index_t i = begin + 1; i < end; i++) { char *ptr = adjusted + i * n_elem_bytes_; mem::CopyBytes(ptr, base, n_elem_bytes_); ot::SemiThaw(ptr); } ot::SemiThaw(base); adjusted_ = adjusted; } }