hi

fastlib/math/math.h

@@ -45,8 +45,8 @@ namespace math {
   /**
    * Rounds a double-precision to an integer, casting it too.
    */
-  inline int RoundInt(double d) {
-    return int(nearbyint(d));
+  inline int64 RoundInt(double d) {
+    return int64(nearbyint(d));
   }
 
   /**

fastlib/thor/distribcache.cc

@@ -160,8 +160,8 @@ void DistributedCache::HandleStatusInformation_(
       DEBUG_ASSERT_MSG(status->owner != my_rank_,
           "Received ownership unexpectedly");
       DEBUG_ASSERT_MSG(status->owner >= 0,
-          "It looks like block %"LI"d is owned by %d of %"LI"d (i'm %d)\n",
-          i, status->owner, blocks_.size(), rpc::rank());
+          "%d It looks like block %"LI"d is owned by %d of %"LI"d (i'm %d)\n",
+          int(n_block_bytes_), i, status->owner, blocks_.size(), rpc::rank());
       DEBUG_ASSERT_MSG(!block->is_dirty(),
           "Remote blocks shouldn't be dirty during a sync.");
       block->value = ~status->owner;
@@ -376,6 +376,7 @@ void DistributedCache::RemoteWrite(blockid_t blockid,
         "Please sync between ownership changes and further reads!");
     block->value = SELF_OWNER_UNALLOCATED; // mark as owner
     block->status = NOT_DIRTY_NEW;
+    DEBUG_ASSERT(block->is_owner());
   }
   mutex_.Unlock();
   Write(blockid, begin, end, buf);
@@ -386,24 +387,11 @@ BlockDevice::blockid_t DistributedCache::AllocBlocks(
   index_t blockid = RemoteAllocBlocks(n_blocks_to_alloc, owner, my_rank_);
   if (owner != my_rank_) {
     // Tell the owner that I've allocated a block in their name.
+    DoOwnerRequest_(owner, owner, blockid, blockid + n_blocks_to_alloc);
   }
   return blockid;
 }
 
-void DistributedCache::DoOwnerRequest_(int dest, int new_owner,
-    blockid_t blockid, blockid_t end_block) {
-  BasicTransaction transaction;
-  transaction.Init(channel_num_);
-  Message *message = transaction.CreateMessage(dest, sizeof(Request));
-  Request *request = message->data_as<Request>();
-  request->type = Request::OWNER;
-  request->field1 = blockid;
-  request->field2 = end_block;
-  request->field3 = new_owner;
-  transaction.Send(message);
-  transaction.Done();
-}
-
 BlockDevice::blockid_t DistributedCache::RemoteAllocBlocks(
     blockid_t n_blocks_to_alloc, int owner, int sender) {
   blockid_t blockid;
@@ -432,8 +420,8 @@ void DistributedCache::MarkOwner_(int owner,
   int32 value = (owner == my_rank_) ? SELF_OWNER_UNALLOCATED : (~owner);
 
   for (blockid_t i = begin; i < end; i++) {
-    if (blocks_[i].is_owner()) {
-      DEBUG_ASSERT(!blocks_[i].is_dirty());
+    if (blocks_[i].is_owner() && value < 0) {
+      blocks_[i].status = NOT_DIRTY_OLD;
       RecycleLocalBlock_(blocks_[i].local_blockid());
     }
     blocks_[i].value = value;
@@ -456,12 +444,26 @@ BlockDevice::blockid_t DistributedCache::DoAllocRequest_(
   return retval;
 }
 
+void DistributedCache::DoOwnerRequest_(int dest, int new_owner,
+    blockid_t blockid, blockid_t end_block) {
+  BasicTransaction transaction;
+  transaction.Init(channel_num_);
+  Message *message = transaction.CreateMessage(dest, sizeof(Request));
+  Request *request = message->data_as<Request>();
+  request->type = Request::OWNER;
+  request->field1 = blockid;
+  request->field2 = end_block;
+  request->field3 = new_owner;
+  transaction.Send(message);
+  transaction.Done();
+}
+
 void DistributedCache::HandleRemoteOwner_(blockid_t block, blockid_t end,
     int new_owner) {
   mutex_.Lock();
   n_blocks_ = max(n_blocks_, end);
   blocks_.Resize(n_blocks_);
-  MarkOwner_(my_rank_, block, end);
+  MarkOwner_(new_owner, block, end);
   mutex_.Unlock();
 }
 
@@ -469,23 +471,21 @@ void DistributedCache::GiveOwnership(blockid_t blockid, int new_owner) {
   // mark whole block as dirty and change its owner.
   mutex_.Lock();
   BlockMetadata *block = &blocks_[blockid];
-  if (unlikely(block->locks == 0)) {
-    DecacheBlock_(blockid);
-    block->locks = 0;
-  }
-  block->status = FULLY_DIRTY;
-  if (!block->is_owner()) {
-    DoOwnerRequest_(block->owner(), new_owner, blockid, blockid + 1);
-  } else {
-    RecycleLocalBlock_(block->local_blockid());
-  }
-  if (new_owner == my_rank_) {
-    block->value = SELF_OWNER_UNALLOCATED;
-  } else {
-    block->value = ~new_owner;
-  }
-  if (unlikely(block->locks == 0)) {
-    EncacheBlock_(blockid);
+  if (block->owner(this) != new_owner) {
+    if (unlikely(block->locks == 0)) {
+      DecacheBlock_(blockid);
+      block->locks = 0;
+    }
+    if (!block->is_owner()) {
+      DoOwnerRequest_(block->owner(), new_owner, blockid, blockid + 1);
+    } else {
+      RecycleLocalBlock_(block->local_blockid());
+    }
+    block->value = (new_owner == my_rank_) ? SELF_OWNER_UNALLOCATED : (~new_owner);
+    block->status = FULLY_DIRTY;
+    if (unlikely(block->locks == 0)) {
+      EncacheBlock_(blockid);
+    }
   }
   mutex_.Unlock();
 }
@@ -680,13 +680,12 @@ void DistributedCache::EncacheBlock_(blockid_t blockid) {
         "Block re-cached: block %d, cache block size %d",
         blockid, n_block_bytes_);
     if (unlikely(i == ASSOC-1)) {
-      const int remote_allowance = ASSOC/2;
       for (;;) {
         if (!blocks_[base_slot[i].blockid].is_owner()) {
           break;
         }
         i--;
-        if (i == remote_allowance) {
+        if (i < REMOTE_ALLOWANCE) {
           // no renote block to purge, take the worst local one
           i = ASSOC-1;
           break;
@@ -704,7 +703,7 @@ void DistributedCache::EncacheBlock_(blockid_t blockid) {
     base_slot[i] = base_slot[i-1];
   }
 
-  base_slot->blockid = blockid;
+  base_slot[0].blockid = blockid;
 }
 
 void DistributedCache::Purge_(blockid_t blockid) {
@@ -713,8 +712,8 @@ void DistributedCache::Purge_(blockid_t blockid) {
 
   DEBUG_ASSERT(block->is_in_core());
   DEBUG_ASSERT_MSG(!block->is_busy(),
-      "Trying to evict a busy block (non-zero lock count %d)",
-      int(block->locks));
+      "Trying to evict a busy block (non-zero lock count %d, block %d, value %d)",
+      int(block->locks), int(blockid), int(block->value));
 
   block->data = NULL;
 

fastlib/thor/distribcache.h

@@ -320,6 +320,8 @@ class DistributedCache : public BlockDevice {
   static const int LOG_ASSOC = 4;
   /** The set associativity of the cache. */
   static const int ASSOC = (1 << LOG_ASSOC);
+  /** Number of remote blocks allowed before we start paging local blocks. */
+  static const int REMOTE_ALLOWANCE = ASSOC/4;
 
   /**
    * Information we keep about the block.