From 6fd54b38dfa2db4e1ef8d5bdb4028158bd2ff11f Mon Sep 17 00:00:00 2001 From: vasiloglou Date: Sat, 26 May 2007 14:20:18 +0000 Subject: [PATCH] fixed mahor bugs that woudl stall the system, --- .../mmanager_with_tpie/memory_manager.h | 34 +++++++------------ .../mmanager_with_tpie/memory_manager_impl.h | 25 ++++++-------- .../u/nvasil/mmengines_comparisons/main.cc | 5 ++- fastlib/u/nvasil/tree/binary_tree.h | 1 + fastlib/u/nvasil/tree/binary_tree_impl.h | 17 ++++++++-- 5 files changed, 43 insertions(+), 39 deletions(-) diff --git a/fastlib/u/nvasil/mmanager_with_tpie/memory_manager.h b/fastlib/u/nvasil/mmanager_with_tpie/memory_manager.h index 26a92df561..51162c53ef 100644 --- a/fastlib/u/nvasil/mmanager_with_tpie/memory_manager.h +++ b/fastlib/u/nvasil/mmanager_with_tpie/memory_manager.h @@ -276,14 +276,23 @@ class MemoryManager { return total_num_of_page_faults_; } - index_t get_num_of_pages(); + index_t get_num_of_pages() { + return num_of_pages_; + } + index_t get_usage() { + return current_offset_; + } void set_cache_size(index_t cache_size) { cache_size_=cache_size; } void set_page_size(int32 page_size) { - page_size_= page_size; + if (page_size_ % system_page_size_ !=0) { + FATAL("Page size must be a multiple of the system page size %i\n", + system_page_size_); + } + page_size_= page_size; } void set_cache_file(string cache_file) { @@ -402,26 +411,9 @@ int FaultHandler(void *fault_address, int serious) { fault_address < allocator->cache_ + allocator->cache_size_){ index_t cache_page = (ptrdiff_t)((char*)fault_address - allocator->cache_) / allocator->page_size_; - // page has to be set as modified + + // page has to be set as modified allocator->SetPageModified(cache_page); - index_t system_page = (ptrdiff_t)fault_address / allocator->system_page_size_; - pair * p=allocator->PagesAffectedBySEGV(system_page); - // if all pages that are covered by the system page are modified - // then the whole page should be set uprotected; - for(index_t i=p->first; i<=p->second; i++) { - if (likely(!allocator->IsPageModified( - static_cast(cache_page + i)))) { - return 1 ; - } - } - char *addr = allocator->cache_ + allocator->system_page_size_ * - ((ptrdiff_t)((char*)fault_address - allocator->cache_) - / allocator->system_page_size_); - if (unlikely(mprotect(addr, - allocator->system_page_size_, PROT_READ | PROT_WRITE) !=0)) { - FATAL("Error %s while trying to change the protection\n", - strerror(errno)); - } return 1; } diff --git a/fastlib/u/nvasil/mmanager_with_tpie/memory_manager_impl.h b/fastlib/u/nvasil/mmanager_with_tpie/memory_manager_impl.h index 4ad8349afa..d83c84a0aa 100644 --- a/fastlib/u/nvasil/mmanager_with_tpie/memory_manager_impl.h +++ b/fastlib/u/nvasil/mmanager_with_tpie/memory_manager_impl.h @@ -348,19 +348,20 @@ inline void MEMORY_MANAGER__::MoveToDisk(index_t paddress){ } off_t disk_offset = paddress * page_size_/kTPIEPageSize; AMI_err ae; - if (unlikely((ae=disk_->seek(disk_offset))!=AMI_ERROR_NO_ERROR)) { + + if (unlikely((ae=disk_->seek(disk_offset))!=AMI_ERROR_NO_ERROR)) { cout << "AMI_ERROR " << ae << "\n"; if (paddress<0) { FATAL("Null pointer exception!\n"); } FATAL("Unable to seek to %llu \n", (unsigned long long)disk_offset); } - - if ((ae=disk_->write_array((Page *)page_address_[paddress], + if ((ae=disk_->write_array((Page *)(page_address_[paddress]), page_size_/kTPIEPageSize))!=AMI_ERROR_NO_ERROR) { cout << "AMI ERROR " << ae << " while transfering block to disk\n"; FATAL(" An error occured whule trying to write on disk\n"); } + } @@ -431,7 +432,7 @@ TEMPLATE__ inline void MEMORY_MANAGER__::SetPageModified(index_t cache_page) { page_modified_[cache_page] = true; // need to set the appropriate system page unprotected - ProtectSysPagesAffected(cache_page, PROT_WRITE); + ProtectSysPagesAffected(cache_page, PROT_WRITE); } // Unprotects all the system pages that include the requested cache_page @@ -482,8 +483,9 @@ inline index_t MEMORY_MANAGER__::LeastNeededPage() { least_recently_used_time = page_timestamp_[i]; } } - DEBUG_ASSERT_MSG(least_recently_used!=-1, - "All paged are locked and cache i stalled. Try to unlock"); + if (unlikely(least_recently_used==-1)) { + FATAL("All paged are locked and cache is stalled. Try to unlock"); + } return least_recently_used ; // return rand() % num_of_pages_; } @@ -525,10 +527,7 @@ inline bool MEMORY_MANAGER__::get_page_modified(index_t cache_page) { return page_modified_[cache_page]; } -TEMPLATE__ -inline index_t MEMORY_MANAGER__::get_num_of_pages() { - return num_of_pages_; -} + TEMPLATE__ inline index_t MEMORY_MANAGER__::GetLastObjectAddress(char *ptr) { @@ -600,12 +599,10 @@ void MEMORY_MANAGER__::CreateNewPageOnDisk() { cout << "AMI_ERROR " << ae << "\n"; FATAL("Unable to seek to %llu \n", (unsigned long long)disk_offset); } - for(index_t i=0; i< num_of_pages_; i++) { - if ((ae=disk_->write_array((Page *)(ptr), - page_size_/kTPIEPageSize))!=AMI_ERROR_NO_ERROR) { + if ((ae=disk_->write_array((Page *)(ptr), + page_size_/kTPIEPageSize))!=AMI_ERROR_NO_ERROR) { cout << "AMI_ERROR " << ae << " during disk_.write_item()\n"; exit(1); - } } delete []ptr; } diff --git a/fastlib/u/nvasil/mmengines_comparisons/main.cc b/fastlib/u/nvasil/mmengines_comparisons/main.cc index 63b47904ae..0b36f88b6d 100644 --- a/fastlib/u/nvasil/mmengines_comparisons/main.cc +++ b/fastlib/u/nvasil/mmengines_comparisons/main.cc @@ -32,7 +32,6 @@ struct Parameters { std::string temp_dir_; std::string memory_engine_; int32 page_size_; - int64 cache_size_; int32 knns_; std::string memory_file_; BinaryDataset data_; @@ -127,11 +126,15 @@ void DuallTreeAllNearestNeighbors(Parameters &args) { fx_timer_start(NULL, "build"); tree.BuildDepthFirst(); fx_timer_stop(NULL, "build"); + printf("Memory usage: %llu\n", + (unsigned long long)TREE::Allocator_t::allocator_->get_usage()); + printf("%s\n", tree.Statistics().c_str()); args.data_.Destruct(); printf("Initializing all nearest neighbor output...\n"); tree.InitAllKNearestNeighborOutput(args.out_file_, args.knns_); printf("Computing all nearest neighbors...\n"); + fflush(stdout); fx_timer_start(NULL, "dualltree"); tree.AllNearestNeighbors(tree.get_parent(), args.knns_); fx_timer_stop(NULL, "dualltree"); diff --git a/fastlib/u/nvasil/tree/binary_tree.h b/fastlib/u/nvasil/tree/binary_tree.h index 3ae147106d..e85d9848fa 100644 --- a/fastlib/u/nvasil/tree/binary_tree.h +++ b/fastlib/u/nvasil/tree/binary_tree.h @@ -68,6 +68,7 @@ class BinaryTree { Result_t *Allocate(int32 num_of_points, int32 knns) { Result_t *result=ptr_+num_; num_+=knns*num_of_points; + printf("%i\n", num_); return result; } private: diff --git a/fastlib/u/nvasil/tree/binary_tree_impl.h b/fastlib/u/nvasil/tree/binary_tree_impl.h index 7243cc9ae1..335c7238d7 100644 --- a/fastlib/u/nvasil/tree/binary_tree_impl.h +++ b/fastlib/u/nvasil/tree/binary_tree_impl.h @@ -431,6 +431,7 @@ void TREE__::InitAllKNearestNeighborOutput(string file, const int32 kChunk=8192; typename Node_t::NNResult *buffer; buffer=new typename Node_t::NNResult[kChunk*knns]; + printf("Generating output file...\n"); for(index_t i=0; iIsLeaf()) { - ptr->set_kneighbors(all_nn_out_.Allocate(ptr->get_num_of_points(), knns), + ptr->set_kneighbors(all_nn_out_.Allocate(ptr->get_num_of_points(), knns), knns); ptr->InitKNeighbors(knns); - ptr->set_min_dist_so_far(numeric_limits::max()); + ptr->set_min_dist_so_far(numeric_limits::max()); + // printf("leaf_id: %i\n", ptr->get_node_id()); ptr.Unlock(); } else { NodePtr_t left = ptr->get_left(); @@ -470,7 +482,6 @@ void TREE__::InitAllKNearestNeighborOutput(typename TREE__::NodePtr_t ptr, NodePtr_t right=ptr->get_right(); ptr.Unlock(); InitAllKNearestNeighborOutput(right, knns); - } }