apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed mahor bugs that woudl stall the system,

Browse Source
This commit is contained in:
vasiloglou
2007-05-26 14:20:18 +00:00
parent b82b7a22c5
commit 6fd54b38df
5 changed files with 43 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastlib/u/nvasil/mmanager_with_tpie/memory_manager.h
+13 -21
View File
@@ -276,14 +276,23 @@ class MemoryManager {
return total_num_of_page_faults_;
}
index_t get_num_of_pages();
index_t get_num_of_pages() {
return num_of_pages_;
}
index_t get_usage() {
return current_offset_;
}
void set_cache_size(index_t cache_size) {
cache_size_=cache_size;
}
void set_page_size(int32 page_size) {
page_size_= page_size;
if (page_size_ % system_page_size_ !=0) {
FATAL("Page size must be a multiple of the system page size %i\n",
system_page_size_);
}
page_size_= page_size;
}
void set_cache_file(string cache_file) {
@@ -402,26 +411,9 @@ int FaultHandler(void *fault_address, int serious) {
fault_address < allocator->cache_ + allocator->cache_size_){
index_t cache_page = (ptrdiff_t)((char*)fault_address - allocator->cache_)
/ allocator->page_size_;
// page has to be set as modified
// page has to be set as modified
allocator->SetPageModified(cache_page);
index_t system_page = (ptrdiff_t)fault_address / allocator->system_page_size_;
pair<index_t, index_t> * p=allocator->PagesAffectedBySEGV(system_page);
// if all pages that are covered by the system page are modified
// then the whole page should be set uprotected;
for(index_t i=p->first; i<=p->second; i++) {
if (likely(!allocator->IsPageModified(
static_cast<index_t>(cache_page + i)))) {
return 1 ;
}
}
char *addr = allocator->cache_ + allocator->system_page_size_ *
((ptrdiff_t)((char*)fault_address - allocator->cache_)
/ allocator->system_page_size_);
if (unlikely(mprotect(addr,
allocator->system_page_size_, PROT_READ | PROT_WRITE) !=0)) {
FATAL("Error %s while trying to change the protection\n",
strerror(errno));
}
return 1;
}
fastlib/u/nvasil/mmanager_with_tpie/memory_manager_impl.h
+11 -14
View File
@@ -348,19 +348,20 @@ inline void MEMORY_MANAGER__::MoveToDisk(index_t paddress){
}
off_t disk_offset = paddress * page_size_/kTPIEPageSize;
AMI_err ae;
if (unlikely((ae=disk_->seek(disk_offset))!=AMI_ERROR_NO_ERROR)) {
if (unlikely((ae=disk_->seek(disk_offset))!=AMI_ERROR_NO_ERROR)) {
cout << "AMI_ERROR " << ae << "\n";
if (paddress<0) {
FATAL("Null pointer exception!\n");
}
FATAL("Unable to seek to %llu \n", (unsigned long long)disk_offset);
}
if ((ae=disk_->write_array((Page *)page_address_[paddress],
if ((ae=disk_->write_array((Page *)(page_address_[paddress]),
page_size_/kTPIEPageSize))!=AMI_ERROR_NO_ERROR) {
cout << "AMI ERROR " << ae << " while transfering block to disk\n";
FATAL(" An error occured whule trying to write on disk\n");
}
}
@@ -431,7 +432,7 @@ TEMPLATE__
inline void MEMORY_MANAGER__::SetPageModified(index_t cache_page) {
page_modified_[cache_page] = true;
// need to set the appropriate system page unprotected
ProtectSysPagesAffected(cache_page, PROT_WRITE);
ProtectSysPagesAffected(cache_page, PROT_WRITE);
}
// Unprotects all the system pages that include the requested cache_page
@@ -482,8 +483,9 @@ inline index_t MEMORY_MANAGER__::LeastNeededPage() {
least_recently_used_time = page_timestamp_[i];
}
}
DEBUG_ASSERT_MSG(least_recently_used!=-1,
"All paged are locked and cache i stalled. Try to unlock");
if (unlikely(least_recently_used==-1)) {
FATAL("All paged are locked and cache is stalled. Try to unlock");
}
return least_recently_used ;
// return rand() % num_of_pages_;
}
@@ -525,10 +527,7 @@ inline bool MEMORY_MANAGER__::get_page_modified(index_t cache_page) {
return page_modified_[cache_page];
}
TEMPLATE__
inline index_t MEMORY_MANAGER__::get_num_of_pages() {
return num_of_pages_;
}
TEMPLATE__
inline index_t MEMORY_MANAGER__::GetLastObjectAddress(char *ptr) {
@@ -600,12 +599,10 @@ void MEMORY_MANAGER__::CreateNewPageOnDisk() {
cout << "AMI_ERROR " << ae << "\n";
FATAL("Unable to seek to %llu \n", (unsigned long long)disk_offset);
}
for(index_t i=0; i< num_of_pages_; i++) {
if ((ae=disk_->write_array((Page *)(ptr),
page_size_/kTPIEPageSize))!=AMI_ERROR_NO_ERROR) {
if ((ae=disk_->write_array((Page *)(ptr),
page_size_/kTPIEPageSize))!=AMI_ERROR_NO_ERROR) {
cout << "AMI_ERROR " << ae << " during disk_.write_item()\n";
exit(1);
}
}
delete []ptr;
}
fastlib/u/nvasil/mmengines_comparisons/main.cc
+4 -1
View File
@@ -32,7 +32,6 @@ struct Parameters {
std::string temp_dir_;
std::string memory_engine_;
int32 page_size_;
int64 cache_size_;
int32 knns_;
std::string memory_file_;
BinaryDataset<float32> data_;
@@ -127,11 +126,15 @@ void DuallTreeAllNearestNeighbors(Parameters &args) {
fx_timer_start(NULL, "build");
tree.BuildDepthFirst();
fx_timer_stop(NULL, "build");
printf("Memory usage: %llu\n",
(unsigned long long)TREE::Allocator_t::allocator_->get_usage());
printf("%s\n", tree.Statistics().c_str());
args.data_.Destruct();
printf("Initializing all nearest neighbor output...\n");
tree.InitAllKNearestNeighborOutput(args.out_file_,
args.knns_);
printf("Computing all nearest neighbors...\n");
fflush(stdout);
fx_timer_start(NULL, "dualltree");
tree.AllNearestNeighbors(tree.get_parent(), args.knns_);
fx_timer_stop(NULL, "dualltree");
fastlib/u/nvasil/tree/binary_tree.h
+1
View File
@@ -68,6 +68,7 @@ class BinaryTree {
Result_t *Allocate(int32 num_of_points, int32 knns) {
Result_t *result=ptr_+num_;
num_+=knns*num_of_points;
printf("%i\n", num_);
return result;
}
private:
fastlib/u/nvasil/tree/binary_tree_impl.h
+14 -3
View File
@@ -431,6 +431,7 @@ void TREE__::InitAllKNearestNeighborOutput(string file,
const int32 kChunk=8192;
typename Node_t::NNResult *buffer;
buffer=new typename Node_t::NNResult[kChunk*knns];
printf("Generating output file...\n");
for(index_t i=0; i<num_of_points_/kChunk; i++) {
fwrite(buffer, sizeof(typename Node_t::NNResult),kChunk*knns, fp );
}
@@ -446,9 +447,19 @@ void TREE__::InitAllKNearestNeighborOutput(string file,
fprintf(stderr, "Unable to map file: %s", strerror(errno));
assert(false);
}
if (madvise(ptr, sizeof(typename Node_t::NNResult)*knns*num_of_points_,
MADV_SEQUENTIAL)!=0) {
NONFATAL("It wasn't possible to advise output, error: %s",
strerror(errno));
}
close(fd);
all_nn_out_.set_ptr(ptr);
printf("Now visiting nodes to initialize output...\n");
fx_timer_start(NULL, "init_knn");
InitAllKNearestNeighborOutput(parent_, knns);
fx_timer_stop(NULL, "init_knn");
}
@@ -457,10 +468,11 @@ void TREE__::InitAllKNearestNeighborOutput(typename TREE__::NodePtr_t ptr,
int32 knns) {
ptr.Lock();
if (ptr->IsLeaf()) {
ptr->set_kneighbors(all_nn_out_.Allocate(ptr->get_num_of_points(), knns),
ptr->set_kneighbors(all_nn_out_.Allocate(ptr->get_num_of_points(), knns),
knns);
ptr->InitKNeighbors(knns);
ptr->set_min_dist_so_far(numeric_limits<Precision_t>::max());
ptr->set_min_dist_so_far(numeric_limits<Precision_t>::max());
// printf("leaf_id: %i\n", ptr->get_node_id());
ptr.Unlock();
} else {
NodePtr_t left = ptr->get_left();
@@ -470,7 +482,6 @@ void TREE__::InitAllKNearestNeighborOutput(typename TREE__::NodePtr_t ptr,
NodePtr_t right=ptr->get_right();
ptr.Unlock();
InitAllKNearestNeighborOutput(right, knns);
}
}