apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
de3c55dfe9fda5bbfc9fbfbc3cbb2eb043bc75ec
mlpack/fastlib/u/nvasil/dataset/binary_dataset.h
T

486 lines
15 KiB
C++
Raw Blame History

/*
* =====================================================================================
*
* Filename: dataset.h
*
* Description:
*
* Version: 1.0
* Created: 04/24/2007 10:25:35 AM EDT
* Revision: none
* Compiler: gcc
*
* Author: Nikolaos Vasiloglou (NV), nvasil@ieee.org
* Company: Georgia Tech Fastlab-ESP Lab
*
* =====================================================================================
*/
#ifndef BINARY_DATASET_
#define BINARY_DATASET_
#include <sys/unistd.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <string>
#include "fastlib/fastlib.h"
#include "u/nvasil/loki/NullType.h"
#include "u/nvasil/tree/point.h"
// BinaryDataset
// Use this class to read a binary data file that is also
// accompanied by a file with the index values (uint64)
// It is templetized so that you can use it with different precisions
// for data
using namespace std;
template <typename PRECISION>
class BinaryDataset {
FORBID_COPY(BinaryDataset<PRECISION>);
template<typename > friend class BinaryDatasetTest;
public:
typedef PRECISION Precision_t;
static const int32 kMinimumPagesToAdvise=10;
friend class Iterator;
class Iterator {
public:
typedef random_access_iterator_tag iterator_category;
typedef CompletePoint<Precision_t> value_type;
typedef index_t difference_type;
typedef CompletePoint<Precision_t>* pointer;
typedef CompletePoint<Precision_t>& reference;
Iterator(){
}
Iterator(BinaryDataset<Precision_t> *dataset) {
set_=dataset;
current_pos_=0;
}
Iterator &operator=(const Iterator &other) {
this->set_=other.set_;
this->current_pos_=other.current_pos_;
return *this;
}
Iterator operator++() {
current_pos_++;
DEBUG_ASSERT_MSG(current_pos_>(signed long long)set_->get_num_of_points(),
"Iterator out of bounds %lli>%lli",
(signed long long)current_pos_,
(signed long long)set_->get_num_of_points());
}
Iterator operator+(index_t i) {
Iterator it(*this);
it.current_pos_+=i;
DEBUG_ASSERT_MSG(it.current_pos_<(signed long long)set_->get_num_of_points()
&& it.current_pos_>=0,
"iterator out of bounds %lli>=%lli\n",
(signed long long)it.current_pos_,
(signed long long)set_->get_num_of_points());
return it;
}
Iterator operator+(const Iterator &other) {
Iterator it(*this);
it.current_pos_+=other.current_pos_;
DEBUG_ASSERT_MSG(it.current_pos_<(signed long long)set_->get_num_of_points()
&& it.current_pos_>=0,
"iterator out of bounds %lli>=%lli\n",
(signed long long)it.current_pos_,
(signed long long)set_->get_num_of_points());
return it;
}
Iterator operator--() {
current_pos_--;
DEBUG_ASSERT_MSG(current_pos_<0,
"Iterator out of bounds %lli <0",
(signed long long)current_pos_ );
}
Iterator operator-(index_t i) {
Iterator it(*this);
it.current_pos_-=i;
DEBUG_ASSERT_MSG(it.current_pos_<(signed long long)set_->get_num_of_points()
&& it.current_pos_>=0,
"iterator out of bounds %lli>=%lli\n",
(unsigned long long)it.current_pos_,
(unsigned long long)set_->get_num_of_points());
return it;
}
index_t operator-(const Iterator &other) {
index_t diff = current_pos_-other.current_pos_;
DEBUG_ASSERT_MSG(current_pos_<(signed long long)set_->get_num_of_points()
&& current_pos_>=0,
"iterator out of bounds %lli>=%lli\n",
(unsigned long long)current_pos_,
(unsigned long long)set_->get_num_of_points());
return diff;
}
Iterator operator/(const int divider) {
Iterator it(*this);
it.current_pos_/=divider;
return it;
}
bool operator==(const Iterator &other) {
if (likely(other.set_==set_)) {
return current_pos_==other.current_pos_;
} else {
return false;
}
}
bool operator!=(const Iterator &other) {
if (likely(other.set_==set_)) {
return current_pos_!=other.current_pos_;
} else {
return true;
}
}
bool operator<=(const Iterator &other) {
if (likely(other.set_==set_)) {
return current_pos_<=other.current_pos_;
} else {
return false;
}
}
bool operator>(const Iterator &other) {
if (likely(other.set_==set_)) {
return current_pos_>other.current_pos_;
} else {
return false;
}
}
bool operator<(const Iterator &other) {
if (likely(other.set_==set_)) {
return current_pos_<other.current_pos_;
} else {
return false;
}
}
CompletePoint<Precision_t> &operator*() {
CompletePoint<Precision_t> point;
point.Alias(set_->At(current_pos_),
set_->get_id(current_pos_),
set_->get_dimension());
return point;
}
CompletePoint<Precision_t> &operator->() {
CompletePoint<Precision_t> point;
point.Alias(set_->At(current_pos_),
set_->get_id(current_pos_),
set_->get_dimension());
return point;
}
private:
BinaryDataset<Precision_t> *set_;
index_t current_pos_;
};
BinaryDataset() {
data_file_="";
index_file_="";
data_=NULL;
index_=NULL;
page_size_ = getpagesize();
}
~BinaryDataset() {
}
// Initializes a the BinaryDataset from two existent files
// one for data and one for the index
success_t Init(string data_file, string index_file) {
data_file_=data_file;
index_file_=index_file;
ReadDimNumOfPointsFromFile(data_file_);
data_=(Precision_t*)MemoryMap(data_file_, sizeof(int32));
index_=(uint64*)MemoryMap(index_file_,0);
return SUCCESS_PASS;
}
// If the index file is not given it assumes it has the same name as
// the data file appended with ind
success_t Init(string data_file) {
string temp=data_file + string(".ind");
Init(data_file, temp);
return SUCCESS_PASS;
}
// Use this to create a new BinarayDataset file, If you omit
// the index file it will create a index file appended with .ind
// extension.
success_t Init(string data_file, uint64 num_of_points,
int32 dimension) {
data_file_=data_file;
index_file_=data_file;
index_file_.append(".ind");
num_of_points_=num_of_points;
Init(data_file_, index_file_, num_of_points_, dimension);
return SUCCESS_PASS;
}
success_t Init(string data_file, string index_file,
uint64 num_of_points, int32 dimension) {
data_file_=data_file;
index_file_=index_file;
num_of_points_=num_of_points;
dimension_=dimension;
CreateDataFile(data_file, dimension, num_of_points);
CreateIndexFile(index_file, num_of_points);
data_=(Precision_t*)MemoryMap(data_file_, sizeof(int32));
index_=(uint64*)MemoryMap(index_file_,0);
return SUCCESS_PASS;
}
Iterator Begin() {
Iterator it(this);
return it;
}
Iterator End() {
Iterator it(this);
it.current_pos_=num_of_points_;
return it;
}
inline void AdviseWillNeed(index_t start, index_t end) {
index_t num_of_bytes1 = (end-start)*dimension_*sizeof(Precision_t);
index_t num_of_bytes2 = (end-start)*sizeof(index_t);
Advise1(data_, start*dimension_*sizeof(Precision_t), num_of_bytes1);
Advise2(index_, start*sizeof(index_t), num_of_bytes2);
}
inline void AdviseWillNotNeed(index_t start, index_t end) {
index_t num_of_bytes1 = (end-start)*dimension_*sizeof(Precision_t);
index_t num_of_bytes2 = (end-start)*sizeof(index_t);
Advise2(data_, start*dimension_*sizeof(Precision_t), num_of_bytes1);
Advise2(index_, start*sizeof(index_t), num_of_bytes2);
}
// Use this to swap the points of a dataset
// swaps the index values as well
inline void Swap(uint64 i, uint64 j) {
Precision_t temp[dimension_];
memcpy(temp, At(j), dimension_ * sizeof(Precision_t));
memcpy(At(j), At(i), dimension_ * sizeof(Precision_t));
memcpy(At(i), temp, dimension_ * sizeof(Precision_t));
uint64 temp_index=index_[i];
index_[i]=index_[j];
index_[j]=temp_index;
}
// Use this for destruction
void Destruct() {
MemoryUnmap(data_, data_file_, sizeof(int32));
MemoryUnmap(index_, index_file_, 0);
}
// returns a matrix on the data
inline Matrix get_data_matrix() {
Matrix matrix;
matrix.Alias(data_, num_of_points_/dimension_, dimension_);
return matrix;
}
// returns a vector on the index
/*inline Vector get_index_vector() {
Vector vector;
vector.Alias(index_, num_of_points_);
return vector;
}*/
inline Point<Precision_t, Loki::NullType> get_point(index_t i) {
Point<Precision_t, Loki::NullType> point;
point.Alias(At(i), get_id(i));
return point;
}
// returns a pointer on the data at the ith point
inline Precision_t* At(uint64 i) {
DEBUG_ASSERT_MSG(i<num_of_points_,
"Attempt to acces data out of range %llu>%llu",
(unsigned long long)i,
(unsigned long long)num_of_points_);
return data_+i*dimension_;
}
// returns a reference on the i,j element
inline Precision_t &At(uint64 i, int32 j) {
DEBUG_ASSERT_MSG(i<num_of_points_,
"Attempt to acces data out of range %lli>%lli",
(unsigned long long)i,
(unsigned long long)num_of_points_);
DEBUG_ASSERT_MSG(j<dimension_,
"Attempt to access element greater that the dimension %lli>%lli",
(unsigned long long)j,
(unsigned long long)dimension_);
return data_[i*dimension_+j];
}
// get the index at i point
inline uint64 get_id(index_t i) {
return index_[i];
}
// set id at point i
inline void set_id(index_t i, index_t value) {
index_[i]=value;
}
// get the number of points
uint64 get_num_of_points() {
return num_of_points_;
}
// get the dimension
int32 get_dimension() {
return dimension_;
}
// get the name of the data file
string get_data_file() {
return data_file_;
}
// get the index file
string get_index_file() {
return index_file_;
}
private:
// system page size
index_t page_size_;
// number of points on the data set
uint64 num_of_points_;
// dimension of the data
int32 dimension_;
// A pointer to the data
Precision_t *data_;
// A pointer to the index
uint64 *index_;
// data file name
string data_file_;
// index file name
string index_file_;
// Reads the dimension and the number of points from a
// data file
void ReadDimNumOfPointsFromFile(string file_name) {
FILE *fp=fopen(file_name.c_str(), "r");
if (unlikely(fp==NULL)) {
FATAL("Error :%s while reading %s\n",
strerror(errno), file_name.c_str());
}
if (unlikely(fread(&dimension_, sizeof(int32), 1, fp)!=1)) {
FATAL("Error :%s while reading %s\n",
strerror(errno), file_name.c_str());
}
fclose(fp);
struct stat info;
if (unlikely(stat(file_name.c_str(), &info)!=0)) {
FATAL("Error %s file %s\n",
strerror(errno), file_name.c_str());
}
num_of_points_ = (info.st_size-sizeof(int32))/
(dimension_*sizeof(Precision_t));
}
// utility function for mapping a file
// offset is used for datafile where we have to read the
// dimension and map the rest of the file
// while in the index file we set it to zero
void* MemoryMap(string file_name, uint64 offset) {
struct stat info;
if (unlikely(stat(file_name.c_str(), &info)!=0)) {
FATAL("Error %s file %s\n",
strerror(errno), file_name.c_str());
}
uint64 map_size = info.st_size;
int fp=open(file_name.c_str(), O_RDWR);
if (fp<0) {
FATAL("Cannot open %s, error %s\n", file_name.c_str(), strerror(errno));
}
void *ptr=mmap(0,
map_size,
PROT_READ | PROT_WRITE, MAP_SHARED,
fp,
0);
ptr = (void*) ((char*)ptr+offset);
if (unlikely(ptr==MAP_FAILED)) {
FATAL("Error %s while mapping %s\n",
strerror(errno), file_name.c_str());
}
close(fp);
return ptr;
}
// same as the above for unmapping
success_t MemoryUnmap(void *ptr, string file_name, uint64 offset) {
struct stat info;
if (unlikely(stat(file_name.c_str(), &info)!=0)) {
NONFATAL("Error %s file %s\n",
strerror(errno), file_name.c_str());
return SUCCESS_FAIL;
}
uint64 map_size = info.st_size;
ptr = (char *)ptr-offset;
if(munmap(ptr , map_size)<0) {
NONFATAL("Error %s while unmapping %s\n",
strerror(errno), file_name.c_str());
return SUCCESS_FAIL;
}
return SUCCESS_PASS;
}
// creates a data file
void CreateDataFile(string file_name,
int32 dimension,
uint64 num_of_points) {
FILE *fp=fopen(file_name.c_str(), "w");
if (unlikely(fwrite(&dimension, sizeof(int32), 1, fp)!=1)) {
FATAL("Error %s, while writing for file %s\n",
strerror(errno), file_name.c_str());
}
uint64 total_size = num_of_points * dimension * sizeof(Precision_t);
const uint64 buffer_length=8192;
char *buffer=new char[buffer_length];
for(uint64 i=0; i<total_size/buffer_length; i++) {
if (unlikely(fwrite(buffer, 1, buffer_length, fp)!=buffer_length))
FATAL("Error %s, while writing for file %s\n",
strerror(errno), file_name.c_str());
}
if (unlikely(fwrite(buffer, 1, total_size % buffer_length, fp)!=
total_size % buffer_length)) {
FATAL("Error %s, while writing for file %s\n",
strerror(errno), file_name.c_str());
}
fclose(fp);
}
// creates an index file
void CreateIndexFile(string file_name,
uint64 num_of_points) {
FILE *fp=fopen(file_name.c_str(), "w");
uint64 total_size = num_of_points * sizeof(uint64);
const uint64 buffer_length=8192;
char *buffer=new char[buffer_length];
for(uint64 i=0; i<total_size/buffer_length; i++) {
if (unlikely(fwrite(buffer, 1, buffer_length, fp)!=buffer_length)) {
FATAL("Error %s, while writing for file %s\n",
strerror(errno), file_name.c_str());
}
}
if (unlikely(fwrite(buffer, 1, total_size % buffer_length, fp)!=
total_size % buffer_length)) {
FATAL("Error %s, while writing for file %s\n",
strerror(errno), file_name.c_str());
}
fclose(fp);
}
void Advise1(void *feed, index_t start, index_t num_of_bytes) {
index_t num_of_pages = (index_t)ceil(1.0*num_of_bytes/page_size_);
if (num_of_pages<kMinimumPagesToAdvise) {
return;
}
char *ptr =(char *)feed+start;
char *temp=NULL;
ptr =temp + ((ptrdiff_t)feed/page_size_)*page_size_;
if (unlikely(madvise(ptr, num_of_pages, MADV_WILLNEED)==-1)) {
FATAL("Failed to advise, error %s", strerror(errno));
}
}
void Advise2(void *feed, index_t start, index_t num_of_bytes) {
index_t num_of_pages = (index_t)floor(1.0*num_of_bytes/page_size_);
if (num_of_pages<kMinimumPagesToAdvise) {
return;
}
char *ptr =(char*)feed+start;
char *temp=NULL;
ptr =temp + (ptrdiff_t)ceil((ptrdiff_t)feed/page_size_)*page_size_;
if (unlikely(madvise(ptr, num_of_pages, MADV_WILLNEED)==-1)) {
FATAL("Failed to advise, error %s", strerror(errno));
}
}
};
#endif // BINARY_DATASET_H_
Reference in New Issue View Git Blame Copy Permalink