diff --git a/fastlib/Doxyfile b/fastlib/Doxyfile index bc7a0d8dfe..0063811be7 100644 --- a/fastlib/Doxyfile +++ b/fastlib/Doxyfile @@ -49,7 +49,7 @@ EXTRACT_STATIC = NO EXTRACT_LOCAL_CLASSES = YES EXTRACT_LOCAL_METHODS = NO HIDE_UNDOC_MEMBERS = NO -HIDE_UNDOC_CLASSES = YES +HIDE_UNDOC_CLASSES = NO HIDE_FRIEND_COMPOUNDS = NO HIDE_IN_BODY_DOCS = NO INTERNAL_DOCS = NO @@ -90,7 +90,6 @@ RECURSIVE = YES EXCLUDE = EXCLUDE_SYMLINKS = YES EXCLUDE_PATTERNS = */auton/* \ - */u/* \ */test/* \ */bin/* \ *test.cc diff --git a/fastlib/base/build.py b/fastlib/base/build.py index cf600f67c9..78ce552915 100644 --- a/fastlib/base/build.py +++ b/fastlib/base/build.py @@ -1,24 +1,21 @@ +def config_doit(sysentry, files, params): + script = files["script"].single(Types.SCRIPT) + outdir = sysentry.bin_dir("arch", "kernel", "compiler") + indir = sysentry.sys.source_dir + sysentry.command("%s --genfiles_dir=%s --source_dir=%s" % (script, outdir, indir)) + gen_headers = ["base/basic_types.h"] + return [(Types.HEADER, sysentry.file(h, "arch", "kernel", "compiler")) + for h in gen_headers] -class ConfigHeadersRule(dep.Rule): - def __init__(self): - self.script = sourcerule(Types.SCRIPT, "config.py") - self.sources = sourcerules(Types.ANY, lglob("config/*.c")) - dep.Rule.__init__(self, script=[self.script], sources=self.sources) - def doit(self, sysentry, state, files, params): - script = files["script"].single(Types.SCRIPT) - outdir = sysentry.bin_dir("arch", "kernel", "compiler") - indir = sysentry.sys.source_dir - sysentry.command("%s --genfiles_dir=%s --source_dir=%s" % (script, outdir, indir)) - gen_headers = ["base/basic_types.h"] - return [(Types.HEADER, sysentry.file(h, "arch", "kernel", "compiler")) - for h in gen_headers] - -register(name = "config_headers", - rule = ConfigHeadersRule()) +customrule( + name = "config_headers", + dependencies = + {"script": [sourcerule(Types.SCRIPT, "../script/config.py")], + "sources": sourcerules(Types.ANY, lglob("config/*.c"))}, + doit_fn = config_doit) librule( sources = ["common.c", "cc.cc", "ccmem.cc"], headers = ["cc.h", "ccmem.h", "common.h", "compiler.h", - "compiler_impl.h", + "compiler_impl.h", "test.h", "fortran.h", "debug.h", "scale.h", ":config_headers"]) - diff --git a/fastlib/base/common.h b/fastlib/base/common.h index 03643d4a07..1e3870dc31 100644 --- a/fastlib/base/common.h +++ b/fastlib/base/common.h @@ -24,6 +24,7 @@ #include "scale.h" #include +#include EXTERN_C_START diff --git a/fastlib/base/compiler.h b/fastlib/base/compiler.h index 6d38f8764f..8b2bccf336 100644 --- a/fastlib/base/compiler.h +++ b/fastlib/base/compiler.h @@ -144,4 +144,14 @@ #define offsetof(structure, field) ((size_t)((&(structure const *)0)->field)) #endif +#ifdef __cplusplus +/** Use constant reference type in C, otherwise use constant pointer. */ +#define CONST_REF const& +/** Use reference type in C, otherwise use pointer. */ +#define REF & +#else +#define CONST_REF const* +#define REF * +#endif + #endif diff --git a/fastlib/base/compiler_impl.h b/fastlib/base/compiler_impl.h index 06428c029a..dab443b135 100644 --- a/fastlib/base/compiler_impl.h +++ b/fastlib/base/compiler_impl.h @@ -23,8 +23,8 @@ #ifdef __GNUC__ #define expect__impl(expr, value) (__builtin_expect((expr), (value))) -#define likely__impl(x) (__builtin_expect((x), 1)) -#define unlikely__impl(x) (__builtin_expect((x), 0)) +#define likely__impl(x) (__builtin_expect(!!(x), 1)) +#define unlikely__impl(x) (__builtin_expect(!!(x), 0)) #define COMPILER_NORETURN__IMPL __attribute__((noreturn)) #define COMPILER_PRINTF__IMPL(format_arg, dotdotdot_arg) \ __attribute__((format(printf, format_arg, dotdotdot_arg))) diff --git a/fastlib/base/debug.h b/fastlib/base/debug.h index 8354391e0f..6381712abf 100644 --- a/fastlib/base/debug.h +++ b/fastlib/base/debug.h @@ -214,4 +214,10 @@ void debug_poison_ptr(T *&x) { "%s == %"L64"d exceeds bound %s == %"L64"d\n", \ #x, STATIC_CAST(uint64, x), #bound, STATIC_CAST(uint64, bound)) +/** + * Asserts that two integers are the same in debug mode. + */ +#define DEBUG_SAME_INT(a, b) \ + DEBUG_ASSERT_MSG(((a) == (b)), "[%s] %d != %d [%s]", #a, (int)(a), (int)(b), #b) + #endif diff --git a/fastlib/base/test.h b/fastlib/base/test.h index 5f4279bfed..6dd0b50057 100644 --- a/fastlib/base/test.h +++ b/fastlib/base/test.h @@ -23,6 +23,14 @@ #define TEST_ASSERT(x) \ DEBUG_ASSERT(x) +#define TEST_DOUBLE_EXACT(a, b) \ + if (unlikely((a) != (b))) \ + FATAL("%.10e (%s) != %.10e (%s)", (double)(a), #a, (double)(b), #b); else + +#define TEST_DOUBLE_APPROX(a, b, absolute_eps) \ + if (unlikely(fabs((a) - (b)) > absolute_eps)) \ + FATAL("%.10e (%s) !~= %.10e (%s)", (double)(a), #a, (double)(b), #b); else + /** * Begin a test suite of a given name. * @@ -49,7 +57,7 @@ } typedef void (*test__void_func)(); -#endif #endif +#endif diff --git a/fastlib/col/col.cc b/fastlib/col/col.cc index 5b4b5e7c3e..66766adfb4 100644 --- a/fastlib/col/col.cc +++ b/fastlib/col/col.cc @@ -72,6 +72,10 @@ index_t String::Split(index_t start_index, while (1) { if (unlikely(*endpos == '\0') || strchr(donechars, *endpos) != NULL) { + // strip extra delimeters from right side + while (endpos > startpos && strchr(delimeters, endpos[-1]) != NULL) { + endpos--; + } done = true; break; } @@ -93,3 +97,31 @@ index_t String::Split(index_t start_index, return pos - begin(); } +void String::TrimLeft(const char *delimeters, String *result) const { + const char *s = begin(); + while (*s != '\0' && strchr(delimeters, *s)) { + s++; + } + result->Copy(s, end() - s); +} + +void String::TrimRight(const char *delimeters, String *result) const { + const char *s = end() - 1; + const char *b = begin(); + while (s >= b && strchr(delimeters, *s)) { + s--; + } + result->Copy(b, s - b + 1); +} + +void String::Trim(const char *delimeters, String *result) const { + const char *b = begin(); + const char *e = end() - 1; + while (e >= b && strchr(delimeters, *e)) { + e--; + } + while (e >= b && strchr(delimeters, *b)) { + b++; + } + result->Copy(b, e - b + 1); +} diff --git a/fastlib/col/heap.h b/fastlib/col/heap.h index 77e06e9fd2..3d7d8be295 100644 --- a/fastlib/col/heap.h +++ b/fastlib/col/heap.h @@ -92,6 +92,8 @@ class MinHeap { /** * Pops and returns the lowest element off the heap. + * + * @return the value associated with the highest priority */ Value Pop() { Value t = entries_[0].value; @@ -103,6 +105,9 @@ class MinHeap { /** * Removes the lowest element from the heap. + * + * Simply pops the top value on the queue, without + * returning it. */ void PopOnly() { Entry entry = *entries_.PopBackPtr(); diff --git a/fastlib/col/string.h b/fastlib/col/string.h index 8a243c871b..699c125feb 100644 --- a/fastlib/col/string.h +++ b/fastlib/col/string.h @@ -22,8 +22,9 @@ * tokenizers that you may find useful. Finally, it is non-templated, * so compiler errors are easier to understand. * - * WARNING: This has gone through rigorous testing -- we expect it to work, - * but there may be some issues. + * WARNING: This has not gone through rigorous testing -- we expect it to + * work, but there may be some issues. You will be just fine using the + * STL string if you need string processing. */ class String { private: @@ -365,6 +366,24 @@ class String { index_t Split(const char *delimeters, ArrayList *result) const { return Split(0, delimeters, "", 0, result); } + + /** + * Creates a new string with the specified characters removed from the left + * part of the string. + */ + void TrimLeft(const char *delimeters, String *result) const; + + /** + * Creates a new string with the specified characters removed from the right + * part of the string. + */ + void TrimRight(const char *delimeters, String *result) const; + + /** + * Creates a new string with the specified characters removed from the left + * and right parts of the string. + */ + void Trim(const char *delimeters, String *result) const; /** * Compares two strings case-insensitively. diff --git a/fastlib/data/crossvalidation.h b/fastlib/data/crossvalidation.h index 6f83a5df25..1e5ff50dc3 100644 --- a/fastlib/data/crossvalidation.h +++ b/fastlib/data/crossvalidation.h @@ -13,7 +13,8 @@ #include "fx/fx.h" /** - * Cross-validator for classifiers, integrating tightly with FastExec. + * Cross-validator for simple classifiers, integrating tightly with + * FastExec. * * Cross-validation runs go under path you give it (kfold_fx_name), * by default "kfold". @@ -51,6 +52,32 @@ * /kfold/0/knn/params/k # this ensures you'll get default params * /kfold * @endcode + * + * Before the cross-validator runs, it will copy parameters from the module + * you specify -- if it is module_root, this will just take the original + * command line parameters that are stored in "/params". In the previous + * example, the command line parameters from "/params/knn/" and + * "/params/kfold/" are used. These parameters are specified by the user + * as "--params/knn/someparameter=3" or "--param/kfold/k=4" to set KNN's + * "someparameter" to 3, and the cross-validator's number of folds to 4. + * + * + * To build a classifier suitable for use with SimpleCrossValidator, you + * must create a class with the following methods: + * + * @code + * class MyClassifier { + * ... + * // Trains on the dataset specified. n_classes is the number of class + * // labels. Tweak parameters can be obtained from the "datanode" passed + * // using fx_param_int, fx_param_double, etc, but passing in "module" as + * // the first parameter instead of NULL. + * // + * void InitTrain(const Dataset& dataset, int n_classes, datanode *module); + * // For a test datum, returns the class label 0 <= label < n_classes + * int Classify(const Vector& test_datum); + * }; + * @endcode */ template class SimpleCrossValidator { @@ -145,12 +172,12 @@ class SimpleCrossValidator { Dataset test; Dataset train; index_t local_n_correct = 0; - datanode *foldmodule = fx_submodule(kfold_module_, - String().InitSprintf("%d", i_folds).c_str(), NULL); - datanode *classifier_module = fx_submodule(foldmodule, - classifier_fx_name_, NULL); + datanode *foldmodule = fx_submodule(kfold_module_, NULL, + String().InitSprintf("%d", i_folds).c_str()); + datanode *classifier_module = fx_submodule(foldmodule, NULL, + classifier_fx_name_); - fx_def_param_node(classifier_module, "", root_module_, + fx_default_param_node(classifier_module, "", root_module_, classifier_fx_name_); data_->SplitTrainTest(n_folds_, i_folds, permutation, &train, &test); diff --git a/fastlib/data/dataset.cc b/fastlib/data/dataset.cc index 5afe930d22..4b07b44ebe 100644 --- a/fastlib/data/dataset.cc +++ b/fastlib/data/dataset.cc @@ -5,6 +5,7 @@ */ #include "file/textfile.h" +#include "base/common.h" #include "dataset.h" @@ -161,15 +162,19 @@ success_t DatasetInfo::InitFromArff(TextLineReader *reader, feature->InitNominal(portions[1]); // TODO: Doesn't support values with spaces { portions[2].Split(1, ", \t", "}%", 0, &feature->value_names()); - } else if (portions[2].EqualsNoCase("numeric") - || portions[2].EqualsNoCase("real")) { - features_.AddBack()->InitContinuous(portions[1]); - } else if (portions[2].EqualsNoCase("integer")) { - features_.AddBack()->InitInteger(portions[1]); } else { - reader->Error( - "ARFF: Only support 'numeric', 'real', and {nominal}."); - result = SUCCESS_FAIL; + String type(portions[2]); + //portions[2].Trim(" \t", &type); + if (type.EqualsNoCase("numeric") + || type.EqualsNoCase("real")) { + features_.AddBack()->InitContinuous(portions[1]); + } else if (type.EqualsNoCase("integer")) { + features_.AddBack()->InitInteger(portions[1]); + } else { + reader->Error( + "ARFF: Only support 'numeric', 'real', and {nominal}."); + result = SUCCESS_FAIL; + } } } } else if (portions[0].EqualsNoCase("@data")) { @@ -353,6 +358,10 @@ success_t DatasetInfo::ReadMatrix(TextLineReader *reader, Matrix *matrix) const break; } + while (*pos == ' ' || *pos == '\t' || *pos == ',') { + pos++; + } + if (*pos != '\0') { for (char *s = reader->Peek().begin(); s < pos; s++) { if (*s == '\0') { @@ -518,3 +527,16 @@ void Dataset::SplitTrainTest(int folds, int fold_number, DEBUG_ASSERT(i_train == train->n_points()); DEBUG_ASSERT(i_test == test->n_points()); } + +void data::Load(const char *fname, Matrix *matrix) { + Dataset dataset; + dataset.InitFromFile(fname); + matrix->Own(&dataset.matrix()); +} + +void data::Save(const char *fname, const Matrix& matrix) { + Dataset dataset; + dataset.AliasMatrix(matrix); + dataset.WriteCsv(fname); +} + diff --git a/fastlib/data/dataset.h b/fastlib/data/dataset.h index 57ce3cbf9f..ab60d1ee7b 100644 --- a/fastlib/data/dataset.h +++ b/fastlib/data/dataset.h @@ -545,8 +545,8 @@ class Dataset { /** * Creates a training and test dataset for k-fold cross validation. * - * The training set will be approximately n_points() / folds, and the - * test set will be all remaining points. This takes as an argument + * The test set will be approximately n_points() / folds, and the + * training set will be all remaining points. This takes as an argument * a permutation to allow use of consistent random permutations. If * an identity permutation is used, the split will be performed strided. * @@ -562,4 +562,40 @@ class Dataset { Dataset *train, Dataset *test) const; }; +/** + * Miscellaneous dataset-related routines. + */ +namespace data { + /** + * Loads a matrix from a file. + * + * This supports any type the Dataset class supports with the + * InitFromFile function: CSV and ARFF. + * + * @code + * Matrix A; + * data::Load("foo.csv", &A); + * @endcode + * + * @param fname the file name to load + * @param matrix a pointer to an uninitialized matrix to load + */ + void Load(const char *fname, Matrix *matrix); + /** + * Saves a matrix to a file. + * + * This saves in CSV format that MATLAB and Excel can handle. + * + * @code + * Matrix matrix_to_save; + * ... matrix_to_save contains the values you want to save + * data::Save("mymatrix.csv", matrix_to_save); + * @endcode + * + * @param fname the file name to load + * @param matrix a pointer to an uninitialized matrix to load + */ + void Save(const char *fname, const Matrix& matrix); +}; + #endif diff --git a/fastlib/data/dataset_test.cc b/fastlib/data/dataset_test.cc index 942237e4b8..36937441c9 100644 --- a/fastlib/data/dataset_test.cc +++ b/fastlib/data/dataset_test.cc @@ -1,8 +1,117 @@ #include "dataset.h" -#include "xrun/xrun.h" #include "math/discrete.h" +#include "base/test.h" +TEST_SUITE_BEGIN(dataset) + +void TestSplitTrainTest() { + Dataset orig; + + orig.InitBlank(); + orig.matrix().Init(1, 12); + orig.info().InitContinuous(1); + + for (int i = 0; i < 12; i++) { + orig.matrix().set(0, i, i); + } + + ArrayList found; + + found.Init(12); + + for (int i = 0; i < 12; i++) { + found[i] = 0; + } + + Dataset train; + Dataset test; + + ArrayList permutation; + + math::MakeIdentityPermutation(12, &permutation); + + orig.SplitTrainTest(5, 1, permutation, + &train, &test); + + DEBUG_ASSERT(test.n_points() == 3); + DEBUG_ASSERT(train.n_points() == 9); + DEBUG_ASSERT_MSG(test.get(0, 0) == 1, "%f", (test.get(0, 0))); + DEBUG_ASSERT_MSG(test.get(0, 1) == 6, "%f", (test.get(0, 1))); + DEBUG_ASSERT_MSG(test.get(0, 2) == 11, "%f", (test.get(0, 2))); + + DEBUG_ASSERT_MSG(train.get(0, 0) == 0, "%f", (train.get(0, 0))); + DEBUG_ASSERT_MSG(train.get(0, 1) == 2, "%f", (train.get(0, 1))); + DEBUG_ASSERT_MSG(train.get(0, 2) == 3, "%f", (train.get(0, 2))); + DEBUG_ASSERT_MSG(train.get(0, 3) == 4, "%f", (train.get(0, 3))); + DEBUG_ASSERT_MSG(train.get(0, 4) == 5, "%f", (train.get(0, 4))); + DEBUG_ASSERT_MSG(train.get(0, 5) == 7, "%f", (train.get(0, 5))); + DEBUG_ASSERT_MSG(train.get(0, 6) == 8, "%f", (train.get(0, 6))); + DEBUG_ASSERT_MSG(train.get(0, 7) == 9, "%f", (train.get(0, 7))); + DEBUG_ASSERT_MSG(train.get(0, 8) == 10, "%f", (train.get(0, 8))); +} + +void AssertSameMatrix(const Matrix& a, const Matrix& b) { + index_t r = a.n_rows(); + index_t c = a.n_cols(); + + TEST_ASSERT(a.n_rows() == b.n_rows()); + TEST_ASSERT(a.n_cols() == b.n_cols()); + + for (index_t ri = 0; ri < r; ri++) { + for (index_t ci = 0; ci < c; ci++) { + DEBUG_ASSERT_MSG(a.get(ri, ci) == b.get(ri, ci), "(%d, %d): %f != %f", + ri, ci, a.get(ri, ci), b.get(ri, ci)); + } + } +} + +void TestLoad() { + Dataset d1; + Dataset d2; + Dataset d3; + Dataset d4; + Dataset d5; + + ASSERT_PASS(d1.InitFromFile("test/fake.arff")); + ASSERT_PASS(d2.InitFromFile("test/fake.csv")); + ASSERT_PASS(d3.InitFromFile("test/fake.csvh")); + ASSERT_PASS(d4.InitFromFile("test/fake.tsv")); + ASSERT_PASS(d5.InitFromFile("test/fake.weird")); + + AssertSameMatrix(d1.matrix(), d2.matrix()); + AssertSameMatrix(d1.matrix(), d3.matrix()); + AssertSameMatrix(d1.matrix(), d4.matrix()); + AssertSameMatrix(d1.matrix(), d5.matrix()); +} + +void TestStoreLoad() { + Dataset d1; + Dataset d2; + Dataset d3; + + ASSERT_PASS(d1.InitFromFile("test/fake.arff")); + d1.WriteCsv("test/fakeout1.csv"); + d1.WriteArff("test/fakeout1.arff"); + + ASSERT_PASS(d2.InitFromFile("test/fakeout1.arff")); + ASSERT_PASS(d3.InitFromFile("test/fakeout1.csv")); + + AssertSameMatrix(d1.matrix(), d2.matrix()); + AssertSameMatrix(d1.matrix(), d3.matrix()); + + DEBUG_ASSERT_MSG(strcmp(d1.info().name(), d2.info().name()) == 0, + "%s != %s", d1.info().name(), d2.info().name()); + for (index_t i = 0; i < d1.info().n_features(); i++) { + DEBUG_ASSERT( + strcmp(d1.info().feature(i).name(), d2.info().feature(i).name()) == 0); + DEBUG_ASSERT(d1.info().feature(i).type() == d2.info().feature(i).type()); + } +} + +TEST_SUITE_END(dataset, TestSplitTrainTest, TestLoad, TestStoreLoad) + +/* int main(int argc, char *argv[]) { xrun_init(argc, argv); const char *in = xrun_param_str("in"); @@ -45,3 +154,4 @@ int main(int argc, char *argv[]) { return 0; } +*/ diff --git a/fastlib/fastlib/build.py b/fastlib/fastlib/build.py index 1b86868112..8bcbd48cde 100644 --- a/fastlib/fastlib/build.py +++ b/fastlib/fastlib/build.py @@ -7,13 +7,3 @@ librule( #, "tree:tree", "par:par" ] ) - -librule( - name = "fastlib_int", - headers = ["fastlib.h"], - deplibs = ["la:la", "base:base", - "fx:fx", "file:file_int", "col:col", - "data:data", "math:math" - "tree:tree", "par:par" - ] - ) diff --git a/fastlib/fastlib/fastlib.h b/fastlib/fastlib/fastlib.h index d9daf85d8e..84ee3ef21e 100644 --- a/fastlib/fastlib/fastlib.h +++ b/fastlib/fastlib/fastlib.h @@ -45,12 +45,15 @@ * comment header files and not source files -- even without Doxygen, header * files remain a natural place for documentation. * - * To generate the doxygen yourself, go in to the code directory and type: + * To generate this HTML documentation yourself, go in to the code directory + * and type: * * @code * doxygen * @endcode * + * Then, visit doc/html/index.html (within the code directory). + * * @section remarks Final Remarks * * This software was written at Georgia Institute of Technology. diff --git a/fastlib/file/build.py b/fastlib/file/build.py index 4aae2ce1b2..ca7bf245d2 100644 --- a/fastlib/file/build.py +++ b/fastlib/file/build.py @@ -5,13 +5,6 @@ librule( deplibs = ["base:base", "col:col"], ) -librule( # internal rule - name = "file_int", - sources = ["serialize.cc"], - headers = ["serialize.h"], - deplibs = [":file"], - ) - binrule( name = "textfile_test", sources = ["textfile_test.cc"], diff --git a/fastlib/file/textfile.h b/fastlib/file/textfile.h index 906224d28e..23e1ee46e7 100644 --- a/fastlib/file/textfile.h +++ b/fastlib/file/textfile.h @@ -328,7 +328,7 @@ class TextWriter { } /** - * Opens a file by name. + * Opens a file by name (initializer). * * @return success or failure */ diff --git a/fastlib/file/textfile_test.cc b/fastlib/file/textfile_test.cc index 41c69015da..b2e7463525 100644 --- a/fastlib/file/textfile_test.cc +++ b/fastlib/file/textfile_test.cc @@ -3,7 +3,7 @@ #include "xrun/xrun.h" -#include +#include void Test1() { TextTokenizer scanner; diff --git a/fastlib/fx/fx.h b/fastlib/fx/fx.h index 698ecfb1c1..41872e09d5 100644 --- a/fastlib/fx/fx.h +++ b/fastlib/fx/fx.h @@ -49,7 +49,7 @@ void fx_done(void); */ int fx_param_exists(struct datanode *module, const char *name); /** - * Obtain a string parameter. + * Obtain a string parameter or use provided default. * * @param module the param's containing module, or NULL for global * @param name the name of the parameter (paths allowed) @@ -58,41 +58,75 @@ int fx_param_exists(struct datanode *module, const char *name); const char *fx_param_str(struct datanode *module, const char *name, const char *def); /** - * Obtain a floating-point parameter. + * Obtain a string parameter, failing if it is not specified. + * + * @param module the param's containing module, or NULL for global + * @param name the name of the parameter (paths allowed) + */ +const char *fx_param_str_req(struct datanode *module, const char *name); +/** + * Obtain a floating-point parameter or use provided default. * * If the parameter is not specified by the user and the default is not * DBL_NAN, then the default value will be stored explicitly in the tree. * * @param module the param's containing module, or NULL for global * @param name the name of the parameter (paths allowed) - * @param def value used if param not given, or DBL_NAN if required + * @param def value used if param not given */ -double fx_param_double(struct datanode *module, const char *name, - double def); +double fx_param_double(struct datanode *module, const char *name, double def); /** - * Obtain an integral parameter. + * Obtain a floating-point parameter, failing if it is not specified. + * + * If the parameter is not specified by the user and the default is not + * DBL_NAN, then the default value will be stored explicitly in the tree. + * + * @param module the param's containing module, or NULL for global + * @param name the name of the parameter (paths allowed) + */ +double fx_param_double_req(struct datanode *module, const char *name); +/** + * Obtain an integral parameter or use provided default. * * If the parameter is not specified by the user and the default is not -1, * then the default value will be stored explicitly in the tree. * * @param module the param's containing module, or NULL for global * @param name the name of the parameter (paths allowed) - * @param def value used if param not given, or -1 if required + * @param def value used if param not given */ -int fx_param_int(struct datanode *module, const char *name, - int def); +int fx_param_int(struct datanode *module, const char *name, int def); /** - * Obtain a boolean parameter. + * Obtain an integral parameter, failing if it is not specified. + * + * If the parameter is not specified by the user and the default is not -1, + * then the default value will be stored explicitly in the tree. + * + * @param module the param's containing module, or NULL for global + * @param name the name of the parameter (paths allowed) + */ +int fx_param_int_req(struct datanode *module, const char *name); +/** + * Obtain a boolean parameter or use provided default. * * Values starting with f, F, n, N, or 0 are false; all others are * true. * * @param module the param's containing module, or NULL for global * @param name the name of the parameter (paths allowed) - * @param def value used if param not given, or NULL if required + * @param def value used if param not given */ -int fx_param_bool(struct datanode *module, const char *name, - const char *def); +int fx_param_bool(struct datanode *module, const char *name, int def); +/** + * Obtain a boolean parameter, failing if it is not specified. + * + * Values starting with f, F, n, N, or 0 are false; all others are + * true. + * + * @param module the param's containing module, or NULL for global + * @param name the name of the parameter (paths allowed) + */ +int fx_param_bool_req(struct datanode *module, const char *name); /** * Obtain a segment of the datastore corresponding to a parameter. * @@ -125,7 +159,8 @@ struct datanode *fx_param_node(struct datanode *module, const char *name); * @param name the name of the parameter (paths allowed) * @param def the value the parameter assumes if unspecified */ -void fx_def_param(struct datanode *module, const char *name, const char *def); +void fx_default_param(struct datanode *module, const char *name, + const char *def); /** * Set a parameter to a given value. * @@ -135,7 +170,8 @@ void fx_def_param(struct datanode *module, const char *name, const char *def); * @param name the name of the parameter (paths allowed) * @param format a format string for the parameter, as in printf */ -void fx_set_param(struct datanode *module, const char *name, const char *val); +void fx_set_param(struct datanode *module, const char *name, + const char *val); /** * Set a parameter to a formatted string. * @@ -172,9 +208,8 @@ void fx_clear_param(struct datanode *module, const char *name); * @param src_module the module to copy it from * @param srcname the name under the module's params to copy it from */ -void fx_def_param_node(struct datanode *dest_module, const char *destname, - struct datanode *src_module, const char *srcname); - +void fx_default_param_node(struct datanode *dest_module, const char *destname, + struct datanode *src_module, const char *srcname); /** * Copy parameters from one module to another, overwriting. * @@ -265,7 +300,7 @@ void fx_timer_stop(struct datanode *module, const char *name); * before they are used. * * @code - * struct datanode *tb_mod = fx_module(NULL, "tree_bldg", "tree_building"); + * struct datanode *tb_mod = fx_module(NULL, "tree_building", "tree_bldg"); * @endcode * * Any existing parameters (e.g. if the submodule is not fresh) are @@ -273,12 +308,13 @@ void fx_timer_stop(struct datanode *module, const char *name); * forwarding. * * @param module the containing module of the submodule - * @param name the name of the submodule (paths allowed) - * @param params_path_template the param node to forward (paths allowed) - * or NULL, printf-style + * @param params the param node to forward (paths allowed) or NULL + * @param name_format the name of the submodule (paths allowed); + * formatted as in printf */ -struct datanode *fx_submodule(struct datanode *module, const char *name, - const char *params_path_template, ...); +COMPILER_PRINTF(3, 4) +struct datanode *fx_submodule(struct datanode *module, const char *param, + const char *name_format, ...); EXTERN_C_END diff --git a/fastlib/la/build.py b/fastlib/la/build.py index 54a5beaed3..9977454c46 100644 --- a/fastlib/la/build.py +++ b/fastlib/la/build.py @@ -1,25 +1,54 @@ +# This first part is dedicated to compiling and installing LAPACK. +# Scroll down later... -class LaProtoRule(dep.Rule): - def __init__(self): - self.script = sourcerule(Types.SCRIPT, "proto.py") - self.sources = [] # TODO: Include files this depends on - dep.Rule.__init__(self, script=[self.script], sources=self.sources) - def doit(self, sysentry, state, files, params): - script = files["script"].single(Types.SCRIPT) - proto_header = sysentry.file("la/proto.h", "arch", "kernel", "compiler") - sysentry.command("python %s --outfile=%s" % (script.name, proto_header.name)) - gen_headers = ["la/proto.h"] - return [(Types.HEADER, proto_header)] +wgetrule( + name = "blaspack_tgz", + type = Types.ANY, + url = "http://www.cc.gatech.edu/~garryb/fastlib/blaspack.tgz") + +def doit_compile_lapack(sysentry, files, params): + blaspack_tgz = files["blaspack_tgz"].single(Types.ANY) + libblaspack = sysentry.file("KEEP/libblaspack.a", "arch", "kernel", "compiler") + workspace_dir = os.path.join(os.path.dirname(libblaspack.name), "libblaspack_workspace") + compiler_info = compilers[params["compiler"]] + compiler = compiler_info.compiler_program("f") + # Make sure we won't rm -rf anything bad + assert "libblaspack_workspace" in workspace_dir + sysentry.command("mkdir -p %s" % workspace_dir) + sysentry.command("cd %s && tar -xzf %s" % (workspace_dir, blaspack_tgz.name)) + sysentry.command("echo '*** Compiling LAPACK with BLAS reference implementation.'") + sysentry.command("echo '... Our compilation differs slightly from regular LAPACK/BLAS:'") + sysentry.command("echo '... NOTE 1: We omit complex-number routines (halves compile time).'") + sysentry.command("echo '... NOTE 2: We require case sensitivity for LAPACK/BLAS string parameters.'") + sysentry.command("echo '... This may take several minutes (about 800 FORTRAN files).'") + # Loop unrolling is not useful on modern architectures (and bloats EXE size). + # Thus, we only compile -O2. + sysentry.command("cd %s && %s -O2 -c src/*.f" % (workspace_dir, compiler)) + sysentry.command("echo '... Almost done with LAPACK/BLAS...'") + noopt = "dlamch slamch" # these have to be compiled without optimization + noopt_real = " ".join(["src/%s.f" % x for x in noopt.split()]) + sysentry.command("cd %s && %s -O0 -c %s" % (workspace_dir, compiler, noopt_real)) + sysentry.command("cd %s && ar r %s *.o" % (workspace_dir, libblaspack.name)) + sysentry.command("echo '... Created archive, cleaning up.'") + sysentry.command("rm -rf %s" % workspace_dir) + sysentry.command("echo '*** Done with LAPACK and BLAS!'") + return [(Types.LINKABLE, libblaspack)] + +customrule( + name = "libblaspack", + dependencies = {"blaspack_tgz": [find(":blaspack_tgz")]}, + doit_fn = doit_compile_lapack) + +#---- This is the main part of LA package librule( - sources = [], - headers = ["matrix.h", "la.h"], - deplibs = ["base:base", "col:col"]) + sources = ["uselapack.cc"], + headers = ["matrix.h", "la.h", "uselapack.h", "clapack.h", "blas.h"], + deplibs = ["base:base", "col:col", ":libblaspack"]) binrule( - name = "lapack_test", - sources = ["lapack.cc"], - headers = [LaProtoRule()], + name = "uselapack_test", + sources = ["uselapack_test.cc"], linkables = [":la"]) binrule( diff --git a/fastlib/la/la.h b/fastlib/la/la.h index f58ac240f7..94882d6bdb 100644 --- a/fastlib/la/la.h +++ b/fastlib/la/la.h @@ -3,26 +3,21 @@ /** * @file la.h * - * Core routines for la algebra. + * Core routines for linear algebra. * - * TODO: Currently lacking much, and eventually we want this to simply - * use BLAS and LAPACK. + * See uselapack.h for more linear algebra routines. */ #ifndef LINEAR_H #define LINEAR_H #include "matrix.h" +#include "uselapack.h" #include "base/scale.h" -#include +#include -/** - * Namespace for linear-algebra helper routines. - * - * Will eventually include BLAS and LAPACk support. - */ namespace la { /** * Finds the Euclidean distance squared between two vectors. @@ -47,193 +42,6 @@ namespace la { DEBUG_ASSERT(x.length() == y.length()); return DistanceSqEuclidean(x.length(), x.ptr(), y.ptr()); } - - - /* these will be replaced with lapack eventually */ -#ifndef LAPACK - /** - * Finds the dot product of two vectors. - */ - inline double VectorDot(index_t length, const double *x, const double *y) { - double sum = 0; - for (index_t i = 0; i < length; i++) { - sum += x[i] * y[i]; - } - return sum; - } - /** - * Finds the dot product of two vectors. - */ - inline double VectorDot(const Vector& x, const Vector& y) { - DEBUG_ASSERT(x.length() == y.length()); - return VectorDot(x.length(), x.ptr(), y.ptr()); - } - - //--- - - /** - * Adds one vector to another (x <- x + y); - */ - inline void VectorAddTo(index_t length, const double *x, double *y) { - do { - *y += *x; - x++; - y++; - } while (--length); - } - - /** - * Adds one vector to another (x <- x + y); - */ - inline void VectorAddTo(const Vector&x, Vector *y) { - DEBUG_ASSERT(x.length() == y->length()); - VectorAddTo(x.length(), x.ptr(), y->ptr()); - } - - /** - * Adds one matrix to another (X <- X + Y); - */ - inline void MatrixAddTo(const Matrix& x, Matrix *y) { - DEBUG_ASSERT(x.n_rows() == y->n_rows()); - DEBUG_ASSERT(x.n_cols() == y->n_cols()); - VectorAddTo(x.n_elements(), x.ptr(), y->ptr()); - } - - //--- - - /** - * Adds a vector times a factor to an existing vector (y <- y + xscale * x). - */ - inline void VectorAddTo(index_t length, - double xscale, const double *x, double *y) { - do { - *y += xscale * (*x); - x++; - y++; - } while (--length); - } - - /** - * Adds a vector times a factor to an existing vector (y <- y + xscale * x). - */ - inline void VectorAddTo(double xscale, const Vector&x, Vector *y) { - DEBUG_ASSERT(x.length() == y->length()); - VectorAddTo(x.length(), xscale, x.ptr(), y->ptr()); - } - - /** - * Adds a matrix times a factor to an existing matrix (Y <- Y + xscale * X). - */ - inline void MatrixAddTo(double xscale, const Matrix& x, Matrix *y) { - DEBUG_ASSERT(x.n_rows() == y->n_rows()); - DEBUG_ASSERT(x.n_cols() == y->n_cols()); - VectorAddTo(x.n_elements(), xscale, x.ptr(), y->ptr()); - } - - //--- - - /** - * Adds two vectors to a new vector (z <- x + y). - */ - inline void VectorAddOverwrite(index_t length, - const double *x, const double *y, double *c) { - for (index_t i = 0; i < length; i++) { - c[i] = x[i] + y[i]; - } - } - - /** - * Adds two vectors to a new vector (z <- x + y). - * @param z will be Initialized to the sum of x and y - */ - inline void VectorAddInit( - const Vector& x, const Vector& y, Vector *z) { - DEBUG_ASSERT(x.length() == y.length()); - z->Init(x.length()); - VectorAddOverwrite(x.length(), x.ptr(), y.ptr(), z->ptr()); - } - - /** - * Adds two matrices to a new matrix (Z <- X + Y). - * @param z will be Initialized to the sum of x and y - */ - inline void MatrixAddInit( - const Matrix& x, const Matrix& y, Matrix *z) { - DEBUG_ASSERT(x.n_rows() == y.n_rows()); - DEBUG_ASSERT(x.n_cols() == y.n_cols()); - z->Init(x.n_rows(), x.n_cols()); - VectorAddOverwrite(x.n_elements(), x.ptr(), y.ptr(), z->ptr()); - } - - //--- - - /** - * Subtracts two vectors into a third (z <- x - y). - */ - inline void VectorSubOverwrite(index_t length, - const double *x, const double *y, double *z) { - for (index_t i = 0; i < length; i++) { - z[i] = x[i] - y[i]; - } - } - - /** - * Subtracts two vectors into a third (z <- x - y). - * @param z will be Initialized to the sum of x and y - */ - inline void VectorSubInit( - const Vector& x, const Vector& y, Vector *z) { - DEBUG_ASSERT(x.length() == y.length()); - z->Init(x.length()); - VectorSubOverwrite(x.length(), x.ptr(), y.ptr(), z->ptr()); - } - - //--- - - /** - * Subtracts a vector from an existing vector (y <- y - x). - */ - inline void VectorSubFrom(index_t length, const double *x, - double *y) { - for (index_t i = 0; i < length; i++) { - y[i] -= x[i]; - } - } - - /** - * Subtracts a vector from an existing vector (y <- y - x). - */ - inline void VectorSubFrom( - const Vector& x, Vector *y) { - DEBUG_ASSERT(x.length() == y->length()); - VectorSubFrom(x.length(), x.ptr(), y->ptr()); - } - - //--- - - /** - * Multiplies a vector in-place by a scale (x <- alpha * x). - */ - inline void VectorScale(index_t length, double alpha, double *x) { - do { - *x++ *= alpha; - } while (--length); - } - - /** - * Multiplies a vector in-place by a scale (x <- alpha * x). - */ - inline void VectorScale(double alpha, Vector *x) { - VectorScale(x->length(), alpha, x->ptr()); - } - - /** - * Multiplies a matrix in-place by a scale (X <- alpha * X). - */ - inline void MatrixScale(double alpha, Matrix *x) { - VectorScale(x->n_elements(), alpha, x->ptr()); - } -#endif }; #endif diff --git a/fastlib/la/matrix.h b/fastlib/la/matrix.h index b0d93955e0..d758db7aab 100644 --- a/fastlib/la/matrix.h +++ b/fastlib/la/matrix.h @@ -16,7 +16,7 @@ #include #include -#include +#include /** * Double-precision vector for use with LAPACK. @@ -236,7 +236,7 @@ class Vector { } /** - * Copies the values from another matrix to this matrix. + * Copies the values from another vector to this vector. * * @param other the vector to copy from */ @@ -245,32 +245,75 @@ class Vector { mem::Copy(ptr_, other.ptr_, length_); } + /** + * Prints to a stream as a debug message. + * @param name a name that will be printed with the vector + */ + void PrintDebug(const char *name = "", FILE *stream = stderr) const { + fprintf(stream, "----- VECTOR %s ------\n", name); + for (index_t i = 0; i < length(); i++) { + fprintf(stream, "%+3.3f ", get(i)); + } + fprintf(stream, "\n"); + } + public: + /** The number of elements in this vector. */ index_t length() const { return length_; } + /** + * A pointer to the C-style array containing the elements of this vector. + */ double *ptr() { return ptr_; } + /** + * A pointer to the C-style array containing the elements of this vector. + */ const double *ptr() const { return ptr_; } + /** + * Gets the i'th element of this vector. + */ double operator [] (index_t i) const { DEBUG_BOUNDS(i, length_); return ptr_[i]; } + /** + * Gets a mutable reference to the i'th element of this vector. + */ double &operator [] (index_t i) { DEBUG_BOUNDS(i, length_); return ptr_[i]; } + /** + * Gets a value to the i'th element of this vector (convenient when + * you have a pointer to a vector). + * + * This is identical to the array subscript operator, except for the + * following reason: + * + * @code + * void FooBar(Vector *v) { + * v->get(0) // much easier to read than (*v)[0] + * } + * @endcode + */ + double get(index_t i) const { + DEBUG_BOUNDS(i, length_); + return ptr_[i]; + } + private: void AssertUninitialized_() const { - DEBUG_ASSERT(length_ == BIG_BAD_NUMBER); + DEBUG_ASSERT_MSG(length_ == BIG_BAD_NUMBER, "Cannot re-init vectors."); } void Uninitialize_() { @@ -380,6 +423,19 @@ class Matrix { // zero SetAll(0); } + + /** + * Makes this a diagonal matrix whose diagonals are the values in v. + */ + void SetDiagonal(const Vector& v) { + DEBUG_ASSERT(n_rows() == v.length()); + DEBUG_ASSERT(n_cols() == v.length()); + SetZero(); + index_t n = v.length(); + for (index_t i = 0; i < n; i++) { + set(i, i, v[i]); + } + } /** * Makes this uninitialized matrix a copy of the other vector. @@ -434,6 +490,24 @@ class Matrix { should_free_ = false; } + /** + * Makes this a 1 row by N column alias of a vector of length N. + * + * @param row_vector the vector to alias + */ + void AliasRowVector(const Vector& row_vector) { + Alias(const_cast(row_vector.ptr()), 1, row_vector.length()); + } + + /** + * Makes this an N row by 1 column alias of a vector of length N. + * + * @param col_vector the vector to alias + */ + void AliasColVector(const Vector& col_vector) { + Alias(const_cast(col_vector.ptr()), col_vector.length(), 1); + } + /** * Makes this a weak copy or alias of the other. * @@ -571,12 +645,14 @@ class Matrix { } /** - * Reduces the number of columns, but REQUIRES that there are no aliases + * Changes the number of columns, but REQUIRES that there are no aliases * to this matrix anywhere else. * + * If the size is increased, the remaining space is not initialized. + * * @param new_n_cols the new number of columns */ - void OwnerReduceColumns(index_t new_n_cols) { + void ResizeNoalias(index_t new_n_cols) { DEBUG_ASSERT(should_free_); // the best assert we can do n_cols_ = new_n_cols; ptr_ = mem::Resize(ptr_, n_elements()); @@ -596,6 +672,31 @@ class Matrix { mem::Swap(ptr_, other->ptr_, n_elements()); } + /** + * Copies the values from another matrix to this matrix. + * + * @param other the vector to copy from + */ + void CopyValues(const Matrix& other) { + DEBUG_ASSERT(n_rows() == other.n_rows()); + DEBUG_ASSERT(n_cols() == other.n_cols()); + mem::Copy(ptr_, other.ptr_, n_elements()); + } + + /** + * Prints to a stream as a debug message. + * @param name a name that will be printed with the matrix + */ + void PrintDebug(const char *name = "", FILE *stream = stderr) const { + fprintf(stream, "----- MATRIX %s ------\n", name); + for (index_t r = 0; r < n_rows(); r++) { + for (index_t c = 0; c < n_cols(); c++) { + fprintf(stream, "%+3.3f ", get(r, c)); + } + fprintf(stream, "\n"); + } + } + public: /** * Returns a pointer to the very beginning of the matrix, stored @@ -680,7 +781,7 @@ class Matrix { private: void AssertUninitialized_() const { - DEBUG_ASSERT(n_rows_ == BIG_BAD_NUMBER); + DEBUG_ASSERT_MSG(n_rows_ == BIG_BAD_NUMBER, "Cannot re-init matrices."); } void Uninitialize_() { diff --git a/fastlib/math/discrete.h b/fastlib/math/discrete.h index 2959d7860b..2e25dea5a8 100644 --- a/fastlib/math/discrete.h +++ b/fastlib/math/discrete.h @@ -12,7 +12,7 @@ #include "base/cc.h" #include "col/arraylist.h" -#include +#include namespace math { /** diff --git a/fastlib/math/geometry.cc b/fastlib/math/geometry.cc index d0c7e63b7d..0692fba052 100644 --- a/fastlib/math/geometry.cc +++ b/fastlib/math/geometry.cc @@ -8,6 +8,7 @@ #include "geometry.h" #include "discrete.h" +#include "math.h" namespace math { diff --git a/fastlib/math/geometry.h b/fastlib/math/geometry.h index cd6c9b9752..9f8382fb79 100644 --- a/fastlib/math/geometry.h +++ b/fastlib/math/geometry.h @@ -11,11 +11,9 @@ #include "base/cc.h" -#include +#include namespace math { - const double PI = 3.141592653589793238462643383279; - /** * Computes the hyper-volume of a hyper-sphere of dimension d. * diff --git a/fastlib/math/kernel.h b/fastlib/math/kernel.h index 19339261cb..bc7459e4d2 100644 --- a/fastlib/math/kernel.h +++ b/fastlib/math/kernel.h @@ -11,8 +11,9 @@ #include "base/common.h" #include "math/geometry.h" +#include "math/math.h" -#include +#include /* More to come soon - Gaussian, Epanechnakov, etc. */ diff --git a/fastlib/math/math.h b/fastlib/math/math.h index 5d6ed34aab..db2e955890 100644 --- a/fastlib/math/math.h +++ b/fastlib/math/math.h @@ -9,14 +9,30 @@ #ifndef MATH_MATH_H #define MATH_MATH_H -#include "discrete.h" -#include "kernel.h" -#include "geometry.h" - /** - * Namespace with a variety of math routines. + * Math routines. + * + * The hope is that this should contain most of the useful math routines + * you can think of. Currently, this is very sparse. */ namespace math { + /** The square root of 2. */ + const double SQRT2 = 1.41421356237309504880; + /** Base of the natural logarithm. */ + const double E = 2.7182818284590452354; + /** Log base 2 of E. */ + const double LOG2_E = 1.4426950408889634074; + /** Log base 10 of E. */ + const double LOG10_E = 0.43429448190325182765; + /** Natural log of 2. */ + const double LN_2 = 0.69314718055994530942; + /** Natural log of 10. */ + const double LN_10 = 2.30258509299404568402; + /** The ratio of the circumference of a circle to its diameter. */ + const double PI = 3.141592653589793238462643383279; + /** The ratio of the radius of a circle to its diameter. */ + const double PI_2 = 1.57079632679489661923; + /** Squares a number. */ template inline T Sqr(T v) { @@ -42,4 +58,8 @@ namespace math { } }; +#include "discrete.h" +#include "kernel.h" +#include "geometry.h" + #endif diff --git a/fastlib/math/math_test.cc b/fastlib/math/math_test.cc index b35b6e516a..80df8cc11f 100644 --- a/fastlib/math/math_test.cc +++ b/fastlib/math/math_test.cc @@ -5,16 +5,16 @@ TEST_SUITE_BEGIN(math) void TestPermutation() { - ArrayList p1; - ArrayList p2; + ArrayList p_rand; + ArrayList p_idnt; ArrayList visited; int n = 3111; - math::MakeRandomPermutation(n, &p1); - math::MakeIdentityPermutation(n, &p2); + math::MakeRandomPermutation(n, &p_rand); + math::MakeIdentityPermutation(n, &p_idnt); for (int i = 0; i < n; i++) { - TEST_ASSERT(p2[i] == i); + TEST_ASSERT(p_idnt[i] == i); } visited.Init(n); @@ -24,7 +24,7 @@ void TestPermutation() { } for (int i = 0; i < n; i++) { - visited[p1[i]]++; + visited[p_rand[i]]++; } for (int i = 0; i < n; i++) { diff --git a/fastlib/py/buildsys.py b/fastlib/py/buildsys.py deleted file mode 100644 index 01a50fa34c..0000000000 --- a/fastlib/py/buildsys.py +++ /dev/null @@ -1,444 +0,0 @@ - -import dep -import util -import glob - -import os - -class Types: - HEADER="buildsys/.h" - GCC_SOURCE="buildsys/gcc" # .c, .cpp, etc - OBJECT="buildsys/.o" - LINKABLE="buildsys/.a" - BINFILE="buildsys/bin" - SCRIPT="buildsys/script" - DIR="buildsys/dir" - PLACEHOLDER="buildsys/placeholder" - MISC="buildsys/misc" - ANY="buildsys/*" - -class CompilerInfo: - pass - -class GCCCompiler(CompilerInfo): - def __init__(self): - self.name = "gcc" - self.mode_dictionary = { - "verbose": "-g -DDEBUG -DVERBOSE", - "debug": "-g3 -DDEBUG", - "check": "-O2 -g -DDEBUG", - "fast": "-O2 -g -fomit-frame-pointer -DNDEBUG", - "unsafe": "-O3 -ffast-math -g -fomit-frame-pointer -DNDEBUG", - "profile" : "-O2 -pg -finline-limit=8 -DPROFILE -DNDEBUG", - "small": "-Os -DNDEBUG" - } - self.command_from_ext = { - "c" : "gcc %s -c %s -o %s -Wall", - "cc" : "g++ %s -c %s -o %s -Wall -Woverloaded-virtual -fno-exceptions -Wparentheses -fno-exceptions", - "f" : "g77 %s -c %s -o %s -Wall -Wno-uninitialized" - } - self.linker = "g++" - self.lflags_start = "-lg2c -lm -lpthread -Wl,-whole-archive" - self.lflags_end = "-Wl,-no-whole-archive" - -class ICCCompiler(CompilerInfo): - def __init__(self): - self.name = "icc" - self.mode_dictionary = { - "verbose": "-g -DDEBUG -DVERBOSE", - "debug": "-g -DDEBUG", - "check": "-O2 -g -DDEBUG", - "fast": "-O3 -g -fomit-frame-pointer -DNDEBUG", - "unsafe": "-O3 -ffast-math -g -fomit-frame-pointer -DNDEBUG", - "profile" : "-O2 -pg -finline-limit=8 -DPROFILE -DNDEBUG", - "small": "-Os" - } - self.command_from_ext = { - "c" : "icc %s -c %s -o %s", - "cc" : "icpc %s -c %s -o %s -fno-exceptions", - "f" : "ifort %s -c %s -o %s -Wall" - } - self.linker = "icpc" - self.lflags_start = "-lm -lpthread -Wl,-whole-archive" - self.lflags_end = "-Wl,-no-whole-archive" - -class MPICompiler(CompilerInfo): - def __init__(self): - self.name = "mpi" - self.mode_dictionary = { - "verbose": "-g -DDEBUG -DVERBOSE", - "debug": "-g -DDEBUG", - "check": "-O2 -g -DDEBUG", - "fast": "-O3 -g -fomit-frame-pointer -DNDEBUG", - "unsafe": "-O3 -ffast-math -g -fomit-frame-pointer -DNDEBUG", - "profile" : "-O2 -pg -finline-limit=8 -DPROFILE -DNDEBUG", - "small": "-Os" - } - self.command_from_ext = { - "c" : "mpicc %s -c %s -o %s", - "cc" : "mpiCC -fno-exceptions %s -c %s -o %s", - "f" : "mpif77 %s -c %s -o %s -Wall" - } - self.linker = "mpiCC" - self.lflags_start = "-lm -lpthread -Wl,-whole-archive" - self.lflags_end = "-Wl,-no-whole-archive" - #self.lflags_start = "-lm -lpthread " - #self.lflags_end = "" - -compiler_choices = [GCCCompiler(), MPICompiler(), ICCCompiler()] -compilers = dict([(c.name, c) for c in compiler_choices]) - -class MakeBuildSys(dep.DepSys): - """Build system. - """ - def __init__(self, source_dir, bin_dir): - dep.DepSys.__init__(self) - self.source_dir = source_dir - self.bin_dir = bin_dir - self.entries = [] - def begin(self, state): - return MakeBuildSysEntry(self, state) - def add_entry(self, entry): - self.entries.append(entry) - def to_makefile(self): - def shorten(fname): - if "/bin/" in fname: - return fname[fname.rindex("/bin/"):] - else: - return fname - lines = [] - self.entries.reverse() - i = 0 - for (outfiles, infiles, commands) in self.entries: - i += 1 - if commands: - lines.append("%s: %s" % (" ".join(outfiles), " ".join(infiles))) - outfiles_short = [shorten(outfile) for outfile in outfiles] - lines.extend(["\t@echo '... Making %s'" % (" ".join(outfiles_short))]) - lines.extend(["\t@" + c for c in commands]) - elif infiles: - lines.append("pseudo_%d: %s" % (i, " ".join(infiles))) - lines.extend(["\t@echo '*** Done with %s'" % (" ".join(infiles))]) - self.entries.reverse() - lines.append("clean:") - lines.append("\trm -rf %s" % (self.bin_dir)) - return lines - -class MakeBuildSysEntry(dep.DepSysEntry): - def __init__(self, sys, state): - dep.DepSysEntry.__init__(self, sys, state) - self.commands = [] - def _make_name(self, simplename, parameterization): - # NOTE: I'm completely ignoring the file list (self.state.files) - dirname = self.bin_dir(*parameterization) - return os.path.join(dirname, simplename) - def bin_dir(self, *pmz): - def name_part(param_name): - if param_name in pmz: - return self.state.params[param_name] - else: - return "COMMON" - - allowed_sequence = ["arch", "kernel", "mode", "compiler"] - (arch, kernel, mode, compiler) = [name_part(x) for x in allowed_sequence] - - for param in pmz: - if not param in allowed_sequence: - raise Exception("I can't deal with extra parameterization yet: '%s' not in '%s'" % (param, allowed_sequence)) - - return os.path.join(self.sys.bin_dir, "%s_%s_%s_%s" % (arch, kernel, mode, compiler)) - def source_file(self, real_path, fake_path): - return dep.DestFile(real_path, fake_path) - def makefile(self): - return self.source_file(os.path.abspath("./Makefile"), "Makefile") - def ensure_writable(self, name): - self.command("mkdir -p %s" % (os.path.dirname(name))) - def command(self, str): - self.commands.append(str) - def end(self, files): - infiles = {} - for file in dep.filemap_to_files(self.state.files): - infiles[file.name] = None - outfiles = [file.name for (classname, file) in files] - self.sys.add_entry((outfiles, list(infiles.keys()), self.commands)) - -class SourceRule(dep.Rule): - def __init__(self, type, real_path, fake_path): - dep.Rule.__init__(self) - self.real_path = real_path - self.fake_path = fake_path - self.type = type - def doit(self, sysentry, state, files, params): - return [(self.type, sysentry.source_file(self.real_path, self.fake_path))] - -class CompileRule(dep.Rule): - def __init__(self, source, headers, cflags): - dep.Rule.__init__(self, source=[source], headers=headers) - self.cflags = cflags - def doit(self, sysentry, state, files, params): - compiler = compilers[params["compiler"]] - source = files["source"].single(Types.GCC_SOURCE) - dot = source.simplename.rindex(".") - sourceextension = source.simplename[dot+1:] - simplename = source.simplename[:dot] + ".o" - object = sysentry.file("obj/" + simplename.replace("/", "_"), "arch", "kernel", "mode", "compiler") - # TODO: -I flags - my_includes = "-I%s -I%s" % (sysentry.bin_dir("arch", "kernel", "compiler"), - sysentry.sys.source_dir) - mode = params["mode"] - my_flags = my_includes + " " + compiler.mode_dictionary[params["mode"]] + " " + self.cflags - if not sourceextension in compiler.command_from_ext: - raise Exception("Don't know how to compile files of type [%s]." % sourceextension) - command_template = compiler.command_from_ext[sourceextension] - (source_dirname, source_basename) = os.path.split(source.name) - compile_cmd = command_template % (my_flags, source_basename, object.name) - sysentry.command("cd " + source_dirname + " && " + compile_cmd) - return [(Types.OBJECT, object)] - -class ArchiveRule(dep.Rule): - def __init__(self, name, objects): - dep.Rule.__init__(self, objects=objects) - self.name = name - def doit(self, sysentry, state, files, params): - objects = files["objects"].many(Types.OBJECT) - libfile = sysentry.file("lib" + self.name + ".a", "arch", "kernel", "mode", "compiler") - sysentry.command("ar r %s %s" % (libfile, " ".join([x.name for x in objects]))) - return [(Types.LINKABLE, libfile)] - #return [(Types.OBJECT, object) for object in objects] - -class HeaderSummaryRule(dep.Rule): - def __init__(self, name, objects): - dep.Rule.__init__(self, objects=objects) - self.name = name - def doit(self, sysentry, state, files, params): - libfile = sysentry.file("lib" + self.name + ".h") - sysentry.command("touch %s" % (libfile)) - return [(Types.PLACEHOLDER, libfile)] - -class LibRule(dep.Rule): - """LibRule returns all relevant archive files for this library and libraries - it depends on. - - In addition, a LibRule exposes header files. - """ - # TODO: "sources" will also natively support C++ files - def __init__(self, name, sources, headers, deplibs, cflags = ""): - source_rules = sources - self.header_rules = headers - # TODO: Be careful about depending on extra stuff? - for dep_lib in deplibs: - self.header_rules.append(dep_lib.header_summary_rule) - self.header_summary_rule = HeaderSummaryRule(name, self.header_rules) - self.compile_rules = [CompileRule(source_rule, self.header_rules, cflags) - for source_rule in source_rules] - self.archive_rule = ArchiveRule(name, self.compile_rules) - dep.Rule.__init__(self, archive=[self.archive_rule], deplibs=deplibs) - def doit(self, sysentry, state, files, params): - return files["deplibs"].to_pairs() + files["archive"].to_pairs() - -class BinRule(dep.Rule): - def __init__(self, name, linkables): - self.name = name - dep.Rule.__init__(self, linkables=linkables) - def doit(self, sysentry, state, files, params): - compiler = compilers[params["compiler"]] - binfile = sysentry.file(self.name, "arch", "kernel", "mode", "compiler") - # TO-DO: Link flags necessary? - lflags_start = compiler.lflags_start - lflags_end = compiler.lflags_end - cflags = compiler.mode_dictionary[params["mode"]] - sysentry.command(compiler.linker + " -o %s %s %s %s %s" % (binfile.name, cflags, - lflags_start, " ".join(files["linkables"].to_names()), lflags_end)) - return [(Types.BINFILE, binfile)] - -class MakefileRule(dep.Rule): - """ - Plug created so that symlinks are re-created whenever the Makefile changes. - """ - def __init__(self): - dep.Rule.__init__(self) - def doit(self, sysentry, state, files, params): - file = sysentry.makefile() - return [(Types.MISC, file)] - -class SymlinkRule(dep.Rule): - def __init__(self, filerules, dest_dir): - dep.Rule.__init__(self, filerules=filerules, makefile=[MakefileRule()]) - self.dest_dir = dest_dir - def doit(self, sysentry, state, files, params): - all = [] - pairs = files["filerules"].to_pairs() - for (classname, file) in pairs: - sourcename = file.name - destname = os.path.join(self.dest_dir, os.path.basename(file.simplename)) - # the symlink's simplename will be the same as the original's - destfile = sysentry.source_file(destname, file.simplename) - sysentry.command("rm -f %s" % destname) - sysentry.command("ln -s %s %s" % (sourcename, destname)) - all.append((classname, destfile)) - sysentry.command("echo '*** Created %d symlinks in %s.'" % (len(pairs), self.dest_dir)) - return all - -# Parameter loader - -BUILD_FILE = "build.py" - -class Loader: - """Responsible for loading build files and helping them stitch together. - - There are concepts of real and fake paths: - - - Real path: Where the file is located exactly on the file system. - - - Fake path: Where results generated by this file are relative to the - build system. If the real path is within the build directories, then - the fake path is just the relative path from the root of the build - system. However, we also support building small code trees that are - not in the build system. In this, the "fake" path starts with "outside" - to indicate it is outside the build system, and has the full path. - - Example: - - Say my build path is - - ~/fastlib/c (this is real_rootpath) - - Someone wants to build ~/fastlib/c/foo/bar, the fake path is "foo/bar". - - But if someone wants to build ~/foo/bar, the fake path is - "outside/home/yourname/foo/bar". - - - See also ../script/fl-build. - """ - - def __init__(self, real_rootpath): - self.real_root = real_rootpath.rstrip(os.sep) - self.thebuildsys = MakeBuildSys( - self.real_root, os.path.join(self.real_root, "bin")) - self.loader_map = {} - - def pathjoin(self, left, right, defaultprefix, lsep = "/", rsep = "/"): - def helper(left, right): - if right and right[0] == ".": - right_index = (right + "/").index(rsep) - right_first = right[:right_index] - right_rest = right[right_index+1:] - if right_first == ".": - return helper(left, right_rest) - else: - try: - left_index = left.rindex(lsep) - except: - left_index = 0 - return helper(left[:left_index], right_rest) - else: - return left + lsep + right.replace(rsep, lsep) - left = left.rstrip(lsep) - right = right.rstrip(rsep) - if not right: - return left - elif right[0] != ".": - if right[0] == rsep: - return right.replace(rsep, lsep) - else: - return defaultprefix + right.replace(rsep, lsep) - else: - return helper(left, right) - - def pathjoin_real(self, left, right): - return self.pathjoin(left, right, self.real_root + "/", os.sep, "/") - - def pathjoin_fake(self, left, right): - return self.pathjoin(left, right, "", "/", "/") - - def find_rule(self, rule_path, cur_real_path, cur_fake_path): - #print "trying: %s" % fullname - (path_rel, rule_name) = rule_path.split(":") - - fake_path = self.pathjoin_fake(cur_fake_path, path_rel) - real_path = self.pathjoin_real(cur_real_path, path_rel) - - if not fake_path in self.loader_map: - self.load(real_path, fake_path) - if not rule_name in self.loader_map[fake_path]: - raise Exception("Rule '%s' not found in '%s'." % (rule_name, real_path)) - #print "Found [%s][%s] aka [%s]" % (fullname, defaultpath, path) - return self.loader_map[fake_path][rule_name] - - def load(self, real_path, fake_path): - """Loads build rules, by exposing a 'register' function.""" - - # !!!!! LOOK AT ME! All the functions in build rules are defined here! - - self.loader_map[fake_path] = {} - selfname = fake_path.split("/")[-1] - def register(name, rule): - if name in self.loader_map[fake_path]: - raise Exception("Duplicate rule %s:%s" % (fake_path, name)) - self.loader_map[fake_path][name] = rule - return rule - def find(rule_name): - return self.find_rule(rule_name, real_path, fake_path) - def pathify_real(name): - return self.pathjoin_real(real_path, name) - def pathify_fake(name): - return self.pathjoin_fake(fake_path, name) - def sourcerule(type, name): - if isinstance(name, str): - # If there is a colon, it is a rule; otherwise, it is just a file. - if ":" in name: - return find(name) - else: - return SourceRule(type, pathify_real("./" + name), pathify_fake("./" + name)) - else: - return name - def sourcerules(type, names): - return [sourcerule(type, name) for name in names] - def lglob(mask, *exclude): - (relpath, namemask) = os.path.split(mask) - full_path = os.path.join(real_path, mask) - basenames = [os.path.basename(f) - for f in glob.glob(os.path.join(real_path, mask))] - return [os.path.join(relpath, basename) - for basename in basenames if not basename in exclude] - def librule(name = selfname, - sources = [], headers = [], deplibs = [], cflags = ""): - return register(name, LibRule(pathify_fake(name), - sourcerules(Types.GCC_SOURCE, sources), - sourcerules(Types.HEADER , headers), - sourcerules(Types.LINKABLE, deplibs), - cflags)) - def unittest(name = selfname + "_unittest", - lib = ":" + selfname, - sources = []): - """Unit tests for the purpose of testing one library. - - All .c or .cc files are compiled and run as unit tests. - - (TODO: Unit test framework) - """ - assert sources - def inttest(name = selfname + "_inttest", - libs = [], - sources = []): - """Integration tests for testing multiple libraries. - """ - assert libs - assert sources - def binrule(name, linkables = [], sources = [], headers = [], cflags = ""): - # (source, headers, cflags) - if sources: - lib = librule(name = name + "__auto", - sources = sources, headers = headers, deplibs = linkables, - cflags = cflags) - linkables = linkables + [lib] - register(name, BinRule(name, sourcerules(Types.LINKABLE, linkables))) - build_file_path = os.path.join(real_path, BUILD_FILE) - print "... Reading %s" % (build_file_path) - text = util.readfile(build_file_path) - exec text in {"register" : register, "Types" : Types, - "find" : find, "dep" : dep, "lglob" : lglob, - "sourcerule" : sourcerule, "sourcerules" : sourcerules, - "librule" : librule, "binrule" : binrule} - diff --git a/fastlib/py/datastore.py b/fastlib/py/datastore.py deleted file mode 100644 index 2b85ed89fa..0000000000 --- a/fastlib/py/datastore.py +++ /dev/null @@ -1,734 +0,0 @@ -import util - -import unittest -import os -import StringIO -import operator -import copy -import sys - -class SexpressionStream: - """Stream for tokenizing an s-expressions.""" - def __init__(self, file): - self.file = file - self.nextchar() - def nextchar(self): - self.peek = self.file.read(1) - return self.peek - def skip(self): - while True: - if self.peek == ';': - while self.peek != "" and self.peek != "\n" and self.peek != "\r": - self.nextchar() - elif self.peek.isspace(): - self.nextchar() - else: - break - def readstr(self): - self.skip() - name = "" - while self.peek.isalnum() or self.peek == "_": - name += self.peek - self.nextchar() - return util.unescape_sexpression(name) - def is_done(self): - return self.peek == "" - -class UnknownField: - def __init__(self, message): - self.message = message - def __repr__(self): - return message - -class DataNode: - """Abstract hierarchial key-value data stucture. - - This is intended more for small data sets, like the output - of a single C program, where the overhead of any more complex - outut mechanism would be a ridiculous annoyance. - - Thinking of the Windows registry should convince you not to use this - for large files. - """ - def __init__(self, val = None): - """Create an empty data store. - """ - self.subnodes = {} - self.val = val - - def get_node(self, path_elements): - """Gets an node via a particular path sequence. - """ - if len(path_elements) == 0: - return self - else: - return self.subnodes[path_elements[0]].get_node(path_elements[1:]) - - def get_node_create(self, path_elements): - """Gets an node via a particular path sequence. - """ - if len(path_elements) != 0: - return self.get_subnode_create(path_elements[0]) \ - .get_node_create(path_elements[1:]) - else: - return self - - def get_subnode_create(self, first): - """Gets a sub-node, creating one if it doesn't exist. - """ - if not first in self.subnodes.keys(): - node = DataNode() - self.subnodes[first] = node - else: - node = self.subnodes[first] - return node - - def get_val(self, path_elements): - """Get a string leaf node by its path sequence. - - Returns None if no such element was found. - """ - try: - return self.get_node(path_elements).val - except: - return None - - def get_val_path(self, path): - """Get a string leaf node by its escaped-and-slash-separated path. - """ - try: - return self.get_node(util.split_path(path)).val - except: - return None - - def set_val(self, path_elements, val): - """Sets a leaf string by a particular sequence of path elements. - """ - self.get_node_create(path_elements).val = val - - def set_val_path(self, path, val): - """Sets a leaf string by a particular escaped-and-slash-separated - path. - """ - self.get_node_create(util.split_path(path)).val = val - - def __write_pathstyle_impl(self, file, prefix): - """Implementation method for writing in path-style output format. - """ - if self.val != None: - file.write(prefix + " " + util.escape_dspath(self.val) + "\n") - for k, v in self.subnodes.iteritems(): - v.__write_pathstyle_impl(file, prefix + "/" + util.escape_dspath(k)) - - def __write_sexpression_impl(self, file, prefix): - """Implementation method for writing in sexpression-style output format. - """ - if self.val != None: - file.write(util.escape_sexpression(self.val)) - for k, v in self.subnodes.iteritems(): - file.write("\n" + prefix + "(" + util.escape_sexpression(k) + " ") - v.__write_sexpression_impl(file, prefix + " ") - file.write(")") - - def write_pathstyle(self, file, prefix = ""): - """Writes the data structure to the output in path-style format, - where every line is a data node, where the left side of the line - is an escaped and slash-separated path and the right side is a - string. - """ - self.__write_pathstyle_impl(file, prefix) - - def write_sexpression(self, file, prefix = ""): - """Writes the data structure to the output in sexpression format. - """ - file.write("; Fx s-expression writer version 1.0") - self.__write_sexpression_impl(file, prefix) - file.write("\n") - - def read_pathstyle(self, file): - """Read a flat-file in key-value format. - """ - for line in file: - try: - parts = line.strip().split(" ") - self.set_val_path(parts[0], util.unescape_dspath(parts[1])) - except: - print "Warning: Line ignored: %s" % (line.strip()) - - def read_sexpression(self, file): - """Read s-expression from a file. - - This only supports a subset of s-expressions that correspond to nodes. - There are two examples that are unsupported: - - (foo bar baz) is invalid because 'bar' and 'baz' cannot both be associated - with the same node. - - (foo (bar a) (bar b)) also cannot work, because there cannot be two 'bar' - nodes associated with foo. - """ - self.__read_sexpression_impl(SexpressionStream(file)) - - def __read_sexpression_impl(self, stream): - try: - more_subexpressions = True - while more_subexpressions: - stream.skip() - if stream.peek.isalnum() or stream.peek == '_': - if self.val != None: - print "Warning: Value reassigned from [%s]." % self.val - self.val = stream.readstr() - elif stream.peek == '(': - # A sub exprsesion - stream.nextchar() - name = stream.readstr() - if name in self.subnodes.keys(): - print "Warning: Node [%s] is specified twice." % name - self.get_subnode_create(name).__read_sexpression_impl(stream) - stream.skip() - if stream.peek != ')': - raise util.ParseError - stream.nextchar() - elif stream.peek == ')': - more_subexpressions = False - # do not gobble - elif stream.peek == '': - more_subexpressions = False - else: - raise util.ParseError() - except: - raise util.ParseError() - - def select(self, pathspecs): - """Selects a natural join of several path specifications. - - See util.select() for more information. - """ - return select(self, self.SELECTFUNCTIONS, pathspecs) - - def select_to_csv_file(self, file, pathspecs, header = True): - """Selects a natural join of several path specifications, and write - to a CSV file. - - See util.select() for more information on how the path specifications - work. - """ - elements = self.select(pathspecs) - if header: - all = [pathspecs] + elements - else: - all = elements - util.write_csv_file(file, all) - - def select_tree(self, matchtree): - """Selects based on a match tree. - """ - return matchtree_walk(self, self.SELECTFUNCTIONS, matchtree) - - def __repr__(self): - """Converts into debugging s-exp representation. - """ - stream = StringIO.StringIO() - self.write_sexpression(stream) - return stream.getvalue() - - def __get_typedval(self): - """Converts into a typed value by guessing its type. - """ - try: - return int(self.val) - except: - try: - return float(self.val) - except: - return self.val - - typedval=property(__get_typedval) - - SELECTFUNCTIONS=(lambda x:x.val, - lambda x:x.subnodes.iterkeys(), - lambda x:x!=None, lambda x,y:x.subnodes.get(y, None), None) - -def sortify(str): - """Makes the string suitable for sorting by returning a tuple, where - the left element is typed and the right element is the string. - """ - val = str - try: - val = float(str) - except: - pass - return (val, str) - -def subcolumns(col_nums, rows): - """Extract certain columns. - - example: - - col_nums = [0, 2, 1] - rows = [ ["ted", "likes", "shrimp", "always"], - ["bill", "eats", "pasta", "sometimes"] ] - - returns: - - [ ["ted", "shrimp", "likes"], - ["bill", "pasta", "eats] ] - """ - return [ [ row[i] for i in col_nums ] for row in rows ] - -def group(groupcols, rows, remove_key = False): - """Groups a matrix by particular columns. - - Example: - - groupcols = [ 0, 1 ] - - rows = [ [ 1, 1, 1, 1 ], - [ 1, 1, 1, 2 ], - [ 1, 1, 2, 1 ], - [ 1, 1, 2, 2 ], - [ 2, 1, 1, 1 ], - [ 2, 1, 1, 2 ], - [ 1, 3, 2, 1 ], - [ 1, 3, 2, 2 ] ] - - returns: - - groups={(1, 1): [ [ 1, 1, 1, 1 ], - [ 1, 1, 1, 2 ], - [ 1, 1, 2, 1 ], - [ 1, 1, 2, 2 ] ], - (2, 1): [ [ 2, 1, 1, 1 ], - [ 2, 1, 1, 2 ] ], - (1, 3): [ [ 1, 3, 2, 1 ], - [ 1, 3, 2, 2 ] ] } - - Note this should work for dictionaries too; if rows is a collection of - dictionaries, and groupcols corresponds to dictionary keys. - """ - groups = {} - - for row in rows: - key = tuple([row[x] for x in groupcols]) - if not key in groups.keys(): - groups[key] = [] - if remove_key: - newrow = [row[i] for i in util.keys(row) if i not in groupcols] - else: - newrow = row - groups[key].append(newrow) - - return groups - -def groupreduce(groups, reductionpairs): - """Takes the result of group, and reduces every specified element. - - Example: - - groups= { (1, 1): [ [ 1, 1, 1, 1 ], - [ 1, 1, 1, 2 ], - [ 1, 1, 2, 1 ], - [ 1, 1, 2, 2 ] ], - (2, 1): [ [ 2, 1, 1, 1 ], - [ 2, 1, 1, 2 ] ], - (1, 3): [ [ 1, 3, 2, 1 ], - [ 1, 3, 2, 2 ] ] } - - reductionpairs = [ (2, operator.add), (3, operator.mul) ] - - Returns: - - reduced = { (1, 1) : [1, 1, 6, 4], - (2, 1) : [2, 1, 2, 2], - (1, 3) : [1, 3, 4, 2] } - - In the "reduced" array, if a column lacks a reduction operator, an - its value from an arbitrary row will be chosen. - """ - reduced = {} - - for (key, rows) in groups.items(): - newrow = copy.copy(rows[0]) # copy the row - for row in rows[1:]: - for (index, reduction) in reductionpairs: - newrow[index] = reduction(newrow[index], row[index]) - reduced[key] = newrow - - return reduced - -def groupsum(groups, indices = [0]): - """Sums all specified columns of a grouping. - """ - reductionpairs = [] - for index in indices: - reductionpairs.append((index, operator.sum)) - return groupreduce(groups, reductionpairs) - -def select(place, (handlefn, listfn, existfn, divefn, bridgefn), pathspecs): - """Yields the desired data on a natural join of pathspecs, sorted. - - The pathspecs are a slash-separated list of pathnames as you would expect - to be stored in a DataNode tree. Path elements may contain a "*" in - which case the sub-elements are listed and each traversed. - - If two different pathspecs contain a "*", the rules are interesting. - If both pathspecs are rooted at the same place up to the "*", the two - pathspecs are traversed together. Otherwise, a cartesian product is - done between the two. Example: - - Say our structure looks like: - - /runs/1/x = 1x - /runs/1/y = 1y - /runs/2/x = 1x - /runs/2/y = 1y - /bill/a = a - /bill/b = b - /constant = q - - ['/q', '/runs/*/x'] repeats the uninteresting one: - ['q','1x], ['q','2x'] - - ['/runs/*/x', '/runs/*/y'] yields related pairs: - ['1x','1y'] , ['2x','2y'] - - ['/runs/*/x', '/bill/*'] yields all combinations: - ['1x','a'], ['1x','b'], ['1y','a'], ['1y','b'] - - ['/runs/*/x', '/runs/*/y', '/bill/*'] yields: - ['1x','1y','a'], ['1x','1y','b'], ['2x','2y','a'], ['2x','2y','b'] - - The resulting data is then sorted first by the first column, then the - second, and so on. Values that look like numbers are sorted in numerical - order. - - The long tuple of functions are the information necessary for iteration - (you likely won't have to specify these yourself): - - - place: the starting 'place', such as a directory name - - listfn: a path to list sub-'place's of a place, such as os.listdir - - existfn: to test the existence of a sub-field like os.path.exists - - divefn: to take a sub-path and add on a new element, such as os.path.join - - bridgefn: more complicated; when a ':' appears at the end of an element, - the divefn is called to get the new 'place' to go to and a new tuple of - relevant functions. for example, in select_fields, the ':' operator - parses the specified file as a DataNode pathstyle file, and plunges into - the DataNode hierarchy - """ - matchtree = DataNode() - for element in pathspecs: - matchtree.set_val_path(element, element) - results = matchtree_walk(place, - (handlefn, listfn, existfn, divefn, bridgefn), matchtree) - def tolist(map): - return [ sortify(map.get(x, "")) for x in pathspecs ] - lines = [ tolist(x) for x in results ] - lines.sort() - def fix(line): - return [ x[1] for x in line ] - return [ fix(line) for line in lines ] - -def matchtree_walk(place, (handlefn, listfn, existfn, divefn, bridgefn), matchtree): - """Implements the natural join over an arbitrary hierarchial data backend. - - See util.select() for information on the behavior. - """ - functions = (handlefn, listfn, existfn, divefn, bridgefn) - def combine(subanswers): - newanswers = [] - for answer1 in answers: - for answer2 in subanswers: - elem = {} - elem.update(answer1) - elem.update(answer2) - newanswers.append(elem) - return newanswers - myanswer = {} - if matchtree.val != None: - myanswer[matchtree.val] = handlefn(place) - answers = [myanswer] - for (matchname, submatchtree) in matchtree.subnodes.iteritems(): - if matchname[-1] == ':': - processfn = bridgefn - matchname = matchname[:-1] - else: - processfn = lambda x:(x, functions) - def subwalkfn((subplace, subfunctions), submatchtree): - return matchtree_walk(subplace, subfunctions, submatchtree) - if matchname == util.STAR: - subanswers = [] - for subname in listfn(place): - subplace = divefn(place, subname) - returned = subwalkfn(processfn(subplace), submatchtree) - if returned != [{}]: # special case: directory that completely failed - subanswers += returned - answers = combine(subanswers) - else: - subplace = divefn(place, matchname) - if existfn(subplace): - answers = combine(subwalkfn(processfn(subplace), submatchtree)) - else: - pass # unknown directories are ignored - return answers - -def select_fields(rootpath, pathspecs): - """Selects first from files, and then from their datanode-pathstyle - contents. - - Pathspecs are in the format: - - runs/*/FILENAME:/info/params/x - runs/*/FILENAME:/info/timers/*/cycles - - Be careful: Both "FILENAME/:/" and "FILENAME/:info" will choke horribly. - Be very careful. - """ - treefunctions = DataNode.SELECTFUNCTIONS - def bridge(filename): - f = open(filename, "r") - try: - result = DataNode() - result.read_pathstyle(f) - return (result, treefunctions) - finally: - f.close() - pathfunctions = (lambda x:x, os.listdir, - os.path.exists, os.path.join, bridge) - return select(rootpath, pathfunctions, pathspecs) - -def select_fields_to_table_file(openfile, rootpath, pathspecs, writefn = util.write_csv_file): - """Selects fields first from files and then from the pathstyle contents, - writing results to an OPEN csv or other type of text file file. - - See util.select_fields(). - """ - headers = [x.strip("/").split("/")[-1] for x in pathspecs] - writefn(openfile, [headers] + select_fields(rootpath, pathspecs)) - -def select_fields_to_csv_file(csvfile, rootpath, pathspecs): - """Selects fields first from files and then from the pathstyle contents, - writing results to an OPEN csv file. - - See util.select_fields(). - """ - select_fields_to_table_file(csvfile, rootpath, pathspecs, util.write_csv_file) - -def select_fields_to_csv(csvfname, rootpath, pathspecs): - """Selects fields first from files and then from the pathstyle contents, - writing results to a to-be-created csv file name. - - See util.select_fields(). - """ - f = open(csvfname, "w") - try: - select_fields_to_csv_file(f, rootpath, pathspecs) - finally: - f.close() - -def select_files(place, pathspecs): - """Selects filenames in the natural-join style. - - See util.select() for info. - """ - functions = (lambda x:x, os.listdir, os.path.exists, os.path.join, None) - return select(place, functions, pathspecs) - -def match_files(place, pathspec): - """Match files based on a single path specification. - - For instance runs/*/SUMMARY matches files named SUMMARY in any - directory called runs. - """ - return [ x[0] for x in select_files(place, [pathspec]) ] - -#---------------------------------------------------------------------------- -# Unit tests - -class IoTest(unittest.TestCase): - def setUp(self): - (tmpdir, tmpfile) = os.path.split(os.tempnam()) - self.tmpdir = "%s/fxsys-util-ut-%s" % (tmpdir, tmpfile) - self.rootpath = self.tmpdir - os.makedirs(self.rootpath) - - def tearDown(self): - util.remove_dir_recursive(self.tmpdir) - - def test_match_files(self): - def fix(x): - return os.path.join(self.rootpath, x) - matchdirs = ["foo/a/bar/a/baz", "foo/a/bar/b/baz", "foo/a/bar/c/baz", - "foo/b/bar/a/baz", "foo/b/bar/c/baz", - "foo/c/bar/b/baz"] - nonmatchdirs = ["foon/a/bar/a/baz", "foo/g/bar/b/bazz", "foo/a/x", "subsume"] - for dir in matchdirs + nonmatchdirs: - os.makedirs(os.path.join(self.rootpath, dir)) - found = match_files(self.rootpath, "foo/*/bar/*/baz") - found.sort() - matchdirs.sort() - matchdirs = [ fix(x) for x in found ] - self.assertEqual(matchdirs, found) - - def test_select_files(self): - def fix(x): - return os.path.join(self.rootpath, x) - leftdirs = ["foo/a/bar/a/baz", "foo/a/bar/b/baz", "foo/a/bar/c/baz", - "foo/b/bar/a/baz", "foo/b/bar/c/baz", - "foo/c/bar/b/baz"] - middirs = ["foo/a/bar/a/bon", "foo/a/bar/b/bon", "foo/a/bar/c/bon", - "foo/b/bar/a/bon", "foo/b/bar/c/bon", - "foo/c/bar/b/bon"] - rightdirs = ["param/a/foo", "param/b/foo", "param/c/foo"] - nondirs = ["foon/a/bar/a/baz", "foo/g/bar/b/bazz", "foo/a/x", "subsume"] - for dir in leftdirs + middirs + rightdirs + nondirs: - os.makedirs(os.path.join(self.rootpath, dir)) - found = select_files(self.rootpath, - ["foo/*/bar/*/baz", "foo/*/bar/*/bon", "param/*/foo"]) - found.sort() - expect = [] - for (leftdir, middir) in zip(leftdirs, middirs): - for rightdir in rightdirs: - expect.append([fix(leftdir), fix(middir), fix(rightdir)]) - expect.sort() - found.sort() - self.assertEqual(expect, found) - - - -class DatastoreTest(unittest.TestCase): - def setUp(self): - self.f = DataNode() - self.f.get_node_create(["walks", "10", "param", "speed"]).val = "SLOW" - self.f.get_node_create(["runs", "10", "param"]).val = "lots of params" - self.f.get_node_create(["runs", "10", "param", "x"]).val = "2" - self.f.set_val(["runs", "10", "param", "y"], "3") - self.f.set_val_path("runs/20/param/x", "4") - self.f.set_val_path("/runs/20/param/y", "5") - self.f.set_val(["runs", "10", "metric", "total performance"], "very good") - self.f.set_val_path("/runs/30/param/x", "8"); - self.f.set_val_path("/runs/30/param/y", "9"); - self.f.set_val_path("/runs/40/param/x", "11"); - - def test_get_val(self): - self.assertEqual(self.f.get_val(["runs", "10", "param", "x"]), "2"); - self.assertEqual(self.f.get_val(["runs", "10", "param", "y"]), "3"); - self.assertEqual(self.f.get_val(["runs", "20", "param", "x"]), "4"); - self.assertEqual(self.f.get_val(["runs", "20", "param", "y"]), "5"); - - def test_get_val_path(self): - self.assertEqual(self.f.get_val_path("runs/20/param/y"), "5"); - self.assertEqual(self.f.get_val_path("/runs/20/param/y"), "5"); - self.assertEqual(self.f.get_val_path("/runs/10/metric/total%20performance"), "very good"); - - def help_test_save_get(self, readfun, writefun): - print "(START)" - first_outstream = StringIO.StringIO() - writefun(self.f, first_outstream) - print first_outstream.getvalue() - print "(REREAD)" - f2 = DataNode() - readfun(f2, StringIO.StringIO(first_outstream.getvalue())) - second_outstream = StringIO.StringIO() - writefun(f2, second_outstream) - print second_outstream.getvalue() - print "(END)" - self.assertEqual(first_outstream.getvalue(), second_outstream.getvalue()); - - def test_pathstyle_save(self): - self.help_test_save_get(DataNode.read_pathstyle, DataNode.write_pathstyle) - - def test_sexpression(self): - self.help_test_save_get(DataNode.read_sexpression, DataNode.write_sexpression) - - def test_select(self): - selected = self.f.select(["runs/*/param/x", "runs/*/param/y"]) - expected = [ - ['2', '3'], - ['4', '5'], - ['8', '9'], - ['11', '']] - selected.sort() - expected.sort() - self.assertEqual(expected, selected) - - def test_group(self): - groupcols = [ 0, 1 ] - rows = [ [ 1, 1, 1, 1 ], - [ 1, 1, 1, 2 ], - [ 1, 1, 2, 1 ], - [ 1, 1, 2, 2 ], - [ 2, 1, 1, 1 ], - [ 2, 1, 1, 2 ], - [ 1, 3, 2, 1 ], - [ 1, 3, 2, 2 ] ] - expect = { (1, 1): [ [ 1, 1, 1, 1 ], - [ 1, 1, 1, 2 ], - [ 1, 1, 2, 1 ], - [ 1, 1, 2, 2 ] ], - (2, 1): [ [ 2, 1, 1, 1 ], - [ 2, 1, 1, 2 ] ], - (1, 3): [ [ 1, 3, 2, 1 ], - [ 1, 3, 2, 2 ] ] } - self.assertEqual(expect, group(groupcols, rows)) - - def test_group_nonint(self): - k_a = "a" - k_b = (3.2, "food") - k_c = "hamburger" - groupcols = [ k_a, k_b ] - rows = [ { k_a:1, k_b:1, k_c:1, "d":1 }, - { k_a:1, k_b:1, k_c:1, "d":2 }, - { k_a:1, k_b:1, k_c:2, "d":1 }, - { k_a:1, k_b:1, k_c:2, "d":2 }, - { k_a:2, k_b:1, k_c:1, "d":1 }, - { k_a:2, k_b:1, k_c:1, "d":2 }, - { k_a:1, k_b:3, k_c:2, "d":1 }, - { k_a:1, k_b:3, k_c:2, "d":2 } ] - expect = { (1, 1): [ { k_a:1, k_b:1, k_c:1, "d":1 }, - { k_a:1, k_b:1, k_c:1, "d":2 }, - { k_a:1, k_b:1, k_c:2, "d":1 }, - { k_a:1, k_b:1, k_c:2, "d":2 } ], - (2, 1): [ { k_a:2, k_b:1, k_c:1, "d":1 }, - { k_a:2, k_b:1, k_c:1, "d":2 } ], - (1, 3): [ { k_a:1, k_b:3, k_c:2, "d":1 }, - { k_a:1, k_b:3, k_c:2, "d":2 } ] } - self.assertEqual(expect, group(groupcols, rows)) - - def test_subcolumns(self): - col_nums = [0, 2, 1] - rows = [ ["ted", "likes", "shrimp", "always"], - ["bill", "eats", "pasta", "sometimes"] ] - expect = [ ["ted", "shrimp", "likes"], - ["bill", "pasta", "eats"] ] - self.assertEqual(expect, subcolumns(col_nums, rows)) - - def test_groupreduce(self): - groups= { (1, 1): [ [ 1, 1, 1, 1 ], - [ 1, 1, 1, 2 ], - [ 1, 1, 2, 1 ], - [ 1, 1, 2, 2 ] ], - (2, 1): [ [ 2, 1, 1, 1 ], - [ 2, 1, 1, 2 ] ], - (1, 3): [ [ 1, 3, 2, 1 ], - [ 1, 3, 2, 2 ] ] } - reductionpairs = [ (2, operator.add), (3, operator.mul) ] - reduced = { (1, 1) : [1, 1, 6, 4], - (2, 1) : [2, 1, 2, 2], - (1, 3) : [1, 3, 4, 2] } - self.assertEqual(reduced, groupreduce(groups, reductionpairs)) - - def test_csv_file_head(self): - stream = StringIO.StringIO() - self.f.select_to_csv_file(stream, ["runs/*/param/x", "runs/*/param/y"]) - expected = "runs/*/param/x, runs/*/param/y\n2, 3\n8, 9\n4, 5\n11, \n"; - self.assertEqual(util.sorted(expected.split("\n")), util.sorted(stream.getvalue().split("\n"))) - - def test_csv_file_nohead(self): - stream = StringIO.StringIO() - self.f.select_to_csv_file(stream, ["runs/*/param/x", "runs/*/param/y"], False) - expected = "2, 3\n8, 9\n4, 5\n11, \n"; - self.assertEqual(util.sorted(expected.split("\n")), util.sorted(stream.getvalue().split("\n"))) - -if __name__ == "__main__": - unittest.main() - test_matching() diff --git a/fastlib/py/dep.py b/fastlib/py/dep.py deleted file mode 100644 index e9d8b046d6..0000000000 --- a/fastlib/py/dep.py +++ /dev/null @@ -1,207 +0,0 @@ -# Simple abstract dependency system suitable for generating things like -# makefiles. - -import util - -import os - -class DestFile: - def __init__(self, name, simplename, dep_files={}, parameterization={}): - # TODO: Separate "my" parameterization from "dep" parameterization - self.name = name - if simplename: - self.simplename = simplename - else: - self.simplename = name - self.parameterization = parameterization - self.dep_files = dep_files - def __repr__(self): - return self.name - def __hash__(self): - return hash(self.name) ^ hash(self.simplename) - def __cmp__(self, other): - if isinstance(other, DestFile): - return cmp(self.name, other.name) - else: - return NotImplemented - -class DepState: - def __init__(self, files, params): - self.files = files - self.params = params - self.parameterization = {} - for file in filemap_to_files(files): - self.parameterization.update(file.parameterization) - -class DepSys: - def __init__(self): - self.runs = {} - def begin(self, state): - raise Exception("DepSys not overridden") - -class DepSysEntry: - """Abstract dependency system entry. - - You must override this and provide a way of generating filenames. - """ - def __init__(self, sys, state): - """ - sys - a DepSys object - state - a DepState object - """ - self.sys = sys - self.state = state - def file(self, simplename, *extra_parameterization): - parameterization = dict(self.state.parameterization) - my_parameterization = dict([(k, self.state.params[k]) - for k in extra_parameterization]) - parameterization.update(my_parameterization) - realname = self._make_name(simplename, parameterization) - self.ensure_writable(realname) - return DestFile(realname, simplename, self.state.files, parameterization) - def _make_name(self, simplename, parameterization): - raise Exception("not implemented") - def ensure_writable(realname): - raise Exception("not implemented") - def end(self, files): - raise Exception("not implemented") - def command(self, str): - raise Exception("not implemented") - -class BadResult: - pass - -class FileCollection: - def __init__(self, pairs): - self.files_by_type = {} - unique_pairs = {} - for pair in pairs: - unique_pairs[pair] = None - for (typename, file) in unique_pairs: - self.files_by_type.setdefault(typename, []).append(file) - def single(self, typename): - result = self.files_by_type[typename] - if len(result) != 1: - raise BadResult() - return result[0] - def many(self, typename): - return self.files_by_type.get(typename, []) - def to_pairs(self): - pairs = [] - for (classname, files) in self.files_by_type.items(): - pairs += [(classname, file) for file in files] - return pairs - def to_files(self): - return util.collapse_once( - [files for (classname, files) in self.files_by_type.items()]) - def to_names(self): - return [f.name for f in self.to_files()] - -def filemap_to_files(filemap): - return util.collapse_once( - [file_collection.to_files() for file_collection in filemap.values()]) - -class Rule: - """ - Generic rule. - - This rule takes in several labelled sets of other rules. - - TODO: Explain the idea of "meta-rule" better - - This is more a meta-rule, which can depend on other meta-rules. - Meta-rules are asked for the exact filenames they produce, which - recursively ask their dependency rules what files they produce. Once a - meta-rule knows what files its dependency rules produce, it can generate - commands to produce its own result files, and tell ITS dependents what - files it generated. - - """ - - # TODO: (requires thought) If multiple rules depend on a single rule, I - # *think* this code, how it is written it will run "doit" for that rule - # multiple times, rather than once. This might be necessary due to - # parameterization, I haven't thought much about it. - - def __init__(self, **dep_map): - self.dep_map = dep_map - def generate(self, sys, params): - def fake_cont(x): - pass - self.run(sys, params, fake_cont) - def run(self, sys, params, continuation): - key = (self, util.dicthash(params)) - if key in sys.runs: - continuation(sys.runs[key]) - else: - all_deps = [] - - for (dep_class, deps) in self.dep_map.items(): - for dep in deps: - all_deps.append((dep_class, dep)) - - dep_count = [len(all_deps)] - dep_files = {} - - def check_done(): - if dep_count[0] == 0: - dep_files_converted = dict(self.dep_map) - for k in self.dep_map: - dep_files_converted[k] = FileCollection(dep_files.get(k, [])) - state = DepState(dep_files_converted, params) - # Also memoize here! - action = sys.begin(state) - my_files = self.doit(action, state, dep_files_converted, params) - action.end(my_files) - sys.runs[key] = my_files - continuation(my_files) - dep_count[0] = -1 - - def my_continuation_for(dep_class): - def impl(single_dep_files): - dep_files.setdefault(dep_class, []).extend(single_dep_files) - dep_count[0] -= 1 - check_done() - return impl - - for (dep_class, dep) in all_deps: - dep.run(sys, params, my_continuation_for(dep_class)) - - check_done() - def doit(self, sysentry, state, dep_files, params): - """ - The doit function is called for all subclasses of this. The doit - function doesn't necessarily perform the action, but is required to - figure out exactly what actions are required. For instance, for the - build system, this generates a Makefile rule, which contains the exact - commands and file names. - - The following are sent to an implementors' doit: - - dep_files: the files "generated" by all rules it depends on. The rules - are grouped into the label of the rule (given when you called - Rule.__init__) and then into the type of the file (like file - extension). The rules you are dependent on were the ones responsible - for assigning the file extension. Each file has a "simple" name and - an "absolute" name. The simple name should be used for generating - result files, and the absolute name is the location of the source - file. - - params: the parameterization of the instantiation; that is, each - file can be created with combinations of different parameters, and the - parameterization is used to identify different "builds" of the same - file. in the case of the build system, this is different operating - systems or compilation modes. certain rules care about certain - parameters, so when they generate destination files, they specify - which parameters they care about. - - the state (an encapsulation of params and input files) - TODO: Remember why this is here, it doesn't seem to be used anywhere - - systentry: a single entry in thie build system. This is equivalent - to a Makefile rule waiting for you to put actions into it. - Unlike this Rule class, this object wants exact specific commands - for a specific parameterization, like a Makefile rule with real - filenames and commands. - You must return: - - a list of key,val pairs, where key is some indicator of file type. - Example is [("header", "foo.h"), ("source", "foo.cc")]. - """ - # TODO: Implement me in subclasses - return util.map_values(FileCollection.to_pairs, dep_files) diff --git a/fastlib/py/fx.py b/fastlib/py/fx.py deleted file mode 100644 index a1cc10dca2..0000000000 --- a/fastlib/py/fx.py +++ /dev/null @@ -1,100 +0,0 @@ -""" -FastExec client library very similar to the C library. - -This provides command-line argument parsing in FASTexec style, and also provides -a global datastore for exporting metrics (and dumping them to a file if -desired). -""" - -import datastore - -import sys - -"""The root datanode for the fx system.""" -datanode = datastore.DataNode() - -"""Extra command-line arguments that do not contain --.""" -extra_args = [] - -def param(name, mustexist = True): - v = datanode.get_val_path("/info/params/" + name.strip("/")) - if mustexist and v == None: - raise "Parameter [%s] not specified." % (name) - return v - -def param_exists(name): - return param(name, False) != None - -def param_default(name, val): - if not param_exists(name): - datanode.set_val_path("/info/params/" + name.strip("/"), val) - -def param_str(name): - # TODO: Validate? - return param(name) - -def param_int(name): - # TODO: Validate? - return int(param(name)) - -def param_float(name): - return float(param(name)) - -def param_bool(name): - str = param(name) - if str == None or (len(str) > 0 and str[0] in "fFnN0"): - return False - else: - return True - -def dump(): - stream = sys.stdout - fname = param("fx/output") - if fname != None: - stream = open(fname, "w") - try: - datanode.write_pathstyle(stream) - finally: - if fname != None: - stream.close() - -# TODO: Configurable - -# TODO: FINISH THIS - -def metric_set(instancename, name, val): - if instancename == None: - path = ["info", "metrics", name] - else: - path = ["instances", instancename, "metrics", name] - datanode.set_val(path, val) - -def subparam_set(instancename, name, val): - path = ["instances", instancename, "params", name] - datanode.set_val(path, val) - -def __parse_args(): - """Parses command-line arguments. - - Note that the keys are in path-style and unescaped as so. - However, the values are assumed not to be escaped. - """ - global extra_args - kvpairs = [] - for arg in sys.argv[1:]: - if arg[0:2] == "--": - if '=' in arg: - i = arg.index("=") - kvpairs.append((arg[2:i], arg[i+1:])) - else: - kvpairs.append((arg[2:], "1")) - else: - extra_args.append(arg) - for (name, val) in kvpairs: - param_default(name, val) - -__parse_args() - -if __name__ == "__main__": - print datanode - print extra_args diff --git a/fastlib/py/fxsys.py b/fastlib/py/fxsys.py deleted file mode 100644 index 4662590b43..0000000000 --- a/fastlib/py/fxsys.py +++ /dev/null @@ -1,289 +0,0 @@ -"""FASTexec system management. - -Allows you to create, manage, and deploy experiments. -""" - -# work items: -# - path translation -# - argument translation (huh?) - -import os -import random -import util -import pm -import sys -import unittest -import work - -class FxSystemException: - pass - -class FxSystem: - """A particular fx system, parameterized by the root of the hierarchy. - - Important attributes: - - rootpath: the root of the system - """ - - def __init__(self, rootpath): - """Creates a new fx-system based on the specified path. - - Rootpath must exist. If the proper FASTexec subdirectories are not present - they will be created, but the root itself must be created. You may - consider making sure the root directory itself has the group - setguid bit if multiple people use the same system. - - - rootpath: absolute path of the root of the fx system - """ - self.rootpath = rootpath - try: - self.validate() - except: - self.create() - - def validate(self): - """Makes sure this is a valid fx system. - """ - if not os.access(os.path.join(self.rootpath, "FXSYS"), os.W_OK): - raise FxSystemException() - # TODO(garryb): Include real validation - - def create(self): - """Sets up this path as an fx system. - - The rootpath must exit, and must already have permissions set up properly. - - If the system is already set up, raises an error exception. - """ - # TODO(garryb): Implement - # TODO(garryb): Set permissions properly - #util.ensuredir(self.rootpath) - util.ensuredir(os.path.join(self.rootpath, "problems")) - util.ensuredir(os.path.join(self.rootpath, "data")) - util.writefile(os.path.join(self.rootpath, "FXSYS"), "FXSYS") - - # TODO(garryb): - - self.validate() - - def translate(self, relpath): - """ - Translates a path alias into absolute path. - - In particular, paths beginning with FXSYS/ are translated to real paths. - """ - relpath = str(relpath) - if relpath.startswith('FXSYS/'): - result = os.path.join(self.rootpath, relpath[len('FXSYS/'):]) - else: - result = os.path.abspath(relpath) - return result - - # All pathname hardcoding should exist in this class, even if it doesn't - # seem the best place to put it. - - def pathof_problem(self, probname): - return self.translate("FXSYS/problems/%s" % probname) - - def pathof_experiment(self, probname, expname): - return self.translate("FXSYS/problems/%s/exps/%s" % (probname, expname)) - - def pathof_run(self, probname, expname, runname): - return self.translate("FXSYS/problems/%s/exps/%s/runs/%s" % (probname, expname, runname)) - - def get_experiment_names(self, probname): - return os.listdir(self.translate("FXSYS/problems/%s/exps" % probname)) - -class Experiment: - """One particular experiment. - - An experiment is a labelled set of runs for a particular purpose. - """ - - def __init__(self, probname, expname, create = True, fxsys = None): - if fxsys == None: - fxsys = default_fxsys() - self.fxsys = fxsys - self.expname = expname - self.probname = probname - self.path = fxsys.pathof_experiment(probname, expname) - if create: - self.create() - - def pathof_run(self, runname): - return self.fxsys.pathof_run(self.probname, self.expname, runname) - - def pathof_statusfile(self, runname): - fullpath = self.pathof_run(runname) - return os.path.join(fullpath, work.WorkEntry.STATUSFILE) - - def create(self): - util.ensuredir(self.path) - - def execute(self, paramset): - wq = work.WorkQueue(self) - wq.add([work.WorkEntry(self, x) for x in paramset.enumerate()]) - wq.inline_exec() - - def get_statuses(self): - """Gets the status, such as work.WorkEntry.FINISHED, for each run that - has been started. It is a key-value map of run name to its execution - status. - - If the status file does not exist, is invalid, or cannot be accessed, - None is returned as the status. - """ - # This should go through and remove decaying STATUS files - statuses = {} - for runname in os.listdir(os.path.join(self.path, "runs")): - try: - lines = util.readlines(self.pathof_statusfile(runname)) - statuses[runname] = lines[0].strip() - except: - statuses[runname] = None - return statuses - - def cleanup(self, purge = False, nuke = False): - """Removes the status files of runs that have not finished. - This is useful you terminate a run and want to restart it -- if you - do not clean it up, the runs that were in progress will not be restarted. - - Be warned -- If you run this while one of the runs is actually - executing, that run will be re-run in parallel. - - purge: whether to completely delete the entire run's directory - (not just the status file) - """ - statuses = self.get_statuses() - for (runname, status) in statuses.items(): - if status != work.WorkEntry.FINISHED: - statusfile = self.pathof_statusfile(runname) - if purge or nuke: - print "Purging stale run %s" % runname - try: - util.remove_dir_recursive(self.pathof_run(runname)) - except OSError: - print "XXX Error deleting the run" - else: - print "Clearing status file of stale run %s" % runname - try: - os.unlink(statusfile) - except: - print "... Status file was already invalid. This run can be re-run." - elif nuke: - print "NUKING COMPLETED RUN %s" % runname - try: - util.remove_dir_recursive(self.pathof_run(runname)) - except OSError: - print "XXX Error deleting the run" - - def print_status(self): - """Prints the log files of any run that is not completed.""" - num_printed = 0 - items = self.get_statuses().items() - items_ordered = [] - items_ordered += [(x,y) for (x,y) in items if y == work.WorkEntry.FINISHED] - items_ordered += [(x,y) for (x,y) in items if y != work.WorkEntry.FINISHED] - for (runname, status) in items_ordered: - print "%s -- %s" % (status, runname) - if status != work.WorkEntry.FINISHED: - logfile = os.path.join(self.pathof_run(runname), work.WorkEntry.LOGFILE) - try: - num_printed += 1 - lines = util.readlines(logfile) - for line in lines: - print " | %s" % line.strip() - except: - print " X No log file" - return num_printed - - def purge(self): - util.remove_dir_recursive(self.path) - -# functions - -class NoConfigFileException: - pass - -def default_fxsys(): - """Gets the default Xsys object, by searching for fx.conf in the current - or parent directories. - """ - - # TODO: Cache this - path = os.getcwd() - - while len(path) > 1 and not os.path.exists(os.path.join(path, "fx.conf")): - path = os.path.dirname(path) - - if len(path) <= 1: - raise NoConfigFileException - - conffile = os.path.join(path, "fx.conf") - lines = util.readlines(conffile) - rootpath = os.path.join(path, lines[0]) - return FxSystem(rootpath) - -# public test helpers - -def test_new_testing_fxsys(): - (tmpdir, tmpfile) = os.path.split(os.tempnam()) - rootpath = "%s/fxsys/%s" % (tmpdir, tmpfile) - util.ensuredir(rootpath) - fxsys = FxSystem(rootpath) - print "Rooted at %s" % rootpath - fxsys.create() - return fxsys - -def test_create_numlist_infiles(fxsys): - infiles = [] - for i in [ 2000, 4000, 6000, 8000, 10000 ]: - fname = fxsys.translate("FXSYS/data/num%d.in" % i) - util.write_random_ints(fname, i) - infiles.append(fname) - return (infiles) - -class TestFx(unittest.TestCase): - - def setUp(self): - self.fxsys = test_new_testing_fxsys() - self.infiles = test_create_numlist_infiles(self.fxsys) - self.paramset = pm.Combine( - pm.Bind(pm.Binfile(), pm.Val("/usr/bin/sort")), - pm.Bind(pm.Extra(), pm.Val("-n")), - pm.Bind(pm.Stdin(), pm.Vals(*self.infiles)), - pm.Bind(pm.Stdout(), "out.txt")) - - def tearDown(self): - util.remove_dir_recursive(self.fxsys.rootpath) - - def test_inline(self): - self.exp = Experiment("xtest-sort", "sort-test_inline", fxsys = self.fxsys) - self.exp.create() - wq = work.WorkQueue(self.exp) - wq.add([work.WorkEntry(self.exp, runspec) for runspec in self.paramset.enumerate()]) - print "Inline first" - wq.inline_exec() - print "Inline second" - wq.inline_exec() - print "Inline end" - - def test_shell(self): - self.exp = Experiment("xtest-sort", "sort-test_shell", fxsys = self.fxsys) - self.exp.create() - wq = work.WorkQueue(self.exp) - wq.add([work.WorkEntry(self.exp, runspec) for runspec in self.paramset.enumerate()]) - print "Writing work queue..." - fname = wq.write() - print "Running %s..." % fname - os.spawnv(os.P_WAIT, "/bin/bash", ["/bin/bash", fname]) - print "Running a second time..." - os.spawnv(os.P_WAIT, "/bin/bash", ["/bin/bash", fname]) - - def test_default(self): - self.exp = Experiment("xtest-sort", "sort-test_default", fxsys = self.fxsys) - self.exp.execute(self.paramset) - -if __name__ == "__main__": - unittest.main() - diff --git a/fastlib/py/pm.py b/fastlib/py/pm.py deleted file mode 100644 index 404c69154e..0000000000 --- a/fastlib/py/pm.py +++ /dev/null @@ -1,382 +0,0 @@ - -import copy -import random -import util -import os -import unittest - -def combine(a, b): - if a == None: - return b - if b == None: - return a - raise Conflict - -class Conflict: - """Exception thrown when two single fields conflict. - - For example, perhaps a Combine is done, but the Binfile - is specified on both sides. - """ - pass - -class RunSpec: - """One particular combination of parameters, that corresponds - to one actual run of the program. - """ - def __init__(self): - """Create an empty run specification. - """ - self.binfile = None - self.wrappers = [] - self.wrapper_info = [] - self.params = {} - self.extra = [] - self.stdin = None - self.stdout = None - self.inputs = [] - self.outputs = [] - def merge(self, other): - """Merge the other's fields into this. - - A Combine exception is raised if singleton parameters - conflict. - - Dictionary-based 'Var' parameters are overwritten. - """ - self.binfile = combine(self.binfile, other.binfile) - self.params.update(other.params) - self.wrappers += other.wrappers - self.wrapper_info += other.wrapper_info - self.extra += other.extra - self.stdin = combine(self.stdin, other.stdin) - self.stdout = combine(self.stdout, other.stdout) - self.inputs += other.inputs - self.outputs += other.outputs - def merged_with(self, other): - """Return a copy of this RunSpec merged with the other's - fields. - """ - c = copy.deepcopy(self) - c.merge(other) - return c - def check(self): - assert self.binfile != None - def generate_name(self): - """Generates a user-friendly filename. - """ - self.check() - allparas = [] - allparas += self.wrapper_info - allparas += [os.path.basename(self.binfile)] - allparas += self.extra - nvp = [pair for pair in self.params.iteritems()] - nvp.sort() # Sort by parameter name to ensure deterministic ordering - for (name, val) in nvp: - if val in self.outputs: - # output files are not important - continue - if val in self.inputs: - val = "%s_%x" % (os.path.basename(val), hash(val) % 65536) - allparas.append("%s=%s" % (name, val)) - if self.stdin != None: - allparas += ["stdin=%s" % os.path.basename(self.stdin)] - return util.sanitize_basename("_".join(allparas)) - def to_args(self): - """Turns to a list of arguments such that args[0] is the binary file, - and the rest are the parameters, suitable for os.exec. - - This does not include the stdin or stdout redirects, so make sure you - handle them separately. - """ - allparas = self.wrappers + [self.binfile] + self.extra - allparas += ["--%s=%s" % (k, v) for (k, v) in self.params.iteritems()] - return allparas - def to_command(self): - """Turns into a single shell command. - """ - # TODO: Escaping - allparas = [] - allparas += [ util.shellquote(x) for x in self.wrappers ] - allparas += [util.shellquote(self.binfile)] - allparas += [ util.shellquote(x) for x in self.extra ] - allparas += ["--%s=%s" % (k, util.shellquote(v)) for (k, v) in self.params.iteritems()] - if self.stdin != None: - allparas += ["<%s" % util.shellquote(self.stdin)] - if self.stdout != None: - allparas += [">%s" % util.shellquote(self.stdout)] - return " ".join(allparas) - -# Parameter sets -- sets of all possible run parameters - -class ParamSet: - """Abstract set of parameters. - """ - def enumerate(self): - """Returns a list of RunSpecs for all runs that should exist.""" - return [] - def print_all(self): - """Prints all commands that would be executed if all were to be - run.""" - for e in self.enumerate(): - print e.to_command() - -class Combine(ParamSet): - """Cartesian product, or all combinations, - of several smaller parameter sets. - - (Technically, this is closer to intersction, but the - actual 'intersection' operation is not performed for - redundant parameters. The handling of redundant - parameters is undefined.) - - Parameters are multiplied so that the last parameters - vary closest together, and the first in the list - vary last. That is, (1 2) x (A B) = - - 1 A - 1 B - 2 A - 2 B - """ - def __init__(self, *factors): - self.factors = factors - def enumerate(self): - all = [ RunSpec() ] - for item in self.factors: - newlist = [] - enumerated = item.enumerate() - for runspec1 in all: - for runspec2 in enumerated: - newlist.append(runspec1.merged_with(runspec2)) - all = newlist - return all - -class Any(ParamSet): - """The union of several smaller parameter sets. - """ - def __init__(self, choices): - self.choices = choices - def enumerate(self): - all = [] - for choice in self.choices: - all += choice.enumerate() - return all - -class Bind(ParamSet): - """Binds a particular RunSpec field to a particular - value set. - """ - def __init__(self, var, valset): - self.var = var - if not isinstance(valset, ValSet): - if isinstance(valset, list): - valset = Vals(*valset) - else: - valset = Val(valset) - self.valset = valset - def enumerate(self): - all = [] - for val in self.valset.enumerate(): - spec = RunSpec() - self.var.set(spec, val) - all.append(spec) - return all - -class CoBind(ParamSet): - """Binds any number of RunSpec fields to any number - of value sets, one value set per fields. - - When enumerated, the enumerations of every value set are zipped together; - all enumerations must be of equal size. - """ - def __init__(self, *pairs): - self.vars = [] - self.valsets = [] - for i in range(0, len(pairs), 2): - self.vars.append(pairs[i]) - self.valsets.append(pairs[i + 1]) - # TODO: Runtime check to make sure each set is the same size - def enumerate(self): - all = [] - valmatrix = [] - for valset in self.valsets: - valmatrix.append(valset.enumerate()) - assert min(map(len, valmatrix)) == max(map(len, valmatrix)) - for i in range(len(valmatrix[0])): - spec = RunSpec() - for j in range(len(valmatrix)): - self.vars[j].set(spec, valmatrix[j][i]) - all.append(spec) - return all - -class CrossValidate(ParamSet): - """ - WARNING! HASN'T BEEN TRIED YET - - Runs several runs over the data set, varying which subset is used for - training and testing. - - NOTE TO SELF: Eventually we would want "optimize over cross-validate" - to work properly (and not find the best portion to cross validate over). - """ - def __init__(self, varname_train, varname_test, count, *files): - """Creates a cross-validation set of runs. - - The parameter names of train and test are provided, and it is assumed - the program being run understands the :x-2/5 and :x2/5 syntax used to - denote cross validation. - - You then provide the number of ways you want to do cross validation, and - also the list of file names, or just a single file name, to run over. - - Example 1: CrossValidate("train", "test", 10, "a.txt") - - Example 2: CrossValidate("train_set", "test_set", 20, "a.txt", "b.txt") - """ - self.varname_train = varname_train - self.var_train = Input(varname_train) - self.varname_test = varname_test - self.var_test = Input(varname_test) - self.files = files - self.count = count - def enumerate(self): - all = [] - for file in self.files: - for i in range(0, count): - spec = RunSpec() - self.var_train.set(spec, file) - self.var_test.set(spec, file) - spec.params[self.varname_train + "/subset"] = ("x-%d/%d" % (i, count)) - spec.params[self.varname_test + "/subset"] = ("x%d/%d" % (i, count)) - all.append(spec) - return all - -# Destinations that can be bound to - -class BindDest: - """Anything that can be bound to, a RunSpec field.""" - def set(self, spec, val): - """Sets the corresponding field in the RunSpec to the - given value.""" - pass - -class Var(BindDest): - """A regular parameter, such as --length. - """ - def __init__(self, name): - self.name = name - def set(self, spec, val): - spec.params[self.name] = val - -class MpiCluster(BindDest): - """A parameter that represents running in MPI. - """ - def __init__(self, machinefile): - """Sample use: - pm.Bind(pm.MpiCluster(os.path.abspath("./amdmachines.txt")), [1, 2, 4, 8, 12]) - """ - self.machinefile = machinefile - def set(self, spec, val): - spec.wrappers += ["mpirun", "-machinefile", self.machinefile, "-np", str(val)] - spec.wrapper_info += ["mpi-%s-%s" % (os.path.basename(self.machinefile), str(val))] - -class Input(Var): - """A parameter that corresponds to be an input file. - - This field is suitable for enforcing file dependencies. - - This will also truncate any characters after ':' operator for dependency - purposes; the ':' is treated special denoting particular kinds of subsets, - used in cross validation. - """ - def set(self, spec, val): - spec.inputs.append(val) - Var.set(self, spec, val) - -class Output(Var): - """A parameter that corresponds to be an output file. - - This field is suitable for enforcing file dependencies. - """ - def set(self, spec, val): - spec.outputs.append(str(val)) - Var.set(self, spec, val) - -class Extra(BindDest): - """An extra parameter that doesn't directly fit into the - fx system. - """ - def set(self, spec, val): - spec.extra.append(val) - -class Stdin(BindDest): - """A parameter that corresponds to be standard input. - - This field is suitable for enforcing file dependencies. - """ - def set(self, spec, val): - spec.stdin = str(val) - -class Stdout(BindDest): - """A parameter that corresponds to be standard output. - - This field is suitable for enforcing file dependencies. - """ - def set(self, spec, val): - spec.stdout = str(val) - -class Binfile(BindDest): - """The binary file to be executed. - """ - def set(self, spec, val): - spec.binfile = str(val) - -# Set of values that can be bound to a destination - -class ValSet: - """An arbitrary set of values that may be bound to a variable. - """ - def enumerate(self): - return [] - -class Vals(ValSet): - """A pre-specified enumeration of values. - """ - def __init__(self, *vals): - self.vals = vals - def enumerate(self): - return self.vals - -class Val(Vals): - """A pre-specified single value. - """ - def __init__(self, val): - Vals.__init__(self, val) - -# Tests - -class ParamTest(unittest.TestCase): - def setUp(self): - self.params = Combine( - Bind(Binfile(), Val("/usr/bin/sort")), - Bind(Input("infile"), Vals("in1.txt", "in2.txt")), - Bind(Var("useless"), Vals(1.1, 1.3, 1.7)), - CoBind(Var("bw1"), Vals(1, 2, 4, 8, 16), Var("bw2"), Vals(0, 1, 2, 3, 4))) - - def test_print(self): - # TODO - Doesn't test anything - self.params.print_all() - - def test_len(self): - self.assertEqual(30, len(self.params.enumerate())) - - def test_product(self): - all = self.params.enumerate() - def count(param, val): - return len([x for x in all if x.params[param] == val]) - self.assertEqual(15, count("infile", "in1.txt")) - self.assertEqual(10, count("useless", 1.3)) - -if __name__ == "__main__": - unittest.main() diff --git a/fastlib/py/util.py b/fastlib/py/util.py deleted file mode 100644 index 02e75eeec0..0000000000 --- a/fastlib/py/util.py +++ /dev/null @@ -1,499 +0,0 @@ -import random -import re -import os -import sys -import StringIO -import unittest - -# TODO: this isn't our own work, I don't remember where it was pulled from -def natsort_key(item): - chunks = re.split('(\d+(?:\.\d+)?)', item) - for ii in range(len(chunks)): - if chunks[ii] and chunks[ii][0] in '0123456789': - if '.' in chunks[ii]: - numtype = float - else: - numtype = int - chunks[ii] = (0, numtype(chunks[ii])) - else: - chunks[ii] = (1, chunks[ii]) - return (chunks, item) - -def natsort(seq): - l = list(seq) - l.sort(key=natsort_key) - return l - -def sorted(l): - """Returns the sorted version of the list. - - (Remove once everyone is running Python 2.4+) - """ - l_copy = list(l) - l_copy.sort() - return l_copy - -def map_values(f, d): - """Returns a copy of the dictionary but with all the values mapped.""" - return dict([(k, f(v)) for (k, v) in d.items()]) - -def map_keys(f, d): - """Returns a copy of the dictionary but with all the values mapped.""" - return dict([(f(k), v) for (k, v) in d.items()]) - -def dicthash(d): - """Turns a dictionary into something hashable.""" - return tuple(sorted(d.items())) - -def collapse_once(collection_of_collection): - """Turns a list of lists into just the items.""" - result = [] - for collection in collection_of_collection: - result += collection - return result - -def remove_dir_recursive(dirname): - """Removes a directory like rm -rf.""" - for subname in os.listdir(dirname): - name = os.path.join(dirname, subname) - if os.path.isdir(name): - remove_dir_recursive(name) - else: - os.remove(name) - os.rmdir(dirname) - - -def testfile(filename): - """Tests if a file exists. - """ - return os.access(filename, os.F_OK) - -def createlock(filename): - """Tries to lock a file for writing. - - Currently this is a stub and just tests for existence, but there is no - real locking semantics. - """ - # TODO: Mode operation - if testfile(filename): - return False - try: - os.open(filename, os.O_CREAT|os.O_EXCL, 0660) - return True - except OSError: - return False - -def writefile(filename, text): - """Writes the text to a file by name. - """ - f = open(filename, "w") - try: - f.write(text) - finally: - f.close() - -def readfile(filename): - """Reads the text from a file. - """ - f = open(filename, "r") - try: - text = f.read() - finally: - f.close() - return text - -def writelines(filename, lines): - """Writes each line to the specified file. - - The Unix newline character will be appended to each line. - """ - # TODO: Unix versus Dos CR/LV - f = open(filename, "w") - try: - f.writelines(["%s\n" % line for line in lines]) - finally: - f.close() - return lines - -def readlines(filename): - """Reads each line from a file to a list, with all whitespace - stripped from the end of each line. - """ - f = open(filename, "r") - try: - lines = f.readlines() - finally: - f.close() - lines = [ l.rstrip() for l in lines ] - return lines - -def read_csv(fname): - """Reads a comma-separated-value file as a matrix. - """ - f = open(fname, "r") - try: - return read_csv_file(f) - finally: - f.close() - -def read_csv_file(f): - """Reads an open comma-separated-value file as a matrix. - """ - return [[s.strip() for s in l.split(",")] for l in f.readlines()] - -def write_csv(fname, lines): - """Writes the specified matrix as a comma-separated-value file. - """ - f = open(fname, "w") - try: - write_csv_file(f, lines) - finally: - f.close() - -def write_csv_file(f, lines): - """Writes the specified matrix as a comma-separated-value open file. - """ - for line in lines: - sanitized = [ str(field).replace(",", ";") for field in line ] - f.write(", ".join(sanitized) + "\n") - -def escape_latex(str): - result = "" - for c in str: - if c == '\\': - result += "$\\backslash$" - elif c in "#%&~$_^{}": - result += "\\" + c - else: - result += c - return result - -def write_latex_table_file(f, lines, align = "r"): - """Writes specified text to a file as a latex table. - """ - def formatline(line): - return " & ".join([escape_latex(field) for field in line]) + " \\\\" - - max_width = max([len(line) for line in lines]) - - # ensure len(align) equals max_width - while len(align) < max_width: - align += align[-1] - align = align[0:max_width] - - f.write("\\documentclass[letter]{article}\n") - f.write("\\begin{document}\n") - f.write("\\begin{tabular}{%s}\n" % ("|".join(align))) - f.write(formatline(lines[0])) - f.write("\n\\hline") - for line in lines[1:]: - f.write(formatline(line)) - f.write("\n\\end{tabular}\n") - f.write("\\end{document}\n") - -def write_random_ints(filename, count): - """Writes a sequence of random integers to a file. - """ - nums = [ "%d" % random.randint(0, 99999999) for i in range(count) ] - writelines(filename, nums) - -def shellquote(s): - """Quotes shell parameters. - - Note that things like newlines and special characters are included literally - in the string, compliant with BASH. - - Perhaps another shellquote function should be written, which uses the - $"string" format, that allows C-like escaping. - """ - result = "" - map = {"$":"\\$", "\"":"\\\"", "`":"\\`", "!":"\\!"} - for c in str(s): - if c in map.keys(): - result += map[c] - else: - result += c - return "\"%s\"" % result - -def sanitize_basename(s): - """Sanitizes the base of a filename. - - This is used in the naming of runs. - """ - result = "" - allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ=-_0123456789" - for c in str(s): - if c in allowed: - result += c - elif c in "./": - result += "-" - return result - -def ensuredir(dirpath): - """Ensure that a directory exists. - """ - if (not os.access(dirpath, os.R_OK|os.W_OK)): - os.makedirs(dirpath) - -def spawn_redirect(rundir, args, infile = None, outfile = None, errfile = None): - """Spawn a new process, optionally redirecting one of standard in, standard - out, or standard error to the specified filenames. - - - rundir: the directory to change to when running the program - - args: the arguments, where args[0] is the file to run - - infile: filename of standard in or None - - outfile: filename of standard out or None - - errfile: filename of standard error or None - """ - pid = os.fork() - if not pid: - try: - if rundir != None: - os.chdir(rundir) - if not infile: - infile = "/dev/null" - if infile != None: - infd = os.open(infile, os.O_RDONLY) - os.close(0) - os.dup2(infd, 0) - os.close(infd) - else: - os.close(0) - if outfile != None: - outfd = os.open(outfile, os.O_WRONLY|os.O_CREAT|os.O_TRUNC) - os.close(1) - os.dup2(outfd, 1) - os.close(outfd) - else: - outfd = None - #os.close(1) - if errfile != None: - # TODO: Mode 755? - errfd = os.open(errfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND) - os.close(2) - os.dup2(errfd, 2) - os.close(errfd) - else: - errfd = None - #os.close(2) -- Show standard error - os.execvp(args[0], args) - finally: - # WALDO - try: - if infile: - os.close(0) - except: - pass - try: - if outfile: - os.close(1) - except: - pass - try: - if errfile: - os.close(2) - except: - pass - os._exit(1) - (p, status) = os.waitpid(pid, 0) - # Status is a 16-bit number. - # The right 8 bits of status is the signal killed (always zero in case - # of Windows). - # The left eight bits are set to the exit value. - if status & 0xFF != 0: - return -1 # It was killed -- the right eight bits are non-zero - else: - return status >> 8 - -def shell(args, infile = None, outfile = None, errfile = None): - result = spawn_redirect(None, args, infile, outfile, errfile) - if result != 0: - print " shell: cannot do: ", args - raise OSError() - -class ParseError: - """Parse error when unqouting. - """ - pass - - -def __dspath_allowed_compute(): - global __dspath_allowed - __dspath_allowed = {} - l = range(ord('A'), ord('Z')) - l += range(ord('a'), ord('z')) - l += range(ord('0'), ord('9')) - l += [ord('.'), ord('-'), ord('_')] - for c in l: - __dspath_allowed[chr(c)] = True - -__dspath_allowed_compute() - -def escape_dspath(str): - """Quote method used for DataNode path elements, encoded in HTML-like - format. - - Only alphanumeric characters, the period, plus sign, minus sign, and - underscore are preserved. Everything else uses a percent (%) sign and - a 2-digit hex code. This is not due to necessity as much as it - is convenient that these characters won't need to be escaped for instance - if you type them from a shell, or in a regular expression. - - (TODO: Decide if the plus sign should also be escaped, for HTML - purposes.) - """ - global __dspath_allowed - result = "" - for c in str: - if c in __dspath_allowed.keys(): - result += c - else: - result += "%%%02X" % ord(c) - return result - -"""This represents the special star character. - -As a string, it looks like only a single star, but it is actually a list. -""" -STAR=tuple('*') - -def unescape_dspath(s): - """Decodes DataNode path elements, by interpreting the special - percent character. - """ - try: - if s == '*': - return STAR - else: - result = "" - hexdigits = "0123456789ABCDEF" - i = 0 - while i < len(s): - c = s[i] - if c == '%': - result += chr(hexdigits.index(s[i+1]) * 16 + hexdigits.index(s[i+2])) - i += 3 - else: - result += c - i += 1 - return result - except: - raise ParseError() - -def combine_path(elements): - """Combines and escapes path elements. - """ - result = "" - for e in elements: - result += "/" + escape_dspath(e) - return result - -def split_path(path): - """Splits and unescapes path elements. - """ - return [unescape_dspath(x) for x in path.strip("/").split("/")] - -def escape_sexpression(str): - """s-expression quoting hack.""" - result = "" - for c in str: - if c.isalnum(): - result += c - else: - result += "_%02X" % ord(c) - return result - -def unescape_sexpression(s): - """Decodes sexpression-escaped elements, by - interpreting the special underscore character. - """ - try: - result = "" - hexdigits = "0123456789ABCDEF" - i = 0 - while i < len(s): - c = s[i] - if c == '_': - result += chr(hexdigits.index(s[i+1]) * 16 + hexdigits.index(s[i+2])) - i += 3 - else: - result += c - i += 1 - return result - except: - raise ParseError() - -def keys(collection): - """Returns the keys of a collection, whether it is an array or dict. - - If it is a list, it returns range(len(x)), otherwise x.keys(). - """ - if isinstance(collection, list): - return range(len(collection)) - else: - return collection.keys() - -def typeconvert_list(items, *type_func_pairs): - return [typeconvert(item, *type_func_pairs) for item in items] - -def typeconvert(item, *type_func_pairs): - for (type, func) in type_func_pairs: - if isinstance(item, type): - item = func(item) - break - return item - - -class Table: - def __init__(self, headings, matrix = []): - self.headings = headings - self.matrix = matrix - def lookup(self, column): - if not isinstance(column, int): - column = self.headings.index(column) - return column - def restricted_fn(self, column, testfn): - column = self.lookup(column) - newmatrix = [line for line in newmatrix if testfn(line[column])] - return Table(self.headings, newmatrix) - def restricted(self, column, allowed): - if not isinstance(allowed, list): - allowed = [allowed] - column = self.lookup(column) - newmatrix = [line for line in self.matrix if line[column] in allowed] - return Table(self.headings, newmatrix) - -def read_table(fname): - """Reads the text from a file. - """ - f = open(fname, "r") - try: - return read_table_file(f) - finally: - f.close() - -def read_table_file(f, headings = None): - return table_from_matrix(read_csv_file(f), headings) - -def table_from_matrix(matrix, headings = None): - if not headings: - headings = matrix[0] - matrix = matrix[1:] - table = Table(headings, matrix) - return table - - -# TODO: Unit tests - -class UtilTest(unittest.TestCase): - def test_typeconvert(self): - self.assertEqual("1", typeconvert(1, (int, str))) - self.assertEqual(1, typeconvert(1, (str, int))) - self.assertEqual([1], typeconvert([1], (int, str))) - def test_typeconvert_list(self): - self.assertEqual(["1", "2", "3", "4"], - typeconvert_list([1, 2, 3, "4"], (int, str))) - -if __name__ == "__main__": - unittest.main() - test_matching() diff --git a/fastlib/py/work.py b/fastlib/py/work.py deleted file mode 100644 index 0151a5f5fe..0000000000 --- a/fastlib/py/work.py +++ /dev/null @@ -1,152 +0,0 @@ -import os -import datastore -import util - -# TODO: Who translates? - -class WorkEntry: - """A single entry on the work queue. - - Important attributes: - - binfile: the binary file to execute - - params: parameters passed, as a raw list - - inputs: all files required by this stage - - outputs: all files written by this stage - """ - - INPUT_NOT_AVAILABLE = "input_not_available" - IN_PROGRESS = "in_progress" - FINISHED = "finished" - ERROR = "error" - STATES = [INPUT_NOT_AVAILABLE, IN_PROGRESS, FINISHED, ERROR] - - STATUSFILE = "STATUS" - LOGFILE = "LOG" - - def __init__(self, experiment, runspec): - """Creates a new work queue entry. - """ - self.experiment = experiment - self.runname = runspec.generate_name() - self.rundir = experiment.pathof_run(self.runname) - self.runspec = runspec - self.logfile = os.path.join(self.rundir, WorkEntry.LOGFILE) - self.statusfile = os.path.join(self.rundir, WorkEntry.STATUSFILE) - - def to_datastore(self, node): - #TODO: FINISH - node.get_subnode_create() - self.fxsys.to_datastore(node) - node.set_val_path("/rundir", self.rundir) - # TODO: serialize the runspec - - def to_bash(self): - conditions = [] - conditions.append("! -e %s" % (util.shellquote(self.statusfile))) - for input in self.runspec.inputs: - conditions.append("-e %s" % util.shellquote(input)) - if self.runspec.stdin != None: - conditions.append("-e %s" % util.shellquote(self.runspec.stdin)) - command = self.runspec.to_command() - id = util.shellquote(os.path.basename(self.runname)) - lines = [ - "echo %s" % id, - "if %s" % " && ".join([ "[ %s ]" % x for x in conditions]), - "then echo ' ... Starting'", - "mkdir -p %s" % util.shellquote(self.rundir), - "cd %s" % util.shellquote(self.rundir), - "echo '%s' >%s" % (WorkEntry.IN_PROGRESS, WorkEntry.STATUSFILE), - "%s 2>%s" % (command, util.shellquote(self.logfile)), - "RV=$?", - "if [ \"$RV\" == 0 ]", - "then echo '%s' >%s" % (WorkEntry.FINISHED, WorkEntry.STATUSFILE), - "echo ' ... Done!'", - "else echo '%s' >%s" % (WorkEntry.ERROR, WorkEntry.STATUSFILE), - "echo ' ... Error' $RV", - "fi", - "fi" - ] - return "; ".join(lines) - - def inline_exec(self): - # TODO: Recover from errors - print "%s" % self.runname - try: - for infile in self.runspec.inputs: - if not os.path.exists(infile): - return self.INPUT_NOT_AVAILABLE - if not os.path.exists(self.rundir): - util.ensuredir(self.rundir) - except OSError: - print "ERROR: Error reading input files: %s" % (str(self.runspec.inputs)) - return self.ERROR - try: - if not util.createlock(self.statusfile): - try: - lines = util.readlines(self.statusfile) - WorkEntry.STATES.index(lines[0]) # raise exception if not a valid state - return lines[0] - except OSError: - print "ERROR: Could not read status file %s." % self.statusfile - return self.ERROR - else: - print " ... %s" % self.runspec.to_command() - util.writefile(self.statusfile, self.IN_PROGRESS) - retval = util.spawn_redirect( - self.rundir, self.runspec.to_args(), - self.runspec.stdin, self.runspec.stdout, self.logfile) - if retval != 0: - status = self.ERROR - print " ... Error %d " % retval - print util.readfile(self.logfile) - else: - status = self.FINISHED - print " ... Done!" - util.writefile(self.statusfile, status) - return status - except WorkQueue: - try: - util.writefile(self.statusfile, self.ERROR) - except: - pass - return self.ERROR - -class WorkQueue: - """A list of work that needs to be done. - - This currently requires to be added in a dependency-friendly order. - """ - - def __init__(self, experiment): - """Creates an empty WorkQueue for the specified system. - """ - - self.experiment = experiment - self.path = experiment.path - self.entries = [] - - def add(self, entries): - self.entries.extend(entries) - - def reorder(self): - # sort by inputs and outputs - pass - - def to_bash_lines(self): - return ["#!/bin/bash"] + [x.to_bash() for x in self.entries] - - def write(self): - fname = os.path.join(self.path, "workqueue.sh") - f = open(fname, "a") - for line in self.to_bash_lines(): - f.write(line) - f.write("\n") - f.close() - return fname - - def inline_exec(self): - # TODO: Handle the return value - for item in self.entries: - result = item.inline_exec() - print result -