hi

fastlib/Doxyfile

@@ -49,7 +49,7 @@ EXTRACT_STATIC         = NO
 EXTRACT_LOCAL_CLASSES  = YES
 EXTRACT_LOCAL_METHODS  = NO
 HIDE_UNDOC_MEMBERS     = NO
-HIDE_UNDOC_CLASSES     = YES
+HIDE_UNDOC_CLASSES     = NO
 HIDE_FRIEND_COMPOUNDS  = NO
 HIDE_IN_BODY_DOCS      = NO
 INTERNAL_DOCS          = NO
@@ -90,7 +90,6 @@ RECURSIVE              = YES
 EXCLUDE                =
 EXCLUDE_SYMLINKS       = YES
 EXCLUDE_PATTERNS       = */auton/* \
-                         */u/* \
                          */test/* \
                          */bin/* \
                          *test.cc

fastlib/base/build.py

@@ -1,24 +1,21 @@
+def config_doit(sysentry, files, params):
+  script = files["script"].single(Types.SCRIPT)
+  outdir = sysentry.bin_dir("arch", "kernel", "compiler")
+  indir = sysentry.sys.source_dir
+  sysentry.command("%s --genfiles_dir=%s --source_dir=%s" % (script, outdir, indir))
+  gen_headers = ["base/basic_types.h"]
+  return [(Types.HEADER, sysentry.file(h, "arch", "kernel", "compiler"))
+      for h in gen_headers]
 
-class ConfigHeadersRule(dep.Rule):
-  def __init__(self):
-    self.script = sourcerule(Types.SCRIPT, "config.py")
-    self.sources = sourcerules(Types.ANY, lglob("config/*.c"))
-    dep.Rule.__init__(self, script=[self.script], sources=self.sources)
-  def doit(self, sysentry, state, files, params):
-    script = files["script"].single(Types.SCRIPT)
-    outdir = sysentry.bin_dir("arch", "kernel", "compiler")
-    indir = sysentry.sys.source_dir
-    sysentry.command("%s --genfiles_dir=%s --source_dir=%s" % (script, outdir, indir))
-    gen_headers = ["base/basic_types.h"]
-    return [(Types.HEADER, sysentry.file(h, "arch", "kernel", "compiler"))
-        for h in gen_headers]
-
-register(name = "config_headers",
-    rule = ConfigHeadersRule())
+customrule(
+    name = "config_headers",
+    dependencies =
+        {"script": [sourcerule(Types.SCRIPT, "../script/config.py")],
+         "sources": sourcerules(Types.ANY, lglob("config/*.c"))},
+    doit_fn = config_doit)
 
 librule(
     sources = ["common.c", "cc.cc", "ccmem.cc"],
     headers = ["cc.h", "ccmem.h", "common.h", "compiler.h",
-               "compiler_impl.h",
+               "compiler_impl.h", "test.h", "fortran.h",
                "debug.h", "scale.h", ":config_headers"])
-

fastlib/base/common.h

@@ -24,6 +24,7 @@
 #include "scale.h"
 
 #include <float.h>
+#include <math.h>
 
 EXTERN_C_START
 

fastlib/base/compiler.h

@@ -144,4 +144,14 @@
 #define offsetof(structure, field) ((size_t)((&(structure const *)0)->field))
 #endif
 
+#ifdef __cplusplus
+/** Use constant reference type in C, otherwise use constant pointer. */
+#define CONST_REF const&
+/** Use reference type in C, otherwise use pointer. */
+#define REF &
+#else
+#define CONST_REF const*
+#define REF *
+#endif
+
 #endif

fastlib/base/compiler_impl.h

@@ -23,8 +23,8 @@
 
 #ifdef __GNUC__
 #define expect__impl(expr, value) (__builtin_expect((expr), (value)))
-#define likely__impl(x) (__builtin_expect((x), 1))
-#define unlikely__impl(x) (__builtin_expect((x), 0))
+#define likely__impl(x) (__builtin_expect(!!(x), 1))
+#define unlikely__impl(x) (__builtin_expect(!!(x), 0))
 #define COMPILER_NORETURN__IMPL __attribute__((noreturn))
 #define COMPILER_PRINTF__IMPL(format_arg, dotdotdot_arg) \
     __attribute__((format(printf, format_arg, dotdotdot_arg)))

fastlib/base/debug.h

@@ -214,4 +214,10 @@ void debug_poison_ptr(T *&x) {
         "%s == %"L64"d exceeds bound %s == %"L64"d\n", \
         #x, STATIC_CAST(uint64, x), #bound, STATIC_CAST(uint64, bound))
 
+/**
+ * Asserts that two integers are the same in debug mode.
+ */
+#define DEBUG_SAME_INT(a, b) \
+    DEBUG_ASSERT_MSG(((a) == (b)), "[%s] %d != %d [%s]", #a, (int)(a), (int)(b), #b)
+
 #endif

fastlib/base/test.h

@@ -23,6 +23,14 @@
 #define TEST_ASSERT(x) \
     DEBUG_ASSERT(x)
 
+#define TEST_DOUBLE_EXACT(a, b) \
+    if (unlikely((a) != (b))) \
+    FATAL("%.10e (%s) != %.10e (%s)", (double)(a), #a, (double)(b), #b); else
+
+#define TEST_DOUBLE_APPROX(a, b, absolute_eps) \
+    if (unlikely(fabs((a) - (b)) > absolute_eps)) \
+    FATAL("%.10e (%s) !~= %.10e (%s)", (double)(a), #a, (double)(b), #b); else
+
 /**
  * Begin a test suite of a given name.
  *
@@ -49,7 +57,7 @@
     }
 
 typedef void (*test__void_func)();
-#endif
 
 #endif
 
+#endif

fastlib/col/col.cc

@@ -72,6 +72,10 @@ index_t String::Split(index_t start_index,
     
     while (1) {
       if (unlikely(*endpos == '\0') || strchr(donechars, *endpos) != NULL) {
+        // strip extra delimeters from right side
+        while (endpos > startpos && strchr(delimeters, endpos[-1]) != NULL) {
+          endpos--;
+        }
         done = true;
         break;
       }
@@ -93,3 +97,31 @@ index_t String::Split(index_t start_index,
   return pos - begin();
 }
 
+void String::TrimLeft(const char *delimeters, String *result) const {
+  const char *s = begin();
+  while (*s != '\0' && strchr(delimeters, *s)) {
+    s++;
+  }
+  result->Copy(s, end() - s);
+}
+
+void String::TrimRight(const char *delimeters, String *result) const {
+  const char *s = end() - 1;
+  const char *b = begin();
+  while (s >= b && strchr(delimeters, *s)) {
+    s--;
+  }
+  result->Copy(b, s - b + 1);
+}
+
+void String::Trim(const char *delimeters, String *result) const {
+  const char *b = begin();
+  const char *e = end() - 1;
+  while (e >= b && strchr(delimeters, *e)) {
+    e--;
+  }
+  while (e >= b && strchr(delimeters, *b)) {
+    b++;
+  }
+  result->Copy(b, e - b + 1);
+}

fastlib/col/heap.h

@@ -92,6 +92,8 @@ class MinHeap {
   
   /**
    * Pops and returns the lowest element off the heap.
+   *
+   * @return the value associated with the highest priority
    */
   Value Pop() {
     Value t = entries_[0].value;
@@ -103,6 +105,9 @@ class MinHeap {
   
   /**
    * Removes the lowest element from the heap.
+   *
+   * Simply pops the top value on the queue, without
+   * returning it.
    */
   void PopOnly() {
     Entry entry = *entries_.PopBackPtr();

fastlib/col/string.h

@@ -22,8 +22,9 @@
  * tokenizers that you may find useful.  Finally, it is non-templated,
  * so compiler errors are easier to understand.
  *
- * WARNING: This has gone through rigorous testing -- we expect it to work,
- * but there may be some issues.
+ * WARNING: This has not gone through rigorous testing -- we expect it to
+ * work, but there may be some issues.  You will be just fine using the
+ * STL string if you need string processing.
  */
 class String {
  private:
@@ -365,6 +366,24 @@ class String {
   index_t Split(const char *delimeters, ArrayList<String> *result) const {
     return Split(0, delimeters, "", 0, result);
   }
+  
+  /**
+   * Creates a new string with the specified characters removed from the left
+   * part of the string.
+   */
+  void TrimLeft(const char *delimeters, String *result) const;
+
+  /**
+   * Creates a new string with the specified characters removed from the right
+   * part of the string.
+   */
+  void TrimRight(const char *delimeters, String *result) const;
+
+  /**
+   * Creates a new string with the specified characters removed from the left
+   * and right parts of the string.
+   */
+  void Trim(const char *delimeters, String *result) const;
 
   /**
    * Compares two strings case-insensitively.

fastlib/data/crossvalidation.h

@@ -13,7 +13,8 @@
 #include "fx/fx.h"
 
 /**
- * Cross-validator for classifiers, integrating tightly with FastExec.
+ * Cross-validator for simple classifiers, integrating tightly with
+ * FastExec.
  *
  * Cross-validation runs go under path you give it (kfold_fx_name),
  * by default "kfold".
@@ -51,6 +52,32 @@
  * /kfold/0/knn/params/k      # this ensures you'll get default params
  * /kfold
  * @endcode
+ *
+ * Before the cross-validator runs, it will copy parameters from the module
+ * you specify -- if it is module_root, this will just take the original
+ * command line parameters that are stored in "/params".  In the previous
+ * example, the command line parameters from "/params/knn/" and
+ * "/params/kfold/" are used.  These parameters are specified by the user
+ * as "--params/knn/someparameter=3" or "--param/kfold/k=4" to set KNN's
+ * "someparameter" to 3, and the cross-validator's number of folds to 4.
+ *
+ *
+ * To build a classifier suitable for use with SimpleCrossValidator, you
+ * must create a class with the following methods:
+ *
+ * @code
+ * class MyClassifier {
+ *   ...
+ *   // Trains on the dataset specified.  n_classes is the number of class
+ *   // labels.  Tweak parameters can be obtained from the "datanode" passed
+ *   // using fx_param_int, fx_param_double, etc, but passing in "module" as
+ *   // the first parameter instead of NULL.
+ *   //
+ *   void InitTrain(const Dataset& dataset, int n_classes, datanode *module);
+ *   // For a test datum, returns the class label 0 <= label < n_classes
+ *   int Classify(const Vector& test_datum);
+ * };
+ * @endcode
  */
 template<class TClassifier>
 class SimpleCrossValidator {
@@ -145,12 +172,12 @@ class SimpleCrossValidator {
       Dataset test;
       Dataset train;
       index_t local_n_correct = 0;
-      datanode *foldmodule = fx_submodule(kfold_module_,
-          String().InitSprintf("%d", i_folds).c_str(), NULL);
-      datanode *classifier_module = fx_submodule(foldmodule,
-          classifier_fx_name_, NULL);
+      datanode *foldmodule = fx_submodule(kfold_module_, NULL,
+          String().InitSprintf("%d", i_folds).c_str());
+      datanode *classifier_module = fx_submodule(foldmodule, NULL,
+          classifier_fx_name_);
       
-      fx_def_param_node(classifier_module, "", root_module_,
+      fx_default_param_node(classifier_module, "", root_module_,
           classifier_fx_name_);
 
       data_->SplitTrainTest(n_folds_, i_folds, permutation, &train, &test);

fastlib/data/dataset.cc

@@ -5,6 +5,7 @@
  */
 
 #include "file/textfile.h"
+#include "base/common.h"
 
 #include "dataset.h"
 
@@ -161,15 +162,19 @@ success_t DatasetInfo::InitFromArff(TextLineReader *reader,
             feature->InitNominal(portions[1]);
             // TODO: Doesn't support values with spaces {
             portions[2].Split(1, ", \t", "}%", 0, &feature->value_names());
-          } else if (portions[2].EqualsNoCase("numeric")
-              || portions[2].EqualsNoCase("real")) {
-            features_.AddBack()->InitContinuous(portions[1]);
-          } else if (portions[2].EqualsNoCase("integer")) {
-            features_.AddBack()->InitInteger(portions[1]);
           } else {
-            reader->Error(
-                "ARFF: Only support 'numeric', 'real', and {nominal}.");
-            result = SUCCESS_FAIL;
+            String type(portions[2]);
+            //portions[2].Trim(" \t", &type);
+            if (type.EqualsNoCase("numeric")
+                || type.EqualsNoCase("real")) {
+              features_.AddBack()->InitContinuous(portions[1]);
+            } else if (type.EqualsNoCase("integer")) {
+              features_.AddBack()->InitInteger(portions[1]);
+            } else {
+              reader->Error(
+                  "ARFF: Only support 'numeric', 'real', and {nominal}.");
+              result = SUCCESS_FAIL;
+            }
           }
         }
       } else if (portions[0].EqualsNoCase("@data")) {
@@ -353,6 +358,10 @@ success_t DatasetInfo::ReadMatrix(TextLineReader *reader, Matrix *matrix) const
       break;
     }
     
+    while (*pos == ' ' || *pos == '\t' || *pos == ',') {
+      pos++;
+    }
+    
     if (*pos != '\0') {
       for (char *s = reader->Peek().begin(); s < pos; s++) {
         if (*s == '\0') {
@@ -518,3 +527,16 @@ void Dataset::SplitTrainTest(int folds, int fold_number,
   DEBUG_ASSERT(i_train == train->n_points());
   DEBUG_ASSERT(i_test == test->n_points());
 }
+
+void data::Load(const char *fname, Matrix *matrix) {
+  Dataset dataset;
+  dataset.InitFromFile(fname);
+  matrix->Own(&dataset.matrix());
+}
+
+void data::Save(const char *fname, const Matrix& matrix) {
+  Dataset dataset;
+  dataset.AliasMatrix(matrix);
+  dataset.WriteCsv(fname);
+}
+

fastlib/data/dataset.h

@@ -545,8 +545,8 @@ class Dataset {
   /**
    * Creates a training and test dataset for k-fold cross validation.
    *
-   * The training set will be approximately n_points() / folds, and the
-   * test set will be all remaining points.  This takes as an argument
+   * The test set will be approximately n_points() / folds, and the
+   * training set will be all remaining points.  This takes as an argument
    * a permutation to allow use of consistent random permutations.  If
    * an identity permutation is used, the split will be performed strided.
    *
@@ -562,4 +562,40 @@ class Dataset {
       Dataset *train, Dataset *test) const;
 };
 
+/**
+ * Miscellaneous dataset-related routines.
+ */
+namespace data {
+  /**
+   * Loads a matrix from a file.
+   *
+   * This supports any type the Dataset class supports with the
+   * InitFromFile function: CSV and ARFF.
+   *
+   * @code
+   * Matrix A;
+   * data::Load("foo.csv", &A);
+   * @endcode
+   *
+   * @param fname the file name to load
+   * @param matrix a pointer to an uninitialized matrix to load
+   */
+  void Load(const char *fname, Matrix *matrix);
+  /**
+   * Saves a matrix to a file.
+   *
+   * This saves in CSV format that MATLAB and Excel can handle.
+   *
+   * @code
+   * Matrix matrix_to_save;
+   * ... matrix_to_save contains the values you want to save
+   * data::Save("mymatrix.csv", matrix_to_save);
+   * @endcode
+   *
+   * @param fname the file name to load
+   * @param matrix a pointer to an uninitialized matrix to load
+   */
+  void Save(const char *fname, const Matrix& matrix);
+};
+
 #endif

fastlib/data/dataset_test.cc

@@ -1,8 +1,117 @@
 #include "dataset.h"
 
-#include "xrun/xrun.h"
 #include "math/discrete.h"
+#include "base/test.h"
 
+TEST_SUITE_BEGIN(dataset)
+
+void TestSplitTrainTest() {
+  Dataset orig;
+  
+  orig.InitBlank();
+  orig.matrix().Init(1, 12);
+  orig.info().InitContinuous(1);
+  
+  for (int i = 0; i < 12; i++) {
+    orig.matrix().set(0, i, i);
+  }
+  
+  ArrayList<int> found;
+  
+  found.Init(12);
+  
+  for (int i = 0; i < 12; i++) {
+    found[i] = 0;
+  }
+  
+  Dataset train;
+  Dataset test;
+  
+  ArrayList<index_t> permutation;
+  
+  math::MakeIdentityPermutation(12, &permutation);
+  
+  orig.SplitTrainTest(5, 1, permutation,
+      &train, &test);
+  
+  DEBUG_ASSERT(test.n_points() == 3);
+  DEBUG_ASSERT(train.n_points() == 9);
+  DEBUG_ASSERT_MSG(test.get(0, 0) == 1, "%f", (test.get(0, 0)));
+  DEBUG_ASSERT_MSG(test.get(0, 1) == 6, "%f", (test.get(0, 1)));
+  DEBUG_ASSERT_MSG(test.get(0, 2) == 11, "%f", (test.get(0, 2)));
+  
+  DEBUG_ASSERT_MSG(train.get(0, 0) == 0, "%f", (train.get(0, 0)));
+  DEBUG_ASSERT_MSG(train.get(0, 1) == 2, "%f", (train.get(0, 1)));
+  DEBUG_ASSERT_MSG(train.get(0, 2) == 3, "%f", (train.get(0, 2)));
+  DEBUG_ASSERT_MSG(train.get(0, 3) == 4, "%f", (train.get(0, 3)));
+  DEBUG_ASSERT_MSG(train.get(0, 4) == 5, "%f", (train.get(0, 4)));
+  DEBUG_ASSERT_MSG(train.get(0, 5) == 7, "%f", (train.get(0, 5)));
+  DEBUG_ASSERT_MSG(train.get(0, 6) == 8, "%f", (train.get(0, 6)));
+  DEBUG_ASSERT_MSG(train.get(0, 7) == 9, "%f", (train.get(0, 7)));
+  DEBUG_ASSERT_MSG(train.get(0, 8) == 10, "%f", (train.get(0, 8)));
+}
+
+void AssertSameMatrix(const Matrix& a, const Matrix& b) {
+  index_t r = a.n_rows();
+  index_t c = a.n_cols();
+  
+  TEST_ASSERT(a.n_rows() == b.n_rows());
+  TEST_ASSERT(a.n_cols() == b.n_cols());
+  
+  for (index_t ri = 0; ri < r; ri++) {
+    for (index_t ci = 0; ci < c; ci++) {
+      DEBUG_ASSERT_MSG(a.get(ri, ci) == b.get(ri, ci), "(%d, %d): %f != %f",
+          ri, ci, a.get(ri, ci), b.get(ri, ci));
+    }
+  }
+}
+
+void TestLoad() {
+  Dataset d1;
+  Dataset d2;
+  Dataset d3;
+  Dataset d4;
+  Dataset d5;
+  
+  ASSERT_PASS(d1.InitFromFile("test/fake.arff"));
+  ASSERT_PASS(d2.InitFromFile("test/fake.csv"));
+  ASSERT_PASS(d3.InitFromFile("test/fake.csvh"));
+  ASSERT_PASS(d4.InitFromFile("test/fake.tsv"));
+  ASSERT_PASS(d5.InitFromFile("test/fake.weird"));
+  
+  AssertSameMatrix(d1.matrix(), d2.matrix());
+  AssertSameMatrix(d1.matrix(), d3.matrix());
+  AssertSameMatrix(d1.matrix(), d4.matrix());
+  AssertSameMatrix(d1.matrix(), d5.matrix());
+}
+
+void TestStoreLoad() {
+  Dataset d1;
+  Dataset d2;
+  Dataset d3;
+  
+  ASSERT_PASS(d1.InitFromFile("test/fake.arff"));
+  d1.WriteCsv("test/fakeout1.csv");
+  d1.WriteArff("test/fakeout1.arff");
+  
+  ASSERT_PASS(d2.InitFromFile("test/fakeout1.arff"));
+  ASSERT_PASS(d3.InitFromFile("test/fakeout1.csv"));
+  
+  AssertSameMatrix(d1.matrix(), d2.matrix());
+  AssertSameMatrix(d1.matrix(), d3.matrix());
+  
+  DEBUG_ASSERT_MSG(strcmp(d1.info().name(), d2.info().name()) == 0,
+      "%s != %s", d1.info().name(), d2.info().name());
+  for (index_t i = 0; i < d1.info().n_features(); i++) {
+    DEBUG_ASSERT(
+      strcmp(d1.info().feature(i).name(), d2.info().feature(i).name()) == 0);
+    DEBUG_ASSERT(d1.info().feature(i).type() == d2.info().feature(i).type());
+  }
+}
+
+TEST_SUITE_END(dataset, TestSplitTrainTest, TestLoad, TestStoreLoad)
+
+/*
 int main(int argc, char *argv[]) {
   xrun_init(argc, argv);
   const char *in = xrun_param_str("in");
@@ -45,3 +154,4 @@ int main(int argc, char *argv[]) {
 
   return 0;
 }
+*/

fastlib/fastlib/build.py

@@ -7,13 +7,3 @@ librule(
               #, "tree:tree", "par:par"
              ]
     )
-
-librule(
-    name = "fastlib_int",
-    headers = ["fastlib.h"],
-    deplibs = ["la:la", "base:base",
-             "fx:fx", "file:file_int", "col:col",
-             "data:data", "math:math"
-              "tree:tree", "par:par"
-             ]
-    )

fastlib/fastlib/fastlib.h

@@ -45,12 +45,15 @@
  * comment header files and not source files -- even without Doxygen, header
  * files remain a natural place for documentation.
  *
- * To generate the doxygen yourself, go in to the code directory and type:
+ * To generate this HTML documentation yourself, go in to the code directory
+ * and type:
  *
  * @code
  * doxygen
  * @endcode
  *
+ * Then, visit doc/html/index.html (within the code directory).
+ *
  * @section remarks Final Remarks
  *
  * This software was written at Georgia Institute of Technology.

fastlib/file/build.py

@@ -5,13 +5,6 @@ librule(
     deplibs = ["base:base", "col:col"],
     )
 
-librule( # internal rule
-    name = "file_int",
-    sources = ["serialize.cc"],
-    headers = ["serialize.h"],
-    deplibs = [":file"],
-    )
-
 binrule(
     name = "textfile_test",
     sources = ["textfile_test.cc"],

fastlib/file/textfile.h

@@ -328,7 +328,7 @@ class TextWriter {
   }
   
   /**
-   * Opens a file by name.
+   * Opens a file by name (initializer).
    *
    * @return success or failure
    */

fastlib/file/textfile_test.cc

@@ -3,7 +3,7 @@
 
 #include "xrun/xrun.h"
 
-#include <cmath>
+#include <math.h>
 
 void Test1() {
   TextTokenizer scanner;

fastlib/fx/fx.h

@@ -49,7 +49,7 @@ void fx_done(void);
  */
 int fx_param_exists(struct datanode *module, const char *name);
 /**
- * Obtain a string parameter.
+ * Obtain a string parameter or use provided default.
  *
  * @param module the param's containing module, or NULL for global
  * @param name the name of the parameter (paths allowed)
@@ -58,41 +58,75 @@ int fx_param_exists(struct datanode *module, const char *name);
 const char *fx_param_str(struct datanode *module, const char *name,
 			 const char *def);
 /**
- * Obtain a floating-point parameter.
+ * Obtain a string parameter, failing if it is not specified.
+ *
+ * @param module the param's containing module, or NULL for global
+ * @param name the name of the parameter (paths allowed)
+ */
+const char *fx_param_str_req(struct datanode *module, const char *name);
+/**
+ * Obtain a floating-point parameter or use provided default.
  *
  * If the parameter is not specified by the user and the default is not
  * DBL_NAN, then the default value will be stored explicitly in the tree.
  *
  * @param module the param's containing module, or NULL for global
  * @param name the name of the parameter (paths allowed)
- * @param def value used if param not given, or DBL_NAN if required
+ * @param def value used if param not given
  */
-double fx_param_double(struct datanode *module, const char *name,
-		       double def);
+double fx_param_double(struct datanode *module, const char *name, double def);
 /**
- * Obtain an integral parameter.
+ * Obtain a floating-point parameter, failing if it is not specified.
+ *
+ * If the parameter is not specified by the user and the default is not
+ * DBL_NAN, then the default value will be stored explicitly in the tree.
+ *
+ * @param module the param's containing module, or NULL for global
+ * @param name the name of the parameter (paths allowed)
+ */
+double fx_param_double_req(struct datanode *module, const char *name);
+/**
+ * Obtain an integral parameter or use provided default.
  *
  * If the parameter is not specified by the user and the default is not -1,
  * then the default value will be stored explicitly in the tree.
  *
  * @param module the param's containing module, or NULL for global
  * @param name the name of the parameter (paths allowed)
- * @param def value used if param not given, or -1 if required
+ * @param def value used if param not given
  */
-int fx_param_int(struct datanode *module, const char *name,
-		 int def);
+int fx_param_int(struct datanode *module, const char *name, int def);
 /**
- * Obtain a boolean parameter.
+ * Obtain an integral parameter, failing if it is not specified.
+ *
+ * If the parameter is not specified by the user and the default is not -1,
+ * then the default value will be stored explicitly in the tree.
+ *
+ * @param module the param's containing module, or NULL for global
+ * @param name the name of the parameter (paths allowed)
+ */
+int fx_param_int_req(struct datanode *module, const char *name);
+/**
+ * Obtain a boolean parameter or use provided default.
  *
  * Values starting with f, F, n, N, or 0 are false; all others are
  * true.
  *
  * @param module the param's containing module, or NULL for global
  * @param name the name of the parameter (paths allowed)
- * @param def value used if param not given, or NULL if required
+ * @param def value used if param not given
  */
-int fx_param_bool(struct datanode *module, const char *name,
-		  const char *def);
+int fx_param_bool(struct datanode *module, const char *name, int def);
+/**
+ * Obtain a boolean parameter, failing if it is not specified.
+ *
+ * Values starting with f, F, n, N, or 0 are false; all others are
+ * true.
+ *
+ * @param module the param's containing module, or NULL for global
+ * @param name the name of the parameter (paths allowed)
+ */
+int fx_param_bool_req(struct datanode *module, const char *name);
 /**
  * Obtain a segment of the datastore corresponding to a parameter.
  *
@@ -125,7 +159,8 @@ struct datanode *fx_param_node(struct datanode *module, const char *name);
  * @param name the name of the parameter (paths allowed)
  * @param def the value the parameter assumes if unspecified
  */
-void fx_def_param(struct datanode *module, const char *name, const char *def);
+void fx_default_param(struct datanode *module, const char *name,
+		      const char *def);
 /**
  * Set a parameter to a given value.
  *
@@ -135,7 +170,8 @@ void fx_def_param(struct datanode *module, const char *name, const char *def);
  * @param name the name of the parameter (paths allowed)
  * @param format a format string for the parameter, as in printf
  */
-void fx_set_param(struct datanode *module, const char *name, const char *val);
+void fx_set_param(struct datanode *module, const char *name,
+		  const char *val);
 /**
  * Set a parameter to a formatted string.
  *
@@ -172,9 +208,8 @@ void fx_clear_param(struct datanode *module, const char *name);
  * @param src_module the module to copy it from
  * @param srcname the name under the module's params to copy it from
  */
-void fx_def_param_node(struct datanode *dest_module, const char *destname,
-		       struct datanode *src_module, const char *srcname);
-
+void fx_default_param_node(struct datanode *dest_module, const char *destname,
+			   struct datanode *src_module, const char *srcname);
 /**
  * Copy parameters from one module to another, overwriting.
  *
@@ -265,7 +300,7 @@ void fx_timer_stop(struct datanode *module, const char *name);
  * before they are used.
  *
  * @code
- * struct datanode *tb_mod = fx_module(NULL, "tree_bldg", "tree_building");
+ * struct datanode *tb_mod = fx_module(NULL, "tree_building", "tree_bldg");
  * @endcode
  *
  * Any existing parameters (e.g. if the submodule is not fresh) are
@@ -273,12 +308,13 @@ void fx_timer_stop(struct datanode *module, const char *name);
  * forwarding.
  *
  * @param module the containing module of the submodule
- * @param name the name of the submodule (paths allowed)
- * @param params_path_template the param node to forward (paths allowed)
- *        or NULL, printf-style
+ * @param params the param node to forward (paths allowed) or NULL
+ * @param name_format the name of the submodule (paths allowed);
+ *        formatted as in printf
  */
-struct datanode *fx_submodule(struct datanode *module, const char *name,
-			      const char *params_path_template, ...);
+COMPILER_PRINTF(3, 4)
+struct datanode *fx_submodule(struct datanode *module, const char *param,
+			      const char *name_format, ...);
 
 EXTERN_C_END
 

fastlib/la/build.py

@@ -1,25 +1,54 @@
+# This first part is dedicated to compiling and installing LAPACK.
+# Scroll down later...
 
-class LaProtoRule(dep.Rule):
-  def __init__(self):
-    self.script = sourcerule(Types.SCRIPT, "proto.py")
-    self.sources = [] # TODO: Include files this depends on
-    dep.Rule.__init__(self, script=[self.script], sources=self.sources)
-  def doit(self, sysentry, state, files, params):
-    script = files["script"].single(Types.SCRIPT)
-    proto_header = sysentry.file("la/proto.h", "arch", "kernel", "compiler")
-    sysentry.command("python %s --outfile=%s" % (script.name, proto_header.name))
-    gen_headers = ["la/proto.h"]
-    return [(Types.HEADER, proto_header)]
+wgetrule(
+    name = "blaspack_tgz",
+    type = Types.ANY,
+    url = "http://www.cc.gatech.edu/~garryb/fastlib/blaspack.tgz")
+
+def doit_compile_lapack(sysentry, files, params):
+  blaspack_tgz = files["blaspack_tgz"].single(Types.ANY)
+  libblaspack = sysentry.file("KEEP/libblaspack.a", "arch", "kernel", "compiler")
+  workspace_dir = os.path.join(os.path.dirname(libblaspack.name), "libblaspack_workspace")
+  compiler_info = compilers[params["compiler"]]
+  compiler = compiler_info.compiler_program("f")
+  # Make sure we won't rm -rf anything bad
+  assert "libblaspack_workspace" in workspace_dir
+  sysentry.command("mkdir -p %s" % workspace_dir)
+  sysentry.command("cd %s && tar -xzf %s" % (workspace_dir, blaspack_tgz.name))
+  sysentry.command("echo '*** Compiling LAPACK with BLAS reference implementation.'")
+  sysentry.command("echo '... Our compilation differs slightly from regular LAPACK/BLAS:'")
+  sysentry.command("echo '... NOTE 1: We omit complex-number routines (halves compile time).'")
+  sysentry.command("echo '... NOTE 2: We require case sensitivity for LAPACK/BLAS string parameters.'")
+  sysentry.command("echo '... This may take several minutes (about 800 FORTRAN files).'")
+  # Loop unrolling is not useful on modern architectures (and bloats EXE size).
+  # Thus, we only compile -O2.
+  sysentry.command("cd %s && %s -O2 -c src/*.f" % (workspace_dir, compiler))
+  sysentry.command("echo '... Almost done with LAPACK/BLAS...'")
+  noopt = "dlamch slamch" # these have to be compiled without optimization
+  noopt_real = " ".join(["src/%s.f" % x for x in noopt.split()])
+  sysentry.command("cd %s && %s -O0 -c %s" % (workspace_dir, compiler, noopt_real))
+  sysentry.command("cd %s && ar r %s *.o" % (workspace_dir, libblaspack.name))
+  sysentry.command("echo '... Created archive, cleaning up.'")
+  sysentry.command("rm -rf %s" % workspace_dir)
+  sysentry.command("echo '*** Done with LAPACK and BLAS!'")
+  return [(Types.LINKABLE, libblaspack)]
+
+customrule(
+    name = "libblaspack",
+    dependencies = {"blaspack_tgz": [find(":blaspack_tgz")]},
+    doit_fn = doit_compile_lapack)
+
+#---- This is the main part of LA package
 
 librule(
-    sources = [],
-    headers = ["matrix.h", "la.h"],
-    deplibs = ["base:base", "col:col"])
+    sources = ["uselapack.cc"],
+    headers = ["matrix.h", "la.h", "uselapack.h", "clapack.h", "blas.h"],
+    deplibs = ["base:base", "col:col", ":libblaspack"])
 
 binrule(
-    name = "lapack_test",
-    sources = ["lapack.cc"],
-    headers = [LaProtoRule()],
+    name = "uselapack_test",
+    sources = ["uselapack_test.cc"],
     linkables = [":la"])
 
 binrule(

fastlib/la/la.h

@@ -3,26 +3,21 @@
 /**
  * @file la.h
  *
- * Core routines for la algebra.
+ * Core routines for linear algebra.
  *
- * TODO: Currently lacking much, and eventually we want this to simply
- * use BLAS and LAPACK.
+ * See uselapack.h for more linear algebra routines.
  */
 
 #ifndef LINEAR_H
 #define LINEAR_H
 
 #include "matrix.h"
+#include "uselapack.h"
 
 #include "base/scale.h"
 
-#include <cmath>
+#include <math.h>
 
-/**
- * Namespace for linear-algebra helper routines.
- *
- * Will eventually include BLAS and LAPACk support.
- */
 namespace la {
   /**
    * Finds the Euclidean distance squared between two vectors.
@@ -47,193 +42,6 @@ namespace la {
     DEBUG_ASSERT(x.length() == y.length());
     return DistanceSqEuclidean(x.length(), x.ptr(), y.ptr());
   }
-  
-  
-  /* these will be replaced with lapack eventually */
-#ifndef LAPACK
-  /**
-   * Finds the dot product of two vectors.
-   */
-  inline double VectorDot(index_t length, const double *x, const double *y) {
-    double sum = 0;
-    for (index_t i = 0; i < length; i++) {
-      sum += x[i] * y[i];
-    }
-    return sum;
-  }
-  /**
-   * Finds the dot product of two vectors.
-   */
-  inline double VectorDot(const Vector& x, const Vector& y) {
-    DEBUG_ASSERT(x.length() == y.length());
-    return VectorDot(x.length(), x.ptr(), y.ptr());
-  }
-  
-  //---
-
-  /**
-   * Adds one vector to another (x <- x + y);
-   */
-  inline void VectorAddTo(index_t length, const double *x, double *y) {
-    do {
-      *y += *x;
-      x++;
-      y++;
-    } while (--length);
-  }
-  
-  /**
-   * Adds one vector to another (x <- x + y);
-   */
-  inline void VectorAddTo(const Vector&x, Vector *y) {
-    DEBUG_ASSERT(x.length() == y->length());
-    VectorAddTo(x.length(), x.ptr(), y->ptr());
-  }
-
-  /**
-   * Adds one matrix to another (X <- X + Y);
-   */
-  inline void MatrixAddTo(const Matrix& x, Matrix *y) {
-    DEBUG_ASSERT(x.n_rows() == y->n_rows());
-    DEBUG_ASSERT(x.n_cols() == y->n_cols());
-    VectorAddTo(x.n_elements(), x.ptr(), y->ptr());
-  }
-
-  //---
-
-  /**
-   * Adds a vector times a factor to an existing vector (y <- y + xscale * x).
-   */
-  inline void VectorAddTo(index_t length,
-      double xscale, const double *x, double *y) {
-    do {
-      *y += xscale * (*x);
-      x++;
-      y++;
-    } while (--length);
-  }
-
-  /**
-   * Adds a vector times a factor to an existing vector (y <- y + xscale * x).
-   */
-  inline void VectorAddTo(double xscale, const Vector&x, Vector *y) {
-    DEBUG_ASSERT(x.length() == y->length());
-    VectorAddTo(x.length(), xscale, x.ptr(), y->ptr());
-  }
-
-  /**
-   * Adds a matrix times a factor to an existing matrix (Y <- Y + xscale * X).
-   */
-  inline void MatrixAddTo(double xscale, const Matrix& x, Matrix *y) {
-    DEBUG_ASSERT(x.n_rows() == y->n_rows());
-    DEBUG_ASSERT(x.n_cols() == y->n_cols());
-    VectorAddTo(x.n_elements(), xscale, x.ptr(), y->ptr());
-  }
-  
-  //---
-
-  /**
-   * Adds two vectors to a new vector (z <- x + y).
-   */
-  inline void VectorAddOverwrite(index_t length,
-      const double *x, const double *y, double *c) {
-    for (index_t i = 0; i < length; i++) {
-      c[i] = x[i] + y[i];
-    }
-  }
-  
-  /**
-   * Adds two vectors to a new vector (z <- x + y).
-   * @param z will be Initialized to the sum of x and y
-   */
-  inline void VectorAddInit(
-      const Vector& x, const Vector& y, Vector *z) {
-    DEBUG_ASSERT(x.length() == y.length());
-    z->Init(x.length());
-    VectorAddOverwrite(x.length(), x.ptr(), y.ptr(), z->ptr());
-  }
-  
-  /**
-   * Adds two matrices to a new matrix (Z <- X + Y).
-   * @param z will be Initialized to the sum of x and y
-   */
-  inline void MatrixAddInit(
-      const Matrix& x, const Matrix& y, Matrix *z) {
-    DEBUG_ASSERT(x.n_rows() == y.n_rows());
-    DEBUG_ASSERT(x.n_cols() == y.n_cols());
-    z->Init(x.n_rows(), x.n_cols());
-    VectorAddOverwrite(x.n_elements(), x.ptr(), y.ptr(), z->ptr());
-  }
-
-  //---
-  
-  /**
-   * Subtracts two vectors into a third (z <- x - y).
-   */
-  inline void VectorSubOverwrite(index_t length,
-      const double *x, const double *y, double *z) {
-    for (index_t i = 0; i < length; i++) {
-      z[i] = x[i] - y[i];
-    }
-  }
-  
-  /**
-   * Subtracts two vectors into a third (z <- x - y).
-   * @param z will be Initialized to the sum of x and y
-   */
-  inline void VectorSubInit(
-      const Vector& x, const Vector& y, Vector *z) {
-    DEBUG_ASSERT(x.length() == y.length());
-    z->Init(x.length());
-    VectorSubOverwrite(x.length(), x.ptr(), y.ptr(), z->ptr());
-  }
-
-  //---
-
-  /**
-   * Subtracts a vector from an existing vector (y <- y - x).
-   */
-  inline void VectorSubFrom(index_t length, const double *x,
-      double *y) {
-    for (index_t i = 0; i < length; i++) {
-      y[i] -= x[i];
-    }
-  }
-    
-  /**
-   * Subtracts a vector from an existing vector (y <- y - x).
-   */
-  inline void VectorSubFrom(
-      const Vector& x, Vector *y) {
-    DEBUG_ASSERT(x.length() == y->length());
-    VectorSubFrom(x.length(), x.ptr(), y->ptr());
-  }
-
-  //---
-  
-  /**
-   * Multiplies a vector in-place by a scale (x <- alpha * x).
-   */
-  inline void VectorScale(index_t length, double alpha, double *x) {
-    do {
-      *x++ *= alpha;
-    } while (--length);
-  }
-
-  /**
-   * Multiplies a vector in-place by a scale (x <- alpha * x).
-   */
-  inline void VectorScale(double alpha, Vector *x) {
-    VectorScale(x->length(), alpha, x->ptr());
-  }
-
-  /**
-   * Multiplies a matrix in-place by a scale (X <- alpha * X).
-   */
-  inline void MatrixScale(double alpha, Matrix *x) {
-    VectorScale(x->n_elements(), alpha, x->ptr());
-  }
-#endif
 };
 
 #endif

fastlib/la/matrix.h

@@ -16,7 +16,7 @@
 
 #include <stdlib.h>
 #include <string.h>
-#include <cmath>
+#include <math.h>
 
 /**
  * Double-precision vector for use with LAPACK.
@@ -236,7 +236,7 @@ class Vector {
   }
   
   /**
-   * Copies the values from another matrix to this matrix.
+   * Copies the values from another vector to this vector.
    *
    * @param other the vector to copy from
    */
@@ -245,32 +245,75 @@ class Vector {
     mem::Copy(ptr_, other.ptr_, length_);
   }
   
+  /**
+   * Prints to a stream as a debug message.
+   * @param name a name that will be printed with the vector
+   */
+  void PrintDebug(const char *name = "", FILE *stream = stderr) const {
+    fprintf(stream, "----- VECTOR %s ------\n", name);
+    for (index_t i = 0; i < length(); i++) {
+      fprintf(stream, "%+3.3f ", get(i));
+    }
+    fprintf(stream, "\n");
+  }
+  
  public:
+  /** The number of elements in this vector. */
   index_t length() const {
     return length_;
   }
   
+  /**
+   * A pointer to the C-style array containing the elements of this vector.
+   */
   double *ptr() {
     return ptr_;
   }
   
+  /**
+   * A pointer to the C-style array containing the elements of this vector.
+   */
   const double *ptr() const {
     return ptr_;
   }
   
+  /**
+   * Gets the i'th element of this vector.
+   */
   double operator [] (index_t i) const {
     DEBUG_BOUNDS(i, length_);
     return ptr_[i];
   }
   
+  /**
+   * Gets a mutable reference to the i'th element of this vector.
+   */
   double &operator [] (index_t i) {
     DEBUG_BOUNDS(i, length_);
     return ptr_[i];
   }
   
+  /**
+   * Gets a value to the i'th element of this vector (convenient when
+   * you have a pointer to a vector).
+   *
+   * This is identical to the array subscript operator, except for the
+   * following reason:
+   *
+   * @code
+   * void FooBar(Vector *v) {
+   *    v->get(0) // much easier to read than (*v)[0]
+   * }
+   * @endcode
+   */
+  double get(index_t i) const {
+    DEBUG_BOUNDS(i, length_);
+    return ptr_[i];
+  }
+  
  private:
   void AssertUninitialized_() const {
-    DEBUG_ASSERT(length_ == BIG_BAD_NUMBER);
+    DEBUG_ASSERT_MSG(length_ == BIG_BAD_NUMBER, "Cannot re-init vectors.");
   }
   
   void Uninitialize_() {
@@ -380,6 +423,19 @@ class Matrix {
     // zero
     SetAll(0);
   }
+  
+  /**
+   * Makes this a diagonal matrix whose diagonals are the values in v.
+   */
+  void SetDiagonal(const Vector& v) {
+    DEBUG_ASSERT(n_rows() == v.length());
+    DEBUG_ASSERT(n_cols() == v.length());
+    SetZero();
+    index_t n = v.length();
+    for (index_t i = 0; i < n; i++) {
+      set(i, i, v[i]);
+    }
+  }
 
   /**
    * Makes this uninitialized matrix a copy of the other vector.
@@ -434,6 +490,24 @@ class Matrix {
     should_free_ = false;
   }
   
+  /**
+   * Makes this a 1 row by N column alias of a vector of length N.
+   *
+   * @param row_vector the vector to alias
+   */
+  void AliasRowVector(const Vector& row_vector) {
+    Alias(const_cast<double*>(row_vector.ptr()), 1, row_vector.length());
+  }
+  
+  /**
+   * Makes this an N row by 1 column alias of a vector of length N.
+   *
+   * @param col_vector the vector to alias
+   */
+  void AliasColVector(const Vector& col_vector) {
+    Alias(const_cast<double*>(col_vector.ptr()), col_vector.length(), 1);
+  }
+  
   /**
    * Makes this a weak copy or alias of the other.
    *
@@ -571,12 +645,14 @@ class Matrix {
   }
   
   /**
-   * Reduces the number of columns, but REQUIRES that there are no aliases
+   * Changes the number of columns, but REQUIRES that there are no aliases
    * to this matrix anywhere else.
    *
+   * If the size is increased, the remaining space is not initialized.
+   *
    * @param new_n_cols the new number of columns
    */
-  void OwnerReduceColumns(index_t new_n_cols) {
+  void ResizeNoalias(index_t new_n_cols) {
     DEBUG_ASSERT(should_free_); // the best assert we can do
     n_cols_ = new_n_cols;
     ptr_ = mem::Resize(ptr_, n_elements());
@@ -596,6 +672,31 @@ class Matrix {
     mem::Swap(ptr_, other->ptr_, n_elements());
   }
   
+  /**
+   * Copies the values from another matrix to this matrix.
+   *
+   * @param other the vector to copy from
+   */
+  void CopyValues(const Matrix& other) {
+    DEBUG_ASSERT(n_rows() == other.n_rows());
+    DEBUG_ASSERT(n_cols() == other.n_cols());
+    mem::Copy(ptr_, other.ptr_, n_elements());
+  }
+
+  /**
+   * Prints to a stream as a debug message.
+   * @param name a name that will be printed with the matrix
+   */
+  void PrintDebug(const char *name = "", FILE *stream = stderr) const {
+    fprintf(stream, "----- MATRIX %s ------\n", name);
+    for (index_t r = 0; r < n_rows(); r++) {
+      for (index_t c = 0; c < n_cols(); c++) {
+        fprintf(stream, "%+3.3f ", get(r, c));
+      }
+      fprintf(stream, "\n");
+    }
+  }
+  
  public:
   /**
    * Returns a pointer to the very beginning of the matrix, stored
@@ -680,7 +781,7 @@ class Matrix {
   
  private:
   void AssertUninitialized_() const {
-    DEBUG_ASSERT(n_rows_ == BIG_BAD_NUMBER);
+    DEBUG_ASSERT_MSG(n_rows_ == BIG_BAD_NUMBER, "Cannot re-init matrices.");
   }
   
   void Uninitialize_() {

fastlib/math/discrete.h

@@ -12,7 +12,7 @@
 #include "base/cc.h"
 #include "col/arraylist.h"
 
-#include <cmath>
+#include <math.h>
 
 namespace math {
   /**

fastlib/math/geometry.cc

@@ -8,6 +8,7 @@
 
 #include "geometry.h"
 #include "discrete.h"
+#include "math.h"
 
 namespace math {
 

fastlib/math/geometry.h

@@ -11,11 +11,9 @@
 
 #include "base/cc.h"
 
-#include <cmath>
+#include <math.h>
 
 namespace math {
-  const double PI = 3.141592653589793238462643383279;
-  
   /**
    * Computes the hyper-volume of a hyper-sphere of dimension d.
    *

fastlib/math/kernel.h

@@ -11,8 +11,9 @@
 
 #include "base/common.h"
 #include "math/geometry.h"
+#include "math/math.h"
 
-#include <cmath>
+#include <math.h>
 
 /* More to come soon - Gaussian, Epanechnakov, etc. */
 

fastlib/math/math.h

@@ -9,14 +9,30 @@
 #ifndef MATH_MATH_H
 #define MATH_MATH_H
 
-#include "discrete.h"
-#include "kernel.h"
-#include "geometry.h"
-
 /**
- * Namespace with a variety of math routines.
+ * Math routines.
+ *
+ * The hope is that this should contain most of the useful math routines
+ * you can think of.  Currently, this is very sparse.
  */
 namespace math {
+  /** The square root of 2. */
+  const double SQRT2 = 1.41421356237309504880;
+  /** Base of the natural logarithm. */
+  const double E = 2.7182818284590452354;
+  /** Log base 2 of E. */
+  const double LOG2_E = 1.4426950408889634074;
+  /** Log base 10 of E. */
+  const double LOG10_E = 0.43429448190325182765;
+  /** Natural log of 2. */
+  const double LN_2 = 0.69314718055994530942;
+  /** Natural log of 10. */
+  const double LN_10 = 2.30258509299404568402;
+  /** The ratio of the circumference of a circle to its diameter. */
+  const double PI = 3.141592653589793238462643383279;
+  /** The ratio of the radius of a circle to its diameter. */
+  const double PI_2 = 1.57079632679489661923;
+
   /** Squares a number. */
   template<typename T>
   inline T Sqr(T v) {
@@ -42,4 +58,8 @@ namespace math {
   }
 };
 
+#include "discrete.h"
+#include "kernel.h"
+#include "geometry.h"
+
 #endif

fastlib/math/math_test.cc

@@ -5,16 +5,16 @@
 TEST_SUITE_BEGIN(math)
 
 void TestPermutation() {
-  ArrayList<index_t> p1;
-  ArrayList<index_t> p2;
+  ArrayList<index_t> p_rand;
+  ArrayList<index_t> p_idnt;
   ArrayList<int> visited;
   int n = 3111;
   
-  math::MakeRandomPermutation(n, &p1);
-  math::MakeIdentityPermutation(n, &p2);
+  math::MakeRandomPermutation(n, &p_rand);
+  math::MakeIdentityPermutation(n, &p_idnt);
   
   for (int i = 0; i < n; i++) {
-    TEST_ASSERT(p2[i] == i);
+    TEST_ASSERT(p_idnt[i] == i);
   }
   
   visited.Init(n);
@@ -24,7 +24,7 @@ void TestPermutation() {
   }
 
   for (int i = 0; i < n; i++) {
-    visited[p1[i]]++;
+    visited[p_rand[i]]++;
   }
 
   for (int i = 0; i < n; i++) {

fastlib/py/buildsys.py

@@ -1,444 +0,0 @@
-
-import dep
-import util
-import glob
-
-import os
-
-class Types:
-  HEADER="buildsys/.h"
-  GCC_SOURCE="buildsys/gcc"  # .c, .cpp, etc
-  OBJECT="buildsys/.o"
-  LINKABLE="buildsys/.a"
-  BINFILE="buildsys/bin"
-  SCRIPT="buildsys/script"
-  DIR="buildsys/dir"
-  PLACEHOLDER="buildsys/placeholder"
-  MISC="buildsys/misc"
-  ANY="buildsys/*"
-
-class CompilerInfo:
-  pass
-
-class GCCCompiler(CompilerInfo):
-  def __init__(self):
-    self.name = "gcc"
-    self.mode_dictionary = {
-      "verbose": "-g -DDEBUG -DVERBOSE",
-      "debug": "-g3 -DDEBUG",
-      "check": "-O2 -g -DDEBUG",
-      "fast": "-O2 -g -fomit-frame-pointer -DNDEBUG",
-      "unsafe": "-O3 -ffast-math -g -fomit-frame-pointer -DNDEBUG",
-      "profile" : "-O2 -pg -finline-limit=8 -DPROFILE -DNDEBUG",
-      "small": "-Os -DNDEBUG"
-    }
-    self.command_from_ext = {
-      "c" : "gcc %s -c %s -o %s -Wall",
-      "cc" : "g++ %s -c %s -o %s -Wall -Woverloaded-virtual -fno-exceptions -Wparentheses -fno-exceptions",
-      "f" : "g77 %s -c %s -o %s -Wall -Wno-uninitialized"
-    }
-    self.linker = "g++"
-    self.lflags_start = "-lg2c -lm -lpthread -Wl,-whole-archive"
-    self.lflags_end = "-Wl,-no-whole-archive"
-
-class ICCCompiler(CompilerInfo):
-  def __init__(self):
-    self.name = "icc"
-    self.mode_dictionary = {
-      "verbose": "-g -DDEBUG -DVERBOSE",
-      "debug": "-g -DDEBUG",
-      "check": "-O2 -g -DDEBUG",
-      "fast": "-O3 -g -fomit-frame-pointer -DNDEBUG",
-      "unsafe": "-O3 -ffast-math -g -fomit-frame-pointer -DNDEBUG",
-      "profile" : "-O2 -pg -finline-limit=8 -DPROFILE -DNDEBUG",
-      "small": "-Os"
-    }
-    self.command_from_ext = {
-      "c" : "icc %s -c %s -o %s",
-      "cc" : "icpc %s -c %s -o %s -fno-exceptions",
-      "f" : "ifort %s -c %s -o %s -Wall"
-    }
-    self.linker = "icpc"
-    self.lflags_start = "-lm -lpthread -Wl,-whole-archive"
-    self.lflags_end = "-Wl,-no-whole-archive"
-
-class MPICompiler(CompilerInfo):
-  def __init__(self):
-    self.name = "mpi"
-    self.mode_dictionary = {
-      "verbose": "-g -DDEBUG -DVERBOSE",
-      "debug": "-g -DDEBUG",
-      "check": "-O2 -g -DDEBUG",
-      "fast": "-O3 -g -fomit-frame-pointer -DNDEBUG",
-      "unsafe": "-O3 -ffast-math -g -fomit-frame-pointer -DNDEBUG",
-      "profile" : "-O2 -pg -finline-limit=8 -DPROFILE -DNDEBUG",
-      "small": "-Os"
-    }
-    self.command_from_ext = {
-      "c" : "mpicc %s -c %s -o %s",
-      "cc" : "mpiCC -fno-exceptions %s -c %s -o %s",
-      "f" : "mpif77 %s -c %s -o %s -Wall"
-    }
-    self.linker = "mpiCC"
-    self.lflags_start = "-lm -lpthread -Wl,-whole-archive"
-    self.lflags_end = "-Wl,-no-whole-archive"
-    #self.lflags_start = "-lm -lpthread "
-    #self.lflags_end = ""
-
-compiler_choices = [GCCCompiler(), MPICompiler(), ICCCompiler()]
-compilers = dict([(c.name, c) for c in compiler_choices])
-
-class MakeBuildSys(dep.DepSys):
-  """Build system.
-  """
-  def __init__(self, source_dir, bin_dir):
-    dep.DepSys.__init__(self)
-    self.source_dir = source_dir
-    self.bin_dir = bin_dir
-    self.entries = []
-  def begin(self, state):
-    return MakeBuildSysEntry(self, state)
-  def add_entry(self, entry):
-    self.entries.append(entry)
-  def to_makefile(self):
-    def shorten(fname):
-      if "/bin/" in fname:
-        return fname[fname.rindex("/bin/"):]
-      else:
-        return fname
-    lines = []
-    self.entries.reverse()
-    i = 0
-    for (outfiles, infiles, commands) in self.entries:
-      i += 1
-      if commands:
-        lines.append("%s: %s" % (" ".join(outfiles), " ".join(infiles)))
-        outfiles_short = [shorten(outfile) for outfile in outfiles]
-        lines.extend(["\t@echo '... Making %s'" % (" ".join(outfiles_short))])
-        lines.extend(["\t@" + c for c in commands])
-      elif infiles:
-        lines.append("pseudo_%d: %s" % (i, " ".join(infiles)))
-        lines.extend(["\t@echo '*** Done with %s'" % (" ".join(infiles))])
-    self.entries.reverse()
-    lines.append("clean:")
-    lines.append("\trm -rf %s" % (self.bin_dir))
-    return lines
-
-class MakeBuildSysEntry(dep.DepSysEntry):
-  def __init__(self, sys, state):
-    dep.DepSysEntry.__init__(self, sys, state)
-    self.commands = []
-  def _make_name(self, simplename, parameterization):
-    # NOTE: I'm completely ignoring the file list (self.state.files)
-    dirname = self.bin_dir(*parameterization)
-    return os.path.join(dirname, simplename)
-  def bin_dir(self, *pmz):
-    def name_part(param_name):
-      if param_name in pmz:
-        return self.state.params[param_name]
-      else:
-        return "COMMON"
-
-    allowed_sequence = ["arch", "kernel", "mode", "compiler"]
-    (arch, kernel, mode, compiler) = [name_part(x) for x in allowed_sequence]
-    
-    for param in pmz:
-      if not param in allowed_sequence:
-        raise Exception("I can't deal with extra parameterization yet: '%s' not in '%s'" % (param, allowed_sequence))
-    
-    return os.path.join(self.sys.bin_dir, "%s_%s_%s_%s" % (arch, kernel, mode, compiler))
-  def source_file(self, real_path, fake_path):
-    return dep.DestFile(real_path, fake_path)
-  def makefile(self):
-    return self.source_file(os.path.abspath("./Makefile"), "Makefile")
-  def ensure_writable(self, name):
-    self.command("mkdir -p %s" % (os.path.dirname(name)))
-  def command(self, str):
-    self.commands.append(str)
-  def end(self, files):
-    infiles = {}
-    for file in dep.filemap_to_files(self.state.files):
-      infiles[file.name] = None
-    outfiles = [file.name for (classname, file) in files]
-    self.sys.add_entry((outfiles, list(infiles.keys()), self.commands))
-
-class SourceRule(dep.Rule):
-  def __init__(self, type, real_path, fake_path):
-    dep.Rule.__init__(self)
-    self.real_path = real_path
-    self.fake_path = fake_path
-    self.type = type
-  def doit(self, sysentry, state, files, params):
-    return [(self.type, sysentry.source_file(self.real_path, self.fake_path))]
-
-class CompileRule(dep.Rule):
-  def __init__(self, source, headers, cflags):
-    dep.Rule.__init__(self, source=[source], headers=headers)
-    self.cflags = cflags
-  def doit(self, sysentry, state, files, params):
-    compiler = compilers[params["compiler"]]
-    source = files["source"].single(Types.GCC_SOURCE)
-    dot = source.simplename.rindex(".")
-    sourceextension = source.simplename[dot+1:]
-    simplename = source.simplename[:dot] + ".o"
-    object = sysentry.file("obj/" + simplename.replace("/", "_"), "arch", "kernel", "mode", "compiler")
-    # TODO: -I flags
-    my_includes = "-I%s -I%s" % (sysentry.bin_dir("arch", "kernel", "compiler"),
-        sysentry.sys.source_dir)
-    mode = params["mode"]
-    my_flags = my_includes + " " + compiler.mode_dictionary[params["mode"]] + " " + self.cflags
-    if not sourceextension in compiler.command_from_ext:
-      raise Exception("Don't know how to compile files of type [%s]." % sourceextension)
-    command_template = compiler.command_from_ext[sourceextension]
-    (source_dirname, source_basename) = os.path.split(source.name)
-    compile_cmd = command_template % (my_flags, source_basename, object.name)
-    sysentry.command("cd " + source_dirname + " && " + compile_cmd)
-    return [(Types.OBJECT, object)]
-
-class ArchiveRule(dep.Rule):
-  def __init__(self, name, objects):
-    dep.Rule.__init__(self, objects=objects)
-    self.name = name
-  def doit(self, sysentry, state, files, params):
-    objects = files["objects"].many(Types.OBJECT)
-    libfile = sysentry.file("lib" + self.name + ".a", "arch", "kernel", "mode", "compiler")
-    sysentry.command("ar r %s %s" % (libfile, " ".join([x.name for x in objects])))
-    return [(Types.LINKABLE, libfile)]
-    #return [(Types.OBJECT, object) for object in objects]
-
-class HeaderSummaryRule(dep.Rule):
-  def __init__(self, name, objects):
-    dep.Rule.__init__(self, objects=objects)
-    self.name = name
-  def doit(self, sysentry, state, files, params):
-    libfile = sysentry.file("lib" + self.name + ".h")
-    sysentry.command("touch %s" % (libfile))
-    return [(Types.PLACEHOLDER, libfile)]
-
-class LibRule(dep.Rule):
-  """LibRule returns all relevant archive files for this library and libraries
-  it depends on.
-  
-  In addition, a LibRule exposes header files.
-  """
-  # TODO: "sources" will also natively support C++ files
-  def __init__(self, name, sources, headers, deplibs, cflags = ""):
-    source_rules = sources
-    self.header_rules = headers
-    # TODO: Be careful about depending on extra stuff?
-    for dep_lib in deplibs:
-      self.header_rules.append(dep_lib.header_summary_rule)
-    self.header_summary_rule = HeaderSummaryRule(name, self.header_rules)
-    self.compile_rules = [CompileRule(source_rule, self.header_rules, cflags)
-        for source_rule in source_rules]
-    self.archive_rule = ArchiveRule(name, self.compile_rules)
-    dep.Rule.__init__(self, archive=[self.archive_rule], deplibs=deplibs)
-  def doit(self, sysentry, state, files, params):
-    return files["deplibs"].to_pairs() + files["archive"].to_pairs()
-
-class BinRule(dep.Rule):
-  def __init__(self, name, linkables):
-    self.name = name
-    dep.Rule.__init__(self, linkables=linkables)
-  def doit(self, sysentry, state, files, params):
-    compiler = compilers[params["compiler"]]
-    binfile = sysentry.file(self.name, "arch", "kernel", "mode", "compiler")
-    # TO-DO: Link flags necessary?
-    lflags_start = compiler.lflags_start
-    lflags_end = compiler.lflags_end
-    cflags = compiler.mode_dictionary[params["mode"]]
-    sysentry.command(compiler.linker + " -o %s %s %s %s %s" % (binfile.name, cflags,
-        lflags_start, " ".join(files["linkables"].to_names()), lflags_end))
-    return [(Types.BINFILE, binfile)]
-
-class MakefileRule(dep.Rule):
-  """
-  Plug created so that symlinks are re-created whenever the Makefile changes.
-  """
-  def __init__(self):
-    dep.Rule.__init__(self)
-  def doit(self, sysentry, state, files, params):
-    file = sysentry.makefile()
-    return [(Types.MISC, file)]
-
-class SymlinkRule(dep.Rule):
-  def __init__(self, filerules, dest_dir):
-    dep.Rule.__init__(self, filerules=filerules, makefile=[MakefileRule()])
-    self.dest_dir = dest_dir
-  def doit(self, sysentry, state, files, params):
-    all = []
-    pairs = files["filerules"].to_pairs()
-    for (classname, file) in pairs:
-      sourcename = file.name
-      destname = os.path.join(self.dest_dir, os.path.basename(file.simplename))
-      # the symlink's simplename will be the same as the original's
-      destfile = sysentry.source_file(destname, file.simplename)
-      sysentry.command("rm -f %s" % destname)
-      sysentry.command("ln -s %s %s" % (sourcename, destname))
-      all.append((classname, destfile))
-    sysentry.command("echo '*** Created %d symlinks in %s.'" % (len(pairs), self.dest_dir))
-    return all
-
-# Parameter loader
-
-BUILD_FILE = "build.py"
-
-class Loader:
-  """Responsible for loading build files and helping them stitch together.
-  
-  There are concepts of real and fake paths:
-  
-  - Real path: Where the file is located exactly on the file system.
-  
-  - Fake path: Where results generated by this file are relative to the
-  build system.  If the real path is within the build directories, then
-  the fake path is just the relative path from the root of the build
-  system.  However, we also support building small code trees that are
-  not in the build system.  In this, the "fake" path starts with "outside"
-  to indicate it is outside the build system, and has the full path.
-  
-  Example:
-  
-  Say my build path is
-  
-    ~/fastlib/c  (this is real_rootpath)
-  
-  Someone wants to build ~/fastlib/c/foo/bar, the fake path is "foo/bar".
-  
-  But if someone wants to build ~/foo/bar, the fake path is
-  "outside/home/yourname/foo/bar".
-  
-  
-  See also ../script/fl-build.
-  """
-  
-  def __init__(self, real_rootpath):
-    self.real_root = real_rootpath.rstrip(os.sep)
-    self.thebuildsys = MakeBuildSys(
-        self.real_root, os.path.join(self.real_root, "bin"))
-    self.loader_map = {}
-  
-  def pathjoin(self, left, right, defaultprefix, lsep = "/", rsep = "/"):
-    def helper(left, right):
-      if right and right[0] == ".":
-        right_index = (right + "/").index(rsep)
-        right_first = right[:right_index]
-        right_rest = right[right_index+1:]
-        if right_first == ".":
-          return helper(left, right_rest)
-        else:
-          try:
-            left_index = left.rindex(lsep)
-          except:
-            left_index = 0
-          return helper(left[:left_index], right_rest)
-      else:
-        return left + lsep + right.replace(rsep, lsep)
-    left = left.rstrip(lsep)
-    right = right.rstrip(rsep)
-    if not right:
-      return left
-    elif right[0] != ".":
-      if right[0] == rsep:
-        return right.replace(rsep, lsep)
-      else:
-        return defaultprefix + right.replace(rsep, lsep)
-    else:
-      return helper(left, right)
-  
-  def pathjoin_real(self, left, right):
-    return self.pathjoin(left, right, self.real_root + "/", os.sep, "/")
-  
-  def pathjoin_fake(self, left, right):
-    return self.pathjoin(left, right, "", "/", "/")
-
-  def find_rule(self, rule_path, cur_real_path, cur_fake_path):
-    #print "trying: %s" % fullname
-    (path_rel, rule_name) = rule_path.split(":")
-    
-    fake_path = self.pathjoin_fake(cur_fake_path, path_rel)
-    real_path = self.pathjoin_real(cur_real_path, path_rel)
-    
-    if not fake_path in self.loader_map:
-      self.load(real_path, fake_path)
-    if not rule_name in self.loader_map[fake_path]:
-      raise Exception("Rule '%s' not found in '%s'." % (rule_name, real_path))
-    #print "Found [%s][%s] aka [%s]" % (fullname, defaultpath, path)
-    return self.loader_map[fake_path][rule_name]
-
-  def load(self, real_path, fake_path):
-    """Loads build rules, by exposing a 'register' function."""
-    
-    # !!!!! LOOK AT ME!  All the functions in build rules are defined here!
-    
-    self.loader_map[fake_path] = {}
-    selfname = fake_path.split("/")[-1]
-    def register(name, rule):
-      if name in self.loader_map[fake_path]:
-        raise Exception("Duplicate rule %s:%s" % (fake_path, name))
-      self.loader_map[fake_path][name] = rule
-      return rule
-    def find(rule_name):
-      return self.find_rule(rule_name, real_path, fake_path)
-    def pathify_real(name):
-      return self.pathjoin_real(real_path, name)
-    def pathify_fake(name):
-      return self.pathjoin_fake(fake_path, name)
-    def sourcerule(type, name):
-      if isinstance(name, str):
-        # If there is a colon, it is a rule; otherwise, it is just a file.
-        if ":" in name:
-          return find(name)
-        else:
-          return SourceRule(type, pathify_real("./" + name), pathify_fake("./" + name))
-      else:
-        return name
-    def sourcerules(type, names):
-      return [sourcerule(type, name) for name in names]
-    def lglob(mask, *exclude):
-      (relpath, namemask) = os.path.split(mask)
-      full_path = os.path.join(real_path, mask)
-      basenames = [os.path.basename(f)
-          for f in glob.glob(os.path.join(real_path, mask))]
-      return [os.path.join(relpath, basename)
-          for basename in basenames if not basename in exclude]
-    def librule(name = selfname,
-        sources = [], headers = [], deplibs = [], cflags = ""):
-      return register(name, LibRule(pathify_fake(name),
-          sourcerules(Types.GCC_SOURCE, sources),
-          sourcerules(Types.HEADER , headers),
-          sourcerules(Types.LINKABLE, deplibs),
-          cflags))
-    def unittest(name = selfname + "_unittest",
-        lib = ":" + selfname,
-        sources = []):
-      """Unit tests for the purpose of testing one library.
-      
-      All .c or .cc files are compiled and run as unit tests.
-      
-      (TODO: Unit test framework)
-      """
-      assert sources
-    def inttest(name = selfname + "_inttest",
-        libs = [],
-        sources = []):
-      """Integration tests for testing multiple libraries.
-      """
-      assert libs
-      assert sources
-    def binrule(name, linkables = [], sources = [], headers = [], cflags = ""):
-      # (source, headers, cflags)
-      if sources:
-        lib = librule(name = name + "__auto",
-            sources = sources, headers = headers, deplibs = linkables,
-            cflags = cflags)
-        linkables = linkables + [lib]
-      register(name, BinRule(name, sourcerules(Types.LINKABLE, linkables)))
-    build_file_path = os.path.join(real_path, BUILD_FILE)
-    print "... Reading %s" % (build_file_path)
-    text = util.readfile(build_file_path)
-    exec text in {"register" : register, "Types" : Types,
-                  "find" : find, "dep" : dep, "lglob" : lglob,
-                  "sourcerule" : sourcerule, "sourcerules" : sourcerules,
-                  "librule" : librule, "binrule" : binrule}
-

fastlib/py/datastore.py

@@ -1,734 +0,0 @@
-import util
-
-import unittest
-import os
-import StringIO
-import operator
-import copy
-import sys
-
-class SexpressionStream:
-  """Stream for tokenizing an s-expressions."""
-  def __init__(self, file):
-    self.file = file
-    self.nextchar()
-  def nextchar(self):
-    self.peek = self.file.read(1)
-    return self.peek
-  def skip(self):
-    while True:
-      if self.peek == ';':
-        while self.peek != "" and self.peek != "\n" and self.peek != "\r":
-          self.nextchar()
-      elif self.peek.isspace():
-        self.nextchar()
-      else:
-        break
-  def readstr(self):
-    self.skip()
-    name = ""
-    while self.peek.isalnum() or self.peek == "_":
-      name += self.peek
-      self.nextchar()
-    return util.unescape_sexpression(name)
-  def is_done(self):
-    return self.peek == ""
-
-class UnknownField:
-  def __init__(self, message):
-    self.message = message
-  def __repr__(self):
-    return message
-
-class DataNode:
-  """Abstract hierarchial key-value data stucture.
-  
-  This is intended more for small data sets, like the output
-  of a single C program, where the overhead of any more complex
-  outut mechanism would be a ridiculous annoyance.
-  
-  Thinking of the Windows registry should convince you not to use this
-  for large files.
-  """
-  def __init__(self, val = None):
-    """Create an empty data store.
-    """
-    self.subnodes = {}
-    self.val = val
-
-  def get_node(self, path_elements):
-    """Gets an node via a particular path sequence.
-    """
-    if len(path_elements) == 0:
-      return self
-    else:
-      return self.subnodes[path_elements[0]].get_node(path_elements[1:])
-
-  def get_node_create(self, path_elements):
-    """Gets an node via a particular path sequence.
-    """
-    if len(path_elements) != 0:
-      return self.get_subnode_create(path_elements[0]) \
-          .get_node_create(path_elements[1:])
-    else:
-      return self
-  
-  def get_subnode_create(self, first):
-    """Gets a sub-node, creating one if it doesn't exist.
-    """
-    if not first in self.subnodes.keys():
-      node = DataNode()
-      self.subnodes[first] = node
-    else:
-      node = self.subnodes[first]
-    return node
-
-  def get_val(self, path_elements):
-    """Get a string leaf node by its path sequence.
-    
-    Returns None if no such element was found.
-    """
-    try:
-      return self.get_node(path_elements).val
-    except:
-      return None
-
-  def get_val_path(self, path):
-    """Get a string leaf node by its escaped-and-slash-separated path.
-    """
-    try:
-      return self.get_node(util.split_path(path)).val
-    except:
-      return None
-
-  def set_val(self, path_elements, val):
-    """Sets a leaf string by a particular sequence of path elements.
-    """
-    self.get_node_create(path_elements).val = val
-
-  def set_val_path(self, path, val):
-    """Sets a leaf string by a particular escaped-and-slash-separated
-    path.
-    """
-    self.get_node_create(util.split_path(path)).val = val
-
-  def __write_pathstyle_impl(self, file, prefix):
-    """Implementation method for writing in path-style output format.
-    """
-    if self.val != None:
-      file.write(prefix + " " + util.escape_dspath(self.val) + "\n")
-    for k, v in self.subnodes.iteritems():
-      v.__write_pathstyle_impl(file, prefix + "/" + util.escape_dspath(k))
-
-  def __write_sexpression_impl(self, file, prefix):
-    """Implementation method for writing in sexpression-style output format.
-    """
-    if self.val != None:
-      file.write(util.escape_sexpression(self.val))
-    for k, v in self.subnodes.iteritems():
-      file.write("\n" + prefix + "(" + util.escape_sexpression(k) + " ")
-      v.__write_sexpression_impl(file, prefix + "  ")
-      file.write(")")
-
-  def write_pathstyle(self, file, prefix = ""):
-    """Writes the data structure to the output in path-style format,
-    where every line is a data node, where the left side of the line
-    is an escaped and slash-separated path and the right side is a
-    string.
-    """
-    self.__write_pathstyle_impl(file, prefix)
-
-  def write_sexpression(self, file, prefix = ""):
-    """Writes the data structure to the output in sexpression format.
-    """
-    file.write("; Fx s-expression writer version 1.0")
-    self.__write_sexpression_impl(file, prefix)
-    file.write("\n")
-
-  def read_pathstyle(self, file):
-    """Read a flat-file in key-value format.
-    """
-    for line in file:
-      try:
-        parts = line.strip().split(" ")
-        self.set_val_path(parts[0], util.unescape_dspath(parts[1]))
-      except:
-        print "Warning: Line ignored: %s" % (line.strip())
-
-  def read_sexpression(self, file):
-    """Read s-expression from a file.
-    
-    This only supports a subset of s-expressions that correspond to nodes.
-    There are two examples that are unsupported:
-    
-    (foo bar baz) is invalid because 'bar' and 'baz' cannot both be associated
-    with the same node.
-    
-    (foo (bar a) (bar b)) also cannot work, because there cannot be two 'bar'
-    nodes associated with foo.
-    """
-    self.__read_sexpression_impl(SexpressionStream(file))
-
-  def __read_sexpression_impl(self, stream):
-    try:
-      more_subexpressions = True
-      while more_subexpressions:
-        stream.skip()
-        if stream.peek.isalnum() or stream.peek == '_':
-          if self.val != None:
-            print "Warning: Value reassigned from [%s]." % self.val
-          self.val = stream.readstr()
-        elif stream.peek == '(':
-          # A sub exprsesion
-          stream.nextchar()
-          name = stream.readstr()
-          if name in self.subnodes.keys():
-            print "Warning: Node [%s] is specified twice." % name
-          self.get_subnode_create(name).__read_sexpression_impl(stream)
-          stream.skip()
-          if stream.peek != ')':
-            raise util.ParseError
-          stream.nextchar()
-        elif stream.peek == ')':
-          more_subexpressions = False
-          # do not gobble
-        elif stream.peek == '':
-          more_subexpressions = False
-        else:
-          raise util.ParseError()
-    except:
-      raise util.ParseError()
-  
-  def select(self, pathspecs):
-    """Selects a natural join of several path specifications.
-    
-    See util.select() for more information.
-    """
-    return select(self, self.SELECTFUNCTIONS, pathspecs)
-  
-  def select_to_csv_file(self, file, pathspecs, header = True):
-    """Selects a natural join of several path specifications, and write
-    to a CSV file.
-    
-    See util.select() for more information on how the path specifications
-    work.
-    """
-    elements = self.select(pathspecs)
-    if header:
-      all = [pathspecs] + elements
-    else:
-      all = elements
-    util.write_csv_file(file, all)
-
-  def select_tree(self, matchtree):
-    """Selects based on a match tree.
-    """
-    return matchtree_walk(self, self.SELECTFUNCTIONS, matchtree)
-
-  def __repr__(self):
-    """Converts into debugging s-exp representation.
-    """
-    stream = StringIO.StringIO()
-    self.write_sexpression(stream)
-    return stream.getvalue()
-  
-  def __get_typedval(self):
-    """Converts into a typed value by guessing its type.
-    """
-    try:
-      return int(self.val)
-    except:
-      try:
-        return float(self.val)
-      except:
-        return self.val
-  
-  typedval=property(__get_typedval)
-  
-  SELECTFUNCTIONS=(lambda x:x.val,
-        lambda x:x.subnodes.iterkeys(),
-        lambda x:x!=None, lambda x,y:x.subnodes.get(y, None), None)
-
-def sortify(str):
-  """Makes the string suitable for sorting by returning a tuple, where
-  the left element is typed and the right element is the string.
-  """
-  val = str
-  try:
-    val = float(str)
-  except:
-    pass
-  return (val, str)
-
-def subcolumns(col_nums, rows):
-  """Extract certain columns.
-  
-  example:
-  
-    col_nums = [0, 2, 1]
-    rows = [ ["ted", "likes", "shrimp", "always"],
-             ["bill", "eats", "pasta", "sometimes"] ]
-  
-  returns:
-  
-    [ ["ted", "shrimp", "likes"],
-      ["bill", "pasta", "eats] ]
-  """
-  return [ [ row[i] for i in col_nums ] for row in rows ]
-
-def group(groupcols, rows, remove_key = False):
-  """Groups a matrix by particular columns.
-  
-  Example:
-  
-     groupcols = [ 0, 1 ]
-     
-     rows = [ [ 1, 1, 1, 1 ],
-              [ 1, 1, 1, 2 ],
-              [ 1, 1, 2, 1 ],
-              [ 1, 1, 2, 2 ],
-              [ 2, 1, 1, 1 ],
-              [ 2, 1, 1, 2 ],
-              [ 1, 3, 2, 1 ],
-              [ 1, 3, 2, 2 ] ]
-  
-  returns:
-  
-     groups={(1, 1): [ [ 1, 1, 1, 1 ],
-                       [ 1, 1, 1, 2 ],
-                       [ 1, 1, 2, 1 ],
-                       [ 1, 1, 2, 2 ] ],
-             (2, 1): [ [ 2, 1, 1, 1 ],
-                       [ 2, 1, 1, 2 ] ],
-             (1, 3): [ [ 1, 3, 2, 1 ],
-                       [ 1, 3, 2, 2 ] ] }
-  
-  Note this should work for dictionaries too; if rows is a collection of
-  dictionaries, and groupcols corresponds to dictionary keys.
-  """
-  groups = {}
-  
-  for row in rows:
-    key = tuple([row[x] for x in groupcols])
-    if not key in groups.keys():
-      groups[key] = []
-    if remove_key:
-      newrow = [row[i] for i in util.keys(row) if i not in groupcols]
-    else:
-      newrow = row
-    groups[key].append(newrow)
-  
-  return groups
-
-def groupreduce(groups, reductionpairs):
-  """Takes the result of group, and reduces every specified element.
-  
-  Example:
-  
-     groups= { (1, 1): [ [ 1, 1, 1, 1 ],
-                         [ 1, 1, 1, 2 ],
-                         [ 1, 1, 2, 1 ],
-                         [ 1, 1, 2, 2 ] ],
-               (2, 1): [ [ 2, 1, 1, 1 ],
-                         [ 2, 1, 1, 2 ] ],
-               (1, 3): [ [ 1, 3, 2, 1 ],
-                         [ 1, 3, 2, 2 ] ] }
-     
-     reductionpairs = [ (2, operator.add), (3, operator.mul) ]
-  
-  Returns:
-  
-     reduced = { (1, 1) : [1, 1, 6, 4],
-                 (2, 1) : [2, 1, 2, 2],
-                 (1, 3) : [1, 3, 4, 2] }
-  
-  In the "reduced" array, if a column lacks a reduction operator, an
-  its value from an arbitrary row will be chosen.
-  """
-  reduced = {}
-  
-  for (key, rows) in groups.items():
-    newrow = copy.copy(rows[0]) # copy the row
-    for row in rows[1:]:
-      for (index, reduction) in reductionpairs:
-        newrow[index] = reduction(newrow[index], row[index])
-    reduced[key] = newrow
-  
-  return reduced  
-
-def groupsum(groups, indices = [0]):
-  """Sums all specified columns of a grouping.
-  """
-  reductionpairs = []
-  for index in indices:
-    reductionpairs.append((index, operator.sum))
-  return groupreduce(groups, reductionpairs)
-
-def select(place, (handlefn, listfn, existfn, divefn, bridgefn), pathspecs):
-  """Yields the desired data on a natural join of pathspecs, sorted.
-  
-  The pathspecs are a slash-separated list of pathnames as you would expect
-  to be stored in a DataNode tree.  Path elements may contain a "*" in
-  which case the sub-elements are listed and each traversed.
-  
-  If two different pathspecs contain a "*", the rules are interesting.
-  If both pathspecs are rooted at the same place up to the "*", the two
-  pathspecs are traversed together.  Otherwise, a cartesian product is
-  done between the two.  Example:
-  
-  Say our structure looks like:
-  
-     /runs/1/x = 1x
-     /runs/1/y = 1y
-     /runs/2/x = 1x
-     /runs/2/y = 1y
-     /bill/a = a
-     /bill/b = b
-     /constant = q
-  
-  ['/q', '/runs/*/x'] repeats the uninteresting one:
-    ['q','1x], ['q','2x']
-  
-  ['/runs/*/x', '/runs/*/y'] yields related pairs:
-    ['1x','1y'] , ['2x','2y']
-  
-  ['/runs/*/x', '/bill/*'] yields all combinations:
-    ['1x','a'], ['1x','b'], ['1y','a'], ['1y','b']
-  
-  ['/runs/*/x', '/runs/*/y', '/bill/*'] yields:
-    ['1x','1y','a'], ['1x','1y','b'], ['2x','2y','a'], ['2x','2y','b']
-  
-  The resulting data is then sorted first by the first column, then the
-  second, and so on.  Values that look like numbers are sorted in numerical
-  order.
-  
-  The long tuple of functions are the information necessary for iteration
-  (you likely won't have to specify these yourself):
-  
-  - place: the starting 'place', such as a directory name
-  - listfn: a path to list sub-'place's of a place, such as os.listdir
-  - existfn: to test the existence of a sub-field like os.path.exists
-  - divefn: to take a sub-path and add on a new element, such as os.path.join
-  - bridgefn: more complicated; when a ':' appears at the end of an element,
-  the divefn is called to get the new 'place' to go to and a new tuple of
-  relevant functions.  for example, in select_fields, the ':' operator
-  parses the specified file as a DataNode pathstyle file, and plunges into
-  the DataNode hierarchy
-  """
-  matchtree = DataNode()
-  for element in pathspecs:
-    matchtree.set_val_path(element, element)
-  results = matchtree_walk(place,
-    (handlefn, listfn, existfn, divefn, bridgefn), matchtree)
-  def tolist(map):
-    return [ sortify(map.get(x, "")) for x in pathspecs ]
-  lines = [ tolist(x) for x in results ]
-  lines.sort()
-  def fix(line):
-    return [ x[1] for x in line ]
-  return [ fix(line) for line in lines ]
-
-def matchtree_walk(place, (handlefn, listfn, existfn, divefn, bridgefn), matchtree):
-  """Implements the natural join over an arbitrary hierarchial data backend.
-  
-  See util.select() for information on the behavior.
-  """
-  functions = (handlefn, listfn, existfn, divefn, bridgefn)
-  def combine(subanswers):
-    newanswers = []
-    for answer1 in answers:
-      for answer2 in subanswers:
-        elem = {}
-        elem.update(answer1)
-        elem.update(answer2)
-        newanswers.append(elem)
-    return newanswers
-  myanswer = {}
-  if matchtree.val != None:
-    myanswer[matchtree.val] = handlefn(place)
-  answers = [myanswer]
-  for (matchname, submatchtree) in matchtree.subnodes.iteritems():
-    if matchname[-1] == ':':
-      processfn = bridgefn
-      matchname = matchname[:-1]
-    else:
-      processfn = lambda x:(x, functions)
-    def subwalkfn((subplace, subfunctions), submatchtree):
-      return matchtree_walk(subplace, subfunctions, submatchtree)
-    if matchname == util.STAR:
-      subanswers = []
-      for subname in listfn(place):
-        subplace = divefn(place, subname)
-        returned = subwalkfn(processfn(subplace), submatchtree)
-        if returned != [{}]: # special case: directory that completely failed
-          subanswers += returned
-      answers = combine(subanswers)
-    else:
-      subplace = divefn(place, matchname)
-      if existfn(subplace):
-        answers = combine(subwalkfn(processfn(subplace), submatchtree))
-      else:
-        pass # unknown directories are ignored
-  return answers
-
-def select_fields(rootpath, pathspecs):
-  """Selects first from files, and then from their datanode-pathstyle
-  contents.
-  
-  Pathspecs are in the format:
-  
-    runs/*/FILENAME:/info/params/x
-    runs/*/FILENAME:/info/timers/*/cycles
-  
-  Be careful: Both "FILENAME/:/" and "FILENAME/:info" will choke horribly.
-  Be very careful.
-  """
-  treefunctions = DataNode.SELECTFUNCTIONS
-  def bridge(filename):
-    f = open(filename, "r")
-    try:
-      result = DataNode()
-      result.read_pathstyle(f)
-      return (result, treefunctions)
-    finally:
-      f.close()
-  pathfunctions = (lambda x:x, os.listdir,
-    os.path.exists, os.path.join, bridge)
-  return select(rootpath, pathfunctions, pathspecs)
-
-def select_fields_to_table_file(openfile, rootpath, pathspecs, writefn = util.write_csv_file):
-  """Selects fields first from files and then from the pathstyle contents,
-  writing results to an OPEN csv or other type of text file file.
-  
-  See util.select_fields().
-  """
-  headers = [x.strip("/").split("/")[-1] for x in pathspecs]
-  writefn(openfile, [headers] + select_fields(rootpath, pathspecs))
-
-def select_fields_to_csv_file(csvfile, rootpath, pathspecs):
-  """Selects fields first from files and then from the pathstyle contents,
-  writing results to an OPEN csv file.
-  
-  See util.select_fields().
-  """
-  select_fields_to_table_file(csvfile, rootpath, pathspecs, util.write_csv_file)
-
-def select_fields_to_csv(csvfname, rootpath, pathspecs):
-  """Selects fields first from files and then from the pathstyle contents,
-  writing results to a to-be-created csv file name.
-  
-  See util.select_fields().
-  """
-  f = open(csvfname, "w")
-  try:
-    select_fields_to_csv_file(f, rootpath, pathspecs)
-  finally:
-    f.close()
-
-def select_files(place, pathspecs):
-  """Selects filenames in the natural-join style.
-  
-  See util.select() for info.
-  """
-  functions = (lambda x:x, os.listdir, os.path.exists, os.path.join, None)
-  return select(place, functions, pathspecs)
-
-def match_files(place, pathspec):
-  """Match files based on a single path specification.
-  
-  For instance runs/*/SUMMARY matches files named SUMMARY in any
-  directory called runs.
-  """
-  return [ x[0] for x in select_files(place, [pathspec]) ]
-
-#----------------------------------------------------------------------------
-# Unit tests
-
-class IoTest(unittest.TestCase):
-  def setUp(self):
-    (tmpdir, tmpfile) = os.path.split(os.tempnam())
-    self.tmpdir = "%s/fxsys-util-ut-%s" % (tmpdir, tmpfile)
-    self.rootpath = self.tmpdir
-    os.makedirs(self.rootpath)
-  
-  def tearDown(self):
-    util.remove_dir_recursive(self.tmpdir)
-  
-  def test_match_files(self):
-    def fix(x):
-      return os.path.join(self.rootpath, x)
-    matchdirs = ["foo/a/bar/a/baz", "foo/a/bar/b/baz", "foo/a/bar/c/baz",
-            "foo/b/bar/a/baz", "foo/b/bar/c/baz",
-            "foo/c/bar/b/baz"]
-    nonmatchdirs = ["foon/a/bar/a/baz", "foo/g/bar/b/bazz", "foo/a/x", "subsume"]
-    for dir in matchdirs + nonmatchdirs:
-      os.makedirs(os.path.join(self.rootpath, dir))
-    found = match_files(self.rootpath, "foo/*/bar/*/baz")
-    found.sort()
-    matchdirs.sort()
-    matchdirs = [ fix(x) for x in found ]
-    self.assertEqual(matchdirs, found)
-    
-  def test_select_files(self):
-    def fix(x):
-      return os.path.join(self.rootpath, x)
-    leftdirs = ["foo/a/bar/a/baz", "foo/a/bar/b/baz", "foo/a/bar/c/baz",
-            "foo/b/bar/a/baz", "foo/b/bar/c/baz",
-            "foo/c/bar/b/baz"]
-    middirs = ["foo/a/bar/a/bon", "foo/a/bar/b/bon", "foo/a/bar/c/bon",
-            "foo/b/bar/a/bon", "foo/b/bar/c/bon",
-            "foo/c/bar/b/bon"]
-    rightdirs = ["param/a/foo", "param/b/foo", "param/c/foo"]
-    nondirs = ["foon/a/bar/a/baz", "foo/g/bar/b/bazz", "foo/a/x", "subsume"]
-    for dir in leftdirs + middirs + rightdirs + nondirs:
-      os.makedirs(os.path.join(self.rootpath, dir))
-    found = select_files(self.rootpath,
-        ["foo/*/bar/*/baz", "foo/*/bar/*/bon", "param/*/foo"])
-    found.sort()
-    expect = []
-    for (leftdir, middir) in zip(leftdirs, middirs):
-      for rightdir in rightdirs:
-        expect.append([fix(leftdir), fix(middir), fix(rightdir)])
-    expect.sort()
-    found.sort()
-    self.assertEqual(expect, found)
-    
-    
-
-class DatastoreTest(unittest.TestCase):
-  def setUp(self):
-    self.f = DataNode()
-    self.f.get_node_create(["walks", "10", "param", "speed"]).val = "SLOW"
-    self.f.get_node_create(["runs", "10", "param"]).val = "lots of params"
-    self.f.get_node_create(["runs", "10", "param", "x"]).val = "2"
-    self.f.set_val(["runs", "10", "param", "y"], "3")
-    self.f.set_val_path("runs/20/param/x", "4")
-    self.f.set_val_path("/runs/20/param/y", "5")
-    self.f.set_val(["runs", "10", "metric", "total performance"], "very good")
-    self.f.set_val_path("/runs/30/param/x", "8");
-    self.f.set_val_path("/runs/30/param/y", "9");
-    self.f.set_val_path("/runs/40/param/x", "11");
-  
-  def test_get_val(self):
-    self.assertEqual(self.f.get_val(["runs", "10", "param", "x"]), "2");
-    self.assertEqual(self.f.get_val(["runs", "10", "param", "y"]), "3");
-    self.assertEqual(self.f.get_val(["runs", "20", "param", "x"]), "4");
-    self.assertEqual(self.f.get_val(["runs", "20", "param", "y"]), "5");
-
-  def test_get_val_path(self):
-    self.assertEqual(self.f.get_val_path("runs/20/param/y"), "5");
-    self.assertEqual(self.f.get_val_path("/runs/20/param/y"), "5");
-    self.assertEqual(self.f.get_val_path("/runs/10/metric/total%20performance"), "very good");
-  
-  def help_test_save_get(self, readfun, writefun):
-    print "(START)"
-    first_outstream = StringIO.StringIO()
-    writefun(self.f, first_outstream)
-    print first_outstream.getvalue()
-    print "(REREAD)"
-    f2 = DataNode()
-    readfun(f2, StringIO.StringIO(first_outstream.getvalue()))
-    second_outstream = StringIO.StringIO()
-    writefun(f2, second_outstream)
-    print second_outstream.getvalue()
-    print "(END)"
-    self.assertEqual(first_outstream.getvalue(), second_outstream.getvalue());
-  
-  def test_pathstyle_save(self):
-    self.help_test_save_get(DataNode.read_pathstyle, DataNode.write_pathstyle)
-  
-  def test_sexpression(self):
-    self.help_test_save_get(DataNode.read_sexpression, DataNode.write_sexpression)
-  
-  def test_select(self):
-    selected = self.f.select(["runs/*/param/x", "runs/*/param/y"])
-    expected = [
-        ['2', '3'],
-        ['4', '5'],
-        ['8', '9'],
-        ['11', '']]
-    selected.sort()
-    expected.sort()
-    self.assertEqual(expected, selected)
-  
-  def test_group(self):
-    groupcols = [ 0, 1 ]
-    rows = [ [ 1, 1, 1, 1 ],
-             [ 1, 1, 1, 2 ],
-             [ 1, 1, 2, 1 ],
-             [ 1, 1, 2, 2 ],
-             [ 2, 1, 1, 1 ],
-             [ 2, 1, 1, 2 ],
-             [ 1, 3, 2, 1 ],
-             [ 1, 3, 2, 2 ] ]
-    expect = { (1, 1): [ [ 1, 1, 1, 1 ],
-                      [ 1, 1, 1, 2 ],
-                      [ 1, 1, 2, 1 ],
-                      [ 1, 1, 2, 2 ] ],
-            (2, 1): [ [ 2, 1, 1, 1 ],
-                      [ 2, 1, 1, 2 ] ],
-            (1, 3): [ [ 1, 3, 2, 1 ],
-                      [ 1, 3, 2, 2 ] ] }
-    self.assertEqual(expect, group(groupcols, rows))
-  
-  def test_group_nonint(self):
-    k_a = "a"
-    k_b = (3.2, "food")
-    k_c = "hamburger"
-    groupcols = [ k_a, k_b ]
-    rows = [ { k_a:1, k_b:1, k_c:1, "d":1 },
-             { k_a:1, k_b:1, k_c:1, "d":2 },
-             { k_a:1, k_b:1, k_c:2, "d":1 },
-             { k_a:1, k_b:1, k_c:2, "d":2 },
-             { k_a:2, k_b:1, k_c:1, "d":1 },
-             { k_a:2, k_b:1, k_c:1, "d":2 },
-             { k_a:1, k_b:3, k_c:2, "d":1 },
-             { k_a:1, k_b:3, k_c:2, "d":2 } ]
-    expect = { (1, 1): [ { k_a:1, k_b:1, k_c:1, "d":1 },
-                         { k_a:1, k_b:1, k_c:1, "d":2 },
-                         { k_a:1, k_b:1, k_c:2, "d":1 },
-                         { k_a:1, k_b:1, k_c:2, "d":2 } ],
-               (2, 1): [ { k_a:2, k_b:1, k_c:1, "d":1 },
-                         { k_a:2, k_b:1, k_c:1, "d":2 } ],
-               (1, 3): [ { k_a:1, k_b:3, k_c:2, "d":1 },
-                         { k_a:1, k_b:3, k_c:2, "d":2 } ] }
-    self.assertEqual(expect, group(groupcols, rows))
-  
-  def test_subcolumns(self):
-    col_nums = [0, 2, 1]
-    rows = [ ["ted", "likes", "shrimp", "always"],
-             ["bill", "eats", "pasta", "sometimes"] ]
-    expect = [ ["ted", "shrimp", "likes"],
-      ["bill", "pasta", "eats"] ]
-    self.assertEqual(expect, subcolumns(col_nums, rows))
-  
-  def test_groupreduce(self):
-    groups= { (1, 1): [ [ 1, 1, 1, 1 ],
-                        [ 1, 1, 1, 2 ],
-                        [ 1, 1, 2, 1 ],
-                        [ 1, 1, 2, 2 ] ],
-              (2, 1): [ [ 2, 1, 1, 1 ],
-                        [ 2, 1, 1, 2 ] ],
-              (1, 3): [ [ 1, 3, 2, 1 ],
-                        [ 1, 3, 2, 2 ] ] }
-    reductionpairs = [ (2, operator.add), (3, operator.mul) ]
-    reduced = { (1, 1) : [1, 1, 6, 4],
-                (2, 1) : [2, 1, 2, 2],
-                (1, 3) : [1, 3, 4, 2] }
-    self.assertEqual(reduced, groupreduce(groups, reductionpairs))
-  
-  def test_csv_file_head(self):
-    stream = StringIO.StringIO()
-    self.f.select_to_csv_file(stream, ["runs/*/param/x", "runs/*/param/y"])
-    expected = "runs/*/param/x, runs/*/param/y\n2, 3\n8, 9\n4, 5\n11, \n";
-    self.assertEqual(util.sorted(expected.split("\n")), util.sorted(stream.getvalue().split("\n")))
-  
-  def test_csv_file_nohead(self):
-    stream = StringIO.StringIO()
-    self.f.select_to_csv_file(stream, ["runs/*/param/x", "runs/*/param/y"], False)
-    expected = "2, 3\n8, 9\n4, 5\n11, \n";
-    self.assertEqual(util.sorted(expected.split("\n")), util.sorted(stream.getvalue().split("\n")))
-
-if __name__ == "__main__":
-  unittest.main()
-  test_matching()

fastlib/py/dep.py

@@ -1,207 +0,0 @@
-# Simple abstract dependency system suitable for generating things like
-# makefiles.
-
-import util
-
-import os
-
-class DestFile:
-  def __init__(self, name, simplename, dep_files={}, parameterization={}):
-    # TODO: Separate "my" parameterization from "dep" parameterization
-    self.name = name
-    if simplename:
-      self.simplename = simplename
-    else:
-      self.simplename = name
-    self.parameterization = parameterization
-    self.dep_files = dep_files
-  def __repr__(self):
-    return self.name
-  def __hash__(self):
-    return hash(self.name) ^ hash(self.simplename)
-  def __cmp__(self, other):
-    if isinstance(other, DestFile):
-      return cmp(self.name, other.name)
-    else:
-      return NotImplemented
-
-class DepState:
-  def __init__(self, files, params):
-    self.files = files
-    self.params = params
-    self.parameterization = {}
-    for file in filemap_to_files(files):
-      self.parameterization.update(file.parameterization)
-
-class DepSys:
-  def __init__(self):
-    self.runs = {}
-  def begin(self, state):
-    raise Exception("DepSys not overridden")
-
-class DepSysEntry:
-  """Abstract dependency system entry.
-  
-  You must override this and provide a way of generating filenames.
-  """
-  def __init__(self, sys, state):
-    """
-    sys - a DepSys object
-    state - a DepState object
-    """
-    self.sys = sys
-    self.state = state
-  def file(self, simplename, *extra_parameterization):
-    parameterization = dict(self.state.parameterization)
-    my_parameterization = dict([(k, self.state.params[k])
-        for k in extra_parameterization])
-    parameterization.update(my_parameterization)
-    realname = self._make_name(simplename, parameterization)
-    self.ensure_writable(realname)
-    return DestFile(realname, simplename, self.state.files, parameterization)
-  def _make_name(self, simplename, parameterization):
-    raise Exception("not implemented")
-  def ensure_writable(realname):
-    raise Exception("not implemented")
-  def end(self, files):
-    raise Exception("not implemented")
-  def command(self, str):
-    raise Exception("not implemented")
-
-class BadResult:
-  pass
-
-class FileCollection:
-  def __init__(self, pairs):
-    self.files_by_type = {}
-    unique_pairs = {}
-    for pair in pairs:
-      unique_pairs[pair] = None
-    for (typename, file) in unique_pairs:
-      self.files_by_type.setdefault(typename, []).append(file)
-  def single(self, typename):
-    result = self.files_by_type[typename]
-    if len(result) != 1:
-      raise BadResult()
-    return result[0]
-  def many(self, typename):
-    return self.files_by_type.get(typename, [])
-  def to_pairs(self):
-    pairs = []
-    for (classname, files) in self.files_by_type.items():
-      pairs += [(classname, file) for file in files]
-    return pairs
-  def to_files(self):
-    return util.collapse_once(
-        [files for (classname, files) in self.files_by_type.items()])
-  def to_names(self):
-    return [f.name for f in self.to_files()]
-
-def filemap_to_files(filemap):
-  return util.collapse_once(
-      [file_collection.to_files() for file_collection in filemap.values()])
-
-class Rule:
-  """
-  Generic rule.
-  
-  This rule takes in several labelled sets of other rules.
-  
-  TODO: Explain the idea of "meta-rule" better
-  
-  This is more a meta-rule, which can depend on other meta-rules.
-  Meta-rules are asked for the exact filenames they produce, which
-  recursively ask their dependency rules what files they produce.  Once a
-  meta-rule knows what files its dependency rules produce, it can generate
-  commands to produce its own result files, and tell ITS dependents what
-  files it generated.
-  
-  """
-  
-  # TODO: (requires thought) If multiple rules depend on a single rule, I
-  # *think* this code, how it is written it will run "doit" for that rule
-  # multiple times, rather than once.  This might be necessary due to
-  # parameterization, I haven't thought much about it.
-  
-  def __init__(self, **dep_map):
-    self.dep_map = dep_map
-  def generate(self, sys, params):
-    def fake_cont(x):
-      pass
-    self.run(sys, params, fake_cont)
-  def run(self, sys, params, continuation):
-    key = (self, util.dicthash(params))
-    if key in sys.runs:
-      continuation(sys.runs[key])
-    else:
-      all_deps = []
-      
-      for (dep_class, deps) in self.dep_map.items():
-        for dep in deps:
-          all_deps.append((dep_class, dep))
-      
-      dep_count = [len(all_deps)]
-      dep_files = {}
-      
-      def check_done():
-        if dep_count[0] == 0:
-          dep_files_converted = dict(self.dep_map)
-          for k in self.dep_map:
-            dep_files_converted[k] = FileCollection(dep_files.get(k, []))
-          state = DepState(dep_files_converted, params)
-          # Also memoize here!
-          action = sys.begin(state)
-          my_files = self.doit(action, state, dep_files_converted, params)
-          action.end(my_files)
-          sys.runs[key] = my_files
-          continuation(my_files)
-          dep_count[0] = -1
-      
-      def my_continuation_for(dep_class):
-        def impl(single_dep_files):
-          dep_files.setdefault(dep_class, []).extend(single_dep_files)
-          dep_count[0] -= 1
-          check_done()
-        return impl
-      
-      for (dep_class, dep) in all_deps:
-        dep.run(sys, params, my_continuation_for(dep_class))
-      
-      check_done()
-  def doit(self, sysentry, state, dep_files, params):
-    """
-    The doit function is called for all subclasses of this.  The doit
-    function doesn't necessarily perform the action, but is required to
-    figure out exactly what actions are required.  For instance, for the
-    build system, this generates a Makefile rule, which contains the exact
-    commands and file names.
-    
-    The following are sent to an implementors' doit:
-      - dep_files: the files "generated" by all rules it depends on.  The rules
-      are grouped into the label of the rule (given when you called
-      Rule.__init__) and then into the type of the file (like file
-      extension).  The rules you are dependent on were the ones responsible
-      for assigning the file extension.  Each file has a "simple" name and
-      an "absolute" name.  The simple name should be used for generating
-      result files, and the absolute name is the location of the source
-      file.
-      - params: the parameterization of the instantiation; that is, each
-      file can be created with combinations of different parameters, and the
-      parameterization is used to identify different "builds" of the same
-      file.  in the case of the build system, this is different operating
-      systems or compilation modes.  certain rules care about certain
-      parameters, so when they generate destination files, they specify
-      which parameters they care about.
-      - the state (an encapsulation of params and input files)
-          TODO: Remember why this is here, it doesn't seem to be used anywhere
-      - systentry: a single entry in thie build system.  This is equivalent
-      to a Makefile rule waiting for you to put actions into it.
-      Unlike this Rule class, this object wants exact specific commands
-      for a specific parameterization, like a Makefile rule with real
-      filenames and commands.
-    You must return:
-      - a list of key,val pairs, where key is some indicator of file type. 
-      Example is [("header", "foo.h"), ("source", "foo.cc")].
-    """
-    # TODO: Implement me in subclasses
-    return util.map_values(FileCollection.to_pairs, dep_files)

fastlib/py/fx.py

@@ -1,100 +0,0 @@
-"""
-FastExec client library very similar to the C library.
-
-This provides command-line argument parsing in FASTexec style, and also provides
-a global datastore for exporting metrics (and dumping them to a file if
-desired).
-"""
-
-import datastore
-
-import sys
-
-"""The root datanode for the fx system."""
-datanode = datastore.DataNode()
-
-"""Extra command-line arguments that do not contain --."""
-extra_args = []
-
-def param(name, mustexist = True):
-  v = datanode.get_val_path("/info/params/" + name.strip("/"))
-  if mustexist and v == None:
-    raise "Parameter [%s] not specified." % (name)
-  return v
-
-def param_exists(name):
-  return param(name, False) != None
-
-def param_default(name, val):
-  if not param_exists(name):
-    datanode.set_val_path("/info/params/" + name.strip("/"), val)
-
-def param_str(name):
-  # TODO: Validate?
-  return param(name)
-
-def param_int(name):
-  # TODO: Validate?
-  return int(param(name))
-
-def param_float(name):
-  return float(param(name))
-
-def param_bool(name):
-  str = param(name)
-  if str == None or (len(str) > 0 and str[0] in "fFnN0"):
-    return False
-  else:
-    return True
-
-def dump():
-  stream = sys.stdout
-  fname = param("fx/output")
-  if fname != None:
-   stream = open(fname, "w")
-  try:
-    datanode.write_pathstyle(stream)
-  finally:
-    if fname != None:
-      stream.close()
-
-# TODO: Configurable
-
-# TODO: FINISH THIS
-
-def metric_set(instancename, name, val):
-  if instancename == None:
-    path = ["info", "metrics", name]
-  else:
-    path = ["instances", instancename, "metrics", name]
-  datanode.set_val(path, val)
-
-def subparam_set(instancename, name, val):
-  path = ["instances", instancename, "params", name]
-  datanode.set_val(path, val)
-
-def __parse_args():
-  """Parses command-line arguments.
-  
-  Note that the keys are in path-style and unescaped as so.
-  However, the values are assumed not to be escaped.
-  """
-  global extra_args
-  kvpairs = []
-  for arg in sys.argv[1:]:
-    if arg[0:2] == "--":
-      if '=' in arg:
-        i = arg.index("=")
-        kvpairs.append((arg[2:i], arg[i+1:]))
-      else:
-        kvpairs.append((arg[2:], "1"))
-    else:
-      extra_args.append(arg)
-  for (name, val) in kvpairs:
-    param_default(name, val)
-
-__parse_args()
-
-if __name__ == "__main__":
-  print datanode
-  print extra_args

fastlib/py/fxsys.py

@@ -1,289 +0,0 @@
-"""FASTexec system management.
-
-Allows you to create, manage, and deploy experiments.
-"""
-
-# work items:
-#  - path translation
-#  - argument translation (huh?)
-
-import os
-import random
-import util
-import pm
-import sys
-import unittest
-import work
-
-class FxSystemException:
-  pass
-
-class FxSystem:
-  """A particular fx system, parameterized by the root of the hierarchy.
-  
-  Important attributes:
-  - rootpath: the root of the system
-  """
-
-  def __init__(self, rootpath):
-    """Creates a new fx-system based on the specified path.
-    
-    Rootpath must exist.  If the proper FASTexec subdirectories are not present
-    they will be created, but the root itself must be created.  You may
-    consider making sure the root directory itself has the group
-    setguid bit if multiple people use the same system.
-    
-    - rootpath: absolute path of the root of the fx system
-    """
-    self.rootpath = rootpath
-    try:
-      self.validate()
-    except:
-      self.create()
-  
-  def validate(self):
-    """Makes sure this is a valid fx system.
-    """
-    if not os.access(os.path.join(self.rootpath, "FXSYS"), os.W_OK):
-      raise FxSystemException()
-    # TODO(garryb): Include real validation
-  
-  def create(self):
-    """Sets up this path as an fx system.
-    
-    The rootpath must exit, and must already have permissions set up properly.
-    
-    If the system is already set up, raises an error exception.
-    """
-    # TODO(garryb): Implement
-    # TODO(garryb): Set permissions properly
-    #util.ensuredir(self.rootpath)
-    util.ensuredir(os.path.join(self.rootpath, "problems"))
-    util.ensuredir(os.path.join(self.rootpath, "data"))
-    util.writefile(os.path.join(self.rootpath, "FXSYS"), "FXSYS")
-    
-    # TODO(garryb):
-    
-    self.validate()
-
-  def translate(self, relpath):
-    """
-    Translates a path alias into absolute path.
-    
-    In particular, paths beginning with FXSYS/ are translated to real paths.
-    """
-    relpath = str(relpath)
-    if relpath.startswith('FXSYS/'):
-      result = os.path.join(self.rootpath, relpath[len('FXSYS/'):])
-    else:
-      result = os.path.abspath(relpath)
-    return result
-
-  # All pathname hardcoding should exist in this class, even if it doesn't
-  # seem the best place to put it.
-  
-  def pathof_problem(self, probname):
-    return self.translate("FXSYS/problems/%s" % probname)
-
-  def pathof_experiment(self, probname, expname):
-    return self.translate("FXSYS/problems/%s/exps/%s" % (probname, expname))
-
-  def pathof_run(self, probname, expname, runname):
-    return self.translate("FXSYS/problems/%s/exps/%s/runs/%s" % (probname, expname, runname))
-
-  def get_experiment_names(self, probname):
-    return os.listdir(self.translate("FXSYS/problems/%s/exps" % probname))
-
-class Experiment:
-  """One particular experiment.
-  
-  An experiment is a labelled set of runs for a particular purpose.
-  """
-  
-  def __init__(self, probname, expname, create = True, fxsys = None):
-    if fxsys == None:
-      fxsys = default_fxsys()
-    self.fxsys = fxsys
-    self.expname = expname
-    self.probname = probname
-    self.path = fxsys.pathof_experiment(probname, expname)
-    if create:
-      self.create()
-  
-  def pathof_run(self, runname):
-    return self.fxsys.pathof_run(self.probname, self.expname, runname)
-  
-  def pathof_statusfile(self, runname):
-    fullpath = self.pathof_run(runname)
-    return os.path.join(fullpath, work.WorkEntry.STATUSFILE)
-  
-  def create(self):
-    util.ensuredir(self.path)
-  
-  def execute(self, paramset):
-    wq = work.WorkQueue(self)
-    wq.add([work.WorkEntry(self, x) for x in paramset.enumerate()])
-    wq.inline_exec()
-  
-  def get_statuses(self):
-    """Gets the status, such as work.WorkEntry.FINISHED, for each run that
-    has been started.  It is a key-value map of run name to its execution
-    status.
-    
-    If the status file does not exist, is invalid, or cannot be accessed,
-    None is returned as the status.
-    """
-    # This should go through and remove decaying STATUS files
-    statuses = {}
-    for runname in os.listdir(os.path.join(self.path, "runs")):
-      try:
-        lines = util.readlines(self.pathof_statusfile(runname))
-        statuses[runname] = lines[0].strip()
-      except:
-        statuses[runname] = None
-    return statuses
-  
-  def cleanup(self, purge = False, nuke = False):
-    """Removes the status files of runs that have not finished.
-    This is useful you terminate a run and want to restart it -- if you
-    do not clean it up, the runs that were in progress will not be restarted.
-    
-    Be warned -- If you run this while one of the runs is actually
-    executing, that run will be re-run in parallel.
-    
-    purge: whether to completely delete the entire run's directory
-           (not just the status file)
-    """
-    statuses = self.get_statuses()
-    for (runname, status) in statuses.items():
-      if status != work.WorkEntry.FINISHED:
-        statusfile = self.pathof_statusfile(runname)
-        if purge or nuke:
-          print "Purging stale run %s" % runname
-          try:
-            util.remove_dir_recursive(self.pathof_run(runname))
-          except OSError:
-            print "XXX Error deleting the run"
-        else:
-          print "Clearing status file of stale run %s" % runname
-          try:
-            os.unlink(statusfile)
-          except:
-            print "... Status file was already invalid.  This run can be re-run."
-      elif nuke:
-        print "NUKING COMPLETED RUN %s" % runname
-        try:
-          util.remove_dir_recursive(self.pathof_run(runname))
-        except OSError:
-          print "XXX Error deleting the run"
-  
-  def print_status(self):
-    """Prints the log files of any run that is not completed."""
-    num_printed = 0
-    items = self.get_statuses().items()
-    items_ordered = []
-    items_ordered += [(x,y) for (x,y) in items if y == work.WorkEntry.FINISHED]
-    items_ordered += [(x,y) for (x,y) in items if y != work.WorkEntry.FINISHED]
-    for (runname, status) in items_ordered:
-      print "%s -- %s" % (status, runname)
-      if status != work.WorkEntry.FINISHED:
-        logfile = os.path.join(self.pathof_run(runname), work.WorkEntry.LOGFILE)
-        try:
-          num_printed += 1
-          lines = util.readlines(logfile)
-          for line in lines:
-            print "  | %s" % line.strip()
-        except:
-          print "  X No log file"
-    return num_printed
-
-  def purge(self):
-    util.remove_dir_recursive(self.path)
-
-# functions
-
-class NoConfigFileException:
-  pass
-
-def default_fxsys():
-  """Gets the default Xsys object, by searching for fx.conf in the current
-  or parent directories.
-  """
-  
-  # TODO: Cache this
-  path = os.getcwd()
-  
-  while len(path) > 1 and not os.path.exists(os.path.join(path, "fx.conf")):
-    path = os.path.dirname(path)
-
-  if len(path) <= 1:
-    raise NoConfigFileException
-  
-  conffile = os.path.join(path, "fx.conf")
-  lines = util.readlines(conffile)
-  rootpath = os.path.join(path, lines[0])
-  return FxSystem(rootpath)
-
-# public test helpers
-
-def test_new_testing_fxsys():
-  (tmpdir, tmpfile) = os.path.split(os.tempnam())
-  rootpath = "%s/fxsys/%s" % (tmpdir, tmpfile)
-  util.ensuredir(rootpath)
-  fxsys = FxSystem(rootpath)
-  print "Rooted at %s" % rootpath
-  fxsys.create()
-  return fxsys
-
-def test_create_numlist_infiles(fxsys):
-  infiles = []
-  for i in [ 2000, 4000, 6000, 8000, 10000 ]:
-    fname = fxsys.translate("FXSYS/data/num%d.in" % i)
-    util.write_random_ints(fname, i)
-    infiles.append(fname)
-  return (infiles)
-
-class TestFx(unittest.TestCase):
-  
-  def setUp(self):
-    self.fxsys = test_new_testing_fxsys()
-    self.infiles = test_create_numlist_infiles(self.fxsys)
-    self.paramset = pm.Combine(
-      pm.Bind(pm.Binfile(), pm.Val("/usr/bin/sort")),
-      pm.Bind(pm.Extra(), pm.Val("-n")),
-      pm.Bind(pm.Stdin(), pm.Vals(*self.infiles)),
-      pm.Bind(pm.Stdout(), "out.txt"))
-  
-  def tearDown(self):
-    util.remove_dir_recursive(self.fxsys.rootpath)
-  
-  def test_inline(self):
-    self.exp = Experiment("xtest-sort", "sort-test_inline", fxsys = self.fxsys)
-    self.exp.create()
-    wq = work.WorkQueue(self.exp)
-    wq.add([work.WorkEntry(self.exp, runspec) for runspec in self.paramset.enumerate()])
-    print "Inline first"
-    wq.inline_exec()
-    print "Inline second"
-    wq.inline_exec()
-    print "Inline end"
-  
-  def test_shell(self):
-    self.exp = Experiment("xtest-sort", "sort-test_shell", fxsys = self.fxsys)
-    self.exp.create()
-    wq = work.WorkQueue(self.exp)
-    wq.add([work.WorkEntry(self.exp, runspec) for runspec in self.paramset.enumerate()])
-    print "Writing work queue..."
-    fname = wq.write()
-    print "Running %s..." % fname
-    os.spawnv(os.P_WAIT, "/bin/bash", ["/bin/bash", fname])
-    print "Running a second time..."
-    os.spawnv(os.P_WAIT, "/bin/bash", ["/bin/bash", fname])
-  
-  def test_default(self):
-    self.exp = Experiment("xtest-sort", "sort-test_default", fxsys = self.fxsys)
-    self.exp.execute(self.paramset)
-
-if __name__ == "__main__":
-  unittest.main()
-

fastlib/py/pm.py

@@ -1,382 +0,0 @@
-
-import copy
-import random
-import util
-import os
-import unittest
-
-def combine(a, b):
-  if a == None:
-    return b
-  if b == None:
-    return a
-  raise Conflict
-
-class Conflict:
-  """Exception thrown when two single fields conflict.
-  
-  For example, perhaps a Combine is done, but the Binfile
-  is specified on both sides.
-  """
-  pass
-
-class RunSpec:
-  """One particular combination of parameters, that corresponds
-  to one actual run of the program.
-  """
-  def __init__(self):
-    """Create an empty run specification.
-    """
-    self.binfile = None
-    self.wrappers = []
-    self.wrapper_info = []
-    self.params = {}
-    self.extra = []
-    self.stdin = None
-    self.stdout = None
-    self.inputs = []
-    self.outputs = []
-  def merge(self, other):
-    """Merge the other's fields into this.
-    
-    A Combine exception is raised if singleton parameters
-    conflict.
-    
-    Dictionary-based 'Var' parameters are overwritten.
-    """
-    self.binfile = combine(self.binfile, other.binfile)
-    self.params.update(other.params)
-    self.wrappers += other.wrappers
-    self.wrapper_info += other.wrapper_info
-    self.extra += other.extra
-    self.stdin = combine(self.stdin, other.stdin)
-    self.stdout = combine(self.stdout, other.stdout)
-    self.inputs += other.inputs
-    self.outputs += other.outputs
-  def merged_with(self, other):
-    """Return a copy of this RunSpec merged with the other's
-    fields.
-    """
-    c = copy.deepcopy(self)
-    c.merge(other)
-    return c
-  def check(self):
-    assert self.binfile != None
-  def generate_name(self):
-    """Generates a user-friendly filename.
-    """
-    self.check()
-    allparas = []
-    allparas += self.wrapper_info
-    allparas += [os.path.basename(self.binfile)]
-    allparas += self.extra
-    nvp = [pair for pair in self.params.iteritems()]
-    nvp.sort() # Sort by parameter name to ensure deterministic ordering
-    for (name, val) in nvp:
-      if val in self.outputs:
-        # output files are not important
-        continue
-      if val in self.inputs:
-        val = "%s_%x" % (os.path.basename(val), hash(val) % 65536)
-      allparas.append("%s=%s" % (name, val))
-    if self.stdin != None:
-      allparas += ["stdin=%s" % os.path.basename(self.stdin)]
-    return util.sanitize_basename("_".join(allparas))
-  def to_args(self):
-    """Turns to a list of arguments such that args[0] is the binary file,
-    and the rest are the parameters, suitable for os.exec.
-    
-    This does not include the stdin or stdout redirects, so make sure you
-    handle them separately.
-    """
-    allparas = self.wrappers + [self.binfile] + self.extra
-    allparas += ["--%s=%s" % (k, v) for (k, v) in self.params.iteritems()]
-    return allparas
-  def to_command(self):
-    """Turns into a single shell command.
-    """
-    # TODO: Escaping
-    allparas = []
-    allparas += [ util.shellquote(x) for x in self.wrappers ]
-    allparas += [util.shellquote(self.binfile)]
-    allparas += [ util.shellquote(x) for x in self.extra ]
-    allparas += ["--%s=%s" % (k, util.shellquote(v)) for (k, v) in self.params.iteritems()]
-    if self.stdin != None:
-      allparas += ["<%s" % util.shellquote(self.stdin)]
-    if self.stdout != None:
-      allparas += [">%s" % util.shellquote(self.stdout)]
-    return " ".join(allparas)
-
-# Parameter sets -- sets of all possible run parameters
-
-class ParamSet:
-  """Abstract set of parameters.
-  """
-  def enumerate(self):
-    """Returns a list of RunSpecs for all runs that should exist."""
-    return []
-  def print_all(self):
-    """Prints all commands that would be executed if all were to be
-    run."""
-    for e in self.enumerate():
-      print e.to_command()
-
-class Combine(ParamSet):
-  """Cartesian product, or all combinations,
-  of several smaller parameter sets.
-  
-  (Technically, this is closer to intersction, but the
-  actual 'intersection' operation is not performed for
-  redundant parameters.  The handling of redundant
-  parameters is undefined.)
-  
-  Parameters are multiplied so that the last parameters
-  vary closest together, and the first in the list
-  vary last.  That is, (1 2) x (A B) =
-  
-     1 A
-     1 B
-     2 A
-     2 B
-  """
-  def __init__(self, *factors):
-    self.factors = factors
-  def enumerate(self):
-    all = [ RunSpec() ]
-    for item in self.factors:
-      newlist = []
-      enumerated = item.enumerate()
-      for runspec1 in all:
-        for runspec2 in enumerated:
-          newlist.append(runspec1.merged_with(runspec2))
-      all = newlist
-    return all
-
-class Any(ParamSet):
-  """The union of several smaller parameter sets.
-  """
-  def __init__(self, choices):
-    self.choices = choices
-  def enumerate(self):
-    all = []
-    for choice in self.choices:
-      all += choice.enumerate()
-    return all
-
-class Bind(ParamSet):
-  """Binds a particular RunSpec field to a particular
-  value set.
-  """
-  def __init__(self, var, valset):
-    self.var = var
-    if not isinstance(valset, ValSet):
-      if isinstance(valset, list):
-        valset = Vals(*valset)
-      else:
-        valset = Val(valset)
-    self.valset = valset
-  def enumerate(self):
-    all = []
-    for val in self.valset.enumerate():
-      spec = RunSpec()
-      self.var.set(spec, val)
-      all.append(spec)
-    return all
-
-class CoBind(ParamSet):
-  """Binds any number of RunSpec fields to any number
-  of value sets, one value set per fields.
-  
-  When enumerated, the enumerations of every value set are zipped together;
-  all enumerations must be of equal size.
-  """
-  def __init__(self, *pairs):
-    self.vars = []
-    self.valsets = []
-    for i in range(0, len(pairs), 2):
-      self.vars.append(pairs[i])
-      self.valsets.append(pairs[i + 1])
-    # TODO: Runtime check to make sure each set is the same size
-  def enumerate(self):
-    all = []
-    valmatrix = []
-    for valset in self.valsets:
-      valmatrix.append(valset.enumerate())
-    assert min(map(len, valmatrix)) == max(map(len, valmatrix))
-    for i in range(len(valmatrix[0])):
-      spec = RunSpec()
-      for j in range(len(valmatrix)):
-        self.vars[j].set(spec, valmatrix[j][i])
-      all.append(spec)
-    return all
-
-class CrossValidate(ParamSet):
-  """
-  WARNING! HASN'T BEEN TRIED YET
-  
-  Runs several runs over the data set, varying which subset is used for
-  training and testing.
-  
-  NOTE TO SELF: Eventually we would want "optimize over cross-validate"
-  to work properly (and not find the best portion to cross validate over).
-  """
-  def __init__(self, varname_train, varname_test, count, *files):
-    """Creates a cross-validation set of runs.
-    
-    The parameter names of train and test are provided, and it is assumed
-    the program being run understands the :x-2/5 and :x2/5 syntax used to
-    denote cross validation.
-    
-    You then provide the number of ways you want to do cross validation, and
-    also the list of file names, or just a single file name, to run over.
-    
-    Example 1: CrossValidate("train", "test", 10, "a.txt")
-    
-    Example 2: CrossValidate("train_set", "test_set", 20, "a.txt", "b.txt")
-    """
-    self.varname_train = varname_train
-    self.var_train = Input(varname_train)
-    self.varname_test = varname_test
-    self.var_test = Input(varname_test)
-    self.files = files
-    self.count = count
-  def enumerate(self):
-    all = []
-    for file in self.files:
-      for i in range(0, count):
-        spec = RunSpec()
-        self.var_train.set(spec, file)
-        self.var_test.set(spec, file)
-        spec.params[self.varname_train + "/subset"] = ("x-%d/%d" % (i, count))
-        spec.params[self.varname_test + "/subset"] = ("x%d/%d" % (i, count))
-        all.append(spec)
-    return all
-
-# Destinations that can be bound to
-
-class BindDest:
-  """Anything that can be bound to, a RunSpec field."""
-  def set(self, spec, val):
-    """Sets the corresponding field in the RunSpec to the
-    given value."""
-    pass
-
-class Var(BindDest):
-  """A regular parameter, such as --length.
-  """
-  def __init__(self, name):
-    self.name = name
-  def set(self, spec, val):
-    spec.params[self.name] = val
-
-class MpiCluster(BindDest):
-  """A parameter that represents running in MPI.
-  """
-  def __init__(self, machinefile):
-    """Sample use:
-      pm.Bind(pm.MpiCluster(os.path.abspath("./amdmachines.txt")), [1, 2, 4, 8, 12])
-    """
-    self.machinefile = machinefile
-  def set(self, spec, val):
-    spec.wrappers += ["mpirun", "-machinefile", self.machinefile, "-np", str(val)]
-    spec.wrapper_info += ["mpi-%s-%s" % (os.path.basename(self.machinefile), str(val))]
-
-class Input(Var):
-  """A parameter that corresponds to be an input file.
-  
-  This field is suitable for enforcing file dependencies.
-  
-  This will also truncate any characters after ':' operator for dependency
-  purposes; the ':' is treated special denoting particular kinds of subsets,
-  used in cross validation.
-  """
-  def set(self, spec, val):
-    spec.inputs.append(val)
-    Var.set(self, spec, val)
-
-class Output(Var):
-  """A parameter that corresponds to be an output file.
-  
-  This field is suitable for enforcing file dependencies.
-  """
-  def set(self, spec, val):
-    spec.outputs.append(str(val))
-    Var.set(self, spec, val)
-
-class Extra(BindDest):
-  """An extra parameter that doesn't directly fit into the
-  fx system.
-  """
-  def set(self, spec, val):
-    spec.extra.append(val)
-
-class Stdin(BindDest):
-  """A parameter that corresponds to be standard input.
-  
-  This field is suitable for enforcing file dependencies.
-  """
-  def set(self, spec, val):
-    spec.stdin = str(val)
-
-class Stdout(BindDest):
-  """A parameter that corresponds to be standard output.
-  
-  This field is suitable for enforcing file dependencies.
-  """
-  def set(self, spec, val):
-    spec.stdout = str(val)
-
-class Binfile(BindDest):
-  """The binary file to be executed.
-  """
-  def set(self, spec, val):
-    spec.binfile = str(val)
-
-# Set of values that can be bound to a destination
-
-class ValSet:
-  """An arbitrary set of values that may be bound to a variable.
-  """
-  def enumerate(self):
-    return []
-
-class Vals(ValSet):
-  """A pre-specified enumeration of values.
-  """
-  def __init__(self, *vals):
-    self.vals = vals
-  def enumerate(self):
-    return self.vals
-
-class Val(Vals):
-  """A pre-specified single value.
-  """
-  def __init__(self, val):
-    Vals.__init__(self, val)
-
-# Tests
-
-class ParamTest(unittest.TestCase):
-  def setUp(self):
-    self.params = Combine(
-      Bind(Binfile(), Val("/usr/bin/sort")),
-      Bind(Input("infile"), Vals("in1.txt", "in2.txt")),
-      Bind(Var("useless"), Vals(1.1, 1.3, 1.7)),
-      CoBind(Var("bw1"), Vals(1, 2, 4, 8, 16), Var("bw2"), Vals(0, 1, 2, 3, 4)))
-  
-  def test_print(self):
-    # TODO - Doesn't test anything
-    self.params.print_all()
-  
-  def test_len(self):
-    self.assertEqual(30, len(self.params.enumerate()))
-
-  def test_product(self):
-    all = self.params.enumerate()
-    def count(param, val):
-      return len([x for x in all if x.params[param] == val])
-    self.assertEqual(15, count("infile", "in1.txt"))
-    self.assertEqual(10, count("useless", 1.3))
-
-if __name__ == "__main__":
-  unittest.main()

fastlib/py/util.py

@@ -1,499 +0,0 @@
-import random
-import re
-import os
-import sys
-import StringIO
-import unittest
-
-# TODO: this isn't our own work, I don't remember where it was pulled from
-def natsort_key(item):
-  chunks = re.split('(\d+(?:\.\d+)?)', item)
-  for ii in range(len(chunks)):
-    if chunks[ii] and chunks[ii][0] in '0123456789':
-      if '.' in chunks[ii]:
-        numtype = float
-      else:
-        numtype = int
-        chunks[ii] = (0, numtype(chunks[ii]))
-    else:
-      chunks[ii] = (1, chunks[ii])
-  return (chunks, item)
-
-def natsort(seq):
-  l = list(seq)
-  l.sort(key=natsort_key)
-  return l
-
-def sorted(l):
-  """Returns the sorted version of the list.
-  
-  (Remove once everyone is running Python 2.4+)
-  """
-  l_copy = list(l)
-  l_copy.sort()
-  return l_copy
-
-def map_values(f, d):
-  """Returns a copy of the dictionary but with all the values mapped."""
-  return dict([(k, f(v)) for (k, v) in d.items()])
-
-def map_keys(f, d):
-  """Returns a copy of the dictionary but with all the values mapped."""
-  return dict([(f(k), v) for (k, v) in d.items()])
-
-def dicthash(d):
-  """Turns a dictionary into something hashable."""
-  return tuple(sorted(d.items()))
-
-def collapse_once(collection_of_collection):
-  """Turns a list of lists into just the items."""
-  result = []
-  for collection in collection_of_collection:
-    result += collection
-  return result
-
-def remove_dir_recursive(dirname):
-  """Removes a directory like rm -rf."""
-  for subname in os.listdir(dirname):
-    name = os.path.join(dirname, subname)
-    if os.path.isdir(name):
-      remove_dir_recursive(name)
-    else:
-      os.remove(name)
-  os.rmdir(dirname)
-
-
-def testfile(filename):
-  """Tests if a file exists.
-  """
-  return os.access(filename, os.F_OK)
-
-def createlock(filename):
-  """Tries to lock a file for writing.
-  
-  Currently this is a stub and just tests for existence, but there is no
-  real locking semantics.
-  """
-  # TODO: Mode operation
-  if testfile(filename):
-    return False
-  try:
-    os.open(filename, os.O_CREAT|os.O_EXCL, 0660)
-    return True
-  except OSError:
-    return False
-
-def writefile(filename, text):
-  """Writes the text to a file by name.
-  """
-  f = open(filename, "w")
-  try:
-    f.write(text)
-  finally:
-    f.close()
-
-def readfile(filename):
-  """Reads the text from a file.
-  """
-  f = open(filename, "r")
-  try:
-    text = f.read()
-  finally:
-    f.close()
-  return text
-
-def writelines(filename, lines):
-  """Writes each line to the specified file.
-  
-  The Unix newline character will be appended to each line.
-  """
-  # TODO: Unix versus Dos CR/LV
-  f = open(filename, "w")
-  try:
-    f.writelines(["%s\n" % line for line in lines])
-  finally:
-    f.close()
-  return lines
-
-def readlines(filename):
-  """Reads each line from a file to a list, with all whitespace
-  stripped from the end of each line.
-  """
-  f = open(filename, "r")
-  try:
-    lines = f.readlines()
-  finally:
-    f.close()
-  lines = [ l.rstrip() for l in lines ]
-  return lines
-
-def read_csv(fname):
-  """Reads a comma-separated-value file as a matrix.
-  """
-  f = open(fname, "r")
-  try:
-    return read_csv_file(f)
-  finally:
-    f.close()
-
-def read_csv_file(f):
-  """Reads an open comma-separated-value file as a matrix.
-  """
-  return [[s.strip() for s in l.split(",")] for l in f.readlines()]
-
-def write_csv(fname, lines):
-  """Writes the specified matrix as a comma-separated-value file.
-  """
-  f = open(fname, "w")
-  try:
-    write_csv_file(f, lines)
-  finally:
-    f.close()
-
-def write_csv_file(f, lines):
-  """Writes the specified matrix as a comma-separated-value open file.
-  """
-  for line in lines:
-    sanitized = [ str(field).replace(",", ";") for field in line ]
-    f.write(", ".join(sanitized) + "\n")
-
-def escape_latex(str):
-  result = ""
-  for c in str:
-    if c == '\\':
-      result += "$\\backslash$"
-    elif c in "#%&~$_^{}":
-      result += "\\" + c
-    else:
-      result += c
-  return result
-
-def write_latex_table_file(f, lines, align = "r"):
-  """Writes specified text to a file as a latex table.
-  """
-  def formatline(line):
-    return " & ".join([escape_latex(field) for field in line]) + " \\\\"
-  
-  max_width = max([len(line) for line in lines])
-  
-  # ensure len(align) equals max_width
-  while len(align) < max_width:
-    align += align[-1]
-  align = align[0:max_width]
-  
-  f.write("\\documentclass[letter]{article}\n")
-  f.write("\\begin{document}\n")
-  f.write("\\begin{tabular}{%s}\n" % ("|".join(align)))
-  f.write(formatline(lines[0]))
-  f.write("\n\\hline")
-  for line in lines[1:]:
-    f.write(formatline(line))
-  f.write("\n\\end{tabular}\n")
-  f.write("\\end{document}\n")
-
-def write_random_ints(filename, count):
-  """Writes a sequence of random integers to a file.
-  """
-  nums = [ "%d" % random.randint(0, 99999999) for i in range(count) ]
-  writelines(filename, nums)
-
-def shellquote(s):
-  """Quotes shell parameters.
-  
-  Note that things like newlines and special characters are included literally
-  in the string, compliant with BASH.
-  
-  Perhaps another shellquote function should be written, which uses the
-  $"string" format, that allows C-like escaping.
-  """
-  result = ""
-  map = {"$":"\\$", "\"":"\\\"", "`":"\\`", "!":"\\!"}
-  for c in str(s):
-    if c in map.keys():
-      result += map[c]
-    else:
-      result += c
-  return "\"%s\"" % result
-
-def sanitize_basename(s):
-  """Sanitizes the base of a filename.
-  
-  This is used in the naming of runs.
-  """
-  result = ""
-  allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ=-_0123456789"
-  for c in str(s):
-    if c in allowed:
-      result += c
-    elif c in "./":
-      result += "-"
-  return result
-
-def ensuredir(dirpath):
-  """Ensure that a directory exists.
-  """
-  if (not os.access(dirpath, os.R_OK|os.W_OK)):
-    os.makedirs(dirpath)
-
-def spawn_redirect(rundir, args, infile = None, outfile = None, errfile = None):
-  """Spawn a new process, optionally redirecting one of standard in, standard
-  out, or standard error to the specified filenames.
-  
-  - rundir: the directory to change to when running the program
-  - args: the arguments, where args[0] is the file to run
-  - infile: filename of standard in or None
-  - outfile: filename of standard out or None
-  - errfile: filename of standard error or None
-  """
-  pid = os.fork()
-  if not pid:
-    try:
-      if rundir != None:
-        os.chdir(rundir)
-      if not infile:
-        infile = "/dev/null"
-      if infile != None:
-        infd = os.open(infile, os.O_RDONLY)
-        os.close(0)
-        os.dup2(infd, 0)
-        os.close(infd)
-      else:
-        os.close(0)
-      if outfile != None:
-        outfd = os.open(outfile, os.O_WRONLY|os.O_CREAT|os.O_TRUNC)
-        os.close(1)
-        os.dup2(outfd, 1)
-        os.close(outfd)
-      else:
-        outfd = None
-        #os.close(1)
-      if errfile != None:
-        # TODO: Mode 755?
-        errfd = os.open(errfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND)
-        os.close(2)
-        os.dup2(errfd, 2)
-        os.close(errfd)
-      else:
-        errfd = None
-        #os.close(2) -- Show standard error
-      os.execvp(args[0], args)
-    finally:
-      # WALDO
-      try:
-        if infile:
-          os.close(0)
-      except:
-        pass
-      try:
-        if outfile:
-          os.close(1)
-      except:
-        pass
-      try:
-        if errfile:
-          os.close(2)
-      except:
-        pass
-      os._exit(1)
-  (p, status) = os.waitpid(pid, 0)
-  # Status is a 16-bit number.
-  # The right 8 bits of status is the signal killed (always zero in case
-  # of Windows).
-  # The left eight bits are set to the exit value.
-  if status & 0xFF != 0:
-    return -1 # It was killed -- the right eight bits are non-zero
-  else:
-    return status >> 8
-
-def shell(args, infile = None, outfile = None, errfile = None):
-  result = spawn_redirect(None, args, infile, outfile, errfile)
-  if result != 0:
-    print "  shell: cannot do: ", args
-    raise OSError()
-
-class ParseError:
-  """Parse error when unqouting.
-  """
-  pass
-
-
-def __dspath_allowed_compute():
-  global __dspath_allowed
-  __dspath_allowed = {}
-  l = range(ord('A'), ord('Z'))
-  l += range(ord('a'), ord('z'))
-  l += range(ord('0'), ord('9'))
-  l += [ord('.'), ord('-'), ord('_')]
-  for c in l:
-    __dspath_allowed[chr(c)] = True
-
-__dspath_allowed_compute()
-
-def escape_dspath(str):
-  """Quote method used for DataNode path elements, encoded in HTML-like
-  format.
-  
-  Only alphanumeric characters, the period, plus sign, minus sign, and
-  underscore are preserved.  Everything else uses a percent (%) sign and
-  a 2-digit hex code.  This is not due to necessity as much as it
-  is convenient that these characters won't need to be escaped for instance
-  if you type them from a shell, or in a regular expression.
-  
-  (TODO: Decide if the plus sign should also be escaped, for HTML
-  purposes.)
-  """
-  global __dspath_allowed
-  result = ""
-  for c in str:
-    if c in __dspath_allowed.keys():
-      result += c
-    else:
-      result += "%%%02X" % ord(c)
-  return result
-
-"""This represents the special star character.
-
-As a string, it looks like only a single star, but it is actually a list.
-"""
-STAR=tuple('*')
-
-def unescape_dspath(s):
-  """Decodes DataNode path elements, by interpreting the special
-  percent character.
-  """
-  try:
-    if s == '*':
-      return STAR
-    else:
-      result = ""
-      hexdigits = "0123456789ABCDEF"
-      i = 0
-      while i < len(s):
-        c = s[i]
-        if c == '%':
-          result += chr(hexdigits.index(s[i+1]) * 16 + hexdigits.index(s[i+2]))
-          i += 3
-        else:
-          result += c
-          i += 1
-      return result
-  except:
-    raise ParseError()
-
-def combine_path(elements):
-  """Combines and escapes path elements.
-  """
-  result = ""
-  for e in elements:
-    result += "/" + escape_dspath(e)
-  return result
-
-def split_path(path):
-  """Splits and unescapes path elements.
-  """
-  return [unescape_dspath(x) for x in path.strip("/").split("/")]
-
-def escape_sexpression(str):
-  """s-expression quoting hack."""
-  result = ""
-  for c in str:
-    if c.isalnum():
-      result += c
-    else:
-      result += "_%02X" % ord(c)
-  return result
-
-def unescape_sexpression(s):
-  """Decodes sexpression-escaped elements, by
-  interpreting the special underscore character.
-  """
-  try:
-    result = ""
-    hexdigits = "0123456789ABCDEF"
-    i = 0
-    while i < len(s):
-      c = s[i]
-      if c == '_':
-        result += chr(hexdigits.index(s[i+1]) * 16 + hexdigits.index(s[i+2]))
-        i += 3
-      else:
-        result += c
-        i += 1
-    return result
-  except:
-    raise ParseError()
-
-def keys(collection):
-  """Returns the keys of a collection, whether it is an array or dict.
-  
-  If it is a list, it returns range(len(x)), otherwise x.keys().
-  """
-  if isinstance(collection, list):
-    return range(len(collection))
-  else:
-    return collection.keys()
-
-def typeconvert_list(items, *type_func_pairs):
-  return [typeconvert(item, *type_func_pairs) for item in items]
-
-def typeconvert(item, *type_func_pairs):
-  for (type, func) in type_func_pairs:
-    if isinstance(item, type):
-      item = func(item)
-      break
-  return item
-
-
-class Table:
-  def __init__(self, headings, matrix = []):
-    self.headings = headings
-    self.matrix = matrix
-  def lookup(self, column):
-    if not isinstance(column, int):
-      column = self.headings.index(column)
-    return column
-  def restricted_fn(self, column, testfn):
-    column = self.lookup(column)
-    newmatrix = [line for line in newmatrix if testfn(line[column])]
-    return Table(self.headings, newmatrix)
-  def restricted(self, column, allowed):
-    if not isinstance(allowed, list):
-      allowed = [allowed]
-    column = self.lookup(column)
-    newmatrix = [line for line in self.matrix if line[column] in allowed]
-    return Table(self.headings, newmatrix)
-
-def read_table(fname):
-  """Reads the text from a file.
-  """
-  f = open(fname, "r")
-  try:
-    return read_table_file(f)
-  finally:
-    f.close()
-
-def read_table_file(f, headings = None):
-  return table_from_matrix(read_csv_file(f), headings)
-
-def table_from_matrix(matrix, headings = None):
-  if not headings:
-    headings = matrix[0]
-    matrix = matrix[1:]
-  table = Table(headings, matrix)
-  return table
-
-
-# TODO: Unit tests
-
-class UtilTest(unittest.TestCase):
-  def test_typeconvert(self):
-    self.assertEqual("1", typeconvert(1, (int, str)))
-    self.assertEqual(1, typeconvert(1, (str, int)))
-    self.assertEqual([1], typeconvert([1], (int, str)))
-  def test_typeconvert_list(self):
-    self.assertEqual(["1", "2", "3", "4"],
-        typeconvert_list([1, 2, 3, "4"], (int, str)))
-
-if __name__ == "__main__":
-  unittest.main()
-  test_matching()

fastlib/py/work.py

@@ -1,152 +0,0 @@
-import os
-import datastore
-import util
-
-# TODO: Who translates?
-
-class WorkEntry:
-  """A single entry on the work queue.
-  
-  Important attributes:
-  - binfile: the binary file to execute
-  - params: parameters passed, as a raw list
-  - inputs: all files required by this stage
-  - outputs: all files written by this stage
-  """
-  
-  INPUT_NOT_AVAILABLE = "input_not_available"
-  IN_PROGRESS = "in_progress"
-  FINISHED = "finished"
-  ERROR = "error"
-  STATES = [INPUT_NOT_AVAILABLE, IN_PROGRESS, FINISHED, ERROR]
-  
-  STATUSFILE = "STATUS"
-  LOGFILE = "LOG"
-  
-  def __init__(self, experiment, runspec):
-    """Creates a new work queue entry.
-    """
-    self.experiment = experiment
-    self.runname = runspec.generate_name()
-    self.rundir = experiment.pathof_run(self.runname)
-    self.runspec = runspec
-    self.logfile = os.path.join(self.rundir, WorkEntry.LOGFILE)
-    self.statusfile = os.path.join(self.rundir, WorkEntry.STATUSFILE)
-  
-  def to_datastore(self, node):
-    #TODO: FINISH
-    node.get_subnode_create()
-    self.fxsys.to_datastore(node)
-    node.set_val_path("/rundir", self.rundir)
-    # TODO: serialize the runspec
-  
-  def to_bash(self):
-    conditions = []
-    conditions.append("! -e %s" % (util.shellquote(self.statusfile)))
-    for input in self.runspec.inputs:
-      conditions.append("-e %s" % util.shellquote(input))
-    if self.runspec.stdin != None:
-      conditions.append("-e %s" % util.shellquote(self.runspec.stdin))
-    command = self.runspec.to_command()
-    id = util.shellquote(os.path.basename(self.runname))
-    lines = [
-        "echo %s" % id,
-        "if %s" % " && ".join([ "[ %s ]" % x for x in conditions]),
-        "then echo ' ... Starting'",
-        "mkdir -p %s" % util.shellquote(self.rundir),
-        "cd %s" % util.shellquote(self.rundir),
-        "echo '%s' >%s" % (WorkEntry.IN_PROGRESS, WorkEntry.STATUSFILE),
-        "%s 2>%s" % (command, util.shellquote(self.logfile)),
-        "RV=$?",
-        "if [ \"$RV\" == 0 ]",
-        "then echo '%s' >%s" % (WorkEntry.FINISHED, WorkEntry.STATUSFILE),
-        "echo ' ... Done!'",
-        "else echo '%s' >%s" % (WorkEntry.ERROR, WorkEntry.STATUSFILE),
-        "echo ' ... Error' $RV",
-        "fi",
-        "fi"
-    ]
-    return "; ".join(lines)
-  
-  def inline_exec(self):
-    # TODO: Recover from errors
-    print "%s" % self.runname
-    try:
-      for infile in self.runspec.inputs:
-        if not os.path.exists(infile):
-          return self.INPUT_NOT_AVAILABLE
-      if not os.path.exists(self.rundir):
-        util.ensuredir(self.rundir)
-    except OSError:
-      print "ERROR: Error reading input files: %s" % (str(self.runspec.inputs))
-      return self.ERROR
-    try:
-      if not util.createlock(self.statusfile):
-        try:
-          lines = util.readlines(self.statusfile)
-          WorkEntry.STATES.index(lines[0]) # raise exception if not a valid state
-          return lines[0]
-        except OSError:
-          print "ERROR: Could not read status file %s." % self.statusfile
-          return self.ERROR
-      else:
-        print " ... %s" % self.runspec.to_command()
-        util.writefile(self.statusfile, self.IN_PROGRESS)
-        retval = util.spawn_redirect(
-          self.rundir, self.runspec.to_args(),
-          self.runspec.stdin, self.runspec.stdout, self.logfile)
-        if retval != 0:
-          status = self.ERROR
-          print " ... Error %d " % retval
-          print util.readfile(self.logfile)
-        else:
-          status = self.FINISHED
-          print " ... Done!"
-        util.writefile(self.statusfile, status)
-        return status
-    except WorkQueue:
-      try:
-        util.writefile(self.statusfile, self.ERROR)
-      except:
-        pass
-      return self.ERROR
-
-class WorkQueue:
-  """A list of work that needs to be done.
-  
-  This currently requires to be added in a dependency-friendly order.
-  """
-  
-  def __init__(self, experiment):
-    """Creates an empty WorkQueue for the specified system.
-    """
-    
-    self.experiment = experiment
-    self.path = experiment.path
-    self.entries = []
-  
-  def add(self, entries):
-    self.entries.extend(entries)
-  
-  def reorder(self):
-    # sort by inputs and outputs
-    pass
-  
-  def to_bash_lines(self):
-    return ["#!/bin/bash"] + [x.to_bash() for x in self.entries]
-  
-  def write(self):
-    fname = os.path.join(self.path, "workqueue.sh")
-    f = open(fname, "a")
-    for line in self.to_bash_lines():
-      f.write(line)
-      f.write("\n")
-    f.close()
-    return fname
-  
-  def inline_exec(self):
-    # TODO: Handle the return value
-    for item in self.entries:
-      result = item.inline_exec()
-      print result
-