import util

import unittest
import os
import StringIO
import operator
import copy
import sys

class SexpressionStream:
  """Stream for tokenizing an s-expressions."""
  def __init__(self, file):
    self.file = file
    self.nextchar()
  def nextchar(self):
    self.peek = self.file.read(1)
    return self.peek
  def skip(self):
    while True:
      if self.peek == ';':
        while self.peek != "" and self.peek != "\n" and self.peek != "\r":
          self.nextchar()
      elif self.peek.isspace():
        self.nextchar()
      else:
        break
  def readstr(self):
    self.skip()
    name = ""
    while self.peek.isalnum() or self.peek == "_":
      name += self.peek
      self.nextchar()
    return util.unescape_sexpression(name)
  def is_done(self):
    return self.peek == ""

class UnknownField:
  def __init__(self, message):
    self.message = message
  def __repr__(self):
    return message

class DataNode:
  """Abstract hierarchial key-value data stucture.
  
  This is intended more for small data sets, like the output
  of a single C program, where the overhead of any more complex
  outut mechanism would be a ridiculous annoyance.
  
  Thinking of the Windows registry should convince you not to use this
  for large files.
  """
  def __init__(self, val = None):
    """Create an empty data store.
    """
    self.subnodes = {}
    self.val = val

  def get_node(self, path_elements):
    """Gets an node via a particular path sequence.
    """
    node = self
    for element in path_elements:
      node = node.subnodes[element]
    return node

  def get_node_create(self, path_elements):
    """Gets an node via a particular path sequence.
    """
    node = self
    for element in path_elements:
      child = node.subnodes.get(element, None)
      if not child:
        child = DataNode()
        node.subnodes[element] = child
      node = child
    return node
  
  def get_subnode_create(self, name):
    """Gets a sub-node, creating one if it doesn't exist.
    """
    node = self.subnodes.get(name, None)
    if not node:
      node = DataNode()
      self.subnodes[name] = node
    return node

  def get_val(self, path_elements):
    """Get a string leaf node by its path sequence.
    
    Returns None if no such element was found.
    """
    try:
      return self.get_node(path_elements).val
    except:
      return None

  def get_val_path(self, path):
    """Get a string leaf node by its escaped-and-slash-separated path.
    """
    try:
      return self.get_node(util.split_path(path)).val
    except:
      return None

  def set_val(self, path_elements, val):
    """Sets a leaf string by a particular sequence of path elements.
    """
    self.get_node_create(path_elements).val = val

  def set_val_path(self, path, val):
    """Sets a leaf string by a particular escaped-and-slash-separated
    path.
    """
    self.get_node_create(util.split_path(path)).val = val

  def __write_pathstyle_impl(self, file, prefix):
    """Implementation method for writing in path-style output format.
    """
    if self.val != None:
      file.write(prefix + " " + util.escape_dspath(self.val) + "\n")
    for k, v in self.subnodes.iteritems():
      v.__write_pathstyle_impl(file, prefix + "/" + util.escape_dspath(k))

  def __write_sexpression_impl(self, file, prefix):
    """Implementation method for writing in sexpression-style output format.
    """
    if self.val != None:
      file.write(util.escape_sexpression(self.val))
    for k, v in self.subnodes.iteritems():
      file.write("\n" + prefix + "(" + util.escape_sexpression(k) + " ")
      v.__write_sexpression_impl(file, prefix + "  ")
      file.write(")")

  def write_pathstyle(self, file, prefix = ""):
    """Writes the data structure to the output in path-style format,
    where every line is a data node, where the left side of the line
    is an escaped and slash-separated path and the right side is a
    string.
    """
    self.__write_pathstyle_impl(file, prefix)

  def write_sexpression(self, file, prefix = ""):
    """Writes the data structure to the output in sexpression format.
    """
    file.write("; Fx s-expression writer version 1.0")
    self.__write_sexpression_impl(file, prefix)
    file.write("\n")

  def read_pathstyle(self, file):
    """Read a flat-file in key-value format.
    """
    for line in file:
      try:
        space = line.index(" ")
        path = line[:space]
        value = line[space+1:].rstrip()
        self.set_val_path(path, util.unescape_dspath(value))
      except ValueError:
        print "Warning: Line ignored: %s" % (line.strip())

  def read_sexpression(self, file):
    """Read s-expression from a file.
    
    This only supports a subset of s-expressions that correspond to nodes.
    There are two examples that are unsupported:
    
    (foo bar baz) is invalid because 'bar' and 'baz' cannot both be associated
    with the same node.
    
    (foo (bar a) (bar b)) also cannot work, because there cannot be two 'bar'
    nodes associated with foo.
    """
    self.__read_sexpression_impl(SexpressionStream(file))

  def __read_sexpression_impl(self, stream):
    try:
      more_subexpressions = True
      while more_subexpressions:
        stream.skip()
        if stream.peek.isalnum() or stream.peek == '_':
          if self.val != None:
            print "Warning: Value reassigned from [%s]." % self.val
          self.val = stream.readstr()
        elif stream.peek == '(':
          # A sub exprsesion
          stream.nextchar()
          name = stream.readstr()
          if name in self.subnodes.keys():
            print "Warning: Node [%s] is specified twice." % name
          self.get_subnode_create(name).__read_sexpression_impl(stream)
          stream.skip()
          if stream.peek != ')':
            raise util.ParseError
          stream.nextchar()
        elif stream.peek == ')':
          more_subexpressions = False
          # do not gobble
        elif stream.peek == '':
          more_subexpressions = False
        else:
          raise util.ParseError()
    except:
      raise util.ParseError()
  
  def select(self, pathspecs):
    """Selects a natural join of several path specifications.
    
    See util.select() for more information.
    """
    return select(self, self.SELECTFUNCTIONS, pathspecs)
  
  def select_to_csv_file(self, file, pathspecs, header = True):
    """Selects a natural join of several path specifications, and write
    to a CSV file.
    
    See util.select() for more information on how the path specifications
    work.
    """
    elements = self.select(pathspecs)
    if header:
      all = [pathspecs] + elements
    else:
      all = elements
    util.write_csv_file(file, all)

  def select_tree(self, matchtree):
    """Selects based on a match tree.
    """
    return matchtree_walk(self, self.SELECTFUNCTIONS, matchtree)

  def __repr__(self):
    """Converts into debugging s-exp representation.
    """
    stream = StringIO.StringIO()
    self.write_sexpression(stream)
    return stream.getvalue()
  
  def __get_typedval(self):
    """Converts into a typed value by guessing its type.
    """
    try:
      return int(self.val)
    except:
      try:
        return float(self.val)
      except:
        return self.val
  
  typedval=property(__get_typedval)
  
  SELECTFUNCTIONS=(lambda x:x.val,
        lambda x:x.subnodes.iterkeys(),
        lambda x:x!=None, lambda x,y:x.subnodes.get(y, None), None)

def sortify(str):
  """Makes the string suitable for sorting by returning a tuple, where
  the left element is typed and the right element is the string.
  """
  val = str
  try:
    val = float(str)
  except:
    pass
  return (val, str)

def subcolumns(col_nums, rows):
  """Extract certain columns.
  
  example:
  
    col_nums = [0, 2, 1]
    rows = [ ["ted", "likes", "shrimp", "always"],
             ["bill", "eats", "pasta", "sometimes"] ]
  
  returns:
  
    [ ["ted", "shrimp", "likes"],
      ["bill", "pasta", "eats] ]
  """
  return [ [ row[i] for i in col_nums ] for row in rows ]

def group(groupcols, rows, remove_key = False):
  """Groups a matrix by particular columns.
  
  Example:
  
     groupcols = [ 0, 1 ]
     
     rows = [ [ 1, 1, 1, 1 ],
              [ 1, 1, 1, 2 ],
              [ 1, 1, 2, 1 ],
              [ 1, 1, 2, 2 ],
              [ 2, 1, 1, 1 ],
              [ 2, 1, 1, 2 ],
              [ 1, 3, 2, 1 ],
              [ 1, 3, 2, 2 ] ]
  
  returns:
  
     groups={(1, 1): [ [ 1, 1, 1, 1 ],
                       [ 1, 1, 1, 2 ],
                       [ 1, 1, 2, 1 ],
                       [ 1, 1, 2, 2 ] ],
             (2, 1): [ [ 2, 1, 1, 1 ],
                       [ 2, 1, 1, 2 ] ],
             (1, 3): [ [ 1, 3, 2, 1 ],
                       [ 1, 3, 2, 2 ] ] }
  
  Note this should work for dictionaries too; if rows is a collection of
  dictionaries, and groupcols corresponds to dictionary keys.
  """
  groups = {}
  
  for row in rows:
    key = tuple([row[x] for x in groupcols])
    if not key in groups.keys():
      groups[key] = []
    if remove_key:
      newrow = [row[i] for i in util.keys(row) if i not in groupcols]
    else:
      newrow = row
    groups[key].append(newrow)
  
  return groups

def groupreduce(groups, reductionpairs):
  """Takes the result of group, and reduces every specified element.
  
  Example:
  
     groups= { (1, 1): [ [ 1, 1, 1, 1 ],
                         [ 1, 1, 1, 2 ],
                         [ 1, 1, 2, 1 ],
                         [ 1, 1, 2, 2 ] ],
               (2, 1): [ [ 2, 1, 1, 1 ],
                         [ 2, 1, 1, 2 ] ],
               (1, 3): [ [ 1, 3, 2, 1 ],
                         [ 1, 3, 2, 2 ] ] }
     
     reductionpairs = [ (2, operator.add), (3, operator.mul) ]
  
  Returns:
  
     reduced = { (1, 1) : [1, 1, 6, 4],
                 (2, 1) : [2, 1, 2, 2],
                 (1, 3) : [1, 3, 4, 2] }
  
  In the "reduced" array, if a column lacks a reduction operator, an
  its value from an arbitrary row will be chosen.
  """
  reduced = {}
  
  for (key, rows) in groups.items():
    newrow = copy.copy(rows[0]) # copy the row
    for row in rows[1:]:
      for (index, reduction) in reductionpairs:
        newrow[index] = reduction(newrow[index], row[index])
    reduced[key] = newrow
  
  return reduced  

def groupsum(groups, indices = [0]):
  """Sums all specified columns of a grouping.
  """
  reductionpairs = []
  for index in indices:
    reductionpairs.append((index, operator.sum))
  return groupreduce(groups, reductionpairs)

def select(place, (handlefn, listfn, existfn, divefn, bridgefn), pathspecs_kv):
  """Yields the desired data on a natural join of pathspecs, sorted.
  
  The pathspecs are a slash-separated list of pathnames as you would expect
  to be stored in a DataNode tree.  Path elements may contain a "*" in
  which case the sub-elements are listed and each traversed.
  
  If two different pathspecs contain a "*", the rules are interesting.
  If both pathspecs are rooted at the same place up to the "*", the two
  pathspecs are traversed together.  Otherwise, a cartesian product is
  done between the two.  Example:
  
  Say our structure looks like:
  
     /runs/1/x = 1x
     /runs/1/y = 1y
     /runs/2/x = 1x
     /runs/2/y = 1y
     /bill/a = a
     /bill/b = b
     /constant = q
  
  ['/q', '/runs/*/x'] repeats the uninteresting one:
    ['q','1x], ['q','2x']
  
  ['/runs/*/x', '/runs/*/y'] yields related pairs:
    ['1x','1y'] , ['2x','2y']
  
  ['/runs/*/x', '/bill/*'] yields all combinations:
    ['1x','a'], ['1x','b'], ['1y','a'], ['1y','b']
  
  ['/runs/*/x', '/runs/*/y', '/bill/*'] yields:
    ['1x','1y','a'], ['1x','1y','b'], ['2x','2y','a'], ['2x','2y','b']
  
  The resulting data is then sorted first by the first column, then the
  second, and so on.  Values that look like numbers are sorted in numerical
  order.
  
  The long tuple of functions are the information necessary for iteration
  (you likely won't have to specify these yourself):
  
  - place: the starting 'place', such as a directory name
  - listfn: a path to list sub-'place's of a place, such as os.listdir
  - existfn: to test the existence of a sub-field like os.path.exists
  - divefn: to take a sub-path and add on a new element, such as os.path.join
  - bridgefn: more complicated; when a ':' appears at the end of an element,
  the divefn is called to get the new 'place' to go to and a new tuple of
  relevant functions.  for example, in select_fields, the ':' operator
  parses the specified file as a DataNode pathstyle file, and plunges into
  the DataNode hierarchy
  
  Also, if a path specification has an "=" sign, this limits the results to
  only those with the particular values specified, i.e. knn/k=2 limits to
  where knn/k is 2.
  """
  matchtree = DataNode()
  pathspecs = []
  pathspec_filters = []
  
  def gen_filter_match_float(index, to_match_float):
    """Generates a numerical-match filter."""
    def filter_match_float(line):
      return float(line[index]) == to_match_float
    return filter_match_float
  def gen_filter_match_str(index, to_match):
    """Generates a string-match filter."""
    def filter_match_str(line):
      return line[index] == to_match
    return filter_match_str
  
  # turn a list of key=val pairs into filters
  for index in range(len(pathspecs_kv)):
    element = pathspecs_kv[index]
    equalsign = element.find("=")
    if equalsign >= 0:
      key = element[:equalsign]
      filter_value = element[equalsign+1:]
      try:
        to_match_float = float(filter_value)
        pathspec_filters += [gen_filter_match_float(index, to_match_float)]
      except:
        pathspec_filters += [gen_filter_match_str(index, filter_value)]
    else:
      key = element
    pathspecs += [key]
  
  for element in pathspecs:
    matchtree.set_val_path(element, element)
  results = matchtree_walk(place,
    (handlefn, listfn, existfn, divefn, bridgefn), matchtree)
  def tolist(map):
    return [ sortify(map.get(x, "")) for x in pathspecs ]
  lines_aug = [ tolist(x) for x in results ]
  lines_aug.sort()
  def unsortify_line(line_aug):
    return [ x[1] for x in line_aug ]
  results = [ unsortify_line(line_aug) for line_aug in lines_aug ]
  
  def matches(line):
    for filter in pathspec_filters:
      if not filter(line):
        return False
    return True
  
  return [ line for line in results if matches(line) ]

def matchtree_walk(place, (handlefn, listfn, existfn, divefn, bridgefn), matchtree):
  """Implements the natural join over an arbitrary hierarchial data backend.
  
  See util.select() for information on the behavior.
  """
  functions = (handlefn, listfn, existfn, divefn, bridgefn)
  def combine(subanswers):
    newanswers = []
    for answer1 in answers:
      for answer2 in subanswers:
        elem = {}
        elem.update(answer1)
        elem.update(answer2)
        newanswers.append(elem)
    return newanswers
  myanswer = {}
  if matchtree.val != None:
    myanswer[matchtree.val] = handlefn(place)
  answers = [myanswer]
  for (matchname, submatchtree) in matchtree.subnodes.iteritems():
    if matchname[-1] == ':':
      processfn = bridgefn
      matchname = matchname[:-1]
    else:
      processfn = lambda x:(x, functions)
    def subwalkfn((subplace, subfunctions), submatchtree):
      return matchtree_walk(subplace, subfunctions, submatchtree)
    if matchname == "*":
      subanswers = []
      for subname in listfn(place):
        subplace = divefn(place, subname)
        returned = subwalkfn(processfn(subplace), submatchtree)
        if returned != [{}]: # special case: directory that completely failed
          subanswers.extend(returned)
      answers = combine(subanswers)
    else:
      subplace = divefn(place, matchname)
      if existfn(subplace):
        answers = combine(subwalkfn(processfn(subplace), submatchtree))
      else:
        pass # unknown directories are ignored
  return answers

def select_fields(rootpath, pathspecs):
  """Selects first from files, and then from their datanode-pathstyle
  contents.
  
  Pathspecs are in the format:
  
    runs/*/FILENAME:/info/params/x
    runs/*/FILENAME:/info/timers/*/cycles
  
  Be careful: Both "FILENAME/:/" and "FILENAME/:info" will choke horribly.
  Be very careful.
  """
  treefunctions = DataNode.SELECTFUNCTIONS
  def bridge(filename):
    f = open(filename, "r")
    try:
      result = DataNode()
      result.read_pathstyle(f)
      return (result, treefunctions)
    finally:
      f.close()
  pathfunctions = (lambda x:x, os.listdir,
    os.path.exists, os.path.join, bridge)
  return select(rootpath, pathfunctions, pathspecs)

def select_fields_to_table_file(openfile, rootpath, pathspecs, writefn = util.write_csv_file):
  """Selects fields first from files and then from the pathstyle contents,
  writing results to an OPEN csv or other type of text file file.
  
  See util.select_fields().
  """
  headers = [x.strip("/").split("/")[-1] for x in pathspecs]
  writefn(openfile, [headers] + select_fields(rootpath, pathspecs))

def select_fields_to_csv_file(csvfile, rootpath, pathspecs):
  """Selects fields first from files and then from the pathstyle contents,
  writing results to an OPEN csv file (or stdout).
  
  See util.select_fields().
  """
  select_fields_to_table_file(csvfile, rootpath, pathspecs, util.write_csv_file)

def select_fields_to_csv(csvfname, rootpath, pathspecs):
  """Selects fields first from files and then from the pathstyle contents,
  writing results to a to-be-created csv file name.
  
  See util.select_fields().
  """
  f = open(csvfname, "w")
  try:
    select_fields_to_csv_file(f, rootpath, pathspecs)
  finally:
    f.close()

def select_files(place, pathspecs):
  """Selects filenames in the natural-join style.
  
  See util.select() for info.
  """
  functions = (lambda x:x, os.listdir, os.path.exists, os.path.join, None)
  return select(place, functions, pathspecs)

def match_files(place, pathspec):
  """Match files based on a single path specification.
  
  For instance runs/*/SUMMARY matches files named SUMMARY in any
  directory called runs.
  """
  return [ x[0] for x in select_files(place, [pathspec]) ]

#----------------------------------------------------------------------------
# Unit tests

class IoTest(unittest.TestCase):
  def setUp(self):
    (tmpdir, tmpfile) = os.path.split(os.tempnam())
    self.tmpdir = "%s/fxsys-util-ut-%s" % (tmpdir, tmpfile)
    self.rootpath = self.tmpdir
    os.makedirs(self.rootpath)
  
  def tearDown(self):
    util.remove_dir_recursive(self.tmpdir)
  
  def test_match_files(self):
    def fix(x):
      return os.path.join(self.rootpath, x)
    matchdirs = ["foo/a/bar/a/baz", "foo/a/bar/b/baz", "foo/a/bar/c/baz",
            "foo/b/bar/a/baz", "foo/b/bar/c/baz",
            "foo/c/bar/b/baz"]
    nonmatchdirs = ["foon/a/bar/a/baz", "foo/g/bar/b/bazz", "foo/a/x", "subsume"]
    for dir in matchdirs + nonmatchdirs:
      os.makedirs(os.path.join(self.rootpath, dir))
    found = match_files(self.rootpath, "foo/*/bar/*/baz")
    found.sort()
    matchdirs.sort()
    matchdirs = [ fix(x) for x in found ]
    self.assertEqual(matchdirs, found)
    
  def test_select_files(self):
    def fix(x):
      return os.path.join(self.rootpath, x)
    leftdirs = ["foo/a/bar/a/baz", "foo/a/bar/b/baz", "foo/a/bar/c/baz",
            "foo/b/bar/a/baz", "foo/b/bar/c/baz",
            "foo/c/bar/b/baz"]
    middirs = ["foo/a/bar/a/bon", "foo/a/bar/b/bon", "foo/a/bar/c/bon",
            "foo/b/bar/a/bon", "foo/b/bar/c/bon",
            "foo/c/bar/b/bon"]
    rightdirs = ["param/a/foo", "param/b/foo", "param/c/foo"]
    nondirs = ["foon/a/bar/a/baz", "foo/g/bar/b/bazz", "foo/a/x", "subsume"]
    for dir in leftdirs + middirs + rightdirs + nondirs:
      os.makedirs(os.path.join(self.rootpath, dir))
    found = select_files(self.rootpath,
        ["foo/*/bar/*/baz", "foo/*/bar/*/bon", "param/*/foo"])
    found.sort()
    expect = []
    for (leftdir, middir) in zip(leftdirs, middirs):
      for rightdir in rightdirs:
        expect.append([fix(leftdir), fix(middir), fix(rightdir)])
    expect.sort()
    found.sort()
    self.assertEqual(expect, found)
    
    

class DatastoreTest(unittest.TestCase):
  def setUp(self):
    self.f = DataNode()
    self.f.get_node_create(["walks", "10", "param", "speed"]).val = "SLOW"
    self.f.get_node_create(["runs", "10", "param"]).val = "lots of params"
    self.f.get_node_create(["runs", "10", "param", "x"]).val = "2"
    self.f.set_val(["runs", "10", "param", "y"], "3")
    self.f.set_val_path("runs/20/param/x", "4")
    self.f.set_val_path("/runs/20/param/y", "5")
    self.f.set_val(["runs", "10", "metric", "total performance"], "very good")
    self.f.set_val_path("/runs/30/param/x", "8");
    self.f.set_val_path("/runs/30/param/y", "9");
    self.f.set_val_path("/runs/40/param/x", "11");
  
  def test_get_val(self):
    self.assertEqual(self.f.get_val(["runs", "10", "param", "x"]), "2");
    self.assertEqual(self.f.get_val(["runs", "10", "param", "y"]), "3");
    self.assertEqual(self.f.get_val(["runs", "20", "param", "x"]), "4");
    self.assertEqual(self.f.get_val(["runs", "20", "param", "y"]), "5");

  def test_get_val_path(self):
    self.assertEqual(self.f.get_val_path("runs/20/param/y"), "5");
    self.assertEqual(self.f.get_val_path("/runs/20/param/y"), "5");
    self.assertEqual(self.f.get_val_path("/runs/10/metric/total%20performance"), "very good");
  
  def help_test_save_get(self, readfun, writefun):
    print "(START)"
    first_outstream = StringIO.StringIO()
    writefun(self.f, first_outstream)
    print first_outstream.getvalue()
    print "(REREAD)"
    f2 = DataNode()
    readfun(f2, StringIO.StringIO(first_outstream.getvalue()))
    second_outstream = StringIO.StringIO()
    writefun(f2, second_outstream)
    print second_outstream.getvalue()
    print "(END)"
    self.assertEqual(first_outstream.getvalue(), second_outstream.getvalue());
  
  def test_pathstyle_save(self):
    self.help_test_save_get(DataNode.read_pathstyle, DataNode.write_pathstyle)
  
  def test_sexpression(self):
    self.help_test_save_get(DataNode.read_sexpression, DataNode.write_sexpression)
  
  def test_select(self):
    selected = self.f.select(["runs/*/param/x", "runs/*/param/y"])
    expected = [
        ['2', '3'],
        ['4', '5'],
        ['8', '9'],
        ['11', '']]
    selected.sort()
    expected.sort()
    self.assertEqual(expected, selected)
  
  def test_group(self):
    groupcols = [ 0, 1 ]
    rows = [ [ 1, 1, 1, 1 ],
             [ 1, 1, 1, 2 ],
             [ 1, 1, 2, 1 ],
             [ 1, 1, 2, 2 ],
             [ 2, 1, 1, 1 ],
             [ 2, 1, 1, 2 ],
             [ 1, 3, 2, 1 ],
             [ 1, 3, 2, 2 ] ]
    expect = { (1, 1): [ [ 1, 1, 1, 1 ],
                      [ 1, 1, 1, 2 ],
                      [ 1, 1, 2, 1 ],
                      [ 1, 1, 2, 2 ] ],
            (2, 1): [ [ 2, 1, 1, 1 ],
                      [ 2, 1, 1, 2 ] ],
            (1, 3): [ [ 1, 3, 2, 1 ],
                      [ 1, 3, 2, 2 ] ] }
    self.assertEqual(expect, group(groupcols, rows))
  
  def test_group_nonint(self):
    k_a = "a"
    k_b = (3.2, "food")
    k_c = "hamburger"
    groupcols = [ k_a, k_b ]
    rows = [ { k_a:1, k_b:1, k_c:1, "d":1 },
             { k_a:1, k_b:1, k_c:1, "d":2 },
             { k_a:1, k_b:1, k_c:2, "d":1 },
             { k_a:1, k_b:1, k_c:2, "d":2 },
             { k_a:2, k_b:1, k_c:1, "d":1 },
             { k_a:2, k_b:1, k_c:1, "d":2 },
             { k_a:1, k_b:3, k_c:2, "d":1 },
             { k_a:1, k_b:3, k_c:2, "d":2 } ]
    expect = { (1, 1): [ { k_a:1, k_b:1, k_c:1, "d":1 },
                         { k_a:1, k_b:1, k_c:1, "d":2 },
                         { k_a:1, k_b:1, k_c:2, "d":1 },
                         { k_a:1, k_b:1, k_c:2, "d":2 } ],
               (2, 1): [ { k_a:2, k_b:1, k_c:1, "d":1 },
                         { k_a:2, k_b:1, k_c:1, "d":2 } ],
               (1, 3): [ { k_a:1, k_b:3, k_c:2, "d":1 },
                         { k_a:1, k_b:3, k_c:2, "d":2 } ] }
    self.assertEqual(expect, group(groupcols, rows))
  
  def test_subcolumns(self):
    col_nums = [0, 2, 1]
    rows = [ ["ted", "likes", "shrimp", "always"],
             ["bill", "eats", "pasta", "sometimes"] ]
    expect = [ ["ted", "shrimp", "likes"],
      ["bill", "pasta", "eats"] ]
    self.assertEqual(expect, subcolumns(col_nums, rows))
  
  def test_groupreduce(self):
    groups= { (1, 1): [ [ 1, 1, 1, 1 ],
                        [ 1, 1, 1, 2 ],
                        [ 1, 1, 2, 1 ],
                        [ 1, 1, 2, 2 ] ],
              (2, 1): [ [ 2, 1, 1, 1 ],
                        [ 2, 1, 1, 2 ] ],
              (1, 3): [ [ 1, 3, 2, 1 ],
                        [ 1, 3, 2, 2 ] ] }
    reductionpairs = [ (2, operator.add), (3, operator.mul) ]
    reduced = { (1, 1) : [1, 1, 6, 4],
                (2, 1) : [2, 1, 2, 2],
                (1, 3) : [1, 3, 4, 2] }
    self.assertEqual(reduced, groupreduce(groups, reductionpairs))
  
  def test_csv_file_head(self):
    stream = StringIO.StringIO()
    self.f.select_to_csv_file(stream, ["runs/*/param/x", "runs/*/param/y"])
    expected = "runs/*/param/x, runs/*/param/y\n2, 3\n8, 9\n4, 5\n11, \n";
    self.assertEqual(util.sorted(expected.split("\n")), util.sorted(stream.getvalue().split("\n")))
  
  def test_csv_file_nohead(self):
    stream = StringIO.StringIO()
    self.f.select_to_csv_file(stream, ["runs/*/param/x", "runs/*/param/y"], False)
    expected = "2, 3\n8, 9\n4, 5\n11, \n";
    self.assertEqual(util.sorted(expected.split("\n")), util.sorted(stream.getvalue().split("\n")))

if __name__ == "__main__":
  unittest.main()
  test_matching()