hi

fastlib/script/fx-rpc

@@ -60,7 +60,12 @@ try:
     chdir_prefix = ["cd", cwd, "&&"]
     if not do_gdb:
       remote_command = [executable] + user_args + rpc_args
+      # arguments: -x -T -n means no X11, no standard input, no TTY
+      # StrictHostKeyChecking=no means don't prompt "Do you really want to connect?"
+      # We turn off number of password prompts.  We'd rather SSH die than
+      # sit indefinitely waiting for a password.
       argv = ["ssh", "-x", "-T", "-n", "-o", "StrictHostKeyChecking=no",
+          "-o", "NumberOfPasswordPrompts=0",
           machinename, " ".join(chdir_prefix + remote_command)]
     else:
       filename = os.path.abspath("fx-rpc.gdb.%d" % rank)
@@ -75,6 +80,7 @@ try:
     if not pid:
       os.execvp(argv[0], argv)
     pids[pid] = rank
+    time.sleep(0.5)
 
   while pids:
     (pid, status) = os.wait()

fastlib/script/fxsys.py

@@ -108,7 +108,7 @@ class AbstractExperiment:
           try:
             os.unlink(statusfile)
           except:
-            print "... Status file was already invalid.  This run can be re-run."
+            print "... Status file was already cleared.  This run can be re-run."
       elif nuke:
         print "NUKING COMPLETED RUN %s" % runname
         try:

fastlib/script/pm.py

@@ -115,7 +115,7 @@ class ParamSet:
     run."""
     for e in self.enumerate():
       #print e.to_command()
-      print e.generate_name() + ": " + e.binfile + " " + " ".join(e.param_args)
+      print e.generate_name(False) + ": " + e.binfile + " " + " ".join(e.param_args)
 
 class Combine(ParamSet):
   """Cartesian product, or all combinations,

fastlib/script/work.py

@@ -92,18 +92,21 @@ class WorkEntry:
         except OSError:
           print (util.ansi.HRED+"ERROR: Could not read status file %s."+util.ansi.CLEAR) % self.statusfile
           return self.ERROR
-      print (util.ansi.HCYAN+" ... %s"+util.ansi.CLEAR) % self.runspec.to_command()
-      util.writefile(self.statusfile, self.IN_PROGRESS)
-      retval = util.spawn_redirect(
-        self.rundir, self.runspec.to_args(),
-        self.runspec.stdin, self.runspec.stdout, self.logfile)
-      if retval != 0:
-        status = self.ERROR
-        print (util.ansi.HRED+" ... Returned error code: %d "+util.ansi.CLEAR) % retval
-        print "".join([" | %s\n" for line in util.readlines(self.logfile)]),
-      else:
-        status = self.FINISHED
-        print (util.ansi.HGREEN+" ... Done!"+util.ansi.CLEAR)
+      for i in range(2):
+        # Try each run twice, to allow for transient errors.
+        print (util.ansi.HCYAN+" ... %s"+util.ansi.CLEAR) % self.runspec.to_command()
+        util.writefile(self.statusfile, self.IN_PROGRESS)
+        retval = util.spawn_redirect(
+          self.rundir, self.runspec.to_args(),
+          self.runspec.stdin, self.runspec.stdout, self.logfile)
+        if retval != 0:
+          status = self.ERROR
+          print (util.ansi.HRED+" ... Returned error code: %d "+util.ansi.CLEAR) % retval
+          print "".join([" | %s\n" for line in util.readlines(self.logfile)]),
+        else:
+          status = self.FINISHED
+          print (util.ansi.HGREEN+" ... Done!"+util.ansi.CLEAR)
+          break
       util.writefile(self.statusfile, status)
       return status
     except WorkQueue: