diff --git a/fastlib/script/fx-rpc b/fastlib/script/fx-rpc index 51a4b605f9..6ddf876e5d 100755 --- a/fastlib/script/fx-rpc +++ b/fastlib/script/fx-rpc @@ -60,7 +60,12 @@ try: chdir_prefix = ["cd", cwd, "&&"] if not do_gdb: remote_command = [executable] + user_args + rpc_args + # arguments: -x -T -n means no X11, no standard input, no TTY + # StrictHostKeyChecking=no means don't prompt "Do you really want to connect?" + # We turn off number of password prompts. We'd rather SSH die than + # sit indefinitely waiting for a password. argv = ["ssh", "-x", "-T", "-n", "-o", "StrictHostKeyChecking=no", + "-o", "NumberOfPasswordPrompts=0", machinename, " ".join(chdir_prefix + remote_command)] else: filename = os.path.abspath("fx-rpc.gdb.%d" % rank) @@ -75,6 +80,7 @@ try: if not pid: os.execvp(argv[0], argv) pids[pid] = rank + time.sleep(0.5) while pids: (pid, status) = os.wait() diff --git a/fastlib/script/fxsys.py b/fastlib/script/fxsys.py index 97ae240f96..76423f9723 100644 --- a/fastlib/script/fxsys.py +++ b/fastlib/script/fxsys.py @@ -108,7 +108,7 @@ class AbstractExperiment: try: os.unlink(statusfile) except: - print "... Status file was already invalid. This run can be re-run." + print "... Status file was already cleared. This run can be re-run." elif nuke: print "NUKING COMPLETED RUN %s" % runname try: diff --git a/fastlib/script/pm.py b/fastlib/script/pm.py index 53327f496c..ae0d495e35 100644 --- a/fastlib/script/pm.py +++ b/fastlib/script/pm.py @@ -115,7 +115,7 @@ class ParamSet: run.""" for e in self.enumerate(): #print e.to_command() - print e.generate_name() + ": " + e.binfile + " " + " ".join(e.param_args) + print e.generate_name(False) + ": " + e.binfile + " " + " ".join(e.param_args) class Combine(ParamSet): """Cartesian product, or all combinations, diff --git a/fastlib/script/work.py b/fastlib/script/work.py index 811dc76f64..e4be145928 100644 --- a/fastlib/script/work.py +++ b/fastlib/script/work.py @@ -92,18 +92,21 @@ class WorkEntry: except OSError: print (util.ansi.HRED+"ERROR: Could not read status file %s."+util.ansi.CLEAR) % self.statusfile return self.ERROR - print (util.ansi.HCYAN+" ... %s"+util.ansi.CLEAR) % self.runspec.to_command() - util.writefile(self.statusfile, self.IN_PROGRESS) - retval = util.spawn_redirect( - self.rundir, self.runspec.to_args(), - self.runspec.stdin, self.runspec.stdout, self.logfile) - if retval != 0: - status = self.ERROR - print (util.ansi.HRED+" ... Returned error code: %d "+util.ansi.CLEAR) % retval - print "".join([" | %s\n" for line in util.readlines(self.logfile)]), - else: - status = self.FINISHED - print (util.ansi.HGREEN+" ... Done!"+util.ansi.CLEAR) + for i in range(2): + # Try each run twice, to allow for transient errors. + print (util.ansi.HCYAN+" ... %s"+util.ansi.CLEAR) % self.runspec.to_command() + util.writefile(self.statusfile, self.IN_PROGRESS) + retval = util.spawn_redirect( + self.rundir, self.runspec.to_args(), + self.runspec.stdin, self.runspec.stdout, self.logfile) + if retval != 0: + status = self.ERROR + print (util.ansi.HRED+" ... Returned error code: %d "+util.ansi.CLEAR) % retval + print "".join([" | %s\n" for line in util.readlines(self.logfile)]), + else: + status = self.FINISHED + print (util.ansi.HGREEN+" ... Done!"+util.ansi.CLEAR) + break util.writefile(self.statusfile, status) return status except WorkQueue: