calibration_train/HDJobSubmitterLocal.py at master · sdobbs/calibration_train · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
# Class to run jobs on the local machine.  Can handle default
# data locations at JLab either inside the counting house at the batch farm
#
# Author: Sean Dobbs (s-dobbs@northwestern.edu), 2015

import os
from subprocess import Popen, PIPE, call

import HDJobUtils
import HDRunFileRAIDList

class HDJobSubmitterLocal:
    """
    def
    """

    def __init__(self,basedir):
        self.VERBOSE = 1
        self.basedir = basedir
        # job defaults
        self.nthreads = 1        # default to 1 thread

    def RunJobs(self, runfile_mapping, server_run_map):
        """
        Runs

        The basic job flow is for each run
        - analyze all of the files in each run at once
        - run the calibrations

        Note that for the first file in each run, we do a "pass 0" over a small sample of events
        for calibration tasks that don't require a full run's worth of data

        [Multiple passes coming in the future]
        """
        if self.workflow is None:
            raise RuntimeError("Tried to create unnamed workflow!")


        # Generate jobs for the first pass over the data
        for run in sorted(runfile_mapping.keys()):
            if self.VERBOSE>0:
                inputfiles="/%s/rawdata/volatile/%s/rawdata/Run%06d/hd_rawdata_*.evio"%(HDRunFileRAIDList.GetRAIDDirFromRun(run,server_run_map),HDJobUtils.GetRunPeriodFromRun(run),run)

                # PASS 0
                print "processing run %d, phase 0 ..."%(int(run))

                # set up command to execute
                if self.nthreads:
                    cmd += " %s/scripts/%s %s %s %06d %03d %d"%(self.basedir,"job_wrapper_local.csh","local_calib_pass0.csh",self.basedir,run,inputfiles,int(self.nthreads))
                else:
                    cmd += " %s/scripts/%s %s %s %06d %03d"%(self.basedir,"job_wrapper_local.csh","local_calib_pass0.csh",self.basedir,run,inputfiles)

                # run command
                os.system(cmd)

                # PASS 1
                print "processing run %d, phase 1 ..."%(int(run))

                # set up command to execute
                if self.nthreads:
                    cmd += " %s/scripts/%s %s %s %06d %03d %d"%(self.basedir,"job_wrapper_local.csh","local_calib_pass1.csh",self.basedir,run,inputfiles,int(self.nthreads))
                else:
                    cmd += " %s/scripts/%s %s %s %06d %03d"%(self.basedir,"job_wrapper_local.csh","local_calib_pass1.csh",self.basedir,run,inputfiles)

                # run command
                os.system(cmd)