Add verbosity to smart-dispatch

scripts/sd-launch-pbs

@@ -7,15 +7,16 @@ import logging
 from smartdispatch import launch_jobs
 from smartdispatch import utils
 
+
+logger = logging.getLogger()
+
+
 LOGS_FOLDERNAME = "SMART_DISPATCH_LOGS"
 CLUSTER_NAME = utils.detect_cluster()
 LAUNCHER = utils.get_launcher(CLUSTER_NAME)
 
 
 def main():
-    # Necessary if we want 'logging.info' to appear in stderr.
-    logging.root.setLevel(logging.INFO)
-
     args = parse_arguments()
 
     launch_jobs(LAUNCHER if args.launcher is None else args.launcher, [args.pbs], CLUSTER_NAME, args.path_job)
@@ -27,8 +28,21 @@ def parse_arguments():
     parser.add_argument('pbs', type=str, help='PBS filename to launch.')
     parser.add_argument('path_job', type=str, help='Path to the job folder.')
 
+    parser.add_argument(
+        '-v', '--verbose', action='count', default=0,
+        help="Print informations about the process.\n"
+             "     -v: INFO\n"
+             "     -vv: DEBUG")
+
     args = parser.parse_args()
 
+    if args.verbose == 0:
+        logging.basicConfig(level=logging.WARNING)
+    elif args.verbose == 1:
+        logging.basicConfig(level=logging.INFO)
+    elif args.verbose >= 2:
+        logging.basicConfig(level=logging.DEBUG)
+
     return args
 
 

scripts/smart-dispatch

@@ -1,9 +1,10 @@
 #!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 
+import argparse
+import logging
 import os
 import sys
-import argparse
 import time as t
 from os.path import join as pjoin
 from textwrap import dedent
@@ -16,9 +17,12 @@ from smartdispatch import get_available_queues
 from smartdispatch import launch_jobs
 from smartdispatch import utils
 
-import logging
 import smartdispatch
 
+
+logger = logging.getLogger()
+
+
 LOGS_FOLDERNAME = "SMART_DISPATCH_LOGS"
 CLUSTER_NAME = utils.detect_cluster()
 AVAILABLE_QUEUES = get_available_queues(CLUSTER_NAME)
@@ -29,27 +33,50 @@ TIMEOUT_EXIT_CODE = 124
 AUTORESUME_TRIGGER_AFTER = '$(($PBS_WALLTIME - 60))'  # By default, 60s before the maximum walltime.
 AUTORESUME_WORKER_CALL_PREFIX = 'timeout -s TERM {trigger_after} '.format(trigger_after=AUTORESUME_TRIGGER_AFTER)
 AUTORESUME_WORKER_CALL_SUFFIX = ' WORKER_PIDS+=" $!"'
-AUTORESUME_PROLOG = 'WORKER_PIDS=""' 
+AUTORESUME_PROLOG = """
+WORKER_PIDS=""
+VERBOSE={verbose}
+"""
 AUTORESUME_EPILOG = """\
 NEED_TO_RESUME=false
+if [ $VERBOSE = true ]; then
+    echo NEED_TO_RESUME=$NEED_TO_RESUME
+    echo WORKER_PIDS=$WORKER_PIDS
+fi
 for WORKER_PID in $WORKER_PIDS; do
+    if [ $VERBOSE = true ]; then
+        echo WORKER_PID=$WORKER_PID
+    fi
     wait "$WORKER_PID"
     RETURN_CODE=$?
+    if [ $VERBOSE = true ]; then
+        echo "RETURN_CODE is $RETURN_CODE while " \
+             "timeout_exit_code is {timeout_exit_code}"
+    fi
     if [ $RETURN_CODE -eq {timeout_exit_code} ]; then
         NEED_TO_RESUME=true
     fi
+    if [ $VERBOSE = true ]; then
+        echo NEED_TO_RESUME=$NEED_TO_RESUME
+    fi
 done
+if [ $VERBOSE = true ]; then
+    echo NEED_TO_RESUME=$NEED_TO_RESUME
+fi
 if [ "$NEED_TO_RESUME" = true ]; then
     echo "Autoresuming using: {{launcher}} $PBS_FILENAME"
-    sd-launch-pbs --launcher {{launcher}} $PBS_FILENAME {{path_job}}
+    if [ $VERBOSE = true]; then
+        VERBOSE_OPTION="-vv"
+    else
+        VERBOSE_OPTION=""
+    fi
+
+    sd-launch-pbs $VERBOSE_OPTION --launcher {{launcher}} $PBS_FILENAME {{path_job}}
 fi
 """.format(timeout_exit_code=TIMEOUT_EXIT_CODE)
 
 
 def main():
-    # Necessary if we want 'logging.info' to appear in stderr.
-    logging.root.setLevel(logging.INFO)
-
     args = parse_arguments()
     path_smartdispatch_logs = pjoin(os.getcwd(), LOGS_FOLDERNAME)
 
@@ -163,8 +190,12 @@ def main():
     prolog = []
     epilog = ['wait']
     if args.autoresume:
-        prolog = [AUTORESUME_PROLOG]
-        epilog = [AUTORESUME_EPILOG.format(launcher=LAUNCHER if args.launcher is None else args.launcher, path_job=path_job)]
+        prolog = [
+            AUTORESUME_PROLOG.format(verbose=str(args.verbose >= 2).lower())]
+        epilog = [
+            AUTORESUME_EPILOG.format(
+                launcher=LAUNCHER if args.launcher is None else args.launcher,
+                path_job=path_job)]
 
     job_generator = job_generator_factory(queue, commands, prolog, epilog, command_params, CLUSTER_NAME, path_job)
 
@@ -187,6 +218,13 @@ def main():
 
 def parse_arguments():
     parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        '-v', '--verbose', action='count', default=0,
+        help="Print informations about the process.\n"
+             "     -v: INFO\n"
+             "     -vv: DEBUG")
+
     parser.add_argument('-q', '--queueName', required=True, help='Queue used (ex: qwork@mp2, qfat256@mp2, gpu_1)')
     parser.add_argument('-n', '--batchName', required=False, help='The name of the batch. Default: The commands launched.')
     parser.add_argument('-t', '--walltime', required=False, help='Set the estimated running time of your jobs using the DD:HH:MM:SS format. Note that they will be killed when this time limit is reached.')
@@ -226,6 +264,13 @@ def parse_arguments():
         if args.coresPerCommand < 1:
             parser.error("coresPerNode must be at least 1")
 
+    if args.verbose == 0:
+        logging.basicConfig(level=logging.WARNING)
+    elif args.verbose == 1:
+        logging.basicConfig(level=logging.INFO)
+    elif args.verbose >= 2:
+        logging.basicConfig(level=logging.DEBUG)
+
     return args