fix(tools): add ignore_errors param to execute_command_parallel (#454)

Author: milindsrivastava1997

asap-tools/experiments/experiment_utils/providers/base.py

@@ -56,6 +56,7 @@ def execute_command_parallel(
         popen: bool = True,
         redirect: bool = False,
         wait: bool = True,
+        ignore_errors: bool = False,
     ) -> List[subprocess.Popen]:
         """
         Execute a command on multiple nodes in parallel.
@@ -68,6 +69,7 @@ def execute_command_parallel(
             popen: Must be True for parallel execution
             redirect: Whether to redirect output to /dev/null
             wait: Whether to wait for all commands to complete
+            ignore_errors: Whether to ignore command execution errors
 
         Returns:
             List of Popen objects for each node

asap-tools/experiments/experiment_utils/providers/cloudlab.py

@@ -78,6 +78,7 @@ def execute_command_parallel(
         popen: bool = True,
         redirect: bool = False,
         wait: bool = True,
+        ignore_errors: bool = False,
     ) -> List[subprocess.Popen]:
         """
         Execute a command on multiple CloudLab nodes in parallel via SSH.

asap-tools/experiments/experiment_utils/providers/cloudlab_local.py

@@ -81,14 +81,20 @@ def execute_command(
 
         # Execute locally
         if popen:
-            return subprocess.Popen(
-                cmd,
-                shell=True,
-                cwd=cmd_dir,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                text=True,
-            )
+            try:
+                return subprocess.Popen(
+                    cmd,
+                    shell=True,
+                    cwd=cmd_dir,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    text=True,
+                )
+            except OSError as e:
+                if ignore_errors:
+                    print(f"Warning: failed to launch '{cmd}' in {cmd_dir!r}: {e}")
+                    return None
+                raise
         else:
             try:
                 result = subprocess.run(
@@ -100,7 +106,7 @@ def execute_command(
                     check=not ignore_errors,
                 )
                 return result
-            except subprocess.CalledProcessError as e:
+            except (subprocess.CalledProcessError, OSError) as e:
                 if ignore_errors:
                     return e
                 raise
@@ -114,6 +120,7 @@ def execute_command_parallel(
         popen: bool = True,
         redirect: bool = False,
         wait: bool = True,
+        ignore_errors: bool = False,
     ) -> List[subprocess.Popen]:
         """
         Execute a command in parallel locally.
@@ -129,6 +136,7 @@ def execute_command_parallel(
             popen: Must be True for parallel execution
             redirect: Whether to redirect output to /dev/null
             wait: Whether to wait for all commands to complete
+            ignore_errors: Whether to ignore command execution errors
 
         Returns:
             List containing single Popen object
@@ -144,6 +152,7 @@ def execute_command_parallel(
             cmd_dir=cmd_dir,
             nohup=nohup,
             popen=True,  # Always return Popen for parallel
+            ignore_errors=ignore_errors,
         )
 
         processes = [process]

asap-tools/experiments/experiment_utils/providers/local.py

@@ -64,13 +64,10 @@ def execute_command(
                     text=True,
                 )
             except OSError as e:
-                # execute_command_parallel has no ignore_errors knob (see
-                # follow-up issue), so this can't be conditional yet. Mirrors
-                # CloudLab's SSH transport: launching `ssh` locally never
-                # fails even if the remote cmd_dir is missing — the failure
-                # is invisible unless something later checks output/health.
-                print(f"Warning: failed to launch '{cmd}' in {cmd_dir!r}: {e}")
-                return None
+                if ignore_errors:
+                    print(f"Warning: failed to launch '{cmd}' in {cmd_dir!r}: {e}")
+                    return None
+                raise
 
         try:
             return subprocess.run(
@@ -101,6 +98,7 @@ def execute_command_parallel(
         popen: bool = True,
         redirect: bool = False,
         wait: bool = True,
+        ignore_errors: bool = False,
     ) -> List[subprocess.Popen]:
         """Execute a command once locally (there is only one local node)."""
         if redirect:
@@ -112,6 +110,7 @@ def execute_command_parallel(
             cmd_dir=cmd_dir,
             nohup=nohup,
             popen=True,
+            ignore_errors=ignore_errors,
         )
         processes = [process]
         if wait:

asap-tools/experiments/experiment_utils/services/misc.py

@@ -79,6 +79,7 @@ def stop(self, **kwargs) -> None:
             nohup=False,
             popen=True,
             wait=True,
+            ignore_errors=True,
         )
 
     def run_workload(