Add stagger delay to fix frontdoor login

main_bu.py

@@ -43,6 +43,8 @@
 logger = logging.getLogger(__name__)
 load_dotenv(override=True)
 PAUSE_AFTER_LOGIN = 10
+LOGIN_STAGGER_DELAY = 1
+_frontdoor_login_lock = asyncio.Lock()
 class BrowserUseFormatter(logging.Formatter):
     def format(self, record):
         if type(record.name) == str and record.name.startswith('browser_use.'):
@@ -130,9 +132,11 @@ async def aevaluate_single_task_bu(
         @controller.action('Login to Salesforce website via frontdoor URL', param_model=NoParamsAction)
         async def login_salesforce(param_model: NoParamsAction, browser: BrowserContextBugFix) -> ActionResult:
             page = await browser.get_current_page()
-            oauth = refresh_access_token(args.org_alias)
-            frontdoor_url = get_frontdoor_url(oauth["access_token"], oauth["instance_url"])
-            await page.goto(frontdoor_url, wait_until="domcontentloaded")
+            async with _frontdoor_login_lock:
+                oauth = refresh_access_token(args.org_alias)
+                frontdoor_url = get_frontdoor_url(oauth["access_token"], oauth["instance_url"])
+                await page.goto(frontdoor_url, wait_until="domcontentloaded")
+                await asyncio.sleep(LOGIN_STAGGER_DELAY)
             await asyncio.sleep(PAUSE_AFTER_LOGIN)
 
             url = page.url

main_cua.py

@@ -20,6 +20,8 @@
 from scuba.helpers.salesforce_commands import authorize_using_access_token, install_initial_data, retrieve_initial_state_metadata, create_project_if_not_exists
 from scuba.helpers.sf_oauth import refresh_access_token
 from envs.remote_docker_env import RemoteDesktopEnv, ContainerConfig, ProviderConfig
+
+LOGIN_STAGGER_DELAY = 1
 from utils import run_evaluate, run_reset, LogFormatter, split_task_config_pool_into_batches
 from args import get_args
 from lib_eval_single_task import evaluate_single_task_vllm, evaluate_single_task_api
@@ -224,7 +226,7 @@ def test(
             with ProcessPool(max_workers=num_envs) as pool:
                 task_idx = 0
                 completed_tasks = 0
-                # Submit initial batch (up to num_envs)
+                # Submit initial batch (up to num_envs), staggered to avoid frontdoor login collisions
                 while task_idx < num_tasks and not env_queue.empty():
                     env_idx = env_queue.get()
                     vllm_idx = vllm_client_idx % vllm_client_count
@@ -236,6 +238,8 @@ def test(
                     running_futures.append((future, env_idx, time.time()))
                     vllm_client_idx += 1
                     task_idx += 1
+                    if task_idx < num_tasks and not env_queue.empty():
+                        time.sleep(LOGIN_STAGGER_DELAY)
 
                 # As tasks finish, submit new ones
                 while completed_tasks < num_tasks:

scripts/test_frontdoor_login.py

@@ -30,17 +30,16 @@
 from scuba.helpers.sf_oauth import refresh_access_token, get_frontdoor_url
 
 PAUSE_AFTER_LOGIN = 12
+LOGIN_STAGGER_DELAY = 1
 
 
-async def launch_and_login(
+async def prepare_browser(
     instance_id: int,
-    oauth: dict,
     headless: bool,
     viewport_width: int,
     viewport_height: int,
 ):
-    instance_url = oauth["instance_url"].rstrip("/")
-
+    """Phase 1: launch browser and get a ready page. Safe to run in parallel."""
     print(f"[Instance {instance_id}] Launching Chromium (headless={headless})...")
     browser_config = BrowserConfig(headless=headless)
     browser = BrowserBugFix(browser_config)
@@ -51,11 +50,24 @@ async def launch_and_login(
     )
     context = BrowserContextBugFix(browser=browser, config=context_config)
 
+    session = await context.get_session()
+    page = session.current_page
+    print(f"[Instance {instance_id}] Browser ready")
+    return browser, context, page
+
+
+async def login_and_navigate(
+    instance_id: int,
+    oauth: dict,
+    page,
+    browser,
+    context,
+    headless: bool,
+):
+    """Phase 2: frontdoor login + post-login navigation. Run with stagger."""
+    instance_url = oauth["instance_url"].rstrip("/")
     success = False
     try:
-        session = await context.get_session()
-        page = session.current_page
-
         frontdoor_url = get_frontdoor_url(oauth["access_token"], oauth["instance_url"])
         print(f"[Instance {instance_id}] Got frontdoor URL ({len(frontdoor_url)} chars)")
 
@@ -140,17 +152,41 @@ async def main():
     test_url = get_frontdoor_url(oauth["access_token"], oauth["instance_url"])
     print(f"  singleaccess: OK ({len(test_url)} chars)")
 
-    print(f"\n--- Launching {args.num_instances} browser(s) ---\n")
-
-    tasks = [
-        launch_and_login(i, oauth, args.headless, args.viewport_width, args.viewport_height)
+    # Phase 1: launch all browsers in parallel
+    print(f"\n--- Phase 1: Launching {args.num_instances} browser(s) in parallel ---\n")
+    prepare_tasks = [
+        prepare_browser(i, args.headless, args.viewport_width, args.viewport_height)
         for i in range(args.num_instances)
     ]
-    raw_results = await asyncio.gather(*tasks, return_exceptions=True)
+    prepared = await asyncio.gather(*prepare_tasks, return_exceptions=True)
+
+    ready = {}
+    for i, result in enumerate(prepared):
+        if isinstance(result, BaseException):
+            print(f"[Instance {i}] FAILED to launch browser: {result}")
+        else:
+            ready[i] = result
+
+    # Phase 2: staggered frontdoor login (~1s apart)
+    print(f"\n--- Phase 2: Logging in {len(ready)} browser(s) (stagger={LOGIN_STAGGER_DELAY}s) ---\n")
+    raw_results: dict[int, tuple | BaseException] = {}
+    for idx, i in enumerate(sorted(ready)):
+        if idx > 0:
+            await asyncio.sleep(LOGIN_STAGGER_DELAY)
+        browser, context, page = ready[i]
+        try:
+            raw_results[i] = await login_and_navigate(
+                i, oauth, page, browser, context, args.headless
+            )
+        except BaseException as e:
+            raw_results[i] = e
 
     successes = 0
     open_browsers: list[tuple[int, object, object]] = []
-    for i, r in enumerate(raw_results):
+    for i in range(args.num_instances):
+        if i not in raw_results:
+            continue
+        r = raw_results[i]
         if isinstance(r, BaseException):
             print(f"[Instance {i}] FAILED with exception: {r}")
             continue