feat(profiling): add @pyroscope/nodejs for continuous CPU/wall profiling (#131)

Initializes Pyroscope SDK before server start for full V8 call-stack
flamegraphs. Gracefully skips when PYROSCOPE_SERVER_ADDRESS is unset.
~2-3% CPU overhead via V8's built-in sampling profiler at 100Hz.

Author: DivMode

package-lock.json

@@ -62,6 +62,7 @@
     "@divmode/rrweb-plugin-console-record": "^0.0.40",
     "@hapi/bourne": "^3.0.0",
     "@hapi/hoek": "^11.0.4",
+    "@pyroscope/nodejs": "^0.4.10",
     "better-sqlite3": "^11.10.0",
     "debug": "^4.4.1",
     "del": "^8.0.1",

package.json

@@ -1,6 +1,25 @@
+import Pyroscope from '@pyroscope/nodejs';
 import { Browserless } from '@browserless.io/browserless';
 import { Effect } from 'effect';
 
+// ── Continuous profiling (Pyroscope) ─────────────────────────────────
+// Must init before any other code to capture full startup profile.
+// Gracefully skips when PYROSCOPE_SERVER_ADDRESS is unset (local dev).
+if (process.env.PYROSCOPE_SERVER_ADDRESS) {
+  Pyroscope.init({
+    serverAddress: process.env.PYROSCOPE_SERVER_ADDRESS,
+    appName: process.env.OTEL_SERVICE_NAME ?? 'browserless',
+    basicAuthUser: process.env.PYROSCOPE_BASIC_AUTH_USER ?? '',
+    basicAuthPassword: process.env.PYROSCOPE_BASIC_AUTH_PASSWORD ?? '',
+    wall: { collectCpuTime: true },
+    tags: {
+      env: process.env.OTEL_DEPLOYMENT_ENVIRONMENT ?? 'production',
+      server: 'flatcar',
+    },
+  });
+  Pyroscope.start();
+}
+
 // ── Fail-fast env validation ─────────────────────────────────────────
 // REQUIRED env vars must be present BEFORE the server accepts connections.
 // Without this, a stale `node --watch` process (started without proper env)

src/index.ts

@@ -1,4 +1,4 @@
-import { Cause, Effect, Exit, FiberMap, Layer, ManagedRuntime, Scope, type Tracer } from 'effect';
+import { Cause, Effect, Exit, FiberMap, Layer, ManagedRuntime, Scope, Semaphore, type Tracer } from 'effect';
 import { CdpSessionId } from '../shared/cloudflare-detection.js';
 import type { TargetId, CloudflareConfig } from '../shared/cloudflare-detection.js';
 import { CloudflareDetector } from './cf/cloudflare-detector.js';
@@ -150,13 +150,20 @@ export class CloudflareSolver {
 
     const proxyOrDirect: SendCommand = (...args) => (self.sendViaProxy || sendCommand)(...args);
 
-    const cdpSenderLayer = Layer.succeed(CdpSender, CdpSender.of({
-      send: (method, params, sessionId, timeoutMs) =>
-        liftSend(sendCommand, method, params, sessionId, timeoutMs),
-      sendViaProxy: (method, params, sessionId, timeoutMs) =>
-        liftSend(proxyOrDirect, method, params, sessionId, timeoutMs),
-      sendViaBrowser: (method, params, sessionId, timeoutMs) =>
-        liftSend(proxyOrDirect, method, params, sessionId, timeoutMs),
+    // Semaphore limits concurrent CDP commands to Chrome — prevents backpressure
+    // when multiple tabs have active detection/solve loops firing simultaneously.
+    const CDP_CONCURRENCY = 3;
+    const cdpSenderLayer = Layer.effect(CdpSender, Effect.gen(function*() {
+      const sem = yield* Semaphore.make(CDP_CONCURRENCY);
+      const throttle = <A, E>(effect: Effect.Effect<A, E>) => sem.withPermits(1)(effect);
+      return CdpSender.of({
+        send: (method, params, sessionId, timeoutMs) =>
+          throttle(liftSend(sendCommand, method, params, sessionId, timeoutMs)),
+        sendViaProxy: (method, params, sessionId, timeoutMs) =>
+          throttle(liftSend(proxyOrDirect, method, params, sessionId, timeoutMs)),
+        sendViaBrowser: (method, params, sessionId, timeoutMs) =>
+          throttle(liftSend(proxyOrDirect, method, params, sessionId, timeoutMs)),
+      });
     }));
 
     const solverEventsLayer = Layer.succeed(SolverEvents, SolverEvents.of({
@@ -196,8 +203,8 @@ export class CloudflareSolver {
     }));
 
     // SolveDispatcher — routes solve attempts through the Effect solver.
-    // Per-solve isolated WS: each solve gets its own WebSocket to Chrome,
-    // Each solve gets its own isolated WS connection, so no concurrency limit needed.
+    // Per-solve isolated WS: each solve gets its own WebSocket to Chrome.
+    // Browser-level sends (originalSender) inherit the Semaphore from cdpSenderLayer.
     const solveDispatcherLayer = Layer.effect(SolveDispatcher, Effect.gen(function*() {
       const solverEvents = yield* SolverEvents;
       const solveDeps = yield* SolveDeps;
@@ -411,7 +418,7 @@ export class CloudflareSolver {
     }
 
     // FiberMap.run auto-interrupts existing fiber for same key.
-    // The detection effect is wrapped in catchAllCause to prevent silent fiber
+    // The detection effect is wrapped in catchCause to prevent silent fiber
     // death — without this, defects (NPE in emitClientEvent, etc.) kill the fiber
     // and pydoll never receives cf.solved/cf.failed (the "events=1" failure mode).
     const guarded = this.detector.detectTurnstileWidgetEffect(targetId, cdpSessionId).pipe(