[pull] main from danny-avila:main

.gitignore

@@ -15,3 +15,4 @@ coverage/**/*
 .cursor/
 .claude/
 .codex/
+src/scripts/activity-labels/results/

AGENTS.md

@@ -147,6 +147,7 @@ These originated from direct Langfuse-team feedback (PRs #288, #316) and must su
 - **Redaction is honored everywhere tool output can surface** — tool spans, and any generation input that embeds tool results (e.g. the activity-label prompt).
 - **Identity and metadata always propagate**: `userId`, `sessionId`, tags, environment, and trace metadata (`messageId`, `parentMessageId`, `agentId`, `agentName`) — including across LangChain callbacks that fire outside the caller's OTEL context.
 - **Trace identity is self-contained.** Root observations never inherit trace ids or parents from foreign ambient OTEL spans (e.g. a host's HTTP auto-instrumentation): the callback handler detaches them so roots stay true roots, deterministic ids apply, and concurrent runs inside one request context (an agent run plus a title run) cannot merge into one trace. Spans created through the Langfuse tracer provider are honored as parents, so hosts can still group runs under their own Langfuse observations deliberately.
+- **Runtime scopes are run-stamped.** LangChain executes non-awaited callbacks on a process-wide background queue, so a callback can run inside a DIFFERENT concurrent run's async context — the ambient scope at callback time is untrustworthy for tenant identity. Scopes and handlers carry a `runId`; a handler only adopts an ambient scope stamped with its own run (same-run agent overlays still win), otherwise its own configuration and seed apply. Without this, concurrent runs leak spans into each other's Langfuse projects.
 - **Deterministic trace ids** when a run opts in (`LangfuseConfig.deterministicTraceId`), so host apps can attach scores/feedback by regenerating the id from the run id.
 
 ### Verifying

package.json

@@ -199,7 +199,9 @@
     "re": "bun run clean && bun install && rm -rf ./dist && turbo build",
     "lint": "eslint \"{,!(node_modules|venv)/**/}*.{js,jsx,ts,tsx}\" --fix",
     "format": "prettier --write .",
-    "audit": "npm audit --omit=dev"
+    "audit": "npm audit --omit=dev",
+    "label:eval": "node -r dotenv/config --loader ./tsconfig-paths-bootstrap.mjs --experimental-specifier-resolution=node ./src/scripts/activity-labels/run.ts",
+    "label:rescore": "node ./src/scripts/activity-labels/rescore.cjs"
   },
   "overrides": {
     "@langchain/openai": "1.5.5",
@@ -224,11 +226,12 @@
     "@langchain/google-gauth": "2.2.0",
     "@langchain/google-genai": "2.2.0",
     "@langchain/google-vertexai": "2.2.0",
-    "@langchain/langgraph": "^1.4.6",
+    "@langchain/langgraph": "1.4.8",
     "@langchain/mistralai": "^1.2.0",
     "@langchain/openai": "1.5.5",
     "@langchain/textsplitters": "^1.0.1",
     "@langchain/xai": "^1.4.3",
+    "@langfuse/core": "^5.4.1",
     "@langfuse/langchain": "^5.4.1",
     "@langfuse/otel": "^5.4.1",
     "@langfuse/tracing": "^5.4.1",

src/graphs/Graph.ts

@@ -1,4 +1,5 @@
 /* eslint-disable no-console */
+import { v4 } from 'uuid';
 import { nanoid } from 'nanoid';
 import { tool } from '@langchain/core/tools';
 import { ToolNode } from '@langchain/langgraph/prebuilt';
@@ -53,6 +54,14 @@
   appendPredecessorHandoffCue,
   removePredecessorHandoffCue,
 } from '@/messages';
+import {
+  attemptInvoke,
+  tryFallbackProviders,
+  getFallbackErrorContext,
+  getFallbackOverflowCandidates,
+  projectMessagesForProvider,
+  resolveServingModelId,
+} from '@/llm/invoke';
 import {
   resetIfNotEmpty,
   isAnthropicLike,
@@ -64,14 +73,14 @@
   sleep,
 } from '@/utils';
 import {
-  attemptInvoke,
-  tryFallbackProviders,
-  getFallbackErrorContext,
-  getFallbackOverflowCandidates,
-  projectMessagesForProvider,
-  resolveServingModelId,
-} from '@/llm/invoke';
-import { v4 } from 'uuid';
+  Constants,
+  GraphNodeKeys,
+  ContentTypes,
+  GraphEvents,
+  Providers,
+  StepTypes,
+  PREEMPT_BOUNDARY_HOOK_TIMEOUT_MS,
+} from '@/common';
 import {
   createLangfuseHandler,
   createLangfuseTraceMetadata,
@@ -83,21 +92,16 @@
   planContextOverflowRecovery,
   translateRecoveryBudget,
 } from '@/llm/contextOverflowRecovery';
+import {
+  hasToolOutputTracingConfig,
+  resolveLangfuseConfig,
+  resolveToolOutputTracingConfig,
+} from '@/langfuseConfig';
 import {
   compactToolContent,
   getToolContentCharLength,
   serializeToolContentBounded,
 } from '@/utils/toolContent';
-import { resolveMaxSeals } from '@/llm/preempt';
-import {
-  Constants,
-  GraphNodeKeys,
-  ContentTypes,
-  GraphEvents,
-  Providers,
-  StepTypes,
-  PREEMPT_BOUNDARY_HOOK_TIMEOUT_MS,
-} from '@/common';
 import {
   annotateMessagesForLLM,
   ToolOutputReferenceRegistry,
@@ -125,12 +129,12 @@
 import { resolveLocalToolsForBinding } from '@/tools/local';
 import { createSummarizeNode } from '@/summarization/node';
 import { messagesStateReducer } from '@/messages/reducer';
-import { resolveLangfuseConfig } from '@/langfuseConfig';
 import { createSchemaOnlyTools } from '@/tools/schema';
 import { AgentContext } from '@/agents/AgentContext';
 import { createFakeStreamingLLM } from '@/llm/fake';
 import { handleToolCalls } from '@/tools/handlers';
 import { isThinkingEnabled } from '@/llm/request';
+import { resolveMaxSeals } from '@/llm/preempt';
 import { initializeModel } from '@/llm/init';
 import { HandlerRegistry } from '@/events';
 import { ChatOpenAI } from '@/llm/openai';
@@ -997,6 +1001,16 @@
   /** Checkpoint scope whose messages match index-keyed tool snapshots. */
   private originalToolContentCheckpointScope?: string;
   runId: string | undefined;
+  /**
+   * Identity used to stamp Langfuse runtime scopes and handlers (see
+   * `LangfuseRuntimeContext.runId`). Carries an opaque per-instance
+   * component: public run ids are unrestricted and may repeat across
+   * concurrently executing runs (retries, duplicate submissions,
+   * tenant-local message ids), and equal stamps would let those runs adopt
+   * each other's scopes. One graph instance = one execution's stamp, shared
+   * by the stream handler and every graph-level scope of that execution.
+   */
+  readonly langfuseScopeRunId: string;
   /**
    * Boundary between historical messages (loaded from conversation state)
    * and messages produced during the current run.  Set once in the state
@@ -1081,6 +1095,7 @@
   }: t.StandardGraphInput) {
     super();
     this.runId = runId;
+    this.langfuseScopeRunId = `${runId ?? 'graph'}:${nanoid()}`;
     this.signal = signal;
     this.langfuse = langfuse;
     this.subagentUsageSink = subagentUsageSink;
@@ -1462,7 +1477,7 @@
   /* Misc.*/
 
   getRunMessages(): BaseMessage[] | undefined {
     if (this.messages == null) {
       return this.cachedRunMessages;
     }
     if (this.messages.length === 0 && this.cachedRunMessages != null) {
@@ -1494,7 +1509,7 @@
     // `messages` can be null/undefined on a graph that has been disposed
     // (clearHeavyState) but is still reachable via a cache (e.g. RedisJobStore's
     // WeakRef) during a HITL resume/reconnect. Guard instead of dereferencing null.
     if (this.messages == null) {
       return undefined;
     }
     return convertMessagesToContent(this.messages.slice(this.startIndex));
@@ -1532,7 +1547,7 @@
     // `contentData` can be null/undefined on a disposed-but-cached graph during a
     // HITL resume/reconnect; without this guard `[...this.contentData]` throws
     // "this.contentData is not iterable".
     if (this.contentData == null) {
       return [];
     }
     if (agentId == null || agentId === '') {
@@ -3057,6 +3072,11 @@
         metadata: {
           ...(config.metadata ?? {}),
           ...traceMetadata,
+          /** Canonical agent identity, stamped OUTSIDE trace-metadata
+           *  filtering: `createLangfuseTraceMetadata` drops values over its
+           *  length cap, but scope trust (`isForeignScope`) needs the id
+           *  verbatim regardless of length. */
+          agentId,
         },
       };
       initializeLangfuseTracing(langfuse);
@@ -3069,6 +3089,16 @@
           tags: ['librechat', 'agent'],
           traceIdSeed:
             langfuse?.deterministicTraceId === true ? this.runId : undefined,
+          runId: this.langfuseScopeRunId,
+          toolOutputTracing: hasToolOutputTracingConfig(
+            this.langfuse,
+            agentContext.langfuse
+          )
+            ? resolveToolOutputTracingConfig(
+              this.langfuse,
+              agentContext.langfuse
+            )
+            : undefined,
         });
         if (langfuseHandler != null) {
           invokeConfig = {
@@ -3089,6 +3119,8 @@
           resolveLangfuseRuntimeScope({
             runLangfuse: this.langfuse,
             langfuseOverlay: agentContext.langfuse,
+            runId: this.langfuseScopeRunId,
+            agentId,
           }),
           () =>
             attemptInvoke(
@@ -3251,6 +3283,8 @@
             resolveLangfuseRuntimeScope({
               runLangfuse: this.langfuse,
               langfuseOverlay: agentContext.langfuse,
+              runId: this.langfuseScopeRunId,
+              agentId,
             }),
             () =>
               tryFallbackProviders({
@@ -3673,7 +3707,10 @@
      * the same run — `haltRun` is first-write-wins — is left alone.
      */
     const halt = this.hookRegistry.getHaltSignal(runId);
-    if (result.preventContinuation === true && halt?.source === 'PreemptBoundary') {
+    if (
+      result.preventContinuation === true &&
+      halt?.source === 'PreemptBoundary'
+    ) {
       this.preemptHaltReason = halt.reason;
       this.hookRegistry.clearHaltSignal(runId);
     }