diff --git a/.scaffold/memory/log/2026-08-24-issue-35-token-stats.md b/.scaffold/memory/log/2026-08-24-issue-35-token-stats.md new file mode 100644 index 0000000..aba8191 --- /dev/null +++ b/.scaffold/memory/log/2026-08-24-issue-35-token-stats.md @@ -0,0 +1,31 @@ +# Closeout — issue #35 token stats (2026-08-24) + +- **Origin:** GitHub Issue #35 — token stats display (tok/s, TTFT), feature, frontend-leaning. +- **Delivery:** branch `feat/token-stats`, PR opened against #35. Key files: + `server/src/routes/chats.js` (messageMetadata on finish), `frontend/src/hooks/useChatStream.js` + (timing + stats into message metadata), `frontend/src/lib/tokenStats.js`, `MessageItem.jsx` + (footer stat line), `useThemeStore.js` + `Settings.jsx` (toggle), `lib/messageShape.js` + (carry `metadata` through `toCanonicalMessage` — without this, stats would not survive reload). +- **Intent changes:** none (no review decisions at closeout time). +- **Requirements satisfied:** + - Usage captured from stream result: server attaches `{usage}` as message metadata on the + `finish` part via `toUIMessageStreamResponse({ messageMetadata })`. + - TTFT measured client-side (send → first assistant text delta) in `useChatStream`. + - Stats persisted in message `metadata.stats` (existing metadata persistence path reused). + - Footer stat line gated on `showTokenStats` setting, default off; disabled = nothing renders, + so zero layout shift. + - Stats survive reload (metadata carried through canonical message shape + DB roundtrip). +- **Remaining scope:** none. Non-goals (cost tracking, historical charts) untouched. +- **Verification ledger:** `bun run test` frontend (52 pass, incl. 7 new tokenStats tests); + `bun test` server (420 pass); `bun run format`; `bun run build`. No e2e run — UI is a + conditional text line, unit tests cover the risky logic (stats math/formatting). +- **Proof:** `frontend/src/lib/tokenStats.test.js` covers buildTokenStats (null guards, + tok/s excluding TTFT, zero-generation guard) and formatTokenStats. +- **Open / next:** PR review. Verify against a live Ollama vs cloud provider for the issue's + acceptance check. +- **Derived decisions (ratify in PR?):** + - Toggle lives in `useThemeStore` (localStorage, per-browser) matching the + `showCodeLineNumbers` pattern, not server-side app settings (which are appName/logoIcon only). + - tok/s = outputTokens / (duration − TTFT) — generation rate, excluding TTFT. + - Server sends raw usage; client composes `stats` (usage + timing) so timing stays honest + (measured where it happens). diff --git a/frontend/src/components/chat/MessageItem.jsx b/frontend/src/components/chat/MessageItem.jsx index 57cc6f5..abc3e22 100644 --- a/frontend/src/components/chat/MessageItem.jsx +++ b/frontend/src/components/chat/MessageItem.jsx @@ -1,5 +1,7 @@ import { MarkdownContent } from "@/components/markdown/MarkdownRenderer"; import { extractTextContent } from "@/lib/messageUtils"; +import { formatTokenStats } from "@/lib/tokenStats"; +import { useThemeStore } from "@/state/useThemeStore"; import { memo } from "@preact/compat"; import { AlertTriangle, Brain, ChevronDown, Sparkles } from "lucide-preact"; import MessageAttachment from "./MessageAttachment"; @@ -41,6 +43,7 @@ const parseThinkingBlocks = (text) => { }; const MessageItem = memo(({ message, onStop, onRegenerate }) => { + const showTokenStats = useThemeStore((state) => state.showTokenStats); const isUser = message.role === "user"; const rawContent = extractTextContent(message); const { thinking, content } = isUser @@ -55,6 +58,7 @@ const MessageItem = memo(({ message, onStop, onRegenerate }) => { (p) => p.type === "tool-invocation" && p.state === "call" ); const sources = isUser ? [] : extractSources(message.parts); + const statsLine = !isUser && showTokenStats ? formatTokenStats(message.metadata?.stats) : ""; const toolErrors = isUser ? [] : message.parts?.filter( @@ -127,6 +131,10 @@ const MessageItem = memo(({ message, onStop, onRegenerate }) => { {sources.length > 0 && } + {statsLine && ( +

{statsLine}

+ )} + {showActions && (
{onStop && ( diff --git a/frontend/src/hooks/useChatStream.js b/frontend/src/hooks/useChatStream.js index 11e29eb..81d86a6 100644 --- a/frontend/src/hooks/useChatStream.js +++ b/frontend/src/hooks/useChatStream.js @@ -1,7 +1,13 @@ import { DefaultChatTransport } from "ai"; import { useChat as useAIChat } from "@ai-sdk/react"; -import { useMemo, useRef } from "preact/hooks"; -import { deduplicateMessages, ensureTimestamp, getMessageTimestamp } from "@/lib/messageUtils"; +import { useEffect, useMemo, useRef } from "preact/hooks"; +import { + deduplicateMessages, + ensureTimestamp, + getMessageTimestamp, + hasTextContent, +} from "@/lib/messageUtils"; +import { buildTokenStats } from "@/lib/tokenStats"; import { MESSAGE_CONSTANTS } from "@faster-chat/shared"; function trimMessageHistory(messages) { @@ -44,6 +50,11 @@ export function useChatStream({ const messageTimestampsRef = useRef(new Map()); + const timingRef = useRef({ sendAt: null, ttftMs: null }); + const resetTiming = () => { + timingRef.current = { sendAt: performance.now(), ttftMs: null }; + }; + const formattedMessages = (persistedMessages ?? []).map((msg) => ensureTimestamp(msg, messageTimestampsRef) ); @@ -98,19 +109,39 @@ export function useChatStream({ const toolParts = message.parts?.filter((p) => p.type === "tool-invocation" && p.state === "result") || []; - const metadata = toolParts.length > 0 ? { toolParts } : null; + + const { sendAt, ttftMs } = timingRef.current; + const stats = buildTokenStats({ + usage: message.metadata?.usage, + ttftMs, + durationMs: sendAt != null ? Math.round(performance.now() - sendAt) : null, + }); + + const metadata = { + ...(toolParts.length > 0 ? { toolParts } : {}), + ...(stats ? { stats } : {}), + }; if (onMessageComplete && content.trim()) { await onMessageComplete({ id: message.id, content, - metadata, + metadata: Object.keys(metadata).length > 0 ? metadata : null, createdAt: getMessageTimestamp(message), }); } }, }); + useEffect(() => { + const { sendAt, ttftMs } = timingRef.current; + if (sendAt == null || ttftMs != null) return; + const last = streamingMessages[streamingMessages.length - 1]; + if (last?.role === "assistant" && hasTextContent(last)) { + timingRef.current.ttftMs = Math.round(performance.now() - sendAt); + } + }, [streamingMessages]); + const isStreaming = status === "streaming" || status === "submitted"; const streamingMessagesWithModel = streamingMessages.map((msg) => ({ @@ -131,13 +162,19 @@ export function useChatStream({ }; if (fileIds.length > 0) message.fileIds = fileIds; messageTimestampsRef.current.set(message.id, message.createdAt); + resetTiming(); await sendMessage(message); } + async function regenerateWithTiming() { + resetTiming(); + await regenerate(); + } + return { messages, send, - regenerate, + regenerate: regenerateWithTiming, stop, status, error, diff --git a/frontend/src/lib/messageShape.js b/frontend/src/lib/messageShape.js index f09ae58..53b89d6 100644 --- a/frontend/src/lib/messageShape.js +++ b/frontend/src/lib/messageShape.js @@ -12,6 +12,7 @@ export function toCanonicalMessage(msg) { parts, fileIds: msg.fileIds || [], model: msg.model || null, + metadata: msg.metadata || null, createdAt: getMessageTimestamp(msg), }; } diff --git a/frontend/src/lib/tokenStats.js b/frontend/src/lib/tokenStats.js new file mode 100644 index 0000000..11b711f --- /dev/null +++ b/frontend/src/lib/tokenStats.js @@ -0,0 +1,39 @@ +/** + * Token performance stats for assistant messages. + * Usage comes from the provider via stream metadata; timing is measured client-side. + */ + +export function buildTokenStats({ usage, ttftMs, durationMs }) { + const outputTokens = usage?.outputTokens; + if (!outputTokens || !durationMs) return null; + + const generationMs = ttftMs != null ? Math.max(durationMs - ttftMs, 1) : durationMs; + + return { + inputTokens: usage.inputTokens ?? null, + outputTokens, + ttftMs: ttftMs ?? null, + durationMs, + tokensPerSecond: Math.round((outputTokens / (generationMs / 1000)) * 10) / 10, + }; +} + +function formatDuration(ms) { + return ms < 1000 ? `${Math.round(ms)}ms` : `${(ms / 1000).toFixed(1)}s`; +} + +export function formatTokenStats(stats) { + if (!stats) return ""; + + const parts = []; + if (stats.ttftMs != null) parts.push(`TTFT ${formatDuration(stats.ttftMs)}`); + if (stats.tokensPerSecond != null) parts.push(`${stats.tokensPerSecond} tok/s`); + if (stats.outputTokens != null) { + parts.push( + stats.inputTokens != null + ? `${stats.inputTokens.toLocaleString("en-US")} in · ${stats.outputTokens.toLocaleString("en-US")} out` + : `${stats.outputTokens.toLocaleString("en-US")} tokens` + ); + } + return parts.join(" · "); +} diff --git a/frontend/src/lib/tokenStats.test.js b/frontend/src/lib/tokenStats.test.js new file mode 100644 index 0000000..813bbd7 --- /dev/null +++ b/frontend/src/lib/tokenStats.test.js @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import { buildTokenStats, formatTokenStats } from "./tokenStats"; + +describe("buildTokenStats", () => { + it("returns null without output tokens", () => { + expect(buildTokenStats({ usage: { inputTokens: 10 }, ttftMs: 100, durationMs: 1000 })).toBe( + null + ); + expect(buildTokenStats({ usage: null, ttftMs: 100, durationMs: 1000 })).toBe(null); + }); + + it("returns null without a duration", () => { + expect(buildTokenStats({ usage: { outputTokens: 50 }, ttftMs: 100, durationMs: null })).toBe( + null + ); + }); + + it("computes tokens/sec over generation time (excludes TTFT)", () => { + const stats = buildTokenStats({ + usage: { inputTokens: 200, outputTokens: 100, totalTokens: 300 }, + ttftMs: 500, + durationMs: 3000, + }); + expect(stats.tokensPerSecond).toBe(40); + expect(stats.ttftMs).toBe(500); + expect(stats.inputTokens).toBe(200); + expect(stats.outputTokens).toBe(100); + }); + + it("falls back to full duration when TTFT is missing", () => { + const stats = buildTokenStats({ + usage: { outputTokens: 50 }, + ttftMs: null, + durationMs: 2000, + }); + expect(stats.tokensPerSecond).toBe(25); + expect(stats.ttftMs).toBe(null); + }); + + it("guards against zero generation time", () => { + const stats = buildTokenStats({ + usage: { outputTokens: 10 }, + ttftMs: 1000, + durationMs: 1000, + }); + expect(stats.tokensPerSecond).toBe(10000); + }); +}); + +describe("formatTokenStats", () => { + it("formats sub-second TTFT in ms and seconds above", () => { + expect(formatTokenStats({ ttftMs: 420, outputTokens: 100 })).toBe("TTFT 420ms · 100 tokens"); + expect(formatTokenStats({ ttftMs: 2500, outputTokens: 100 })).toContain("TTFT 2.5s"); + }); + + it("shows in/out counts when both are known", () => { + const line = formatTokenStats({ + ttftMs: 300, + tokensPerSecond: 42.1, + inputTokens: 1240, + outputTokens: 156, + }); + expect(line).toBe("TTFT 300ms · 42.1 tok/s · 1,240 in · 156 out"); + }); + + it("omits missing values and handles null stats", () => { + expect(formatTokenStats(null)).toBe(""); + expect(formatTokenStats({ tokensPerSecond: 12.5 })).toBe("12.5 tok/s"); + }); +}); diff --git a/frontend/src/pages/authenticated/Settings.jsx b/frontend/src/pages/authenticated/Settings.jsx index bf94018..51a2730 100644 --- a/frontend/src/pages/authenticated/Settings.jsx +++ b/frontend/src/pages/authenticated/Settings.jsx @@ -30,6 +30,8 @@ const Settings = () => { const { returnToChat, isReturning } = useReturnToChat(); const showCodeLineNumbers = useThemeStore((state) => state.showCodeLineNumbers); const setShowCodeLineNumbers = useThemeStore((state) => state.setShowCodeLineNumbers); + const showTokenStats = useThemeStore((state) => state.showTokenStats); + const setShowTokenStats = useThemeStore((state) => state.setShowTokenStats); return (
@@ -100,6 +102,20 @@ const Settings = () => { aria-label="Show line numbers in code blocks" />
+ +
+
+ +

+ Show speed and token counts under assistant replies +

+
+ +
{/* Typography */} diff --git a/frontend/src/state/useThemeStore.js b/frontend/src/state/useThemeStore.js index 49759b7..2c49a88 100644 --- a/frontend/src/state/useThemeStore.js +++ b/frontend/src/state/useThemeStore.js @@ -150,6 +150,8 @@ export const useThemeStore = create( chatFontSize: "medium", // Code blocks showCodeLineNumbers: false, + // Assistant message token stats + showTokenStats: false, // Initialize theme on app start initializeTheme: async () => { @@ -241,6 +243,11 @@ export const useThemeStore = create( setShowCodeLineNumbers: (show) => { set({ showCodeLineNumbers: Boolean(show) }); }, + + // Toggle token stats under assistant messages + setShowTokenStats: (show) => { + set({ showTokenStats: Boolean(show) }); + }, }), { name: "theme-store-v3", @@ -251,6 +258,7 @@ export const useThemeStore = create( chatFont: state.chatFont, chatFontSize: state.chatFontSize, showCodeLineNumbers: state.showCodeLineNumbers, + showTokenStats: state.showTokenStats, }), } ) diff --git a/server/src/routes/chats.js b/server/src/routes/chats.js index 1780eb2..299af47 100644 --- a/server/src/routes/chats.js +++ b/server/src/routes/chats.js @@ -478,6 +478,16 @@ chatsRouter.post( console.error("Stream error:", error); return humanizeProviderError(error, providerLabel); }, + messageMetadata: ({ part }) => { + if (part.type !== "finish" || !part.totalUsage) return undefined; + return { + usage: { + inputTokens: part.totalUsage.inputTokens ?? null, + outputTokens: part.totalUsage.outputTokens ?? null, + totalTokens: part.totalUsage.totalTokens ?? null, + }, + }; + }, }); } catch (error) { console.error("Completion error:", error);