diff --git a/app/app-services.ts b/app/app-services.ts index fc7a44c4e4a9..1c93fcac073b 100644 --- a/app/app-services.ts +++ b/app/app-services.ts @@ -67,6 +67,8 @@ export { StreamAvatarService } from 'services/stream-avatar/stream-avatar-servic export { StreamAvatarApiService } from 'services/stream-avatar/stream-avatar-api-service'; export { AutomationsService } from 'services/stream-avatar/automations-service'; export { AutomationsEngineService } from 'services/stream-avatar/automations-engine-service'; +export { KevinSupportService } from 'services/stream-avatar/kevin-support-service'; +export { AgentToolsService } from 'services/stream-avatar/v2/agent-tools'; export { OnboardingV2Service } from 'services/onboarding/onboarding-v2'; // ONLINE SERVICES @@ -214,6 +216,8 @@ import { VirtualWebcamService } from 'services/virtual-webcam'; import { StreamAvatarApiService } from 'services/stream-avatar/stream-avatar-api-service'; import { AutomationsService } from 'services/stream-avatar/automations-service'; import { AutomationsEngineService } from 'services/stream-avatar/automations-engine-service'; +import { KevinSupportService } from 'services/stream-avatar/kevin-support-service'; +import { AgentToolsService } from 'services/stream-avatar/v2/agent-tools'; export const AppServices = { AppService, @@ -305,4 +309,6 @@ export const AppServices = { StreamAvatarApiService, AutomationsService, AutomationsEngineService, + KevinSupportService, + AgentToolsService, }; diff --git a/app/components-react/agent/KevinApprovalBubble.m.less b/app/components-react/agent/KevinApprovalBubble.m.less new file mode 100644 index 000000000000..dc37a60170f4 --- /dev/null +++ b/app/components-react/agent/KevinApprovalBubble.m.less @@ -0,0 +1,95 @@ +@import '../../styles/index'; + +/** + * A callout above the footer's Kevin icon, pointing down at it. + * + * `fixed`, not `absolute`: the footer clips on both axes (`overflow-y: hidden` + * on `.footer`, `overflow-x: auto` on it and on `.footer--left`), so a child + * positioned above the bar disappears. Fixed escapes ancestor overflow, and no + * ancestor sets a transform that would turn it back into a containing block. + * The component sets left/bottom from the icon's measured rect. + * + * Kept in the React tree rather than portalled to document.body: the theme is a + * class on the main window's root div, and everything below reads its CSS vars. + * Portalling out renders the card unstyled and light against a dark app. + * + * Deliberately the same fills, border and radius as the inline card in + * KevinSupport.m.less, so the two read as one component in two places. + */ +.bubble { + position: fixed; + width: 320px; + padding: 12px 14px; + background-color: var(--section-alt); + border: 1px solid var(--border); + border-radius: 12px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.4); + color: var(--title); + font-size: 14px; + line-height: 1.5; + // Above the footer and the OBS display, below a real modal (.ant-modal-wrap + // is 1003 in Main.m.less). + z-index: 1002; + cursor: default; + + // The arrow, pointing down at the icon. Two triangles so the border reads as + // a continuous outline rather than stopping where the bubble ends. Nothing + // here may sit inside an overflow container -- both are outside the box, and + // an `overflow` on this element would clip them and add a stray scrollbar. + &::before, + &::after { + content: ''; + position: absolute; + top: 100%; + left: 18px; + width: 0; + height: 0; + border-style: solid; + } + + &::before { + border-width: 8px 8px 0 8px; + border-color: var(--border) transparent transparent transparent; + } + + &::after { + border-width: 7px 7px 0 7px; + border-color: var(--section-alt) transparent transparent transparent; + left: 19px; + } +} + +/* Left accent marks a decision point, matching `.approval` in + KevinSupport.m.less. Stacked when one turn proposes more than one. */ +.approval { + padding-left: 10px; + border-left: 3px solid var(--warning); + + & + & { + margin-top: 12px; + padding-top: 12px; + border-top: 1px solid var(--border); + } +} + +.summary { + font-weight: 600; + margin-bottom: 4px; +} + +.warning { + color: var(--warning); + font-size: 12px; + margin-bottom: 8px; +} + +.actions { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-top: 10px; + + button { + margin: 0; + } +} diff --git a/app/components-react/agent/KevinApprovalBubble.tsx b/app/components-react/agent/KevinApprovalBubble.tsx new file mode 100644 index 000000000000..8ddf97525072 --- /dev/null +++ b/app/components-react/agent/KevinApprovalBubble.tsx @@ -0,0 +1,114 @@ +import React, { useLayoutEffect, useState } from 'react'; +import { Services } from 'components-react/service-provider'; +import { useVuex } from 'components-react/hooks'; +import { $t } from 'services/i18n'; +import styles from './KevinApprovalBubble.m.less'; + +interface Props { + /** The footer icon this points at. Measured, never positioned against. */ + anchorRef: React.RefObject; +} + +/** + * The approval prompt, shown above the Kevin icon in the studio footer. + * + * An approval used to force the support window open and focus it, which is an + * ambush mid-stream: a window jumps in front of whatever the streamer was doing, + * for a decision that is usually a single "yes". This carries the whole decision + * instead, so they answer without leaving the editor. + * + * Positioned `fixed` from the icon's measured rect rather than absolutely inside + * the footer, because the footer scrolls: `.footer` sets `overflow-y: hidden` + * and `overflow-x: auto`, and `.footer--left` another `overflow-x: auto`. A + * child positioned above the bar is clipped by both, which is invisible rather + * than merely misplaced. Fixed escapes that; it stays in the React tree so it + * keeps inheriting the theme class's CSS variables. + * + * It duplicates the card in KevinSupport.tsx rather than sharing one, because + * the two differ in everything but the three buttons: that one is a turn in a + * conversation, this one is a floating callout with no room for an avatar or a + * lead-in. Both call the same `resolveApproval`, which is the part that matters. + */ +export default function KevinApprovalBubble({ anchorRef }: Props) { + const { KevinSupportService, WindowsService } = Services; + + const { pendingApprovals, chatFocused } = useVuex(() => ({ + pendingApprovals: KevinSupportService.state.pendingApprovals, + // The state entry is deleted when the window closes, so `undefined` covers + // "closed" and `false` covers "open but behind something" in one read. + chatFocused: !!WindowsService.state['kevin-support']?.isFocused, + })); + + // Nothing to add when the streamer is already looking at the chat — the card + // in there is the live surface, and two copies of one decision is worse than + // one in the wrong place. + const show = !chatFocused && pendingApprovals.length > 0; + + const [anchor, setAnchor] = useState(null); + + useLayoutEffect(() => { + if (!show) return; + const measure = () => setAnchor(anchorRef.current?.getBoundingClientRect() ?? null); + measure(); + // ponytail: re-measured on resize only. The icon also shifts when the + // performance metrics beside it change width, or if the footer is scrolled + // horizontally — watch those with a ResizeObserver if it ever looks off. + window.addEventListener('resize', measure); + return () => window.removeEventListener('resize', measure); + }, [show, anchorRef]); + + if (!show || !anchor) return <>; + + return ( + // role="alert" rather than "alertdialog": the content is what should be + // announced, and an alertdialog would need a label naming a prompt whose + // whole text is already the summary below. +
+ {pendingApprovals.map(approval => ( +
+
{approval.summary}
+ {approval.risk === 'irreversible' && ( +
{$t('This cannot be undone.')}
+ )} + {approval.risk === 'external' && ( +
{$t('This will be visible to your viewers.')}
+ )} +
+ + + +
+
+ ))} +
+ ); +} diff --git a/app/components-react/agent/KevinSupport.m.less b/app/components-react/agent/KevinSupport.m.less new file mode 100644 index 000000000000..efc4cafe285d --- /dev/null +++ b/app/components-react/agent/KevinSupport.m.less @@ -0,0 +1,314 @@ +@import '../../styles/index'; +@import '../../styles/ultra'; + +.window { + overflow: hidden; +} + +.body { + display: flex; + flex-direction: column; + padding: 0 !important; + height: 100%; + overflow: hidden; + background-color: var(--background); +} + +.content { + display: flex; + flex: 1 0 0; + flex-direction: column; + gap: 20px; + min-height: 0; + padding: 20px 24px; +} + +// -- empty state ------------------------------------------------------------- + +.empty-state { + display: flex; + flex: 1 0 0; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 6px; + min-height: 0; + padding: 16px 0; + text-align: center; +} + +.empty-title { + .weight(@bold); + + margin: 6px 0 0; + color: var(--title); + font-size: 18px; +} + +.empty-body { + max-width: 420px; + margin: 0; + color: var(--paragraph); + font-size: 13px; + line-height: 1.5; +} + +// -- suggested prompts ------------------------------------------------------- + +.suggestions { + display: flex; + flex-direction: column; + gap: 12px; +} + +.suggestions-title { + .weight(@bold); + + color: var(--paragraph); + font-size: 11px; + text-transform: uppercase; +} + +.prompt-row { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.prompt { + .weight(@medium); + + padding: 8px 12px; + color: var(--title); + font-size: 12px; + background-color: var(--section); + border: 1px solid var(--border); + border-radius: 100px; + cursor: pointer; + + &:hover { + border-color: var(--teal); + } + + &:focus { + outline: none; + } +} + +// -- transcript -------------------------------------------------------------- + +.messages { + flex: 1 0 0; + min-height: 0; +} + +.message { + display: flex; + gap: 10px; + align-items: flex-start; + margin-bottom: 20px; +} + +.from-user { + justify-content: flex-end; +} + +.from-agent { + justify-content: flex-start; +} + +// The Figma badge draws at roughly 16x14 inside its 28px frame; the top margin +// nudges it down onto the bubble's first line of text instead of its own top. +.avatar { + flex-shrink: 0; + width: 16px; + height: 14px; + margin-top: 7px; +} + +.bubble { + max-width: 550px; + padding: 10px 14px; + color: var(--title); + font-size: 14px; + line-height: 1.5; + white-space: pre-wrap; + overflow-wrap: anywhere; + background-color: var(--section-alt); + border: 1px solid var(--border); + border-radius: 0 12px 12px 12px; + + .from-user & { + background-color: var(--chat-bubble-user); + border-color: var(--chat-bubble-user-border); + color: var(--white); + border-radius: 12px 12px 0 12px; + } +} + +.link { + color: var(--teal); + text-decoration: underline; + cursor: pointer; +} + +// -- errors ------------------------------------------------------------------ + +.error { + padding: 8px 24px; + color: var(--warning); + font-size: 12px; +} + +// -- composer ---------------------------------------------------------------- + +.composer { + display: flex; + flex-shrink: 0; + gap: 12px; + align-items: flex-end; + padding: 12px 16px; + background-color: var(--section); + border-top: 1px solid var(--border); +} + +.input { + flex: 1 0 0; + min-width: 0; + padding: 8px 12px !important; + color: var(--title); + font-size: 13px; + background-color: var(--background) !important; + border: 1px solid var(--border) !important; + border-radius: 6px !important; + resize: none; + + &:focus { + border-color: var(--focus-border) !important; + box-shadow: none !important; + } +} + +.send { + display: inline-flex; + align-items: center; + gap: 6px; + flex-shrink: 0; + height: 36px; + padding: 8px 16px; +} + +/* Inline tool-approval card. Shares the agent bubble, but wider and with a + left accent so it reads as a decision point rather than another reply. */ +.approval { + .bubble { + max-width: 100%; + // The bubble's default radius rounds its bottom-left corner, which would + // clip the straight warning accent below -- square both left corners here. + border-radius: 0 12px 12px 0; + border-left: 3px solid var(--warning); + } +} + +.approvalSummary { + font-weight: 600; + margin-bottom: 4px; +} + +.approvalWarning { + color: var(--warning); + font-size: 12px; + margin-bottom: 8px; +} + +.approvalActions { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-top: 8px; + + button { + margin: 0; + } +} + +/* Shown when an approval is the only thing in the window, so the card is not + floating with no explanation of where it came from. */ +.approvalContext { + color: var(--paragraph); + font-size: 12px; + text-align: center; + padding: 12px 16px 4px; +} + +// -- interaction quota -------------------------------------------------------- +// Mirrors the Automations usage meter (EditAutomations.m.less) so the two read +// as the same control. Right-aligned and out of the flex flow so it sits in the +// window's top-right corner without taking height from the conversation. + +.usage-meter { + display: flex; + flex-direction: column; + align-items: flex-end; + gap: 6px; + flex-shrink: 0; + margin-bottom: -8px; + font-size: 13px; + white-space: nowrap; +} + +.usage-row { + display: flex; + align-items: center; + gap: 8px; +} + +.usage-text { + color: var(--title); +} + +.usage-info { + color: var(--icon); + cursor: help; +} + +.usage-track { + width: 140px; + height: 6px; + border-radius: 3px; + background: var(--border); + overflow: hidden; +} + +.usage-fill { + height: 100%; + border-radius: 3px; + background: var(--teal); + transition: width 150ms ease; +} + +// Spent, not merely nearly spent. The bar is the thing seen at a glance, so it +// should not still read as healthy on the request that gets refused. +.usage-fill-full { + background: var(--warning); +} + +.upgrade-link { + display: inline-flex; + align-items: center; + gap: 6px; + font-weight: 600; + cursor: pointer; + + &:hover .upgrade-text { + opacity: 0.85; + } +} + +.upgrade-text { + .ultra-text('warm'); +} + +.at-cap-note { + color: var(--paragraph); +} diff --git a/app/components-react/agent/KevinSupport.tsx b/app/components-react/agent/KevinSupport.tsx new file mode 100644 index 000000000000..3e2ba18690a0 --- /dev/null +++ b/app/components-react/agent/KevinSupport.tsx @@ -0,0 +1,398 @@ +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import cx from 'classnames'; +import { Input, Tooltip } from 'antd'; +import * as remote from '@electron/remote'; +import { $t } from 'services/i18n'; +import { Services } from 'components-react/service-provider'; +import { useVuex } from 'components-react/hooks'; +import Scrollable from 'components-react/shared/Scrollable'; +import { ModalLayout } from 'components-react/shared/ModalLayout'; +import KevinSvg from 'components-react/shared/KevinSvg'; +import UltraIcon from 'components-react/shared/UltraIcon'; +import { KevinChatIcon, SendIcon } from 'components-react/shared/icons'; +import { INTERACTION_LIMITS, ULTRA_PLUS_TIER, promptUpgrade, upgrade } from './support-limits'; +import styles from './KevinSupport.m.less'; + +// $t() must be called at render time, not module load, so the strings pick up a +// language change — and inline so they stay extractable. The chip's label and +// the message it sends are independent strings: a short label reads well as a +// pill, but the agent needs the actual question spelled out. +const suggestedPrompts = () => [ + { label: $t('Setup alerts & widgets'), prompt: $t('How do I setup alerts & widgets?') }, + { label: $t('Mute mic'), prompt: $t('Mute my microphone') }, + { + label: $t('Connect streaming platforms'), + prompt: $t('How do I connect more streaming platforms?'), + }, + { label: $t('Sidekick'), prompt: $t('How do I setup Streamlabs Sidekick?') }, +]; + +// [label](url) **bold** *italic* `code` +const INLINE_MD = /\[([^\]]+)\]\(([^)\s]+)\)|\*\*([^*]+)\*\*|\*([^*]+)\*|`([^`]+)`/g; + +// agent text is server-provided; only http(s) may reach openExternal. +function safeHttpUrl(raw: string): string | null { + try { + const u = new URL(raw); + return u.protocol === 'http:' || u.protocol === 'https:' ? u.href : null; + } catch { + return null; + } +} + +/** + * the agent's technical-assistant mode emits inline markdown only — + * links, emphasis, the odd code span — never lists or headings, because its + * system prompt forbids them. So one regex pass beats pulling in react-markdown. + * Swap in a real renderer if replies ever start using block-level markdown. + */ +function renderText(text: string): React.ReactNode[] { + // A fresh regex per call: emphasis recurses, and a shared /g literal would have + // its lastIndex clobbered by the inner call — an infinite loop, not a wrong result. + const re = new RegExp(INLINE_MD.source, 'g'); + const nodes: React.ReactNode[] = []; + let lastIndex = 0; + let match: RegExpExecArray | null; + + // eslint-disable-next-line no-cond-assign + while ((match = re.exec(text)) !== null) { + if (match.index > lastIndex) nodes.push(text.slice(lastIndex, match.index)); + + const [, label, url, bold, italic, code] = match; + const key = `${match.index}`; + + if (url) { + const safe = safeHttpUrl(url); + if (!safe) { + nodes.push(match[0]); + } else { + nodes.push( + { + e.preventDefault(); + remote.shell.openExternal(safe); + }} + className={styles.link} + > + {label} + , + ); + } + } else if (bold) { + // Recurse so `**[label](url)**` stays a link instead of rendering raw. + nodes.push({renderText(bold)}); + } else if (italic) { + nodes.push({renderText(italic)}); + } else { + nodes.push({code}); + } + + lastIndex = match.index + match[0].length; + } + + if (lastIndex < text.length) nodes.push(text.slice(lastIndex)); + return nodes; +} + +/** + * Interactions used, top right, mirroring the Automations usage meter. + * + * Counts come from the server on every request, so this reflects the quota that + * actually applies rather than one derived here. The tier constants are only + * used to name the next tier's allowance in the tooltip and the upsell. + */ +function UsageMeter(p: { tier: string; rateLimit: { current: number; maximum: number } | null }) { + const atTopTier = p.tier === ULTRA_PLUS_TIER; + + // The server reports the real counts, but only once it has handled a request, + // so waiting for them left the meter absent until after the first message -- + // which is exactly when someone on the free tier most wants to see what their + // allowance is. The tier's own limit stands in until then, the way the + // Automations meter derives its numbers locally, and the server's figures + // replace it the moment they arrive. + const current = p.rateLimit?.current ?? 0; + const maximum = p.rateLimit?.maximum ?? INTERACTION_LIMITS[p.tier] ?? INTERACTION_LIMITS.free; + + const pct = maximum > 0 ? Math.min(100, Math.round((current / maximum) * 100)) : 0; + // maximum > 0 guards the degenerate case: 0 >= 0 would offer an upgrade to + // someone whose quota simply has not been reported yet. + const atCap = maximum > 0 && current >= maximum; + + return ( +
+
+ + {$t('%{count}/%{max} interactions used', { count: current, max: maximum })} + + + + +
+
+
+
+ + {atCap && !atTopTier && ( + upgrade(p.tier, 'meter')}> + + + {p.tier === 'ultra' + ? $t('Upgrade to Ultra+ for more') + : $t('Upgrade to Ultra for more')} + + + )} + {atCap && atTopTier && ( + {$t('Monthly limit reached')} + )} +
+ ); +} + +export default function KevinSupport() { + const { KevinSupportService, UserService } = Services; + + const { + messages, + pending, + error, + pendingApprovals, + rateLimit, + rateLimitRefusals, + tier, + } = useVuex(() => ({ + messages: KevinSupportService.state.messages, + pending: KevinSupportService.state.pending, + error: KevinSupportService.state.error, + pendingApprovals: KevinSupportService.state.pendingApprovals, + rateLimit: KevinSupportService.state.rateLimit, + rateLimitRefusals: KevinSupportService.state.rateLimitRefusals, + // Read reactively: a subscription that lapses while the window is open has + // to move the meter and the upsell with it, which a one-shot call at render + // time never would. + tier: UserService.views.tier, + })); + + const [draft, setDraft] = useState(''); + const listRef = useRef(null); + + // A pending approval is content, even with no messages behind it: an approval + // raised by a voice request through the avatar plugin arrives on a Desktop + // chat that has never been used. Gating purely on messages.length showed the + // "How can we help you today?" empty state while an approval sat unanswered + // in state, and the run expired. + const isEmpty = useMemo(() => messages.length === 0 && pendingApprovals.length === 0, [ + messages.length, + pendingApprovals.length, + ]); + + useEffect(() => { + KevinSupportService.actions.connect(); + }, []); + + // Every refused request gets an answer, which is how Automations behaves: it + // prompts on each blocked action rather than once per period. Keyed on the + // refusal count and not on `exceeded`, because that latches true for the rest + // of the period and a later attempt would otherwise be swallowed in silence -- + // the quota error is no longer shown as a banner, so this modal is the only + // thing that tells them why nothing happened. + useEffect(() => { + if (rateLimitRefusals > 0) promptUpgrade(tier); + }, [rateLimitRefusals]); + + useEffect(() => { + // Scroll the OverlayScrollbars viewport itself. scrollIntoView() would walk up + // and scroll every scrollable ancestor including the document, which drags the + // window's title bar off the top of the screen. + const viewport = listRef.current?.closest('.os-viewport') as HTMLElement | null; + if (viewport) viewport.scrollTop = viewport.scrollHeight; + }, [messages.length, pending, pendingApprovals.length]); + + const send = useCallback( + (text: string) => { + const trimmed = text.trim(); + if (!trimmed || pending) return; + KevinSupportService.actions.sendMessage(trimmed); + setDraft(''); + }, + [pending], + ); + + const onKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (e.key !== 'Enter' || e.shiftKey) return; + e.preventDefault(); + send(draft); + }, + [draft, send], + ); + + return ( + +
+ + + {isEmpty ? ( +
+ +

{$t('How can we help you today?')}

+

+ {$t( + 'Ask me any question about alerts, widgets, audio delay, or connected streaming platforms.', + )} +

+
+ ) : ( + // Scrollable forwards `className` into OverlayScrollbars' options (a + // scrollbar-theme name), not onto the DOM node — so the height has to be + // constrained by this wrapper plus `style`, never by a CSS-module class. +
+ +
+ {messages.map((message, i) => ( +
+ {!message.isUser && ( + + )} +
{renderText(message.text)}
+
+ ))} + {/* An approval is a turn in the conversation, not a modal over + it: the agent asked for something and is waiting on an + answer. Rendering it inline also means it cannot be missed + behind another window. */} + {pendingApprovals.length > 0 && messages.length === 0 && ( +
+ {$t('Your avatar is asking permission for something you requested by voice.')} +
+ )} + + {pendingApprovals.map(approval => ( +
+ +
+
{approval.summary}
+ {approval.risk === 'irreversible' && ( +
{$t('This cannot be undone.')}
+ )} + {approval.risk === 'external' && ( +
+ {$t('This will be visible to your viewers.')} +
+ )} +
+ + + +
+
+
+ ))} + + {pending && pendingApprovals.length === 0 && ( +
+ +
+ +
+
+ )} +
+ +
+ )} + + {isEmpty && ( +
+ {$t('Suggested Prompts')} +
+ {suggestedPrompts().map(({ label, prompt }) => ( + + ))} +
+
+ )} +
+ + {error &&
{error}
} + +
+ setDraft(e.target.value)} + onKeyDown={onKeyDown} + placeholder={$t('Type your message...')} + autoSize={{ minRows: 1, maxRows: 5 }} + bordered={false} + /> + +
+ + ); +} diff --git a/app/components-react/agent/kevin-analytics.ts b/app/components-react/agent/kevin-analytics.ts new file mode 100644 index 000000000000..1b6cb2aa8c89 --- /dev/null +++ b/app/components-react/agent/kevin-analytics.ts @@ -0,0 +1,42 @@ +import { Services } from 'components-react/service-provider'; + +/** + * Every support-chat analytics action. + * + * Exported because `KevinSupportService` fires five of these from the worker and + * imports this type to stay checked against the same list -- a type-only import, + * so no runtime dependency runs the wrong way. The Automations version skipped + * that step and its service-side 'automation_fired' is absent from the union. + */ +export type TSupportChatAction = + | 'chat_opened' + | 'message_sent' + | 'run_ended' + | 'tool_executed' + | 'approval_requested' + | 'approval_resolved' + | 'limit_reached' + | 'upsell_clicked'; + +/** Where an approval was answered. The footer bubble exists so the chat window need not be. */ +export type TApprovalSurface = 'chat' | 'footer'; +/** Which upgrade affordance was clicked. */ +export type TUpsellSource = 'meter' | 'modal'; +/** How the spent quota was announced -- Ultra+ gets a toast, everyone else the modal. */ +export type TLimitSurface = 'modal' | 'toast'; + +export const KevinAnalytics = { + track(action: TSupportChatAction, payload?: Record) { + Services.UsageStatisticsService.actions.recordAnalyticsEvent('SupportChat', { + action, + ...payload, + }); + }, + /** The footer icon was clicked, whether or not that created a new window. */ + chatOpened: () => KevinAnalytics.track('chat_opened'), + /** The user hit the cap and was told so. */ + limitReached: (payload: { tier: string; max: number; surface: TLimitSurface }) => + KevinAnalytics.track('limit_reached', payload), + upsellClicked: (payload: { tier: string; target: string; source: TUpsellSource }) => + KevinAnalytics.track('upsell_clicked', payload), +}; diff --git a/app/components-react/agent/support-limits.tsx b/app/components-react/agent/support-limits.tsx new file mode 100644 index 000000000000..fadd277bf7ac --- /dev/null +++ b/app/components-react/agent/support-limits.tsx @@ -0,0 +1,90 @@ +import React from 'react'; +import { message } from 'antd'; +import { Services } from 'components-react/service-provider'; +import { $t } from 'services/i18n'; +import UltraIcon from 'components-react/shared/UltraIcon'; +import { promptAction } from 'components-react/modals'; +import { KevinAnalytics, TUpsellSource } from './kevin-analytics'; + +/** + * Support-chat interaction allowances, mirroring the server's RateLimitService. + * + * Kept here only so the tooltip and the upsell can name the next tier's number. + * The server is the authority and its counts arrive on `v2:rateLimit` -- nothing + * here gates a request, and a drift between these numbers and the server's is a + * wrong sentence, never a wrong decision. + * + * Note the free tier is a LIFETIME allowance, not monthly: it does not come back + * next month, which is the whole reason the upsell is worth showing. + */ +export const INTERACTION_LIMITS: Record = { + free: 100, + ultra: 1000, + ultra_plus: 5000, +}; + +/** Matches automations-limits: only 'free' and 'ultra' come back from the API today. */ +export const ULTRA_PLUS_TIER = 'ultra_plus'; + +/** Sends the user to checkout for whatever tier sits above the one they are on. */ +export function upgrade(currentTier: string, source: TUpsellSource) { + const toUltraPlus = currentTier === 'ultra'; + + KevinAnalytics.upsellClicked({ + tier: currentTier, + target: toUltraPlus ? ULTRA_PLUS_TIER : 'ultra', + source, + }); + + // A distinct refl from Automations' 'slobs-automations', so the two upsells + // are separable in the Ultra conversion funnel rather than one blended number. + Services.MagicLinkService.actions.linkToPrime('slobs-support-chat', { + event: 'SupportChat', + ...(toUltraPlus ? { tier: ULTRA_PLUS_TIER } : {}), + }); +} + +/** + * The "you are out of interactions" upsell. + * + * Called when the server says the limit is spent. Deliberately not a gate: the + * server already refused the request and said so, and a second client-side + * refusal would be a guess layered on an answer we already have. + */ +export function promptUpgrade(tier: string) { + const max = INTERACTION_LIMITS[tier] ?? INTERACTION_LIMITS.free; + const atTopTier = tier === ULTRA_PLUS_TIER; + + // Ahead of the branch, so the Ultra+ toast counts as a limit hit too. The + // recordShown below deliberately stays where it is -- it pairs with the + // recordUltra that linkToPrime fires, and only the modal has a click to pair with. + KevinAnalytics.limitReached({ tier, max, surface: atTopTier ? 'toast' : 'modal' }); + + if (atTopTier) { + message.warning($t("You've used all %{max} of this month's support interactions.", { max }), 5); + return; + } + + // Pairs with the recordUltra that linkToPrime fires, so shown-vs-clicked is a + // straight ratio within one stream. + Services.UsageStatisticsService.actions.recordShown('SupportChat', 'slobs-support-chat'); + + const toUltraPlus = tier === 'ultra'; + promptAction({ + title: $t('Interaction limit reached'), + message: toUltraPlus + ? $t('Ultra includes %{max} support interactions a month. Upgrade to Ultra+ for %{next}.', { + max, + next: INTERACTION_LIMITS[ULTRA_PLUS_TIER], + }) + : $t( + 'Free accounts include %{max} support interactions in total. Upgrade to Ultra for %{next} every month.', + { max, next: INTERACTION_LIMITS.ultra }, + ), + icon: , + btnText: toUltraPlus ? $t('Upgrade to Ultra+') : $t('Upgrade to Ultra'), + fn: () => upgrade(tier, 'modal'), + cancelBtnPosition: 'left', + cancelBtnText: $t('Not now'), + }); +} diff --git a/app/components-react/index.ts b/app/components-react/index.ts index 436a489dbbfb..9e3492f7e400 100644 --- a/app/components-react/index.ts +++ b/app/components-react/index.ts @@ -40,6 +40,7 @@ import DismissableBadge from './shared/DismissableBadge'; import UltraIcon from './shared/UltraIcon'; import EditTransform from './windows/EditTransform'; import EditAutomations from './windows/stream-avatar-automations/EditAutomations'; +import KevinSupport from './agent/KevinSupport'; import MarketingModal from './windows/MarketingModal'; import Main from './windows/Main'; import Loader from './pages/Loader'; @@ -96,6 +97,7 @@ export const components = { UltraIcon, EditTransform, EditAutomations, + KevinSupport, Blank, MarketingModal, Main: createRoot(Main), diff --git a/app/components-react/root/StudioFooter.m.less b/app/components-react/root/StudioFooter.m.less index 5d53cd3bdc9b..9174ff2505f0 100644 --- a/app/components-react/root/StudioFooter.m.less +++ b/app/components-react/root/StudioFooter.m.less @@ -38,6 +38,34 @@ } } +/* Measured by the approval bubble, which is positioned `fixed` from this + element's rect to escape the footer's clipping. */ +.kevin-anchor { + display: flex; +} + +.kevin-icon { + display: flex; + align-items: center; + padding: 0 12px; + background: transparent; + border: none; + color: var(--paragraph); + + &:hover { + cursor: pointer; + color: var(--title); + } + + &:focus { + outline: none; + } + + &:focus-visible { + outline: 1px solid var(--teal); + } +} + .warning { color: var(--warning); } diff --git a/app/components-react/root/StudioFooter.tsx b/app/components-react/root/StudioFooter.tsx index 5c2f8570732d..ade45906b803 100644 --- a/app/components-react/root/StudioFooter.tsx +++ b/app/components-react/root/StudioFooter.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useCallback, useMemo, memo } from 'react'; +import React, { useState, useEffect, useCallback, useMemo, useRef, memo } from 'react'; import cx from 'classnames'; import { EStreamQuality } from '../../services/performance'; import { EStreamingState, EReplayBufferState, ERecordingState } from '../../services/streaming'; @@ -14,6 +14,9 @@ import { Tooltip } from 'antd'; import { confirmAsync } from 'components-react/modals'; import RecordingSwitcher from 'components-react/windows/go-live/RecordingSwitcher'; import { EAvailableFeatures } from 'services/incremental-rollout'; +import { KevinChatIcon } from 'components-react/shared/icons'; +import KevinApprovalBubble from 'components-react/agent/KevinApprovalBubble'; +import { KevinAnalytics } from 'components-react/agent/kevin-analytics'; function StudioFooterComponent() { const { @@ -95,6 +98,29 @@ function StudioFooterComponent() { UsageStatisticsService.actions.recordFeatureUsage('PerformanceStatistics'); }, []); + const kevinAnchorRef = useRef(null); + + const openKevinSupport = useCallback(() => { + // A one-off window, not showWindow(): there is only one shared `child` window, + // so showWindow would close whatever the user already had open. Support needs + // to sit alongside the thing being asked about. The fixed windowId means a + // second click restores and focuses the existing window instead of duplicating. + WindowsService.actions.createOneOffWindow( + { + componentName: 'KevinSupport', + title: $t('Streamlabs Desktop Support'), + queryParams: {}, + size: { width: 900, height: 640, minWidth: 560, minHeight: 420 }, + }, + 'kevin-support', + ); + // Tracked on the click, not on the window's mount effect: the fixed windowId + // means a second click only refocuses, so the component never remounts and + // the reach for support would go unrecorded. + KevinAnalytics.chatOpened(); + UsageStatisticsService.actions.recordFeatureUsage('KevinSupportChat'); + }, []); + const toggleReplayBuffer = useCallback(() => { if (replayBufferStatus === EReplayBufferState.Offline) { StreamingService.actions.startReplayBuffer(); @@ -143,6 +169,23 @@ function StudioFooterComponent() { /> + {isLoggedIn && ( + // The wrapper exists only to give the approval bubble something to + // measure; the bubble positions itself `fixed`, since the footer clips. +
+ + + + +
+ )}
diff --git a/app/components-react/shared/icons/KevinChatIcon.tsx b/app/components-react/shared/icons/KevinChatIcon.tsx new file mode 100644 index 000000000000..06a34773ace4 --- /dev/null +++ b/app/components-react/shared/icons/KevinChatIcon.tsx @@ -0,0 +1,27 @@ +import React from 'react'; +import SvgContainer from 'components-react/shared/SvgContainer'; + +export default function KevinChatIcon(p: { className?: string; style?: React.CSSProperties }) { + return ; +} + +const kevinChatSvg = ` + + + + + +`; diff --git a/app/components-react/shared/icons/SendIcon.tsx b/app/components-react/shared/icons/SendIcon.tsx new file mode 100644 index 000000000000..d54294311a24 --- /dev/null +++ b/app/components-react/shared/icons/SendIcon.tsx @@ -0,0 +1,17 @@ +import React from 'react'; +import SvgContainer from 'components-react/shared/SvgContainer'; + +export default function SendIcon(p: { className?: string; style?: React.CSSProperties }) { + return ; +} + +const sendSvg = ` + + + +`; diff --git a/app/components-react/shared/icons/index.ts b/app/components-react/shared/icons/index.ts index 331b902b517d..c377d1e86fcc 100644 --- a/app/components-react/shared/icons/index.ts +++ b/app/components-react/shared/icons/index.ts @@ -1 +1,3 @@ export { default as AutomationsIcon } from './AutomationsIcon'; +export { default as KevinChatIcon } from './KevinChatIcon'; +export { default as SendIcon } from './SendIcon'; diff --git a/app/components/shared/ReactComponentList.tsx b/app/components/shared/ReactComponentList.tsx index acd2e6b867aa..77c7d4e38fcc 100644 --- a/app/components/shared/ReactComponentList.tsx +++ b/app/components/shared/ReactComponentList.tsx @@ -81,6 +81,14 @@ export class EditTransform extends ReactComponent {} }) export class EditAutomations extends ReactComponent {} +@Component({ + props: { + name: { default: 'KevinSupport' }, + wrapperStyles: { default: () => ({ height: '100%' }) }, + }, +}) +export class KevinSupport extends ReactComponent {} + @Component({ props: { name: { default: 'GoLiveWindow' }, diff --git a/app/i18n/en-US/ai.json b/app/i18n/en-US/ai.json index d5412148d5dc..d6d09e161182 100644 --- a/app/i18n/en-US/ai.json +++ b/app/i18n/en-US/ai.json @@ -26,5 +26,33 @@ "Open Display Frame": "Open Display Frame", "Active Process": "Active Process", "Selected Game": "Selected Game", - "There was an error installing Streamlabs AI.": "There was an error installing Streamlabs AI." + "There was an error installing Streamlabs AI.": "There was an error installing Streamlabs AI.", + "Streamlabs Desktop Support": "Streamlabs Desktop Support", + "How can we help you today?": "How can we help you today?", + "Ask me any question about alerts, widgets, audio delay, or connected streaming platforms.": "Ask me any question about alerts, widgets, audio delay, or connected streaming platforms.", + "Suggested Prompts": "Suggested Prompts", + "Setup alerts & widgets": "Setup alerts & widgets", + "How do I setup alerts & widgets?": "How do I setup alerts & widgets?", + "Mute mic": "Mute mic", + "Mute my microphone": "Mute my microphone", + "Connect streaming platforms": "Connect streaming platforms", + "How do I connect more streaming platforms?": "How do I connect more streaming platforms?", + "How do I setup Streamlabs Sidekick?": "How do I setup Streamlabs Sidekick?", + "Type your message...": "Type your message...", + "Send": "Send", + "Log in to use Streamlabs Desktop Support.": "Log in to use Streamlabs Desktop Support.", + "Could not connect to Streamlabs Desktop Support. Please try again.": "Could not connect to Streamlabs Desktop Support. Please try again.", + "Something went wrong. Please try again.": "Something went wrong. Please try again.", + "%{count}/%{max} interactions used": "%{count}/%{max} interactions used", + "Free includes %{free} interactions in total. Ultra includes %{ultra} a month and Ultra+ %{ultraPlus}.": "Free includes %{free} interactions in total. Ultra includes %{ultra} a month and Ultra+ %{ultraPlus}.", + "Upgrade to Ultra+ for more": "Upgrade to Ultra+ for more", + "Upgrade to Ultra for more": "Upgrade to Ultra for more", + "Monthly limit reached": "Monthly limit reached", + "Interaction limit reached": "Interaction limit reached", + "Ultra includes %{max} support interactions a month. Upgrade to Ultra+ for %{next}.": "Ultra includes %{max} support interactions a month. Upgrade to Ultra+ for %{next}.", + "Free accounts include %{max} support interactions in total. Upgrade to Ultra for %{next} every month.": "Free accounts include %{max} support interactions in total. Upgrade to Ultra for %{next} every month.", + "You've used all %{max} of this month's support interactions.": "You've used all %{max} of this month's support interactions.", + "Upgrade to Ultra+": "Upgrade to Ultra+", + "Upgrade to Ultra": "Upgrade to Ultra", + "Not now": "Not now" } diff --git a/app/i18n/en-US/stream-avatar-agent.json b/app/i18n/en-US/stream-avatar-agent.json new file mode 100644 index 000000000000..8faca229e960 --- /dev/null +++ b/app/i18n/en-US/stream-avatar-agent.json @@ -0,0 +1,10 @@ +{ + "Allow this action?": "Allow this action?", + "Your avatar wants to: %{action}": "Your avatar wants to: %{action}", + "Your avatar is asking permission for something you requested by voice.": "Your avatar is asking permission for something you requested by voice.", + "This cannot be undone.": "This cannot be undone.", + "This will be visible to your viewers.": "This will be visible to your viewers.", + "Allow once": "Allow once", + "Always allow": "Always allow", + "Deny": "Deny" +} diff --git a/app/i18n/fallback.ts b/app/i18n/fallback.ts index 81c7bb0e3c27..2daf37a1bb6f 100644 --- a/app/i18n/fallback.ts +++ b/app/i18n/fallback.ts @@ -72,6 +72,7 @@ const fallbackDictionary = { ...require('./en-US/dual-output.json'), ...require('./en-US/patreon.json'), ...require('./en-US/stream-avatar-automations.json'), + ...require('./en-US/stream-avatar-agent.json'), }; export default fallbackDictionary; diff --git a/app/services/stream-avatar/automations-service.ts b/app/services/stream-avatar/automations-service.ts index 8e9fab286ccf..524d6905e49e 100644 --- a/app/services/stream-avatar/automations-service.ts +++ b/app/services/stream-avatar/automations-service.ts @@ -19,7 +19,7 @@ interface IAutomationsState { const RETRY_BASE_DELAY_MS = 2000; const RETRY_MAX_DELAY_MS = 30000; -// ponytail: give up after this many auto-retries. Without a cap a client whose +// give up after this many auto-retries. Without a cap a client whose // fetch fails for a non-transient reason (403, bad account state) polls // /automations + /token every 30s for the whole session. The window has a // "retry" button, and login re-arms the counter. diff --git a/app/services/stream-avatar/kevin-support-service.ts b/app/services/stream-avatar/kevin-support-service.ts new file mode 100644 index 000000000000..33a1739f5705 --- /dev/null +++ b/app/services/stream-avatar/kevin-support-service.ts @@ -0,0 +1,571 @@ +import { InitAfter } from 'services/core'; +import { StatefulService, mutation } from 'services/core/stateful-service'; +import { Inject } from 'services/core/injector'; +import { UserService } from 'services/user'; +import { $t } from 'services/i18n'; +import { HostsService } from 'services/hosts'; +import { UsageStatisticsService } from 'services/usage-statistics'; +import Utils from 'services/utils'; +import { importSocketIOClient } from 'util/slow-imports'; +// Type-only: the action vocabulary lives with the component-side helpers, and a +// type import is erased, so nothing in the services layer depends on React at runtime. +import type { TApprovalSurface, TSupportChatAction } from 'components-react/agent/kevin-analytics'; +import { StreamAvatarApiService } from './stream-avatar-api-service'; +import { AgentToolsService, TOOL_GROUPS } from './v2/agent-tools'; +import { + V2_NAMESPACE, + V2_PROTOCOL_VERSION, + V2_TOOL_PROTOCOL_VERSION, + V2ApprovalDecision, + V2ApprovalRequestPayload, + V2ReadyPayload, + V2RunEndedPayload, + V2TextPayload, + V2ToolInvokePayload, +} from './v2/protocol'; + +const CONNECT_TIMEOUT_MS = 15000; + +export interface IKevinMessage { + /** Groups the burst of text packets that make up one assistant reply. */ + interactionId: string; + isUser: boolean; + text: string; + date: number; +} + +export interface IKevinSupportState { + messages: IKevinMessage[]; + /** A reply has been requested and the run has not ended yet. */ + pending: boolean; + connected: boolean; + connecting: boolean; + error: string | null; + /** + * Interaction quota, as last reported by the server. `exceeded` is the + * server's own verdict on the request it just refused -- the UI shows the + * upsell from this rather than comparing current against maximum itself, + * because only the server knows whether the allowance is monthly or lifetime. + */ + rateLimit: { current: number; maximum: number; exceeded: boolean } | null; + /** + * How many requests the server has refused for quota. Increments per refusal + * rather than latching a boolean, because `exceeded` stays true for the rest + * of the period: the UI needs to answer every attempt, not just the first. + */ + rateLimitRefusals: number; + /** + * Sensitive tool calls waiting on a human. The worker owns the socket but + * cannot render, so a UI window observes this via useVuex and answers + * through resolveApproval(). + */ + pendingApprovals: V2ApprovalRequestPayload[]; +} + +/** + * Streamlabs Desktop Support chat ("Kevin"), on the agent API's `/v2` namespace. + * + * Beyond chat, this connection is now how the agent reaches OBS: the server + * emits `v2:tool.invoke`, we run it against the services layer, and reply with + * a correlated `v2:tool.result`. Nothing uses an acknowledgement callback, so + * a slow tool or a human sitting on an approval never blocks the server's + * agent loop. + * + * Still worker-window only, so the conversation and any in-flight tool call + * survive the support window being closed. + * + * socket.io-client here is v2, which has no `auth` option — identity rides the + * query string. The server accepts that and `allowEIO3` lets a v2 client speak + * to its 4.x server. Note that connection-state recovery is a v4-protocol + * feature and never engages for us, so every reconnect is a fresh session that + * resyncs from `v2:ready`. + */ +@InitAfter('UserService') +export class KevinSupportService extends StatefulService { + static initialState: IKevinSupportState = { + messages: [], + pending: false, + connected: false, + connecting: false, + error: null, + rateLimit: null, + rateLimitRefusals: 0, + pendingApprovals: [], + }; + + @Inject() private streamAvatarApiService: StreamAvatarApiService; + @Inject() private userService: UserService; + @Inject() private hostsService: HostsService; + @Inject() private agentToolsService: AgentToolsService; + @Inject() private usageStatisticsService: UsageStatisticsService; + + private io: SocketIOClientStatic; + private socket: SocketIOClient.Socket | null = null; + private connectPromise: Promise | null = null; + + init() { + if (!Utils.isWorkerWindow()) return; + + this.userService.userLogout.subscribe(() => { + this.disconnect(); + // The minted JWT outlives the session it was minted for, so an account + // switch would reconnect this socket as the previous user. AutomationsService + // clears it on logout for the same reason. + this.streamAvatarApiService.clearToken(); + this.RESET(); + }); + + // Connect at startup, not when the chat window first opens. + // + // Desktop is the approval surface for the whole product: an approval can be + // raised by a voice request through the avatar plugin, with this window + // never having been opened. Connecting lazily meant no `desktop` device was + // attached at that moment, so the server fell back to the plugin — and the + // rule "Desktop handles approvals whenever connected" silently degraded to + // "whenever the user happened to open the chat". + this.userService.userLogin.subscribe(() => void this.connect()); + if (this.userService.isLoggedIn) void this.connect(); + } + + /** Idempotent; concurrent callers share one in-flight connect. */ + async connect(): Promise { + if (!Utils.isWorkerWindow()) return; + if (this.state.connected && this.socket?.connected) return; + if (this.connectPromise) return this.connectPromise; + + this.connectPromise = this.openSocket().finally(() => { + this.connectPromise = null; + }); + + return this.connectPromise; + } + + /** Resolves once `v2:ready` lands, not merely once the socket object exists. */ + private async openSocket(): Promise { + if (!this.userService.isLoggedIn) { + this.SET_ERROR($t('Log in to use Streamlabs Desktop Support.')); + return; + } + + this.SET_CONNECTING(true); + this.SET_ERROR(null); + + try { + if (!this.io) this.io = (await importSocketIOClient()).default; + + const token = await this.streamAvatarApiService.getToken(); + const protocol = Utils.getAvatarEnvironment() === 'local' ? 'http://' : 'https://'; + const url = + `${protocol}${this.hostsService.streamAvatarApi}${V2_NAMESPACE}` + + `?token=${token}&role=desktop&tv=${V2_TOOL_PROTOCOL_VERSION}`; + + this.socket?.disconnect(); + this.log('--', 'connecting', { url: url.replace(/token=[^&]+/, 'token=***') }); + const socket = this.io(url, { transports: ['websocket'] }); + this.socket = socket; + + this.traceUnhandled(socket, [ + 'v2:text', + 'v2:run.started', + 'v2:presence', + 'v2:run.ended', + 'v2:tool.invoke', + 'v2:approval.request', + 'v2:approval.resolved', + 'v2:rateLimit', + 'v2:error', + ]); + + socket.on('v2:run.started', (p: { runId: string }) => { + this.log('in', 'v2:run.started', p); + this.SET_ERROR(null); + this.SET_PENDING(true); + }); + + socket.on('v2:presence', (p: { roles: string[]; sourceCount: number }) => { + this.log('in', 'v2:presence', p); + }); + + socket.on('v2:text', (p: V2TextPayload) => { + this.log('in', 'v2:text', { runId: p?.packetId?.runId, kind: p?.kind, text: p?.text }); + this.handleText(p); + }); + socket.on('v2:run.ended', (p: V2RunEndedPayload) => { + this.log('in', 'v2:run.ended', p); + this.handleRunEnded(p); + }); + socket.on('v2:tool.invoke', (p: V2ToolInvokePayload) => { + this.log('in', 'v2:tool.invoke', { callId: p?.callId, tool: p?.tool, args: p?.args }); + this.handleToolInvoke(p); + }); + socket.on('v2:approval.request', (p: V2ApprovalRequestPayload) => { + this.log('in', 'v2:approval.request', { + approvalId: p?.approvalId, + tool: p?.tool, + risk: p?.risk, + summary: p?.summary, + }); + // Nothing is opened or focused here. The footer bubble + // (KevinApprovalBubble) shows the prompt whenever the support window is + // closed or buried, so a decision no longer costs the streamer a window + // jumping in front of whatever they were doing mid-stream. + // + // Tracked here and not in ADD_APPROVAL (mutations stay pure) and not on + // the v2:ready replay below, which re-sends every still-live prompt -- + // counting those would turn one reconnect into a spike of new requests. + this.track('approval_requested', { tool: p.tool, risk: p.risk }); + this.ADD_APPROVAL(p); + }); + socket.on('v2:approval.resolved', (p: { approvalId: string }) => + this.REMOVE_APPROVAL(p.approvalId), + ); + socket.on('v2:rateLimit', (p: { current: number; maximum: number; exceeded?: boolean }) => + this.SET_RATE_LIMIT({ + current: p.current, + maximum: p.maximum, + exceeded: p.exceeded === true, + }), + ); + socket.on('v2:error', (p: { code: string; message: string }) => { + this.log('in', 'v2:error', p); + // An error means this attempt is over. A refused request never starts a + // run, so `v2:run.ended` never arrives and nothing else would clear + // this -- the spinner span forever and Send stayed disabled. + this.SET_PENDING(false); + + // Quota is answered by the upgrade modal, not by a red line: showing + // both says the same thing twice, and only one of them is actionable. + if (p.code === 'rate_limit') return; + this.SET_ERROR(p.message || $t('Something went wrong. Please try again.')); + }); + + socket.on('disconnect', (reason: string) => { + this.log('--', 'disconnect', { reason }); + this.SET_CONNECTED(false); + this.SET_CONNECTING(false); + this.SET_PENDING(false); + // Prompts belong to a live session; a stale one cannot be answered. + this.CLEAR_APPROVALS(); + }); + + await new Promise((resolve, reject) => { + const timeout = window.setTimeout(() => reject(new Error('timed out')), CONNECT_TIMEOUT_MS); + const settle = (err?: Error) => { + clearTimeout(timeout); + err ? reject(err) : resolve(); + }; + + socket.on('connect', () => { + this.log('out', 'v2:hello', { role: 'desktop' }); + socket.emit('v2:hello', { + protocolVersion: V2_PROTOCOL_VERSION, + toolProtocolVersion: V2_TOOL_PROTOCOL_VERSION, + role: 'desktop', + deviceId: this.deviceId(), + }); + }); + + socket.on('v2:ready', (ready: V2ReadyPayload) => { + this.log('in', 'v2:ready', { + role: ready?.role, + tools: ready?.tools, + activeRunIds: ready?.activeRunIds, + pendingApprovals: ready?.pendingApprovals?.length ?? 0, + }); + // Replayed approvals: a prompt raised while we were reconnecting is + // still live server-side and must reappear here. + this.SET_APPROVALS(ready.pendingApprovals ?? []); + this.SET_CONNECTING(false); + this.SET_CONNECTED(true); + settle(); + }); + + socket.on('connect_error', (e: unknown) => { + this.log('--', 'connect_error', { error: String(e) }); + settle(new Error('connect_error')); + }); + + // A refusal during the handshake — bad protocol version, internal error + // — is terminal: v2:ready will never arrive. Without this the connect + // sits here for the full CONNECT_TIMEOUT_MS and then reports a generic + // failure instead of what the server actually said. The handler + // registered above runs first and has already recorded that message; a + // v2:error after v2:ready lands on a settled promise and does nothing. + socket.on('v2:error', () => settle(new Error('v2:error'))); + }); + } catch (e: unknown) { + console.error('[KevinSupport] connect failed', e); + this.socket?.disconnect(); + this.socket = null; + this.SET_CONNECTING(false); + this.SET_CONNECTED(false); + this.SET_PENDING(false); + // Keep whatever the v2:error handler recorded — that is the server saying + // why. The generic line is for a socket that never got far enough to say + // anything, and overwriting with it was how a real reason got lost. + if (!this.state.error) { + this.SET_ERROR($t('Could not connect to Streamlabs Desktop Support. Please try again.')); + } + } + } + + /** + * Wire tracing. Event names are always logged: this socket is low-traffic + * (text chat plus the occasional tool call), and the failure mode it exists + * to catch — a packet the server sent to a room this device never joined — + * is otherwise completely silent on the client. Payload detail (chat text, + * tool arguments) is dev-only, so production logs never carry it. + */ + private log(direction: 'in' | 'out' | '--', event: string, detail?: unknown) { + const body = + detail === undefined || !Utils.isDevMode() ? '' : ` ${JSON.stringify(detail).slice(0, 400)}`; + console.log(`[KevinSupport ${direction}] ${event}${body}`); + } + + /** + * Feature analytics, worker-side. No `.actions` — we are already in the worker, + * so this is a plain call on the singleton, the way AutomationsEngineService + * records 'automation_fired'. Nothing here reports errors: connection failures + * and caught exceptions stay in the log above. `success` and `reason` are + * outcome fields on a usage event, which is a different thing. + */ + private track(action: TSupportChatAction, payload?: Record) { + this.usageStatisticsService.recordAnalyticsEvent('SupportChat', { action, ...payload }); + } + + /** + * Catch-all so we can see events the server sends that we do NOT handle. + * socket.io v2 exposes onevent rather than onAny. + */ + private traceUnhandled(socket: SocketIOClient.Socket, handled: string[]) { + const known = new Set([...handled, 'connect', 'disconnect', 'connect_error', 'v2:ready']); + const anySocket = (socket as unknown) as { + onevent: (packet: { data?: unknown[] }) => void; + }; + const original = anySocket.onevent.bind(anySocket); + anySocket.onevent = (packet: { data?: unknown[] }) => { + const name = String(packet?.data?.[0] ?? ''); + if (name && !known.has(name)) this.log('in', `${name} (UNHANDLED)`, packet?.data?.[1]); + original(packet); + }; + } + + /** Stable per-install id so a reconnect is recognised as the same device. */ + private deviceId(): string { + const KEY = 'sa.v2.desktopDeviceId'; + try { + const existing = localStorage.getItem(KEY); + if (existing) return existing; + const fresh = `desktop-${Date.now()}-${Math.floor(Math.random() * 1e6)}`; + localStorage.setItem(KEY, fresh); + return fresh; + } catch { + return `desktop-${Date.now()}`; + } + } + + // ── inbound ──────────────────────────────────────────────────────────────── + + private handleText(packet: V2TextPayload) { + if (!packet?.text) return; + + // kind === 'links' is the source footer for a knowledge answer. It appends + // to the reply it belongs to, or opens its own bubble when the question was + // asked by voice and the spoken answer never came through here. + const interactionId = packet.packetId?.runId ?? ''; + const existing = this.state.messages.find(m => !m.isUser && m.interactionId === interactionId); + + if (existing) { + this.APPEND_TEXT(interactionId, packet.text); + } else { + this.ADD_MESSAGE({ interactionId, isUser: false, text: packet.text, date: Date.now() }); + } + } + + private handleRunEnded(packet: V2RunEndedPayload) { + this.SET_PENDING(false); + // Paired with message_sent, this is the answer rate: how often a question + // actually produced a finished reply rather than being cancelled or lost. + this.track('run_ended', { reason: packet.reason }); + if (packet.reason === 'error' && packet.message) this.SET_ERROR(packet.message); + } + + /** + * Executes a tool the server routed here and replies. Always replies — the + * agent loop is parked on this callId and would otherwise wait out its + * timeout before telling the user anything. + */ + private async handleToolInvoke(invoke: V2ToolInvokePayload) { + const outcome = this.agentToolsService.canExecute(invoke.tool) + ? await this.agentToolsService.execute(invoke.tool, invoke.args ?? {}) + : { + ok: false as const, + code: 'unknown_tool', + message: `Desktop cannot run ${invoke.tool}.`, + }; + + this.log('out', 'v2:tool.result', { callId: invoke.callId, ok: outcome.ok }); + // Here rather than inside AgentToolsService.execute(): the unknown_tool + // fallback above short-circuits execute entirely, and a tool the server + // thinks we have but we do not is exactly the failure worth seeing. + // + // `code` is 'unknown_tool' | 'failed', and absent on success, so a chart of + // failure reasons does not have to filter out a null bucket. Narrowed with + // `in` rather than on `outcome.ok`: strictNullChecks is off for this file, + // and the boolean discriminant does not narrow the union without it. + this.track('tool_executed', { + tool: invoke.tool, + tool_group: TOOL_GROUPS[invoke.tool] ?? 'other', + success: outcome.ok, + ...('code' in outcome ? { code: outcome.code } : {}), + }); + this.socket?.emit('v2:tool.result', { callId: invoke.callId, outcome }); + } + + // ── outbound ─────────────────────────────────────────────────────────────── + + async sendMessage(text: string): Promise { + const trimmed = text.trim(); + if (!trimmed) return; + + await this.connect(); + if (!this.state.connected || !this.socket) return; // openSocket already set the error + + this.SET_ERROR(null); + this.ADD_MESSAGE({ + interactionId: `local-${Date.now()}`, + isUser: true, + text: trimmed, + date: Date.now(), + }); + this.SET_PENDING(true); + this.log('out', 'v2:input.text', { text: trimmed.slice(0, 80) }); + this.socket.emit('v2:input.text', { text: trimmed, responseType: 'text' }); + // Below the guard above, so a send dropped for want of a socket is not + // counted as one. No payload: the count is the signal, and nothing about + // what was typed belongs in analytics. + this.track('message_sent'); + } + + /** + * Answers a pending approval. Called from a UI window through + * `KevinSupportService.actions.resolveApproval(...)`, since the prompt cannot + * render in the worker. + * + * `surface` is passed in because the chat window and the footer bubble reach + * this method identically — it is the only way to tell whether the bubble is + * earning its keep. + */ + resolveApproval( + approvalId: string, + decision: V2ApprovalDecision, + surface: TApprovalSurface = 'chat', + ) { + this.log('out', 'v2:approval.resolve', { approvalId, decision }); + // Read before REMOVE_APPROVAL drops the entry: the caller only knows the id, + // and which tool was being gated is the whole point of the event. + const approval = this.state.pendingApprovals.find(a => a.approvalId === approvalId); + this.track('approval_resolved', { + tool: approval?.tool ?? 'unknown', + risk: approval?.risk ?? 'unknown', + decision, + surface, + }); + this.socket?.emit('v2:approval.resolve', { approvalId, decision }); + // Optimistic: the server confirms with v2:approval.resolved, but the + // prompt should not linger while that round-trips. + this.REMOVE_APPROVAL(approvalId); + } + + clearConversation() { + this.CLEAR_MESSAGES(); + } + + disconnect() { + this.socket?.disconnect(); + this.socket = null; + this.SET_CONNECTED(false); + this.SET_CONNECTING(false); + this.SET_PENDING(false); + this.CLEAR_APPROVALS(); + } + + @mutation() + private ADD_MESSAGE(message: IKevinMessage) { + this.state.messages.push(message); + } + + @mutation() + private APPEND_TEXT(interactionId: string, text: string) { + const message = this.state.messages.find(m => !m.isUser && m.interactionId === interactionId); + if (message) message.text += text; + } + + @mutation() + private CLEAR_MESSAGES() { + this.state.messages = []; + } + + @mutation() + private SET_PENDING(pending: boolean) { + this.state.pending = pending; + } + + @mutation() + private SET_CONNECTED(connected: boolean) { + this.state.connected = connected; + } + + @mutation() + private SET_CONNECTING(connecting: boolean) { + this.state.connecting = connecting; + } + + @mutation() + private SET_ERROR(error: string | null) { + this.state.error = error; + } + + @mutation() + private SET_RATE_LIMIT(rateLimit: { current: number; maximum: number; exceeded: boolean }) { + if (rateLimit.exceeded) this.state.rateLimitRefusals += 1; + this.state.rateLimit = rateLimit; + } + + @mutation() + private ADD_APPROVAL(approval: V2ApprovalRequestPayload) { + // The server replays outstanding approvals on reconnect; do not stack one + // we are already showing. + if (this.state.pendingApprovals.some(a => a.approvalId === approval.approvalId)) return; + this.state.pendingApprovals.push(approval); + } + + @mutation() + private REMOVE_APPROVAL(approvalId: string) { + this.state.pendingApprovals = this.state.pendingApprovals.filter( + a => a.approvalId !== approvalId, + ); + } + + @mutation() + private SET_APPROVALS(approvals: V2ApprovalRequestPayload[]) { + this.state.pendingApprovals = approvals; + } + + @mutation() + private CLEAR_APPROVALS() { + this.state.pendingApprovals = []; + } + + @mutation() + private RESET() { + this.state.messages = []; + this.state.pending = false; + this.state.connected = false; + this.state.connecting = false; + this.state.error = null; + this.state.rateLimit = null; + this.state.rateLimitRefusals = 0; + this.state.pendingApprovals = []; + } +} diff --git a/app/services/stream-avatar/stream-avatar-api-service.ts b/app/services/stream-avatar/stream-avatar-api-service.ts index 2d8e2994476d..7eb4072a96b2 100644 --- a/app/services/stream-avatar/stream-avatar-api-service.ts +++ b/app/services/stream-avatar/stream-avatar-api-service.ts @@ -166,15 +166,4 @@ export class StreamAvatarApiService extends Service { body: JSON.stringify({ instruction, response }), }); } - - async sendTrigger( - name: string, - parameters: Record, - response: 'text' | 'tts' = 'tts', - ): Promise { - await this.authedFetch('/agent/trigger', { - method: 'POST', - body: JSON.stringify({ trigger: { name, parameters }, response }), - }); - } } diff --git a/app/services/stream-avatar/v2/agent-tools.ts b/app/services/stream-avatar/v2/agent-tools.ts new file mode 100644 index 000000000000..8eccc4dcc700 --- /dev/null +++ b/app/services/stream-avatar/v2/agent-tools.ts @@ -0,0 +1,424 @@ +import { Inject } from 'services/core/injector'; +import { Service } from 'services/core/service'; +import { ScenesService } from 'services/scenes'; +import { SourcesService } from 'services/sources'; +import { AudioService } from 'services/audio'; +import { StreamingService } from 'services/streaming'; +import { + SourceFiltersService, + EFilterDisplayType, + TSourceFilterType, +} from 'services/source-filters'; +import { TObsValue } from 'components/obs/inputs/ObsInput'; +import { PerformanceService } from 'services/performance'; +import { DiagnosticsService } from 'services/diagnostics'; +import { VideoSettingsService } from 'services/settings-v2/video'; +import { V2ToolOutcome } from './protocol'; + +/** + * Every filter this service adds is named with this prefix. + * + * It is how a previous preset is found and cleared before the next goes on. + * Matching the prefix rather than the chosen preset's own filters matters: + * switching from `too_quiet` to `background_noise` must not leave the limiter + * behind, stacked underneath the new chain. + */ +const FILTER_PREFIX = 'Sidekick '; + +interface MicFilter { + name: string; + type: TSourceFilterType; + settings: Dictionary; +} + +/** + * Mic filter chains, chosen by what the streamer says is wrong. + * + * One chain for everybody did not work: the same gate that suits a decent mic in + * a quiet room sits below a noisy room's floor and does nothing, and compressing + * someone who is simply too quiet without makeup gain leaves them quieter still. + * + * These are still fixed numbers — a starting point the streamer can tune in the + * Filters dialog, not a measurement of their actual mic. Fitting a specific mic + * means sampling its real noise floor, which is a bigger piece of work; see the + * note in the plan if these turn out to miss often. + * + * Order within a chain is application order, which is signal order in OBS. + */ +const MIC_PRESETS: Record = { + // Nothing specific said. The balanced chain. + general: [ + { + name: `${FILTER_PREFIX}Noise Suppression`, + type: 'noise_suppress_filter_v2', + settings: { method: 'rnnoise' }, + }, + { + name: `${FILTER_PREFIX}Noise Gate`, + type: 'noise_gate_filter', + settings: { open_threshold: -26, close_threshold: -32 }, + }, + { + name: `${FILTER_PREFIX}Compressor`, + type: 'compressor_filter', + settings: { ratio: 4, threshold: -18 }, + }, + ], + + // Fan, keyboard, air conditioning, room tone. RNNoise does the real work and + // needs no threshold; the gate is kept moderate rather than aggressive so it + // cannot eat the front of a quietly spoken word. No compressor on purpose — + // compressing a noisy signal lifts the room tone between words. + background_noise: [ + { + name: `${FILTER_PREFIX}Noise Suppression`, + type: 'noise_suppress_filter_v2', + settings: { method: 'rnnoise' }, + }, + { + name: `${FILTER_PREFIX}Noise Gate`, + type: 'noise_gate_filter', + settings: { open_threshold: -30, close_threshold: -36 }, + }, + ], + + // Makeup gain is the actual fix. OBS has no standalone gain filter in + // TSourceFilterType, so it rides on the compressor's output_gain, with a + // limiter so that gain cannot clip. No gate: they have told us level is the + // problem, and a gate is the one filter that can make quiet speech vanish. + too_quiet: [ + { + name: `${FILTER_PREFIX}Compressor`, + type: 'compressor_filter', + settings: { ratio: 3, threshold: -24, output_gain: 12 }, + }, + { name: `${FILTER_PREFIX}Limiter`, type: 'limiter_filter', settings: { threshold: -3 } }, + ], + + // Peaks and troughs, or clipping on the loud moments. + uneven_volume: [ + { + name: `${FILTER_PREFIX}Compressor`, + type: 'compressor_filter', + settings: { ratio: 4, threshold: -18, output_gain: 4 }, + }, + { name: `${FILTER_PREFIX}Limiter`, type: 'limiter_filter', settings: { threshold: -2 } }, + ], +}; + +/** The ticket form Settings > Get Support links to (`Support.tsx`). */ +const SUPPORT_TICKET_URL = + 'https://support.streamlabs.com/hc/en-us/requests/new?ticket_form_id=473667'; + +/** + * Coarse buckets for the `tool_executed` analytics event, so a chart can say + * "audio is where people need help" without listing thirteen tool names. + * + * Keep in step with `handlers` below -- a tool missing from here is not an error, + * it just lands in 'other', which is how the avatar plugin's COMMAND_GROUPS reads + * its own map too. + */ +export const TOOL_GROUPS: Record = { + scene_list: 'scene', + scene_switch: 'scene', + source_list: 'source', + source_set_visible: 'source', + source_set_muted: 'audio', + mic_enhance: 'audio', + stream_status: 'stream_control', + stream_stop: 'stream_control', + stream_health: 'stream_control', + replay_save: 'clip', + watch_replay: 'clip', + diagnostics_report: 'support', + support_open_ticket: 'support', +}; + +/** + * The tools the agent may execute on this machine. + * + * Hand-written rather than generated from the `platform-apps` `@apiMethod()` + * decorators: that surface is a permission API for third-party apps, keyed by + * resourceId and shaped for RPC. This one is ~10 curated actions with + * LLM-facing descriptions and name-based addressing. Same underlying services, + * different contract. + * + * Two conventions carried over from the automations engine: + * - scenes and sources are addressed **by name**, never by id — ids are not + * stable across scene collections, and names are what the model sees; + * - every handler returns something the agent can say out loud, so a refusal + * or a miss reads as an explanation rather than a silent no-op. + * + * Runs in the worker window, like everything else in the services layer. + */ +export class AgentToolsService extends Service { + @Inject() private scenesService: ScenesService; + @Inject() private sourcesService: SourcesService; + @Inject() private audioService: AudioService; + @Inject() private streamingService: StreamingService; + @Inject() private sourceFiltersService: SourceFiltersService; + @Inject() private performanceService: PerformanceService; + @Inject() private diagnosticsService: DiagnosticsService; + @Inject() private videoSettingsService: VideoSettingsService; + + /** + * The microphone, resolved rather than asked for. + * + * "Make my mic sound better" must not depend on the model guessing the exact + * name out of source_list. `Mic/Aux` is the app's own default name for it + * (scene-collections seeds it), with a "Microphone…" prefix match as the + * fallback for anyone who renamed theirs — the same lookup the plugin's + * audio_set_muted uses. + */ + private findMicSource() { + const audioSources = this.audioService.views.sourcesForCurrentScene; + return ( + audioSources.find(s => s.name === 'Mic/Aux') ?? + audioSources.find(s => s.name.toLowerCase().startsWith('mic')) + ); + } + + private get handlers(): Record) => Promise> { + return { + scene_list: async () => ({ + activeScene: this.scenesService.views.activeScene?.name ?? null, + scenes: this.scenesService.views.scenes.map(s => s.name), + }), + + scene_switch: async args => { + const name = String(args.scene ?? ''); + const scene = this.scenesService.views.scenes.find(s => s.name === name); + if (!scene) { + throw new Error( + `No scene named "${name}". Available: ${this.scenesService.views.scenes + .map(s => s.name) + .join(', ')}`, + ); + } + this.scenesService.makeSceneActive(scene.id); + return { switchedTo: scene.name }; + }, + + source_list: async () => { + const scene = this.scenesService.views.activeScene; + if (!scene) return { scene: null, sources: [] }; + return { + scene: scene.name, + sources: scene.getItems().map(item => ({ + name: item.name, + visible: item.visible, + muted: this.audioService.views.getSource(item.sourceId)?.muted ?? false, + })), + }; + }, + + source_set_visible: async args => { + const name = String(args.source ?? ''); + const visible = args.visible !== false; + const scene = this.scenesService.views.activeScene; + const item = scene?.getItems().find(i => i.name === name); + if (!item) throw new Error(`No source named "${name}" in the current scene.`); + item.setVisibility(visible); + return { source: name, visible }; + }, + + source_set_muted: async args => { + const name = String(args.source ?? ''); + const muted = args.muted !== false; + const source = this.sourcesService.views.sources.find(s => s.name === name); + if (!source) throw new Error(`No source named "${name}".`); + const audioSource = this.audioService.views.getSource(source.sourceId); + if (!audioSource) throw new Error(`"${name}" has no audio to mute.`); + audioSource.setMuted(muted); + return { source: name, muted }; + }, + + stream_status: async () => ({ + streaming: this.streamingService.views.isStreaming, + recording: this.streamingService.views.isRecording, + replayBufferRunning: this.streamingService.views.isReplayBufferActive, + }), + + stream_stop: async () => { + if (!this.streamingService.views.isStreaming) { + return { stopped: false, reason: 'not currently streaming' }; + } + // stopStreaming() is deprecated in favour of toggleStreaming(); the + // isStreaming guard above is what makes the toggle unambiguous. + this.streamingService.actions.toggleStreaming(); + return { stopped: true }; + }, + + replay_save: async () => { + if (!this.streamingService.views.isReplayBufferActive) { + throw new Error('The replay buffer is not running.'); + } + this.streamingService.actions.saveReplay(); + return { saved: true }; + }, + + // Shows what replay_save already wrote; it does not capture one. Saving + // is asynchronous — the file only lands on replayBufferFileWrite — so + // folding a save in here would race the 15s tool timeout and switch to a + // scene still playing the previous replay. + watch_replay: async () => { + // An Instant Replay source is a plain ffmpeg_source wearing the + // 'replay' properties manager. There is no OBS type id for it, so the + // manager tag is the only thing that identifies one. + const source = this.sourcesService.views.sources.find( + s => s.propertiesManagerType === 'replay', + ); + if (!source) { + throw new Error( + 'No Instant Replay source found. The streamer needs to add one (Add Source → Instant Replay) before this can do anything.', + ); + } + + // first scene wins when the source sits in several. Pick the + // one nearest the active scene if that ever turns out to matter. + const [item] = this.scenesService.views.getSceneItemsBySourceId(source.sourceId); + if (!item) { + throw new Error( + `The Instant Replay source "${source.name}" is not in any scene, so there is nowhere to switch to.`, + ); + } + + const scene = this.scenesService.views.getScene(item.sceneId); + this.scenesService.makeSceneActive(item.sceneId); + return { scene: scene?.name ?? item.sceneId, source: source.name }; + }, + + mic_enhance: async args => { + const problem = String(args.problem ?? 'general'); + + // No OBS filter fixes room reverb or speaker bleed, so say that rather + // than applying something that will not help. Checked before the mic + // lookup: the advice holds whether or not a mic source exists. + if (problem === 'echo') { + return { + applied: false, + reason: + 'Echo is a room and monitoring problem, not something an audio filter can remove. The fixes are ' + + 'listening on headphones instead of speakers, moving the mic closer and pointing it away from the room, ' + + 'and putting something soft on the hard surfaces around it.', + }; + } + + const mic = this.findMicSource(); + if (!mic) { + throw new Error( + 'No microphone source found. The streamer needs to add a Mic/Aux source before this can do anything.', + ); + } + + // Always clear ours first, by prefix rather than by the chosen preset: + // switching presets must not leave the previous one's filters stacked + // underneath, and asking twice must not end up with two gates fighting. + const existing = this.sourceFiltersService.views.filtersBySourceId(mic.sourceId, true); + const removed = existing + .filter(f => f.name.startsWith(FILTER_PREFIX)) + .map(f => { + this.sourceFiltersService.remove(mic.sourceId, f.name); + return f.name; + }); + + if (problem === 'none') { + return { source: mic.name, problem, applied: false, filters: [], removed }; + } + + const preset = MIC_PRESETS[problem] ?? MIC_PRESETS.general; + for (const filter of preset) { + this.sourceFiltersService.add( + mic.sourceId, + filter.type, + filter.name, + { ...filter.settings }, + EFilterDisplayType.Normal, + ); + } + + return { + source: mic.name, + problem, + applied: true, + filters: preset.map(f => f.name), + removed, + }; + }, + + stream_health: async () => { + const { state } = this.performanceService; + const base = this.videoSettingsService.baseResolution; + const output = this.videoSettingsService.outputResolutions.horizontal; + + return { + // The service's own verdict. Lead with it rather than re-deriving one + // from the percentages — the thresholds live in one place for a reason. + streamQuality: this.performanceService.views.streamQuality, + cpuPercent: state.CPU, + frameRate: state.frameRate, + droppedFrames: state.numberDroppedFrames, + droppedFramesPercent: state.percentageDroppedFrames, + skippedFrames: state.numberSkippedFrames, + skippedFramesPercent: state.percentageSkippedFrames, + laggedFrames: state.numberLaggedFrames, + laggedFramesPercent: state.percentageLaggedFrames, + streamingBandwidthKbps: state.streamingBandwidth, + baseResolution: `${base.baseWidth}x${base.baseHeight}`, + outputResolution: `${output.outputWidth}x${output.outputHeight}`, + streaming: this.streamingService.views.isStreaming, + recording: this.streamingService.views.isRecording, + }; + }, + + diagnostics_report: async () => { + const report = await this.diagnosticsService.uploadReport(); + if (!report?.report_code) { + throw new Error( + 'The diagnostic report did not upload. Ask them to try again in a moment.', + ); + } + return { reportCode: report.report_code }; + }, + + support_open_ticket: async () => { + // @electron/remote, the way application-menu.ts opens external links. + require('@electron/remote').shell.openExternal(SUPPORT_TICKET_URL); + return { opened: true, url: SUPPORT_TICKET_URL }; + }, + }; + } + + canExecute(tool: string): boolean { + // Own-property, not `in`: the tool name comes off the socket, and `in` + // walks the prototype — 'toString' would report as executable and then + // dispatch to Object.prototype.toString as though it were a tool. + return Object.prototype.hasOwnProperty.call(this.handlers, tool); + } + + /** + * Runs a tool, converting any throw into a tool error. The agent loop is + * parked on this call's id, so it must always get an answer. + */ + async execute(tool: string, args: Record): Promise { + // Read the getter once — it rebuilds the map on every access — and guard it + // the same way canExecute does, since this is reachable on its own. + const handlers = this.handlers; + const handler = Object.prototype.hasOwnProperty.call(handlers, tool) + ? handlers[tool] + : undefined; + if (!handler) { + return { ok: false, code: 'unknown_tool', message: `Desktop cannot run ${tool}.` }; + } + + try { + return { ok: true, result: await handler(args ?? {}) }; + } catch (e: unknown) { + return { + ok: false, + code: 'failed', + message: e instanceof Error ? e.message : `${tool} failed.`, + }; + } + } +} diff --git a/app/services/stream-avatar/v2/protocol.ts b/app/services/stream-avatar/v2/protocol.ts new file mode 100644 index 000000000000..87cf598b67d9 --- /dev/null +++ b/app/services/stream-avatar/v2/protocol.ts @@ -0,0 +1,361 @@ +/** Socket.IO namespace. Legacy clients stay on "/". */ +export const V2_NAMESPACE = '/v2'; + +/** + * Wire protocol version. Bump on any breaking change to the event set or to a + * payload shape. Sent by the client in `v2:hello`; the server refuses a major + * mismatch with `v2:error{code:"protocol_version"}`. + */ +export const V2_PROTOCOL_VERSION = 1; + +/** + * Tool surface version, advertised per device in `v2:hello`. Independent of + * V2_PROTOCOL_VERSION because the tool set grows far more often than the + * transport changes, and because Desktop ships on a slower release cadence than + * the API deploys. A tool declaring `minToolVersion: N` is hidden from the + * model entirely unless an attached device advertises `toolProtocolVersion >= N`. + * Bump whenever a device gains a new executable tool, and pin the new tool with + * `minToolVersion`. Tools without one default to 1, so a bump never hides + * anything from clients that are already out there. + */ +export const V2_TOOL_PROTOCOL_VERSION = 1; + +// ─── identity ──────────────────────────────────────────────────────────────── + +/** + * Which surface a socket is. One user may have several attached at once, and + * tool calls route by role. + * app — the plugin settings panel (Streamlabs Desktop embedded browser) + * source — the avatar browser source; render/playback only, executes no tools + * desktop — the native Streamlabs Desktop app; owns OBS + */ +export type V2DeviceRole = 'app' | 'source' | 'desktop'; + +export const V2_DEVICE_ROLES = ['app', 'source', 'desktop'] as const; + +/** Where a tool actually runs. */ +export type V2ToolExecutor = 'server' | 'app' | 'desktop'; + +/** Requested output modality for a run. */ +export type V2ResponseType = 'text' | 'tts' | 'both'; + +// ─── policy ────────────────────────────────────────────────────────────────── + +/** + * Server-owned risk classification. Never read from model output. + * read — observes only + * reversible — mutates something the user can trivially undo + * irreversible — cannot be undone (ending a broadcast) + * external — visible outside the machine (public chat, published clip) + */ +export type V2ToolRisk = 'read' | 'reversible' | 'irreversible' | 'external'; + +/** Per-user override, persisted in the existing settings jsonb column. */ +export type V2ToolPolicyMode = 'auto' | 'ask' | 'never'; + +/** Default gating by risk, before any per-user override. */ +export const V2_DEFAULT_POLICY: Record = { + read: 'auto', + reversible: 'auto', + irreversible: 'ask', + external: 'ask', +}; + +export type V2ApprovalDecision = 'approve' | 'deny' | 'always'; + +/** How an approval finished. "cancelled" means the run was aborted under it. */ +export type V2ApprovalOutcome = 'approved' | 'denied' | 'expired' | 'cancelled'; + +/** Why a run stopped. "lost" is emitted on resync when the process no longer has it. */ +export type V2RunEndReason = 'complete' | 'cancelled' | 'error' | 'lost'; + +// ─── timings ───────────────────────────────────────────────────────────────── + +/** Tool dispatch deadline. Exceeding it yields a tool error, never a hang. */ +export const V2_TOOL_TIMEOUT_MS = 15_000; + +/** Approval deadline. Exceeding it resolves as a denial. */ +export const V2_APPROVAL_TIMEOUT_MS = 60_000; + +// ─── shared payload fragments ──────────────────────────────────────────────── + +export interface V2ToolCall { + /** Correlates `v2:tool.invoke` with `v2:tool.result`. Also the idempotency key. */ + callId: string; + runId: string; + /** Registry name, e.g. "scene.switch". */ + tool: string; + args: Record; +} + +export interface V2ToolOk { + ok: true; + /** JSON-safe. Serialized straight into the model's tool result message. */ + result: unknown; +} + +export interface V2ToolErr { + ok: false; + /** Machine-readable: "timeout" | "unknown_tool" | "denied" | "unreachable" | "failed". */ + code: string; + /** Shown to the model so it can explain itself to the user. */ + message: string; + /** True only when a human declined. The model is told refusal, not failure. */ + denied?: boolean; +} + +export type V2ToolOutcome = V2ToolOk | V2ToolErr; + +export interface V2PacketId { + runId: string; + /** Groups the text packet and its synthesized audio. */ + utteranceId: string; +} + +// ─── client → server ───────────────────────────────────────────────────────── + +export interface V2HelloPayload { + protocolVersion: number; + toolProtocolVersion: number; + role: V2DeviceRole; + /** Stable per install, so reconnects are recognised as the same device. */ + deviceId: string; + /** Settings schema version, for the migration path already in the codebase. */ + settingsVersion?: number; +} + +export interface V2TextInput { + text: string; + responseType: V2ResponseType; +} + +export interface V2TriggerInput { + name: string; + parameters?: Record; + responseType: V2ResponseType; +} + +export interface V2InstructionInput { + instruction: string; + responseType: V2ResponseType; +} + +export interface V2AudioStart { + responseType: V2ResponseType; +} + +export interface V2AudioChunk { + /** + * Float32 PCM samples at 16 kHz, as a plain array. + * NOTE: verbose on the wire (~7 bytes/sample as JSON). Kept for parity with + * the legacy path; base64 Int16 is the obvious upgrade if bandwidth bites. + */ + samples: number[]; +} + +export interface V2ToolResultPayload { + callId: string; + outcome: V2ToolOutcome; +} + +export interface V2ApprovalResolvePayload { + approvalId: string; + decision: V2ApprovalDecision; +} + +export interface V2RunCancelPayload { + /** Omit to cancel every run for this user. */ + runId?: string; +} + +/** Client-pushed session state. Fire-and-forget; last write wins. */ +export interface V2StatePayload { + scenes?: string[]; + sources?: Array<{ name: string; visible?: boolean; muted?: boolean; scene?: string }>; + sceneTree?: unknown; + currentScene?: string; + voice?: string; + personality?: { type: string; traits: string[] }; + game?: string; + /** Base64 frame. Sensitive — only sent on explicit user action. */ + vision?: string; + displayName?: string; + secretsEnabled?: boolean; +} + +export interface V2ChatMessagePayload { + author: string; + text: string; +} + +/** + * Persist settings. Fire-and-forget by design: v1 used an ack-based RPC for + * this, but it is a debounced autosave — nothing in the UI waits on the result, + * and a failed write is recoverable on the next save. The read direction is a + * push (`v2:ready`), so no request/response machinery exists in v2 at all. + */ +export interface V2SettingsUpdatePayload { + settings: unknown; + settingsVersion: number; +} + +export interface V2ClientToServerEvents { + 'v2:hello': (p: V2HelloPayload) => void; + 'v2:input.text': (p: V2TextInput) => void; + 'v2:input.trigger': (p: V2TriggerInput) => void; + 'v2:input.instruction': (p: V2InstructionInput) => void; + /** Also the barge-in signal: aborts every in-flight run for this user. */ + 'v2:input.audio.start': (p: V2AudioStart) => void; + 'v2:input.audio.chunk': (p: V2AudioChunk) => void; + 'v2:input.audio.end': () => void; + 'v2:tool.result': (p: V2ToolResultPayload) => void; + 'v2:approval.resolve': (p: V2ApprovalResolvePayload) => void; + 'v2:run.cancel': (p: V2RunCancelPayload) => void; + 'v2:state': (p: V2StatePayload) => void; + 'v2:chat.message': (p: V2ChatMessagePayload) => void; + 'v2:settings.update': (p: V2SettingsUpdatePayload) => void; + /** Viseme stream for the 2D avatar. High frequency, never logged. */ + 'v2:animate': (p: { viseme: string; duration?: number }) => void; +} + +// ─── server → client ───────────────────────────────────────────────────────── + +export interface V2ReadyPayload { + protocolVersion: number; + userId: number; + deviceId: string; + role: V2DeviceRole; + displayName: string; + isPro: boolean; + tier: string | null; + /** Tool names visible to the model right now, given attached devices and policy. */ + tools: string[]; + /** Runs still alive on the server, so a reconnecting client can resync. */ + activeRunIds: string[]; + /** Approvals still awaiting a human, replayed so a reconnecting client re-prompts. */ + pendingApprovals: V2ApprovalRequestPayload[]; + /** + * Everything v1 fetched through four separate ack-based RPCs + * (getSettings / getVoices / getAgents2D), pushed once instead. Read-once + * data does not need a request/response channel, and removing it is what + * lets v2 have no acks anywhere. + */ + settings: unknown; + settingsVersion: number; + voices: unknown[]; + agents2D: unknown[]; +} + +export interface V2RunStartedPayload { + runId: string; + responseType: V2ResponseType; +} + +export interface V2TextPayload { + packetId: V2PacketId; + text: string; + /** Last text packet of the run. */ + final: boolean; + /** Present when the text is a link footer or similar non-spoken addendum. */ + kind?: 'speech' | 'links'; +} + +export interface V2AudioPayload { + packetId: V2PacketId; + /** Base64 WAV. */ + audio: string; + timestamps?: unknown; +} + +export interface V2IntentPayload { + runId: string; + name: string; + parameters?: Record; +} + +export interface V2RunEndedPayload { + runId: string; + reason: V2RunEndReason; + /** Set when reason is "error". */ + message?: string; +} + +export interface V2ToolInvokePayload extends V2ToolCall { + /** Client should self-abandon past this and not reply. */ + timeoutMs: number; +} + +export interface V2ToolCancelPayload { + callId: string; +} + +export interface V2ApprovalRequestPayload { + approvalId: string; + runId: string; + callId: string; + tool: string; + args: Record; + risk: V2ToolRisk; + /** Short human sentence for the prompt, e.g. "Stop the stream". */ + summary: string; + /** Epoch ms. Client should dismiss its own prompt at this point. */ + expiresAt: number; +} + +export interface V2ApprovalResolvedPayload { + approvalId: string; + outcome: V2ApprovalOutcome; + /** Role of the device that answered, so other devices can say who did. */ + by?: V2DeviceRole; +} + +export interface V2RateLimitPayload { + current: number; + maximum: number; + exceeded: boolean; +} + +export interface V2ErrorPayload { + runId?: string; + /** "protocol_version" | "auth" | "rate_limit" | "internal". */ + code: string; + message: string; +} + +export interface V2PresencePayload { + /** Attached roles for this user right now. Drives client affordances. */ + roles: V2DeviceRole[]; + sourceCount: number; +} + +export interface V2ServerToClientEvents { + 'v2:ready': (p: V2ReadyPayload) => void; + 'v2:run.started': (p: V2RunStartedPayload) => void; + 'v2:text': (p: V2TextPayload) => void; + 'v2:audio': (p: V2AudioPayload) => void; + 'v2:intent': (p: V2IntentPayload) => void; + 'v2:run.ended': (p: V2RunEndedPayload) => void; + 'v2:tool.invoke': (p: V2ToolInvokePayload) => void; + 'v2:tool.cancel': (p: V2ToolCancelPayload) => void; + 'v2:approval.request': (p: V2ApprovalRequestPayload) => void; + 'v2:approval.resolved': (p: V2ApprovalResolvedPayload) => void; + /** Relayed viseme, panel -> avatar browser sources. */ + 'v2:animate': (p: { viseme: string; duration?: number }) => void; + /** + * Automation preview: play a pre-recorded CDN voice line for a condition, + * bypassing the agent entirely. Triggered from Desktop's automation test + * button via REST. + */ + 'v2:bark': (p: { conditionType: string }) => void; + 'v2:rateLimit': (p: V2RateLimitPayload) => void; + 'v2:presence': (p: V2PresencePayload) => void; + 'v2:error': (p: V2ErrorPayload) => void; +} + +// ─── rooms ─────────────────────────────────────────────────────────────────── + +/** Everything the user has attached. Approvals broadcast here. */ +export const v2UserRoom = (userId: number | string) => `v2:user-${userId}`; + +/** One role's sockets for a user. Audio fans out to the source room. */ +export const v2RoleRoom = (userId: number | string, role: V2DeviceRole) => `v2:${role}-${userId}`; diff --git a/app/services/usage-statistics.ts b/app/services/usage-statistics.ts index c973253c8ff4..0964de27a50d 100644 --- a/app/services/usage-statistics.ts +++ b/app/services/usage-statistics.ts @@ -62,6 +62,7 @@ export type TAnalyticsEvent = | 'WidgetRemoved' | 'GamePulse' | 'Automations' + | 'SupportChat' | 'LiveOutputEditing'; // Refls are used as uuids for ultra components and should be updated for new ulta components. @@ -82,6 +83,7 @@ export type TUltraRefl = | 'slobs-stream-settings' | 'slobs-live-output-editing' | 'slobs-automations' + | 'slobs-support-chat' | string; interface IAnalyticsEvent { diff --git a/app/services/windows.ts b/app/services/windows.ts index 1b6be8f86d1b..030beeacb76e 100644 --- a/app/services/windows.ts +++ b/app/services/windows.ts @@ -39,6 +39,7 @@ import { RecentEventsWindow, EditTransform, EditAutomations, + KevinSupport, Blank, Main, MultistreamChatInfo, @@ -99,6 +100,7 @@ export function getComponents() { PlatformAppPopOut, EditTransform, EditAutomations, + KevinSupport, OverlayPlaceholder, BrowserSourceInteraction, EventFilterMenu, @@ -139,6 +141,11 @@ export interface IWindowOptions extends Electron.BrowserWindowConstructorOptions }; scaleFactor: number; isShown: boolean; + /** + * Live focus state, not a construction option — same as scaleFactor above. + * Only maintained for one-off windows; main and child never set it. + */ + isFocused?: boolean; title?: string; center?: boolean; position?: { @@ -271,6 +278,11 @@ export class WindowsService extends StatefulService { } } + private setOneOffFocused(windowId: string, isFocused: boolean) { + if (!this.state[windowId]) return; + this.UPDATE_ONE_OFF_WINDOW(windowId, { isFocused }); + } + getWindowIdFromElectronId(electronWindowId: number) { return Object.keys(this.windows).find(win => this.windows[win].id === electronWindowId); } @@ -490,6 +502,9 @@ export class WindowsService extends StatefulService { this.DELETE_ONE_OFF_WINDOW(windowId); }); + newWindow.on('focus', () => this.setOneOffFocused(windowId, true)); + newWindow.on('blur', () => this.setOneOffFocused(windowId, false)); + this.updateScaleFactor(windowId); newWindow.on('move', () => this.updateScaleFactor(windowId)); diff --git a/app/themes.g.less b/app/themes.g.less index fdd3a42be571..3bb36ce4d800 100644 --- a/app/themes.g.less +++ b/app/themes.g.less @@ -39,6 +39,8 @@ --new-badge-text: @purple-light; --info-badge: @navy-dark; --info-badge-text: @blue-dark; + --chat-bubble-user: @navy; + --chat-bubble-user-border: lighten(@dark-5, 4%); --beta-text: @purple-light; --badge: @purple-dark; --highlighter-icon: @light-4;