diff --git a/dashboard/src/lib/transfer-board.test.ts b/dashboard/src/lib/transfer-board.test.ts index a3ff66448..be2e0f17a 100644 --- a/dashboard/src/lib/transfer-board.test.ts +++ b/dashboard/src/lib/transfer-board.test.ts @@ -1,14 +1,34 @@ import { readFileSync } from 'node:fs' import { describe, expect, it } from 'vitest' -import { RECOVERY_GUIDE, recoveryModeLabel, type RecoveryMode } from './transfer' +import { + RECOVERY_GUIDE, + recoveryModeLabel, + transferRecoveryCommands, + tradeRecoveryCommands, + type RecoveryCommand, + type RecoveryMode, + type TransferCategory +} from './transfer' +import type { UsdcBridgeDirection } from './api/UsdcBridgeDirection' // The Grafana board (observability/) shows the same recovery commands as the -// SPA from its own copy: recovery-guide.json for the guide. These tests fail -// when the copy drifts from this file's builders. They also cover the board's +// SPA from its own copies: recovery-guide.json for the guide and +// recovery-commands.js for the per-row dialog. These tests fail when a copy +// drifts from this file's builders. They also cover the board's // status-history.js, which orders a row's status entries in its dialog. const panels = new URL('../../../observability/liquidity-panels/', import.meta.url) +type BoardCommand = { command: string; description: string; mode: RecoveryMode } +type BoardTransfer = { kind: TransferCategory; id: string; status: string; direction?: string } +type BoardBuilders = { + tradeCommands: (client: string, symbol: string) => BoardCommand[] + transferCommands: (client: string, transfer: BoardTransfer) => BoardCommand[] +} + +// A plain script in the board, an ES module through its export line here. +const board = (await import(new URL('recovery-commands.js', panels).href)) as BoardBuilders + type StatusEntry = { time: number; kind: string; status: string; direction?: string } type StatusHistory = { latest: ( @@ -19,6 +39,12 @@ type StatusHistory = { // A plain script in the board, an ES module through its export line here. const statusHistory = (await import(new URL('status-history.js', panels).href)) as StatusHistory +const CLIENT = 'st0x-liquidity-client --env production' +const PROD = { simulateSourceId: null, backendPort: null, clientEnv: 'production' } as const + +const withoutLabel = (commands: RecoveryCommand[]): BoardCommand[] => + commands.map(({ command, description, mode }) => ({ command, description, mode })) + describe('board recovery-guide.json', () => { it('matches RECOVERY_GUIDE and the mode labels', () => { const guide = JSON.parse(readFileSync(new URL('recovery-guide.json', panels), 'utf8')) as { @@ -32,6 +58,60 @@ describe('board recovery-guide.json', () => { }) }) +describe('board recovery-commands.js', () => { + it('matches tradeRecoveryCommands', () => { + expect(board.tradeCommands(CLIENT, 'MSTR')).toEqual( + withoutLabel(tradeRecoveryCommands({ deployment: PROD, symbol: 'MSTR' })) + ) + }) + + it('matches transferRecoveryCommands for every equity status', () => { + const kinds: TransferCategory[] = ['equity_mint', 'equity_redemption'] + for (const kind of kinds) { + for (const status of ['wrapping', 'sending', 'failed', 'completed', 'reconciled']) { + expect(board.transferCommands(CLIENT, { kind, id: 'X1', status })).toEqual( + withoutLabel(transferRecoveryCommands({ deployment: PROD, kind, id: 'X1', status })) + ) + } + } + }) + + it('matches transferRecoveryCommands for every usdc bridge status and direction', () => { + // The exporter does not log the bridge's postBurn flag, so the board + // matches the SPA without it: a failed bridge offers no reconcile. + const directions: (UsdcBridgeDirection | null)[] = ['alpaca_to_base', 'base_to_alpaca', null] + for (const direction of directions) { + for (const status of [ + 'converting', + 'withdrawing', + 'bridging', + 'depositing', + 'failed', + 'completed' + ]) { + expect( + board.transferCommands(CLIENT, { + kind: 'usdc_bridge', + id: 'B1', + status, + ...(direction === null ? {} : { direction }) + }) + ).toEqual( + withoutLabel( + transferRecoveryCommands({ + deployment: PROD, + kind: 'usdc_bridge', + id: 'B1', + status, + direction + }) + ) + ) + } + } + }) +}) + describe('board status-history.js', () => { const bridge = (status: string, time: number, direction = 'alpaca_to_base'): StatusEntry => ({ time, diff --git a/observability/README.md b/observability/README.md index a8f182a65..a56c946c2 100644 --- a/observability/README.md +++ b/observability/README.md @@ -41,11 +41,12 @@ into the board JSON. The plugin is installed on the box by `T0Trade/t0.devops` (`GF_INSTALL_PLUGINS`); the PR check's throwaway Grafana does not have it, so it loads these panels as "Panel plugin not found" and still passes. -The recovery commands in the header's guide (`recovery-guide.json`) are a copy -of the SPA's (`dashboard/src/lib/transfer.ts`), in the operations client form -for the environment the board shows. `dashboard/src/lib/transfer-board.test.ts` -fails when the copy drifts, so change the SPA and the copy together. It runs in -the dashboard build of CI. +The recovery commands in the header's guide (`recovery-guide.json`) and in the +row dialog (`recovery-commands.js`) are copies of the SPA's +(`dashboard/src/lib/transfer.ts`), in the operations client form for the +environment the board shows. `dashboard/src/lib/transfer-board.test.ts` fails +when a copy drifts, so change the SPA and the copies together. It runs in the +dashboard build of CI. The Dashboard tab's inventory is three Grafana tables, for their column filters: USD at Alpaca, USD on each chain, and Equities. Per-chain rows (USD ยท Onchain) diff --git a/observability/dashboards/liquidity/t0-liquidity.json b/observability/dashboards/liquidity/t0-liquidity.json index 9e7746387..5bde516e6 100644 --- a/observability/dashboards/liquidity/t0-liquidity.json +++ b/observability/dashboards/liquidity/t0-liquidity.json @@ -2567,8 +2567,8 @@ "content": "
", "defaultContent": "
", "helpers": "", - "afterRender": "// The detail dialog's status history: a Trades or Rebalances row's status-log\n// entries in order, and the status the row is at. The generator prepends this\n// file to detail.js; dashboard/src/lib/transfer-board.test.ts imports it as a\n// module through the export line, which the generator drops.\n\n// A USDC bridge's in-flight statuses in lifecycle order per direction. Their\n// log times can go backwards (WithdrawalComplete carries confirmed_at, the\n// BridgingSubmitting after it the older initiated_at), so two in-flight\n// entries of a bridge are ordered by this. Anything else, failed included\n// (the bot can recover a bridge out of a failure), orders by time.\n//\n// Known limit: a Base to Alpaca bridge recovered out of DepositFailed reaches\n// converting, which the bot stamps with the bridge's initiated_at. That entry\n// sorts before the failed one, so the dialog keeps showing the status before\n// it (failed, or depositing) until the bridge completes.\nconst BRIDGE_LIFECYCLE = {\n alpaca_to_base: ['converting', 'withdrawing', 'bridging', 'depositing'],\n base_to_alpaca: ['withdrawing', 'bridging', 'depositing', 'converting'],\n};\nconst stage = (entry) => {\n const lifecycle = entry.kind === 'usdc_bridge' ? BRIDGE_LIFECYCLE[entry.direction] : null;\n const index = lifecycle ? lifecycle.indexOf(String(entry.status || '').toLowerCase()) : -1;\n return index >= 0 ? index : null;\n};\n\n// The row at its latest status, with its status history oldest first: by\n// time (ties keep Cloud Logging's newest-first order reversed), then each\n// pair of neighbouring in-flight bridge entries put in lifecycle order.\nconst latest = (entries) => {\n if (entries.length === 0) return null;\n const history = [...entries].reverse().sort((left, right) => left.time - right.time);\n for (let swapped = true; swapped; ) {\n swapped = false;\n for (let index = 1; index < history.length; index++) {\n const [before, after] = [stage(history[index - 1]), stage(history[index])];\n if (before !== null && after !== null && before > after) {\n [history[index - 1], history[index]] = [history[index], history[index - 1]];\n swapped = true;\n }\n }\n }\n const last = history[history.length - 1];\n return { ...last, first: Math.min(...entries.map((entry) => entry.time)), history };\n};\n\n\n\n// Business Text \"afterRender\" code for the Liquidity bot's detail panel: the\n// SPA's trade and transfer detail dialogs (trade-history-panel.svelte,\n// transfer-panel.svelte in ST0x-Technology/st0x.liquidity) for the row whose\n// \u24d8 was clicked in the Trades or Rebalances table.\n//\n// The click sets the hidden `detail` variable to the row's id. A and B are\n// the Trades and Rebalances tables' own results (the newest 500 status\n// entries of each log, through the Dashboard datasource), and the script\n// picks the id's entries out of them. C, the bot's event timeline, has no\n// query until the bot logs its events, so the dialog shows the status\n// history. Closing the dialog clears the variable.\n//\n// The panel is one empty column of the header row; only the dialog shows.\n//\n// latest is not defined here: the generator prepends it from\n// status-history.js (see detail_panel()).\n\nconst theme = context.grafana.theme;\nconst root = context.element;\nroot.style.setProperty('--det-muted', theme.colors.text.secondary);\nroot.style.setProperty('--det-border', theme.colors.border.weak);\nroot.style.setProperty('--det-stripe', theme.colors.background.secondary);\nroot.style.setProperty('--det-card', theme.colors.background.primary);\nroot.style.setProperty('--det-text', theme.colors.text.primary);\n// Light theme swaps the status colours for darker shades (detail.css).\nroot.classList.toggle('det-light', !theme.isDark);\n\n// Alternate rows in the board's Trades and Rebalances\n// tables, like the SPA's. Grafana's table has no striping option, but its\n// grid marks every row rdg-row-even or rdg-row-odd. This panel lives only on\n// the Dashboard tab, so the rule goes into the page while the panel is\n// mounted and comes out when the board is left. Grafana can mount the panel\n// on a new element (tab switch, view mode), so each render takes the rule\n// over for its own element and restarts the watcher, and a watcher removes\n// the rule only while its element still owns it.\nlet stripes = document.getElementById('liquidity-table-stripes');\nif (!stripes) {\n stripes = document.createElement('style');\n stripes.id = 'liquidity-table-stripes';\n document.head.appendChild(stripes);\n}\nstripes.textContent = `.rdg-row-even { background: ${theme.isDark ? 'rgba(255, 255, 255, 0.03)' : 'rgba(0, 0, 0, 0.03)'}; }`;\nroot.__stripesOwner = root.__stripesOwner || Math.random().toString(36).slice(2);\nstripes.dataset.owner = root.__stripesOwner;\nclearInterval(root.__stripesWatch);\nroot.__stripesWatch = setInterval(() => {\n if (root.isConnected) return;\n clearInterval(root.__stripesWatch);\n if (stripes.dataset.owner === root.__stripesOwner) stripes.remove();\n}, 2000);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The id the tables asked for. Entries of any other id are a previous\n// query's data, still shown while the new one loads.\nconst wanted = context.grafana.replaceVariables('${detail}');\n\n// One entry per log line: its time and the exporter's payload. The plugin\n// ships the payload as the JSON body and again as flattened\n// `jsonPayload.` labels; the labels are the fallback.\nconst entriesOf = (refId) => {\n const frame = (context.panelData?.series || []).find((series) => series.refId === refId);\n if (!frame) return [];\n const column = (name) => frame.fields.find((field) => field.name === name);\n const times = column('timestamp');\n const bodies = column('body');\n const labels = column('labels');\n const count = times ? times.values.length : 0;\n // A JSON object, or null for anything else (no column, bad JSON, a string).\n const parseObject = (value) => {\n if (value !== null && typeof value === 'object') return value;\n try {\n const parsed = JSON.parse(value);\n return parsed !== null && typeof parsed === 'object' ? parsed : null;\n } catch (error) {\n return null;\n }\n };\n const fromLabels = (index) => {\n const payload = {};\n const flat = labels ? parseObject(labels.values[index]) : null;\n for (const [key, value] of Object.entries(flat || {})) {\n if (key.startsWith('jsonPayload.')) payload[key.slice('jsonPayload.'.length)] = value;\n }\n return payload;\n };\n const entries = [];\n for (let index = 0; index < count; index++) {\n const payload = (bodies && parseObject(bodies.values[index])) || fromLabels(index);\n if (payload.id && String(payload.id) === wanted) entries.push({ time: Number(times.values[index]), ...payload });\n }\n return entries;\n};\n\n// The SPA's formatUtc: \"Oct 6, 12:24:39 UTC\".\nconst MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];\nconst utc = (value) => {\n const date = new Date(value);\n if (Number.isNaN(date.getTime())) return '\u2014';\n const pad = (number) => String(number).padStart(2, '0');\n return `${MONTHS[date.getUTCMonth()]} ${date.getUTCDate()}, ${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())} UTC`;\n};\n\n// The SPA's formatDecimal(value, 3): fixed decimals, comma groups, and the\n// raw value when it does not parse.\nconst decimal = (value) => {\n const number = Number(value);\n if (value === '' || value === null || value === undefined || Number.isNaN(number)) return escapeHtml(value || '\u2014');\n return number.toLocaleString('en-US', { minimumFractionDigits: 3, maximumFractionDigits: 3 });\n};\n\nconst VENUES = {\n raindex: ['Raindex', 'det-blue', true],\n bebop: ['Bebop', 'det-purple', true],\n uniswap_v4: ['Uniswap v4', 'det-pink', true],\n unknown_onchain: ['Unknown Onchain', '', true],\n alpaca: ['Alpaca', 'det-orange', false],\n dry_run: ['DryRun', '', false],\n};\nconst venue = (key) => VENUES[key] || [key || '\u2014', '', false];\n\nconst TYPES = {\n equity_mint: 'Mint',\n equity_redemption: 'Redeem',\n alpaca_to_base: 'Alpaca \u2192 Raindex',\n base_to_alpaca: 'Raindex \u2192 Alpaca',\n};\nconst typeOf = (transfer) => (transfer.kind === 'usdc_bridge' ? transfer.direction : transfer.kind);\nconst typeLabel = (transfer) => TYPES[typeOf(transfer)] || transfer.kind || '\u2014';\nconst assetOf = (transfer) => (transfer.kind === 'usdc_bridge' || !transfer.symbol ? 'USDC' : transfer.symbol);\n\n// The SPA's statusStyle colours.\nconst statusClass = (status) => {\n const lower = String(status || '').toLowerCase();\n if (lower.includes('completed') || lower.includes('filled') || lower.includes('deposited') || lower.includes('confirmed')) return 'det-green';\n if (lower.includes('reconciled')) return 'det-amber';\n if (lower.includes('failed')) return 'det-red';\n if (lower.includes('cancelled')) return 'det-orange';\n return 'det-blue';\n};\nconst statusLabel = (status) => {\n const text = String(status || '\u2014').replace(/_/g, ' ');\n return text.charAt(0).toUpperCase() + text.slice(1);\n};\n\n// --------------------------------------------------------------------------\n// The dialog\n// --------------------------------------------------------------------------\n\n// Block explorers by the bot's chain wire name. A chain without one, or an\n// unknown chain, shows the hash unlinked: a link to the wrong explorer would\n// read as \"this tx does not exist\".\nconst EXPLORERS = {\n base: 'https://basescan.org',\n ethereum: 'https://etherscan.io',\n hyperevm: 'https://hyperevmscan.io',\n};\nconst shortHash = (hash) => `${escapeHtml(hash.slice(0, 10))}\u2026${escapeHtml(hash.slice(-8))}`;\nconst txLink = (hash, chain) =>\n EXPLORERS[chain]\n ? `${shortHash(hash)} \u2197`\n : `${shortHash(hash)}`;\n\n// The bot's event timeline, once the panel queries it as C: each step with\n// its time and fields, like the SPA's dialog.\n// Without it, the exporter's status log: one step per status change.\nconst events = entriesOf('C').sort((left, right) => (left.sequence ?? 0) - (right.sequence ?? 0));\nconst humanize = (step) => String(step || '').replace(/([a-z0-9])([A-Z])/g, '$1 $2');\nconst isTxHash = (value) => typeof value === 'string' && /^0x[0-9a-fA-F]{64}$/.test(value);\nconst eventValue = (key, value, chain) => {\n if (key === 'error') return `${escapeHtml(value)}`;\n if (isTxHash(value)) return txLink(value, chain);\n if (value !== null && typeof value === 'object') return `${escapeHtml(JSON.stringify(value))}`;\n return escapeHtml(value);\n};\n// A payload's own `chain` names where its hashes live; else the row's chain\n// (a trade's), else none, which leaves the hashes unlinked.\nconst eventFields = (payload, rowChain) => {\n const fields = payload && typeof payload === 'object' ? payload : {};\n const chain = typeof fields.chain === 'string' ? fields.chain : rowChain;\n return Object.entries(fields)\n .filter(([key]) => !/(_at|At|timestamp)$/.test(key))\n .map(([key, value]) => `
${escapeHtml(key.replace(/_/g, ' '))}${eventValue(key, value, chain)}
`)\n .join('');\n};\n\n// The board loads the newest 500 events of all rows, so an older row can have\n// only its later events here. Then the timeline starts after the row's first\n// status entry, and says so.\nconst timelineIncomplete = (history) =>\n history.length > 0 &&\n Math.min(...events.map((event) => event.time)) > Math.min(...history.map((entry) => entry.time)) + 120000;\n\nconst timeline = (history, chain) =>\n events.length > 0\n ? `\n
Event timeline
\n ${timelineIncomplete(history) ? '

Older events of this row are outside the newest 500 the board loads; the full timeline is in the SPA at liquidity.t0trade.com.

' : ''}\n
${events\n .map(\n (event) => `\n
\n \n
${escapeHtml(humanize(event.step))}
\n
${utc(event.time)}
\n ${eventFields(event.payload, chain)}\n
`\n )\n .join('')}
`\n : `\n
Status history
\n
${history\n .map(\n (entry) => `\n
\n \n
${escapeHtml(statusLabel(entry.status))}
\n
${utc(entry.time)}
\n ${entry.error ? `
${escapeHtml(entry.error)}
` : ''}\n
`\n )\n .join('')}
\n

From the exporter's status log. The full event timeline is in the SPA at liquidity.t0trade.com.

`;\n\n// valueHtml is trusted HTML: a caller escapes any data it puts in it.\nconst field = (name, valueHtml) => `
${escapeHtml(name)}${valueHtml}
`;\n\nconst tradeDialog = (trade) => {\n const [venueName, venueClass, onchain] = venue(trade.venue);\n const id = String(trade.id);\n // An onchain trade id is `chain:tx_hash:log_index` (OnChainTradeId), or\n // `tx_hash:log_index` from before the chain was part of it (Base).\n const parts = id.split(':');\n const [chain, hash, log] = parts.length === 3 ? parts : ['base', ...parts];\n const idCell =\n onchain && parts.length >= 2 && isTxHash(hash)\n ? `${txLink(hash, chain)} (${escapeHtml(chain)}, log ${escapeHtml(log)})`\n : `${escapeHtml(id)}`;\n return `\n
\n \n ${escapeHtml(trade.direction)}\n ${escapeHtml(trade.symbol)}\n ${decimal(trade.shares)} ${onchain ? 'wrapped shares' : 'shares'}\n ${escapeHtml(venueName)}\n \n \n
\n
\n
\n ${field('ID', idCell)}\n ${field('Occurred At', utc(trade.first))}\n ${field('Status', `${escapeHtml(statusLabel(trade.status))}${trade.error ? `
${escapeHtml(trade.error)}
` : ''}`)}\n
\n ${timeline(trade.history, onchain ? chain : null)}\n
`;\n};\n\nconst transferDialog = (transfer) => {\n return `\n
\n \n ${escapeHtml(typeLabel(transfer))}\n ${escapeHtml(assetOf(transfer))}\n ${decimal(transfer.amount)}\n \n \n
\n
\n
\n ${field('ID', `${escapeHtml(transfer.id)}`)}\n ${field('Started', utc(transfer.started_at || transfer.first))}\n ${field('Updated', utc(transfer.time))}\n ${field('Status', `${escapeHtml(statusLabel(transfer.status))}`)}\n
\n ${timeline(transfer.history, null)}\n
`;\n};\n\n// --------------------------------------------------------------------------\n// Open, refresh or close the one dialog\n// --------------------------------------------------------------------------\n\n// Kept on the element across renders: the plugin re-runs this code on every\n// refresh and query, into the same element.\nconst state = root.__det || (root.__det = { dismissed: null });\n\n// The variable can lag the close by a render (a refresh that was already\n// running), so an id the operator just closed is not reopened until the\n// variable has moved off it.\nif (state.dismissed !== null && state.dismissed !== wanted) state.dismissed = null;\n\n// Switching Environment keeps the tables' old rows on screen until the new\n// queries return, and those rows belong to the other project. Hold back\n// every row after a switch until a query has run (Grafana showed the panel\n// Loading) and then succeeded. Frames alone cannot tell: Grafana replays and\n// copies the old ones, and keeps them on an error. A Cloud Logging query\n// takes over a second, so the Loading state shows; if it does not, or shows\n// only on the switch render, the dialog says Loading\u2026 until the next\n// refresh, which fails safe. Known\n// limit: a query cancelled from the refresh picker after the switch also\n// ends Done with the old rows, and that releases the hold. The memory\n// lives on the window, not the element, so a remount (tab switch, view mode)\n// keeps it. The first render after the page loads is not held back: its rows\n// are for the environment it opened on.\nconst env = context.grafana.replaceVariables('${env}');\nconst loading = context.panelData?.state === 'Loading';\nconst failed = context.panelData?.state === 'Error';\nconst envState = window.__liqDetailEnv || (window.__liqDetailEnv = { env: undefined, held: false, sawLoading: false });\nconst switched = envState.env !== undefined && envState.env !== env;\nenvState.env = env;\nif (switched) {\n // A Loading state on this render can be a run that started before the\n // switch, so only a later one counts.\n envState.held = true;\n envState.sawLoading = false;\n} else if (envState.held) {\n // A query that failed does not count as the one that ran.\n if (loading) envState.sawLoading = true;\n else if (failed) envState.sawLoading = false;\n else if (envState.sawLoading) envState.held = false;\n}\n\n// A failed refresh can keep the old frames, so it shows the error, not a\n// row and its commands that may be out of date.\nconst trade = envState.held || failed ? null : latest(entriesOf('A'));\nconst transfer = trade || envState.held || failed ? null : latest(entriesOf('B'));\n// The dialog opens on the click, before the query returns, and says so when\n// the id has no entries in the panel's window.\nconst message = (text) => `\n
${escapeHtml(wanted)}
\n

${escapeHtml(text)}

`;\nconst queryError = () => {\n const errors = context.panelData?.errors || (context.panelData?.error ? [context.panelData.error] : []);\n return errors.map((error) => error.message).filter(Boolean).join('; ') || 'unknown error';\n};\nconst html = trade\n ? tradeDialog(trade)\n : transfer\n ? transferDialog(transfer)\n : wanted === ''\n ? null\n : failed\n ? message(`The Trades or Rebalances query failed (${queryError()}), so this row cannot be shown. Refresh the board to retry.`)\n : loading || envState.held\n ? message('Loading\u2026')\n : message('This id is not in the newest 500 status entries any more. Search the Logs tab for it.');\n\n// Reuse the dialog when it is still in the element, so a refresh with the\n// dialog open swaps its content without closing and reopening it.\nlet dialog = root.querySelector('dialog.det-dialog');\nif (!dialog) {\n dialog = document.createElement('dialog');\n dialog.className = 'det-dialog';\n root.appendChild(dialog);\n}\n\n// Property handlers, not listeners: each render replaces them instead of\n// stacking another copy.\ndialog.onclose = () => {\n state.dismissed = dialog.detId || null;\n dialog.detId = '';\n // An empty value, not null: removing the URL parameter leaves a textbox\n // variable at its old value, so the same row could not be reopened.\n if (context.grafana.replaceVariables('${detail}') !== '') {\n context.grafana.locationService.partial({ 'var-detail': '' }, true);\n }\n};\n// A click on the backdrop closes it, like the SPA.\ndialog.onclick = (event) => {\n if (event.target === dialog) dialog.close();\n};\n\nif (html === null || state.dismissed === wanted) {\n if (dialog.open) dialog.close();\n} else {\n if (dialog.detHtml !== html) {\n // A refresh that brings a new status keeps the operator's scroll.\n const body = dialog.querySelector('.det-dialog-body');\n const scroll = dialog.detId === wanted && body ? body.scrollTop : 0;\n dialog.innerHTML = html;\n dialog.detHtml = html;\n const next = dialog.querySelector('.det-dialog-body');\n if (next) next.scrollTop = scroll;\n }\n dialog.detId = wanted;\n dialog.querySelector('[data-close]').onclick = () => dialog.close();\n if (!dialog.open) dialog.showModal();\n}\n", - "styles": "/* Business Text \"styles\" for the Liquidity bot's trade and transfer detail\n dialog. Theme colours come in as --det-* variables set by detail.js;\n status, venue and side colours are the SPA's (Tailwind 500). */\n\n.det-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.det-muted { color: var(--det-muted); }\n.det-break { word-break: break-all; }\n\n.det-green { color: #22c55e; }\n.det-red { color: #ef4444; }\n.det-amber { color: #f59e0b; }\n.det-orange { color: #f97316; }\n.det-blue { color: #3b82f6; }\n.det-purple { color: #a855f7; }\n.det-pink { color: #ec4899; }\n\n/* Tailwind 700 on the light theme, so status text keeps a 4.5:1 contrast\n against the white dialog. */\n.det-light .det-green { color: #15803d; }\n.det-light .det-red { color: #b91c1c; }\n.det-light .det-amber { color: #b45309; }\n.det-light .det-orange { color: #c2410c; }\n.det-light .det-blue { color: #1d4ed8; }\n.det-light .det-purple { color: #7e22ce; }\n.det-light .det-pink { color: #be185d; }\n\n.det-dialog {\n width: min(36rem, 92vw);\n padding: 0;\n border: 1px solid var(--det-border);\n border-radius: 8px;\n background: var(--det-card);\n color: var(--det-text);\n font-size: 14px;\n white-space: normal;\n}\n\n.det-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n\n.det-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--det-border);\n}\n\n.det-title-line {\n display: flex;\n flex-wrap: wrap;\n align-items: center;\n gap: 8px;\n font-size: 16px;\n font-weight: 600;\n}\n\n.det-title-line .det-mono { font-size: 14px; font-weight: 400; }\n\n.det-close {\n border: 0;\n background: transparent;\n color: var(--det-muted);\n font-size: 20px;\n line-height: 1;\n cursor: pointer;\n}\n\n.det-close:hover { color: var(--det-text); }\n\n.det-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 14px;\n}\n\n.det-dialog-body a { color: #3b82f6; }\n.det-light .det-dialog-body a { color: #1d4ed8; }\n\n.det-fields { margin-bottom: 16px; }\n\n.det-field {\n display: flex;\n gap: 8px;\n padding: 2px 0;\n}\n\n.det-field > .det-muted { flex-shrink: 0; }\n\n.det-section-title {\n margin-bottom: 8px;\n font-weight: 600;\n}\n\n.det-timeline {\n margin-bottom: 8px;\n padding-left: 16px;\n border-left: 2px solid var(--det-border);\n}\n\n.det-step {\n position: relative;\n padding-bottom: 12px;\n}\n\n.det-dot {\n position: absolute;\n top: 6px;\n left: -21px;\n width: 8px;\n height: 8px;\n border-radius: 9999px;\n background: currentColor;\n}\n", + "afterRender": "const MODE_LABELS = {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"};\n\n// The board's per-row recovery commands: the client form of the SPA's\n// tradeRecoveryCommands and transferRecoveryCommands\n// (dashboard/src/lib/transfer.ts). The generator prepends this file to\n// detail.js; dashboard/src/lib/transfer-board.test.ts runs it against the SPA's\n// builders so the two cannot drift, importing it as a module through the\n// export line, which the generator drops. `client` is the client invocation\n// with its --env.\n\nconst tradeCommands = (client, symbol) => [\n {\n command: `${client} debug position release-hedge ${symbol} --order-id --reason \"\"`,\n description: \"Clear a position's stuck pending offchain order so normal hedging can retry. Needs the order id.\",\n mode: 'requires-bot',\n },\n {\n command: `${client} debug position set ${symbol} --target-net [--price-usdc ] --reason \"\"`,\n description:\n 'Override the net exposure after a manual correction. The target is signed: negative is short, 0 is flat. The price is required for a nonzero target unless the position already has a last price.',\n mode: 'requires-bot',\n },\n {\n command: `${client} debug view rebuild position --id ${symbol}`,\n description: 'Replay all events to reconstruct a corrupted position view.',\n mode: 'requires-bot',\n },\n];\n\nconst USDC_FAIL_ALPACA_TO_BASE =\n 'Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal ' +\n 'or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum ' +\n 'that no CCTP burn left the bot wallet after this transfer started (its Started time) and ' +\n 'that the wallet has no pending transaction (pending nonce equals latest nonce); if you ' +\n 'are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held ' +\n 'until you settle them with reconcile-usdc.';\n\nconst USDC_FAIL_BASE_TO_ALPACA =\n 'Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, ' +\n 'and it does not check the chain. First confirm on Base that the bot wallet made no ' +\n \"OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 \" +\n 'on the orderbook either (a withdrawal sent before the inventory migration). Only a ' +\n 'USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in ' +\n 'the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 ' +\n 'sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the ' +\n 'network accepted but has not mined is not in the logs yet, so also wait until the ' +\n \"transfer's attempt timeout has passed and confirm the wallet has no pending \" +\n 'transaction (no pending withdraw4, and pending nonce equals latest nonce on more than ' +\n 'one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal ' +\n 'landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, ' +\n 'and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot ' +\n 'wallet back to the vault by hand. If nothing landed, the resume keeps redriving: ' +\n 'repeat the checks, and once they show that no withdrawal landed and none is pending, ' +\n 'take the next step. If none landed, close the nonce before you fail the transfer: ' +\n 'stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself ' +\n 'at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market ' +\n 'fee, wait for its required confirmations, and check the withdrawal logs again. If a ' +\n 'withdrawal mined instead, start the bot and run resume-usdc, and do not fail the ' +\n 'transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after ' +\n 'you restart the bot. If a withdrawal for this transfer mines later anyway, move its ' +\n 'USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is ' +\n 'still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the ' +\n 'bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn ' +\n 'left the bot wallet after this transfer started and the wallet has no pending ' +\n 'transaction. If any of that is not certain, start the bot and run resume-usdc; else ' +\n 'run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.';\n\nconst USDC_BRIDGING_BASE_TO_ALPACA =\n ' If the burn keeps failing, the running bot refuses fail-usdc-transfer here: see ' +\n 'fail-usdc-transfer in the CLI recovery guide for the offline path.';\n\nconst USDC_CONVERTING_ALPACA_TO_BASE =\n ' Alpaca to Base: this status covers two bot states. If the bot recorded the ' +\n 'conversion as complete, this resume continues with the Alpaca withdrawal. If not, the ' +\n 'bot cannot re-drive the USD to USDC conversion (it did not store the broker order ' +\n 'id), so this resume fails the transfer (ConversionFailed) and releases the guard, ' +\n 'without checking Alpaca. So first confirm at Alpaca that no USD to USDC order for ' +\n 'this transfer is still open (it is filled, cancelled or rejected), above all after a ' +\n 'conversion outcome unresolved alert; if one is open, cancel it or wait for it. Do not ' +\n 'move USDC by hand before you run the resume. If it ends in ConversionFailed, settle ' +\n 'any converted USDC by hand at Alpaca (reconcile-usdc does not apply). A later failure ' +\n 'has its own recovery.';\n\n// usdcResumeNote in transfer.ts: the extra resume text for a bridge's status\n// and direction.\nconst usdcResumeNote = (status, direction) => {\n if (direction === 'base_to_alpaca' && status === 'bridging') return USDC_BRIDGING_BASE_TO_ALPACA;\n if (direction !== 'base_to_alpaca' && status === 'converting') return USDC_CONVERTING_ALPACA_TO_BASE;\n return '';\n};\n\n// usdcFailCheck in transfer.ts: the check before fail-usdc-transfer, or null\n// where the running bot always refuses it.\nconst usdcFailCheck = (status, direction) => {\n if (direction === 'alpaca_to_base') {\n return status === 'withdrawing' || status === 'bridging' ? USDC_FAIL_ALPACA_TO_BASE : null;\n }\n if (direction === 'base_to_alpaca') return status === 'withdrawing' ? USDC_FAIL_BASE_TO_ALPACA : null;\n return status === 'withdrawing' || status === 'bridging'\n ? `${USDC_FAIL_ALPACA_TO_BASE} ${USDC_FAIL_BASE_TO_ALPACA}`\n : null;\n};\n\nconst transferCommands = (client, transfer) => {\n const status = String(transfer.status || '').toLowerCase();\n const failed = status === 'failed';\n const terminal = failed || status === 'completed' || status === 'reconciled';\n if (status === 'completed' || status === 'reconciled') return [];\n const id = transfer.id;\n\n if (transfer.kind === 'usdc_bridge') {\n // Reconcile needs the bridge's postBurn flag, which the exporter does not\n // log; the SPA offers nothing without it either.\n if (failed) return [];\n const direction =\n { alpaca_to_base: 'alpaca-to-base', base_to_alpaca: 'base-to-alpaca' }[transfer.direction] ||\n '';\n const resume = {\n command: `${client} debug resume-usdc ${direction} ${id}`,\n description:\n \"Re-drive this USDC bridge on the bot's transfer worker from the stage it stopped at.\" +\n usdcResumeNote(status, transfer.direction || null),\n mode: 'requires-bot',\n };\n const check = usdcFailCheck(status, transfer.direction || null);\n if (check === null) return [resume];\n return [\n resume,\n {\n command: `${client} debug fail-usdc-transfer ${id} --reason \"\"`,\n description: `Check the transfer first. ${check}`,\n mode: 'requires-bot',\n },\n ];\n }\n\n const kind = { equity_mint: 'mint', equity_redemption: 'redemption' }[transfer.kind];\n if (!kind) return [];\n const commands = [\n {\n command: `${client} debug recheck ${kind} ${id}`,\n description: 'Ask the running bot to re-poll the provider and complete the transfer if it settled.',\n mode: 'requires-bot',\n },\n ];\n if (!terminal) {\n commands.push({\n command: `${client} debug resume`,\n description:\n 'Re-drive ALL interrupted mints and redemptions via the bot (no id; best-effort per transfer, each succeeds or fails independently and failures are reported as counts).',\n mode: 'requires-bot',\n });\n commands.push({\n command: `${client} debug fail-equity-transfer ${kind} ${id} --reason \"\"`,\n description: 'Force this stuck transfer into the terminal Failed state. Use when it is permanently stuck.',\n mode: 'requires-bot',\n });\n }\n if (failed) {\n commands.push({\n command: `${client} debug reconcile-equity ${kind} ${id} --reason \"\"`,\n description: 'Mark a Failed transfer as Reconciled once its residue was handled out-of-band (bookkeeping).',\n mode: 'requires-bot',\n });\n }\n return commands;\n};\n\n\n// The detail dialog's status history: a Trades or Rebalances row's status-log\n// entries in order, and the status the row is at. The generator prepends this\n// file to detail.js; dashboard/src/lib/transfer-board.test.ts imports it as a\n// module through the export line, which the generator drops.\n\n// A USDC bridge's in-flight statuses in lifecycle order per direction. Their\n// log times can go backwards (WithdrawalComplete carries confirmed_at, the\n// BridgingSubmitting after it the older initiated_at), so two in-flight\n// entries of a bridge are ordered by this. Anything else, failed included\n// (the bot can recover a bridge out of a failure), orders by time.\n//\n// Known limit: a Base to Alpaca bridge recovered out of DepositFailed reaches\n// converting, which the bot stamps with the bridge's initiated_at. That entry\n// sorts before the failed one, so the dialog keeps showing the status before\n// it (failed, or depositing) until the bridge completes.\nconst BRIDGE_LIFECYCLE = {\n alpaca_to_base: ['converting', 'withdrawing', 'bridging', 'depositing'],\n base_to_alpaca: ['withdrawing', 'bridging', 'depositing', 'converting'],\n};\nconst stage = (entry) => {\n const lifecycle = entry.kind === 'usdc_bridge' ? BRIDGE_LIFECYCLE[entry.direction] : null;\n const index = lifecycle ? lifecycle.indexOf(String(entry.status || '').toLowerCase()) : -1;\n return index >= 0 ? index : null;\n};\n\n// The row at its latest status, with its status history oldest first: by\n// time (ties keep Cloud Logging's newest-first order reversed), then each\n// pair of neighbouring in-flight bridge entries put in lifecycle order.\nconst latest = (entries) => {\n if (entries.length === 0) return null;\n const history = [...entries].reverse().sort((left, right) => left.time - right.time);\n for (let swapped = true; swapped; ) {\n swapped = false;\n for (let index = 1; index < history.length; index++) {\n const [before, after] = [stage(history[index - 1]), stage(history[index])];\n if (before !== null && after !== null && before > after) {\n [history[index - 1], history[index]] = [history[index], history[index - 1]];\n swapped = true;\n }\n }\n }\n const last = history[history.length - 1];\n return { ...last, first: Math.min(...entries.map((entry) => entry.time)), history };\n};\n\n\n\n// Business Text \"afterRender\" code for the Liquidity bot's detail panel: the\n// SPA's trade and transfer detail dialogs (trade-history-panel.svelte,\n// transfer-panel.svelte in ST0x-Technology/st0x.liquidity) for the row whose\n// \u24d8 was clicked in the Trades or Rebalances table.\n//\n// The click sets the hidden `detail` variable to the row's id. A and B are\n// the Trades and Rebalances tables' own results (the newest 500 status\n// entries of each log, through the Dashboard datasource), and the script\n// picks the id's entries out of them. C, the bot's event timeline, has no\n// query until the bot logs its events, so the dialog shows the status\n// history. Closing the dialog clears the variable.\n//\n// The panel is one empty column of the header row; only the dialog shows.\n//\n// MODE_LABELS, tradeCommands, transferCommands and latest are not defined\n// here: the generator prepends them from recovery-guide.json,\n// recovery-commands.js and status-history.js (see detail_panel()).\n\nconst theme = context.grafana.theme;\nconst root = context.element;\nroot.style.setProperty('--det-muted', theme.colors.text.secondary);\nroot.style.setProperty('--det-border', theme.colors.border.weak);\nroot.style.setProperty('--det-stripe', theme.colors.background.secondary);\nroot.style.setProperty('--det-card', theme.colors.background.primary);\nroot.style.setProperty('--det-text', theme.colors.text.primary);\n// Light theme swaps the status colours for darker shades (detail.css).\nroot.classList.toggle('det-light', !theme.isDark);\n\n// Alternate rows in the board's Trades and Rebalances\n// tables, like the SPA's. Grafana's table has no striping option, but its\n// grid marks every row rdg-row-even or rdg-row-odd. This panel lives only on\n// the Dashboard tab, so the rule goes into the page while the panel is\n// mounted and comes out when the board is left. Grafana can mount the panel\n// on a new element (tab switch, view mode), so each render takes the rule\n// over for its own element and restarts the watcher, and a watcher removes\n// the rule only while its element still owns it.\nlet stripes = document.getElementById('liquidity-table-stripes');\nif (!stripes) {\n stripes = document.createElement('style');\n stripes.id = 'liquidity-table-stripes';\n document.head.appendChild(stripes);\n}\nstripes.textContent = `.rdg-row-even { background: ${theme.isDark ? 'rgba(255, 255, 255, 0.03)' : 'rgba(0, 0, 0, 0.03)'}; }`;\nroot.__stripesOwner = root.__stripesOwner || Math.random().toString(36).slice(2);\nstripes.dataset.owner = root.__stripesOwner;\nclearInterval(root.__stripesWatch);\nroot.__stripesWatch = setInterval(() => {\n if (root.isConnected) return;\n clearInterval(root.__stripesWatch);\n if (stripes.dataset.owner === root.__stripesOwner) stripes.remove();\n}, 2000);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The id the tables asked for. Entries of any other id are a previous\n// query's data, still shown while the new one loads.\nconst wanted = context.grafana.replaceVariables('${detail}');\n\n// One entry per log line: its time and the exporter's payload. The plugin\n// ships the payload as the JSON body and again as flattened\n// `jsonPayload.` labels; the labels are the fallback.\nconst entriesOf = (refId) => {\n const frame = (context.panelData?.series || []).find((series) => series.refId === refId);\n if (!frame) return [];\n const column = (name) => frame.fields.find((field) => field.name === name);\n const times = column('timestamp');\n const bodies = column('body');\n const labels = column('labels');\n const count = times ? times.values.length : 0;\n // A JSON object, or null for anything else (no column, bad JSON, a string).\n const parseObject = (value) => {\n if (value !== null && typeof value === 'object') return value;\n try {\n const parsed = JSON.parse(value);\n return parsed !== null && typeof parsed === 'object' ? parsed : null;\n } catch (error) {\n return null;\n }\n };\n const fromLabels = (index) => {\n const payload = {};\n const flat = labels ? parseObject(labels.values[index]) : null;\n for (const [key, value] of Object.entries(flat || {})) {\n if (key.startsWith('jsonPayload.')) payload[key.slice('jsonPayload.'.length)] = value;\n }\n return payload;\n };\n const entries = [];\n for (let index = 0; index < count; index++) {\n const payload = (bodies && parseObject(bodies.values[index])) || fromLabels(index);\n if (payload.id && String(payload.id) === wanted) entries.push({ time: Number(times.values[index]), ...payload });\n }\n return entries;\n};\n\n// The SPA's formatUtc: \"Oct 6, 12:24:39 UTC\".\nconst MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];\nconst utc = (value) => {\n const date = new Date(value);\n if (Number.isNaN(date.getTime())) return '\u2014';\n const pad = (number) => String(number).padStart(2, '0');\n return `${MONTHS[date.getUTCMonth()]} ${date.getUTCDate()}, ${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())} UTC`;\n};\n\n// The SPA's formatDecimal(value, 3): fixed decimals, comma groups, and the\n// raw value when it does not parse.\nconst decimal = (value) => {\n const number = Number(value);\n if (value === '' || value === null || value === undefined || Number.isNaN(number)) return escapeHtml(value || '\u2014');\n return number.toLocaleString('en-US', { minimumFractionDigits: 3, maximumFractionDigits: 3 });\n};\n\nconst VENUES = {\n raindex: ['Raindex', 'det-blue', true],\n bebop: ['Bebop', 'det-purple', true],\n uniswap_v4: ['Uniswap v4', 'det-pink', true],\n unknown_onchain: ['Unknown Onchain', '', true],\n alpaca: ['Alpaca', 'det-orange', false],\n dry_run: ['DryRun', '', false],\n};\nconst venue = (key) => VENUES[key] || [key || '\u2014', '', false];\n\nconst TYPES = {\n equity_mint: 'Mint',\n equity_redemption: 'Redeem',\n alpaca_to_base: 'Alpaca \u2192 Raindex',\n base_to_alpaca: 'Raindex \u2192 Alpaca',\n};\nconst typeOf = (transfer) => (transfer.kind === 'usdc_bridge' ? transfer.direction : transfer.kind);\nconst typeLabel = (transfer) => TYPES[typeOf(transfer)] || transfer.kind || '\u2014';\nconst assetOf = (transfer) => (transfer.kind === 'usdc_bridge' || !transfer.symbol ? 'USDC' : transfer.symbol);\n\n// The SPA's statusStyle colours.\nconst statusClass = (status) => {\n const lower = String(status || '').toLowerCase();\n if (lower.includes('completed') || lower.includes('filled') || lower.includes('deposited') || lower.includes('confirmed')) return 'det-green';\n if (lower.includes('reconciled')) return 'det-amber';\n if (lower.includes('failed')) return 'det-red';\n if (lower.includes('cancelled')) return 'det-orange';\n return 'det-blue';\n};\nconst statusLabel = (status) => {\n const text = String(status || '\u2014').replace(/_/g, ' ');\n return text.charAt(0).toUpperCase() + text.slice(1);\n};\n\n// --------------------------------------------------------------------------\n// Recovery commands (recovery-commands.js, prepended): the operations client,\n// which signs in with Google and calls the running bot's ops API, so no SSH\n// is needed and every command needs the bot.\n// --------------------------------------------------------------------------\n\n// The environment the board shows: the client takes the selector's text.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\nconst CLIENT = `st0x-liquidity-client --env ${ENV}`;\n\nconst modeClass = (mode) => (mode === 'requires-bot' ? 'det-amber' : 'det-red');\nconst commandBlock = (commands, note) =>\n commands.length === 0 && !note\n ? ''\n : `
CLI commands
${note ? `

${escapeHtml(note)}

` : ''}${commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command)}
\n
${escapeHtml(entry.description)}
\n ${escapeHtml(MODE_LABELS[entry.mode] || entry.mode)}\n
`\n )\n .join('')}
`;\n\n// --------------------------------------------------------------------------\n// The dialog\n// --------------------------------------------------------------------------\n\n// Block explorers by the bot's chain wire name. A chain without one, or an\n// unknown chain, shows the hash unlinked: a link to the wrong explorer would\n// read as \"this tx does not exist\".\nconst EXPLORERS = {\n base: 'https://basescan.org',\n ethereum: 'https://etherscan.io',\n hyperevm: 'https://hyperevmscan.io',\n};\nconst shortHash = (hash) => `${escapeHtml(hash.slice(0, 10))}\u2026${escapeHtml(hash.slice(-8))}`;\nconst txLink = (hash, chain) =>\n EXPLORERS[chain]\n ? `${shortHash(hash)} \u2197`\n : `${shortHash(hash)}`;\n\n// The bot's event timeline, once the panel queries it as C: each step with\n// its time and fields, like the SPA's dialog.\n// Without it, the exporter's status log: one step per status change.\nconst events = entriesOf('C').sort((left, right) => (left.sequence ?? 0) - (right.sequence ?? 0));\nconst humanize = (step) => String(step || '').replace(/([a-z0-9])([A-Z])/g, '$1 $2');\nconst isTxHash = (value) => typeof value === 'string' && /^0x[0-9a-fA-F]{64}$/.test(value);\nconst eventValue = (key, value, chain) => {\n if (key === 'error') return `${escapeHtml(value)}`;\n if (isTxHash(value)) return txLink(value, chain);\n if (value !== null && typeof value === 'object') return `${escapeHtml(JSON.stringify(value))}`;\n return escapeHtml(value);\n};\n// A payload's own `chain` names where its hashes live; else the row's chain\n// (a trade's), else none, which leaves the hashes unlinked.\nconst eventFields = (payload, rowChain) => {\n const fields = payload && typeof payload === 'object' ? payload : {};\n const chain = typeof fields.chain === 'string' ? fields.chain : rowChain;\n return Object.entries(fields)\n .filter(([key]) => !/(_at|At|timestamp)$/.test(key))\n .map(([key, value]) => `
${escapeHtml(key.replace(/_/g, ' '))}${eventValue(key, value, chain)}
`)\n .join('');\n};\n\n// The board loads the newest 500 events of all rows, so an older row can have\n// only its later events here. Then the timeline starts after the row's first\n// status entry, and says so.\nconst timelineIncomplete = (history) =>\n history.length > 0 &&\n Math.min(...events.map((event) => event.time)) > Math.min(...history.map((entry) => entry.time)) + 120000;\n\nconst timeline = (history, chain) =>\n events.length > 0\n ? `\n
Event timeline
\n ${timelineIncomplete(history) ? '

Older events of this row are outside the newest 500 the board loads; the full timeline is in the SPA at liquidity.t0trade.com.

' : ''}\n
${events\n .map(\n (event) => `\n
\n \n
${escapeHtml(humanize(event.step))}
\n
${utc(event.time)}
\n ${eventFields(event.payload, chain)}\n
`\n )\n .join('')}
`\n : `\n
Status history
\n
${history\n .map(\n (entry) => `\n
\n \n
${escapeHtml(statusLabel(entry.status))}
\n
${utc(entry.time)}
\n ${entry.error ? `
${escapeHtml(entry.error)}
` : ''}\n
`\n )\n .join('')}
\n

From the exporter's status log. The full event timeline is in the SPA at liquidity.t0trade.com.

`;\n\n// valueHtml is trusted HTML: a caller escapes any data it puts in it.\nconst field = (name, valueHtml) => `
${escapeHtml(name)}${valueHtml}
`;\n\nconst tradeDialog = (trade) => {\n const [venueName, venueClass, onchain] = venue(trade.venue);\n const id = String(trade.id);\n // An onchain trade id is `chain:tx_hash:log_index` (OnChainTradeId), or\n // `tx_hash:log_index` from before the chain was part of it (Base).\n const parts = id.split(':');\n const [chain, hash, log] = parts.length === 3 ? parts : ['base', ...parts];\n const idCell =\n onchain && parts.length >= 2 && isTxHash(hash)\n ? `${txLink(hash, chain)} (${escapeHtml(chain)}, log ${escapeHtml(log)})`\n : `${escapeHtml(id)}`;\n return `\n
\n \n ${escapeHtml(trade.direction)}\n ${escapeHtml(trade.symbol)}\n ${decimal(trade.shares)} ${onchain ? 'wrapped shares' : 'shares'}\n ${escapeHtml(venueName)}\n \n \n
\n
\n
\n ${field('ID', idCell)}\n ${field('Occurred At', utc(trade.first))}\n ${field('Status', `${escapeHtml(statusLabel(trade.status))}${trade.error ? `
${escapeHtml(trade.error)}
` : ''}`)}\n
\n ${timeline(trade.history, onchain ? chain : null)}\n ${commandBlock(tradeCommands(CLIENT, trade.symbol))}\n
`;\n};\n\nconst transferDialog = (transfer) => {\n const usdcFailed = transfer.kind === 'usdc_bridge' && String(transfer.status).toLowerCase() === 'failed';\n return `\n
\n \n ${escapeHtml(typeLabel(transfer))}\n ${escapeHtml(assetOf(transfer))}\n ${decimal(transfer.amount)}\n \n \n
\n
\n
\n ${field('ID', `${escapeHtml(transfer.id)}`)}\n ${field('Started', utc(transfer.started_at || transfer.first))}\n ${field('Updated', utc(transfer.time))}\n ${field('Status', `${escapeHtml(statusLabel(transfer.status))}`)}\n
\n ${timeline(transfer.history, null)}\n ${commandBlock(\n transferCommands(CLIENT, transfer),\n usdcFailed\n ? 'Reconcile applies only when the funds left their source venue, which the logs do not show. Check the transfer in the SPA before you reconcile it.'\n : ''\n )}\n
`;\n};\n\n// --------------------------------------------------------------------------\n// Open, refresh or close the one dialog\n// --------------------------------------------------------------------------\n\n// Kept on the element across renders: the plugin re-runs this code on every\n// refresh and query, into the same element.\nconst state = root.__det || (root.__det = { dismissed: null });\n\n// The variable can lag the close by a render (a refresh that was already\n// running), so an id the operator just closed is not reopened until the\n// variable has moved off it.\nif (state.dismissed !== null && state.dismissed !== wanted) state.dismissed = null;\n\n// Switching Environment keeps the tables' old rows on screen until the new\n// queries return, and those rows belong to the other project. Hold back\n// every row after a switch until a query has run (Grafana showed the panel\n// Loading) and then succeeded. Frames alone cannot tell: Grafana replays and\n// copies the old ones, and keeps them on an error. A Cloud Logging query\n// takes over a second, so the Loading state shows; if it does not, or shows\n// only on the switch render, the dialog says Loading\u2026 until the next\n// refresh, which fails safe. Known\n// limit: a query cancelled from the refresh picker after the switch also\n// ends Done with the old rows, and that releases the hold. The memory\n// lives on the window, not the element, so a remount (tab switch, view mode)\n// keeps it. The first render after the page loads is not held back: its rows\n// are for the environment it opened on.\nconst env = context.grafana.replaceVariables('${env}');\nconst loading = context.panelData?.state === 'Loading';\nconst failed = context.panelData?.state === 'Error';\nconst envState = window.__liqDetailEnv || (window.__liqDetailEnv = { env: undefined, held: false, sawLoading: false });\nconst switched = envState.env !== undefined && envState.env !== env;\nenvState.env = env;\nif (switched) {\n // A Loading state on this render can be a run that started before the\n // switch, so only a later one counts.\n envState.held = true;\n envState.sawLoading = false;\n} else if (envState.held) {\n // A query that failed does not count as the one that ran.\n if (loading) envState.sawLoading = true;\n else if (failed) envState.sawLoading = false;\n else if (envState.sawLoading) envState.held = false;\n}\n\n// A failed refresh can keep the old frames, so it shows the error, not a\n// row and its commands that may be out of date.\nconst trade = envState.held || failed ? null : latest(entriesOf('A'));\nconst transfer = trade || envState.held || failed ? null : latest(entriesOf('B'));\n// The dialog opens on the click, before the query returns, and says so when\n// the id has no entries in the panel's window.\nconst message = (text) => `\n
${escapeHtml(wanted)}
\n

${escapeHtml(text)}

`;\nconst queryError = () => {\n const errors = context.panelData?.errors || (context.panelData?.error ? [context.panelData.error] : []);\n return errors.map((error) => error.message).filter(Boolean).join('; ') || 'unknown error';\n};\nconst html = trade\n ? tradeDialog(trade)\n : transfer\n ? transferDialog(transfer)\n : wanted === ''\n ? null\n : failed\n ? message(`The Trades or Rebalances query failed (${queryError()}), so this row cannot be shown. Refresh the board to retry.`)\n : loading || envState.held\n ? message('Loading\u2026')\n : message('This id is not in the newest 500 status entries any more. Search the Logs tab for it.');\n\n// Reuse the dialog when it is still in the element, so a refresh with the\n// dialog open swaps its content without closing and reopening it.\nlet dialog = root.querySelector('dialog.det-dialog');\nif (!dialog) {\n dialog = document.createElement('dialog');\n dialog.className = 'det-dialog';\n root.appendChild(dialog);\n}\n\n// Property handlers, not listeners: each render replaces them instead of\n// stacking another copy.\ndialog.onclose = () => {\n state.dismissed = dialog.detId || null;\n dialog.detId = '';\n // An empty value, not null: removing the URL parameter leaves a textbox\n // variable at its old value, so the same row could not be reopened.\n if (context.grafana.replaceVariables('${detail}') !== '') {\n context.grafana.locationService.partial({ 'var-detail': '' }, true);\n }\n};\n// A click on the backdrop closes it, like the SPA.\ndialog.onclick = (event) => {\n if (event.target === dialog) dialog.close();\n};\n\nif (html === null || state.dismissed === wanted) {\n if (dialog.open) dialog.close();\n} else {\n if (dialog.detHtml !== html) {\n // A refresh that brings a new status keeps the operator's scroll.\n const body = dialog.querySelector('.det-dialog-body');\n const scroll = dialog.detId === wanted && body ? body.scrollTop : 0;\n dialog.innerHTML = html;\n dialog.detHtml = html;\n const next = dialog.querySelector('.det-dialog-body');\n if (next) next.scrollTop = scroll;\n }\n dialog.detId = wanted;\n dialog.querySelector('[data-close]').onclick = () => dialog.close();\n if (!dialog.open) dialog.showModal();\n}\n", + "styles": "/* Business Text \"styles\" for the Liquidity bot's trade and transfer detail\n dialog. Theme colours come in as --det-* variables set by detail.js;\n status, venue and side colours are the SPA's (Tailwind 500). */\n\n.det-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.det-muted { color: var(--det-muted); }\n.det-break { word-break: break-all; }\n\n.det-green { color: #22c55e; }\n.det-red { color: #ef4444; }\n.det-amber { color: #f59e0b; }\n.det-orange { color: #f97316; }\n.det-blue { color: #3b82f6; }\n.det-purple { color: #a855f7; }\n.det-pink { color: #ec4899; }\n\n/* Tailwind 700 on the light theme, so status text keeps a 4.5:1 contrast\n against the white dialog. */\n.det-light .det-green { color: #15803d; }\n.det-light .det-red { color: #b91c1c; }\n.det-light .det-amber { color: #b45309; }\n.det-light .det-orange { color: #c2410c; }\n.det-light .det-blue { color: #1d4ed8; }\n.det-light .det-purple { color: #7e22ce; }\n.det-light .det-pink { color: #be185d; }\n\n.det-dialog {\n width: min(36rem, 92vw);\n padding: 0;\n border: 1px solid var(--det-border);\n border-radius: 8px;\n background: var(--det-card);\n color: var(--det-text);\n font-size: 14px;\n white-space: normal;\n}\n\n.det-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n\n.det-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--det-border);\n}\n\n.det-title-line {\n display: flex;\n flex-wrap: wrap;\n align-items: center;\n gap: 8px;\n font-size: 16px;\n font-weight: 600;\n}\n\n.det-title-line .det-mono { font-size: 14px; font-weight: 400; }\n\n.det-close {\n border: 0;\n background: transparent;\n color: var(--det-muted);\n font-size: 20px;\n line-height: 1;\n cursor: pointer;\n}\n\n.det-close:hover { color: var(--det-text); }\n\n.det-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 14px;\n}\n\n.det-dialog-body a { color: #3b82f6; }\n.det-light .det-dialog-body a { color: #1d4ed8; }\n\n.det-fields { margin-bottom: 16px; }\n\n.det-field {\n display: flex;\n gap: 8px;\n padding: 2px 0;\n}\n\n.det-field > .det-muted { flex-shrink: 0; }\n\n.det-section-title,\n.det-cli-title {\n margin-bottom: 8px;\n font-weight: 600;\n}\n\n.det-timeline {\n margin-bottom: 8px;\n padding-left: 16px;\n border-left: 2px solid var(--det-border);\n}\n\n.det-step {\n position: relative;\n padding-bottom: 12px;\n}\n\n.det-dot {\n position: absolute;\n top: 6px;\n left: -21px;\n width: 8px;\n height: 8px;\n border-radius: 9999px;\n background: currentColor;\n}\n\n.det-cli {\n margin-top: 16px;\n padding-top: 12px;\n border-top: 1px solid var(--det-border);\n}\n\n.det-command {\n margin-bottom: 8px;\n padding: 8px 12px;\n border: 1px solid var(--det-border);\n border-radius: 6px;\n background: var(--det-stripe);\n}\n\n.det-command pre {\n margin: 0 0 6px;\n padding: 6px 8px;\n border-radius: 4px;\n background: var(--det-card);\n font-size: 12px;\n white-space: pre-wrap;\n word-break: break-all;\n}\n\n.det-mode {\n display: inline-block;\n margin-top: 4px;\n font-size: 12px;\n}\n", "wrap": false } } diff --git a/observability/gen-t0-liquidity.py b/observability/gen-t0-liquidity.py index 8f9bf1a7d..6d161d119 100644 --- a/observability/gen-t0-liquidity.py +++ b/observability/gen-t0-liquidity.py @@ -1260,13 +1260,21 @@ def detail_panel(x, y, trades_id, transfers_id): which makes Grafana redraw the panel when the variable changes. liquidity-panels/detail.js builds the dialog. renderMode "data" hands the script both frames, where "allRows" would draw a frame picker. - The status-history order is prepended from status-history.js. + MODE_LABELS (how each recovery command runs) is prepended from + recovery-guide.json, like the header's guide, and the per-row command + builders from recovery-commands.js, and the status-history order from + status-history.js. """ - with open(os.path.join(HERE, "liquidity-panels", "status-history.js")) as f: - # Its export line is for the SPA test; afterRender is not a module. - history = "".join(line for line in f if not line.startswith("export ")) + with open(os.path.join(HERE, "liquidity-panels", "recovery-guide.json")) as f: + mode_labels = json.load(f)["modeLabels"] + commands = "" + for module in ("recovery-commands.js", "status-history.js"): + with open(os.path.join(HERE, "liquidity-panels", module)) as f: + # Its export line is for the SPA test; afterRender is not a module. + commands += "".join(line for line in f if not line.startswith("export ")) + "\n" with open(os.path.join(HERE, "liquidity-panels", "detail.js")) as f: - after_render = history + "\n\n" + f.read() + after_render = (f"const MODE_LABELS = {json.dumps(mode_labels)};\n\n" + + commands + "\n" + f.read()) with open(os.path.join(HERE, "liquidity-panels", "detail.css")) as f: styles = f.read() return { diff --git a/observability/liquidity-panels/detail.css b/observability/liquidity-panels/detail.css index b89aee5f3..cb5f1754e 100644 --- a/observability/liquidity-panels/detail.css +++ b/observability/liquidity-panels/detail.css @@ -87,7 +87,8 @@ .det-field > .det-muted { flex-shrink: 0; } -.det-section-title { +.det-section-title, +.det-cli-title { margin-bottom: 8px; font-weight: 600; } @@ -112,3 +113,33 @@ border-radius: 9999px; background: currentColor; } + +.det-cli { + margin-top: 16px; + padding-top: 12px; + border-top: 1px solid var(--det-border); +} + +.det-command { + margin-bottom: 8px; + padding: 8px 12px; + border: 1px solid var(--det-border); + border-radius: 6px; + background: var(--det-stripe); +} + +.det-command pre { + margin: 0 0 6px; + padding: 6px 8px; + border-radius: 4px; + background: var(--det-card); + font-size: 12px; + white-space: pre-wrap; + word-break: break-all; +} + +.det-mode { + display: inline-block; + margin-top: 4px; + font-size: 12px; +} diff --git a/observability/liquidity-panels/detail.js b/observability/liquidity-panels/detail.js index e8d430231..ada0517c8 100644 --- a/observability/liquidity-panels/detail.js +++ b/observability/liquidity-panels/detail.js @@ -12,8 +12,9 @@ // // The panel is one empty column of the header row; only the dialog shows. // -// latest is not defined here: the generator prepends it from -// status-history.js (see detail_panel()). +// MODE_LABELS, tradeCommands, transferCommands and latest are not defined +// here: the generator prepends them from recovery-guide.json, +// recovery-commands.js and status-history.js (see detail_panel()). const theme = context.grafana.theme; const root = context.element; @@ -144,6 +145,31 @@ const statusLabel = (status) => { return text.charAt(0).toUpperCase() + text.slice(1); }; +// -------------------------------------------------------------------------- +// Recovery commands (recovery-commands.js, prepended): the operations client, +// which signs in with Google and calls the running bot's ops API, so no SSH +// is needed and every command needs the bot. +// -------------------------------------------------------------------------- + +// The environment the board shows: the client takes the selector's text. +const ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production'; +const CLIENT = `st0x-liquidity-client --env ${ENV}`; + +const modeClass = (mode) => (mode === 'requires-bot' ? 'det-amber' : 'det-red'); +const commandBlock = (commands, note) => + commands.length === 0 && !note + ? '' + : `
CLI commands
${note ? `

${escapeHtml(note)}

` : ''}${commands + .map( + (entry) => ` +
+ 
${escapeHtml(entry.command)}
+
${escapeHtml(entry.description)}
+ ${escapeHtml(MODE_LABELS[entry.mode] || entry.mode)} +
` + ) + .join('')}
`; + // -------------------------------------------------------------------------- // The dialog // -------------------------------------------------------------------------- @@ -254,10 +280,12 @@ const tradeDialog = (trade) => { ${field('Status', `${escapeHtml(statusLabel(trade.status))}${trade.error ? `
${escapeHtml(trade.error)}
` : ''}`)} ${timeline(trade.history, onchain ? chain : null)} + ${commandBlock(tradeCommands(CLIENT, trade.symbol))} `; }; const transferDialog = (transfer) => { + const usdcFailed = transfer.kind === 'usdc_bridge' && String(transfer.status).toLowerCase() === 'failed'; return `
@@ -275,6 +303,12 @@ const transferDialog = (transfer) => { ${field('Status', `${escapeHtml(statusLabel(transfer.status))}`)}
${timeline(transfer.history, null)} + ${commandBlock( + transferCommands(CLIENT, transfer), + usdcFailed + ? 'Reconcile applies only when the funds left their source venue, which the logs do not show. Check the transfer in the SPA before you reconcile it.' + : '' + )} `; }; diff --git a/observability/liquidity-panels/recovery-commands.js b/observability/liquidity-panels/recovery-commands.js new file mode 100644 index 000000000..4862cad95 --- /dev/null +++ b/observability/liquidity-panels/recovery-commands.js @@ -0,0 +1,167 @@ +// The board's per-row recovery commands: the client form of the SPA's +// tradeRecoveryCommands and transferRecoveryCommands +// (dashboard/src/lib/transfer.ts). The generator prepends this file to +// detail.js; dashboard/src/lib/transfer-board.test.ts runs it against the SPA's +// builders so the two cannot drift, importing it as a module through the +// export line, which the generator drops. `client` is the client invocation +// with its --env. + +const tradeCommands = (client, symbol) => [ + { + command: `${client} debug position release-hedge ${symbol} --order-id --reason ""`, + description: "Clear a position's stuck pending offchain order so normal hedging can retry. Needs the order id.", + mode: 'requires-bot', + }, + { + command: `${client} debug position set ${symbol} --target-net [--price-usdc ] --reason ""`, + description: + 'Override the net exposure after a manual correction. The target is signed: negative is short, 0 is flat. The price is required for a nonzero target unless the position already has a last price.', + mode: 'requires-bot', + }, + { + command: `${client} debug view rebuild position --id ${symbol}`, + description: 'Replay all events to reconstruct a corrupted position view.', + mode: 'requires-bot', + }, +]; + +const USDC_FAIL_ALPACA_TO_BASE = + 'Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal ' + + 'or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum ' + + 'that no CCTP burn left the bot wallet after this transfer started (its Started time) and ' + + 'that the wallet has no pending transaction (pending nonce equals latest nonce); if you ' + + 'are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held ' + + 'until you settle them with reconcile-usdc.'; + +const USDC_FAIL_BASE_TO_ALPACA = + 'Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, ' + + 'and it does not check the chain. First confirm on Base that the bot wallet made no ' + + "OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 " + + 'on the orderbook either (a withdrawal sent before the inventory migration). Only a ' + + 'USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in ' + + 'the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 ' + + 'sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the ' + + 'network accepted but has not mined is not in the logs yet, so also wait until the ' + + "transfer's attempt timeout has passed and confirm the wallet has no pending " + + 'transaction (no pending withdraw4, and pending nonce equals latest nonce on more than ' + + 'one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal ' + + 'landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, ' + + 'and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot ' + + 'wallet back to the vault by hand. If nothing landed, the resume keeps redriving: ' + + 'repeat the checks, and once they show that no withdrawal landed and none is pending, ' + + 'take the next step. If none landed, close the nonce before you fail the transfer: ' + + 'stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself ' + + 'at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market ' + + 'fee, wait for its required confirmations, and check the withdrawal logs again. If a ' + + 'withdrawal mined instead, start the bot and run resume-usdc, and do not fail the ' + + 'transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after ' + + 'you restart the bot. If a withdrawal for this transfer mines later anyway, move its ' + + 'USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is ' + + 'still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the ' + + 'bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn ' + + 'left the bot wallet after this transfer started and the wallet has no pending ' + + 'transaction. If any of that is not certain, start the bot and run resume-usdc; else ' + + 'run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.'; + +const USDC_BRIDGING_BASE_TO_ALPACA = + ' If the burn keeps failing, the running bot refuses fail-usdc-transfer here: see ' + + 'fail-usdc-transfer in the CLI recovery guide for the offline path.'; + +const USDC_CONVERTING_ALPACA_TO_BASE = + ' Alpaca to Base: this status covers two bot states. If the bot recorded the ' + + 'conversion as complete, this resume continues with the Alpaca withdrawal. If not, the ' + + 'bot cannot re-drive the USD to USDC conversion (it did not store the broker order ' + + 'id), so this resume fails the transfer (ConversionFailed) and releases the guard, ' + + 'without checking Alpaca. So first confirm at Alpaca that no USD to USDC order for ' + + 'this transfer is still open (it is filled, cancelled or rejected), above all after a ' + + 'conversion outcome unresolved alert; if one is open, cancel it or wait for it. Do not ' + + 'move USDC by hand before you run the resume. If it ends in ConversionFailed, settle ' + + 'any converted USDC by hand at Alpaca (reconcile-usdc does not apply). A later failure ' + + 'has its own recovery.'; + +// usdcResumeNote in transfer.ts: the extra resume text for a bridge's status +// and direction. +const usdcResumeNote = (status, direction) => { + if (direction === 'base_to_alpaca' && status === 'bridging') return USDC_BRIDGING_BASE_TO_ALPACA; + if (direction !== 'base_to_alpaca' && status === 'converting') return USDC_CONVERTING_ALPACA_TO_BASE; + return ''; +}; + +// usdcFailCheck in transfer.ts: the check before fail-usdc-transfer, or null +// where the running bot always refuses it. +const usdcFailCheck = (status, direction) => { + if (direction === 'alpaca_to_base') { + return status === 'withdrawing' || status === 'bridging' ? USDC_FAIL_ALPACA_TO_BASE : null; + } + if (direction === 'base_to_alpaca') return status === 'withdrawing' ? USDC_FAIL_BASE_TO_ALPACA : null; + return status === 'withdrawing' || status === 'bridging' + ? `${USDC_FAIL_ALPACA_TO_BASE} ${USDC_FAIL_BASE_TO_ALPACA}` + : null; +}; + +const transferCommands = (client, transfer) => { + const status = String(transfer.status || '').toLowerCase(); + const failed = status === 'failed'; + const terminal = failed || status === 'completed' || status === 'reconciled'; + if (status === 'completed' || status === 'reconciled') return []; + const id = transfer.id; + + if (transfer.kind === 'usdc_bridge') { + // Reconcile needs the bridge's postBurn flag, which the exporter does not + // log; the SPA offers nothing without it either. + if (failed) return []; + const direction = + { alpaca_to_base: 'alpaca-to-base', base_to_alpaca: 'base-to-alpaca' }[transfer.direction] || + ''; + const resume = { + command: `${client} debug resume-usdc ${direction} ${id}`, + description: + "Re-drive this USDC bridge on the bot's transfer worker from the stage it stopped at." + + usdcResumeNote(status, transfer.direction || null), + mode: 'requires-bot', + }; + const check = usdcFailCheck(status, transfer.direction || null); + if (check === null) return [resume]; + return [ + resume, + { + command: `${client} debug fail-usdc-transfer ${id} --reason ""`, + description: `Check the transfer first. ${check}`, + mode: 'requires-bot', + }, + ]; + } + + const kind = { equity_mint: 'mint', equity_redemption: 'redemption' }[transfer.kind]; + if (!kind) return []; + const commands = [ + { + command: `${client} debug recheck ${kind} ${id}`, + description: 'Ask the running bot to re-poll the provider and complete the transfer if it settled.', + mode: 'requires-bot', + }, + ]; + if (!terminal) { + commands.push({ + command: `${client} debug resume`, + description: + 'Re-drive ALL interrupted mints and redemptions via the bot (no id; best-effort per transfer, each succeeds or fails independently and failures are reported as counts).', + mode: 'requires-bot', + }); + commands.push({ + command: `${client} debug fail-equity-transfer ${kind} ${id} --reason ""`, + description: 'Force this stuck transfer into the terminal Failed state. Use when it is permanently stuck.', + mode: 'requires-bot', + }); + } + if (failed) { + commands.push({ + command: `${client} debug reconcile-equity ${kind} ${id} --reason ""`, + description: 'Mark a Failed transfer as Reconciled once its residue was handled out-of-band (bookkeeping).', + mode: 'requires-bot', + }); + } + return commands; +}; + +export { tradeCommands, transferCommands };