From 64f224d071f12eaf33427d2927313ef771d3ad54 Mon Sep 17 00:00:00 2001 From: JuaniRios Date: Thu, 8 Oct 2026 19:37:06 -0300 Subject: [PATCH] [RAI-2998] Open a detail dialog from each liquidity trade and rebalance row --- dashboard/src/lib/transfer-board.test.ts | 68 ++- observability/README.md | 8 + .../dashboards/liquidity/t0-liquidity.json | 259 ++++++++---- .../liquidity/tabs/t0-liquidity-logs.json | 4 +- .../liquidity/tabs/t0-liquidity-orders.json | 10 +- .../tabs/t0-liquidity-performance.json | 100 ++--- .../liquidity/tabs/t0-liquidity-pnl.json | 42 +- observability/gen-t0-liquidity.py | 179 ++++++-- observability/liquidity-panels/detail.css | 114 +++++ observability/liquidity-panels/detail.js | 390 ++++++++++++++++++ .../liquidity-panels/status-history.js | 46 +++ 11 files changed, 1032 insertions(+), 188 deletions(-) create mode 100644 observability/liquidity-panels/detail.css create mode 100644 observability/liquidity-panels/detail.js create mode 100644 observability/liquidity-panels/status-history.js diff --git a/dashboard/src/lib/transfer-board.test.ts b/dashboard/src/lib/transfer-board.test.ts index 4dc288d55b..a3ff664485 100644 --- a/dashboard/src/lib/transfer-board.test.ts +++ b/dashboard/src/lib/transfer-board.test.ts @@ -4,10 +4,21 @@ import { RECOVERY_GUIDE, recoveryModeLabel, type RecoveryMode } from './transfer // The Grafana board (observability/) shows the same recovery commands as the // SPA from its own copy: recovery-guide.json for the guide. These tests fail -// when the copy drifts from this file's builders. +// when the copy drifts from this file's builders. They also cover the board's +// status-history.js, which orders a row's status entries in its dialog. const panels = new URL('../../../observability/liquidity-panels/', import.meta.url) +type StatusEntry = { time: number; kind: string; status: string; direction?: string } +type StatusHistory = { + latest: ( + entries: StatusEntry[] + ) => (StatusEntry & { first: number; history: StatusEntry[] }) | null +} + +// A plain script in the board, an ES module through its export line here. +const statusHistory = (await import(new URL('status-history.js', panels).href)) as StatusHistory + describe('board recovery-guide.json', () => { it('matches RECOVERY_GUIDE and the mode labels', () => { const guide = JSON.parse(readFileSync(new URL('recovery-guide.json', panels), 'utf8')) as { @@ -20,3 +31,58 @@ describe('board recovery-guide.json', () => { } }) }) + +describe('board status-history.js', () => { + const bridge = (status: string, time: number, direction = 'alpaca_to_base'): StatusEntry => ({ + time, + kind: 'usdc_bridge', + status, + direction + }) + // Cloud Logging returns the entries newest first. + const order = (entries: StatusEntry[]) => { + const row = statusHistory.latest(entries) + return { + status: row?.status, + first: row?.first, + history: row?.history.map((entry) => entry.status) + } + } + + it('orders in-flight bridge statuses by lifecycle when their times go backwards', () => { + // WithdrawalComplete carries confirmed_at, the BridgingSubmitting after it + // the older initiated_at. + expect(order([bridge('withdrawing', 200), bridge('bridging', 100)])).toEqual({ + status: 'bridging', + first: 100, + history: ['withdrawing', 'bridging'] + }) + expect( + order([bridge('bridging', 100, 'base_to_alpaca'), bridge('converting', 50, 'base_to_alpaca')]) + ).toEqual({ status: 'converting', first: 50, history: ['bridging', 'converting'] }) + }) + + it('orders in-flight bridge statuses that share a time by lifecycle, in either order', () => { + for (const entries of [ + [bridge('bridging', 100), bridge('withdrawing', 100)], + [bridge('withdrawing', 100), bridge('bridging', 100)] + ]) { + expect(order(entries).history).toEqual(['withdrawing', 'bridging']) + } + }) + + it('orders a failed entry by time, so a recovered bridge shows its newer status', () => { + expect( + order([bridge('bridging', 300), bridge('failed', 200), bridge('bridging', 100)]) + ).toEqual({ status: 'bridging', first: 100, history: ['bridging', 'failed', 'bridging'] }) + expect(order([bridge('failed', 200), bridge('depositing', 100)]).status).toBe('failed') + }) + + it('orders other rows by time, ties newest last as Cloud Logging returned them', () => { + const mint = (status: string, time: number): StatusEntry => ({ time, kind: 'mint', status }) + expect( + order([mint('wrapping', 100), mint('minting', 100), mint('pending', 50)]).history + ).toEqual(['pending', 'minting', 'wrapping']) + expect(statusHistory.latest([])).toBeNull() + }) +}) diff --git a/observability/README.md b/observability/README.md index e1685db32a..a8f182a655 100644 --- a/observability/README.md +++ b/observability/README.md @@ -59,6 +59,14 @@ band verdict (a symbol rebalanced on Base, with a balance, with no vault on another chain, while the default Base target is set), so it is grey on the `exporter` source, which cannot see the other chains. +Trades and Rebalances are Grafana tables too. A row's ⓘ sets the hidden `detail` +variable to the row's id; the `detail` panel, a second Business Text panel in +the header row's last column, then opens that row's dialog. It reads the two +tables' results through the Dashboard datasource (so Rebalances queries as refId +B) and runs no query of its own, so a click opens the dialog at once. The dialog +shows the row's status history (`status-history.js`); the bot's event timeline +joins it once the bot logs its events. Closing the dialog clears the variable. + Keep the Dashboard tab's panel heights (`TRADES_H`, `TRANSFERS_H` and the `native_inventory` heights in the generator) unless you check a new set on a preview board: reload it at a few window heights and confirm both columns end diff --git a/observability/dashboards/liquidity/t0-liquidity.json b/observability/dashboards/liquidity/t0-liquidity.json index 473b0295c9..9e7746387e 100644 --- a/observability/dashboards/liquidity/t0-liquidity.json +++ b/observability/dashboards/liquidity/t0-liquidity.json @@ -71,6 +71,17 @@ "value": "t0-liquidity|t0-liquidity-staging" } ] + }, + { + "type": "textbox", + "name": "detail", + "hide": 2, + "query": "", + "current": { + "text": "", + "value": "" + }, + "options": [] } ] }, @@ -179,7 +190,7 @@ "transparent": true, "gridPos": { "h": 1, - "w": 24, + "w": 23, "x": 0, "y": 0 }, @@ -1599,8 +1610,9 @@ { "id": 5, "type": "table", - "title": "Trade History", - "description": "Direct Raindex fills, fills routed through supported adapters such as Bebop, and the corresponding Alpaca hedge trades placed to offset exposure. One row per trade at its latest status. Built from the newest 500 status entries, so a busy period can push older rows out of the 30-day window.", + "title": "Trades", + "description": "Direct Raindex fills, fills routed through supported adapters such as Bebop, and the corresponding Alpaca hedge trades placed to offset exposure. One row per trade at its latest status; its \u24d8 opens the details. Built from the newest 500 status entries, so a busy period can push older rows out of the 30-day window.", + "pluginVersion": "13.1.0", "datasource": { "type": "googlecloud-logging-datasource", "uid": "cloudlog" @@ -1673,7 +1685,7 @@ "jsonPayload.direction": "Side", "jsonPayload.shares": "Size", "jsonPayload.status": "Status", - "timestamp": "Time", + "timestamp": "Last updated", "jsonPayload.id": "Id" } } @@ -1686,7 +1698,7 @@ "operation": "groupby", "aggregations": [] }, - "Time": { + "Last updated": { "operation": "aggregate", "aggregations": [ "first" @@ -1728,19 +1740,18 @@ { "id": "organize", "options": { - "excludeByName": { - "Id": true - }, + "excludeByName": {}, "indexByName": { - "Time (first)": 0, - "Asset (first)": 1, - "Venue (first)": 2, - "Side (first)": 3, - "Size (first)": 4, - "Status (first)": 5 + "Id": 0, + "Last updated (first)": 1, + "Asset (first)": 2, + "Venue (first)": 3, + "Side (first)": 4, + "Size (first)": 5, + "Status (first)": 6 }, "renameByName": { - "Time (first)": "Time", + "Last updated (first)": "Last updated", "Asset (first)": "Asset", "Venue (first)": "Venue", "Side (first)": "Side", @@ -1755,7 +1766,7 @@ "fields": {}, "sort": [ { - "field": "Time", + "field": "Last updated", "desc": true } ] @@ -1789,7 +1800,7 @@ { "matcher": { "id": "byName", - "options": "Time" + "options": "Last updated" }, "properties": [ { @@ -1801,12 +1812,53 @@ { "matcher": { "id": "byName", - "options": "Time" + "options": "Id" + }, + "properties": [ + { + "id": "displayName", + "value": " " + }, + { + "id": "custom.width", + "value": 50 + }, + { + "id": "custom.align", + "value": "center" + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "custom.cellOptions", + "value": { + "type": "data-links" + } + }, + { + "id": "links", + "value": [ + { + "title": "\u24d8", + "url": "/d/${__dashboard.uid}/${__url.params:exclude:var-detail}&var-detail=${__data.fields[\"jsonPayload.id\"]}", + "targetBlank": false, + "oneClick": true + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Last updated" }, "properties": [ { "id": "custom.width", - "value": 170 + "value": 185 } ] }, @@ -1818,7 +1870,7 @@ "properties": [ { "id": "custom.width", - "value": 80 + "value": 85 } ] }, @@ -1830,7 +1882,7 @@ "properties": [ { "id": "custom.width", - "value": 110 + "value": 115 }, { "id": "mappings", @@ -1888,7 +1940,7 @@ "properties": [ { "id": "custom.width", - "value": 70 + "value": 75 }, { "id": "mappings", @@ -1926,7 +1978,7 @@ "properties": [ { "id": "custom.width", - "value": 110 + "value": 100 } ] }, @@ -2028,14 +2080,15 @@ }, "options": { "showHeader": true, - "cellHeight": "sm" + "cellHeight": "md" } }, { "id": 6, "type": "table", - "title": "Cross-venue Transfers", - "description": "Asset movements between venues to rebalance inventory: equity mints (Alpaca to onchain), redemptions (onchain to Alpaca), and USDC bridges (Base/Ethereum via CCTP). One row per transfer at its latest status. A USDC bridge can show the previous status for a while: the exporter stamps each status with the transfer's updatedAt, which is not always later than the one before (bridging starts from initiated_at). Built from the newest 500 status entries, so a busy period can push older rows out of the 30-day window.", + "title": "Rebalances", + "description": "Asset movements between venues to rebalance inventory: equity mints (Alpaca to onchain), redemptions (onchain to Alpaca), and USDC bridges (Base/Ethereum via CCTP). One row per transfer at its latest status; its \u24d8 opens the details. A USDC bridge can show the previous status for a while: the exporter stamps each status with the transfer's updatedAt, which is not always later than the one before (bridging starts from initiated_at). Built from the newest 500 status entries, so a busy period can push older rows out of the 30-day window.", + "pluginVersion": "13.1.0", "datasource": { "type": "googlecloud-logging-datasource", "uid": "cloudlog" @@ -2044,7 +2097,7 @@ "maxDataPoints": 500, "targets": [ { - "refId": "A", + "refId": "B", "datasource": { "type": "googlecloud-logging-datasource", "uid": "cloudlog" @@ -2085,7 +2138,6 @@ "options": { "include": { "names": [ - "jsonPayload.started_at", "jsonPayload.symbol", "jsonPayload.amount", "jsonPayload.status", @@ -2103,10 +2155,6 @@ { "targetField": "jsonPayload.amount", "destinationType": "number" - }, - { - "targetField": "jsonPayload.started_at", - "destinationType": "time" } ] } @@ -2117,11 +2165,10 @@ "excludeByName": {}, "indexByName": {}, "renameByName": { - "jsonPayload.started_at": "Started", "jsonPayload.symbol": "Asset", "jsonPayload.amount": "Amount", "jsonPayload.status": "Status", - "timestamp": "Updated", + "timestamp": "Last updated", "jsonPayload.id": "Id" } } @@ -2134,7 +2181,7 @@ "operation": "groupby", "aggregations": [] }, - "Started": { + "Last updated": { "operation": "aggregate", "aggregations": [ "first" @@ -2163,12 +2210,6 @@ "aggregations": [ "first" ] - }, - "Updated": { - "operation": "aggregate", - "aggregations": [ - "first" - ] } } } @@ -2176,24 +2217,21 @@ { "id": "organize", "options": { - "excludeByName": { - "Id": true - }, + "excludeByName": {}, "indexByName": { - "Started (first)": 0, - "Type (first)": 1, - "Asset (first)": 2, - "Amount (first)": 3, - "Status (first)": 4, - "Updated (first)": 5 + "Id": 0, + "Last updated (first)": 1, + "Type (first)": 2, + "Asset (first)": 3, + "Amount (first)": 4, + "Status (first)": 5 }, "renameByName": { - "Started (first)": "Started", + "Last updated (first)": "Last updated", "Type (first)": "Type", "Asset (first)": "Asset", "Amount (first)": "Amount", - "Status (first)": "Status", - "Updated (first)": "Updated" + "Status (first)": "Status" } } }, @@ -2203,7 +2241,7 @@ "fields": {}, "sort": [ { - "field": "Updated", + "field": "Last updated", "desc": true } ] @@ -2237,7 +2275,7 @@ { "matcher": { "id": "byName", - "options": "Updated" + "options": "Last updated" }, "properties": [ { @@ -2249,24 +2287,53 @@ { "matcher": { "id": "byName", - "options": "Started" + "options": "Id" }, "properties": [ { - "id": "unit", - "value": "time:MMM D, HH:mm:ss [UTC]" + "id": "displayName", + "value": " " + }, + { + "id": "custom.width", + "value": 50 + }, + { + "id": "custom.align", + "value": "center" + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "custom.cellOptions", + "value": { + "type": "data-links" + } + }, + { + "id": "links", + "value": [ + { + "title": "\u24d8", + "url": "/d/${__dashboard.uid}/${__url.params:exclude:var-detail}&var-detail=${__data.fields[\"jsonPayload.id\"]}", + "targetBlank": false, + "oneClick": true + } + ] } ] }, { "matcher": { "id": "byName", - "options": "Started" + "options": "Last updated" }, "properties": [ { "id": "custom.width", - "value": 135 + "value": 185 } ] }, @@ -2278,7 +2345,7 @@ "properties": [ { "id": "custom.width", - "value": 125 + "value": 165 }, { "id": "mappings", @@ -2320,7 +2387,7 @@ "properties": [ { "id": "custom.width", - "value": 60 + "value": 80 }, { "id": "mappings", @@ -2347,7 +2414,7 @@ "properties": [ { "id": "custom.width", - "value": 85 + "value": 115 } ] }, @@ -2357,10 +2424,6 @@ "options": "Status" }, "properties": [ - { - "id": "custom.width", - "value": 110 - }, { "id": "mappings", "value": [ @@ -2448,19 +2511,65 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Updated" - }, - "properties": [] } ] }, "options": { "showHeader": true, - "cellHeight": "sm" + "cellHeight": "md" + } + }, + { + "id": 7, + "type": "marcusolsson-dynamictext-panel", + "title": "", + "description": "The detail dialog of a Trades or Rebalances row: open it from the row's \u24d8.", + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "timeFrom": "30d", + "hideTimeOverride": true, + "maxDataPoints": 500, + "targets": [ + { + "refId": "A", + "panelId": 5, + "withTransforms": false, + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + } + }, + { + "refId": "B", + "panelId": 6, + "withTransforms": false, + "datasource": { + "type": "datasource", + "uid": "-- Dashboard --" + } + } + ], + "transparent": true, + "gridPos": { + "h": 1, + "w": 1, + "x": 23, + "y": 0 + }, + "options": { + "renderMode": "data", + "editor": { + "format": "auto", + "language": "html" + }, + "content": "
", + "defaultContent": "
", + "helpers": "", + "afterRender": "// The detail dialog's status history: a Trades or Rebalances row's status-log\n// entries in order, and the status the row is at. The generator prepends this\n// file to detail.js; dashboard/src/lib/transfer-board.test.ts imports it as a\n// module through the export line, which the generator drops.\n\n// A USDC bridge's in-flight statuses in lifecycle order per direction. Their\n// log times can go backwards (WithdrawalComplete carries confirmed_at, the\n// BridgingSubmitting after it the older initiated_at), so two in-flight\n// entries of a bridge are ordered by this. Anything else, failed included\n// (the bot can recover a bridge out of a failure), orders by time.\n//\n// Known limit: a Base to Alpaca bridge recovered out of DepositFailed reaches\n// converting, which the bot stamps with the bridge's initiated_at. That entry\n// sorts before the failed one, so the dialog keeps showing the status before\n// it (failed, or depositing) until the bridge completes.\nconst BRIDGE_LIFECYCLE = {\n alpaca_to_base: ['converting', 'withdrawing', 'bridging', 'depositing'],\n base_to_alpaca: ['withdrawing', 'bridging', 'depositing', 'converting'],\n};\nconst stage = (entry) => {\n const lifecycle = entry.kind === 'usdc_bridge' ? BRIDGE_LIFECYCLE[entry.direction] : null;\n const index = lifecycle ? lifecycle.indexOf(String(entry.status || '').toLowerCase()) : -1;\n return index >= 0 ? index : null;\n};\n\n// The row at its latest status, with its status history oldest first: by\n// time (ties keep Cloud Logging's newest-first order reversed), then each\n// pair of neighbouring in-flight bridge entries put in lifecycle order.\nconst latest = (entries) => {\n if (entries.length === 0) return null;\n const history = [...entries].reverse().sort((left, right) => left.time - right.time);\n for (let swapped = true; swapped; ) {\n swapped = false;\n for (let index = 1; index < history.length; index++) {\n const [before, after] = [stage(history[index - 1]), stage(history[index])];\n if (before !== null && after !== null && before > after) {\n [history[index - 1], history[index]] = [history[index], history[index - 1]];\n swapped = true;\n }\n }\n }\n const last = history[history.length - 1];\n return { ...last, first: Math.min(...entries.map((entry) => entry.time)), history };\n};\n\n\n\n// Business Text \"afterRender\" code for the Liquidity bot's detail panel: the\n// SPA's trade and transfer detail dialogs (trade-history-panel.svelte,\n// transfer-panel.svelte in ST0x-Technology/st0x.liquidity) for the row whose\n// \u24d8 was clicked in the Trades or Rebalances table.\n//\n// The click sets the hidden `detail` variable to the row's id. A and B are\n// the Trades and Rebalances tables' own results (the newest 500 status\n// entries of each log, through the Dashboard datasource), and the script\n// picks the id's entries out of them. C, the bot's event timeline, has no\n// query until the bot logs its events, so the dialog shows the status\n// history. Closing the dialog clears the variable.\n//\n// The panel is one empty column of the header row; only the dialog shows.\n//\n// latest is not defined here: the generator prepends it from\n// status-history.js (see detail_panel()).\n\nconst theme = context.grafana.theme;\nconst root = context.element;\nroot.style.setProperty('--det-muted', theme.colors.text.secondary);\nroot.style.setProperty('--det-border', theme.colors.border.weak);\nroot.style.setProperty('--det-stripe', theme.colors.background.secondary);\nroot.style.setProperty('--det-card', theme.colors.background.primary);\nroot.style.setProperty('--det-text', theme.colors.text.primary);\n// Light theme swaps the status colours for darker shades (detail.css).\nroot.classList.toggle('det-light', !theme.isDark);\n\n// Alternate rows in the board's Trades and Rebalances\n// tables, like the SPA's. Grafana's table has no striping option, but its\n// grid marks every row rdg-row-even or rdg-row-odd. This panel lives only on\n// the Dashboard tab, so the rule goes into the page while the panel is\n// mounted and comes out when the board is left. Grafana can mount the panel\n// on a new element (tab switch, view mode), so each render takes the rule\n// over for its own element and restarts the watcher, and a watcher removes\n// the rule only while its element still owns it.\nlet stripes = document.getElementById('liquidity-table-stripes');\nif (!stripes) {\n stripes = document.createElement('style');\n stripes.id = 'liquidity-table-stripes';\n document.head.appendChild(stripes);\n}\nstripes.textContent = `.rdg-row-even { background: ${theme.isDark ? 'rgba(255, 255, 255, 0.03)' : 'rgba(0, 0, 0, 0.03)'}; }`;\nroot.__stripesOwner = root.__stripesOwner || Math.random().toString(36).slice(2);\nstripes.dataset.owner = root.__stripesOwner;\nclearInterval(root.__stripesWatch);\nroot.__stripesWatch = setInterval(() => {\n if (root.isConnected) return;\n clearInterval(root.__stripesWatch);\n if (stripes.dataset.owner === root.__stripesOwner) stripes.remove();\n}, 2000);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The id the tables asked for. Entries of any other id are a previous\n// query's data, still shown while the new one loads.\nconst wanted = context.grafana.replaceVariables('${detail}');\n\n// One entry per log line: its time and the exporter's payload. The plugin\n// ships the payload as the JSON body and again as flattened\n// `jsonPayload.` labels; the labels are the fallback.\nconst entriesOf = (refId) => {\n const frame = (context.panelData?.series || []).find((series) => series.refId === refId);\n if (!frame) return [];\n const column = (name) => frame.fields.find((field) => field.name === name);\n const times = column('timestamp');\n const bodies = column('body');\n const labels = column('labels');\n const count = times ? times.values.length : 0;\n // A JSON object, or null for anything else (no column, bad JSON, a string).\n const parseObject = (value) => {\n if (value !== null && typeof value === 'object') return value;\n try {\n const parsed = JSON.parse(value);\n return parsed !== null && typeof parsed === 'object' ? parsed : null;\n } catch (error) {\n return null;\n }\n };\n const fromLabels = (index) => {\n const payload = {};\n const flat = labels ? parseObject(labels.values[index]) : null;\n for (const [key, value] of Object.entries(flat || {})) {\n if (key.startsWith('jsonPayload.')) payload[key.slice('jsonPayload.'.length)] = value;\n }\n return payload;\n };\n const entries = [];\n for (let index = 0; index < count; index++) {\n const payload = (bodies && parseObject(bodies.values[index])) || fromLabels(index);\n if (payload.id && String(payload.id) === wanted) entries.push({ time: Number(times.values[index]), ...payload });\n }\n return entries;\n};\n\n// The SPA's formatUtc: \"Oct 6, 12:24:39 UTC\".\nconst MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];\nconst utc = (value) => {\n const date = new Date(value);\n if (Number.isNaN(date.getTime())) return '\u2014';\n const pad = (number) => String(number).padStart(2, '0');\n return `${MONTHS[date.getUTCMonth()]} ${date.getUTCDate()}, ${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())} UTC`;\n};\n\n// The SPA's formatDecimal(value, 3): fixed decimals, comma groups, and the\n// raw value when it does not parse.\nconst decimal = (value) => {\n const number = Number(value);\n if (value === '' || value === null || value === undefined || Number.isNaN(number)) return escapeHtml(value || '\u2014');\n return number.toLocaleString('en-US', { minimumFractionDigits: 3, maximumFractionDigits: 3 });\n};\n\nconst VENUES = {\n raindex: ['Raindex', 'det-blue', true],\n bebop: ['Bebop', 'det-purple', true],\n uniswap_v4: ['Uniswap v4', 'det-pink', true],\n unknown_onchain: ['Unknown Onchain', '', true],\n alpaca: ['Alpaca', 'det-orange', false],\n dry_run: ['DryRun', '', false],\n};\nconst venue = (key) => VENUES[key] || [key || '\u2014', '', false];\n\nconst TYPES = {\n equity_mint: 'Mint',\n equity_redemption: 'Redeem',\n alpaca_to_base: 'Alpaca \u2192 Raindex',\n base_to_alpaca: 'Raindex \u2192 Alpaca',\n};\nconst typeOf = (transfer) => (transfer.kind === 'usdc_bridge' ? transfer.direction : transfer.kind);\nconst typeLabel = (transfer) => TYPES[typeOf(transfer)] || transfer.kind || '\u2014';\nconst assetOf = (transfer) => (transfer.kind === 'usdc_bridge' || !transfer.symbol ? 'USDC' : transfer.symbol);\n\n// The SPA's statusStyle colours.\nconst statusClass = (status) => {\n const lower = String(status || '').toLowerCase();\n if (lower.includes('completed') || lower.includes('filled') || lower.includes('deposited') || lower.includes('confirmed')) return 'det-green';\n if (lower.includes('reconciled')) return 'det-amber';\n if (lower.includes('failed')) return 'det-red';\n if (lower.includes('cancelled')) return 'det-orange';\n return 'det-blue';\n};\nconst statusLabel = (status) => {\n const text = String(status || '\u2014').replace(/_/g, ' ');\n return text.charAt(0).toUpperCase() + text.slice(1);\n};\n\n// --------------------------------------------------------------------------\n// The dialog\n// --------------------------------------------------------------------------\n\n// Block explorers by the bot's chain wire name. A chain without one, or an\n// unknown chain, shows the hash unlinked: a link to the wrong explorer would\n// read as \"this tx does not exist\".\nconst EXPLORERS = {\n base: 'https://basescan.org',\n ethereum: 'https://etherscan.io',\n hyperevm: 'https://hyperevmscan.io',\n};\nconst shortHash = (hash) => `${escapeHtml(hash.slice(0, 10))}\u2026${escapeHtml(hash.slice(-8))}`;\nconst txLink = (hash, chain) =>\n EXPLORERS[chain]\n ? `${shortHash(hash)} \u2197`\n : `${shortHash(hash)}`;\n\n// The bot's event timeline, once the panel queries it as C: each step with\n// its time and fields, like the SPA's dialog.\n// Without it, the exporter's status log: one step per status change.\nconst events = entriesOf('C').sort((left, right) => (left.sequence ?? 0) - (right.sequence ?? 0));\nconst humanize = (step) => String(step || '').replace(/([a-z0-9])([A-Z])/g, '$1 $2');\nconst isTxHash = (value) => typeof value === 'string' && /^0x[0-9a-fA-F]{64}$/.test(value);\nconst eventValue = (key, value, chain) => {\n if (key === 'error') return `${escapeHtml(value)}`;\n if (isTxHash(value)) return txLink(value, chain);\n if (value !== null && typeof value === 'object') return `${escapeHtml(JSON.stringify(value))}`;\n return escapeHtml(value);\n};\n// A payload's own `chain` names where its hashes live; else the row's chain\n// (a trade's), else none, which leaves the hashes unlinked.\nconst eventFields = (payload, rowChain) => {\n const fields = payload && typeof payload === 'object' ? payload : {};\n const chain = typeof fields.chain === 'string' ? fields.chain : rowChain;\n return Object.entries(fields)\n .filter(([key]) => !/(_at|At|timestamp)$/.test(key))\n .map(([key, value]) => `
${escapeHtml(key.replace(/_/g, ' '))}${eventValue(key, value, chain)}
`)\n .join('');\n};\n\n// The board loads the newest 500 events of all rows, so an older row can have\n// only its later events here. Then the timeline starts after the row's first\n// status entry, and says so.\nconst timelineIncomplete = (history) =>\n history.length > 0 &&\n Math.min(...events.map((event) => event.time)) > Math.min(...history.map((entry) => entry.time)) + 120000;\n\nconst timeline = (history, chain) =>\n events.length > 0\n ? `\n
Event timeline
\n ${timelineIncomplete(history) ? '

Older events of this row are outside the newest 500 the board loads; the full timeline is in the SPA at liquidity.t0trade.com.

' : ''}\n
${events\n .map(\n (event) => `\n
\n \n
${escapeHtml(humanize(event.step))}
\n
${utc(event.time)}
\n ${eventFields(event.payload, chain)}\n
`\n )\n .join('')}
`\n : `\n
Status history
\n
${history\n .map(\n (entry) => `\n
\n \n
${escapeHtml(statusLabel(entry.status))}
\n
${utc(entry.time)}
\n ${entry.error ? `
${escapeHtml(entry.error)}
` : ''}\n
`\n )\n .join('')}
\n

From the exporter's status log. The full event timeline is in the SPA at liquidity.t0trade.com.

`;\n\n// valueHtml is trusted HTML: a caller escapes any data it puts in it.\nconst field = (name, valueHtml) => `
${escapeHtml(name)}${valueHtml}
`;\n\nconst tradeDialog = (trade) => {\n const [venueName, venueClass, onchain] = venue(trade.venue);\n const id = String(trade.id);\n // An onchain trade id is `chain:tx_hash:log_index` (OnChainTradeId), or\n // `tx_hash:log_index` from before the chain was part of it (Base).\n const parts = id.split(':');\n const [chain, hash, log] = parts.length === 3 ? parts : ['base', ...parts];\n const idCell =\n onchain && parts.length >= 2 && isTxHash(hash)\n ? `${txLink(hash, chain)} (${escapeHtml(chain)}, log ${escapeHtml(log)})`\n : `${escapeHtml(id)}`;\n return `\n
\n \n ${escapeHtml(trade.direction)}\n ${escapeHtml(trade.symbol)}\n ${decimal(trade.shares)} ${onchain ? 'wrapped shares' : 'shares'}\n ${escapeHtml(venueName)}\n \n \n
\n
\n
\n ${field('ID', idCell)}\n ${field('Occurred At', utc(trade.first))}\n ${field('Status', `${escapeHtml(statusLabel(trade.status))}${trade.error ? `
${escapeHtml(trade.error)}
` : ''}`)}\n
\n ${timeline(trade.history, onchain ? chain : null)}\n
`;\n};\n\nconst transferDialog = (transfer) => {\n return `\n
\n \n ${escapeHtml(typeLabel(transfer))}\n ${escapeHtml(assetOf(transfer))}\n ${decimal(transfer.amount)}\n \n \n
\n
\n
\n ${field('ID', `${escapeHtml(transfer.id)}`)}\n ${field('Started', utc(transfer.started_at || transfer.first))}\n ${field('Updated', utc(transfer.time))}\n ${field('Status', `${escapeHtml(statusLabel(transfer.status))}`)}\n
\n ${timeline(transfer.history, null)}\n
`;\n};\n\n// --------------------------------------------------------------------------\n// Open, refresh or close the one dialog\n// --------------------------------------------------------------------------\n\n// Kept on the element across renders: the plugin re-runs this code on every\n// refresh and query, into the same element.\nconst state = root.__det || (root.__det = { dismissed: null });\n\n// The variable can lag the close by a render (a refresh that was already\n// running), so an id the operator just closed is not reopened until the\n// variable has moved off it.\nif (state.dismissed !== null && state.dismissed !== wanted) state.dismissed = null;\n\n// Switching Environment keeps the tables' old rows on screen until the new\n// queries return, and those rows belong to the other project. Hold back\n// every row after a switch until a query has run (Grafana showed the panel\n// Loading) and then succeeded. Frames alone cannot tell: Grafana replays and\n// copies the old ones, and keeps them on an error. A Cloud Logging query\n// takes over a second, so the Loading state shows; if it does not, or shows\n// only on the switch render, the dialog says Loading\u2026 until the next\n// refresh, which fails safe. Known\n// limit: a query cancelled from the refresh picker after the switch also\n// ends Done with the old rows, and that releases the hold. The memory\n// lives on the window, not the element, so a remount (tab switch, view mode)\n// keeps it. The first render after the page loads is not held back: its rows\n// are for the environment it opened on.\nconst env = context.grafana.replaceVariables('${env}');\nconst loading = context.panelData?.state === 'Loading';\nconst failed = context.panelData?.state === 'Error';\nconst envState = window.__liqDetailEnv || (window.__liqDetailEnv = { env: undefined, held: false, sawLoading: false });\nconst switched = envState.env !== undefined && envState.env !== env;\nenvState.env = env;\nif (switched) {\n // A Loading state on this render can be a run that started before the\n // switch, so only a later one counts.\n envState.held = true;\n envState.sawLoading = false;\n} else if (envState.held) {\n // A query that failed does not count as the one that ran.\n if (loading) envState.sawLoading = true;\n else if (failed) envState.sawLoading = false;\n else if (envState.sawLoading) envState.held = false;\n}\n\n// A failed refresh can keep the old frames, so it shows the error, not a\n// row and its commands that may be out of date.\nconst trade = envState.held || failed ? null : latest(entriesOf('A'));\nconst transfer = trade || envState.held || failed ? null : latest(entriesOf('B'));\n// The dialog opens on the click, before the query returns, and says so when\n// the id has no entries in the panel's window.\nconst message = (text) => `\n
${escapeHtml(wanted)}
\n

${escapeHtml(text)}

`;\nconst queryError = () => {\n const errors = context.panelData?.errors || (context.panelData?.error ? [context.panelData.error] : []);\n return errors.map((error) => error.message).filter(Boolean).join('; ') || 'unknown error';\n};\nconst html = trade\n ? tradeDialog(trade)\n : transfer\n ? transferDialog(transfer)\n : wanted === ''\n ? null\n : failed\n ? message(`The Trades or Rebalances query failed (${queryError()}), so this row cannot be shown. Refresh the board to retry.`)\n : loading || envState.held\n ? message('Loading\u2026')\n : message('This id is not in the newest 500 status entries any more. Search the Logs tab for it.');\n\n// Reuse the dialog when it is still in the element, so a refresh with the\n// dialog open swaps its content without closing and reopening it.\nlet dialog = root.querySelector('dialog.det-dialog');\nif (!dialog) {\n dialog = document.createElement('dialog');\n dialog.className = 'det-dialog';\n root.appendChild(dialog);\n}\n\n// Property handlers, not listeners: each render replaces them instead of\n// stacking another copy.\ndialog.onclose = () => {\n state.dismissed = dialog.detId || null;\n dialog.detId = '';\n // An empty value, not null: removing the URL parameter leaves a textbox\n // variable at its old value, so the same row could not be reopened.\n if (context.grafana.replaceVariables('${detail}') !== '') {\n context.grafana.locationService.partial({ 'var-detail': '' }, true);\n }\n};\n// A click on the backdrop closes it, like the SPA.\ndialog.onclick = (event) => {\n if (event.target === dialog) dialog.close();\n};\n\nif (html === null || state.dismissed === wanted) {\n if (dialog.open) dialog.close();\n} else {\n if (dialog.detHtml !== html) {\n // A refresh that brings a new status keeps the operator's scroll.\n const body = dialog.querySelector('.det-dialog-body');\n const scroll = dialog.detId === wanted && body ? body.scrollTop : 0;\n dialog.innerHTML = html;\n dialog.detHtml = html;\n const next = dialog.querySelector('.det-dialog-body');\n if (next) next.scrollTop = scroll;\n }\n dialog.detId = wanted;\n dialog.querySelector('[data-close]').onclick = () => dialog.close();\n if (!dialog.open) dialog.showModal();\n}\n", + "styles": "/* Business Text \"styles\" for the Liquidity bot's trade and transfer detail\n dialog. Theme colours come in as --det-* variables set by detail.js;\n status, venue and side colours are the SPA's (Tailwind 500). */\n\n.det-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.det-muted { color: var(--det-muted); }\n.det-break { word-break: break-all; }\n\n.det-green { color: #22c55e; }\n.det-red { color: #ef4444; }\n.det-amber { color: #f59e0b; }\n.det-orange { color: #f97316; }\n.det-blue { color: #3b82f6; }\n.det-purple { color: #a855f7; }\n.det-pink { color: #ec4899; }\n\n/* Tailwind 700 on the light theme, so status text keeps a 4.5:1 contrast\n against the white dialog. */\n.det-light .det-green { color: #15803d; }\n.det-light .det-red { color: #b91c1c; }\n.det-light .det-amber { color: #b45309; }\n.det-light .det-orange { color: #c2410c; }\n.det-light .det-blue { color: #1d4ed8; }\n.det-light .det-purple { color: #7e22ce; }\n.det-light .det-pink { color: #be185d; }\n\n.det-dialog {\n width: min(36rem, 92vw);\n padding: 0;\n border: 1px solid var(--det-border);\n border-radius: 8px;\n background: var(--det-card);\n color: var(--det-text);\n font-size: 14px;\n white-space: normal;\n}\n\n.det-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n\n.det-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--det-border);\n}\n\n.det-title-line {\n display: flex;\n flex-wrap: wrap;\n align-items: center;\n gap: 8px;\n font-size: 16px;\n font-weight: 600;\n}\n\n.det-title-line .det-mono { font-size: 14px; font-weight: 400; }\n\n.det-close {\n border: 0;\n background: transparent;\n color: var(--det-muted);\n font-size: 20px;\n line-height: 1;\n cursor: pointer;\n}\n\n.det-close:hover { color: var(--det-text); }\n\n.det-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 14px;\n}\n\n.det-dialog-body a { color: #3b82f6; }\n.det-light .det-dialog-body a { color: #1d4ed8; }\n\n.det-fields { margin-bottom: 16px; }\n\n.det-field {\n display: flex;\n gap: 8px;\n padding: 2px 0;\n}\n\n.det-field > .det-muted { flex-shrink: 0; }\n\n.det-section-title {\n margin-bottom: 8px;\n font-weight: 600;\n}\n\n.det-timeline {\n margin-bottom: 8px;\n padding-left: 16px;\n border-left: 2px solid var(--det-border);\n}\n\n.det-step {\n position: relative;\n padding-bottom: 12px;\n}\n\n.det-dot {\n position: absolute;\n top: 6px;\n left: -21px;\n width: 8px;\n height: 8px;\n border-radius: 9999px;\n background: currentColor;\n}\n", + "wrap": false } } ] diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json b/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json index 239bcf02db..428d76e7ed 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json @@ -308,7 +308,7 @@ ], "panels": [ { - "id": 83, + "id": 84, "type": "marcusolsson-dynamictext-panel", "title": "", "description": "The SPA header: settings pills, Config, the CLI recovery guide, the deployed commit, uptime, and whether the bot is up: the exporter's liq_up, or the bot scrape's up on the bot source.", @@ -372,7 +372,7 @@ "hideTimeOverride": true }, { - "id": 84, + "id": 85, "type": "table", "title": "Log History", "description": "The bot's structured log lines, re-emitted by the exporter with real severity and target (the raw container stream is all-INFO). Filter with the level/category/target/search variables above. \u26a0\ufe0f Only the newest matching entries, about the panel's pixel width of them. Narrow the filters, or use the SPA for deep digs. DEBUG/TRACE are not shipped.", diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json b/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json index 77d9490337..d1982327dc 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json @@ -146,7 +146,7 @@ ], "panels": [ { - "id": 7, + "id": 8, "type": "marcusolsson-dynamictext-panel", "title": "", "description": "The SPA header: settings pills, Config, the CLI recovery guide, the deployed commit, uptime, and whether the bot is up: the exporter's liq_up, or the bot scrape's up on the bot source.", @@ -210,7 +210,7 @@ "hideTimeOverride": true }, { - "id": 8, + "id": 9, "type": "table", "title": "Raindex Orders", "description": "The SPA's Orders tab, columns in SPA order (Output | Input | Vault Balance | IO Ratio | Order Hash | Created). Shipped by the exporter's collect_orders() as one Cloud Logging row per order per poll cycle (logName liquidity-orders); insertId buckets by poll-hour so a still-active order dedupes to ~1 stored row/hour rather than one per 60s poll \u2014 with a 2h window that means a couple of rows per order is expected, not a bug. Created is the order's own on-chain creation time (jsonPayload.created_at), not the row's ingestion time. The datasource returns at most about the panel's pixel width in entries.", @@ -495,7 +495,7 @@ } }, { - "id": 9, + "id": 10, "type": "stat", "title": "", "description": "In-flight broker orders right now (/orders/pending).", @@ -577,7 +577,7 @@ } }, { - "id": 10, + "id": 11, "type": "stat", "title": "", "description": "Raindex orders owned by the bot (upstream REST API total).", @@ -659,7 +659,7 @@ } }, { - "id": 11, + "id": 12, "type": "stat", "title": "", "description": "1 when the bot's upstream REST API for Raindex orders is unavailable.", diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json b/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json index 23ec2e20b4..18c2fbc1d4 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json @@ -187,7 +187,7 @@ ], "panels": [ { - "id": 33, + "id": 34, "type": "marcusolsson-dynamictext-panel", "title": "", "description": "The SPA header: settings pills, Config, the CLI recovery guide, the deployed commit, uptime, and whether the bot is up: the exporter's liq_up, or the bot scrape's up on the bot source.", @@ -251,7 +251,7 @@ "hideTimeOverride": true }, { - "id": 34, + "id": 35, "type": "stat", "title": "Detection latency", "description": "p95, 24h. Fill on-chain \u2192 bot noticed. good \u226430s, warn \u2264120s.", @@ -340,7 +340,7 @@ } }, { - "id": 35, + "id": 36, "type": "stat", "title": "Exposure window", "description": "p95, 24h. Fill on-chain \u2192 hedge filled: how long capital stays exposed. good \u226460s, warn \u2264300s.", @@ -429,7 +429,7 @@ } }, { - "id": 36, + "id": 37, "type": "stat", "title": "Errors (24h)", "description": "ERROR-level log lines in 24h. good \u22640, warn \u226410. Money-at-risk lifecycle failures are broken out per event type in the 'Errors & warnings by module' table below.", @@ -518,7 +518,7 @@ } }, { - "id": 37, + "id": 38, "type": "stat", "title": "Oldest unhedged fill", "description": "Age of the oldest on-chain fill not yet hedged. good \u22645m, warn \u226430m; '\u2014' = nothing unhedged (the SPA shows 'none').", @@ -607,7 +607,7 @@ } }, { - "id": 38, + "id": 39, "type": "stat", "title": "Block lag", "description": "Chain blocks the fill-detection checkpoint trails by. good \u226430, warn \u2264300.", @@ -696,7 +696,7 @@ } }, { - "id": 39, + "id": 40, "type": "stat", "title": "", "description": "SPA secondary line: p50 detection latency.", @@ -782,7 +782,7 @@ } }, { - "id": 40, + "id": 41, "type": "stat", "title": "", "description": "SPA secondary line: fill sample count backing the detection latency percentiles.", @@ -868,7 +868,7 @@ } }, { - "id": 41, + "id": 42, "type": "stat", "title": "", "description": "SPA secondary line: p50 exposure-window latency.", @@ -954,7 +954,7 @@ } }, { - "id": 42, + "id": 43, "type": "stat", "title": "", "description": "SPA secondary line: hedge sample count backing the exposure-window percentiles.", @@ -1040,7 +1040,7 @@ } }, { - "id": 43, + "id": 44, "type": "stat", "title": "", "description": "SPA secondary line: WARN-level log lines in 24h.", @@ -1126,7 +1126,7 @@ } }, { - "id": 44, + "id": 45, "type": "stat", "title": "", "description": "SPA secondary line: money-at-risk lifecycle failure events in 24h (the 'Errors & warnings by module' table breaks these out by event type).", @@ -1212,7 +1212,7 @@ } }, { - "id": 45, + "id": 46, "type": "stat", "title": "", "description": "SPA secondary line: apalis job queues with a failed or killed job right now (instantaneous, not windowed) \u2014 the SPA folds killed into the same 'queue(s) failed' phrase.", @@ -1302,7 +1302,7 @@ } }, { - "id": 46, + "id": 47, "type": "stat", "title": "", "description": "SPA secondary line: on-chain fills still uncovered by a hedge, summed across symbols.", @@ -1388,7 +1388,7 @@ } }, { - "id": 47, + "id": 48, "type": "stat", "title": "", "description": "SPA secondary line: symbols with an open (unhedged) exposure \u2014 0 is the SPA's 'all fills hedged'.", @@ -1484,7 +1484,7 @@ } }, { - "id": 48, + "id": 49, "type": "stat", "title": "", "description": "SPA secondary line: age of the current block-lag sample.", @@ -1570,7 +1570,7 @@ } }, { - "id": 49, + "id": 50, "type": "stat", "title": "", "description": "SPA secondary line: order-fill monitor poll ticks skipped (overrun) in 24h.", @@ -1656,7 +1656,7 @@ } }, { - "id": 50, + "id": 51, "type": "bargauge", "title": "Hedge cycle stages (p95, 24h)", "description": "The SPA waterfall's aggregate form: p95 per stage. Per-cycle waterfalls live in the SPA.", @@ -1734,7 +1734,7 @@ } }, { - "id": 51, + "id": 52, "type": "timeseries", "title": "Latency percentiles over time \u2014 $stage", "description": "p50/p90/p99 of the selected hedge stage, from the exporter's rolling 24h summary scraped once a minute.", @@ -1890,7 +1890,7 @@ } }, { - "id": 52, + "id": 53, "type": "table", "title": "Errors & warnings by module (24h)", "description": "The SPA's per-target error/warning counts. The Logs tab has the lines themselves. Click a module to jump to the Logs board pre-filtered on it (the SPA's 'logs \u2192' button); sparkline cells are skipped as low-value/fiddly here.", @@ -2095,7 +2095,7 @@ } }, { - "id": 53, + "id": 54, "type": "bargauge", "title": "USDC rebalance stages (p95, 30d)", "description": "Stage timings for USDC rebalances. From the server's stageSummary; per-operation waterfalls live in the SPA.", @@ -2165,7 +2165,7 @@ } }, { - "id": 54, + "id": 55, "type": "bargauge", "title": "Equity rebalance stages (p95, 30d)", "description": "Stage timings for equity mints/redemptions.", @@ -2235,7 +2235,7 @@ } }, { - "id": 55, + "id": 56, "type": "table", "title": "Dependency health (24h)", "description": "Every RPC / broker operation: calls, errors, latency percentiles.", @@ -2395,7 +2395,7 @@ } }, { - "id": 56, + "id": 57, "type": "timeseries", "title": "CCTP attestation time", "description": "Duration of the most recent CCTP attestation per rebalance kind, sampled once per scrape. The SPA can chart the full attestationTrend history for the selected range; this gauge only has 'last value at scrape time', so the line only accrues from whenever this panel started being scraped forward \u2014 a short/flat history here does not mean attestations were fast, it means the metric is young.", @@ -2516,7 +2516,7 @@ } }, { - "id": 57, + "id": 58, "type": "timeseries", "title": "Block lag over time", "description": "The ingestion-health chart: how far fill detection trails the chain.", @@ -2587,7 +2587,7 @@ } }, { - "id": 58, + "id": 59, "type": "stat", "title": "Poll cycles (24h)", "description": "Order-fill monitor poll cycles.", @@ -2668,7 +2668,7 @@ } }, { - "id": 59, + "id": 60, "type": "stat", "title": "Poll errors (24h)", "description": "Failed poll cycles.", @@ -2753,7 +2753,7 @@ } }, { - "id": 60, + "id": 61, "type": "stat", "title": "Skipped ticks (24h)", "description": "Poll ticks skipped (overrun).", @@ -2838,7 +2838,7 @@ } }, { - "id": 61, + "id": 62, "type": "bargauge", "title": "Poll duration percentiles", "description": "The monitor's own cycle duration.", @@ -2908,7 +2908,7 @@ } }, { - "id": 62, + "id": 63, "type": "table", "title": "Job queues", "description": "The bot's apalis job queues (instantaneous whole-table counts, not windowed). killed or failed above zero is the SPA's critical Errors state.", @@ -3094,7 +3094,7 @@ } }, { - "id": 63, + "id": 64, "type": "row", "title": "Hedge execution quality (native bot metrics)", "collapsed": true, @@ -3106,7 +3106,7 @@ }, "panels": [ { - "id": 64, + "id": 65, "type": "timeseries", "title": "Hedge fill latency percentiles", "description": "Wall-clock seconds from hedge order placement to confirmed fill. The app builds these as a metrics-rs summary (no explicit buckets), so the quantiles come straight off the {quantile=...} label rather than from histogram_quantile. This is the metrics-side view of the SPA's 'Latency percentiles over time' chart \u2014 that chart's per-stage waterfall (detection / decision / submission / execution) comes from the telemetry tables in SQLite and has no metric behind it.", @@ -3218,7 +3218,7 @@ } }, { - "id": 65, + "id": 66, "type": "timeseries", "title": "Hedge fill latency by symbol (1h average)", "description": "Average seconds between placing a hedge and it filling, per symbol, derived from the summary's _sum / _count. Spikes mean capital sits exposed longer on that name.", @@ -3293,7 +3293,7 @@ } }, { - "id": 66, + "id": 67, "type": "timeseries", "title": "Broker errors per hour, by kind", "description": "Alpaca API failures, split by kind (e.g. place_order_failed) and symbol. Empty is the healthy state.", @@ -3363,7 +3363,7 @@ } }, { - "id": 67, + "id": 68, "type": "timeseries", "title": "Hedge scans skipped, by reason", "description": "Extended-hours buys the position scan dropped before enqueueing a hedge job, because reference-price resolution or crossing failed. Each one is an on-chain fill that did not get hedged on that pass.", @@ -3433,7 +3433,7 @@ } }, { - "id": 68, + "id": 69, "type": "timeseries", "title": "Hedges dead-lettered, by reason", "description": "Hedge attempts the process gave up on: a permanent or retry-budget-exhausted pricing failure, or broker rate-limiting that outlived the reschedule budget. Anything here is an unhedged fill that needs a human.", @@ -3503,7 +3503,7 @@ } }, { - "id": 69, + "id": 70, "type": "timeseries", "title": "Which reference priced each extended-hours limit", "description": "hedge_price_source_total by source (primary_quote / mark / delayed_sip_quote) and path (ordinary_extended / close_flatten). Shows which fallback legs are load-bearing \u2014 a rise in delayed_sip_quote means we are pricing off stale data.", @@ -3577,7 +3577,7 @@ ] }, { - "id": 70, + "id": 71, "type": "row", "title": "Close-flatten (end-of-session flattening) (native bot metrics)", "collapsed": true, @@ -3589,7 +3589,7 @@ }, "panels": [ { - "id": 71, + "id": 72, "type": "timeseries", "title": "Attempts, by post-close gap reason", "description": "Close-flatten hedge pricing attempts, by the post-close gap reason that triggered them.", @@ -3659,7 +3659,7 @@ } }, { - "id": 72, + "id": 73, "type": "timeseries", "title": "Blocked before submission, by reason", "description": "Close-flatten attempts stopped before they reached the broker. close_time_unknown carries no symbol label; the rest are per symbol.", @@ -3729,7 +3729,7 @@ } }, { - "id": 73, + "id": 74, "type": "timeseries", "title": "Placements, by cross applied", "description": "Limit orders priced and cleared for submission, bucketed to whole percent of cross. Counts attempts, not orders: a retried or re-driven placement of the same hedge counts again, and an attempt the broker later rejects still counts.", @@ -3799,7 +3799,7 @@ } }, { - "id": 74, + "id": 75, "type": "timeseries", "title": "Outcomes", "description": "Terminal broker-state dispatches for close-flatten placements (filled / cancelled / failed). A placement can be observed again until its follow-up job commits the terminal aggregate transition, so treat these as dispatches rather than a strict order count.", @@ -3871,7 +3871,7 @@ ] }, { - "id": 75, + "id": 76, "type": "row", "title": "Data integrity & ingestion health (native bot metrics)", "collapsed": true, @@ -3883,7 +3883,7 @@ }, "panels": [ { - "id": 76, + "id": 77, "type": "stat", "title": "Ambiguous settlements (24h)", "description": "Inventory settlements quarantined because one transaction emitted multiple OperatorDeposits or multiple OperatorWithdraws and the legs could not be safely paired. Each one is a held-back settlement.", @@ -3973,7 +3973,7 @@ } }, { - "id": 77, + "id": 78, "type": "stat", "title": "Unpaired settlement legs (24h)", "description": "OperatorDeposit / OperatorWithdraw legs with no same-transaction counterpart in the batch.", @@ -4063,7 +4063,7 @@ } }, { - "id": 78, + "id": 79, "type": "stat", "title": "Unusable snapshot marks (24h)", "description": "Nonzero equity balances captured with a missing or stale USD mark. These make the portfolio snapshot \u2014 and therefore the PnL replay in the SPA \u2014 incomplete for that interval.", @@ -4153,7 +4153,7 @@ } }, { - "id": 79, + "id": 80, "type": "stat", "title": "PnL ledger checkpoint lag", "description": "How many events the PnL ledger's checkpoint trails the event store by. The PnL numbers themselves are computed by the backend from SQLite and are not exported \u2014 this gauge is the one PnL signal that reaches Grafana, and it answers 'is the PnL panel's input keeping up?'.", @@ -4247,7 +4247,7 @@ } }, { - "id": 80, + "id": 81, "type": "timeseries", "title": "Scrape health", "description": "The Ops Agent's own view of the bot's /metrics endpoint: 1 = scraped successfully, 0 = the endpoint did not answer. Every other panel on this dashboard goes blank while this is 0, so check it first before concluding the bot is idle.", @@ -4320,7 +4320,7 @@ } }, { - "id": 81, + "id": 82, "type": "timeseries", "title": "Scrape duration and series count", "description": "How long the scrape takes and how many series come back. A series count that collapses to near zero while up=1 means the bot restarted and its counters reset.", @@ -4411,7 +4411,7 @@ } }, { - "id": 82, + "id": 83, "type": "timeseries", "title": "Bot-gas redrives and PnL catch-up failures", "description": "bot_gas_redrive_total: receipt-cost enqueue failures redriven instead of failing the triggering job. pnl_ledger_catch_up_failures_total: the PnL ledger failing to catch up to the event store. Both are self-healing retries \u2014 a steady rate means something underneath is not healing.", diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json b/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json index 627fa4defa..eb5277c175 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json @@ -193,7 +193,7 @@ ], "panels": [ { - "id": 12, + "id": 13, "type": "marcusolsson-dynamictext-panel", "title": "", "description": "The SPA header: settings pills, Config, the CLI recovery guide, the deployed commit, uptime, and whether the bot is up: the exporter's liq_up, or the bot scrape's up on the bot source.", @@ -257,7 +257,7 @@ "hideTimeOverride": true }, { - "id": 13, + "id": 14, "type": "stat", "title": "", "description": "Gross minus costs plus revenues. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -351,7 +351,7 @@ } }, { - "id": 14, + "id": 15, "type": "stat", "title": "", "description": "Closed-lot fill replay before costs. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -445,7 +445,7 @@ } }, { - "id": 15, + "id": 16, "type": "stat", "title": "", "description": "Cost/revenue ledger entries. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -531,7 +531,7 @@ } }, { - "id": 16, + "id": 17, "type": "stat", "title": "", "description": "Dividends and broker ledger credits. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -625,7 +625,7 @@ } }, { - "id": 17, + "id": 18, "type": "stat", "title": "", "description": "Broker hedge stream. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -719,7 +719,7 @@ } }, { - "id": 18, + "id": 19, "type": "stat", "title": "", "description": "Users passively crossed open inventory. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -813,7 +813,7 @@ } }, { - "id": 19, + "id": 20, "type": "stat", "title": "", "description": "Closed delayed or offchain-origin exposure. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -907,7 +907,7 @@ } }, { - "id": 20, + "id": 21, "type": "stat", "title": "", "description": "Requires historical snapshots. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -1001,7 +1001,7 @@ } }, { - "id": 21, + "id": 22, "type": "stat", "title": "", "description": "All directional streams together. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -1095,7 +1095,7 @@ } }, { - "id": 22, + "id": 23, "type": "stat", "title": "", "description": "Every stream summed. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -1189,7 +1189,7 @@ } }, { - "id": 23, + "id": 24, "type": "stat", "title": "", "description": "Open replay inventory in shares (the SPA's muted tile).", @@ -1271,7 +1271,7 @@ } }, { - "id": 24, + "id": 25, "type": "stat", "title": "", "description": "Total fills in the PnL database sample.", @@ -1353,7 +1353,7 @@ } }, { - "id": 25, + "id": 26, "type": "stat", "title": "", "description": "Average deployed capital from daily snapshots \u2014 computed by the bot, never rendered by the SPA.", @@ -1435,7 +1435,7 @@ } }, { - "id": 26, + "id": 27, "type": "stat", "title": "", "description": "Annualized return on that capital \u2014 computed by the bot, never rendered by the SPA.", @@ -1525,7 +1525,7 @@ } }, { - "id": 27, + "id": 28, "type": "stat", "title": "", "description": "Days of snapshot coverage behind the capital figures.", @@ -1607,7 +1607,7 @@ } }, { - "id": 28, + "id": 29, "type": "table", "title": "Per Asset PnL", "description": "The SPA's Per Asset PnL table (closed-lot replay per symbol; Lots = matched lot count; Volume = both legs of the matched shares, without the SPA's USD sub-line).", @@ -2137,7 +2137,7 @@ } }, { - "id": 29, + "id": 30, "type": "barchart", "title": "PnL by Asset per Time Unit", "description": "One bar per calendar day, split by asset and stacked, straight from the bot's daily PnL buckets. A day with no closed lot for an asset is a zero, as in the SPA.", @@ -2284,7 +2284,7 @@ } }, { - "id": 30, + "id": 31, "type": "barchart", "title": "PnL by PnL-Stream per Time Unit", "description": "The same days split by PnL stream, the SPA's four component streams only, since directional_exposure and total are tile-only aggregates there and stacking them beside their own components double-counts.", @@ -2492,7 +2492,7 @@ } }, { - "id": 31, + "id": 32, "type": "barchart", "title": "Cumulative PnL by Asset", "description": "Running total per asset through the selected range. Unstacked: the SPA draws one independent line per asset here, not a stack. The last day's bars sum to the Total PnL tile above.", @@ -2639,7 +2639,7 @@ } }, { - "id": 32, + "id": 33, "type": "barchart", "title": "Cumulative PnL by PnL-Stream", "description": "Running total per stream, unstacked like the SPA's lines. Grafana has no categorical-axis line chart, so these are bars over the same day buckets: the numbers are the SPA's, the shape is not.", diff --git a/observability/gen-t0-liquidity.py b/observability/gen-t0-liquidity.py index a9b3a1073f..8f9bf1a7df 100644 --- a/observability/gen-t0-liquidity.py +++ b/observability/gen-t0-liquidity.py @@ -664,7 +664,7 @@ def tab_links(active): return links -def pills(y): +def pills(y, w=24): """The SPA's HeaderBar and SettingsBar as one Business Text row, repeated on every tab like the SPA repeats its header: the settings pills (broker, Equity and USDC targets with their bands, Trigger, @@ -678,6 +678,9 @@ def pills(y): one in the lookback window. The recovery guide is static data exported from the SPA (liquidity-panels/recovery-guide.json) and prepended to the script. + + w: the Dashboard tab narrows it to 23 to fit the detail panel beside + it (see detail_panel()). """ def named(expr, name): return f'label_replace({expr}, "k", "{name}", "", "")' @@ -713,7 +716,7 @@ def named(expr, name): "datasource": CM, "targets": [promql(expr, instant=True)], "transparent": True, - "gridPos": {"h": 1, "w": 24, "x": 0, "y": y}, + "gridPos": {"h": 1, "w": w, "x": 0, "y": y}, "transformations": [ {"id": "labelsToFields", "options": {}}, {"id": "merge", "options": {}}, @@ -823,7 +826,7 @@ def make_dashboard(uid, title, description, panels, links, variables): # stacked right. # ========================================================================== panels = [] -panels += pills(0) +panels += pills(0, w=23) def native_table(title, desc, expr, row_label, columns, w, h, x, y, first_col, sort_by=None, widths=None, bars=(), @@ -1222,16 +1225,96 @@ def per_chain(labelled, base): # fit left it a header and a scrollbar on windows under ~1600px tall. panels += native_inventory(w=13, x=0, y=1, heights=(8, 9, 35)) +# A row's detail dialog is the hidden `detail` variable: a table row's link +# sets it to the row's id, the detail panel picks that id out of the Trades +# and Rebalances tables' own results (it runs no query of its own) and +# opens the dialog, and closing the dialog clears the variable again. +# Do not add $detail to its A and B: they are the tables' results, not +# queries. +# +# The id is read by the field NAME. The groupBy key keeps the raw field name +# (jsonPayload.id); organize's rename only set its display name, and the Id +# column's override replaces that with a blank header. ${__url.params:...} +# already starts with "?", so the URL appends straight to it. +DETAIL_URL = ('/d/${__dashboard.uid}/${__url.params:exclude:var-detail}' + '&var-detail=${__data.fields["jsonPayload.id"]}') + + +DETAIL_VAR = {"type": "textbox", "name": "detail", "hide": 2, "query": "", + "current": {"text": "", "value": ""}, "options": []} + + +def detail_panel(x, y, trades_id, transfers_id): + """The SPA's trade and transfer detail dialogs, opened from a row of the + Trades or Rebalances table (see DETAIL_URL). + + The panel itself draws nothing; it takes the header row's last column so + the tab's panel heights stay as they are. It reads the Trades and + Rebalances tables' own results (the newest 500 entries of each log) + through the Dashboard datasource, so the board scans each log once per + refresh, and not per click: a query filtered on one id still scans the + whole window (about 13s for 30 days), so a click only picks the id out + of data the browser already has and the dialog opens at once. The + Dashboard datasource keeps each table's refId: Trades A, Rebalances B. The queries do not use + `$detail`, so a click does not re-run them; the script reads `${detail}`, + which makes Grafana redraw the panel when the variable changes. + liquidity-panels/detail.js builds the dialog. renderMode "data" hands + the script both frames, where "allRows" would draw a frame picker. + The status-history order is prepended from status-history.js. + """ + with open(os.path.join(HERE, "liquidity-panels", "status-history.js")) as f: + # Its export line is for the SPA test; afterRender is not a module. + history = "".join(line for line in f if not line.startswith("export ")) + with open(os.path.join(HERE, "liquidity-panels", "detail.js")) as f: + after_render = history + "\n\n" + f.read() + with open(os.path.join(HERE, "liquidity-panels", "detail.css")) as f: + styles = f.read() + return { + "id": nid(), "type": "marcusolsson-dynamictext-panel", "title": "", + "description": "The detail dialog of a Trades or Rebalances row: " + "open it from the row's ⓘ.", + "datasource": {"type": "datasource", "uid": "-- Mixed --"}, + "timeFrom": "30d", "hideTimeOverride": True, + "maxDataPoints": 500, + "targets": [ + *({"refId": ref, "panelId": panel_id, "withTransforms": False, + "datasource": {"type": "datasource", "uid": "-- Dashboard --"}} + for ref, panel_id in (("A", trades_id), ("B", transfers_id))), + ], + "transparent": True, + "gridPos": {"h": 1, "w": 1, "x": x, "y": y}, + "options": { + "renderMode": "data", + "editor": {"format": "auto", "language": "html"}, + "content": "
", + "defaultContent": "
", + "helpers": "", + "afterRender": after_render, + "styles": styles, + "wrap": False, + }, + } + + def latest_status_table(title, desc, log, fields, shown, w, h, x, y, body_regex=None, number_columns=None, time_columns=(), - overrides=(), sort_by=None): + overrides=(), sort_by=None, ref="A"): """A Cloud Logging table with one row per entity at its latest status, - like the SPA's Trade History and Cross-venue Transfers cards. + like the SPA's Trade History and Cross-venue Transfers cards (here + Trades and Rebalances). The exporter writes one log entry per status change (insertId `:`), so the raw feed holds several rows per trade or transfer. The plugin returns entries newest first, so groupBy on - jsonPayload.id with `first` keeps each entity's latest status. + jsonPayload.id with `first` keeps each entity's entry with the highest + timestamp. That is usually the latest status, but a status's timestamp + can be earlier than the one before it: a USDC bridge stamps + WithdrawalComplete with confirmed_at and the BridgingSubmitting after it + with the older initiated_at, so such a row can show withdrawing. The + detail dialog orders a bridge's statuses by its lifecycle instead. + + The id stays as the first column, an ⓘ that opens the row's detail + dialog (see DETAIL_URL). fields: {payload leaf: column name}; the entry's own timestamp is always kept, as fields["timestamp"]. @@ -1257,6 +1340,9 @@ def latest_status_table(title, desc, log, fields, shown, w, h, x, y, for name in time_columns]) return { "id": nid(), "type": "table", "title": title, "description": desc, + # Without a version Grafana runs the table panel's old-version + # migration on load, which can drop field overrides. + "pluginVersion": "13.1.0", "datasource": CL, "timeFrom": "30d", # The plugin takes the row limit from maxDataPoints, which Grafana @@ -1264,7 +1350,7 @@ def latest_status_table(title, desc, log, fields, shown, w, h, x, y, # 500 newest entries load in about 1.5s and still leave well over # the SPA's 100 rows after grouping by id. "maxDataPoints": 500, - "targets": [cloudlog(log)], + "targets": [cloudlog(log, ref=ref)], "gridPos": {"h": h, "w": w, "x": x, "y": y}, "transformations": [ # `labels` is a JSON object per row; extractFields lifts each @@ -1292,9 +1378,10 @@ def latest_status_table(title, desc, log, fields, shown, w, h, x, y, for name in shown}, }}}, {"id": "organize", - "options": {"excludeByName": {"Id": True}, - "indexByName": {f"{name} (first)": index - for index, name in enumerate(shown)}, + "options": {"excludeByName": {}, + "indexByName": {"Id": 0, **{ + f"{name} (first)": index + 1 + for index, name in enumerate(shown)}}, "renameByName": {f"{name} (first)": name for name in shown}}}, {"id": "sortBy", "options": {"fields": {}, "sort": [ @@ -1310,14 +1397,31 @@ def latest_status_table(title, desc, log, fields, shown, w, h, x, y, *[{"matcher": {"id": "byName", "options": name}, "properties": TIME_FMT} for name in [fields["timestamp"], *time_columns]], + # The ⓘ is the link's title: a data-links cell draws the + # title in place of the value. A value mapping would do the + # same, but it would also turn ${__data.fields[...]} into + # the ⓘ, since data links read a field's display text. + {"matcher": {"id": "byName", "options": "Id"}, + "properties": [ + {"id": "displayName", "value": " "}, + # Grafana's minimum column width; see native_inventory. + {"id": "custom.width", "value": 50}, + {"id": "custom.align", "value": "center"}, + {"id": "custom.filterable", "value": False}, + {"id": "custom.cellOptions", "value": {"type": "data-links"}}, + {"id": "links", "value": [{ + "title": "ⓘ", "url": DETAIL_URL, + "targetBlank": False, "oneClick": True}]}]}, *overrides, ], }, - "options": {"showHeader": True, "cellHeight": "sm"}, + # Medium rows: small ones read crowded, large ones waste the card. + "options": {"showHeader": True, "cellHeight": "md"}, } def column(name, px, mappings=None, color_text=False): + # px None: the column stretches over the rest of the card. properties = [{"id": "custom.width", "value": px}] if px else [] if mappings: properties.append({"id": "mappings", "value": mappings}) @@ -1329,43 +1433,45 @@ def column(name, px, mappings=None, color_text=False): panels.append(latest_status_table( - "Trade History", + "Trades", "Direct Raindex fills, fills routed through supported adapters such as " "Bebop, and the corresponding Alpaca hedge trades placed to offset " - "exposure. One row per trade at its latest status. Built from the " - "newest 500 status entries, so a busy period can push older rows out " - "of the 30-day window.", + "exposure. One row per trade at its latest status; its ⓘ " + "opens the details. Built from the newest 500 status entries, so a " + "busy period can push older rows out of the 30-day window.", 'logName="projects/$env/logs/liquidity-trades"', - fields={"timestamp": "Time", "symbol": "Asset", "venue": "Venue", + # One timestamp, first: when the row last changed status. + fields={"timestamp": "Last updated", "symbol": "Asset", "venue": "Venue", "direction": "Side", "shares": "Size", "status": "Status"}, - shown=["Time", "Asset", "Venue", "Side", "Size", "Status"], + shown=["Last updated", "Asset", "Venue", "Side", "Size", "Status"], w=11, h=TRADES_H, x=13, y=1, number_columns={"Size": 3}, overrides=[ - # No width on the last column: it takes what is left, so the - # table fills the card like the SPA's. - column("Time", 170), column("Asset", 80), - column("Venue", 110, VENUE_MAPPINGS, color_text=True), - column("Side", 70, SIDE_MAPPINGS, color_text=True), - column("Size", 110), + column("Last updated", 185), column("Asset", 85), + column("Venue", 115, VENUE_MAPPINGS, color_text=True), + column("Side", 75, SIDE_MAPPINGS, color_text=True), + column("Size", 100), column("Status", None, STATUS_CAP_MAPPINGS, color_text=True), ], )) panels.append(latest_status_table( - "Cross-venue Transfers", + "Rebalances", "Asset movements between venues to rebalance inventory: equity mints " "(Alpaca to onchain), redemptions (onchain to Alpaca), and USDC bridges " - "(Base/Ethereum via CCTP). One row per transfer at its latest status. " + "(Base/Ethereum via CCTP). One row per transfer at its latest status; " + "its ⓘ opens the details. " "A USDC bridge can show the previous status for a while: the exporter " "stamps each status with the transfer's updatedAt, which is not always " "later than the one before (bridging starts from initiated_at). " "Built from the newest 500 status entries, so a busy period can push " "older rows out of the 30-day window.", 'logName="projects/$env/logs/liquidity-transfers"', - fields={"timestamp": "Updated", "started_at": "Started", "symbol": "Asset", + # One timestamp, first: when the row last changed status. The start + # time is in the row's detail dialog. + fields={"timestamp": "Last updated", "symbol": "Asset", "amount": "Amount", "status": "Status"}, - shown=["Started", "Type", "Asset", "Amount", "Status", "Updated"], + shown=["Last updated", "Type", "Asset", "Amount", "Status"], # Type reads the direction for a USDC bridge and the kind for an equity # transfer, like the SPA's transferTypeLabel. A bridge's kind is # usdc_bridge and an equity transfer's direction is "", so each row can @@ -1374,18 +1480,23 @@ def column(name, px, mappings=None, color_text=False): '|equity_mint|equity_redemption)"', w=11, h=TRANSFERS_H, x=13, y=1 + TRADES_H, number_columns={"Amount": 3}, - time_columns=["Started"], overrides=[ - column("Started", 135), column("Type", 125, TYPE_MAPPINGS), + column("Last updated", 185), column("Type", 165, TYPE_MAPPINGS), # A USDC bridge carries no symbol; the SPA's Asset column reads # "USDC" for it. - column("Asset", 60, [{"type": "special", "options": { + column("Asset", 80, [{"type": "special", "options": { "match": "empty", "result": {"text": "USDC", "index": 0}}}]), - column("Amount", 85), - column("Status", 110, STATUS_DOT_MAPPINGS, color_text=True), - column("Updated", None), + column("Amount", 115), + column("Status", None, STATUS_DOT_MAPPINGS, color_text=True), ], + # B, not A: the detail panel reads both tables through the Dashboard + # datasource, which keeps each table's refId. + ref="B", )) +panels.append(detail_panel( + x=23, y=0, + trades_id=next(p["id"] for p in panels if p.get("title") == "Trades"), + transfers_id=next(p["id"] for p in panels if p.get("title") == "Rebalances"))) dashboards.append(make_dashboard( @@ -1404,7 +1515,7 @@ def column(name, px, mappings=None, color_text=False): "compute start-iap-tunnel 8080 " "--local-host-port=localhost:8080 --zone europe-west3-b " "--project $env (VM name = project id)", - panels, tab_links("Dashboard"), [ENV_VAR, SOURCE_VAR])) + panels, tab_links("Dashboard"), [ENV_VAR, SOURCE_VAR, DETAIL_VAR])) # ========================================================================== # Tab 2: Orders — the SPA's Raindex Orders table (+ ops extras). diff --git a/observability/liquidity-panels/detail.css b/observability/liquidity-panels/detail.css new file mode 100644 index 0000000000..b89aee5f3f --- /dev/null +++ b/observability/liquidity-panels/detail.css @@ -0,0 +1,114 @@ +/* Business Text "styles" for the Liquidity bot's trade and transfer detail + dialog. Theme colours come in as --det-* variables set by detail.js; + status, venue and side colours are the SPA's (Tailwind 500). */ + +.det-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; } +.det-muted { color: var(--det-muted); } +.det-break { word-break: break-all; } + +.det-green { color: #22c55e; } +.det-red { color: #ef4444; } +.det-amber { color: #f59e0b; } +.det-orange { color: #f97316; } +.det-blue { color: #3b82f6; } +.det-purple { color: #a855f7; } +.det-pink { color: #ec4899; } + +/* Tailwind 700 on the light theme, so status text keeps a 4.5:1 contrast + against the white dialog. */ +.det-light .det-green { color: #15803d; } +.det-light .det-red { color: #b91c1c; } +.det-light .det-amber { color: #b45309; } +.det-light .det-orange { color: #c2410c; } +.det-light .det-blue { color: #1d4ed8; } +.det-light .det-purple { color: #7e22ce; } +.det-light .det-pink { color: #be185d; } + +.det-dialog { + width: min(36rem, 92vw); + padding: 0; + border: 1px solid var(--det-border); + border-radius: 8px; + background: var(--det-card); + color: var(--det-text); + font-size: 14px; + white-space: normal; +} + +.det-dialog::backdrop { background: rgba(0, 0, 0, 0.5); } + +.det-dialog-head { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 20px; + border-bottom: 1px solid var(--det-border); +} + +.det-title-line { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; + font-size: 16px; + font-weight: 600; +} + +.det-title-line .det-mono { font-size: 14px; font-weight: 400; } + +.det-close { + border: 0; + background: transparent; + color: var(--det-muted); + font-size: 20px; + line-height: 1; + cursor: pointer; +} + +.det-close:hover { color: var(--det-text); } + +.det-dialog-body { + max-height: 70vh; + overflow-y: auto; + padding: 16px 20px; + font-size: 14px; +} + +.det-dialog-body a { color: #3b82f6; } +.det-light .det-dialog-body a { color: #1d4ed8; } + +.det-fields { margin-bottom: 16px; } + +.det-field { + display: flex; + gap: 8px; + padding: 2px 0; +} + +.det-field > .det-muted { flex-shrink: 0; } + +.det-section-title { + margin-bottom: 8px; + font-weight: 600; +} + +.det-timeline { + margin-bottom: 8px; + padding-left: 16px; + border-left: 2px solid var(--det-border); +} + +.det-step { + position: relative; + padding-bottom: 12px; +} + +.det-dot { + position: absolute; + top: 6px; + left: -21px; + width: 8px; + height: 8px; + border-radius: 9999px; + background: currentColor; +} diff --git a/observability/liquidity-panels/detail.js b/observability/liquidity-panels/detail.js new file mode 100644 index 0000000000..e8d4302312 --- /dev/null +++ b/observability/liquidity-panels/detail.js @@ -0,0 +1,390 @@ +// Business Text "afterRender" code for the Liquidity bot's detail panel: the +// SPA's trade and transfer detail dialogs (trade-history-panel.svelte, +// transfer-panel.svelte in ST0x-Technology/st0x.liquidity) for the row whose +// ⓘ was clicked in the Trades or Rebalances table. +// +// The click sets the hidden `detail` variable to the row's id. A and B are +// the Trades and Rebalances tables' own results (the newest 500 status +// entries of each log, through the Dashboard datasource), and the script +// picks the id's entries out of them. C, the bot's event timeline, has no +// query until the bot logs its events, so the dialog shows the status +// history. Closing the dialog clears the variable. +// +// The panel is one empty column of the header row; only the dialog shows. +// +// latest is not defined here: the generator prepends it from +// status-history.js (see detail_panel()). + +const theme = context.grafana.theme; +const root = context.element; +root.style.setProperty('--det-muted', theme.colors.text.secondary); +root.style.setProperty('--det-border', theme.colors.border.weak); +root.style.setProperty('--det-stripe', theme.colors.background.secondary); +root.style.setProperty('--det-card', theme.colors.background.primary); +root.style.setProperty('--det-text', theme.colors.text.primary); +// Light theme swaps the status colours for darker shades (detail.css). +root.classList.toggle('det-light', !theme.isDark); + +// Alternate rows in the board's Trades and Rebalances +// tables, like the SPA's. Grafana's table has no striping option, but its +// grid marks every row rdg-row-even or rdg-row-odd. This panel lives only on +// the Dashboard tab, so the rule goes into the page while the panel is +// mounted and comes out when the board is left. Grafana can mount the panel +// on a new element (tab switch, view mode), so each render takes the rule +// over for its own element and restarts the watcher, and a watcher removes +// the rule only while its element still owns it. +let stripes = document.getElementById('liquidity-table-stripes'); +if (!stripes) { + stripes = document.createElement('style'); + stripes.id = 'liquidity-table-stripes'; + document.head.appendChild(stripes); +} +stripes.textContent = `.rdg-row-even { background: ${theme.isDark ? 'rgba(255, 255, 255, 0.03)' : 'rgba(0, 0, 0, 0.03)'}; }`; +root.__stripesOwner = root.__stripesOwner || Math.random().toString(36).slice(2); +stripes.dataset.owner = root.__stripesOwner; +clearInterval(root.__stripesWatch); +root.__stripesWatch = setInterval(() => { + if (root.isConnected) return; + clearInterval(root.__stripesWatch); + if (stripes.dataset.owner === root.__stripesOwner) stripes.remove(); +}, 2000); + +const escapeHtml = (text) => + String(text ?? '').replace(/[&<>"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[char]); + +// The id the tables asked for. Entries of any other id are a previous +// query's data, still shown while the new one loads. +const wanted = context.grafana.replaceVariables('${detail}'); + +// One entry per log line: its time and the exporter's payload. The plugin +// ships the payload as the JSON body and again as flattened +// `jsonPayload.` labels; the labels are the fallback. +const entriesOf = (refId) => { + const frame = (context.panelData?.series || []).find((series) => series.refId === refId); + if (!frame) return []; + const column = (name) => frame.fields.find((field) => field.name === name); + const times = column('timestamp'); + const bodies = column('body'); + const labels = column('labels'); + const count = times ? times.values.length : 0; + // A JSON object, or null for anything else (no column, bad JSON, a string). + const parseObject = (value) => { + if (value !== null && typeof value === 'object') return value; + try { + const parsed = JSON.parse(value); + return parsed !== null && typeof parsed === 'object' ? parsed : null; + } catch (error) { + return null; + } + }; + const fromLabels = (index) => { + const payload = {}; + const flat = labels ? parseObject(labels.values[index]) : null; + for (const [key, value] of Object.entries(flat || {})) { + if (key.startsWith('jsonPayload.')) payload[key.slice('jsonPayload.'.length)] = value; + } + return payload; + }; + const entries = []; + for (let index = 0; index < count; index++) { + const payload = (bodies && parseObject(bodies.values[index])) || fromLabels(index); + if (payload.id && String(payload.id) === wanted) entries.push({ time: Number(times.values[index]), ...payload }); + } + return entries; +}; + +// The SPA's formatUtc: "Oct 6, 12:24:39 UTC". +const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; +const utc = (value) => { + const date = new Date(value); + if (Number.isNaN(date.getTime())) return '—'; + const pad = (number) => String(number).padStart(2, '0'); + return `${MONTHS[date.getUTCMonth()]} ${date.getUTCDate()}, ${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())} UTC`; +}; + +// The SPA's formatDecimal(value, 3): fixed decimals, comma groups, and the +// raw value when it does not parse. +const decimal = (value) => { + const number = Number(value); + if (value === '' || value === null || value === undefined || Number.isNaN(number)) return escapeHtml(value || '—'); + return number.toLocaleString('en-US', { minimumFractionDigits: 3, maximumFractionDigits: 3 }); +}; + +const VENUES = { + raindex: ['Raindex', 'det-blue', true], + bebop: ['Bebop', 'det-purple', true], + uniswap_v4: ['Uniswap v4', 'det-pink', true], + unknown_onchain: ['Unknown Onchain', '', true], + alpaca: ['Alpaca', 'det-orange', false], + dry_run: ['DryRun', '', false], +}; +const venue = (key) => VENUES[key] || [key || '—', '', false]; + +const TYPES = { + equity_mint: 'Mint', + equity_redemption: 'Redeem', + alpaca_to_base: 'Alpaca → Raindex', + base_to_alpaca: 'Raindex → Alpaca', +}; +const typeOf = (transfer) => (transfer.kind === 'usdc_bridge' ? transfer.direction : transfer.kind); +const typeLabel = (transfer) => TYPES[typeOf(transfer)] || transfer.kind || '—'; +const assetOf = (transfer) => (transfer.kind === 'usdc_bridge' || !transfer.symbol ? 'USDC' : transfer.symbol); + +// The SPA's statusStyle colours. +const statusClass = (status) => { + const lower = String(status || '').toLowerCase(); + if (lower.includes('completed') || lower.includes('filled') || lower.includes('deposited') || lower.includes('confirmed')) return 'det-green'; + if (lower.includes('reconciled')) return 'det-amber'; + if (lower.includes('failed')) return 'det-red'; + if (lower.includes('cancelled')) return 'det-orange'; + return 'det-blue'; +}; +const statusLabel = (status) => { + const text = String(status || '—').replace(/_/g, ' '); + return text.charAt(0).toUpperCase() + text.slice(1); +}; + +// -------------------------------------------------------------------------- +// The dialog +// -------------------------------------------------------------------------- + +// Block explorers by the bot's chain wire name. A chain without one, or an +// unknown chain, shows the hash unlinked: a link to the wrong explorer would +// read as "this tx does not exist". +const EXPLORERS = { + base: 'https://basescan.org', + ethereum: 'https://etherscan.io', + hyperevm: 'https://hyperevmscan.io', +}; +const shortHash = (hash) => `${escapeHtml(hash.slice(0, 10))}…${escapeHtml(hash.slice(-8))}`; +const txLink = (hash, chain) => + EXPLORERS[chain] + ? `${shortHash(hash)} ↗` + : `${shortHash(hash)}`; + +// The bot's event timeline, once the panel queries it as C: each step with +// its time and fields, like the SPA's dialog. +// Without it, the exporter's status log: one step per status change. +const events = entriesOf('C').sort((left, right) => (left.sequence ?? 0) - (right.sequence ?? 0)); +const humanize = (step) => String(step || '').replace(/([a-z0-9])([A-Z])/g, '$1 $2'); +const isTxHash = (value) => typeof value === 'string' && /^0x[0-9a-fA-F]{64}$/.test(value); +const eventValue = (key, value, chain) => { + if (key === 'error') return `${escapeHtml(value)}`; + if (isTxHash(value)) return txLink(value, chain); + if (value !== null && typeof value === 'object') return `${escapeHtml(JSON.stringify(value))}`; + return escapeHtml(value); +}; +// A payload's own `chain` names where its hashes live; else the row's chain +// (a trade's), else none, which leaves the hashes unlinked. +const eventFields = (payload, rowChain) => { + const fields = payload && typeof payload === 'object' ? payload : {}; + const chain = typeof fields.chain === 'string' ? fields.chain : rowChain; + return Object.entries(fields) + .filter(([key]) => !/(_at|At|timestamp)$/.test(key)) + .map(([key, value]) => `
${escapeHtml(key.replace(/_/g, ' '))}${eventValue(key, value, chain)}
`) + .join(''); +}; + +// The board loads the newest 500 events of all rows, so an older row can have +// only its later events here. Then the timeline starts after the row's first +// status entry, and says so. +const timelineIncomplete = (history) => + history.length > 0 && + Math.min(...events.map((event) => event.time)) > Math.min(...history.map((entry) => entry.time)) + 120000; + +const timeline = (history, chain) => + events.length > 0 + ? ` +
Event timeline
+ ${timelineIncomplete(history) ? '

Older events of this row are outside the newest 500 the board loads; the full timeline is in the SPA at liquidity.t0trade.com.

' : ''} +
${events + .map( + (event) => ` +
+ +
${escapeHtml(humanize(event.step))}
+
${utc(event.time)}
+ ${eventFields(event.payload, chain)} +
` + ) + .join('')}
` + : ` +
Status history
+
${history + .map( + (entry) => ` +
+ +
${escapeHtml(statusLabel(entry.status))}
+
${utc(entry.time)}
+ ${entry.error ? `
${escapeHtml(entry.error)}
` : ''} +
` + ) + .join('')}
+

From the exporter's status log. The full event timeline is in the SPA at liquidity.t0trade.com.

`; + +// valueHtml is trusted HTML: a caller escapes any data it puts in it. +const field = (name, valueHtml) => `
${escapeHtml(name)}${valueHtml}
`; + +const tradeDialog = (trade) => { + const [venueName, venueClass, onchain] = venue(trade.venue); + const id = String(trade.id); + // An onchain trade id is `chain:tx_hash:log_index` (OnChainTradeId), or + // `tx_hash:log_index` from before the chain was part of it (Base). + const parts = id.split(':'); + const [chain, hash, log] = parts.length === 3 ? parts : ['base', ...parts]; + const idCell = + onchain && parts.length >= 2 && isTxHash(hash) + ? `${txLink(hash, chain)} (${escapeHtml(chain)}, log ${escapeHtml(log)})` + : `${escapeHtml(id)}`; + return ` +
+ + ${escapeHtml(trade.direction)} + ${escapeHtml(trade.symbol)} + ${decimal(trade.shares)} ${onchain ? 'wrapped shares' : 'shares'} + ${escapeHtml(venueName)} + + +
+
+
+ ${field('ID', idCell)} + ${field('Occurred At', utc(trade.first))} + ${field('Status', `${escapeHtml(statusLabel(trade.status))}${trade.error ? `
${escapeHtml(trade.error)}
` : ''}`)} +
+ ${timeline(trade.history, onchain ? chain : null)} +
`; +}; + +const transferDialog = (transfer) => { + return ` +
+ + ${escapeHtml(typeLabel(transfer))} + ${escapeHtml(assetOf(transfer))} + ${decimal(transfer.amount)} + + +
+
+
+ ${field('ID', `${escapeHtml(transfer.id)}`)} + ${field('Started', utc(transfer.started_at || transfer.first))} + ${field('Updated', utc(transfer.time))} + ${field('Status', `${escapeHtml(statusLabel(transfer.status))}`)} +
+ ${timeline(transfer.history, null)} +
`; +}; + +// -------------------------------------------------------------------------- +// Open, refresh or close the one dialog +// -------------------------------------------------------------------------- + +// Kept on the element across renders: the plugin re-runs this code on every +// refresh and query, into the same element. +const state = root.__det || (root.__det = { dismissed: null }); + +// The variable can lag the close by a render (a refresh that was already +// running), so an id the operator just closed is not reopened until the +// variable has moved off it. +if (state.dismissed !== null && state.dismissed !== wanted) state.dismissed = null; + +// Switching Environment keeps the tables' old rows on screen until the new +// queries return, and those rows belong to the other project. Hold back +// every row after a switch until a query has run (Grafana showed the panel +// Loading) and then succeeded. Frames alone cannot tell: Grafana replays and +// copies the old ones, and keeps them on an error. A Cloud Logging query +// takes over a second, so the Loading state shows; if it does not, or shows +// only on the switch render, the dialog says Loading… until the next +// refresh, which fails safe. Known +// limit: a query cancelled from the refresh picker after the switch also +// ends Done with the old rows, and that releases the hold. The memory +// lives on the window, not the element, so a remount (tab switch, view mode) +// keeps it. The first render after the page loads is not held back: its rows +// are for the environment it opened on. +const env = context.grafana.replaceVariables('${env}'); +const loading = context.panelData?.state === 'Loading'; +const failed = context.panelData?.state === 'Error'; +const envState = window.__liqDetailEnv || (window.__liqDetailEnv = { env: undefined, held: false, sawLoading: false }); +const switched = envState.env !== undefined && envState.env !== env; +envState.env = env; +if (switched) { + // A Loading state on this render can be a run that started before the + // switch, so only a later one counts. + envState.held = true; + envState.sawLoading = false; +} else if (envState.held) { + // A query that failed does not count as the one that ran. + if (loading) envState.sawLoading = true; + else if (failed) envState.sawLoading = false; + else if (envState.sawLoading) envState.held = false; +} + +// A failed refresh can keep the old frames, so it shows the error, not a +// row and its commands that may be out of date. +const trade = envState.held || failed ? null : latest(entriesOf('A')); +const transfer = trade || envState.held || failed ? null : latest(entriesOf('B')); +// The dialog opens on the click, before the query returns, and says so when +// the id has no entries in the panel's window. +const message = (text) => ` +
${escapeHtml(wanted)}
+

${escapeHtml(text)}

`; +const queryError = () => { + const errors = context.panelData?.errors || (context.panelData?.error ? [context.panelData.error] : []); + return errors.map((error) => error.message).filter(Boolean).join('; ') || 'unknown error'; +}; +const html = trade + ? tradeDialog(trade) + : transfer + ? transferDialog(transfer) + : wanted === '' + ? null + : failed + ? message(`The Trades or Rebalances query failed (${queryError()}), so this row cannot be shown. Refresh the board to retry.`) + : loading || envState.held + ? message('Loading…') + : message('This id is not in the newest 500 status entries any more. Search the Logs tab for it.'); + +// Reuse the dialog when it is still in the element, so a refresh with the +// dialog open swaps its content without closing and reopening it. +let dialog = root.querySelector('dialog.det-dialog'); +if (!dialog) { + dialog = document.createElement('dialog'); + dialog.className = 'det-dialog'; + root.appendChild(dialog); +} + +// Property handlers, not listeners: each render replaces them instead of +// stacking another copy. +dialog.onclose = () => { + state.dismissed = dialog.detId || null; + dialog.detId = ''; + // An empty value, not null: removing the URL parameter leaves a textbox + // variable at its old value, so the same row could not be reopened. + if (context.grafana.replaceVariables('${detail}') !== '') { + context.grafana.locationService.partial({ 'var-detail': '' }, true); + } +}; +// A click on the backdrop closes it, like the SPA. +dialog.onclick = (event) => { + if (event.target === dialog) dialog.close(); +}; + +if (html === null || state.dismissed === wanted) { + if (dialog.open) dialog.close(); +} else { + if (dialog.detHtml !== html) { + // A refresh that brings a new status keeps the operator's scroll. + const body = dialog.querySelector('.det-dialog-body'); + const scroll = dialog.detId === wanted && body ? body.scrollTop : 0; + dialog.innerHTML = html; + dialog.detHtml = html; + const next = dialog.querySelector('.det-dialog-body'); + if (next) next.scrollTop = scroll; + } + dialog.detId = wanted; + dialog.querySelector('[data-close]').onclick = () => dialog.close(); + if (!dialog.open) dialog.showModal(); +} diff --git a/observability/liquidity-panels/status-history.js b/observability/liquidity-panels/status-history.js new file mode 100644 index 0000000000..e8a4a1839f --- /dev/null +++ b/observability/liquidity-panels/status-history.js @@ -0,0 +1,46 @@ +// The detail dialog's status history: a Trades or Rebalances row's status-log +// entries in order, and the status the row is at. The generator prepends this +// file to detail.js; dashboard/src/lib/transfer-board.test.ts imports it as a +// module through the export line, which the generator drops. + +// A USDC bridge's in-flight statuses in lifecycle order per direction. Their +// log times can go backwards (WithdrawalComplete carries confirmed_at, the +// BridgingSubmitting after it the older initiated_at), so two in-flight +// entries of a bridge are ordered by this. Anything else, failed included +// (the bot can recover a bridge out of a failure), orders by time. +// +// Known limit: a Base to Alpaca bridge recovered out of DepositFailed reaches +// converting, which the bot stamps with the bridge's initiated_at. That entry +// sorts before the failed one, so the dialog keeps showing the status before +// it (failed, or depositing) until the bridge completes. +const BRIDGE_LIFECYCLE = { + alpaca_to_base: ['converting', 'withdrawing', 'bridging', 'depositing'], + base_to_alpaca: ['withdrawing', 'bridging', 'depositing', 'converting'], +}; +const stage = (entry) => { + const lifecycle = entry.kind === 'usdc_bridge' ? BRIDGE_LIFECYCLE[entry.direction] : null; + const index = lifecycle ? lifecycle.indexOf(String(entry.status || '').toLowerCase()) : -1; + return index >= 0 ? index : null; +}; + +// The row at its latest status, with its status history oldest first: by +// time (ties keep Cloud Logging's newest-first order reversed), then each +// pair of neighbouring in-flight bridge entries put in lifecycle order. +const latest = (entries) => { + if (entries.length === 0) return null; + const history = [...entries].reverse().sort((left, right) => left.time - right.time); + for (let swapped = true; swapped; ) { + swapped = false; + for (let index = 1; index < history.length; index++) { + const [before, after] = [stage(history[index - 1]), stage(history[index])]; + if (before !== null && after !== null && before > after) { + [history[index - 1], history[index]] = [history[index], history[index - 1]]; + swapped = true; + } + } + } + const last = history[history.length - 1]; + return { ...last, first: Math.min(...entries.map((entry) => entry.time)), history }; +}; + +export { latest };