observability: show each liquidity row's recovery commands in its detail dialog [RAI-3100]

dashboard/src/lib/transfer-board.test.ts

@@ -1,14 +1,34 @@
 import { readFileSync } from 'node:fs'
 import { describe, expect, it } from 'vitest'
-import { RECOVERY_GUIDE, recoveryModeLabel, type RecoveryMode } from './transfer'
+import {
+  RECOVERY_GUIDE,
+  recoveryModeLabel,
+  transferRecoveryCommands,
+  tradeRecoveryCommands,
+  type RecoveryCommand,
+  type RecoveryMode,
+  type TransferCategory
+} from './transfer'
+import type { UsdcBridgeDirection } from './api/UsdcBridgeDirection'
 
 // The Grafana board (observability/) shows the same recovery commands as the
-// SPA from its own copy: recovery-guide.json for the guide. These tests fail
-// when the copy drifts from this file's builders. They also cover the board's
+// SPA from its own copies: recovery-guide.json for the guide and
+// recovery-commands.js for the per-row dialog. These tests fail when a copy
+// drifts from this file's builders. They also cover the board's
 // status-history.js, which orders a row's status entries in its dialog.
 
 const panels = new URL('../../../observability/liquidity-panels/', import.meta.url)
 
+type BoardCommand = { command: string; description: string; mode: RecoveryMode }
+type BoardTransfer = { kind: TransferCategory; id: string; status: string; direction?: string }
+type BoardBuilders = {
+  tradeCommands: (client: string, symbol: string) => BoardCommand[]
+  transferCommands: (client: string, transfer: BoardTransfer) => BoardCommand[]
+}
+
+// A plain script in the board, an ES module through its export line here.
+const board = (await import(new URL('recovery-commands.js', panels).href)) as BoardBuilders
+
 type StatusEntry = { time: number; kind: string; status: string; direction?: string }
 type StatusHistory = {
   latest: (
@@ -19,6 +39,12 @@ type StatusHistory = {
 // A plain script in the board, an ES module through its export line here.
 const statusHistory = (await import(new URL('status-history.js', panels).href)) as StatusHistory
 
+const CLIENT = 'st0x-liquidity-client --env production'
+const PROD = { simulateSourceId: null, backendPort: null, clientEnv: 'production' } as const
+
+const withoutLabel = (commands: RecoveryCommand[]): BoardCommand[] =>
+  commands.map(({ command, description, mode }) => ({ command, description, mode }))
+
 describe('board recovery-guide.json', () => {
   it('matches RECOVERY_GUIDE and the mode labels', () => {
     const guide = JSON.parse(readFileSync(new URL('recovery-guide.json', panels), 'utf8')) as {
@@ -32,6 +58,60 @@ describe('board recovery-guide.json', () => {
   })
 })
 
+describe('board recovery-commands.js', () => {
+  it('matches tradeRecoveryCommands', () => {
+    expect(board.tradeCommands(CLIENT, 'MSTR')).toEqual(
+      withoutLabel(tradeRecoveryCommands({ deployment: PROD, symbol: 'MSTR' }))
+    )
+  })
+
+  it('matches transferRecoveryCommands for every equity status', () => {
+    const kinds: TransferCategory[] = ['equity_mint', 'equity_redemption']
+    for (const kind of kinds) {
+      for (const status of ['wrapping', 'sending', 'failed', 'completed', 'reconciled']) {
+        expect(board.transferCommands(CLIENT, { kind, id: 'X1', status })).toEqual(
+          withoutLabel(transferRecoveryCommands({ deployment: PROD, kind, id: 'X1', status }))
+        )
+      }
+    }
+  })
+
+  it('matches transferRecoveryCommands for every usdc bridge status and direction', () => {
+    // The exporter does not log the bridge's postBurn flag, so the board
+    // matches the SPA without it: a failed bridge offers no reconcile.
+    const directions: (UsdcBridgeDirection | null)[] = ['alpaca_to_base', 'base_to_alpaca', null]
+    for (const direction of directions) {
+      for (const status of [
+        'converting',
+        'withdrawing',
+        'bridging',
+        'depositing',
+        'failed',
+        'completed'
+      ]) {
+        expect(
+          board.transferCommands(CLIENT, {
+            kind: 'usdc_bridge',
+            id: 'B1',
+            status,
+            ...(direction === null ? {} : { direction })
+          })
+        ).toEqual(
+          withoutLabel(
+            transferRecoveryCommands({
+              deployment: PROD,
+              kind: 'usdc_bridge',
+              id: 'B1',
+              status,
+              direction
+            })
+          )
+        )
+      }
+    }
+  })
+})
+
 describe('board status-history.js', () => {
   const bridge = (status: string, time: number, direction = 'alpaca_to_base'): StatusEntry => ({
     time,

observability/README.md

@@ -41,11 +41,12 @@ into the board JSON. The plugin is installed on the box by `T0Trade/t0.devops`
 (`GF_INSTALL_PLUGINS`); the PR check's throwaway Grafana does not have it, so it
 loads these panels as "Panel plugin not found" and still passes.
 
-The recovery commands in the header's guide (`recovery-guide.json`) are a copy
-of the SPA's (`dashboard/src/lib/transfer.ts`), in the operations client form
-for the environment the board shows. `dashboard/src/lib/transfer-board.test.ts`
-fails when the copy drifts, so change the SPA and the copy together. It runs in
-the dashboard build of CI.
+The recovery commands in the header's guide (`recovery-guide.json`) and in the
+row dialog (`recovery-commands.js`) are copies of the SPA's
+(`dashboard/src/lib/transfer.ts`), in the operations client form for the
+environment the board shows. `dashboard/src/lib/transfer-board.test.ts` fails
+when a copy drifts, so change the SPA and the copies together. It runs in the
+dashboard build of CI.
 
 The Dashboard tab's inventory is three Grafana tables, for their column filters:
 USD at Alpaca, USD on each chain, and Equities. Per-chain rows (USD · Onchain)

observability/gen-t0-liquidity.py

@@ -1260,13 +1260,21 @@ def detail_panel(x, y, trades_id, transfers_id):
     which makes Grafana redraw the panel when the variable changes.
     liquidity-panels/detail.js builds the dialog. renderMode "data" hands
     the script both frames, where "allRows" would draw a frame picker.
-    The status-history order is prepended from status-history.js.
+    MODE_LABELS (how each recovery command runs) is prepended from
+    recovery-guide.json, like the header's guide, and the per-row command
+    builders from recovery-commands.js, and the status-history order from
+    status-history.js.
     """
-    with open(os.path.join(HERE, "liquidity-panels", "status-history.js")) as f:
-        # Its export line is for the SPA test; afterRender is not a module.
-        history = "".join(line for line in f if not line.startswith("export "))
+    with open(os.path.join(HERE, "liquidity-panels", "recovery-guide.json")) as f:
+        mode_labels = json.load(f)["modeLabels"]
+    commands = ""
+    for module in ("recovery-commands.js", "status-history.js"):
+        with open(os.path.join(HERE, "liquidity-panels", module)) as f:
+            # Its export line is for the SPA test; afterRender is not a module.
+            commands += "".join(line for line in f if not line.startswith("export ")) + "\n"
     with open(os.path.join(HERE, "liquidity-panels", "detail.js")) as f:
-        after_render = history + "\n\n" + f.read()
+        after_render = (f"const MODE_LABELS = {json.dumps(mode_labels)};\n\n"
+                        + commands + "\n" + f.read())
     with open(os.path.join(HERE, "liquidity-panels", "detail.css")) as f:
         styles = f.read()
     return {

observability/liquidity-panels/detail.css

@@ -87,7 +87,8 @@
 
 .det-field > .det-muted { flex-shrink: 0; }
 
-.det-section-title {
+.det-section-title,
+.det-cli-title {
   margin-bottom: 8px;
   font-weight: 600;
 }
@@ -112,3 +113,33 @@
   border-radius: 9999px;
   background: currentColor;
 }
+
+.det-cli {
+  margin-top: 16px;
+  padding-top: 12px;
+  border-top: 1px solid var(--det-border);
+}
+
+.det-command {
+  margin-bottom: 8px;
+  padding: 8px 12px;
+  border: 1px solid var(--det-border);
+  border-radius: 6px;
+  background: var(--det-stripe);
+}
+
+.det-command pre {
+  margin: 0 0 6px;
+  padding: 6px 8px;
+  border-radius: 4px;
+  background: var(--det-card);
+  font-size: 12px;
+  white-space: pre-wrap;
+  word-break: break-all;
+}
+
+.det-mode {
+  display: inline-block;
+  margin-top: 4px;
+  font-size: 12px;
+}

observability/liquidity-panels/detail.js

@@ -12,8 +12,9 @@
 //
 // The panel is one empty column of the header row; only the dialog shows.
 //
-// latest is not defined here: the generator prepends it from
-// status-history.js (see detail_panel()).
+// MODE_LABELS, tradeCommands, transferCommands and latest are not defined
+// here: the generator prepends them from recovery-guide.json,
+// recovery-commands.js and status-history.js (see detail_panel()).
 
 const theme = context.grafana.theme;
 const root = context.element;
@@ -144,6 +145,31 @@ const statusLabel = (status) => {
   return text.charAt(0).toUpperCase() + text.slice(1);
 };
 
+// --------------------------------------------------------------------------
+// Recovery commands (recovery-commands.js, prepended): the operations client,
+// which signs in with Google and calls the running bot's ops API, so no SSH
+// is needed and every command needs the bot.
+// --------------------------------------------------------------------------
+
+// The environment the board shows: the client takes the selector's text.
+const ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';
+const CLIENT = `st0x-liquidity-client --env ${ENV}`;
+
+const modeClass = (mode) => (mode === 'requires-bot' ? 'det-amber' : 'det-red');
+const commandBlock = (commands, note) =>
+  commands.length === 0 && !note
+    ? ''
+    : `<div class="det-cli"><div class="det-cli-title">CLI commands</div>${note ? `<p class="det-muted">${escapeHtml(note)}</p>` : ''}${commands
+        .map(
+          (entry) => `
+        <div class="det-command">
+          <pre>${escapeHtml(entry.command)}</pre>
+          <div>${escapeHtml(entry.description)}</div>
+          <span class="det-mode ${modeClass(entry.mode)}">${escapeHtml(MODE_LABELS[entry.mode] || entry.mode)}</span>
+        </div>`
+        )
+        .join('')}</div>`;
+
 // --------------------------------------------------------------------------
 // The dialog
 // --------------------------------------------------------------------------
@@ -254,10 +280,12 @@ const tradeDialog = (trade) => {
       ${field('Status', `<span class="${statusClass(trade.status)}">${escapeHtml(statusLabel(trade.status))}</span>${trade.error ? `<div class="det-red">${escapeHtml(trade.error)}</div>` : ''}`)}
     </div>
     ${timeline(trade.history, onchain ? chain : null)}
+    ${commandBlock(tradeCommands(CLIENT, trade.symbol))}
   </div>`;
 };
 
 const transferDialog = (transfer) => {
+  const usdcFailed = transfer.kind === 'usdc_bridge' && String(transfer.status).toLowerCase() === 'failed';
   return `
   <div class="det-dialog-head">
     <span class="det-title-line">
@@ -275,6 +303,12 @@ const transferDialog = (transfer) => {
       ${field('Status', `<span class="${statusClass(transfer.status)}">${escapeHtml(statusLabel(transfer.status))}</span>`)}
     </div>
     ${timeline(transfer.history, null)}
+    ${commandBlock(
+      transferCommands(CLIENT, transfer),
+      usdcFailed
+        ? 'Reconcile applies only when the funds left their source venue, which the logs do not show. Check the transfer in the SPA before you reconcile it.'
+        : ''
+    )}
   </div>`;
 };