diff --git a/dashboard/src/lib/transfer-board.test.ts b/dashboard/src/lib/transfer-board.test.ts new file mode 100644 index 000000000..4dc288d55 --- /dev/null +++ b/dashboard/src/lib/transfer-board.test.ts @@ -0,0 +1,22 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' +import { RECOVERY_GUIDE, recoveryModeLabel, type RecoveryMode } from './transfer' + +// The Grafana board (observability/) shows the same recovery commands as the +// SPA from its own copy: recovery-guide.json for the guide. These tests fail +// when the copy drifts from this file's builders. + +const panels = new URL('../../../observability/liquidity-panels/', import.meta.url) + +describe('board recovery-guide.json', () => { + it('matches RECOVERY_GUIDE and the mode labels', () => { + const guide = JSON.parse(readFileSync(new URL('recovery-guide.json', panels), 'utf8')) as { + modeLabels: Record + groups: unknown + } + expect(guide.groups).toEqual(RECOVERY_GUIDE) + for (const mode of ['requires-bot', 'direct-db'] as const) { + expect(guide.modeLabels[mode]).toBe(recoveryModeLabel(mode)) + } + }) +}) diff --git a/observability/README.md b/observability/README.md index a73f57d10..d11689d35 100644 --- a/observability/README.md +++ b/observability/README.md @@ -35,6 +35,18 @@ every `liq_` selector and fails on one without it, because an unpinned selector adds both sources together. Stage 4 of the migration makes `bot` the default (`SOURCE_VAR` in the generator). See `docs/observability.md`, "Board source". +The header row is a Business Text panel (`marcusolsson-dynamictext-panel`): its +JavaScript and CSS live in `liquidity-panels/`, and the generator inlines them +into the board JSON. The plugin is installed on the box by `T0Trade/t0.devops` +(`GF_INSTALL_PLUGINS`); the PR check's throwaway Grafana does not have it, so it +loads these panels as "Panel plugin not found" and still passes. + +The recovery commands in the header's guide (`recovery-guide.json`) are a copy +of the SPA's (`dashboard/src/lib/transfer.ts`), in the operations client form +for the environment the board shows. `dashboard/src/lib/transfer-board.test.ts` +fails when the copy drifts, so change the SPA and the copy together. It runs in +the dashboard build of CI. + `alerting/liquidity.rules.yml` is the alert rules, in the `Alerts` folder. Each rule's `uid` is permanent and must be unique across every repo's file. Removing a rule from the file does not remove it from Grafana: add its `uid` to diff --git a/observability/dashboards/liquidity/t0-liquidity.json b/observability/dashboards/liquidity/t0-liquidity.json index a48ee0e25..e2438a864 100644 --- a/observability/dashboards/liquidity/t0-liquidity.json +++ b/observability/dashboards/liquidity/t0-liquidity.json @@ -77,9 +77,8 @@ "links": [ { "type": "link", - "title": "\u25b8 Dashboard", + "title": "\u25b8 \ud83d\udcca\u00a0Dashboard", "url": "/d/t0-liquidity/?theme=light&autofitpanels", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -89,9 +88,8 @@ }, { "type": "link", - "title": "Orders", + "title": "\ud83d\udccb\u00a0Orders", "url": "/d/t0-liquidity-orders/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -101,9 +99,8 @@ }, { "type": "link", - "title": "PnL", + "title": "\ud83d\udcb5\u00a0PnL", "url": "/d/t0-liquidity-pnl/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -113,9 +110,8 @@ }, { "type": "link", - "title": "Performance", + "title": "\u26a1\u00a0Performance", "url": "/d/t0-liquidity-performance/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -125,9 +121,8 @@ }, { "type": "link", - "title": "Logs", + "title": "\ud83d\udcdc\u00a0Logs", "url": "/d/t0-liquidity-logs/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -152,9 +147,9 @@ "panels": [ { "id": 1, - "type": "stat", + "type": "marcusolsson-dynamictext-panel", "title": "", - "description": "The SPA header's connection light: is the bot process up and answering? The exporter source reads liq_up, which is 0 when the exporter cannot reach the bot's /health. The bot never emits liq_up, so the bot source reads the scrape's own up. '\u2014' means the source is not reporting.", + "description": "The SPA header: settings pills, Config, the CLI recovery guide, the deployed commit, uptime, and whether the bot is up: the exporter's liq_up, or the bot scrape's up on the bot source.", "datasource": { "type": "stackdriver", "uid": "cloudmon" @@ -169,7 +164,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"})", + "expr": "label_replace(max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"}), \"k\", \"up\", \"\", \"\") or label_replace(time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"}), \"k\", \"uptime\", \"\", \"\") or label_replace(max by (git_commit) (timestamp(liq_bot_info{job=~\"$source\"})), \"k\", \"commit\", \"\", \"\") or label_replace(max by (broker, log_level, wallet_kind, wallet_address, orderbook, turnkey_organization, server_port) (liq_settings_info{job=~\"$source\"}), \"k\", \"info\", \"\", \"\") or label_replace(max(liq_settings_equity_target{job=~\"$source\"}), \"k\", \"equity_target\", \"\", \"\") or label_replace(max(liq_settings_equity_deviation{job=~\"$source\"}), \"k\", \"equity_deviation\", \"\", \"\") or label_replace(max(liq_settings_usdc_target{job=~\"$source\"}), \"k\", \"usdc_target\", \"\", \"\") or label_replace(max(liq_settings_usdc_deviation{job=~\"$source\"}), \"k\", \"usdc_deviation\", \"\", \"\") or label_replace(max(liq_settings_execution_threshold_usd{job=~\"$source\"}), \"k\", \"trigger\", \"\", \"\") or label_replace(max(liq_settings_cash_reserved{job=~\"$source\"}), \"k\", \"cash_reserved\", \"\", \"\") or label_replace(max(liq_settings_order_polling_seconds{job=~\"$source\"}), \"k\", \"order_polling\", \"\", \"\") or label_replace(max(liq_settings_inventory_poll_seconds{job=~\"$source\"}), \"k\", \"inventory_polling\", \"\", \"\") or label_replace(max(liq_settings_deployment_block{job=~\"$source\"}), \"k\", \"deployment_block\", \"\", \"\")", "step": "60s" }, "timeSeriesList": { @@ -184,602 +179,38 @@ "transparent": true, "gridPos": { "h": 1, - "w": 3, + "w": 24, "x": 0, "y": 0 }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "value", - "options": { - "1": { - "text": "Connected", - "color": "green", - "index": 0 - }, - "0": { - "text": "Disconnected", - "color": "red", - "index": 1 - } - } - }, - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short", - "displayName": "Bot" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 2, - "type": "stat", - "title": "", - "description": "Deployed commit, from /health, cut to 7 characters like the SPA header. The stackdriver plugin runs a range query even for an instant target, so over the dashboard's 24h the previous commit is a second series and its name drew over this one. PromQL also keeps a stopped series for its 5m lookback, so topk on the sample timestamp keeps only the commit with the newest sample, over a 1m window.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=~\"$source\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true, - "legendFormat": "{{git_commit}}" - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 3, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "1m", - "hideTimeOverride": true - }, - { - "id": 3, - "type": "stat", - "title": "", - "description": "Bot process uptime, from /health uptimeSeconds.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 6, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "s", - "displayName": "Uptime", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 4, - "type": "stat", - "title": "", - "description": "The SPA settings bar's Broker pill.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true, - "legendFormat": "{{broker}}" - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 9, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 5, - "type": "stat", - "title": "", - "description": "Equity rebalance target. The SPA pill also shows the band: target +/- liq_settings_equity_deviation.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_equity_target{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 12, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "percentunit", - "displayName": "Equity", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 6, - "type": "stat", - "title": "", - "description": "USDC rebalance target. The SPA pill also shows the band: target +/- liq_settings_usdc_deviation.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ + "transformations": [ { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_usdc_target{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 16, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "percentunit", - "displayName": "USDC", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 + "id": "labelsToFields", + "options": {} }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 7, - "type": "stat", - "title": "", - "description": "Hedge execution threshold (settings bar 'Trigger').", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true + "id": "merge", + "options": {} } ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 20, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "currencyUSD", - "displayName": "Trigger", - "decimals": 0 - }, - "overrides": [] - }, "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 + "renderMode": "allRows", + "editor": { + "format": "auto", + "language": "html" }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } + "content": "
", + "defaultContent": "No bot data.", + "helpers": "", + "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum that no CCTP burn left the bot wallet after this transfer started (its Started time) and that the wallet has no pending transaction (pending nonce equals latest nonce); if you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, and pending nonce equals latest nonce on more than one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once they show that no withdrawal landed and none is pending, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn left the bot wallet after this transfer started and the wallet has no pending transaction. If any of that is not certain, start the bot and run resume-usdc; else run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", + "styles": "/* Business Text \"styles\" for the Liquidity bot header row and its two\n dialogs. Theme colours come in as --hdr-* variables set by header.js. */\n\n.hdr {\n display: flex;\n align-items: center;\n justify-content: space-between;\n gap: 8px;\n height: 100%;\n font-size: 13px;\n white-space: nowrap;\n}\n\n.hdr-left,\n.hdr-right {\n display: flex;\n align-items: center;\n gap: 8px;\n min-width: 0;\n}\n\n.hdr-pill {\n display: inline-flex;\n align-items: center;\n gap: 4px;\n padding: 2px 10px;\n border-radius: 9999px;\n background: var(--hdr-pill);\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n border: 0;\n}\n\n.hdr-button {\n color: var(--hdr-muted);\n cursor: pointer;\n}\n\n.hdr-button:hover,\n.hdr-guide:hover,\n.hdr-close:hover {\n color: var(--hdr-text);\n}\n\n.hdr-guide {\n padding: 2px 8px;\n border: 1px solid var(--hdr-border);\n border-radius: 4px;\n background: transparent;\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n cursor: pointer;\n}\n\n.hdr-badge {\n padding: 2px 8px;\n border-radius: 9999px;\n font-size: 13px;\n line-height: 18px;\n font-weight: 600;\n}\n\n.hdr-ok { background: #f5f5f5; color: #171717; }\n.hdr-bad { background: #ef4444; color: #fff; }\n.hdr-unknown { border: 1px solid var(--hdr-border); color: var(--hdr-muted); }\n\n.hdr-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.hdr-muted { color: var(--hdr-muted); }\n\n.hdr-dialog {\n padding: 0;\n border: 1px solid var(--hdr-border);\n border-radius: 8px;\n background: var(--hdr-card);\n color: var(--hdr-text);\n white-space: normal;\n}\n\n.hdr-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n.hdr-narrow { width: min(28rem, 92vw); }\n.hdr-wide { width: min(42rem, 92vw); }\n\n.hdr-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--hdr-border);\n font-size: 14px;\n font-weight: 600;\n}\n\n.hdr-close {\n border: 0;\n background: transparent;\n color: var(--hdr-muted);\n font-size: 18px;\n line-height: 1;\n cursor: pointer;\n}\n\n.hdr-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 13px;\n}\n\n.hdr-row {\n display: flex;\n justify-content: space-between;\n gap: 16px;\n padding: 4px 0;\n}\n\n.hdr-trunc {\n overflow: hidden;\n text-overflow: ellipsis;\n white-space: nowrap;\n}\n\n.hdr-rule { border-top: 1px solid var(--hdr-border); margin: 8px 0; }\n.hdr-intro { margin: 0 0 16px; }\n.hdr-group { margin-bottom: 20px; }\n.hdr-group h3 { margin: 0 0 8px; font-size: 13px; text-transform: capitalize; }\n\n.hdr-command {\n margin-bottom: 12px;\n padding: 8px 12px;\n border: 1px solid var(--hdr-border);\n border-radius: 6px;\n background: var(--hdr-pill);\n}\n\n.hdr-command pre {\n margin: 0;\n padding: 8px;\n border-radius: 4px;\n background: var(--hdr-card);\n font-size: 12px;\n white-space: pre-wrap;\n word-break: break-all;\n}\n\n.hdr-grid {\n display: grid;\n grid-template-columns: auto 1fr;\n gap: 4px 12px;\n margin-top: 6px;\n font-size: 12px;\n}\n\n.hdr-amber { color: #f59e0b; }\n.hdr-red { color: #ef4444; }\n\n/* Tailwind 700 on the light theme, like detail.css, for a 4.5:1 contrast\n against the white dialog. */\n.hdr-light .hdr-amber { color: #b45309; }\n.hdr-light .hdr-red { color: #b91c1c; }\n", + "wrap": false }, "timeFrom": "5m", "hideTimeOverride": true }, { - "id": 8, + "id": 2, "type": "table", "title": "Inventory", "description": "The SPA's USDC row: Raindex + Alpaca Total + Inflight = Total; Ratio = Raindex/(Raindex+Alpaca Total); \u0394 = deviation from the target ratio in band widths, colored outside +/-1, where the SPA colors its bar.", @@ -1093,7 +524,7 @@ "hideTimeOverride": false }, { - "id": 9, + "id": 3, "type": "table", "title": "Alpaca", "description": "Rebalanceable = max(0, withdrawable \u2212 reserve). Counter-tradeable deliberately equals Alpaca Total (reserve NOT subtracted, same as the SPA).", @@ -1276,7 +707,7 @@ "hideTimeOverride": false }, { - "id": 10, + "id": 4, "type": "table", "title": "Wallets", "description": "USDC seen in the bot's Ethereum and Base wallets: sanity checks, not part of Total.", @@ -1438,7 +869,7 @@ "hideTimeOverride": true }, { - "id": 11, + "id": 5, "type": "table", "title": "", "description": "The SPA inventory equity table: CT / Rebal / Ext status lights from settings; Raindex / Inflight / Alpaca / Total share balances; Ratio = onchain/(onchain+offchain); \u0394 = deviation from the target ratio in band widths, colored outside +/-1; Exposure = net \u00d7 last price. Unwrapped/Wrapped are wallet-observed and not part of Total. Sorted CT-first (desc) so counter-tradeable assets float to the top, like the SPA.", @@ -2024,7 +1455,7 @@ "hideTimeOverride": false }, { - "id": 12, + "id": 6, "type": "table", "title": "Trade History", "description": "Direct Raindex fills, fills routed through supported adapters such as Bebop, and the corresponding Alpaca hedge trades placed to offset exposure. One row per trade at its latest status. Built from the newest 500 status entries, so a busy period can push older rows out of the 30-day window.", @@ -2459,7 +1890,7 @@ } }, { - "id": 13, + "id": 7, "type": "table", "title": "Cross-venue Transfers", "description": "Asset movements between venues to rebalance inventory: equity mints (Alpaca to onchain), redemptions (onchain to Alpaca), and USDC bridges (Base/Ethereum via CCTP). One row per transfer at its latest status. A USDC bridge can show the previous status for a while: the exporter stamps each status with the transfer's updatedAt, which is not always later than the one before (bridging starts from initiated_at). Built from the newest 500 status entries, so a busy period can push older rows out of the 30-day window.", diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json b/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json index 615f155ec..428d76e7e 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json @@ -239,9 +239,8 @@ "links": [ { "type": "link", - "title": "Dashboard", + "title": "\ud83d\udcca\u00a0Dashboard", "url": "/d/t0-liquidity/?theme=light&autofitpanels", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -251,9 +250,8 @@ }, { "type": "link", - "title": "Orders", + "title": "\ud83d\udccb\u00a0Orders", "url": "/d/t0-liquidity-orders/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -263,9 +261,8 @@ }, { "type": "link", - "title": "PnL", + "title": "\ud83d\udcb5\u00a0PnL", "url": "/d/t0-liquidity-pnl/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -275,9 +272,8 @@ }, { "type": "link", - "title": "Performance", + "title": "\u26a1\u00a0Performance", "url": "/d/t0-liquidity-performance/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -287,9 +283,8 @@ }, { "type": "link", - "title": "\u25b8 Logs", + "title": "\u25b8 \ud83d\udcdc\u00a0Logs", "url": "/d/t0-liquidity-logs/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -313,10 +308,10 @@ ], "panels": [ { - "id": 108, - "type": "stat", + "id": 84, + "type": "marcusolsson-dynamictext-panel", "title": "", - "description": "The SPA header's connection light: is the bot process up and answering? The exporter source reads liq_up, which is 0 when the exporter cannot reach the bot's /health. The bot never emits liq_up, so the bot source reads the scrape's own up. '\u2014' means the source is not reporting.", + "description": "The SPA header: settings pills, Config, the CLI recovery guide, the deployed commit, uptime, and whether the bot is up: the exporter's liq_up, or the bot scrape's up on the bot source.", "datasource": { "type": "stackdriver", "uid": "cloudmon" @@ -331,7 +326,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"})", + "expr": "label_replace(max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"}), \"k\", \"up\", \"\", \"\") or label_replace(time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"}), \"k\", \"uptime\", \"\", \"\") or label_replace(max by (git_commit) (timestamp(liq_bot_info{job=~\"$source\"})), \"k\", \"commit\", \"\", \"\") or label_replace(max by (broker, log_level, wallet_kind, wallet_address, orderbook, turnkey_organization, server_port) (liq_settings_info{job=~\"$source\"}), \"k\", \"info\", \"\", \"\") or label_replace(max(liq_settings_equity_target{job=~\"$source\"}), \"k\", \"equity_target\", \"\", \"\") or label_replace(max(liq_settings_equity_deviation{job=~\"$source\"}), \"k\", \"equity_deviation\", \"\", \"\") or label_replace(max(liq_settings_usdc_target{job=~\"$source\"}), \"k\", \"usdc_target\", \"\", \"\") or label_replace(max(liq_settings_usdc_deviation{job=~\"$source\"}), \"k\", \"usdc_deviation\", \"\", \"\") or label_replace(max(liq_settings_execution_threshold_usd{job=~\"$source\"}), \"k\", \"trigger\", \"\", \"\") or label_replace(max(liq_settings_cash_reserved{job=~\"$source\"}), \"k\", \"cash_reserved\", \"\", \"\") or label_replace(max(liq_settings_order_polling_seconds{job=~\"$source\"}), \"k\", \"order_polling\", \"\", \"\") or label_replace(max(liq_settings_inventory_poll_seconds{job=~\"$source\"}), \"k\", \"inventory_polling\", \"\", \"\") or label_replace(max(liq_settings_deployment_block{job=~\"$source\"}), \"k\", \"deployment_block\", \"\", \"\")", "step": "60s" }, "timeSeriesList": { @@ -346,602 +341,38 @@ "transparent": true, "gridPos": { "h": 1, - "w": 3, + "w": 24, "x": 0, "y": 0 }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "value", - "options": { - "1": { - "text": "Connected", - "color": "green", - "index": 0 - }, - "0": { - "text": "Disconnected", - "color": "red", - "index": 1 - } - } - }, - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short", - "displayName": "Bot" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 109, - "type": "stat", - "title": "", - "description": "Deployed commit, from /health, cut to 7 characters like the SPA header. The stackdriver plugin runs a range query even for an instant target, so over the dashboard's 24h the previous commit is a second series and its name drew over this one. PromQL also keeps a stopped series for its 5m lookback, so topk on the sample timestamp keeps only the commit with the newest sample, over a 1m window.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=~\"$source\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true, - "legendFormat": "{{git_commit}}" - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 3, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "1m", - "hideTimeOverride": true - }, - { - "id": 110, - "type": "stat", - "title": "", - "description": "Bot process uptime, from /health uptimeSeconds.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 6, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "s", - "displayName": "Uptime", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 111, - "type": "stat", - "title": "", - "description": "The SPA settings bar's Broker pill.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true, - "legendFormat": "{{broker}}" - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 9, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 112, - "type": "stat", - "title": "", - "description": "Equity rebalance target. The SPA pill also shows the band: target +/- liq_settings_equity_deviation.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ + "transformations": [ { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_equity_target{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 12, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "percentunit", - "displayName": "Equity", - "decimals": 0 + "id": "labelsToFields", + "options": {} }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 113, - "type": "stat", - "title": "", - "description": "USDC rebalance target. The SPA pill also shows the band: target +/- liq_settings_usdc_deviation.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_usdc_target{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 16, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "percentunit", - "displayName": "USDC", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 114, - "type": "stat", - "title": "", - "description": "Hedge execution threshold (settings bar 'Trigger').", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true + "id": "merge", + "options": {} } ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 20, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "currencyUSD", - "displayName": "Trigger", - "decimals": 0 - }, - "overrides": [] - }, "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 + "renderMode": "allRows", + "editor": { + "format": "auto", + "language": "html" }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } + "content": "
", + "defaultContent": "No bot data.", + "helpers": "", + "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum that no CCTP burn left the bot wallet after this transfer started (its Started time) and that the wallet has no pending transaction (pending nonce equals latest nonce); if you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, and pending nonce equals latest nonce on more than one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once they show that no withdrawal landed and none is pending, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn left the bot wallet after this transfer started and the wallet has no pending transaction. If any of that is not certain, start the bot and run resume-usdc; else run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", + "styles": "/* Business Text \"styles\" for the Liquidity bot header row and its two\n dialogs. Theme colours come in as --hdr-* variables set by header.js. */\n\n.hdr {\n display: flex;\n align-items: center;\n justify-content: space-between;\n gap: 8px;\n height: 100%;\n font-size: 13px;\n white-space: nowrap;\n}\n\n.hdr-left,\n.hdr-right {\n display: flex;\n align-items: center;\n gap: 8px;\n min-width: 0;\n}\n\n.hdr-pill {\n display: inline-flex;\n align-items: center;\n gap: 4px;\n padding: 2px 10px;\n border-radius: 9999px;\n background: var(--hdr-pill);\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n border: 0;\n}\n\n.hdr-button {\n color: var(--hdr-muted);\n cursor: pointer;\n}\n\n.hdr-button:hover,\n.hdr-guide:hover,\n.hdr-close:hover {\n color: var(--hdr-text);\n}\n\n.hdr-guide {\n padding: 2px 8px;\n border: 1px solid var(--hdr-border);\n border-radius: 4px;\n background: transparent;\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n cursor: pointer;\n}\n\n.hdr-badge {\n padding: 2px 8px;\n border-radius: 9999px;\n font-size: 13px;\n line-height: 18px;\n font-weight: 600;\n}\n\n.hdr-ok { background: #f5f5f5; color: #171717; }\n.hdr-bad { background: #ef4444; color: #fff; }\n.hdr-unknown { border: 1px solid var(--hdr-border); color: var(--hdr-muted); }\n\n.hdr-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.hdr-muted { color: var(--hdr-muted); }\n\n.hdr-dialog {\n padding: 0;\n border: 1px solid var(--hdr-border);\n border-radius: 8px;\n background: var(--hdr-card);\n color: var(--hdr-text);\n white-space: normal;\n}\n\n.hdr-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n.hdr-narrow { width: min(28rem, 92vw); }\n.hdr-wide { width: min(42rem, 92vw); }\n\n.hdr-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--hdr-border);\n font-size: 14px;\n font-weight: 600;\n}\n\n.hdr-close {\n border: 0;\n background: transparent;\n color: var(--hdr-muted);\n font-size: 18px;\n line-height: 1;\n cursor: pointer;\n}\n\n.hdr-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 13px;\n}\n\n.hdr-row {\n display: flex;\n justify-content: space-between;\n gap: 16px;\n padding: 4px 0;\n}\n\n.hdr-trunc {\n overflow: hidden;\n text-overflow: ellipsis;\n white-space: nowrap;\n}\n\n.hdr-rule { border-top: 1px solid var(--hdr-border); margin: 8px 0; }\n.hdr-intro { margin: 0 0 16px; }\n.hdr-group { margin-bottom: 20px; }\n.hdr-group h3 { margin: 0 0 8px; font-size: 13px; text-transform: capitalize; }\n\n.hdr-command {\n margin-bottom: 12px;\n padding: 8px 12px;\n border: 1px solid var(--hdr-border);\n border-radius: 6px;\n background: var(--hdr-pill);\n}\n\n.hdr-command pre {\n margin: 0;\n padding: 8px;\n border-radius: 4px;\n background: var(--hdr-card);\n font-size: 12px;\n white-space: pre-wrap;\n word-break: break-all;\n}\n\n.hdr-grid {\n display: grid;\n grid-template-columns: auto 1fr;\n gap: 4px 12px;\n margin-top: 6px;\n font-size: 12px;\n}\n\n.hdr-amber { color: #f59e0b; }\n.hdr-red { color: #ef4444; }\n\n/* Tailwind 700 on the light theme, like detail.css, for a 4.5:1 contrast\n against the white dialog. */\n.hdr-light .hdr-amber { color: #b45309; }\n.hdr-light .hdr-red { color: #b91c1c; }\n", + "wrap": false }, "timeFrom": "5m", "hideTimeOverride": true }, { - "id": 115, + "id": 85, "type": "table", "title": "Log History", "description": "The bot's structured log lines, re-emitted by the exporter with real severity and target (the raw container stream is all-INFO). Filter with the level/category/target/search variables above. \u26a0\ufe0f Only the newest matching entries, about the panel's pixel width of them. Narrow the filters, or use the SPA for deep digs. DEBUG/TRACE are not shipped.", diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json b/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json index 8057dc6d6..d1982327d 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json @@ -77,9 +77,8 @@ "links": [ { "type": "link", - "title": "Dashboard", + "title": "\ud83d\udcca\u00a0Dashboard", "url": "/d/t0-liquidity/?theme=light&autofitpanels", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -89,9 +88,8 @@ }, { "type": "link", - "title": "\u25b8 Orders", + "title": "\u25b8 \ud83d\udccb\u00a0Orders", "url": "/d/t0-liquidity-orders/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -101,9 +99,8 @@ }, { "type": "link", - "title": "PnL", + "title": "\ud83d\udcb5\u00a0PnL", "url": "/d/t0-liquidity-pnl/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -113,9 +110,8 @@ }, { "type": "link", - "title": "Performance", + "title": "\u26a1\u00a0Performance", "url": "/d/t0-liquidity-performance/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -125,9 +121,8 @@ }, { "type": "link", - "title": "Logs", + "title": "\ud83d\udcdc\u00a0Logs", "url": "/d/t0-liquidity-logs/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -151,10 +146,10 @@ ], "panels": [ { - "id": 14, - "type": "stat", + "id": 8, + "type": "marcusolsson-dynamictext-panel", "title": "", - "description": "The SPA header's connection light: is the bot process up and answering? The exporter source reads liq_up, which is 0 when the exporter cannot reach the bot's /health. The bot never emits liq_up, so the bot source reads the scrape's own up. '\u2014' means the source is not reporting.", + "description": "The SPA header: settings pills, Config, the CLI recovery guide, the deployed commit, uptime, and whether the bot is up: the exporter's liq_up, or the bot scrape's up on the bot source.", "datasource": { "type": "stackdriver", "uid": "cloudmon" @@ -169,7 +164,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"})", + "expr": "label_replace(max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"}), \"k\", \"up\", \"\", \"\") or label_replace(time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"}), \"k\", \"uptime\", \"\", \"\") or label_replace(max by (git_commit) (timestamp(liq_bot_info{job=~\"$source\"})), \"k\", \"commit\", \"\", \"\") or label_replace(max by (broker, log_level, wallet_kind, wallet_address, orderbook, turnkey_organization, server_port) (liq_settings_info{job=~\"$source\"}), \"k\", \"info\", \"\", \"\") or label_replace(max(liq_settings_equity_target{job=~\"$source\"}), \"k\", \"equity_target\", \"\", \"\") or label_replace(max(liq_settings_equity_deviation{job=~\"$source\"}), \"k\", \"equity_deviation\", \"\", \"\") or label_replace(max(liq_settings_usdc_target{job=~\"$source\"}), \"k\", \"usdc_target\", \"\", \"\") or label_replace(max(liq_settings_usdc_deviation{job=~\"$source\"}), \"k\", \"usdc_deviation\", \"\", \"\") or label_replace(max(liq_settings_execution_threshold_usd{job=~\"$source\"}), \"k\", \"trigger\", \"\", \"\") or label_replace(max(liq_settings_cash_reserved{job=~\"$source\"}), \"k\", \"cash_reserved\", \"\", \"\") or label_replace(max(liq_settings_order_polling_seconds{job=~\"$source\"}), \"k\", \"order_polling\", \"\", \"\") or label_replace(max(liq_settings_inventory_poll_seconds{job=~\"$source\"}), \"k\", \"inventory_polling\", \"\", \"\") or label_replace(max(liq_settings_deployment_block{job=~\"$source\"}), \"k\", \"deployment_block\", \"\", \"\")", "step": "60s" }, "timeSeriesList": { @@ -184,602 +179,38 @@ "transparent": true, "gridPos": { "h": 1, - "w": 3, + "w": 24, "x": 0, "y": 0 }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "value", - "options": { - "1": { - "text": "Connected", - "color": "green", - "index": 0 - }, - "0": { - "text": "Disconnected", - "color": "red", - "index": 1 - } - } - }, - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short", - "displayName": "Bot" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 15, - "type": "stat", - "title": "", - "description": "Deployed commit, from /health, cut to 7 characters like the SPA header. The stackdriver plugin runs a range query even for an instant target, so over the dashboard's 24h the previous commit is a second series and its name drew over this one. PromQL also keeps a stopped series for its 5m lookback, so topk on the sample timestamp keeps only the commit with the newest sample, over a 1m window.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ + "transformations": [ { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=~\"$source\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true, - "legendFormat": "{{git_commit}}" - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 3, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 + "id": "labelsToFields", + "options": {} }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "1m", - "hideTimeOverride": true - }, - { - "id": 16, - "type": "stat", - "title": "", - "description": "Bot process uptime, from /health uptimeSeconds.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true + "id": "merge", + "options": {} } ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 6, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "s", - "displayName": "Uptime", - "decimals": 0 - }, - "overrides": [] - }, "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 + "renderMode": "allRows", + "editor": { + "format": "auto", + "language": "html" }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 17, - "type": "stat", - "title": "", - "description": "The SPA settings bar's Broker pill.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true, - "legendFormat": "{{broker}}" - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 9, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 18, - "type": "stat", - "title": "", - "description": "Equity rebalance target. The SPA pill also shows the band: target +/- liq_settings_equity_deviation.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_equity_target{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 12, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "percentunit", - "displayName": "Equity", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 19, - "type": "stat", - "title": "", - "description": "USDC rebalance target. The SPA pill also shows the band: target +/- liq_settings_usdc_deviation.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_usdc_target{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 16, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "percentunit", - "displayName": "USDC", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 20, - "type": "stat", - "title": "", - "description": "Hedge execution threshold (settings bar 'Trigger').", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 20, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "currencyUSD", - "displayName": "Trigger", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } + "content": "
", + "defaultContent": "No bot data.", + "helpers": "", + "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum that no CCTP burn left the bot wallet after this transfer started (its Started time) and that the wallet has no pending transaction (pending nonce equals latest nonce); if you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, and pending nonce equals latest nonce on more than one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once they show that no withdrawal landed and none is pending, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn left the bot wallet after this transfer started and the wallet has no pending transaction. If any of that is not certain, start the bot and run resume-usdc; else run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", + "styles": "/* Business Text \"styles\" for the Liquidity bot header row and its two\n dialogs. Theme colours come in as --hdr-* variables set by header.js. */\n\n.hdr {\n display: flex;\n align-items: center;\n justify-content: space-between;\n gap: 8px;\n height: 100%;\n font-size: 13px;\n white-space: nowrap;\n}\n\n.hdr-left,\n.hdr-right {\n display: flex;\n align-items: center;\n gap: 8px;\n min-width: 0;\n}\n\n.hdr-pill {\n display: inline-flex;\n align-items: center;\n gap: 4px;\n padding: 2px 10px;\n border-radius: 9999px;\n background: var(--hdr-pill);\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n border: 0;\n}\n\n.hdr-button {\n color: var(--hdr-muted);\n cursor: pointer;\n}\n\n.hdr-button:hover,\n.hdr-guide:hover,\n.hdr-close:hover {\n color: var(--hdr-text);\n}\n\n.hdr-guide {\n padding: 2px 8px;\n border: 1px solid var(--hdr-border);\n border-radius: 4px;\n background: transparent;\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n cursor: pointer;\n}\n\n.hdr-badge {\n padding: 2px 8px;\n border-radius: 9999px;\n font-size: 13px;\n line-height: 18px;\n font-weight: 600;\n}\n\n.hdr-ok { background: #f5f5f5; color: #171717; }\n.hdr-bad { background: #ef4444; color: #fff; }\n.hdr-unknown { border: 1px solid var(--hdr-border); color: var(--hdr-muted); }\n\n.hdr-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.hdr-muted { color: var(--hdr-muted); }\n\n.hdr-dialog {\n padding: 0;\n border: 1px solid var(--hdr-border);\n border-radius: 8px;\n background: var(--hdr-card);\n color: var(--hdr-text);\n white-space: normal;\n}\n\n.hdr-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n.hdr-narrow { width: min(28rem, 92vw); }\n.hdr-wide { width: min(42rem, 92vw); }\n\n.hdr-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--hdr-border);\n font-size: 14px;\n font-weight: 600;\n}\n\n.hdr-close {\n border: 0;\n background: transparent;\n color: var(--hdr-muted);\n font-size: 18px;\n line-height: 1;\n cursor: pointer;\n}\n\n.hdr-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 13px;\n}\n\n.hdr-row {\n display: flex;\n justify-content: space-between;\n gap: 16px;\n padding: 4px 0;\n}\n\n.hdr-trunc {\n overflow: hidden;\n text-overflow: ellipsis;\n white-space: nowrap;\n}\n\n.hdr-rule { border-top: 1px solid var(--hdr-border); margin: 8px 0; }\n.hdr-intro { margin: 0 0 16px; }\n.hdr-group { margin-bottom: 20px; }\n.hdr-group h3 { margin: 0 0 8px; font-size: 13px; text-transform: capitalize; }\n\n.hdr-command {\n margin-bottom: 12px;\n padding: 8px 12px;\n border: 1px solid var(--hdr-border);\n border-radius: 6px;\n background: var(--hdr-pill);\n}\n\n.hdr-command pre {\n margin: 0;\n padding: 8px;\n border-radius: 4px;\n background: var(--hdr-card);\n font-size: 12px;\n white-space: pre-wrap;\n word-break: break-all;\n}\n\n.hdr-grid {\n display: grid;\n grid-template-columns: auto 1fr;\n gap: 4px 12px;\n margin-top: 6px;\n font-size: 12px;\n}\n\n.hdr-amber { color: #f59e0b; }\n.hdr-red { color: #ef4444; }\n\n/* Tailwind 700 on the light theme, like detail.css, for a 4.5:1 contrast\n against the white dialog. */\n.hdr-light .hdr-amber { color: #b45309; }\n.hdr-light .hdr-red { color: #b91c1c; }\n", + "wrap": false }, "timeFrom": "5m", "hideTimeOverride": true }, { - "id": 21, + "id": 9, "type": "table", "title": "Raindex Orders", "description": "The SPA's Orders tab, columns in SPA order (Output | Input | Vault Balance | IO Ratio | Order Hash | Created). Shipped by the exporter's collect_orders() as one Cloud Logging row per order per poll cycle (logName liquidity-orders); insertId buckets by poll-hour so a still-active order dedupes to ~1 stored row/hour rather than one per 60s poll \u2014 with a 2h window that means a couple of rows per order is expected, not a bug. Created is the order's own on-chain creation time (jsonPayload.created_at), not the row's ingestion time. The datasource returns at most about the panel's pixel width in entries.", @@ -1064,7 +495,7 @@ } }, { - "id": 22, + "id": 10, "type": "stat", "title": "", "description": "In-flight broker orders right now (/orders/pending).", @@ -1146,7 +577,7 @@ } }, { - "id": 23, + "id": 11, "type": "stat", "title": "", "description": "Raindex orders owned by the bot (upstream REST API total).", @@ -1228,7 +659,7 @@ } }, { - "id": 24, + "id": 12, "type": "stat", "title": "", "description": "1 when the bot's upstream REST API for Raindex orders is unavailable.", diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json b/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json index f6c6754dd..18c2fbc1d 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json @@ -118,9 +118,8 @@ "links": [ { "type": "link", - "title": "Dashboard", + "title": "\ud83d\udcca\u00a0Dashboard", "url": "/d/t0-liquidity/?theme=light&autofitpanels", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -130,9 +129,8 @@ }, { "type": "link", - "title": "Orders", + "title": "\ud83d\udccb\u00a0Orders", "url": "/d/t0-liquidity-orders/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -142,9 +140,8 @@ }, { "type": "link", - "title": "PnL", + "title": "\ud83d\udcb5\u00a0PnL", "url": "/d/t0-liquidity-pnl/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -154,9 +151,8 @@ }, { "type": "link", - "title": "\u25b8 Performance", + "title": "\u25b8 \u26a1\u00a0Performance", "url": "/d/t0-liquidity-performance/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -166,9 +162,8 @@ }, { "type": "link", - "title": "Logs", + "title": "\ud83d\udcdc\u00a0Logs", "url": "/d/t0-liquidity-logs/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -192,374 +187,10 @@ ], "panels": [ { - "id": 52, - "type": "stat", - "title": "", - "description": "The SPA header's connection light: is the bot process up and answering? The exporter source reads liq_up, which is 0 when the exporter cannot reach the bot's /health. The bot never emits liq_up, so the bot source reads the scrape's own up. '\u2014' means the source is not reporting.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 0, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "value", - "options": { - "1": { - "text": "Connected", - "color": "green", - "index": 0 - }, - "0": { - "text": "Disconnected", - "color": "red", - "index": 1 - } - } - }, - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short", - "displayName": "Bot" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 53, - "type": "stat", - "title": "", - "description": "Deployed commit, from /health, cut to 7 characters like the SPA header. The stackdriver plugin runs a range query even for an instant target, so over the dashboard's 24h the previous commit is a second series and its name drew over this one. PromQL also keeps a stopped series for its 5m lookback, so topk on the sample timestamp keeps only the commit with the newest sample, over a 1m window.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=~\"$source\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true, - "legendFormat": "{{git_commit}}" - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 3, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "1m", - "hideTimeOverride": true - }, - { - "id": 54, - "type": "stat", - "title": "", - "description": "Bot process uptime, from /health uptimeSeconds.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 6, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "s", - "displayName": "Uptime", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 55, - "type": "stat", - "title": "", - "description": "The SPA settings bar's Broker pill.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true, - "legendFormat": "{{broker}}" - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 9, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 56, - "type": "stat", + "id": 34, + "type": "marcusolsson-dynamictext-panel", "title": "", - "description": "Equity rebalance target. The SPA pill also shows the band: target +/- liq_settings_equity_deviation.", + "description": "The SPA header: settings pills, Config, the CLI recovery guide, the deployed commit, uptime, and whether the bot is up: the exporter's liq_up, or the bot scrape's up on the bot source.", "datasource": { "type": "stackdriver", "uid": "cloudmon" @@ -574,7 +205,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_equity_target{job=~\"$source\"})", + "expr": "label_replace(max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"}), \"k\", \"up\", \"\", \"\") or label_replace(time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"}), \"k\", \"uptime\", \"\", \"\") or label_replace(max by (git_commit) (timestamp(liq_bot_info{job=~\"$source\"})), \"k\", \"commit\", \"\", \"\") or label_replace(max by (broker, log_level, wallet_kind, wallet_address, orderbook, turnkey_organization, server_port) (liq_settings_info{job=~\"$source\"}), \"k\", \"info\", \"\", \"\") or label_replace(max(liq_settings_equity_target{job=~\"$source\"}), \"k\", \"equity_target\", \"\", \"\") or label_replace(max(liq_settings_equity_deviation{job=~\"$source\"}), \"k\", \"equity_deviation\", \"\", \"\") or label_replace(max(liq_settings_usdc_target{job=~\"$source\"}), \"k\", \"usdc_target\", \"\", \"\") or label_replace(max(liq_settings_usdc_deviation{job=~\"$source\"}), \"k\", \"usdc_deviation\", \"\", \"\") or label_replace(max(liq_settings_execution_threshold_usd{job=~\"$source\"}), \"k\", \"trigger\", \"\", \"\") or label_replace(max(liq_settings_cash_reserved{job=~\"$source\"}), \"k\", \"cash_reserved\", \"\", \"\") or label_replace(max(liq_settings_order_polling_seconds{job=~\"$source\"}), \"k\", \"order_polling\", \"\", \"\") or label_replace(max(liq_settings_inventory_poll_seconds{job=~\"$source\"}), \"k\", \"inventory_polling\", \"\", \"\") or label_replace(max(liq_settings_deployment_block{job=~\"$source\"}), \"k\", \"deployment_block\", \"\", \"\")", "step": "60s" }, "timeSeriesList": { @@ -589,238 +220,38 @@ "transparent": true, "gridPos": { "h": 1, - "w": 4, - "x": 12, + "w": 24, + "x": 0, "y": 0 }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "percentunit", - "displayName": "Equity", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 57, - "type": "stat", - "title": "", - "description": "USDC rebalance target. The SPA pill also shows the band: target +/- liq_settings_usdc_deviation.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ + "transformations": [ { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_usdc_target{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 16, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "percentunit", - "displayName": "USDC", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 + "id": "labelsToFields", + "options": {} }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 58, - "type": "stat", - "title": "", - "description": "Hedge execution threshold (settings bar 'Trigger').", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true + "id": "merge", + "options": {} } ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 20, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "currencyUSD", - "displayName": "Trigger", - "decimals": 0 - }, - "overrides": [] - }, "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 + "renderMode": "allRows", + "editor": { + "format": "auto", + "language": "html" }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } + "content": "
", + "defaultContent": "No bot data.", + "helpers": "", + "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum that no CCTP burn left the bot wallet after this transfer started (its Started time) and that the wallet has no pending transaction (pending nonce equals latest nonce); if you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, and pending nonce equals latest nonce on more than one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once they show that no withdrawal landed and none is pending, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn left the bot wallet after this transfer started and the wallet has no pending transaction. If any of that is not certain, start the bot and run resume-usdc; else run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", + "styles": "/* Business Text \"styles\" for the Liquidity bot header row and its two\n dialogs. Theme colours come in as --hdr-* variables set by header.js. */\n\n.hdr {\n display: flex;\n align-items: center;\n justify-content: space-between;\n gap: 8px;\n height: 100%;\n font-size: 13px;\n white-space: nowrap;\n}\n\n.hdr-left,\n.hdr-right {\n display: flex;\n align-items: center;\n gap: 8px;\n min-width: 0;\n}\n\n.hdr-pill {\n display: inline-flex;\n align-items: center;\n gap: 4px;\n padding: 2px 10px;\n border-radius: 9999px;\n background: var(--hdr-pill);\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n border: 0;\n}\n\n.hdr-button {\n color: var(--hdr-muted);\n cursor: pointer;\n}\n\n.hdr-button:hover,\n.hdr-guide:hover,\n.hdr-close:hover {\n color: var(--hdr-text);\n}\n\n.hdr-guide {\n padding: 2px 8px;\n border: 1px solid var(--hdr-border);\n border-radius: 4px;\n background: transparent;\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n cursor: pointer;\n}\n\n.hdr-badge {\n padding: 2px 8px;\n border-radius: 9999px;\n font-size: 13px;\n line-height: 18px;\n font-weight: 600;\n}\n\n.hdr-ok { background: #f5f5f5; color: #171717; }\n.hdr-bad { background: #ef4444; color: #fff; }\n.hdr-unknown { border: 1px solid var(--hdr-border); color: var(--hdr-muted); }\n\n.hdr-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.hdr-muted { color: var(--hdr-muted); }\n\n.hdr-dialog {\n padding: 0;\n border: 1px solid var(--hdr-border);\n border-radius: 8px;\n background: var(--hdr-card);\n color: var(--hdr-text);\n white-space: normal;\n}\n\n.hdr-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n.hdr-narrow { width: min(28rem, 92vw); }\n.hdr-wide { width: min(42rem, 92vw); }\n\n.hdr-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--hdr-border);\n font-size: 14px;\n font-weight: 600;\n}\n\n.hdr-close {\n border: 0;\n background: transparent;\n color: var(--hdr-muted);\n font-size: 18px;\n line-height: 1;\n cursor: pointer;\n}\n\n.hdr-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 13px;\n}\n\n.hdr-row {\n display: flex;\n justify-content: space-between;\n gap: 16px;\n padding: 4px 0;\n}\n\n.hdr-trunc {\n overflow: hidden;\n text-overflow: ellipsis;\n white-space: nowrap;\n}\n\n.hdr-rule { border-top: 1px solid var(--hdr-border); margin: 8px 0; }\n.hdr-intro { margin: 0 0 16px; }\n.hdr-group { margin-bottom: 20px; }\n.hdr-group h3 { margin: 0 0 8px; font-size: 13px; text-transform: capitalize; }\n\n.hdr-command {\n margin-bottom: 12px;\n padding: 8px 12px;\n border: 1px solid var(--hdr-border);\n border-radius: 6px;\n background: var(--hdr-pill);\n}\n\n.hdr-command pre {\n margin: 0;\n padding: 8px;\n border-radius: 4px;\n background: var(--hdr-card);\n font-size: 12px;\n white-space: pre-wrap;\n word-break: break-all;\n}\n\n.hdr-grid {\n display: grid;\n grid-template-columns: auto 1fr;\n gap: 4px 12px;\n margin-top: 6px;\n font-size: 12px;\n}\n\n.hdr-amber { color: #f59e0b; }\n.hdr-red { color: #ef4444; }\n\n/* Tailwind 700 on the light theme, like detail.css, for a 4.5:1 contrast\n against the white dialog. */\n.hdr-light .hdr-amber { color: #b45309; }\n.hdr-light .hdr-red { color: #b91c1c; }\n", + "wrap": false }, "timeFrom": "5m", "hideTimeOverride": true }, { - "id": 59, + "id": 35, "type": "stat", "title": "Detection latency", "description": "p95, 24h. Fill on-chain \u2192 bot noticed. good \u226430s, warn \u2264120s.", @@ -909,7 +340,7 @@ } }, { - "id": 60, + "id": 36, "type": "stat", "title": "Exposure window", "description": "p95, 24h. Fill on-chain \u2192 hedge filled: how long capital stays exposed. good \u226460s, warn \u2264300s.", @@ -998,7 +429,7 @@ } }, { - "id": 61, + "id": 37, "type": "stat", "title": "Errors (24h)", "description": "ERROR-level log lines in 24h. good \u22640, warn \u226410. Money-at-risk lifecycle failures are broken out per event type in the 'Errors & warnings by module' table below.", @@ -1087,7 +518,7 @@ } }, { - "id": 62, + "id": 38, "type": "stat", "title": "Oldest unhedged fill", "description": "Age of the oldest on-chain fill not yet hedged. good \u22645m, warn \u226430m; '\u2014' = nothing unhedged (the SPA shows 'none').", @@ -1176,7 +607,7 @@ } }, { - "id": 63, + "id": 39, "type": "stat", "title": "Block lag", "description": "Chain blocks the fill-detection checkpoint trails by. good \u226430, warn \u2264300.", @@ -1265,7 +696,7 @@ } }, { - "id": 64, + "id": 40, "type": "stat", "title": "", "description": "SPA secondary line: p50 detection latency.", @@ -1351,7 +782,7 @@ } }, { - "id": 65, + "id": 41, "type": "stat", "title": "", "description": "SPA secondary line: fill sample count backing the detection latency percentiles.", @@ -1437,7 +868,7 @@ } }, { - "id": 66, + "id": 42, "type": "stat", "title": "", "description": "SPA secondary line: p50 exposure-window latency.", @@ -1523,7 +954,7 @@ } }, { - "id": 67, + "id": 43, "type": "stat", "title": "", "description": "SPA secondary line: hedge sample count backing the exposure-window percentiles.", @@ -1609,7 +1040,7 @@ } }, { - "id": 68, + "id": 44, "type": "stat", "title": "", "description": "SPA secondary line: WARN-level log lines in 24h.", @@ -1695,7 +1126,7 @@ } }, { - "id": 69, + "id": 45, "type": "stat", "title": "", "description": "SPA secondary line: money-at-risk lifecycle failure events in 24h (the 'Errors & warnings by module' table breaks these out by event type).", @@ -1781,7 +1212,7 @@ } }, { - "id": 70, + "id": 46, "type": "stat", "title": "", "description": "SPA secondary line: apalis job queues with a failed or killed job right now (instantaneous, not windowed) \u2014 the SPA folds killed into the same 'queue(s) failed' phrase.", @@ -1871,7 +1302,7 @@ } }, { - "id": 71, + "id": 47, "type": "stat", "title": "", "description": "SPA secondary line: on-chain fills still uncovered by a hedge, summed across symbols.", @@ -1957,7 +1388,7 @@ } }, { - "id": 72, + "id": 48, "type": "stat", "title": "", "description": "SPA secondary line: symbols with an open (unhedged) exposure \u2014 0 is the SPA's 'all fills hedged'.", @@ -2053,7 +1484,7 @@ } }, { - "id": 73, + "id": 49, "type": "stat", "title": "", "description": "SPA secondary line: age of the current block-lag sample.", @@ -2139,7 +1570,7 @@ } }, { - "id": 74, + "id": 50, "type": "stat", "title": "", "description": "SPA secondary line: order-fill monitor poll ticks skipped (overrun) in 24h.", @@ -2225,7 +1656,7 @@ } }, { - "id": 75, + "id": 51, "type": "bargauge", "title": "Hedge cycle stages (p95, 24h)", "description": "The SPA waterfall's aggregate form: p95 per stage. Per-cycle waterfalls live in the SPA.", @@ -2303,7 +1734,7 @@ } }, { - "id": 76, + "id": 52, "type": "timeseries", "title": "Latency percentiles over time \u2014 $stage", "description": "p50/p90/p99 of the selected hedge stage, from the exporter's rolling 24h summary scraped once a minute.", @@ -2459,7 +1890,7 @@ } }, { - "id": 77, + "id": 53, "type": "table", "title": "Errors & warnings by module (24h)", "description": "The SPA's per-target error/warning counts. The Logs tab has the lines themselves. Click a module to jump to the Logs board pre-filtered on it (the SPA's 'logs \u2192' button); sparkline cells are skipped as low-value/fiddly here.", @@ -2664,7 +2095,7 @@ } }, { - "id": 78, + "id": 54, "type": "bargauge", "title": "USDC rebalance stages (p95, 30d)", "description": "Stage timings for USDC rebalances. From the server's stageSummary; per-operation waterfalls live in the SPA.", @@ -2734,7 +2165,7 @@ } }, { - "id": 79, + "id": 55, "type": "bargauge", "title": "Equity rebalance stages (p95, 30d)", "description": "Stage timings for equity mints/redemptions.", @@ -2804,7 +2235,7 @@ } }, { - "id": 80, + "id": 56, "type": "table", "title": "Dependency health (24h)", "description": "Every RPC / broker operation: calls, errors, latency percentiles.", @@ -2964,7 +2395,7 @@ } }, { - "id": 81, + "id": 57, "type": "timeseries", "title": "CCTP attestation time", "description": "Duration of the most recent CCTP attestation per rebalance kind, sampled once per scrape. The SPA can chart the full attestationTrend history for the selected range; this gauge only has 'last value at scrape time', so the line only accrues from whenever this panel started being scraped forward \u2014 a short/flat history here does not mean attestations were fast, it means the metric is young.", @@ -3085,7 +2516,7 @@ } }, { - "id": 82, + "id": 58, "type": "timeseries", "title": "Block lag over time", "description": "The ingestion-health chart: how far fill detection trails the chain.", @@ -3156,7 +2587,7 @@ } }, { - "id": 83, + "id": 59, "type": "stat", "title": "Poll cycles (24h)", "description": "Order-fill monitor poll cycles.", @@ -3237,7 +2668,7 @@ } }, { - "id": 84, + "id": 60, "type": "stat", "title": "Poll errors (24h)", "description": "Failed poll cycles.", @@ -3322,7 +2753,7 @@ } }, { - "id": 85, + "id": 61, "type": "stat", "title": "Skipped ticks (24h)", "description": "Poll ticks skipped (overrun).", @@ -3407,7 +2838,7 @@ } }, { - "id": 86, + "id": 62, "type": "bargauge", "title": "Poll duration percentiles", "description": "The monitor's own cycle duration.", @@ -3477,7 +2908,7 @@ } }, { - "id": 87, + "id": 63, "type": "table", "title": "Job queues", "description": "The bot's apalis job queues (instantaneous whole-table counts, not windowed). killed or failed above zero is the SPA's critical Errors state.", @@ -3663,7 +3094,7 @@ } }, { - "id": 88, + "id": 64, "type": "row", "title": "Hedge execution quality (native bot metrics)", "collapsed": true, @@ -3675,7 +3106,7 @@ }, "panels": [ { - "id": 89, + "id": 65, "type": "timeseries", "title": "Hedge fill latency percentiles", "description": "Wall-clock seconds from hedge order placement to confirmed fill. The app builds these as a metrics-rs summary (no explicit buckets), so the quantiles come straight off the {quantile=...} label rather than from histogram_quantile. This is the metrics-side view of the SPA's 'Latency percentiles over time' chart \u2014 that chart's per-stage waterfall (detection / decision / submission / execution) comes from the telemetry tables in SQLite and has no metric behind it.", @@ -3787,7 +3218,7 @@ } }, { - "id": 90, + "id": 66, "type": "timeseries", "title": "Hedge fill latency by symbol (1h average)", "description": "Average seconds between placing a hedge and it filling, per symbol, derived from the summary's _sum / _count. Spikes mean capital sits exposed longer on that name.", @@ -3862,7 +3293,7 @@ } }, { - "id": 91, + "id": 67, "type": "timeseries", "title": "Broker errors per hour, by kind", "description": "Alpaca API failures, split by kind (e.g. place_order_failed) and symbol. Empty is the healthy state.", @@ -3932,7 +3363,7 @@ } }, { - "id": 92, + "id": 68, "type": "timeseries", "title": "Hedge scans skipped, by reason", "description": "Extended-hours buys the position scan dropped before enqueueing a hedge job, because reference-price resolution or crossing failed. Each one is an on-chain fill that did not get hedged on that pass.", @@ -4002,7 +3433,7 @@ } }, { - "id": 93, + "id": 69, "type": "timeseries", "title": "Hedges dead-lettered, by reason", "description": "Hedge attempts the process gave up on: a permanent or retry-budget-exhausted pricing failure, or broker rate-limiting that outlived the reschedule budget. Anything here is an unhedged fill that needs a human.", @@ -4072,7 +3503,7 @@ } }, { - "id": 94, + "id": 70, "type": "timeseries", "title": "Which reference priced each extended-hours limit", "description": "hedge_price_source_total by source (primary_quote / mark / delayed_sip_quote) and path (ordinary_extended / close_flatten). Shows which fallback legs are load-bearing \u2014 a rise in delayed_sip_quote means we are pricing off stale data.", @@ -4146,7 +3577,7 @@ ] }, { - "id": 95, + "id": 71, "type": "row", "title": "Close-flatten (end-of-session flattening) (native bot metrics)", "collapsed": true, @@ -4158,7 +3589,7 @@ }, "panels": [ { - "id": 96, + "id": 72, "type": "timeseries", "title": "Attempts, by post-close gap reason", "description": "Close-flatten hedge pricing attempts, by the post-close gap reason that triggered them.", @@ -4228,7 +3659,7 @@ } }, { - "id": 97, + "id": 73, "type": "timeseries", "title": "Blocked before submission, by reason", "description": "Close-flatten attempts stopped before they reached the broker. close_time_unknown carries no symbol label; the rest are per symbol.", @@ -4298,7 +3729,7 @@ } }, { - "id": 98, + "id": 74, "type": "timeseries", "title": "Placements, by cross applied", "description": "Limit orders priced and cleared for submission, bucketed to whole percent of cross. Counts attempts, not orders: a retried or re-driven placement of the same hedge counts again, and an attempt the broker later rejects still counts.", @@ -4368,7 +3799,7 @@ } }, { - "id": 99, + "id": 75, "type": "timeseries", "title": "Outcomes", "description": "Terminal broker-state dispatches for close-flatten placements (filled / cancelled / failed). A placement can be observed again until its follow-up job commits the terminal aggregate transition, so treat these as dispatches rather than a strict order count.", @@ -4440,7 +3871,7 @@ ] }, { - "id": 100, + "id": 76, "type": "row", "title": "Data integrity & ingestion health (native bot metrics)", "collapsed": true, @@ -4452,7 +3883,7 @@ }, "panels": [ { - "id": 101, + "id": 77, "type": "stat", "title": "Ambiguous settlements (24h)", "description": "Inventory settlements quarantined because one transaction emitted multiple OperatorDeposits or multiple OperatorWithdraws and the legs could not be safely paired. Each one is a held-back settlement.", @@ -4542,7 +3973,7 @@ } }, { - "id": 102, + "id": 78, "type": "stat", "title": "Unpaired settlement legs (24h)", "description": "OperatorDeposit / OperatorWithdraw legs with no same-transaction counterpart in the batch.", @@ -4632,7 +4063,7 @@ } }, { - "id": 103, + "id": 79, "type": "stat", "title": "Unusable snapshot marks (24h)", "description": "Nonzero equity balances captured with a missing or stale USD mark. These make the portfolio snapshot \u2014 and therefore the PnL replay in the SPA \u2014 incomplete for that interval.", @@ -4722,7 +4153,7 @@ } }, { - "id": 104, + "id": 80, "type": "stat", "title": "PnL ledger checkpoint lag", "description": "How many events the PnL ledger's checkpoint trails the event store by. The PnL numbers themselves are computed by the backend from SQLite and are not exported \u2014 this gauge is the one PnL signal that reaches Grafana, and it answers 'is the PnL panel's input keeping up?'.", @@ -4816,7 +4247,7 @@ } }, { - "id": 105, + "id": 81, "type": "timeseries", "title": "Scrape health", "description": "The Ops Agent's own view of the bot's /metrics endpoint: 1 = scraped successfully, 0 = the endpoint did not answer. Every other panel on this dashboard goes blank while this is 0, so check it first before concluding the bot is idle.", @@ -4889,7 +4320,7 @@ } }, { - "id": 106, + "id": 82, "type": "timeseries", "title": "Scrape duration and series count", "description": "How long the scrape takes and how many series come back. A series count that collapses to near zero while up=1 means the bot restarted and its counters reset.", @@ -4980,7 +4411,7 @@ } }, { - "id": 107, + "id": 83, "type": "timeseries", "title": "Bot-gas redrives and PnL catch-up failures", "description": "bot_gas_redrive_total: receipt-cost enqueue failures redriven instead of failing the triggering job. pnl_ledger_catch_up_failures_total: the PnL ledger failing to catch up to the event store. Both are self-healing retries \u2014 a steady rate means something underneath is not healing.", diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json b/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json index 71fadf3b5..eb5277c17 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json @@ -124,9 +124,8 @@ "links": [ { "type": "link", - "title": "Dashboard", + "title": "\ud83d\udcca\u00a0Dashboard", "url": "/d/t0-liquidity/?theme=light&autofitpanels", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -136,9 +135,8 @@ }, { "type": "link", - "title": "Orders", + "title": "\ud83d\udccb\u00a0Orders", "url": "/d/t0-liquidity-orders/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -148,9 +146,8 @@ }, { "type": "link", - "title": "\u25b8 PnL", + "title": "\u25b8 \ud83d\udcb5\u00a0PnL", "url": "/d/t0-liquidity-pnl/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -160,9 +157,8 @@ }, { "type": "link", - "title": "Performance", + "title": "\u26a1\u00a0Performance", "url": "/d/t0-liquidity-performance/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -172,9 +168,8 @@ }, { "type": "link", - "title": "Logs", + "title": "\ud83d\udcdc\u00a0Logs", "url": "/d/t0-liquidity-logs/?theme=light", - "icon": "dashboard", "targetBlank": false, "keepTime": true, "includeVars": true, @@ -198,374 +193,10 @@ ], "panels": [ { - "id": 25, - "type": "stat", - "title": "", - "description": "The SPA header's connection light: is the bot process up and answering? The exporter source reads liq_up, which is 0 when the exporter cannot reach the bot's /health. The bot never emits liq_up, so the bot source reads the scrape's own up. '\u2014' means the source is not reporting.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 0, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "value", - "options": { - "1": { - "text": "Connected", - "color": "green", - "index": 0 - }, - "0": { - "text": "Disconnected", - "color": "red", - "index": 1 - } - } - }, - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short", - "displayName": "Bot" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 26, - "type": "stat", - "title": "", - "description": "Deployed commit, from /health, cut to 7 characters like the SPA header. The stackdriver plugin runs a range query even for an instant target, so over the dashboard's 24h the previous commit is a second series and its name drew over this one. PromQL also keeps a stopped series for its 5m lookback, so topk on the sample timestamp keeps only the commit with the newest sample, over a 1m window.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=~\"$source\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true, - "legendFormat": "{{git_commit}}" - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 3, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "1m", - "hideTimeOverride": true - }, - { - "id": 27, - "type": "stat", - "title": "", - "description": "Bot process uptime, from /health uptimeSeconds.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 6, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "s", - "displayName": "Uptime", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 28, - "type": "stat", - "title": "", - "description": "The SPA settings bar's Broker pill.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true, - "legendFormat": "{{broker}}" - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 3, - "x": 9, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 29, - "type": "stat", + "id": 13, + "type": "marcusolsson-dynamictext-panel", "title": "", - "description": "Equity rebalance target. The SPA pill also shows the band: target +/- liq_settings_equity_deviation.", + "description": "The SPA header: settings pills, Config, the CLI recovery guide, the deployed commit, uptime, and whether the bot is up: the exporter's liq_up, or the bot scrape's up on the bot source.", "datasource": { "type": "stackdriver", "uid": "cloudmon" @@ -580,7 +211,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_equity_target{job=~\"$source\"})", + "expr": "label_replace(max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"}), \"k\", \"up\", \"\", \"\") or label_replace(time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"}), \"k\", \"uptime\", \"\", \"\") or label_replace(max by (git_commit) (timestamp(liq_bot_info{job=~\"$source\"})), \"k\", \"commit\", \"\", \"\") or label_replace(max by (broker, log_level, wallet_kind, wallet_address, orderbook, turnkey_organization, server_port) (liq_settings_info{job=~\"$source\"}), \"k\", \"info\", \"\", \"\") or label_replace(max(liq_settings_equity_target{job=~\"$source\"}), \"k\", \"equity_target\", \"\", \"\") or label_replace(max(liq_settings_equity_deviation{job=~\"$source\"}), \"k\", \"equity_deviation\", \"\", \"\") or label_replace(max(liq_settings_usdc_target{job=~\"$source\"}), \"k\", \"usdc_target\", \"\", \"\") or label_replace(max(liq_settings_usdc_deviation{job=~\"$source\"}), \"k\", \"usdc_deviation\", \"\", \"\") or label_replace(max(liq_settings_execution_threshold_usd{job=~\"$source\"}), \"k\", \"trigger\", \"\", \"\") or label_replace(max(liq_settings_cash_reserved{job=~\"$source\"}), \"k\", \"cash_reserved\", \"\", \"\") or label_replace(max(liq_settings_order_polling_seconds{job=~\"$source\"}), \"k\", \"order_polling\", \"\", \"\") or label_replace(max(liq_settings_inventory_poll_seconds{job=~\"$source\"}), \"k\", \"inventory_polling\", \"\", \"\") or label_replace(max(liq_settings_deployment_block{job=~\"$source\"}), \"k\", \"deployment_block\", \"\", \"\")", "step": "60s" }, "timeSeriesList": { @@ -595,238 +226,38 @@ "transparent": true, "gridPos": { "h": 1, - "w": 4, - "x": 12, + "w": 24, + "x": 0, "y": 0 }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "percentunit", - "displayName": "Equity", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 30, - "type": "stat", - "title": "", - "description": "USDC rebalance target. The SPA pill also shows the band: target +/- liq_settings_usdc_deviation.", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ + "transformations": [ { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_usdc_target{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true - } - ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 16, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "percentunit", - "displayName": "USDC", - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 + "id": "labelsToFields", + "options": {} }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "timeFrom": "5m", - "hideTimeOverride": true - }, - { - "id": 31, - "type": "stat", - "title": "", - "description": "Hedge execution threshold (settings bar 'Trigger').", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "stackdriver", - "uid": "cloudmon" - }, - "queryType": "promQL", - "promQLQuery": { - "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd{job=~\"$source\"})", - "step": "60s" - }, - "timeSeriesList": { - "projectName": "", - "filters": [], - "view": "FULL", - "groupBys": [] - }, - "instant": true + "id": "merge", + "options": {} } ], - "transparent": true, - "gridPos": { - "h": 1, - "w": 4, - "x": 20, - "y": 0 - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "special", - "options": { - "match": "null+nan", - "result": { - "text": "\u2014", - "color": "text", - "index": 99 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "currencyUSD", - "displayName": "Trigger", - "decimals": 0 - }, - "overrides": [] - }, "options": { - "colorMode": "none", - "graphMode": "none", - "textMode": "value_and_name", - "justifyMode": "center", - "text": { - "valueSize": 14, - "titleSize": 12 - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } + "renderMode": "allRows", + "editor": { + "format": "auto", + "language": "html" + }, + "content": "
", + "defaultContent": "No bot data.", + "helpers": "", + "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum that no CCTP burn left the bot wallet after this transfer started (its Started time) and that the wallet has no pending transaction (pending nonce equals latest nonce); if you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, and pending nonce equals latest nonce on more than one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once they show that no withdrawal landed and none is pending, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn left the bot wallet after this transfer started and the wallet has no pending transaction. If any of that is not certain, start the bot and run resume-usdc; else run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", + "styles": "/* Business Text \"styles\" for the Liquidity bot header row and its two\n dialogs. Theme colours come in as --hdr-* variables set by header.js. */\n\n.hdr {\n display: flex;\n align-items: center;\n justify-content: space-between;\n gap: 8px;\n height: 100%;\n font-size: 13px;\n white-space: nowrap;\n}\n\n.hdr-left,\n.hdr-right {\n display: flex;\n align-items: center;\n gap: 8px;\n min-width: 0;\n}\n\n.hdr-pill {\n display: inline-flex;\n align-items: center;\n gap: 4px;\n padding: 2px 10px;\n border-radius: 9999px;\n background: var(--hdr-pill);\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n border: 0;\n}\n\n.hdr-button {\n color: var(--hdr-muted);\n cursor: pointer;\n}\n\n.hdr-button:hover,\n.hdr-guide:hover,\n.hdr-close:hover {\n color: var(--hdr-text);\n}\n\n.hdr-guide {\n padding: 2px 8px;\n border: 1px solid var(--hdr-border);\n border-radius: 4px;\n background: transparent;\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n cursor: pointer;\n}\n\n.hdr-badge {\n padding: 2px 8px;\n border-radius: 9999px;\n font-size: 13px;\n line-height: 18px;\n font-weight: 600;\n}\n\n.hdr-ok { background: #f5f5f5; color: #171717; }\n.hdr-bad { background: #ef4444; color: #fff; }\n.hdr-unknown { border: 1px solid var(--hdr-border); color: var(--hdr-muted); }\n\n.hdr-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.hdr-muted { color: var(--hdr-muted); }\n\n.hdr-dialog {\n padding: 0;\n border: 1px solid var(--hdr-border);\n border-radius: 8px;\n background: var(--hdr-card);\n color: var(--hdr-text);\n white-space: normal;\n}\n\n.hdr-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n.hdr-narrow { width: min(28rem, 92vw); }\n.hdr-wide { width: min(42rem, 92vw); }\n\n.hdr-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--hdr-border);\n font-size: 14px;\n font-weight: 600;\n}\n\n.hdr-close {\n border: 0;\n background: transparent;\n color: var(--hdr-muted);\n font-size: 18px;\n line-height: 1;\n cursor: pointer;\n}\n\n.hdr-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 13px;\n}\n\n.hdr-row {\n display: flex;\n justify-content: space-between;\n gap: 16px;\n padding: 4px 0;\n}\n\n.hdr-trunc {\n overflow: hidden;\n text-overflow: ellipsis;\n white-space: nowrap;\n}\n\n.hdr-rule { border-top: 1px solid var(--hdr-border); margin: 8px 0; }\n.hdr-intro { margin: 0 0 16px; }\n.hdr-group { margin-bottom: 20px; }\n.hdr-group h3 { margin: 0 0 8px; font-size: 13px; text-transform: capitalize; }\n\n.hdr-command {\n margin-bottom: 12px;\n padding: 8px 12px;\n border: 1px solid var(--hdr-border);\n border-radius: 6px;\n background: var(--hdr-pill);\n}\n\n.hdr-command pre {\n margin: 0;\n padding: 8px;\n border-radius: 4px;\n background: var(--hdr-card);\n font-size: 12px;\n white-space: pre-wrap;\n word-break: break-all;\n}\n\n.hdr-grid {\n display: grid;\n grid-template-columns: auto 1fr;\n gap: 4px 12px;\n margin-top: 6px;\n font-size: 12px;\n}\n\n.hdr-amber { color: #f59e0b; }\n.hdr-red { color: #ef4444; }\n\n/* Tailwind 700 on the light theme, like detail.css, for a 4.5:1 contrast\n against the white dialog. */\n.hdr-light .hdr-amber { color: #b45309; }\n.hdr-light .hdr-red { color: #b91c1c; }\n", + "wrap": false }, "timeFrom": "5m", "hideTimeOverride": true }, { - "id": 32, + "id": 14, "type": "stat", "title": "", "description": "Gross minus costs plus revenues. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -920,7 +351,7 @@ } }, { - "id": 33, + "id": 15, "type": "stat", "title": "", "description": "Closed-lot fill replay before costs. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -1014,7 +445,7 @@ } }, { - "id": 34, + "id": 16, "type": "stat", "title": "", "description": "Cost/revenue ledger entries. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -1100,7 +531,7 @@ } }, { - "id": 35, + "id": 17, "type": "stat", "title": "", "description": "Dividends and broker ledger credits. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -1194,7 +625,7 @@ } }, { - "id": 36, + "id": 18, "type": "stat", "title": "", "description": "Broker hedge stream. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -1288,7 +719,7 @@ } }, { - "id": 37, + "id": 19, "type": "stat", "title": "", "description": "Users passively crossed open inventory. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -1382,7 +813,7 @@ } }, { - "id": 38, + "id": 20, "type": "stat", "title": "", "description": "Closed delayed or offchain-origin exposure. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -1476,7 +907,7 @@ } }, { - "id": 39, + "id": 21, "type": "stat", "title": "", "description": "Requires historical snapshots. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -1570,7 +1001,7 @@ } }, { - "id": 40, + "id": 22, "type": "stat", "title": "", "description": "All directional streams together. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -1664,7 +1095,7 @@ } }, { - "id": 41, + "id": 23, "type": "stat", "title": "", "description": "Every stream summed. (SPA PnL tile, computed by the bot's replay \u2014 5-min refresh cadence.)", @@ -1758,7 +1189,7 @@ } }, { - "id": 42, + "id": 24, "type": "stat", "title": "", "description": "Open replay inventory in shares (the SPA's muted tile).", @@ -1840,7 +1271,7 @@ } }, { - "id": 43, + "id": 25, "type": "stat", "title": "", "description": "Total fills in the PnL database sample.", @@ -1922,7 +1353,7 @@ } }, { - "id": 44, + "id": 26, "type": "stat", "title": "", "description": "Average deployed capital from daily snapshots \u2014 computed by the bot, never rendered by the SPA.", @@ -2004,7 +1435,7 @@ } }, { - "id": 45, + "id": 27, "type": "stat", "title": "", "description": "Annualized return on that capital \u2014 computed by the bot, never rendered by the SPA.", @@ -2094,7 +1525,7 @@ } }, { - "id": 46, + "id": 28, "type": "stat", "title": "", "description": "Days of snapshot coverage behind the capital figures.", @@ -2176,7 +1607,7 @@ } }, { - "id": 47, + "id": 29, "type": "table", "title": "Per Asset PnL", "description": "The SPA's Per Asset PnL table (closed-lot replay per symbol; Lots = matched lot count; Volume = both legs of the matched shares, without the SPA's USD sub-line).", @@ -2706,7 +2137,7 @@ } }, { - "id": 48, + "id": 30, "type": "barchart", "title": "PnL by Asset per Time Unit", "description": "One bar per calendar day, split by asset and stacked, straight from the bot's daily PnL buckets. A day with no closed lot for an asset is a zero, as in the SPA.", @@ -2853,7 +2284,7 @@ } }, { - "id": 49, + "id": 31, "type": "barchart", "title": "PnL by PnL-Stream per Time Unit", "description": "The same days split by PnL stream, the SPA's four component streams only, since directional_exposure and total are tile-only aggregates there and stacking them beside their own components double-counts.", @@ -3061,7 +2492,7 @@ } }, { - "id": 50, + "id": 32, "type": "barchart", "title": "Cumulative PnL by Asset", "description": "Running total per asset through the selected range. Unstacked: the SPA draws one independent line per asset here, not a stack. The last day's bars sum to the Total PnL tile above.", @@ -3208,7 +2639,7 @@ } }, { - "id": 51, + "id": 33, "type": "barchart", "title": "Cumulative PnL by PnL-Stream", "description": "Running total per stream, unstacked like the SPA's lines. Grafana has no categorical-axis line chart, so these are bars over the same day buckets: the numbers are the SPA's, the shape is not.", diff --git a/observability/gen-t0-liquidity.py b/observability/gen-t0-liquidity.py index 034ae3e9b..0bded1191 100644 --- a/observability/gen-t0-liquidity.py +++ b/observability/gen-t0-liquidity.py @@ -678,27 +678,32 @@ def width(px): # theme=light so the suite renders in the SPA's light look regardless of # the viewer's Grafana theme preference. # ========================================================================== +# Each tab carries its own symbol in its label. Grafana's link buttons take +# only seven built-in icons (external link, dashboard, question, info, bolt, +# doc, cloud), too few to tell five tabs apart. TABS = [ - ("Dashboard", "t0-liquidity"), - ("Orders", "t0-liquidity-orders"), - ("PnL", "t0-liquidity-pnl"), - ("Performance", "t0-liquidity-performance"), - ("Logs", "t0-liquidity-logs"), + ("Dashboard", "t0-liquidity", "πŸ“Š"), + ("Orders", "t0-liquidity-orders", "πŸ“‹"), + ("PnL", "t0-liquidity-pnl", "πŸ’΅"), + ("Performance", "t0-liquidity-performance", "⚑"), + ("Logs", "t0-liquidity-logs", "πŸ“œ"), ] def tab_links(active): links = [] - for name, uid in TABS: + for name, uid, symbol in TABS: links.append({ "type": "link", - "title": f"β–Έ {name}" if name == active else name, + # A no-break space: Grafana trims a plain one after the symbol. + "title": (f"β–Έ {symbol}\u00a0{name}" if name == active + else f"{symbol}\u00a0{name}"), # autofitpanels stretches the board to the window height, like # the SPA's full-height layout. Only the Dashboard tab: the # other tabs hold too many panels to squeeze into one screen. "url": f"/d/{uid}/?theme=light" + ("&autofitpanels" if uid == "t0-liquidity" else ""), - "icon": "dashboard", "targetBlank": False, + "targetBlank": False, "keepTime": True, "includeVars": True, "asDropdown": False, "tags": [], "tooltip": "", }) @@ -712,69 +717,71 @@ def tab_links(active): def pills(y): - """The SPA's header + settings bar as one thin pill row, repeated on - every tab like the SPA repeats its header. One grid row tall: a pill - with an empty title gets a hover-only header, so the value fills the - 30px row instead of sitting under a title line. - - The SPA has no Mode or CT-assets pill, so this row has neither. A stat cannot - join two series into one string, so the Equity and USDC pills show - the target only; the band is in their descriptions.""" - def pill(title, desc, expr, display=None, unit="short", decimals=None, - mappings=None, legend=None, text_mode="value_and_name", x=0, - w=3): - return {**stat(title, desc, expr, display=display, unit=unit, - decimals=decimals, mappings=mappings, legend=legend, - text_mode=text_mode, - color_mode="none" if not mappings else "value", - w=w, h=1, x=x, y=y, value_size=14), - **LATEST_ONLY_HIDDEN} - - return [ - pill("", "The SPA header's connection light: is the bot process up " - "and answering? The exporter source reads liq_up, which is 0 " - "when the exporter cannot reach the bot's /health. The bot " - "never emits liq_up, so the bot source reads the scrape's own " - "up. 'β€”' means the source is not reporting.", - LIQ_UP, - display="Bot", - mappings=[{"type": "value", "options": { - "1": {"text": "Connected", "color": "green", "index": 0}, - "0": {"text": "Disconnected", "color": "red", "index": 1}}}], - x=0, w=3), - {**pill("", "Deployed commit, from /health, cut to 7 " - "characters like the SPA header. The stackdriver plugin " - "runs a range query even for an instant target, so over " - "the dashboard's 24h the previous commit is a second series " - "and its name drew over this one. PromQL also keeps a " - "stopped series for its 5m lookback, so topk on the sample " - "timestamp keeps only the commit with the newest sample, " - "over a 1m window.", - # max by drops every other label: this datasource ignores - # legendFormat, so the series name must be the commit alone. - 'max by (git_commit) (topk(1, label_replace(' - 'timestamp(liq_bot_info), "git_commit", "$1", "git_commit", ' - '"(.{7}).*")))', - legend="{{git_commit}}", text_mode="name", x=3, w=3), - "timeFrom": "1m"}, - pill("", "Bot process uptime, from /health uptimeSeconds.", - "time() - max(liq_bot_start_timestamp_seconds)", - display="Uptime", unit="s", decimals=0, x=6, w=3), - pill("", "The SPA settings bar's Broker pill.", - "sum by (broker) (liq_settings_info)", - legend="{{broker}}", text_mode="name", x=9, w=3), - pill("", "Equity rebalance target. The SPA pill also shows the " - "band: target +/- liq_settings_equity_deviation.", - "max(liq_settings_equity_target)", display="Equity", - unit="percentunit", decimals=0, x=12, w=4), - pill("", "USDC rebalance target. The SPA pill also shows the " - "band: target +/- liq_settings_usdc_deviation.", - "max(liq_settings_usdc_target)", display="USDC", - unit="percentunit", decimals=0, x=16, w=4), - pill("", "Hedge execution threshold (settings bar 'Trigger').", - "max(liq_settings_execution_threshold_usd)", display="Trigger", - unit="currencyUSD", decimals=0, x=20, w=4), - ] + """The SPA's HeaderBar and SettingsBar as one Business Text row, + repeated on every tab like the SPA repeats its header: the settings + pills (broker, Equity and USDC targets with their bands, Trigger, + Reserve) and a Config button on the left; the CLI recovery guide + button, a UTC clock, the commit, uptime, and the connection badge on + the right. Config and the recovery guide open the SPA's dialogs. + + Every series carries a `k` label naming it; liquidity-panels/header.js + reads the rows by `k`. The commit series is its sample timestamp, so + the script can keep the newest commit when a deploy leaves the previous + one in the lookback window. The recovery guide is static data exported + from the SPA (liquidity-panels/recovery-guide.json) and prepended to + the script. + """ + def named(expr, name): + return f'label_replace({expr}, "k", "{name}", "", "")' + + expr = " or ".join([ + named(LIQ_UP, "up"), + named("time() - max(liq_bot_start_timestamp_seconds)", "uptime"), + named("max by (git_commit) (timestamp(liq_bot_info))", "commit"), + named("max by (broker, log_level, wallet_kind, wallet_address, orderbook, " + "turnkey_organization, server_port) (liq_settings_info)", "info"), + named("max(liq_settings_equity_target)", "equity_target"), + named("max(liq_settings_equity_deviation)", "equity_deviation"), + named("max(liq_settings_usdc_target)", "usdc_target"), + named("max(liq_settings_usdc_deviation)", "usdc_deviation"), + named("max(liq_settings_execution_threshold_usd)", "trigger"), + named("max(liq_settings_cash_reserved)", "cash_reserved"), + named("max(liq_settings_order_polling_seconds)", "order_polling"), + named("max(liq_settings_inventory_poll_seconds)", "inventory_polling"), + named("max(liq_settings_deployment_block)", "deployment_block"), + ]) + with open(os.path.join(HERE, "liquidity-panels", "recovery-guide.json")) as f: + guide = json.load(f) + with open(os.path.join(HERE, "liquidity-panels", "header.js")) as f: + after_render = f"const RECOVERY_GUIDE = {json.dumps(guide)};\n\n" + f.read() + with open(os.path.join(HERE, "liquidity-panels", "header.css")) as f: + styles = f.read() + return [{ + "id": nid(), "type": "marcusolsson-dynamictext-panel", "title": "", + "description": "The SPA header: settings pills, Config, the CLI " + "recovery guide, the deployed commit, uptime, and " + "whether the bot is up: the exporter's liq_up, or " + "the bot scrape's up on the bot source.", + "datasource": CM, + "targets": [promql(expr, instant=True)], + "transparent": True, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": y}, + "transformations": [ + {"id": "labelsToFields", "options": {}}, + {"id": "merge", "options": {}}, + ], + "options": { + "renderMode": "allRows", + "editor": {"format": "auto", "language": "html"}, + "content": "
", + "defaultContent": "No bot data.", + "helpers": "", + "afterRender": after_render, + "styles": styles, + "wrap": False, + }, + **LATEST_ONLY_HIDDEN, + }] # The SPA's PnL range buttons (1W / 1M / YTD / 1Y / All). PnL is a windowed diff --git a/observability/liquidity-panels/header.css b/observability/liquidity-panels/header.css new file mode 100644 index 000000000..2ce12e4c2 --- /dev/null +++ b/observability/liquidity-panels/header.css @@ -0,0 +1,161 @@ +/* Business Text "styles" for the Liquidity bot header row and its two + dialogs. Theme colours come in as --hdr-* variables set by header.js. */ + +.hdr { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + height: 100%; + font-size: 13px; + white-space: nowrap; +} + +.hdr-left, +.hdr-right { + display: flex; + align-items: center; + gap: 8px; + min-width: 0; +} + +.hdr-pill { + display: inline-flex; + align-items: center; + gap: 4px; + padding: 2px 10px; + border-radius: 9999px; + background: var(--hdr-pill); + color: var(--hdr-text); + font-size: 13px; + line-height: 18px; + border: 0; +} + +.hdr-button { + color: var(--hdr-muted); + cursor: pointer; +} + +.hdr-button:hover, +.hdr-guide:hover, +.hdr-close:hover { + color: var(--hdr-text); +} + +.hdr-guide { + padding: 2px 8px; + border: 1px solid var(--hdr-border); + border-radius: 4px; + background: transparent; + color: var(--hdr-text); + font-size: 13px; + line-height: 18px; + cursor: pointer; +} + +.hdr-badge { + padding: 2px 8px; + border-radius: 9999px; + font-size: 13px; + line-height: 18px; + font-weight: 600; +} + +.hdr-ok { background: #f5f5f5; color: #171717; } +.hdr-bad { background: #ef4444; color: #fff; } +.hdr-unknown { border: 1px solid var(--hdr-border); color: var(--hdr-muted); } + +.hdr-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; } +.hdr-muted { color: var(--hdr-muted); } + +.hdr-dialog { + padding: 0; + border: 1px solid var(--hdr-border); + border-radius: 8px; + background: var(--hdr-card); + color: var(--hdr-text); + white-space: normal; +} + +.hdr-dialog::backdrop { background: rgba(0, 0, 0, 0.5); } +.hdr-narrow { width: min(28rem, 92vw); } +.hdr-wide { width: min(42rem, 92vw); } + +.hdr-dialog-head { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 20px; + border-bottom: 1px solid var(--hdr-border); + font-size: 14px; + font-weight: 600; +} + +.hdr-close { + border: 0; + background: transparent; + color: var(--hdr-muted); + font-size: 18px; + line-height: 1; + cursor: pointer; +} + +.hdr-dialog-body { + max-height: 70vh; + overflow-y: auto; + padding: 16px 20px; + font-size: 13px; +} + +.hdr-row { + display: flex; + justify-content: space-between; + gap: 16px; + padding: 4px 0; +} + +.hdr-trunc { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.hdr-rule { border-top: 1px solid var(--hdr-border); margin: 8px 0; } +.hdr-intro { margin: 0 0 16px; } +.hdr-group { margin-bottom: 20px; } +.hdr-group h3 { margin: 0 0 8px; font-size: 13px; text-transform: capitalize; } + +.hdr-command { + margin-bottom: 12px; + padding: 8px 12px; + border: 1px solid var(--hdr-border); + border-radius: 6px; + background: var(--hdr-pill); +} + +.hdr-command pre { + margin: 0; + padding: 8px; + border-radius: 4px; + background: var(--hdr-card); + font-size: 12px; + white-space: pre-wrap; + word-break: break-all; +} + +.hdr-grid { + display: grid; + grid-template-columns: auto 1fr; + gap: 4px 12px; + margin-top: 6px; + font-size: 12px; +} + +.hdr-amber { color: #f59e0b; } +.hdr-red { color: #ef4444; } + +/* Tailwind 700 on the light theme, like detail.css, for a 4.5:1 contrast + against the white dialog. */ +.hdr-light .hdr-amber { color: #b45309; } +.hdr-light .hdr-red { color: #b91c1c; } diff --git a/observability/liquidity-panels/header.js b/observability/liquidity-panels/header.js new file mode 100644 index 000000000..e95471d63 --- /dev/null +++ b/observability/liquidity-panels/header.js @@ -0,0 +1,216 @@ +// Business Text "afterRender" code for the Liquidity bot header: the SPA's +// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte, +// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with +// the same Config and CLI recovery guide dialogs. +// +// Input: context.data, rows with a `k` column naming the value (up, uptime, +// commit, info, equity_target, ...) plus the string labels some of them +// carry (git_commit on commit; broker, wallet_kind, ... on info). +// +// RECOVERY_GUIDE is not defined here: the generator prepends it from +// recovery-guide.json (see pills() in the generator), because the template output is +// sanitized and cannot carry data into this code. + +const rows = Array.isArray(context.data) ? context.data : []; +const value = {}; +const labels = {}; +// The stackdriver datasource runs a range query even for an instant target, +// so each `k` can arrive at several times; keep the newest. +const seen = {}; +for (const row of rows) { + const time = Number(row.Time) || 0; + if (row.k !== 'commit') { + if (row.k in seen && seen[row.k] > time) continue; + seen[row.k] = time; + } + if (row.k === 'commit') { + // A range query keeps the previous commit's series for a few minutes + // after a deploy. The commit row's value is its sample timestamp, so the + // newest one is the running commit. + if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value }; + } else if (row.k === 'info') { + labels.info = row; + } else { + value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value); + } +} +const info = labels.info || {}; + +const theme = context.grafana.theme; +const root = context.element; +// The guide's commands target the environment the board shows: the client +// takes the selector's text, production or staging. +const ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production'; +// The plugin pads each rendered row by 8px, which makes this one-row panel +// scroll; the row has no room for it. +root.style.padding = '0'; +root.style.height = '100%'; +root.style.overflow = 'hidden'; +root.style.setProperty('--hdr-muted', theme.colors.text.secondary); +// Light theme swaps the mode colours for darker shades (header.css). +root.classList.toggle('hdr-light', !theme.isDark); +root.style.setProperty('--hdr-border', theme.colors.border.weak); +root.style.setProperty('--hdr-pill', theme.colors.background.secondary); +root.style.setProperty('--hdr-card', theme.colors.background.primary); +root.style.setProperty('--hdr-text', theme.colors.text.primary); + +const escapeHtml = (text) => + String(text ?? '').replace(/[&<>"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[char]); + +// The SPA's fmtPct and band strings: "50%" and "45%–55%". +const pct = (fraction) => `${(fraction * 100).toFixed(0)}%`; +const band = (target, deviation) => + target === null || target === undefined + ? '' + : deviation === null || deviation === undefined + ? '' + : ` (${pct(target - deviation)}–${pct(target + deviation)})`; + +// The SPA's formatUptime. +const uptime = (seconds) => { + if (seconds === null || seconds === undefined) return ''; + const days = Math.floor(seconds / 86400); + const hours = Math.floor((seconds % 86400) / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + if (days > 0) return `up ${days}d ${hours}h`; + if (hours > 0) return `up ${hours}h ${minutes}m`; + return `up ${minutes}m`; +}; + +const clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC'; + +const pill = (inner) => `${inner}`; +// 1 up, 0 down; no value means the source is not reporting, which says +// nothing about the bot. +const connection = + value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data']; + +const pills = [ + info.broker ? pill(`${escapeHtml(info.broker)}`) : '', + value.equity_target !== null && value.equity_target !== undefined + ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`) + : '', + value.usdc_target !== null && value.usdc_target !== undefined + ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`) + : '', + value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '', + value.cash_reserved !== null && value.cash_reserved !== undefined + ? pill(`Reserve $${value.cash_reserved}`) + : '', + '', +].join(''); + +const configRow = (name, text) => + `
${name}${escapeHtml(text)}
`; +const seconds = (number) => (number === null || number === undefined ? '' : `${number}s`); + +const configDialog = ` + +
Configuration
+
+ ${ + info.wallet_kind + ? configRow('Wallet', info.wallet_kind) + + configRow('Address', info.wallet_address) + + (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') + + '
' + : '' + } + ${configRow('Exported log level', info.log_level)} + ${configRow('Server port', info.server_port)} + ${configRow('Deployment block', value.deployment_block ?? '')} + ${configRow('Order polling', seconds(value.order_polling))} + ${configRow('Inventory polling', seconds(value.inventory_polling))} + ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')} + ${configRow('Orderbook', info.orderbook)} +
+`; + +const guide = RECOVERY_GUIDE; +const modeClass = (mode) => + mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red'; +const guideDialog = ` + +
CLI recovery guide
+
+

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

+ ${guide.groups + .map( + (group) => ` +
+

${escapeHtml(group.object)}

+ ${group.commands + .map( + (entry) => ` +
+ 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
+
+ What${escapeHtml(entry.description)} + When${escapeHtml(entry.whenToUse)} + Applies to${escapeHtml(entry.appliesTo)} + Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)} +
+
` + ) + .join('')} +
` + ) + .join('')} +
+`; + +// The board refreshes every minute and each render replaces the markup, which +// would close a dialog the operator is reading. Remember it to reopen it at +// the same scroll position. +const openDialog = root.querySelector('dialog[open]'); +const reopen = openDialog && { + name: openDialog.dataset.dialog, + scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0, +}; + +root.innerHTML = ` +
+
${pills}
+
+ + ${clock()} + ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''} + ${uptime(value.uptime)} + ${connection[1]} +
+
+${configDialog} +${guideDialog}`; + +root.querySelectorAll('[data-open]').forEach((button) => { + button.addEventListener('click', () => root.querySelector(`[data-dialog="${button.dataset.open}"]`).showModal()); +}); +root.querySelectorAll('dialog').forEach((dialog) => { + dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close()); + // A click on the backdrop closes it, like the SPA. + dialog.addEventListener('click', (event) => { + if (event.target === dialog) dialog.close(); + }); +}); +if (reopen) { + const dialog = root.querySelector(`[data-dialog="${reopen.name}"]`); + if (dialog) { + dialog.showModal(); + const body = dialog.querySelector('.hdr-dialog-body'); + if (body) body.scrollTop = reopen.scroll; + } +} + +// The SPA's ticking UTC clock. One interval per element, replaced on every +// render so a refresh never stacks them. Grafana can also mount a new +// element (tab switch, view mode), so an interval whose element left the +// page stops itself. +clearInterval(root.__hdrClock); +root.__hdrClock = setInterval(() => { + if (!root.isConnected) { + clearInterval(root.__hdrClock); + return; + } + const node = root.querySelector('[data-clock]'); + if (node) node.textContent = clock(); +}, 1000); diff --git a/observability/liquidity-panels/recovery-guide.json b/observability/liquidity-panels/recovery-guide.json new file mode 100644 index 000000000..0ad0266e9 --- /dev/null +++ b/observability/liquidity-panels/recovery-guide.json @@ -0,0 +1,118 @@ +{ + "source": "ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand", + "modeLabels": { + "requires-bot": "REST β€” requires the running bot", + "direct-db": "direct DB β€” stop the bot / ensure it is not driving this id" + }, + "groups": [ + { + "object": "transfer", + "commands": [ + { + "command": "st0x-liquidity-client --env production debug recheck ", + "description": "Re-poll the provider and complete the transfer if it has settled.", + "whenToUse": "A transfer is stuck or failed but the provider may have settled it.", + "appliesTo": "Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.", + "mode": "requires-bot" + }, + { + "command": "st0x-liquidity-client --env production debug resume", + "description": "Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).", + "whenToUse": "Equity transfers were interrupted mid-flight and need re-driving via the bot.", + "appliesTo": "All in-flight equity transfers", + "mode": "requires-bot" + }, + { + "command": "st0x-liquidity-client --env production debug resume-usdc ", + "description": "Re-drive a single USDC bridge on the bot's transfer worker.", + "whenToUse": "A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).", + "appliesTo": "USDC bridge (in-flight)", + "mode": "requires-bot" + }, + { + "command": "st0x-liquidity-client --env production debug fail-equity-transfer --reason \"\"", + "description": "Force a stuck transfer into the terminal Failed state.", + "whenToUse": "A mint/redemption is permanently stuck and unrecoverable.", + "appliesTo": "Equity mint / redemption (non-terminal)", + "mode": "requires-bot" + }, + { + "command": "st0x-liquidity-client --env production debug fail-usdc-transfer --reason \"\"", + "description": "Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum that no CCTP burn left the bot wallet after this transfer started (its Started time) and that the wallet has no pending transaction (pending nonce equals latest nonce); if you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, and pending nonce equals latest nonce on more than one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once they show that no withdrawal landed and none is pending, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn left the bot wallet after this transfer started and the wallet has no pending transaction. If any of that is not certain, start the bot and run resume-usdc; else run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.", + "whenToUse": "A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.", + "appliesTo": "USDC bridge (pre-burn stuck)", + "mode": "requires-bot" + }, + { + "command": "st0x-liquidity-client --env production debug reconcile-equity --reason \"\"", + "description": "Mark a terminally-failed equity transfer Reconciled after handling residue manually.", + "whenToUse": "An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).", + "appliesTo": "Failed equity mint / redemption", + "mode": "requires-bot" + }, + { + "command": "st0x-liquidity-client --env production debug reconcile-usdc --reason ", + "description": "Mark a failed USDC bridge Reconciled after its funds were settled manually.", + "whenToUse": "A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.", + "appliesTo": "USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)", + "mode": "requires-bot" + } + ] + }, + { + "object": "position", + "commands": [ + { + "command": "st0x-liquidity-client --env production debug position release-hedge --order-id --reason \"\"", + "description": "Clear a position's pending offchain order pointer so hedging can retry.", + "whenToUse": "A position is wedged on a hedge order that never resolved.", + "appliesTo": "Position with a stuck pending offchain order", + "mode": "requires-bot" + }, + { + "command": "st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \"\"", + "description": "Override a position’s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.", + "whenToUse": "The recorded net exposure has drifted from reality and must be set explicitly.", + "appliesTo": "Any position", + "mode": "requires-bot" + } + ] + }, + { + "object": "view", + "commands": [ + { + "command": "st0x-liquidity-client --env production debug view rebuild (--id | --all)", + "description": "Replay all events to reconstruct a corrupted materialized view.", + "whenToUse": "A view became corrupted (e.g. lost updates from optimistic-lock conflicts).", + "appliesTo": "Position / offchain-order / vault-registry views", + "mode": "requires-bot" + } + ] + }, + { + "object": "cctp", + "commands": [ + { + "command": "st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain ", + "description": "Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.", + "whenToUse": "A CCTP burn succeeded but attestation polling was interrupted before the mint.", + "appliesTo": "CCTP cross-chain USDC transfer", + "mode": "requires-bot" + } + ] + }, + { + "object": "trade", + "commands": [ + { + "command": "st0x-liquidity-client --env production debug process-tx [--chain ]", + "description": "Re-account a missed onchain fill: record the trade and place the hedge.", + "whenToUse": "The bot missed an onchain fill and the position/hedge was never updated.", + "appliesTo": "Onchain (Raindex) fills", + "mode": "requires-bot" + } + ] + } + ] +}