From 8427df3f55603b0109ed55c5743c8d433b162ed0 Mon Sep 17 00:00:00 2001 From: JuaniRios Date: Sat, 10 Oct 2026 09:50:19 -0300 Subject: [PATCH] docs: close the burn nonce before failing a pre-burn USDC transfer --- dashboard/src/lib/transfer.test.ts | 168 ++++++++++++------ dashboard/src/lib/transfer.ts | 72 +++++--- docs/cli-ops.md | 146 +++++++++++++-- .../dashboards/liquidity/t0-liquidity.json | 4 +- .../liquidity/tabs/t0-liquidity-logs.json | 2 +- .../liquidity/tabs/t0-liquidity-orders.json | 2 +- .../tabs/t0-liquidity-performance.json | 2 +- .../liquidity/tabs/t0-liquidity-pnl.json | 2 +- .../liquidity-panels/recovery-commands.js | 72 +++++--- .../liquidity-panels/recovery-guide.json | 2 +- 10 files changed, 348 insertions(+), 124 deletions(-) diff --git a/dashboard/src/lib/transfer.test.ts b/dashboard/src/lib/transfer.test.ts index 88bba3b0f..52c05073a 100644 --- a/dashboard/src/lib/transfer.test.ts +++ b/dashboard/src/lib/transfer.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'node:fs' import { describe, expect, it } from 'vitest' import { kindLabel, @@ -459,7 +460,9 @@ describe('transferRecoveryCommands', () => { const text = resumeText(direction) expect(text.includes('this resume continues with the Alpaca withdrawal')).toBe(true) expect(text.includes('cannot re-drive the USD to USDC conversion')).toBe(true) - expect(text.includes('fails the transfer (ConversionFailed) and releases the guard')).toBe(true) + expect(text.includes('fails the transfer (ConversionFailed) and releases the guard')).toBe( + true + ) // After ConversionCancelNotSettled or ConversionOrderNotFound the order // can still fill, and the resume does not ask Alpaca, so check it first. const check = text.indexOf('no USD to USDC order for this transfer is still open') @@ -467,14 +470,22 @@ describe('transferRecoveryCommands', () => { expect(check).toBeLessThan(text.indexOf('Do not move USDC by hand before you run the resume')) // Settling at Alpaca fits only a ConversionFailed end; a later failure // (say an Alpaca to Base BridgingFailed) has funds off Alpaca. - expect(text.includes('If it ends in ConversionFailed, settle any converted USDC by hand at Alpaca')).toBe(true) + expect( + text.includes('If it ends in ConversionFailed, settle any converted USDC by hand at Alpaca') + ).toBe(true) expect(text.includes('reconcile-usdc does not apply')).toBe(true) expect(text.includes('A later failure has its own recovery')).toBe(true) } }) const bridgeLabels = (status: string, direction: 'alpaca_to_base' | 'base_to_alpaca' | null) => - transferRecoveryCommands({ deployment: PROD, kind: 'usdc_bridge', id: 'BRIDGE001', status, direction }) + transferRecoveryCommands({ + deployment: PROD, + kind: 'usdc_bridge', + id: 'BRIDGE001', + status, + direction + }) it('offers fail-usdc-transfer only where the running bot can accept it', () => { // Alpaca to Base: WithdrawalComplete (withdrawing) and a BridgingSubmitting @@ -500,58 +511,84 @@ describe('transferRecoveryCommands', () => { ) const toBase = failText('withdrawing', 'alpaca_to_base') expect(toBase.startsWith('Check the transfer first.')).toBe(true) - // The Alpaca to Base burn runs on Ethereum, and a nonce is per chain. - // The wallet burned on every earlier transfer too, so the check is anchored. + // The checks and the nonce close live in one place, the runbook, so the + // dialog gates the fail on them instead of keeping a second copy. The + // Alpaca to Base burn runs on Ethereum, and a nonce is per chain. + expect( + toBase.includes( + 'do not fail the transfer until you have done every check and the nonce close on Ethereum in docs/cli-ops.md, "Clearing a pre-burn guard latch", with the bot stopped' + ) + ).toBe(true) + expect( + toBase.includes('If a burn landed or you are not certain, run resume-usdc instead') + ).toBe(true) + // A restart sends the burn of a completed withdrawal, so the fail is the + // offline command, before the start. expect( - toBase.includes('verify on Ethereum that no CCTP burn left the bot wallet after this transfer started') + toBase.includes( + 'fail it with the offline stox fail-usdc-transfer before you start the bot again' + ) ).toBe(true) - expect(toBase.includes('pending nonce equals latest nonce')).toBe(true) + expect(toBase.includes('the guard stays held until you settle them with reconcile-usdc')).toBe( + true + ) expect(toBase.includes('OperatorWithdraw')).toBe(false) - // A Base to Alpaca burn comes after the vault withdrawal, so the check is - // on the withdrawal, and a landed one is adopted with resume-usdc. + // A Base to Alpaca transfer's checks and nonce close run on Base. const toAlpaca = failText('withdrawing', 'base_to_alpaca') + expect(toAlpaca.includes('only while the vault withdrawal is unrecorded')).toBe(true) + // The runbook below this stack has no full withdrawal procedure, so the + // dialog keeps it: both withdrawal events, the cash vault, the pending and + // queued checks, the nonce close and the recheck. expect(toAlpaca.includes('OperatorWithdraw')).toBe(true) - // A withdrawal sent before the inventory migration emits the orderbook's - // WithdrawV2, not OperatorWithdraw, so the check names both. expect(toAlpaca.includes('no WithdrawV2 on the orderbook either')).toBe(true) - // Only the cash vault's USDC counts, and the hooks' inventory-sent - // WithdrawV2 is not the bot's withdrawal. expect(toAlpaca.includes('cash vault_id in the bot config')).toBe(true) - expect(toAlpaca.includes('WithdrawV2 with the inventory as sender does not count')).toBe(true) - expect(toAlpaca.includes('run resume-usdc instead')).toBe(true) - expect(toAlpaca.includes('offline stox fail-usdc-transfer')).toBe(true) - // A log scan misses an unmined transaction, so the check also waits out - // the attempt and asks for no pending one. - expect(toAlpaca.includes('attempt timeout has passed')).toBe(true) - expect(toAlpaca.includes('pending nonce equals latest nonce on more than one RPC provider')).toBe(true) - // A provider can miss a pending withdraw, so the nonce is closed with a - // self-transfer and the logs checked again before the transfer is failed. - const cancel = toAlpaca.indexOf('close the nonce before you fail the transfer') - expect(cancel).toBeGreaterThan(-1) - expect(toAlpaca.includes('0-value transfer with no calldata from the bot wallet to itself')).toBe(true) - expect(toAlpaca.indexOf('check the withdrawal logs again')).toBeGreaterThan(cancel) - expect(toAlpaca.indexOf('Otherwise run the offline stox fail-usdc-transfer')).toBeGreaterThan( - toAlpaca.indexOf('check the withdrawal logs again') - ) - // The bot is stopped at that point, and the client resume needs it running. expect( - toAlpaca.includes('If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer') + toAlpaca.includes('pending nonce equals latest nonce on more than one RPC provider') ).toBe(true) - // Between reading the board and stopping the bot, the bot can broadcast a - // burn it does not record, and the offline command does not check the chain. - expect(toAlpaca.includes('no CCTP burn left the bot wallet')).toBe(true) - // A burn sent just before the stop can still be pending after it. - expect(toAlpaca.includes('stop the bot, then confirm')).toBe(true) - expect(toAlpaca.includes('the wallet has no pending transaction')).toBe(true) - expect(toAlpaca.includes('on Base that no CCTP burn left the bot wallet after this transfer started')).toBe( - true - ) - expect(toAlpaca.includes('on Ethereum')).toBe(false) + // A queued send waits behind a nonce gap; only empty nonces are filled, + // and only after the kept-send check. + const fill = toAlpaca.indexOf('fill each empty nonce below it') + expect(fill).toBeGreaterThan(-1) + expect(toAlpaca.indexOf('signed and keeps holds that nonce')).toBeGreaterThan(fill) + const close = toAlpaca.indexOf('close the nonce before you fail the transfer') + expect(close).toBeGreaterThan(-1) + expect(toAlpaca.indexOf('check the withdrawal logs again')).toBeGreaterThan(close) + expect( + toAlpaca.includes( + 'If a withdrawal mined instead, a transaction is pending or queued, or you cannot tell, start the bot and run resume-usdc' + ) + ).toBe(true) + // Resume adopts only the full amount; anything else ends in + // WithdrawalFailed with the USDC in the wallet. + expect(toAlpaca.includes('any other amount ends the transfer in WithdrawalFailed')).toBe(true) + // The unrecorded withdrawal can be failed through the live route too. + expect( + toAlpaca.includes( + 'Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot' + ) + ).toBe(true) + // Once recorded, the burn checks apply, with the runbook as the source. + expect(toAlpaca.includes('"Clearing a pre-burn guard latch"')).toBe(true) + // A late burn is finished with the bot stopped, for the amount the mint + // printed. + expect(toAlpaca.includes('Keep the bot stopped until that USDC is at Alpaca')).toBe(true) + expect(toAlpaca.includes('pending, queued and kept-send checks on Ethereum')).toBe(true) + expect(toAlpaca.includes('divided by 1,000,000 (it prints USDC base units)')).toBe(true) + // A burn found by the recheck of a confirmed withdrawal is resumed, + // never failed. + expect( + toAlpaca.includes( + 'If a burn landed, something is pending or queued, or you are not certain, start the bot and run resume-usdc, and do not fail the transfer' + ) + ).toBe(true) + // Its withdrawal checks and nonce close run on Base; only a late burn's + // mint and deposit are on Ethereum. + expect(toAlpaca.includes('to itself on Ethereum')).toBe(false) const unknown = failText('withdrawing', null) - expect(unknown.includes('verify on Ethereum')).toBe(true) - expect(unknown.includes('OperatorWithdraw')).toBe(true) + expect(unknown.includes('nonce close on Ethereum')).toBe(true) + expect(unknown.includes('nonce close on Base')).toBe(true) expect(unknown.includes('no reconcile is needed')).toBe(false) }) @@ -560,10 +597,12 @@ describe('transferRecoveryCommands', () => { bridgeLabels(status, direction).find((entry) => entry.label === 'Resume')?.description ?? '' // The running bot refuses fail-usdc-transfer once the vault withdrawal // confirmed, so the dialog offers no Fail there and the resume says where to go. - expect(bridgeLabels('bridging', 'base_to_alpaca').map((entry) => entry.label)).toEqual(['Resume']) - expect(resumeText('bridging', 'base_to_alpaca').includes('CLI recovery guide for the offline path')).toBe( - true - ) + expect(bridgeLabels('bridging', 'base_to_alpaca').map((entry) => entry.label)).toEqual([ + 'Resume' + ]) + expect( + resumeText('bridging', 'base_to_alpaca').includes('CLI recovery guide for the offline path') + ).toBe(true) for (const [status, direction] of [ ['withdrawing', 'base_to_alpaca'], ['depositing', 'base_to_alpaca'], @@ -582,7 +621,9 @@ describe('transferRecoveryCommands', () => { status: 'bridging', direction: 'alpaca_to_base' }) - expect(commandFor(toBase, 'Resume')).toBe(`${CLIENT} debug resume-usdc alpaca-to-base BRIDGE001`) + expect(commandFor(toBase, 'Resume')).toBe( + `${CLIENT} debug resume-usdc alpaca-to-base BRIDGE001` + ) const toAlpaca = transferRecoveryCommands({ deployment: PROD, @@ -779,7 +820,9 @@ describe('tradeRecoveryCommands', () => { expect(commandFor(commands, 'Set position')).toBe( `${CLIENT} debug position set MSTR --target-net [--price-usdc ] --reason ""` ) - expect(commandFor(commands, 'Rebuild view')).toBe(`${CLIENT} debug view rebuild position --id MSTR`) + expect(commandFor(commands, 'Rebuild view')).toBe( + `${CLIENT} debug view rebuild position --id MSTR` + ) }) it('uses the mock cli prefix in a simulation build', () => { @@ -804,7 +847,9 @@ describe('tradeRecoveryCommands', () => { describe('forClientEnv', () => { it('retargets a guide command at staging and leaves production as is', () => { const command = `${CLIENT} debug resume` - expect(forClientEnv(command, 'staging')).toBe('st0x-liquidity-client --env staging debug resume') + expect(forClientEnv(command, 'staging')).toBe( + 'st0x-liquidity-client --env staging debug resume' + ) expect(forClientEnv(command, 'production')).toBe(command) expect(forClientEnv(command, null)).toBe( 'st0x-liquidity-client --env debug resume' @@ -845,7 +890,9 @@ describe('RECOVERY_GUIDE', () => { ) // The bot refuses an Alpaca to Base deposit and one with no onchain deposit // transaction, so the guide sends those to manual settlement instead. - expect(recheck?.appliesTo.includes('failed Base to Alpaca USDC deposit with an onchain')).toBe(true) + expect(recheck?.appliesTo.includes('failed Base to Alpaca USDC deposit with an onchain')).toBe( + true + ) expect(recheck?.appliesTo.includes('Alpaca to Base deposit needs manual settlement')).toBe(true) }) @@ -855,7 +902,9 @@ describe('RECOVERY_GUIDE', () => { ) // The same check as the converting note in the transfer dialog. expect( - resume?.whenToUse.includes('first confirm at Alpaca that no USD to USDC order for the transfer is still open') + resume?.whenToUse.includes( + 'first confirm at Alpaca that no USD to USDC order for the transfer is still open' + ) ).toBe(true) }) }) @@ -875,3 +924,18 @@ describe('recoveryModeColor', () => { expect(recoveryModeColor('direct-db').text).toBe('text-destructive') }) }) + +describe('runbook headings the recovery text names', () => { + it('exist in docs/cli-ops.md', () => { + // The fail gates send operators to these sections by name, so a renamed + // heading must fail here instead of leaving every gate pointing nowhere. + const runbook = readFileSync(new URL('../../../docs/cli-ops.md', import.meta.url), 'utf8') + const headings = runbook + .split('\n') + .filter((line) => line.startsWith('#')) + .map((line) => line.replace(/^#+\s*/, '')) + for (const heading of ['Clearing a pre-burn guard latch']) { + expect(headings).toContain(heading) + } + }) +}) diff --git a/dashboard/src/lib/transfer.ts b/dashboard/src/lib/transfer.ts index 2a96bf15c..6c6591b92 100644 --- a/dashboard/src/lib/transfer.ts +++ b/dashboard/src/lib/transfer.ts @@ -456,12 +456,16 @@ const usdcDirectionToClientArg = (direction: UsdcBridgeDirection | null): string } const USDC_FAIL_ALPACA_TO_BASE = - 'Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal ' + - 'or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum ' + - 'that no CCTP burn left the bot wallet after this transfer started (its Started time) and ' + - 'that the wallet has no pending transaction (pending nonce equals latest nonce); if you ' + - 'are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held ' + - 'until you settle them with reconcile-usdc.' + 'Alpaca to Base: the bot accepts this only before the burn, from a completed ' + + 'withdrawal or a burn submission with no recorded burn, and it does not check the ' + + 'chain. A burn the bot sent but did not record can still land, so do not fail the ' + + 'transfer until you have done every check and the nonce close on Ethereum in ' + + 'docs/cli-ops.md, "Clearing a pre-burn guard latch", with the bot stopped. Then fail ' + + 'it with the offline stox fail-usdc-transfer before you start the bot again: a restart ' + + 'sends the burn of a completed withdrawal (at a burn submission, this command after a ' + + 'restart works too). If a burn landed or you are not certain, run resume-usdc instead. ' + + 'The funds left Alpaca, so the guard stays held until you settle them with ' + + 'reconcile-usdc.' const USDC_FAIL_BASE_TO_ALPACA = 'Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, ' + @@ -473,25 +477,43 @@ const USDC_FAIL_BASE_TO_ALPACA = 'sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the ' + 'network accepted but has not mined is not in the logs yet, so also wait until the ' + "transfer's attempt timeout has passed and confirm the wallet has no pending " + - 'transaction (no pending withdraw4, and pending nonce equals latest nonce on more than ' + - 'one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal ' + - 'landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, ' + - 'and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot ' + - 'wallet back to the vault by hand. If nothing landed, the resume keeps redriving: ' + - 'repeat the checks, and once they show that no withdrawal landed and none is pending, ' + - 'take the next step. If none landed, close the nonce before you fail the transfer: ' + - 'stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself ' + - 'at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market ' + - 'fee, wait for its required confirmations, and check the withdrawal logs again. If a ' + - 'withdrawal mined instead, start the bot and run resume-usdc, and do not fail the ' + - 'transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after ' + - 'you restart the bot. If a withdrawal for this transfer mines later anyway, move its ' + - 'USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is ' + - 'still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the ' + - 'bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn ' + - 'left the bot wallet after this transfer started and the wallet has no pending ' + - 'transaction. If any of that is not certain, start the bot and run resume-usdc; else ' + - 'run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.' + 'transaction (no pending withdraw4, pending nonce equals latest nonce on more than one ' + + 'RPC provider, and the explorer shows no queued transaction from the wallet). A queued ' + + 'transaction waits behind a nonce gap: with the bot stopped, fill each empty nonce ' + + 'below it, in order, with a self-transfer as below, after making sure no send the bot ' + + 'signed and keeps holds that nonce, then repeat the checks. These checks cannot prove ' + + 'that nothing is pending. If a withdrawal landed or you are not certain, run ' + + 'resume-usdc instead: it adopts a withdrawal of the full amount, and any other amount ' + + 'ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to ' + + 'the cash vault by hand. If nothing landed, the resume keeps redriving: repeat the ' + + 'checks, and once every check is clean, take the next step. If none landed, close the ' + + 'nonce before you fail the transfer: stop the bot, check again that the wallet has no ' + + 'pending or queued transaction, make sure no send the bot signed and keeps holds that ' + + 'nonce (if one does, start the bot so it rebroadcasts the send instead), send a 0-value ' + + 'transfer with no calldata from the bot wallet to itself at the latest nonce, with ' + + 'maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required ' + + 'confirmations, and check the withdrawal logs again. The self-transfer closes only ' + + 'that nonce. If a withdrawal mined instead, a transaction is pending or queued, or you ' + + 'cannot tell, start the bot and run resume-usdc, and do not fail the transfer. ' + + 'Otherwise run the offline stox fail-usdc-transfer, or this command after you restart ' + + 'the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the ' + + 'bot wallet back to the cash vault by hand. While a recorded withdrawal is still ' + + 'confirming, run resume-usdc. Once it has confirmed the bot refuses: do every check ' + + 'and the nonce close on Base in docs/cli-ops.md, "Clearing a pre-burn guard latch". If ' + + 'a burn landed, something is pending or queued, or you are not certain, start the bot ' + + 'and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox ' + + 'fail-usdc-transfer and move the wallet USDC back by hand. If ' + + 'a burn for this transfer mines later anyway, resume and reconcile refuse the failed ' + + 'transfer. Keep the bot stopped until that USDC is at Alpaca: the offline commands send ' + + 'from the bot wallet and replace whatever is pending at their nonce, so first run the ' + + 'pending, queued and kept-send checks on Ethereum as docs/cli-ops.md describes, and let ' + + 'any kept send mine. Mint it with stox ' + + "cctp complete-mint --burn-tx --source-chain base (it adopts a relayer's mint; " + + 'do not mint by hand, since the MessageSent log holds a placeholder nonce). Then send ' + + 'the amount received that complete-mint printed, divided by 1,000,000 (it prints USDC ' + + 'base units), to Alpaca with stox alpaca-deposit -a , and do not move that USDC ' + + 'back to the vault. Once Alpaca credits it, convert ' + + 'it with stox alpaca-convert -d to-usd -a , as the bot would have.' /// A Base to Alpaca bridge stuck at `bridging` (its vault withdrawal confirmed): /// the running bot refuses `fail-usdc-transfer` there, so the resume command diff --git a/docs/cli-ops.md b/docs/cli-ops.md index 02522b3c6..856b6e00d 100644 --- a/docs/cli-ops.md +++ b/docs/cli-ops.md @@ -895,17 +895,106 @@ the guard -- settle the funds with `transfer reconcile --kind usdc`. For BaseToAlpaca the funds moved out of the Raindex vault into the market maker wallet: the failure reconciles to source and the guard clears on the next restart, and the wallet USDC is moved back to the vault by hand. The offline -command is safe to run once the bot is stopped. +command is safe to run once the bot is stopped, but read the transfer's state +again after the stop: a bot that moved on before the stop may have reached +`BridgingSubmitting` and broadcast a burn. If it did, follow the checks and the +nonce close below. `BridgingSubmitting` is NOT unconditionally safe. A crash at this state may have already broadcast a CCTP burn whose `BridgingInitiated` event never persisted. Before running this command on a `BridgingSubmitting` transfer, verify on-chain that no recent CCTP burn was submitted from the market-maker wallet (e.g. via -`cast` against the Circle CCTP contract or by inspecting recent wallet txs). - -- **If no burn is found**: run `fail-usdc-transfer`. For BaseToAlpaca the guard - then clears on restart; for AlpacaToBase it stays held until - `transfer reconcile --kind usdc` settles the withdrawn funds. +`cast` against the Circle CCTP contract or by inspecting recent wallet txs). The +burn runs on Ethereum for AlpacaToBase and on Base for BaseToAlpaca. Look there +for a `DepositForBurn` from the bot wallet after the transfer started. A burn +the network accepted but has not mined is not in the logs yet, and a send queued +behind a nonce gap does not count as pending, so the checks below look at both. +These checks cannot prove that nothing is pending: the bot recorded no burn hash +for this transfer, and a provider can hold a transaction that no other provider +shows. + +- **If no burn is found**: close the nonce before you fail the transfer. + 1. Stop the bot so that it sends nothing more from that wallet. + 2. On the burn's chain, confirm that the bot wallet has no pending + transaction: pending nonce equal to latest nonce on more than one RPC + provider, and no queued transaction from the wallet in the explorer's + pending list (or `txpool_inspect` where the provider supports it). A queued + transaction waits behind a nonce gap. For each gap nonce below it, in + order, first run the step 3 check on that nonce: a kept send that dropped + out of every mempool is a common cause of a gap, and filling its nonce + means it can never mine. Then fill the nonce with a self-transfer as in + step 4, so the queued transaction can mine, and look for the burn again. If + a burn mined, start the bot, then follow the step for a found burn below; + otherwise repeat this step. Do not close the queued transaction's own nonce + unless it is this transfer's burn. If you cannot tell, do not fail the + transfer: start the bot, then follow the step for an uncertain burn. + 3. Make sure no send the bot signed and keeps holds that nonce: a redemption's + vault withdrawal or its send to the issuer, a BaseToAlpaca deposit send, or + a capital burn, which the bot rebroadcasts at startup. Check every open + send of that kind on that chain, not only this transfer's events, and the + startup log's rebroadcasts. If one does, start the bot so that it + rebroadcasts the send, let it mine, and start again from step 1. + 4. From the bot wallet, send a 0-value transfer with no calldata to the wallet + itself at that nonce, with `maxFeePerGas` and `maxPriorityFeePerGas` well + above the current market fee. Wait until it has the chain's required + confirmations. + 5. The self-transfer closes only that nonce. Look for the burn again, and + repeat the pending and queued checks from step 2. If a burn mined, + something is pending or queued, or you cannot tell, do not fail the + transfer: start the bot, then follow the step for a found or an uncertain + burn. Otherwise run the offline `fail-usdc-transfer` before you start the + bot again (at `BridgingSubmitting`, for AlpacaToBase, the live route after + a restart works too; at `WithdrawalComplete` a restart sends the burn). For + BaseToAlpaca the guard then clears on restart; for AlpacaToBase it stays + held until `transfer reconcile --kind usdc` settles the withdrawn funds. +- **If a burn for this transfer mines after the fail**, its USDC is in flight to + the destination chain (Base for AlpacaToBase, Ethereum for BaseToAlpaca). Mint + it with + `stox cctp complete-mint --burn-tx --source-chain + ` + (the burn's chain) with the bot stopped: the offline command sends from the + bot's wallet, reads the nonce itself, and fee-bumps until it replaces whatever + is pending at that nonce, so beside a live bot it can replace or block one of + the bot's sends. Stopping the bot does not remove sends it already broadcast + on the mint's chain, so before each offline mint or deposit, run the step 2 + pending and queued checks and the step 3 kept-send check on that command's + chain, and let any kept send mine first. For AlpacaToBase, whose guard stays + held until the reconcile, you can instead use the live + `st0x-liquidity-client --env debug cctp complete-mint` with the bot + running: it takes the recovery lock and pauses the USDC driver. It fetches the + Circle message and attestation by the burn's hash, and adopts the mint when a + relayer already minted it. Do not build the mint by hand: the burn's + `MessageSent` log holds a placeholder, not the nonce. If the mint reverts + because the attestation expired and no relayer minted it, request a new + attestation from Circle (`POST /v2/reattest/{nonce}`, with the nonce from the + attested message, not from `MessageSent`), wait for it, and run + `complete-mint` again. The amount received is the burn amount less the CCTP + fee. The offline command prints "Amount received" in USDC base units: divide + it by 1,000,000 for the whole-USDC amount the deposit commands below take. The + live route returns `amountReceived` in whole USDC: use it as is (an + undecodable result gives base units). For AlpacaToBase, then deposit that + amount to the cash vault on Base + (`st0x-liquidity-client --env capital vault-deposit --amount + --network base --token --vault-id ` + with the bot running, or the offline `stox vault-deposit` with the same + options with it stopped) and reconcile with `--kind usdc`. For BaseToAlpaca, + resume and reconcile both refuse the failure, and the guard clears on restart + while that USDC is in flight, so a running bot can plan another transfer from + a balance that leaves it out. Keep the bot stopped through the offline mint + and the deposit (`stox alpaca-deposit` also sends from the bot's Ethereum + wallet and replaces what is pending at its nonce). Send the amount received, + in whole USDC as above, to Alpaca with `stox alpaca-deposit -a `, and + do not move the wallet USDC that burn took back to the vault. Once Alpaca + credits it, convert it with `stox alpaca-convert -d to-usd -a `, as + the bot would have: until then it does not count as broker cash for hedging. + There is no reconcile step. +- **If you are not certain** that no burn is pending: run + `transfer resume --kind usdc`. At `BridgingSubmitting` it adopts a burn that + landed. If it finds none, it fails closed and the transfer stays at + `BridgingSubmitting` (it never sends a second burn). Repeat the checks above, + and once every check is clean, follow the first step. At `WithdrawalComplete` + no burn was sent yet, and a resume or a restart sends it and the transfer goes + on. - **If a burn IS found** while the aggregate is still `BridgingSubmitting`: do NOT run `fail-usdc-transfer` (strands the burned funds) and do NOT run `transfer reconcile` (its preflight rejects `BridgingSubmitting` -- it only @@ -1387,15 +1476,42 @@ a recorded burn as post-burn, `transfer resume` re-derives `Dropped`, and never landed (the USDC never left the market-maker wallet), it clears the recorded hash, returning the aggregate to `BridgingSubmitting` with no recorded burn. It does NOT release the guard on its own -- run `fail-usdc-transfer` next -(if the burn never landed) to release it, or `transfer resume --kind usdc` to -continue the bridge. - -Safe to run with the bot live: this command only loads the aggregate and sends a -single CQRS command, and the precondition is a `Dropped`-latched transfer whose -job has already fail-closed -- no active job is driving that rebalance, so there -is no race. Do NOT run it against a transfer that is still being actively -processed (one that has not yet latched); always confirm the latch and the -burn-absent on-chain state first. +(if the burn never landed) to release it. `transfer resume --kind usdc` after +the clear adopts the burn if it landed and otherwise fails closed: it does not +burn again. A load-balanced RPC can report a burn as dropped while a node still +holds it, so before the fail run the checks in "Clearing a pre-burn guard latch" +above, with the bot stopped before any self-transfer here (a live bot can send +from that wallet at one of these nonces, and the above-market self-transfer +would replace it): look for that burn and for anything pending or queued from +the wallet. Then compare the dropped burn's nonce with the wallet's latest +nonce. Read the burn's nonce from the bot's `Transaction submitted` log line for +the recorded burn tx hash (it logs `tx_hash` and `nonce`); an explorer usually +has no record of a dropped transaction. If a different transaction mined at the +burn's nonce, that burn can never mine and needs no close: go on with the checks +for the latest nonce as in pre-burn steps 1 to 5. Otherwise the self-transfer at +the burn's nonce must replace the burn wherever a node still holds it, so give +it a `maxFeePerGas` and a `maxPriorityFeePerGas` at least 10% above the burn's +and above the market fee, or well above the market fee when the burn's fees are +not known. If the burn's nonce equals the latest nonce, send that self-transfer. +If it is above the latest nonce, send that self-transfer first, then fill the +empty nonces from the latest up to it, in order, running step 3 of the pre-burn +checks on each nonce before you fill it (a kept send can hold one). Do not +replace a transaction queued at one of those nonces: only the burn's own nonce +is replaced. Wait for the confirmations, check that the transaction mined at the +burn's nonce is your self-transfer, and look for a `DepositForBurn` again. If a +burn mined, start the bot and resume, which adopts it. If the replacement did +not hold, or a transaction mined at a filled nonce is neither your self-transfer +nor the burn, do not fail: repeat the checks until they are clean. If you cannot +find the burn's nonce, fall back to pre-burn steps 1 to 5 for the latest nonce. + +Only `clear-pending-burn` below is safe to run with the bot live: run the +`fail-usdc-transfer` after it only once the checks above are done, with the bot +stopped. `clear-pending-burn` is safe live because it only loads the aggregate +and sends a single CQRS command, and the precondition is a `Dropped`-latched +transfer whose job has already fail-closed -- no active job is driving that +rebalance, so there is no race. Do NOT run it against a transfer that is still +being actively processed (one that has not yet latched); always confirm the +latch and the burn-absent on-chain state first. stox clear-pending-burn --id --reason "dropped burn verified absent on-chain" stox fail-usdc-transfer --id --reason "pre-burn crash, burn never landed" diff --git a/observability/dashboards/liquidity/t0-liquidity.json b/observability/dashboards/liquidity/t0-liquidity.json index 219f69ba9..578d3962e 100644 --- a/observability/dashboards/liquidity/t0-liquidity.json +++ b/observability/dashboards/liquidity/t0-liquidity.json @@ -213,7 +213,7 @@ "content": "
", "defaultContent": "No bot data.", "helpers": "", - "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum that no CCTP burn left the bot wallet after this transfer started (its Started time) and that the wallet has no pending transaction (pending nonce equals latest nonce); if you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, and pending nonce equals latest nonce on more than one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once they show that no withdrawal landed and none is pending, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn left the bot wallet after this transfer started and the wallet has no pending transaction. If any of that is not certain, start the bot and run resume-usdc; else run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", + "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn, and it does not check the chain. A burn the bot sent but did not record can still land, so do not fail the transfer until you have done every check and the nonce close on Ethereum in docs/cli-ops.md, \\\"Clearing a pre-burn guard latch\\\", with the bot stopped. Then fail it with the offline stox fail-usdc-transfer before you start the bot again: a restart sends the burn of a completed withdrawal (at a burn submission, this command after a restart works too). If a burn landed or you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, pending nonce equals latest nonce on more than one RPC provider, and the explorer shows no queued transaction from the wallet). A queued transaction waits behind a nonce gap: with the bot stopped, fill each empty nonce below it, in order, with a self-transfer as below, after making sure no send the bot signed and keeps holds that nonce, then repeat the checks. These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a withdrawal of the full amount, and any other amount ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the cash vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once every check is clean, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, check again that the wallet has no pending or queued transaction, make sure no send the bot signed and keeps holds that nonce (if one does, start the bot so it rebroadcasts the send instead), send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. The self-transfer closes only that nonce. If a withdrawal mined instead, a transaction is pending or queued, or you cannot tell, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the cash vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: do every check and the nonce close on Base in docs/cli-ops.md, \\\"Clearing a pre-burn guard latch\\\". If a burn landed, something is pending or queued, or you are not certain, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer and move the wallet USDC back by hand. If a burn for this transfer mines later anyway, resume and reconcile refuse the failed transfer. Keep the bot stopped until that USDC is at Alpaca: the offline commands send from the bot wallet and replace whatever is pending at their nonce, so first run the pending, queued and kept-send checks on Ethereum as docs/cli-ops.md describes, and let any kept send mine. Mint it with stox cctp complete-mint --burn-tx --source-chain base (it adopts a relayer's mint; do not mint by hand, since the MessageSent log holds a placeholder nonce). Then send the amount received that complete-mint printed, divided by 1,000,000 (it prints USDC base units), to Alpaca with stox alpaca-deposit -a , and do not move that USDC back to the vault. Once Alpaca credits it, convert it with stox alpaca-convert -d to-usd -a , as the bot would have.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", "styles": "/* Business Text \"styles\" for the Liquidity bot header row and its two\n dialogs. Theme colours come in as --hdr-* variables set by header.js. */\n\n.hdr {\n display: flex;\n align-items: center;\n justify-content: space-between;\n gap: 8px;\n height: 100%;\n font-size: 13px;\n white-space: nowrap;\n}\n\n.hdr-left,\n.hdr-right {\n display: flex;\n align-items: center;\n gap: 8px;\n min-width: 0;\n}\n\n.hdr-pill {\n display: inline-flex;\n align-items: center;\n gap: 4px;\n padding: 2px 10px;\n border-radius: 9999px;\n background: var(--hdr-pill);\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n border: 0;\n}\n\n.hdr-button {\n color: var(--hdr-muted);\n cursor: pointer;\n}\n\n.hdr-button:hover,\n.hdr-guide:hover,\n.hdr-close:hover {\n color: var(--hdr-text);\n}\n\n.hdr-guide {\n padding: 2px 8px;\n border: 1px solid var(--hdr-border);\n border-radius: 4px;\n background: transparent;\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n cursor: pointer;\n}\n\n.hdr-badge {\n padding: 2px 8px;\n border-radius: 9999px;\n font-size: 13px;\n line-height: 18px;\n font-weight: 600;\n}\n\n.hdr-ok { background: #f5f5f5; color: #171717; }\n.hdr-bad { background: #ef4444; color: #fff; }\n.hdr-unknown { border: 1px solid var(--hdr-border); color: var(--hdr-muted); }\n\n.hdr-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.hdr-muted { color: var(--hdr-muted); }\n\n.hdr-dialog {\n padding: 0;\n border: 1px solid var(--hdr-border);\n border-radius: 8px;\n background: var(--hdr-card);\n color: var(--hdr-text);\n white-space: normal;\n}\n\n.hdr-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n.hdr-narrow { width: min(28rem, 92vw); }\n.hdr-wide { width: min(42rem, 92vw); }\n\n.hdr-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--hdr-border);\n font-size: 14px;\n font-weight: 600;\n}\n\n.hdr-close {\n border: 0;\n background: transparent;\n color: var(--hdr-muted);\n font-size: 18px;\n line-height: 1;\n cursor: pointer;\n}\n\n.hdr-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 13px;\n}\n\n.hdr-row {\n display: flex;\n justify-content: space-between;\n gap: 16px;\n padding: 4px 0;\n}\n\n.hdr-trunc {\n overflow: hidden;\n text-overflow: ellipsis;\n white-space: nowrap;\n}\n\n.hdr-rule { border-top: 1px solid var(--hdr-border); margin: 8px 0; }\n.hdr-intro { margin: 0 0 16px; }\n.hdr-group { margin-bottom: 20px; }\n.hdr-group h3 { margin: 0 0 8px; font-size: 13px; text-transform: capitalize; }\n\n.hdr-command {\n margin-bottom: 12px;\n padding: 8px 12px;\n border: 1px solid var(--hdr-border);\n border-radius: 6px;\n background: var(--hdr-pill);\n}\n\n.hdr-command pre {\n margin: 0;\n padding: 8px;\n border-radius: 4px;\n background: var(--hdr-card);\n font-size: 12px;\n white-space: pre-wrap;\n word-break: break-all;\n}\n\n.hdr-grid {\n display: grid;\n grid-template-columns: auto 1fr;\n gap: 4px 12px;\n margin-top: 6px;\n font-size: 12px;\n}\n\n.hdr-amber { color: #f59e0b; }\n.hdr-red { color: #ef4444; }\n\n/* Tailwind 700 on the light theme, like detail.css, for a 4.5:1 contrast\n against the white dialog. */\n.hdr-light .hdr-amber { color: #b45309; }\n.hdr-light .hdr-red { color: #b91c1c; }\n", "wrap": false }, @@ -2738,7 +2738,7 @@ "content": "
", "defaultContent": "
", "helpers": "", - "afterRender": "const MODE_LABELS = {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"};\n\n// The board's per-row recovery commands: the client form of the SPA's\n// tradeRecoveryCommands and transferRecoveryCommands\n// (dashboard/src/lib/transfer.ts). The generator prepends this file to\n// detail.js; dashboard/src/lib/transfer-board.test.ts runs it against the SPA's\n// builders so the two cannot drift, importing it as a module through the\n// export line, which the generator drops. `client` is the client invocation\n// with its --env.\n\nconst tradeCommands = (client, symbol) => [\n {\n command: `${client} debug position release-hedge ${symbol} --order-id --reason \"\"`,\n description: \"Clear a position's stuck pending offchain order so normal hedging can retry. Needs the order id.\",\n mode: 'requires-bot',\n },\n {\n command: `${client} debug position set ${symbol} --target-net [--price-usdc ] --reason \"\"`,\n description:\n 'Override the net exposure after a manual correction. The target is signed: negative is short, 0 is flat. The price is required for a nonzero target unless the position already has a last price.',\n mode: 'requires-bot',\n },\n {\n command: `${client} debug view rebuild position --id ${symbol}`,\n description: 'Replay all events to reconstruct a corrupted position view.',\n mode: 'requires-bot',\n },\n];\n\nconst USDC_FAIL_ALPACA_TO_BASE =\n 'Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal ' +\n 'or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum ' +\n 'that no CCTP burn left the bot wallet after this transfer started (its Started time) and ' +\n 'that the wallet has no pending transaction (pending nonce equals latest nonce); if you ' +\n 'are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held ' +\n 'until you settle them with reconcile-usdc.';\n\nconst USDC_FAIL_BASE_TO_ALPACA =\n 'Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, ' +\n 'and it does not check the chain. First confirm on Base that the bot wallet made no ' +\n \"OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 \" +\n 'on the orderbook either (a withdrawal sent before the inventory migration). Only a ' +\n 'USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in ' +\n 'the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 ' +\n 'sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the ' +\n 'network accepted but has not mined is not in the logs yet, so also wait until the ' +\n \"transfer's attempt timeout has passed and confirm the wallet has no pending \" +\n 'transaction (no pending withdraw4, and pending nonce equals latest nonce on more than ' +\n 'one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal ' +\n 'landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, ' +\n 'and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot ' +\n 'wallet back to the vault by hand. If nothing landed, the resume keeps redriving: ' +\n 'repeat the checks, and once they show that no withdrawal landed and none is pending, ' +\n 'take the next step. If none landed, close the nonce before you fail the transfer: ' +\n 'stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself ' +\n 'at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market ' +\n 'fee, wait for its required confirmations, and check the withdrawal logs again. If a ' +\n 'withdrawal mined instead, start the bot and run resume-usdc, and do not fail the ' +\n 'transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after ' +\n 'you restart the bot. If a withdrawal for this transfer mines later anyway, move its ' +\n 'USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is ' +\n 'still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the ' +\n 'bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn ' +\n 'left the bot wallet after this transfer started and the wallet has no pending ' +\n 'transaction. If any of that is not certain, start the bot and run resume-usdc; else ' +\n 'run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.';\n\nconst USDC_BRIDGING_BASE_TO_ALPACA =\n ' If the burn keeps failing, the running bot refuses fail-usdc-transfer here: see ' +\n 'fail-usdc-transfer in the CLI recovery guide for the offline path.';\n\nconst USDC_CONVERTING_ALPACA_TO_BASE =\n ' Alpaca to Base: this status covers two bot states. If the bot recorded the ' +\n 'conversion as complete, this resume continues with the Alpaca withdrawal. If not, the ' +\n 'bot cannot re-drive the USD to USDC conversion (it did not store the broker order ' +\n 'id), so this resume fails the transfer (ConversionFailed) and releases the guard, ' +\n 'without checking Alpaca. So first confirm at Alpaca that no USD to USDC order for ' +\n 'this transfer is still open (it is filled, cancelled or rejected), above all after a ' +\n 'conversion outcome unresolved alert; if one is open, cancel it or wait for it. Do not ' +\n 'move USDC by hand before you run the resume. If it ends in ConversionFailed, settle ' +\n 'any converted USDC by hand at Alpaca (reconcile-usdc does not apply). A later failure ' +\n 'has its own recovery.';\n\n// usdcResumeNote in transfer.ts: the extra resume text for a bridge's status\n// and direction.\nconst usdcResumeNote = (status, direction) => {\n if (direction === 'base_to_alpaca' && status === 'bridging') return USDC_BRIDGING_BASE_TO_ALPACA;\n if (direction !== 'base_to_alpaca' && status === 'converting') return USDC_CONVERTING_ALPACA_TO_BASE;\n return '';\n};\n\n// usdcFailCheck in transfer.ts: the check before fail-usdc-transfer, or null\n// where the running bot always refuses it.\nconst usdcFailCheck = (status, direction) => {\n if (direction === 'alpaca_to_base') {\n return status === 'withdrawing' || status === 'bridging' ? USDC_FAIL_ALPACA_TO_BASE : null;\n }\n if (direction === 'base_to_alpaca') return status === 'withdrawing' ? USDC_FAIL_BASE_TO_ALPACA : null;\n return status === 'withdrawing' || status === 'bridging'\n ? `${USDC_FAIL_ALPACA_TO_BASE} ${USDC_FAIL_BASE_TO_ALPACA}`\n : null;\n};\n\nconst transferCommands = (client, transfer) => {\n const status = String(transfer.status || '').toLowerCase();\n const failed = status === 'failed';\n const terminal = failed || status === 'completed' || status === 'reconciled';\n if (status === 'completed' || status === 'reconciled') return [];\n const id = transfer.id;\n\n if (transfer.kind === 'usdc_bridge') {\n // Reconcile needs the bridge's postBurn flag, which the exporter does not\n // log; the SPA offers nothing without it either.\n if (failed) return [];\n const direction =\n { alpaca_to_base: 'alpaca-to-base', base_to_alpaca: 'base-to-alpaca' }[transfer.direction] ||\n '';\n const resume = {\n command: `${client} debug resume-usdc ${direction} ${id}`,\n description:\n \"Re-drive this USDC bridge on the bot's transfer worker from the stage it stopped at.\" +\n usdcResumeNote(status, transfer.direction || null),\n mode: 'requires-bot',\n };\n const check = usdcFailCheck(status, transfer.direction || null);\n if (check === null) return [resume];\n return [\n resume,\n {\n command: `${client} debug fail-usdc-transfer ${id} --reason \"\"`,\n description: `Check the transfer first. ${check}`,\n mode: 'requires-bot',\n },\n ];\n }\n\n const kind = { equity_mint: 'mint', equity_redemption: 'redemption' }[transfer.kind];\n if (!kind) return [];\n const commands = [\n {\n command: `${client} debug recheck ${kind} ${id}`,\n description: 'Ask the running bot to re-poll the provider and complete the transfer if it settled.',\n mode: 'requires-bot',\n },\n ];\n if (!terminal) {\n commands.push({\n command: `${client} debug resume`,\n description:\n 'Re-drive ALL interrupted mints and redemptions via the bot (no id; best-effort per transfer, each succeeds or fails independently and failures are reported as counts).',\n mode: 'requires-bot',\n });\n commands.push({\n command: `${client} debug fail-equity-transfer ${kind} ${id} --reason \"\"`,\n description: 'Force this stuck transfer into the terminal Failed state. Use when it is permanently stuck.',\n mode: 'requires-bot',\n });\n }\n if (failed) {\n commands.push({\n command: `${client} debug reconcile-equity ${kind} ${id} --reason \"\"`,\n description: 'Mark a Failed transfer as Reconciled once its residue was handled out-of-band (bookkeeping).',\n mode: 'requires-bot',\n });\n }\n return commands;\n};\n\n\n// The detail dialog's status history: a Trades or Rebalances row's status-log\n// entries in order, and the status the row is at. The generator prepends this\n// file to detail.js; dashboard/src/lib/transfer-board.test.ts imports it as a\n// module through the export line, which the generator drops.\n\n// A USDC bridge's in-flight statuses in lifecycle order per direction. Their\n// log times can go backwards (WithdrawalComplete carries confirmed_at, the\n// BridgingSubmitting after it the older initiated_at), so two in-flight\n// entries of a bridge are ordered by this. Anything else, failed included\n// (the bot can recover a bridge out of a failure), orders by time.\n//\n// Known limit: a Base to Alpaca bridge recovered out of DepositFailed reaches\n// converting, which the bot stamps with the bridge's initiated_at. That entry\n// sorts before the failed one, so the dialog keeps showing the status before\n// it (failed, or depositing) until the bridge completes.\nconst BRIDGE_LIFECYCLE = {\n alpaca_to_base: ['converting', 'withdrawing', 'bridging', 'depositing'],\n base_to_alpaca: ['withdrawing', 'bridging', 'depositing', 'converting'],\n};\nconst stage = (entry) => {\n const lifecycle = entry.kind === 'usdc_bridge' ? BRIDGE_LIFECYCLE[entry.direction] : null;\n const index = lifecycle ? lifecycle.indexOf(String(entry.status || '').toLowerCase()) : -1;\n return index >= 0 ? index : null;\n};\n\n// The event store's sequence in a bot line's `event_id`\n// (`::`), or null for an exporter entry.\nconst sequenceOf = (entry) => {\n const match = /:(\\d+)$/.exec(entry.event_id || '');\n return match ? Number(match[1]) : null;\n};\n\n// The row at its latest status, with its status history oldest first. The\n// bot's lines carry the commit order, so they sort by sequence. Exporter\n// entries sort by time (ties keep Cloud Logging's newest-first order\n// reversed), then each pair of neighbouring in-flight bridge entries is put\n// in lifecycle order.\nconst latest = (entries) => {\n if (entries.length === 0) return null;\n if (entries.every((entry) => sequenceOf(entry) !== null)) {\n const history = [...entries].sort((left, right) => sequenceOf(left) - sequenceOf(right));\n const last = history[history.length - 1];\n return { ...last, first: Math.min(...entries.map((entry) => entry.time)), history };\n }\n const history = [...entries].reverse().sort((left, right) => left.time - right.time);\n for (let swapped = true; swapped; ) {\n swapped = false;\n for (let index = 1; index < history.length; index++) {\n const [before, after] = [stage(history[index - 1]), stage(history[index])];\n if (before !== null && after !== null && before > after) {\n [history[index - 1], history[index]] = [history[index], history[index - 1]];\n swapped = true;\n }\n }\n }\n const last = history[history.length - 1];\n return { ...last, first: Math.min(...entries.map((entry) => entry.time)), history };\n};\n\n\n// The detail dialog's reading of Cloud Logging frames: a row's entries, and\n// the bot's event timeline. The generator prepends this file to detail.js;\n// dashboard/src/lib/transfer-board.test.ts imports it as a module through\n// the export line, which the generator drops.\n\n// A JSON object, or null for anything else (no value, bad JSON, a string).\nconst parseObject = (value) => {\n if (value !== null && typeof value === 'object') return value;\n try {\n const parsed = JSON.parse(value);\n return parsed !== null && typeof parsed === 'object' ? parsed : null;\n } catch (error) {\n return null;\n }\n};\n\n// The entries of `id` in one frame of the plugin's log-lines shape, newest\n// first like the frame: each line's time and payload. The exporter's body\n// is its JSON payload. A bot line's body is its message, so its payload\n// comes from the flattened `jsonPayload.` labels, which the exporter's\n// lines carry too. A line from another project than `project` belongs to\n// the environment the board showed before a switch and is dropped. A line\n// whose `event_id` came before (the bot's lines, shipped twice) is dropped.\nconst lineEntries = (frame, id, project) => {\n if (!frame) return [];\n const column = (name) => frame.fields.find((field) => field.name === name);\n const times = column('timestamp');\n const bodies = column('body');\n const labels = column('labels');\n const count = times ? times.values.length : 0;\n const seen = new Set();\n const entries = [];\n for (let index = 0; index < count; index++) {\n const flat = (labels && parseObject(labels.values[index])) || {};\n const lineProject = flat['resource.labels.project_id'];\n if (lineProject && lineProject !== project) continue;\n const fromLabels = {};\n for (const [key, value] of Object.entries(flat)) {\n if (key.startsWith('jsonPayload.')) fromLabels[key.slice('jsonPayload.'.length)] = value;\n }\n const payload = (bodies && parseObject(bodies.values[index])) || fromLabels;\n if (!payload.id || String(payload.id) !== id) continue;\n if (payload.event_id) {\n if (seen.has(payload.event_id)) continue;\n seen.add(payload.event_id);\n }\n entries.push({ time: Number(times.values[index]), ...payload });\n }\n return entries;\n};\n\n// A row's event timeline from its liq_event entries, oldest first: the\n// events of a trade, or of a transfer of `kind` (each transfer kind is its\n// own aggregate). A trade's events are not matched on venue, because a\n// venue correction changes it. Each payload is parsed from its JSON string.\nconst eventTimeline = (entries, parent, kind) =>\n entries\n .filter((entry) => entry.parent === parent && (parent !== 'transfer' || entry.kind === kind))\n .map((entry) => ({ ...entry, payload: parseObject(entry.payload) || {} }))\n .sort((left, right) => Number(left.sequence) - Number(right.sequence));\n\n// Whether a row's event timeline misses events: the event store numbers an\n// aggregate's events from 1 without gaps, so a timeline that does not start\n// at 1, skips a number, or ends before the event of the row's latest status\n// line (`rowEventId`, `::`) lacks lines: lines not\n// loaded yet (the events scan ran before the row's), the newest-500 window,\n// or a line the bot never wrote.\nconst timelineIncomplete = (events, rowEventId) => {\n const match = /:(\\d+)$/.exec(rowEventId || '');\n const last = events.length > 0 ? Number(events[events.length - 1].sequence) : 0;\n return (\n events.some((event, index) => Number(event.sequence) !== index + 1) ||\n (match !== null && Number(match[1]) > last)\n );\n};\n\n// The panel errors that concern the source shown. A failed query of the\n// other source, or of the event timeline, leaves the rows usable, so it does\n// not block them. An error without a refId blocks: it cannot be placed.\nconst blockingErrors = (errors, source) => {\n const other = source === 'bot' ? 'exporter-' : 'bot-';\n return errors.filter(\n (error) => !error.refId || (error.refId !== 'bot-events' && !error.refId.startsWith(other))\n );\n};\n\n// What the panel's last refresh says about the queries the source shown\n// reads. Grafana's Mixed datasource keeps only the last failing query's\n// errors (runRequest's processResponsePacket overwrites error and errors per\n// packet), so an Error state whose listed errors do not block can still hide\n// a failed table query, and its frames can be the ones the failed refresh\n// kept. `failed`: a listed error blocks, or the state is Error with none\n// listed. `unsure`: the state is Error and only errors that do not block are\n// listed, so a row shown can be out of date. `eventsFailed`: the event query\n// is among the listed errors, whether Grafana gave a list or one error.\nconst queryState = (panelData, source) => {\n const listed = panelData?.errors || (panelData?.error ? [panelData.error] : []);\n const blocking = blockingErrors(listed, source);\n const isError = panelData?.state === 'Error';\n return {\n listed,\n blocking,\n failed: isError && (listed.length === 0 || blocking.length > 0),\n unsure: isError && listed.length > 0 && blocking.length === 0,\n eventsFailed: listed.some((error) => error.refId === 'bot-events'),\n };\n};\n\n\n\n// Business Text \"afterRender\" code for the Liquidity bot's detail panel: the\n// SPA's trade and transfer detail dialogs (trade-history-panel.svelte,\n// transfer-panel.svelte in ST0x-Technology/st0x.liquidity) for the row whose\n// \u24d8 was clicked in the Trades or Rebalances table.\n//\n// The click sets the hidden `detail` variable to the row's id. A and B are\n// the Trades and Rebalances tables' own results through the Dashboard\n// datasource: each table's two frames, the exporter's and the bot's (the\n// newest 500 status entries of each log), and the script reads the frames\n// of the `source` picked. On the bot source, bot-events is the bot's event\n// timeline (the newest 500 liq_event lines); the exporter has no event\n// log, so there the dialog shows the status history. Closing the dialog\n// clears the variable.\n//\n// The panel is one empty column of the header row; only the dialog shows.\n//\n// MODE_LABELS, tradeCommands, transferCommands, latest, lineEntries,\n// eventTimeline, timelineIncomplete and queryState are not defined here: the\n// generator prepends them from recovery-guide.json, recovery-commands.js,\n// status-history.js and log-lines.js (see detail_panel()).\n\nconst theme = context.grafana.theme;\nconst root = context.element;\nroot.style.setProperty('--det-muted', theme.colors.text.secondary);\nroot.style.setProperty('--det-border', theme.colors.border.weak);\nroot.style.setProperty('--det-stripe', theme.colors.background.secondary);\nroot.style.setProperty('--det-card', theme.colors.background.primary);\nroot.style.setProperty('--det-text', theme.colors.text.primary);\n// Light theme swaps the status colours for darker shades (detail.css).\nroot.classList.toggle('det-light', !theme.isDark);\n\n// Alternate rows in the board's Trades and Rebalances\n// tables, like the SPA's. Grafana's table has no striping option, but its\n// grid marks every row rdg-row-even or rdg-row-odd. This panel lives only on\n// the Dashboard tab, so the rule goes into the page while the panel is\n// mounted and comes out when the board is left. Grafana can mount the panel\n// on a new element (tab switch, view mode), so each render takes the rule\n// over for its own element and restarts the watcher, and a watcher removes\n// the rule only while its element still owns it.\nlet stripes = document.getElementById('liquidity-table-stripes');\nif (!stripes) {\n stripes = document.createElement('style');\n stripes.id = 'liquidity-table-stripes';\n document.head.appendChild(stripes);\n}\nstripes.textContent = `.rdg-row-even { background: ${theme.isDark ? 'rgba(255, 255, 255, 0.03)' : 'rgba(0, 0, 0, 0.03)'}; }`;\nroot.__stripesOwner = root.__stripesOwner || Math.random().toString(36).slice(2);\nstripes.dataset.owner = root.__stripesOwner;\nclearInterval(root.__stripesWatch);\nroot.__stripesWatch = setInterval(() => {\n if (root.isConnected) return;\n clearInterval(root.__stripesWatch);\n if (stripes.dataset.owner === root.__stripesOwner) stripes.remove();\n}, 2000);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The id the tables asked for. Entries of any other id are a previous\n// query's data, still shown while the new one loads.\nconst wanted = context.grafana.replaceVariables('${detail}');\n\n// The board's environment is its project, and `source` names the frames.\nconst env = context.grafana.replaceVariables('${env}');\nconst SOURCE = context.grafana.replaceVariables('${source:text}') === 'bot' ? 'bot' : 'exporter';\n\n// The wanted id's entries in one frame, from the board's project only (see\n// lineEntries).\nconst entriesOf = (refId) =>\n lineEntries(\n (context.panelData?.series || []).find((series) => series.refId === refId),\n wanted,\n env\n );\n\n// The SPA's formatUtc: \"Oct 6, 12:24:39 UTC\".\nconst MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];\nconst utc = (value) => {\n const date = new Date(value);\n if (Number.isNaN(date.getTime())) return '\u2014';\n const pad = (number) => String(number).padStart(2, '0');\n return `${MONTHS[date.getUTCMonth()]} ${date.getUTCDate()}, ${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())} UTC`;\n};\n\n// The SPA's formatDecimal(value, 3): fixed decimals, comma groups, and the\n// raw value when it does not parse.\nconst decimal = (value) => {\n const number = Number(value);\n if (value === '' || value === null || value === undefined || Number.isNaN(number)) return escapeHtml(value || '\u2014');\n return number.toLocaleString('en-US', { minimumFractionDigits: 3, maximumFractionDigits: 3 });\n};\n\nconst VENUES = {\n raindex: ['Raindex', 'det-blue', true],\n bebop: ['Bebop', 'det-purple', true],\n uniswap_v4: ['Uniswap v4', 'det-pink', true],\n unknown_onchain: ['Unknown Onchain', '', true],\n alpaca: ['Alpaca', 'det-orange', false],\n dry_run: ['DryRun', '', false],\n};\nconst venue = (key) => VENUES[key] || [key || '\u2014', '', false];\n\nconst TYPES = {\n equity_mint: 'Mint',\n equity_redemption: 'Redeem',\n alpaca_to_base: 'Alpaca \u2192 Raindex',\n base_to_alpaca: 'Raindex \u2192 Alpaca',\n};\nconst typeOf = (transfer) => (transfer.kind === 'usdc_bridge' ? transfer.direction : transfer.kind);\nconst typeLabel = (transfer) => TYPES[typeOf(transfer)] || transfer.kind || '\u2014';\nconst assetOf = (transfer) => (transfer.kind === 'usdc_bridge' || !transfer.symbol ? 'USDC' : transfer.symbol);\n\n// The SPA's statusStyle colours.\nconst statusClass = (status) => {\n const lower = String(status || '').toLowerCase();\n if (lower.includes('completed') || lower.includes('filled') || lower.includes('deposited') || lower.includes('confirmed')) return 'det-green';\n if (lower.includes('reconciled')) return 'det-amber';\n if (lower.includes('failed')) return 'det-red';\n if (lower.includes('cancelled')) return 'det-orange';\n return 'det-blue';\n};\nconst statusLabel = (status) => {\n const text = String(status || '\u2014').replace(/_/g, ' ');\n return text.charAt(0).toUpperCase() + text.slice(1);\n};\n\n// --------------------------------------------------------------------------\n// Recovery commands (recovery-commands.js, prepended): the operations client,\n// which signs in with Google and calls the running bot's ops API, so no SSH\n// is needed and every command needs the bot.\n// --------------------------------------------------------------------------\n\n// The environment the board shows: the client takes the selector's text.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\nconst CLIENT = `st0x-liquidity-client --env ${ENV}`;\n\nconst modeClass = (mode) => (mode === 'requires-bot' ? 'det-amber' : 'det-red');\nconst commandBlock = (commands, note) =>\n commands.length === 0 && !note\n ? ''\n : `
CLI commands
${note ? `

${escapeHtml(note)}

` : ''}${commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command)}
\n
${escapeHtml(entry.description)}
\n ${escapeHtml(MODE_LABELS[entry.mode] || entry.mode)}\n
`\n )\n .join('')}
`;\n\n// --------------------------------------------------------------------------\n// The dialog\n// --------------------------------------------------------------------------\n\n// Block explorers by the bot's chain wire name. A chain without one, or an\n// unknown chain, shows the hash unlinked: a link to the wrong explorer would\n// read as \"this tx does not exist\".\nconst EXPLORERS = {\n base: 'https://basescan.org',\n ethereum: 'https://etherscan.io',\n hyperevm: 'https://hyperevmscan.io',\n};\nconst shortHash = (hash) => `${escapeHtml(hash.slice(0, 10))}\u2026${escapeHtml(hash.slice(-8))}`;\nconst txLink = (hash, chain) =>\n EXPLORERS[chain]\n ? `${shortHash(hash)} \u2197`\n : `${shortHash(hash)}`;\n\n// The bot's event timeline: each step with its time and fields, like the\n// SPA's dialog. On the exporter source, the status history instead: one\n// step per status change.\nconst eventLines = SOURCE === 'bot' ? entriesOf('bot-events') : [];\n// Only an error of a query this source reads fails the dialog (see\n// queryState). An Error state that lists no error fails it too, and one that\n// lists only other errors marks the row as possibly out of date.\nconst queries = queryState(context.panelData, SOURCE);\nconst eventsFailed = queries.eventsFailed;\nconst listedError = (errors) => errors.map((error) => error.message).filter(Boolean).join('; ') || 'unknown error';\n// Grafana can drop a failed table query's error when another query also\n// failed, so a row shown after a failed refresh says it can be out of date.\nconst staleNotice = () =>\n queries.unsure\n ? `

A query failed on the last refresh (${escapeHtml(listedError(queries.listed))}). Grafana may not list every query that failed, so this row and its CLI commands can be out of date. Refresh the board and check the row again before you run a command.

`\n : '';\nconst humanize = (step) => String(step || '').replace(/([a-z0-9])([A-Z])/g, '$1 $2');\nconst isTxHash = (value) => typeof value === 'string' && /^0x[0-9a-fA-F]{64}$/.test(value);\nconst eventValue = (key, value, chain) => {\n if (key === 'error') return `${escapeHtml(value)}`;\n if (isTxHash(value)) return txLink(value, chain);\n if (value !== null && typeof value === 'object') return `${escapeHtml(JSON.stringify(value))}`;\n return escapeHtml(value);\n};\n// A payload's own `chain` names where its hashes live; else the row's chain\n// (a trade's), else none, which leaves the hashes unlinked.\nconst eventFields = (payload, rowChain) => {\n const fields = payload && typeof payload === 'object' ? payload : {};\n const chain = typeof fields.chain === 'string' ? fields.chain : rowChain;\n return Object.entries(fields)\n .filter(([key]) => !/(_at|At|timestamp)$/.test(key))\n .map(([key, value]) => `
${escapeHtml(key.replace(/_/g, ' '))}${eventValue(key, value, chain)}
`)\n .join('');\n};\n\n// The events scan and the table scans run at different moments, so a row's\n// newest status can also be ahead of its events until the next refresh.\nconst timeline = (events, history, chain, rowEventId) =>\n events.length > 0\n ? `\n
Event timeline
\n ${eventsFailed ? '

The event query failed, so this timeline can be out of date.

' : ''}\n ${timelineIncomplete(events, rowEventId) ? '

Some events of this row are not here: they are not loaded yet (refresh), outside the newest 500 event lines the board loads, or the bot missed a line. The full timeline is in the SPA at liquidity.t0trade.com.

' : ''}\n
${events\n .map(\n (event) => `\n
\n \n
${escapeHtml(humanize(event.step))}
\n
${utc(event.time)}
\n ${eventFields(event.payload, chain)}\n
`\n )\n .join('')}
`\n : `\n
Status history
\n
${history\n .map(\n (entry) => `\n
\n \n
${escapeHtml(statusLabel(entry.status))}
\n
${utc(entry.time)}
\n ${entry.error ? `
${escapeHtml(entry.error)}
` : ''}\n
`\n )\n .join('')}
\n

${SOURCE === 'exporter' ? \"From the exporter's status log.\" : eventsFailed ? \"From the bot's status lines: the event query failed.\" : \"From the bot's status lines; its event lines for this row are not in the newest 500.\"} The full event timeline is in the SPA at liquidity.t0trade.com.

`;\n\n// valueHtml is trusted HTML: a caller escapes any data it puts in it.\nconst field = (name, valueHtml) => `
${escapeHtml(name)}${valueHtml}
`;\n\nconst tradeDialog = (trade) => {\n const [venueName, venueClass, onchain] = venue(trade.venue);\n const id = String(trade.id);\n // An onchain trade id is `chain:tx_hash:log_index` (OnChainTradeId), or\n // `tx_hash:log_index` from before the chain was part of it (Base).\n const parts = id.split(':');\n const [chain, hash, log] = parts.length === 3 ? parts : ['base', ...parts];\n const idCell =\n onchain && parts.length >= 2 && isTxHash(hash)\n ? `${txLink(hash, chain)} (${escapeHtml(chain)}, log ${escapeHtml(log)})`\n : `${escapeHtml(id)}`;\n return `\n
\n \n ${escapeHtml(trade.direction)}\n ${escapeHtml(trade.symbol)}\n ${decimal(trade.shares)} ${onchain ? 'wrapped shares' : 'shares'}\n ${escapeHtml(venueName)}\n \n \n
\n
\n ${staleNotice()}\n
\n ${field('ID', idCell)}\n ${field('Occurred At', utc(trade.occurred_at || trade.first))}\n ${field('Status', `${escapeHtml(statusLabel(trade.status))}${trade.error ? `
${escapeHtml(trade.error)}
` : ''}`)}\n
\n ${timeline(eventTimeline(eventLines, 'trade'), trade.history, onchain ? chain : null, trade.event_id)}\n ${commandBlock(tradeCommands(CLIENT, trade.symbol))}\n
`;\n};\n\nconst transferDialog = (transfer) => {\n const usdcFailed = transfer.kind === 'usdc_bridge' && String(transfer.status).toLowerCase() === 'failed';\n return `\n
\n \n ${escapeHtml(typeLabel(transfer))}\n ${escapeHtml(assetOf(transfer))}\n ${decimal(transfer.amount)}\n \n \n
\n
\n ${staleNotice()}\n
\n ${field('ID', `${escapeHtml(transfer.id)}`)}\n ${field('Started', utc(transfer.started_at || transfer.first))}\n ${field('Updated', utc(transfer.time))}\n ${field('Status', `${escapeHtml(statusLabel(transfer.status))}`)}\n
\n ${timeline(eventTimeline(eventLines, 'transfer', transfer.kind), transfer.history, null, transfer.event_id)}\n ${commandBlock(\n transferCommands(CLIENT, transfer),\n usdcFailed\n ? 'Reconcile applies only when the funds left their source venue, which the logs do not show. Check the transfer in the SPA before you reconcile it.'\n : ''\n )}\n
`;\n};\n\n// --------------------------------------------------------------------------\n// Open, refresh or close the one dialog\n// --------------------------------------------------------------------------\n\n// Kept on the element across renders: the plugin re-runs this code on every\n// refresh and query, into the same element.\nconst state = root.__det || (root.__det = { dismissed: null });\n\n// The variable can lag the close by a render (a refresh that was already\n// running), so an id the operator just closed is not reopened until the\n// variable has moved off it.\nif (state.dismissed !== null && state.dismissed !== wanted) state.dismissed = null;\n\n// Switching Environment keeps the tables' old rows on screen until the new\n// queries return, and those rows belong to the other project. Hold back\n// every row after a switch until a query has run (Grafana showed the panel\n// Loading) and then succeeded. Frames alone cannot tell: Grafana replays and\n// copies the old ones, and keeps them on an error. A Cloud Logging query\n// takes over a second, so the Loading state shows; if it does not, or shows\n// only on the switch render, the dialog says Loading\u2026 until the next\n// refresh, which fails safe. A query cancelled from the refresh picker\n// after the switch also ends Done with the old rows and releases the hold,\n// but each row carries its project (lineEntries), so a row of the other\n// project still does not show. The memory lives on the window, not the\n// element, so a remount (tab switch, view mode) keeps it. The first render\n// after the page loads is not held back: its rows are for the environment\n// it opened on.\nconst loading = context.panelData?.state === 'Loading';\nconst failed = queries.failed;\nconst envState = window.__liqDetailEnv || (window.__liqDetailEnv = { env: undefined, held: false, sawLoading: false });\nconst switched = envState.env !== undefined && envState.env !== env;\nenvState.env = env;\nif (switched) {\n // A Loading state on this render can be a run that started before the\n // switch, so only a later one counts.\n envState.held = true;\n envState.sawLoading = false;\n} else if (envState.held) {\n // A query that failed does not count as the one that ran.\n if (loading) envState.sawLoading = true;\n else if (failed) envState.sawLoading = false;\n else if (envState.sawLoading) envState.held = false;\n}\n\n// A failed refresh can keep the old frames, so it shows the error, not a\n// row and its commands that may be out of date.\nconst trade = envState.held || failed ? null : latest(entriesOf(`${SOURCE}-trades`));\nconst transfer = trade || envState.held || failed ? null : latest(entriesOf(`${SOURCE}-transfers`));\n// The dialog opens on the click, before the query returns, and says so when\n// the id has no entries in the panel's window.\nconst message = (text) => `\n
${escapeHtml(wanted)}
\n

${escapeHtml(text)}

`;\nconst queryError = () => listedError(queries.blocking);\nconst html = trade\n ? tradeDialog(trade)\n : transfer\n ? transferDialog(transfer)\n : wanted === ''\n ? null\n : failed\n ? message(`The Trades or Rebalances query failed (${queryError()}), so this row cannot be shown. Refresh the board to retry.`)\n : loading || envState.held\n ? message('Loading\u2026')\n : message('This id is not in the newest 500 status entries any more. Search the Logs tab for it.');\n\n// Reuse the dialog when it is still in the element, so a refresh with the\n// dialog open swaps its content without closing and reopening it.\nlet dialog = root.querySelector('dialog.det-dialog');\nif (!dialog) {\n dialog = document.createElement('dialog');\n dialog.className = 'det-dialog';\n root.appendChild(dialog);\n}\n\n// Property handlers, not listeners: each render replaces them instead of\n// stacking another copy.\ndialog.onclose = () => {\n state.dismissed = dialog.detId || null;\n dialog.detId = '';\n // An empty value, not null: removing the URL parameter leaves a textbox\n // variable at its old value, so the same row could not be reopened.\n if (context.grafana.replaceVariables('${detail}') !== '') {\n context.grafana.locationService.partial({ 'var-detail': '' }, true);\n }\n};\n// A click on the backdrop closes it, like the SPA.\ndialog.onclick = (event) => {\n if (event.target === dialog) dialog.close();\n};\n\nif (html === null || state.dismissed === wanted) {\n if (dialog.open) dialog.close();\n} else {\n if (dialog.detHtml !== html) {\n // A refresh that brings a new status keeps the operator's scroll.\n const body = dialog.querySelector('.det-dialog-body');\n const scroll = dialog.detId === wanted && body ? body.scrollTop : 0;\n dialog.innerHTML = html;\n dialog.detHtml = html;\n const next = dialog.querySelector('.det-dialog-body');\n if (next) next.scrollTop = scroll;\n }\n dialog.detId = wanted;\n dialog.querySelector('[data-close]').onclick = () => dialog.close();\n if (!dialog.open) dialog.showModal();\n}\n", + "afterRender": "const MODE_LABELS = {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"};\n\n// The board's per-row recovery commands: the client form of the SPA's\n// tradeRecoveryCommands and transferRecoveryCommands\n// (dashboard/src/lib/transfer.ts). The generator prepends this file to\n// detail.js; dashboard/src/lib/transfer-board.test.ts runs it against the SPA's\n// builders so the two cannot drift, importing it as a module through the\n// export line, which the generator drops. `client` is the client invocation\n// with its --env.\n\nconst tradeCommands = (client, symbol) => [\n {\n command: `${client} debug position release-hedge ${symbol} --order-id --reason \"\"`,\n description: \"Clear a position's stuck pending offchain order so normal hedging can retry. Needs the order id.\",\n mode: 'requires-bot',\n },\n {\n command: `${client} debug position set ${symbol} --target-net [--price-usdc ] --reason \"\"`,\n description:\n 'Override the net exposure after a manual correction. The target is signed: negative is short, 0 is flat. The price is required for a nonzero target unless the position already has a last price.',\n mode: 'requires-bot',\n },\n {\n command: `${client} debug view rebuild position --id ${symbol}`,\n description: 'Replay all events to reconstruct a corrupted position view.',\n mode: 'requires-bot',\n },\n];\n\nconst USDC_FAIL_ALPACA_TO_BASE =\n 'Alpaca to Base: the bot accepts this only before the burn, from a completed ' +\n 'withdrawal or a burn submission with no recorded burn, and it does not check the ' +\n 'chain. A burn the bot sent but did not record can still land, so do not fail the ' +\n 'transfer until you have done every check and the nonce close on Ethereum in ' +\n 'docs/cli-ops.md, \"Clearing a pre-burn guard latch\", with the bot stopped. Then fail ' +\n 'it with the offline stox fail-usdc-transfer before you start the bot again: a restart ' +\n 'sends the burn of a completed withdrawal (at a burn submission, this command after a ' +\n 'restart works too). If a burn landed or you are not certain, run resume-usdc instead. ' +\n 'The funds left Alpaca, so the guard stays held until you settle them with ' +\n 'reconcile-usdc.';\n\nconst USDC_FAIL_BASE_TO_ALPACA =\n 'Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, ' +\n 'and it does not check the chain. First confirm on Base that the bot wallet made no ' +\n \"OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 \" +\n 'on the orderbook either (a withdrawal sent before the inventory migration). Only a ' +\n 'USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in ' +\n 'the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 ' +\n 'sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the ' +\n 'network accepted but has not mined is not in the logs yet, so also wait until the ' +\n \"transfer's attempt timeout has passed and confirm the wallet has no pending \" +\n 'transaction (no pending withdraw4, pending nonce equals latest nonce on more than one ' +\n 'RPC provider, and the explorer shows no queued transaction from the wallet). A queued ' +\n 'transaction waits behind a nonce gap: with the bot stopped, fill each empty nonce ' +\n 'below it, in order, with a self-transfer as below, after making sure no send the bot ' +\n 'signed and keeps holds that nonce, then repeat the checks. These checks cannot prove ' +\n 'that nothing is pending. If a withdrawal landed or you are not certain, run ' +\n 'resume-usdc instead: it adopts a withdrawal of the full amount, and any other amount ' +\n 'ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to ' +\n 'the cash vault by hand. If nothing landed, the resume keeps redriving: repeat the ' +\n 'checks, and once every check is clean, take the next step. If none landed, close the ' +\n 'nonce before you fail the transfer: stop the bot, check again that the wallet has no ' +\n 'pending or queued transaction, make sure no send the bot signed and keeps holds that ' +\n 'nonce (if one does, start the bot so it rebroadcasts the send instead), send a 0-value ' +\n 'transfer with no calldata from the bot wallet to itself at the latest nonce, with ' +\n 'maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required ' +\n 'confirmations, and check the withdrawal logs again. The self-transfer closes only ' +\n 'that nonce. If a withdrawal mined instead, a transaction is pending or queued, or you ' +\n 'cannot tell, start the bot and run resume-usdc, and do not fail the transfer. ' +\n 'Otherwise run the offline stox fail-usdc-transfer, or this command after you restart ' +\n 'the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the ' +\n 'bot wallet back to the cash vault by hand. While a recorded withdrawal is still ' +\n 'confirming, run resume-usdc. Once it has confirmed the bot refuses: do every check ' +\n 'and the nonce close on Base in docs/cli-ops.md, \"Clearing a pre-burn guard latch\". If ' +\n 'a burn landed, something is pending or queued, or you are not certain, start the bot ' +\n 'and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox ' +\n 'fail-usdc-transfer and move the wallet USDC back by hand. If ' +\n 'a burn for this transfer mines later anyway, resume and reconcile refuse the failed ' +\n 'transfer. Keep the bot stopped until that USDC is at Alpaca: the offline commands send ' +\n 'from the bot wallet and replace whatever is pending at their nonce, so first run the ' +\n 'pending, queued and kept-send checks on Ethereum as docs/cli-ops.md describes, and let ' +\n 'any kept send mine. Mint it with stox ' +\n \"cctp complete-mint --burn-tx --source-chain base (it adopts a relayer's mint; \" +\n 'do not mint by hand, since the MessageSent log holds a placeholder nonce). Then send ' +\n 'the amount received that complete-mint printed, divided by 1,000,000 (it prints USDC ' +\n 'base units), to Alpaca with stox alpaca-deposit -a , and do not move that USDC ' +\n 'back to the vault. Once Alpaca credits it, convert ' +\n 'it with stox alpaca-convert -d to-usd -a , as the bot would have.';\n\nconst USDC_BRIDGING_BASE_TO_ALPACA =\n ' If the burn keeps failing, the running bot refuses fail-usdc-transfer here: see ' +\n 'fail-usdc-transfer in the CLI recovery guide for the offline path.';\n\nconst USDC_CONVERTING_ALPACA_TO_BASE =\n ' Alpaca to Base: this status covers two bot states. If the bot recorded the ' +\n 'conversion as complete, this resume continues with the Alpaca withdrawal. If not, the ' +\n 'bot cannot re-drive the USD to USDC conversion (it did not store the broker order ' +\n 'id), so this resume fails the transfer (ConversionFailed) and releases the guard, ' +\n 'without checking Alpaca. So first confirm at Alpaca that no USD to USDC order for ' +\n 'this transfer is still open (it is filled, cancelled or rejected), above all after a ' +\n 'conversion outcome unresolved alert; if one is open, cancel it or wait for it. Do not ' +\n 'move USDC by hand before you run the resume. If it ends in ConversionFailed, settle ' +\n 'any converted USDC by hand at Alpaca (reconcile-usdc does not apply). A later failure ' +\n 'has its own recovery.';\n\n// usdcResumeNote in transfer.ts: the extra resume text for a bridge's status\n// and direction.\nconst usdcResumeNote = (status, direction) => {\n if (direction === 'base_to_alpaca' && status === 'bridging') return USDC_BRIDGING_BASE_TO_ALPACA;\n if (direction !== 'base_to_alpaca' && status === 'converting') return USDC_CONVERTING_ALPACA_TO_BASE;\n return '';\n};\n\n// usdcFailCheck in transfer.ts: the check before fail-usdc-transfer, or null\n// where the running bot always refuses it.\nconst usdcFailCheck = (status, direction) => {\n if (direction === 'alpaca_to_base') {\n return status === 'withdrawing' || status === 'bridging' ? USDC_FAIL_ALPACA_TO_BASE : null;\n }\n if (direction === 'base_to_alpaca') return status === 'withdrawing' ? USDC_FAIL_BASE_TO_ALPACA : null;\n return status === 'withdrawing' || status === 'bridging'\n ? `${USDC_FAIL_ALPACA_TO_BASE} ${USDC_FAIL_BASE_TO_ALPACA}`\n : null;\n};\n\nconst transferCommands = (client, transfer) => {\n const status = String(transfer.status || '').toLowerCase();\n const failed = status === 'failed';\n const terminal = failed || status === 'completed' || status === 'reconciled';\n if (status === 'completed' || status === 'reconciled') return [];\n const id = transfer.id;\n\n if (transfer.kind === 'usdc_bridge') {\n // Reconcile needs the bridge's postBurn flag, which the exporter does not\n // log; the SPA offers nothing without it either.\n if (failed) return [];\n const direction =\n { alpaca_to_base: 'alpaca-to-base', base_to_alpaca: 'base-to-alpaca' }[transfer.direction] ||\n '';\n const resume = {\n command: `${client} debug resume-usdc ${direction} ${id}`,\n description:\n \"Re-drive this USDC bridge on the bot's transfer worker from the stage it stopped at.\" +\n usdcResumeNote(status, transfer.direction || null),\n mode: 'requires-bot',\n };\n const check = usdcFailCheck(status, transfer.direction || null);\n if (check === null) return [resume];\n return [\n resume,\n {\n command: `${client} debug fail-usdc-transfer ${id} --reason \"\"`,\n description: `Check the transfer first. ${check}`,\n mode: 'requires-bot',\n },\n ];\n }\n\n const kind = { equity_mint: 'mint', equity_redemption: 'redemption' }[transfer.kind];\n if (!kind) return [];\n const commands = [\n {\n command: `${client} debug recheck ${kind} ${id}`,\n description: 'Ask the running bot to re-poll the provider and complete the transfer if it settled.',\n mode: 'requires-bot',\n },\n ];\n if (!terminal) {\n commands.push({\n command: `${client} debug resume`,\n description:\n 'Re-drive ALL interrupted mints and redemptions via the bot (no id; best-effort per transfer, each succeeds or fails independently and failures are reported as counts).',\n mode: 'requires-bot',\n });\n commands.push({\n command: `${client} debug fail-equity-transfer ${kind} ${id} --reason \"\"`,\n description: 'Force this stuck transfer into the terminal Failed state. Use when it is permanently stuck.',\n mode: 'requires-bot',\n });\n }\n if (failed) {\n commands.push({\n command: `${client} debug reconcile-equity ${kind} ${id} --reason \"\"`,\n description: 'Mark a Failed transfer as Reconciled once its residue was handled out-of-band (bookkeeping).',\n mode: 'requires-bot',\n });\n }\n return commands;\n};\n\n\n// The detail dialog's status history: a Trades or Rebalances row's status-log\n// entries in order, and the status the row is at. The generator prepends this\n// file to detail.js; dashboard/src/lib/transfer-board.test.ts imports it as a\n// module through the export line, which the generator drops.\n\n// A USDC bridge's in-flight statuses in lifecycle order per direction. Their\n// log times can go backwards (WithdrawalComplete carries confirmed_at, the\n// BridgingSubmitting after it the older initiated_at), so two in-flight\n// entries of a bridge are ordered by this. Anything else, failed included\n// (the bot can recover a bridge out of a failure), orders by time.\n//\n// Known limit: a Base to Alpaca bridge recovered out of DepositFailed reaches\n// converting, which the bot stamps with the bridge's initiated_at. That entry\n// sorts before the failed one, so the dialog keeps showing the status before\n// it (failed, or depositing) until the bridge completes.\nconst BRIDGE_LIFECYCLE = {\n alpaca_to_base: ['converting', 'withdrawing', 'bridging', 'depositing'],\n base_to_alpaca: ['withdrawing', 'bridging', 'depositing', 'converting'],\n};\nconst stage = (entry) => {\n const lifecycle = entry.kind === 'usdc_bridge' ? BRIDGE_LIFECYCLE[entry.direction] : null;\n const index = lifecycle ? lifecycle.indexOf(String(entry.status || '').toLowerCase()) : -1;\n return index >= 0 ? index : null;\n};\n\n// The event store's sequence in a bot line's `event_id`\n// (`::`), or null for an exporter entry.\nconst sequenceOf = (entry) => {\n const match = /:(\\d+)$/.exec(entry.event_id || '');\n return match ? Number(match[1]) : null;\n};\n\n// The row at its latest status, with its status history oldest first. The\n// bot's lines carry the commit order, so they sort by sequence. Exporter\n// entries sort by time (ties keep Cloud Logging's newest-first order\n// reversed), then each pair of neighbouring in-flight bridge entries is put\n// in lifecycle order.\nconst latest = (entries) => {\n if (entries.length === 0) return null;\n if (entries.every((entry) => sequenceOf(entry) !== null)) {\n const history = [...entries].sort((left, right) => sequenceOf(left) - sequenceOf(right));\n const last = history[history.length - 1];\n return { ...last, first: Math.min(...entries.map((entry) => entry.time)), history };\n }\n const history = [...entries].reverse().sort((left, right) => left.time - right.time);\n for (let swapped = true; swapped; ) {\n swapped = false;\n for (let index = 1; index < history.length; index++) {\n const [before, after] = [stage(history[index - 1]), stage(history[index])];\n if (before !== null && after !== null && before > after) {\n [history[index - 1], history[index]] = [history[index], history[index - 1]];\n swapped = true;\n }\n }\n }\n const last = history[history.length - 1];\n return { ...last, first: Math.min(...entries.map((entry) => entry.time)), history };\n};\n\n\n// The detail dialog's reading of Cloud Logging frames: a row's entries, and\n// the bot's event timeline. The generator prepends this file to detail.js;\n// dashboard/src/lib/transfer-board.test.ts imports it as a module through\n// the export line, which the generator drops.\n\n// A JSON object, or null for anything else (no value, bad JSON, a string).\nconst parseObject = (value) => {\n if (value !== null && typeof value === 'object') return value;\n try {\n const parsed = JSON.parse(value);\n return parsed !== null && typeof parsed === 'object' ? parsed : null;\n } catch (error) {\n return null;\n }\n};\n\n// The entries of `id` in one frame of the plugin's log-lines shape, newest\n// first like the frame: each line's time and payload. The exporter's body\n// is its JSON payload. A bot line's body is its message, so its payload\n// comes from the flattened `jsonPayload.` labels, which the exporter's\n// lines carry too. A line from another project than `project` belongs to\n// the environment the board showed before a switch and is dropped. A line\n// whose `event_id` came before (the bot's lines, shipped twice) is dropped.\nconst lineEntries = (frame, id, project) => {\n if (!frame) return [];\n const column = (name) => frame.fields.find((field) => field.name === name);\n const times = column('timestamp');\n const bodies = column('body');\n const labels = column('labels');\n const count = times ? times.values.length : 0;\n const seen = new Set();\n const entries = [];\n for (let index = 0; index < count; index++) {\n const flat = (labels && parseObject(labels.values[index])) || {};\n const lineProject = flat['resource.labels.project_id'];\n if (lineProject && lineProject !== project) continue;\n const fromLabels = {};\n for (const [key, value] of Object.entries(flat)) {\n if (key.startsWith('jsonPayload.')) fromLabels[key.slice('jsonPayload.'.length)] = value;\n }\n const payload = (bodies && parseObject(bodies.values[index])) || fromLabels;\n if (!payload.id || String(payload.id) !== id) continue;\n if (payload.event_id) {\n if (seen.has(payload.event_id)) continue;\n seen.add(payload.event_id);\n }\n entries.push({ time: Number(times.values[index]), ...payload });\n }\n return entries;\n};\n\n// A row's event timeline from its liq_event entries, oldest first: the\n// events of a trade, or of a transfer of `kind` (each transfer kind is its\n// own aggregate). A trade's events are not matched on venue, because a\n// venue correction changes it. Each payload is parsed from its JSON string.\nconst eventTimeline = (entries, parent, kind) =>\n entries\n .filter((entry) => entry.parent === parent && (parent !== 'transfer' || entry.kind === kind))\n .map((entry) => ({ ...entry, payload: parseObject(entry.payload) || {} }))\n .sort((left, right) => Number(left.sequence) - Number(right.sequence));\n\n// Whether a row's event timeline misses events: the event store numbers an\n// aggregate's events from 1 without gaps, so a timeline that does not start\n// at 1, skips a number, or ends before the event of the row's latest status\n// line (`rowEventId`, `::`) lacks lines: lines not\n// loaded yet (the events scan ran before the row's), the newest-500 window,\n// or a line the bot never wrote.\nconst timelineIncomplete = (events, rowEventId) => {\n const match = /:(\\d+)$/.exec(rowEventId || '');\n const last = events.length > 0 ? Number(events[events.length - 1].sequence) : 0;\n return (\n events.some((event, index) => Number(event.sequence) !== index + 1) ||\n (match !== null && Number(match[1]) > last)\n );\n};\n\n// The panel errors that concern the source shown. A failed query of the\n// other source, or of the event timeline, leaves the rows usable, so it does\n// not block them. An error without a refId blocks: it cannot be placed.\nconst blockingErrors = (errors, source) => {\n const other = source === 'bot' ? 'exporter-' : 'bot-';\n return errors.filter(\n (error) => !error.refId || (error.refId !== 'bot-events' && !error.refId.startsWith(other))\n );\n};\n\n// What the panel's last refresh says about the queries the source shown\n// reads. Grafana's Mixed datasource keeps only the last failing query's\n// errors (runRequest's processResponsePacket overwrites error and errors per\n// packet), so an Error state whose listed errors do not block can still hide\n// a failed table query, and its frames can be the ones the failed refresh\n// kept. `failed`: a listed error blocks, or the state is Error with none\n// listed. `unsure`: the state is Error and only errors that do not block are\n// listed, so a row shown can be out of date. `eventsFailed`: the event query\n// is among the listed errors, whether Grafana gave a list or one error.\nconst queryState = (panelData, source) => {\n const listed = panelData?.errors || (panelData?.error ? [panelData.error] : []);\n const blocking = blockingErrors(listed, source);\n const isError = panelData?.state === 'Error';\n return {\n listed,\n blocking,\n failed: isError && (listed.length === 0 || blocking.length > 0),\n unsure: isError && listed.length > 0 && blocking.length === 0,\n eventsFailed: listed.some((error) => error.refId === 'bot-events'),\n };\n};\n\n\n\n// Business Text \"afterRender\" code for the Liquidity bot's detail panel: the\n// SPA's trade and transfer detail dialogs (trade-history-panel.svelte,\n// transfer-panel.svelte in ST0x-Technology/st0x.liquidity) for the row whose\n// \u24d8 was clicked in the Trades or Rebalances table.\n//\n// The click sets the hidden `detail` variable to the row's id. A and B are\n// the Trades and Rebalances tables' own results through the Dashboard\n// datasource: each table's two frames, the exporter's and the bot's (the\n// newest 500 status entries of each log), and the script reads the frames\n// of the `source` picked. On the bot source, bot-events is the bot's event\n// timeline (the newest 500 liq_event lines); the exporter has no event\n// log, so there the dialog shows the status history. Closing the dialog\n// clears the variable.\n//\n// The panel is one empty column of the header row; only the dialog shows.\n//\n// MODE_LABELS, tradeCommands, transferCommands, latest, lineEntries,\n// eventTimeline, timelineIncomplete and queryState are not defined here: the\n// generator prepends them from recovery-guide.json, recovery-commands.js,\n// status-history.js and log-lines.js (see detail_panel()).\n\nconst theme = context.grafana.theme;\nconst root = context.element;\nroot.style.setProperty('--det-muted', theme.colors.text.secondary);\nroot.style.setProperty('--det-border', theme.colors.border.weak);\nroot.style.setProperty('--det-stripe', theme.colors.background.secondary);\nroot.style.setProperty('--det-card', theme.colors.background.primary);\nroot.style.setProperty('--det-text', theme.colors.text.primary);\n// Light theme swaps the status colours for darker shades (detail.css).\nroot.classList.toggle('det-light', !theme.isDark);\n\n// Alternate rows in the board's Trades and Rebalances\n// tables, like the SPA's. Grafana's table has no striping option, but its\n// grid marks every row rdg-row-even or rdg-row-odd. This panel lives only on\n// the Dashboard tab, so the rule goes into the page while the panel is\n// mounted and comes out when the board is left. Grafana can mount the panel\n// on a new element (tab switch, view mode), so each render takes the rule\n// over for its own element and restarts the watcher, and a watcher removes\n// the rule only while its element still owns it.\nlet stripes = document.getElementById('liquidity-table-stripes');\nif (!stripes) {\n stripes = document.createElement('style');\n stripes.id = 'liquidity-table-stripes';\n document.head.appendChild(stripes);\n}\nstripes.textContent = `.rdg-row-even { background: ${theme.isDark ? 'rgba(255, 255, 255, 0.03)' : 'rgba(0, 0, 0, 0.03)'}; }`;\nroot.__stripesOwner = root.__stripesOwner || Math.random().toString(36).slice(2);\nstripes.dataset.owner = root.__stripesOwner;\nclearInterval(root.__stripesWatch);\nroot.__stripesWatch = setInterval(() => {\n if (root.isConnected) return;\n clearInterval(root.__stripesWatch);\n if (stripes.dataset.owner === root.__stripesOwner) stripes.remove();\n}, 2000);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The id the tables asked for. Entries of any other id are a previous\n// query's data, still shown while the new one loads.\nconst wanted = context.grafana.replaceVariables('${detail}');\n\n// The board's environment is its project, and `source` names the frames.\nconst env = context.grafana.replaceVariables('${env}');\nconst SOURCE = context.grafana.replaceVariables('${source:text}') === 'bot' ? 'bot' : 'exporter';\n\n// The wanted id's entries in one frame, from the board's project only (see\n// lineEntries).\nconst entriesOf = (refId) =>\n lineEntries(\n (context.panelData?.series || []).find((series) => series.refId === refId),\n wanted,\n env\n );\n\n// The SPA's formatUtc: \"Oct 6, 12:24:39 UTC\".\nconst MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];\nconst utc = (value) => {\n const date = new Date(value);\n if (Number.isNaN(date.getTime())) return '\u2014';\n const pad = (number) => String(number).padStart(2, '0');\n return `${MONTHS[date.getUTCMonth()]} ${date.getUTCDate()}, ${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())} UTC`;\n};\n\n// The SPA's formatDecimal(value, 3): fixed decimals, comma groups, and the\n// raw value when it does not parse.\nconst decimal = (value) => {\n const number = Number(value);\n if (value === '' || value === null || value === undefined || Number.isNaN(number)) return escapeHtml(value || '\u2014');\n return number.toLocaleString('en-US', { minimumFractionDigits: 3, maximumFractionDigits: 3 });\n};\n\nconst VENUES = {\n raindex: ['Raindex', 'det-blue', true],\n bebop: ['Bebop', 'det-purple', true],\n uniswap_v4: ['Uniswap v4', 'det-pink', true],\n unknown_onchain: ['Unknown Onchain', '', true],\n alpaca: ['Alpaca', 'det-orange', false],\n dry_run: ['DryRun', '', false],\n};\nconst venue = (key) => VENUES[key] || [key || '\u2014', '', false];\n\nconst TYPES = {\n equity_mint: 'Mint',\n equity_redemption: 'Redeem',\n alpaca_to_base: 'Alpaca \u2192 Raindex',\n base_to_alpaca: 'Raindex \u2192 Alpaca',\n};\nconst typeOf = (transfer) => (transfer.kind === 'usdc_bridge' ? transfer.direction : transfer.kind);\nconst typeLabel = (transfer) => TYPES[typeOf(transfer)] || transfer.kind || '\u2014';\nconst assetOf = (transfer) => (transfer.kind === 'usdc_bridge' || !transfer.symbol ? 'USDC' : transfer.symbol);\n\n// The SPA's statusStyle colours.\nconst statusClass = (status) => {\n const lower = String(status || '').toLowerCase();\n if (lower.includes('completed') || lower.includes('filled') || lower.includes('deposited') || lower.includes('confirmed')) return 'det-green';\n if (lower.includes('reconciled')) return 'det-amber';\n if (lower.includes('failed')) return 'det-red';\n if (lower.includes('cancelled')) return 'det-orange';\n return 'det-blue';\n};\nconst statusLabel = (status) => {\n const text = String(status || '\u2014').replace(/_/g, ' ');\n return text.charAt(0).toUpperCase() + text.slice(1);\n};\n\n// --------------------------------------------------------------------------\n// Recovery commands (recovery-commands.js, prepended): the operations client,\n// which signs in with Google and calls the running bot's ops API, so no SSH\n// is needed and every command needs the bot.\n// --------------------------------------------------------------------------\n\n// The environment the board shows: the client takes the selector's text.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\nconst CLIENT = `st0x-liquidity-client --env ${ENV}`;\n\nconst modeClass = (mode) => (mode === 'requires-bot' ? 'det-amber' : 'det-red');\nconst commandBlock = (commands, note) =>\n commands.length === 0 && !note\n ? ''\n : `
CLI commands
${note ? `

${escapeHtml(note)}

` : ''}${commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command)}
\n
${escapeHtml(entry.description)}
\n ${escapeHtml(MODE_LABELS[entry.mode] || entry.mode)}\n
`\n )\n .join('')}
`;\n\n// --------------------------------------------------------------------------\n// The dialog\n// --------------------------------------------------------------------------\n\n// Block explorers by the bot's chain wire name. A chain without one, or an\n// unknown chain, shows the hash unlinked: a link to the wrong explorer would\n// read as \"this tx does not exist\".\nconst EXPLORERS = {\n base: 'https://basescan.org',\n ethereum: 'https://etherscan.io',\n hyperevm: 'https://hyperevmscan.io',\n};\nconst shortHash = (hash) => `${escapeHtml(hash.slice(0, 10))}\u2026${escapeHtml(hash.slice(-8))}`;\nconst txLink = (hash, chain) =>\n EXPLORERS[chain]\n ? `${shortHash(hash)} \u2197`\n : `${shortHash(hash)}`;\n\n// The bot's event timeline: each step with its time and fields, like the\n// SPA's dialog. On the exporter source, the status history instead: one\n// step per status change.\nconst eventLines = SOURCE === 'bot' ? entriesOf('bot-events') : [];\n// Only an error of a query this source reads fails the dialog (see\n// queryState). An Error state that lists no error fails it too, and one that\n// lists only other errors marks the row as possibly out of date.\nconst queries = queryState(context.panelData, SOURCE);\nconst eventsFailed = queries.eventsFailed;\nconst listedError = (errors) => errors.map((error) => error.message).filter(Boolean).join('; ') || 'unknown error';\n// Grafana can drop a failed table query's error when another query also\n// failed, so a row shown after a failed refresh says it can be out of date.\nconst staleNotice = () =>\n queries.unsure\n ? `

A query failed on the last refresh (${escapeHtml(listedError(queries.listed))}). Grafana may not list every query that failed, so this row and its CLI commands can be out of date. Refresh the board and check the row again before you run a command.

`\n : '';\nconst humanize = (step) => String(step || '').replace(/([a-z0-9])([A-Z])/g, '$1 $2');\nconst isTxHash = (value) => typeof value === 'string' && /^0x[0-9a-fA-F]{64}$/.test(value);\nconst eventValue = (key, value, chain) => {\n if (key === 'error') return `${escapeHtml(value)}`;\n if (isTxHash(value)) return txLink(value, chain);\n if (value !== null && typeof value === 'object') return `${escapeHtml(JSON.stringify(value))}`;\n return escapeHtml(value);\n};\n// A payload's own `chain` names where its hashes live; else the row's chain\n// (a trade's), else none, which leaves the hashes unlinked.\nconst eventFields = (payload, rowChain) => {\n const fields = payload && typeof payload === 'object' ? payload : {};\n const chain = typeof fields.chain === 'string' ? fields.chain : rowChain;\n return Object.entries(fields)\n .filter(([key]) => !/(_at|At|timestamp)$/.test(key))\n .map(([key, value]) => `
${escapeHtml(key.replace(/_/g, ' '))}${eventValue(key, value, chain)}
`)\n .join('');\n};\n\n// The events scan and the table scans run at different moments, so a row's\n// newest status can also be ahead of its events until the next refresh.\nconst timeline = (events, history, chain, rowEventId) =>\n events.length > 0\n ? `\n
Event timeline
\n ${eventsFailed ? '

The event query failed, so this timeline can be out of date.

' : ''}\n ${timelineIncomplete(events, rowEventId) ? '

Some events of this row are not here: they are not loaded yet (refresh), outside the newest 500 event lines the board loads, or the bot missed a line. The full timeline is in the SPA at liquidity.t0trade.com.

' : ''}\n
${events\n .map(\n (event) => `\n
\n \n
${escapeHtml(humanize(event.step))}
\n
${utc(event.time)}
\n ${eventFields(event.payload, chain)}\n
`\n )\n .join('')}
`\n : `\n
Status history
\n
${history\n .map(\n (entry) => `\n
\n \n
${escapeHtml(statusLabel(entry.status))}
\n
${utc(entry.time)}
\n ${entry.error ? `
${escapeHtml(entry.error)}
` : ''}\n
`\n )\n .join('')}
\n

${SOURCE === 'exporter' ? \"From the exporter's status log.\" : eventsFailed ? \"From the bot's status lines: the event query failed.\" : \"From the bot's status lines; its event lines for this row are not in the newest 500.\"} The full event timeline is in the SPA at liquidity.t0trade.com.

`;\n\n// valueHtml is trusted HTML: a caller escapes any data it puts in it.\nconst field = (name, valueHtml) => `
${escapeHtml(name)}${valueHtml}
`;\n\nconst tradeDialog = (trade) => {\n const [venueName, venueClass, onchain] = venue(trade.venue);\n const id = String(trade.id);\n // An onchain trade id is `chain:tx_hash:log_index` (OnChainTradeId), or\n // `tx_hash:log_index` from before the chain was part of it (Base).\n const parts = id.split(':');\n const [chain, hash, log] = parts.length === 3 ? parts : ['base', ...parts];\n const idCell =\n onchain && parts.length >= 2 && isTxHash(hash)\n ? `${txLink(hash, chain)} (${escapeHtml(chain)}, log ${escapeHtml(log)})`\n : `${escapeHtml(id)}`;\n return `\n
\n \n ${escapeHtml(trade.direction)}\n ${escapeHtml(trade.symbol)}\n ${decimal(trade.shares)} ${onchain ? 'wrapped shares' : 'shares'}\n ${escapeHtml(venueName)}\n \n \n
\n
\n ${staleNotice()}\n
\n ${field('ID', idCell)}\n ${field('Occurred At', utc(trade.occurred_at || trade.first))}\n ${field('Status', `${escapeHtml(statusLabel(trade.status))}${trade.error ? `
${escapeHtml(trade.error)}
` : ''}`)}\n
\n ${timeline(eventTimeline(eventLines, 'trade'), trade.history, onchain ? chain : null, trade.event_id)}\n ${commandBlock(tradeCommands(CLIENT, trade.symbol))}\n
`;\n};\n\nconst transferDialog = (transfer) => {\n const usdcFailed = transfer.kind === 'usdc_bridge' && String(transfer.status).toLowerCase() === 'failed';\n return `\n
\n \n ${escapeHtml(typeLabel(transfer))}\n ${escapeHtml(assetOf(transfer))}\n ${decimal(transfer.amount)}\n \n \n
\n
\n ${staleNotice()}\n
\n ${field('ID', `${escapeHtml(transfer.id)}`)}\n ${field('Started', utc(transfer.started_at || transfer.first))}\n ${field('Updated', utc(transfer.time))}\n ${field('Status', `${escapeHtml(statusLabel(transfer.status))}`)}\n
\n ${timeline(eventTimeline(eventLines, 'transfer', transfer.kind), transfer.history, null, transfer.event_id)}\n ${commandBlock(\n transferCommands(CLIENT, transfer),\n usdcFailed\n ? 'Reconcile applies only when the funds left their source venue, which the logs do not show. Check the transfer in the SPA before you reconcile it.'\n : ''\n )}\n
`;\n};\n\n// --------------------------------------------------------------------------\n// Open, refresh or close the one dialog\n// --------------------------------------------------------------------------\n\n// Kept on the element across renders: the plugin re-runs this code on every\n// refresh and query, into the same element.\nconst state = root.__det || (root.__det = { dismissed: null });\n\n// The variable can lag the close by a render (a refresh that was already\n// running), so an id the operator just closed is not reopened until the\n// variable has moved off it.\nif (state.dismissed !== null && state.dismissed !== wanted) state.dismissed = null;\n\n// Switching Environment keeps the tables' old rows on screen until the new\n// queries return, and those rows belong to the other project. Hold back\n// every row after a switch until a query has run (Grafana showed the panel\n// Loading) and then succeeded. Frames alone cannot tell: Grafana replays and\n// copies the old ones, and keeps them on an error. A Cloud Logging query\n// takes over a second, so the Loading state shows; if it does not, or shows\n// only on the switch render, the dialog says Loading\u2026 until the next\n// refresh, which fails safe. A query cancelled from the refresh picker\n// after the switch also ends Done with the old rows and releases the hold,\n// but each row carries its project (lineEntries), so a row of the other\n// project still does not show. The memory lives on the window, not the\n// element, so a remount (tab switch, view mode) keeps it. The first render\n// after the page loads is not held back: its rows are for the environment\n// it opened on.\nconst loading = context.panelData?.state === 'Loading';\nconst failed = queries.failed;\nconst envState = window.__liqDetailEnv || (window.__liqDetailEnv = { env: undefined, held: false, sawLoading: false });\nconst switched = envState.env !== undefined && envState.env !== env;\nenvState.env = env;\nif (switched) {\n // A Loading state on this render can be a run that started before the\n // switch, so only a later one counts.\n envState.held = true;\n envState.sawLoading = false;\n} else if (envState.held) {\n // A query that failed does not count as the one that ran.\n if (loading) envState.sawLoading = true;\n else if (failed) envState.sawLoading = false;\n else if (envState.sawLoading) envState.held = false;\n}\n\n// A failed refresh can keep the old frames, so it shows the error, not a\n// row and its commands that may be out of date.\nconst trade = envState.held || failed ? null : latest(entriesOf(`${SOURCE}-trades`));\nconst transfer = trade || envState.held || failed ? null : latest(entriesOf(`${SOURCE}-transfers`));\n// The dialog opens on the click, before the query returns, and says so when\n// the id has no entries in the panel's window.\nconst message = (text) => `\n
${escapeHtml(wanted)}
\n

${escapeHtml(text)}

`;\nconst queryError = () => listedError(queries.blocking);\nconst html = trade\n ? tradeDialog(trade)\n : transfer\n ? transferDialog(transfer)\n : wanted === ''\n ? null\n : failed\n ? message(`The Trades or Rebalances query failed (${queryError()}), so this row cannot be shown. Refresh the board to retry.`)\n : loading || envState.held\n ? message('Loading\u2026')\n : message('This id is not in the newest 500 status entries any more. Search the Logs tab for it.');\n\n// Reuse the dialog when it is still in the element, so a refresh with the\n// dialog open swaps its content without closing and reopening it.\nlet dialog = root.querySelector('dialog.det-dialog');\nif (!dialog) {\n dialog = document.createElement('dialog');\n dialog.className = 'det-dialog';\n root.appendChild(dialog);\n}\n\n// Property handlers, not listeners: each render replaces them instead of\n// stacking another copy.\ndialog.onclose = () => {\n state.dismissed = dialog.detId || null;\n dialog.detId = '';\n // An empty value, not null: removing the URL parameter leaves a textbox\n // variable at its old value, so the same row could not be reopened.\n if (context.grafana.replaceVariables('${detail}') !== '') {\n context.grafana.locationService.partial({ 'var-detail': '' }, true);\n }\n};\n// A click on the backdrop closes it, like the SPA.\ndialog.onclick = (event) => {\n if (event.target === dialog) dialog.close();\n};\n\nif (html === null || state.dismissed === wanted) {\n if (dialog.open) dialog.close();\n} else {\n if (dialog.detHtml !== html) {\n // A refresh that brings a new status keeps the operator's scroll.\n const body = dialog.querySelector('.det-dialog-body');\n const scroll = dialog.detId === wanted && body ? body.scrollTop : 0;\n dialog.innerHTML = html;\n dialog.detHtml = html;\n const next = dialog.querySelector('.det-dialog-body');\n if (next) next.scrollTop = scroll;\n }\n dialog.detId = wanted;\n dialog.querySelector('[data-close]').onclick = () => dialog.close();\n if (!dialog.open) dialog.showModal();\n}\n", "styles": "/* Business Text \"styles\" for the Liquidity bot's trade and transfer detail\n dialog. Theme colours come in as --det-* variables set by detail.js;\n status, venue and side colours are the SPA's (Tailwind 500). */\n\n.det-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.det-muted { color: var(--det-muted); }\n.det-break { word-break: break-all; }\n\n.det-green { color: #22c55e; }\n.det-red { color: #ef4444; }\n.det-amber { color: #f59e0b; }\n.det-orange { color: #f97316; }\n.det-blue { color: #3b82f6; }\n.det-purple { color: #a855f7; }\n.det-pink { color: #ec4899; }\n\n/* Tailwind 700 on the light theme, so status text keeps a 4.5:1 contrast\n against the white dialog. */\n.det-light .det-green { color: #15803d; }\n.det-light .det-red { color: #b91c1c; }\n.det-light .det-amber { color: #b45309; }\n.det-light .det-orange { color: #c2410c; }\n.det-light .det-blue { color: #1d4ed8; }\n.det-light .det-purple { color: #7e22ce; }\n.det-light .det-pink { color: #be185d; }\n\n.det-dialog {\n width: min(36rem, 92vw);\n padding: 0;\n border: 1px solid var(--det-border);\n border-radius: 8px;\n background: var(--det-card);\n color: var(--det-text);\n font-size: 14px;\n white-space: normal;\n}\n\n.det-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n\n.det-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--det-border);\n}\n\n.det-title-line {\n display: flex;\n flex-wrap: wrap;\n align-items: center;\n gap: 8px;\n font-size: 16px;\n font-weight: 600;\n}\n\n.det-title-line .det-mono { font-size: 14px; font-weight: 400; }\n\n.det-close {\n border: 0;\n background: transparent;\n color: var(--det-muted);\n font-size: 20px;\n line-height: 1;\n cursor: pointer;\n}\n\n.det-close:hover { color: var(--det-text); }\n\n.det-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 14px;\n}\n\n.det-dialog-body a { color: #3b82f6; }\n.det-light .det-dialog-body a { color: #1d4ed8; }\n\n.det-fields { margin-bottom: 16px; }\n\n.det-field {\n display: flex;\n gap: 8px;\n padding: 2px 0;\n}\n\n.det-field > .det-muted { flex-shrink: 0; }\n\n.det-section-title,\n.det-cli-title {\n margin-bottom: 8px;\n font-weight: 600;\n}\n\n.det-timeline {\n margin-bottom: 8px;\n padding-left: 16px;\n border-left: 2px solid var(--det-border);\n}\n\n.det-step {\n position: relative;\n padding-bottom: 12px;\n}\n\n.det-dot {\n position: absolute;\n top: 6px;\n left: -21px;\n width: 8px;\n height: 8px;\n border-radius: 9999px;\n background: currentColor;\n}\n\n.det-cli {\n margin-top: 16px;\n padding-top: 12px;\n border-top: 1px solid var(--det-border);\n}\n\n.det-command {\n margin-bottom: 8px;\n padding: 8px 12px;\n border: 1px solid var(--det-border);\n border-radius: 6px;\n background: var(--det-stripe);\n}\n\n.det-command pre {\n margin: 0 0 6px;\n padding: 6px 8px;\n border-radius: 4px;\n background: var(--det-card);\n font-size: 12px;\n white-space: pre-wrap;\n word-break: break-all;\n}\n\n.det-mode {\n display: inline-block;\n margin-top: 4px;\n font-size: 12px;\n}\n", "wrap": false } diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json b/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json index 9319c3659..29195e2f4 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json @@ -364,7 +364,7 @@ "content": "
", "defaultContent": "No bot data.", "helpers": "", - "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum that no CCTP burn left the bot wallet after this transfer started (its Started time) and that the wallet has no pending transaction (pending nonce equals latest nonce); if you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, and pending nonce equals latest nonce on more than one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once they show that no withdrawal landed and none is pending, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn left the bot wallet after this transfer started and the wallet has no pending transaction. If any of that is not certain, start the bot and run resume-usdc; else run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", + "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn, and it does not check the chain. A burn the bot sent but did not record can still land, so do not fail the transfer until you have done every check and the nonce close on Ethereum in docs/cli-ops.md, \\\"Clearing a pre-burn guard latch\\\", with the bot stopped. Then fail it with the offline stox fail-usdc-transfer before you start the bot again: a restart sends the burn of a completed withdrawal (at a burn submission, this command after a restart works too). If a burn landed or you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, pending nonce equals latest nonce on more than one RPC provider, and the explorer shows no queued transaction from the wallet). A queued transaction waits behind a nonce gap: with the bot stopped, fill each empty nonce below it, in order, with a self-transfer as below, after making sure no send the bot signed and keeps holds that nonce, then repeat the checks. These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a withdrawal of the full amount, and any other amount ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the cash vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once every check is clean, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, check again that the wallet has no pending or queued transaction, make sure no send the bot signed and keeps holds that nonce (if one does, start the bot so it rebroadcasts the send instead), send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. The self-transfer closes only that nonce. If a withdrawal mined instead, a transaction is pending or queued, or you cannot tell, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the cash vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: do every check and the nonce close on Base in docs/cli-ops.md, \\\"Clearing a pre-burn guard latch\\\". If a burn landed, something is pending or queued, or you are not certain, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer and move the wallet USDC back by hand. If a burn for this transfer mines later anyway, resume and reconcile refuse the failed transfer. Keep the bot stopped until that USDC is at Alpaca: the offline commands send from the bot wallet and replace whatever is pending at their nonce, so first run the pending, queued and kept-send checks on Ethereum as docs/cli-ops.md describes, and let any kept send mine. Mint it with stox cctp complete-mint --burn-tx --source-chain base (it adopts a relayer's mint; do not mint by hand, since the MessageSent log holds a placeholder nonce). Then send the amount received that complete-mint printed, divided by 1,000,000 (it prints USDC base units), to Alpaca with stox alpaca-deposit -a , and do not move that USDC back to the vault. Once Alpaca credits it, convert it with stox alpaca-convert -d to-usd -a , as the bot would have.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", "styles": "/* Business Text \"styles\" for the Liquidity bot header row and its two\n dialogs. Theme colours come in as --hdr-* variables set by header.js. */\n\n.hdr {\n display: flex;\n align-items: center;\n justify-content: space-between;\n gap: 8px;\n height: 100%;\n font-size: 13px;\n white-space: nowrap;\n}\n\n.hdr-left,\n.hdr-right {\n display: flex;\n align-items: center;\n gap: 8px;\n min-width: 0;\n}\n\n.hdr-pill {\n display: inline-flex;\n align-items: center;\n gap: 4px;\n padding: 2px 10px;\n border-radius: 9999px;\n background: var(--hdr-pill);\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n border: 0;\n}\n\n.hdr-button {\n color: var(--hdr-muted);\n cursor: pointer;\n}\n\n.hdr-button:hover,\n.hdr-guide:hover,\n.hdr-close:hover {\n color: var(--hdr-text);\n}\n\n.hdr-guide {\n padding: 2px 8px;\n border: 1px solid var(--hdr-border);\n border-radius: 4px;\n background: transparent;\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n cursor: pointer;\n}\n\n.hdr-badge {\n padding: 2px 8px;\n border-radius: 9999px;\n font-size: 13px;\n line-height: 18px;\n font-weight: 600;\n}\n\n.hdr-ok { background: #f5f5f5; color: #171717; }\n.hdr-bad { background: #ef4444; color: #fff; }\n.hdr-unknown { border: 1px solid var(--hdr-border); color: var(--hdr-muted); }\n\n.hdr-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.hdr-muted { color: var(--hdr-muted); }\n\n.hdr-dialog {\n padding: 0;\n border: 1px solid var(--hdr-border);\n border-radius: 8px;\n background: var(--hdr-card);\n color: var(--hdr-text);\n white-space: normal;\n}\n\n.hdr-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n.hdr-narrow { width: min(28rem, 92vw); }\n.hdr-wide { width: min(42rem, 92vw); }\n\n.hdr-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--hdr-border);\n font-size: 14px;\n font-weight: 600;\n}\n\n.hdr-close {\n border: 0;\n background: transparent;\n color: var(--hdr-muted);\n font-size: 18px;\n line-height: 1;\n cursor: pointer;\n}\n\n.hdr-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 13px;\n}\n\n.hdr-row {\n display: flex;\n justify-content: space-between;\n gap: 16px;\n padding: 4px 0;\n}\n\n.hdr-trunc {\n overflow: hidden;\n text-overflow: ellipsis;\n white-space: nowrap;\n}\n\n.hdr-rule { border-top: 1px solid var(--hdr-border); margin: 8px 0; }\n.hdr-intro { margin: 0 0 16px; }\n.hdr-group { margin-bottom: 20px; }\n.hdr-group h3 { margin: 0 0 8px; font-size: 13px; text-transform: capitalize; }\n\n.hdr-command {\n margin-bottom: 12px;\n padding: 8px 12px;\n border: 1px solid var(--hdr-border);\n border-radius: 6px;\n background: var(--hdr-pill);\n}\n\n.hdr-command pre {\n margin: 0;\n padding: 8px;\n border-radius: 4px;\n background: var(--hdr-card);\n font-size: 12px;\n white-space: pre-wrap;\n word-break: break-all;\n}\n\n.hdr-grid {\n display: grid;\n grid-template-columns: auto 1fr;\n gap: 4px 12px;\n margin-top: 6px;\n font-size: 12px;\n}\n\n.hdr-amber { color: #f59e0b; }\n.hdr-red { color: #ef4444; }\n\n/* Tailwind 700 on the light theme, like detail.css, for a 4.5:1 contrast\n against the white dialog. */\n.hdr-light .hdr-amber { color: #b45309; }\n.hdr-light .hdr-red { color: #b91c1c; }\n", "wrap": false }, diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json b/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json index d1982327d..f100e9361 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json @@ -202,7 +202,7 @@ "content": "
", "defaultContent": "No bot data.", "helpers": "", - "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum that no CCTP burn left the bot wallet after this transfer started (its Started time) and that the wallet has no pending transaction (pending nonce equals latest nonce); if you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, and pending nonce equals latest nonce on more than one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once they show that no withdrawal landed and none is pending, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn left the bot wallet after this transfer started and the wallet has no pending transaction. If any of that is not certain, start the bot and run resume-usdc; else run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", + "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn, and it does not check the chain. A burn the bot sent but did not record can still land, so do not fail the transfer until you have done every check and the nonce close on Ethereum in docs/cli-ops.md, \\\"Clearing a pre-burn guard latch\\\", with the bot stopped. Then fail it with the offline stox fail-usdc-transfer before you start the bot again: a restart sends the burn of a completed withdrawal (at a burn submission, this command after a restart works too). If a burn landed or you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, pending nonce equals latest nonce on more than one RPC provider, and the explorer shows no queued transaction from the wallet). A queued transaction waits behind a nonce gap: with the bot stopped, fill each empty nonce below it, in order, with a self-transfer as below, after making sure no send the bot signed and keeps holds that nonce, then repeat the checks. These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a withdrawal of the full amount, and any other amount ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the cash vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once every check is clean, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, check again that the wallet has no pending or queued transaction, make sure no send the bot signed and keeps holds that nonce (if one does, start the bot so it rebroadcasts the send instead), send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. The self-transfer closes only that nonce. If a withdrawal mined instead, a transaction is pending or queued, or you cannot tell, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the cash vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: do every check and the nonce close on Base in docs/cli-ops.md, \\\"Clearing a pre-burn guard latch\\\". If a burn landed, something is pending or queued, or you are not certain, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer and move the wallet USDC back by hand. If a burn for this transfer mines later anyway, resume and reconcile refuse the failed transfer. Keep the bot stopped until that USDC is at Alpaca: the offline commands send from the bot wallet and replace whatever is pending at their nonce, so first run the pending, queued and kept-send checks on Ethereum as docs/cli-ops.md describes, and let any kept send mine. Mint it with stox cctp complete-mint --burn-tx --source-chain base (it adopts a relayer's mint; do not mint by hand, since the MessageSent log holds a placeholder nonce). Then send the amount received that complete-mint printed, divided by 1,000,000 (it prints USDC base units), to Alpaca with stox alpaca-deposit -a , and do not move that USDC back to the vault. Once Alpaca credits it, convert it with stox alpaca-convert -d to-usd -a , as the bot would have.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", "styles": "/* Business Text \"styles\" for the Liquidity bot header row and its two\n dialogs. Theme colours come in as --hdr-* variables set by header.js. */\n\n.hdr {\n display: flex;\n align-items: center;\n justify-content: space-between;\n gap: 8px;\n height: 100%;\n font-size: 13px;\n white-space: nowrap;\n}\n\n.hdr-left,\n.hdr-right {\n display: flex;\n align-items: center;\n gap: 8px;\n min-width: 0;\n}\n\n.hdr-pill {\n display: inline-flex;\n align-items: center;\n gap: 4px;\n padding: 2px 10px;\n border-radius: 9999px;\n background: var(--hdr-pill);\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n border: 0;\n}\n\n.hdr-button {\n color: var(--hdr-muted);\n cursor: pointer;\n}\n\n.hdr-button:hover,\n.hdr-guide:hover,\n.hdr-close:hover {\n color: var(--hdr-text);\n}\n\n.hdr-guide {\n padding: 2px 8px;\n border: 1px solid var(--hdr-border);\n border-radius: 4px;\n background: transparent;\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n cursor: pointer;\n}\n\n.hdr-badge {\n padding: 2px 8px;\n border-radius: 9999px;\n font-size: 13px;\n line-height: 18px;\n font-weight: 600;\n}\n\n.hdr-ok { background: #f5f5f5; color: #171717; }\n.hdr-bad { background: #ef4444; color: #fff; }\n.hdr-unknown { border: 1px solid var(--hdr-border); color: var(--hdr-muted); }\n\n.hdr-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.hdr-muted { color: var(--hdr-muted); }\n\n.hdr-dialog {\n padding: 0;\n border: 1px solid var(--hdr-border);\n border-radius: 8px;\n background: var(--hdr-card);\n color: var(--hdr-text);\n white-space: normal;\n}\n\n.hdr-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n.hdr-narrow { width: min(28rem, 92vw); }\n.hdr-wide { width: min(42rem, 92vw); }\n\n.hdr-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--hdr-border);\n font-size: 14px;\n font-weight: 600;\n}\n\n.hdr-close {\n border: 0;\n background: transparent;\n color: var(--hdr-muted);\n font-size: 18px;\n line-height: 1;\n cursor: pointer;\n}\n\n.hdr-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 13px;\n}\n\n.hdr-row {\n display: flex;\n justify-content: space-between;\n gap: 16px;\n padding: 4px 0;\n}\n\n.hdr-trunc {\n overflow: hidden;\n text-overflow: ellipsis;\n white-space: nowrap;\n}\n\n.hdr-rule { border-top: 1px solid var(--hdr-border); margin: 8px 0; }\n.hdr-intro { margin: 0 0 16px; }\n.hdr-group { margin-bottom: 20px; }\n.hdr-group h3 { margin: 0 0 8px; font-size: 13px; text-transform: capitalize; }\n\n.hdr-command {\n margin-bottom: 12px;\n padding: 8px 12px;\n border: 1px solid var(--hdr-border);\n border-radius: 6px;\n background: var(--hdr-pill);\n}\n\n.hdr-command pre {\n margin: 0;\n padding: 8px;\n border-radius: 4px;\n background: var(--hdr-card);\n font-size: 12px;\n white-space: pre-wrap;\n word-break: break-all;\n}\n\n.hdr-grid {\n display: grid;\n grid-template-columns: auto 1fr;\n gap: 4px 12px;\n margin-top: 6px;\n font-size: 12px;\n}\n\n.hdr-amber { color: #f59e0b; }\n.hdr-red { color: #ef4444; }\n\n/* Tailwind 700 on the light theme, like detail.css, for a 4.5:1 contrast\n against the white dialog. */\n.hdr-light .hdr-amber { color: #b45309; }\n.hdr-light .hdr-red { color: #b91c1c; }\n", "wrap": false }, diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json b/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json index 18c2fbc1d..799d71d4a 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json @@ -243,7 +243,7 @@ "content": "
", "defaultContent": "No bot data.", "helpers": "", - "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum that no CCTP burn left the bot wallet after this transfer started (its Started time) and that the wallet has no pending transaction (pending nonce equals latest nonce); if you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, and pending nonce equals latest nonce on more than one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once they show that no withdrawal landed and none is pending, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn left the bot wallet after this transfer started and the wallet has no pending transaction. If any of that is not certain, start the bot and run resume-usdc; else run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", + "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn, and it does not check the chain. A burn the bot sent but did not record can still land, so do not fail the transfer until you have done every check and the nonce close on Ethereum in docs/cli-ops.md, \\\"Clearing a pre-burn guard latch\\\", with the bot stopped. Then fail it with the offline stox fail-usdc-transfer before you start the bot again: a restart sends the burn of a completed withdrawal (at a burn submission, this command after a restart works too). If a burn landed or you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, pending nonce equals latest nonce on more than one RPC provider, and the explorer shows no queued transaction from the wallet). A queued transaction waits behind a nonce gap: with the bot stopped, fill each empty nonce below it, in order, with a self-transfer as below, after making sure no send the bot signed and keeps holds that nonce, then repeat the checks. These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a withdrawal of the full amount, and any other amount ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the cash vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once every check is clean, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, check again that the wallet has no pending or queued transaction, make sure no send the bot signed and keeps holds that nonce (if one does, start the bot so it rebroadcasts the send instead), send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. The self-transfer closes only that nonce. If a withdrawal mined instead, a transaction is pending or queued, or you cannot tell, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the cash vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: do every check and the nonce close on Base in docs/cli-ops.md, \\\"Clearing a pre-burn guard latch\\\". If a burn landed, something is pending or queued, or you are not certain, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer and move the wallet USDC back by hand. If a burn for this transfer mines later anyway, resume and reconcile refuse the failed transfer. Keep the bot stopped until that USDC is at Alpaca: the offline commands send from the bot wallet and replace whatever is pending at their nonce, so first run the pending, queued and kept-send checks on Ethereum as docs/cli-ops.md describes, and let any kept send mine. Mint it with stox cctp complete-mint --burn-tx --source-chain base (it adopts a relayer's mint; do not mint by hand, since the MessageSent log holds a placeholder nonce). Then send the amount received that complete-mint printed, divided by 1,000,000 (it prints USDC base units), to Alpaca with stox alpaca-deposit -a , and do not move that USDC back to the vault. Once Alpaca credits it, convert it with stox alpaca-convert -d to-usd -a , as the bot would have.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", "styles": "/* Business Text \"styles\" for the Liquidity bot header row and its two\n dialogs. Theme colours come in as --hdr-* variables set by header.js. */\n\n.hdr {\n display: flex;\n align-items: center;\n justify-content: space-between;\n gap: 8px;\n height: 100%;\n font-size: 13px;\n white-space: nowrap;\n}\n\n.hdr-left,\n.hdr-right {\n display: flex;\n align-items: center;\n gap: 8px;\n min-width: 0;\n}\n\n.hdr-pill {\n display: inline-flex;\n align-items: center;\n gap: 4px;\n padding: 2px 10px;\n border-radius: 9999px;\n background: var(--hdr-pill);\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n border: 0;\n}\n\n.hdr-button {\n color: var(--hdr-muted);\n cursor: pointer;\n}\n\n.hdr-button:hover,\n.hdr-guide:hover,\n.hdr-close:hover {\n color: var(--hdr-text);\n}\n\n.hdr-guide {\n padding: 2px 8px;\n border: 1px solid var(--hdr-border);\n border-radius: 4px;\n background: transparent;\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n cursor: pointer;\n}\n\n.hdr-badge {\n padding: 2px 8px;\n border-radius: 9999px;\n font-size: 13px;\n line-height: 18px;\n font-weight: 600;\n}\n\n.hdr-ok { background: #f5f5f5; color: #171717; }\n.hdr-bad { background: #ef4444; color: #fff; }\n.hdr-unknown { border: 1px solid var(--hdr-border); color: var(--hdr-muted); }\n\n.hdr-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.hdr-muted { color: var(--hdr-muted); }\n\n.hdr-dialog {\n padding: 0;\n border: 1px solid var(--hdr-border);\n border-radius: 8px;\n background: var(--hdr-card);\n color: var(--hdr-text);\n white-space: normal;\n}\n\n.hdr-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n.hdr-narrow { width: min(28rem, 92vw); }\n.hdr-wide { width: min(42rem, 92vw); }\n\n.hdr-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--hdr-border);\n font-size: 14px;\n font-weight: 600;\n}\n\n.hdr-close {\n border: 0;\n background: transparent;\n color: var(--hdr-muted);\n font-size: 18px;\n line-height: 1;\n cursor: pointer;\n}\n\n.hdr-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 13px;\n}\n\n.hdr-row {\n display: flex;\n justify-content: space-between;\n gap: 16px;\n padding: 4px 0;\n}\n\n.hdr-trunc {\n overflow: hidden;\n text-overflow: ellipsis;\n white-space: nowrap;\n}\n\n.hdr-rule { border-top: 1px solid var(--hdr-border); margin: 8px 0; }\n.hdr-intro { margin: 0 0 16px; }\n.hdr-group { margin-bottom: 20px; }\n.hdr-group h3 { margin: 0 0 8px; font-size: 13px; text-transform: capitalize; }\n\n.hdr-command {\n margin-bottom: 12px;\n padding: 8px 12px;\n border: 1px solid var(--hdr-border);\n border-radius: 6px;\n background: var(--hdr-pill);\n}\n\n.hdr-command pre {\n margin: 0;\n padding: 8px;\n border-radius: 4px;\n background: var(--hdr-card);\n font-size: 12px;\n white-space: pre-wrap;\n word-break: break-all;\n}\n\n.hdr-grid {\n display: grid;\n grid-template-columns: auto 1fr;\n gap: 4px 12px;\n margin-top: 6px;\n font-size: 12px;\n}\n\n.hdr-amber { color: #f59e0b; }\n.hdr-red { color: #ef4444; }\n\n/* Tailwind 700 on the light theme, like detail.css, for a 4.5:1 contrast\n against the white dialog. */\n.hdr-light .hdr-amber { color: #b45309; }\n.hdr-light .hdr-red { color: #b91c1c; }\n", "wrap": false }, diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json b/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json index eb5277c17..372d4a3b7 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json @@ -249,7 +249,7 @@ "content": "
", "defaultContent": "No bot data.", "helpers": "", - "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum that no CCTP burn left the bot wallet after this transfer started (its Started time) and that the wallet has no pending transaction (pending nonce equals latest nonce); if you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, and pending nonce equals latest nonce on more than one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once they show that no withdrawal landed and none is pending, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn left the bot wallet after this transfer started and the wallet has no pending transaction. If any of that is not certain, start the bot and run resume-usdc; else run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", + "afterRender": "const RECOVERY_GUIDE = {\"source\": \"ST0x-Technology/st0x.liquidity dashboard/src/lib/transfer.ts RECOVERY_GUIDE, same commit as this file; regenerate from there, do not edit by hand\", \"modeLabels\": {\"requires-bot\": \"REST \\u2014 requires the running bot\", \"direct-db\": \"direct DB \\u2014 stop the bot / ensure it is not driving this id\"}, \"groups\": [{\"object\": \"transfer\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug recheck \", \"description\": \"Re-poll the provider and complete the transfer if it has settled.\", \"whenToUse\": \"A transfer is stuck or failed but the provider may have settled it.\", \"appliesTo\": \"Equity mint / redemption (any non-completed state), failed Base to Alpaca USDC deposit with an onchain deposit transaction. A failed Alpaca to Base deposit needs manual settlement, then reconcile-usdc.\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume\", \"description\": \"Re-drive ALL interrupted mints and redemptions (no id; best-effort per transfer, failures reported as counts).\", \"whenToUse\": \"Equity transfers were interrupted mid-flight and need re-driving via the bot.\", \"appliesTo\": \"All in-flight equity transfers\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug resume-usdc \", \"description\": \"Re-drive a single USDC bridge on the bot's transfer worker.\", \"whenToUse\": \"A USDC bridge stopped mid-flight, before or after the burn. For an Alpaca to Base bridge at converting, first confirm at Alpaca that no USD to USDC order for the transfer is still open: the resume can fail it without asking Alpaca (see its transfer dialog).\", \"appliesTo\": \"USDC bridge (in-flight)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-equity-transfer --reason \\\"\\\"\", \"description\": \"Force a stuck transfer into the terminal Failed state.\", \"whenToUse\": \"A mint/redemption is permanently stuck and unrecoverable.\", \"appliesTo\": \"Equity mint / redemption (non-terminal)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug fail-usdc-transfer --reason \\\"\\\"\", \"description\": \"Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn, and it does not check the chain. A burn the bot sent but did not record can still land, so do not fail the transfer until you have done every check and the nonce close on Ethereum in docs/cli-ops.md, \\\"Clearing a pre-burn guard latch\\\", with the bot stopped. Then fail it with the offline stox fail-usdc-transfer before you start the bot again: a restart sends the burn of a completed withdrawal (at a burn submission, this command after a restart works too). If a burn landed or you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, pending nonce equals latest nonce on more than one RPC provider, and the explorer shows no queued transaction from the wallet). A queued transaction waits behind a nonce gap: with the bot stopped, fill each empty nonce below it, in order, with a self-transfer as below, after making sure no send the bot signed and keeps holds that nonce, then repeat the checks. These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a withdrawal of the full amount, and any other amount ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the cash vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once every check is clean, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, check again that the wallet has no pending or queued transaction, make sure no send the bot signed and keeps holds that nonce (if one does, start the bot so it rebroadcasts the send instead), send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. The self-transfer closes only that nonce. If a withdrawal mined instead, a transaction is pending or queued, or you cannot tell, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the cash vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: do every check and the nonce close on Base in docs/cli-ops.md, \\\"Clearing a pre-burn guard latch\\\". If a burn landed, something is pending or queued, or you are not certain, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer and move the wallet USDC back by hand. If a burn for this transfer mines later anyway, resume and reconcile refuse the failed transfer. Keep the bot stopped until that USDC is at Alpaca: the offline commands send from the bot wallet and replace whatever is pending at their nonce, so first run the pending, queued and kept-send checks on Ethereum as docs/cli-ops.md describes, and let any kept send mine. Mint it with stox cctp complete-mint --burn-tx --source-chain base (it adopts a relayer's mint; do not mint by hand, since the MessageSent log holds a placeholder nonce). Then send the amount received that complete-mint printed, divided by 1,000,000 (it prints USDC base units), to Alpaca with stox alpaca-deposit -a , and do not move that USDC back to the vault. Once Alpaca credits it, convert it with stox alpaca-convert -d to-usd -a , as the bot would have.\", \"whenToUse\": \"A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.\", \"appliesTo\": \"USDC bridge (pre-burn stuck)\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-equity --reason \\\"\\\"\", \"description\": \"Mark a terminally-failed equity transfer Reconciled after handling residue manually.\", \"whenToUse\": \"An equity transfer is in a terminal failure and its residue was settled out-of-band (bookkeeping).\", \"appliesTo\": \"Failed equity mint / redemption\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug reconcile-usdc --reason \", \"description\": \"Mark a failed USDC bridge Reconciled after its funds were settled manually.\", \"whenToUse\": \"A USDC bridge failed after its funds left the source venue, and they were settled out-of-band.\", \"appliesTo\": \"USDC failure with funds off the source venue (DepositFailed, any AlpacaToBase BridgingFailed, a post-burn BaseToAlpaca BridgingFailed, or BaseToAlpaca ConversionFailed)\", \"mode\": \"requires-bot\"}]}, {\"object\": \"position\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug position release-hedge --order-id --reason \\\"\\\"\", \"description\": \"Clear a position's pending offchain order pointer so hedging can retry.\", \"whenToUse\": \"A position is wedged on a hedge order that never resolved.\", \"appliesTo\": \"Position with a stuck pending offchain order\", \"mode\": \"requires-bot\"}, {\"command\": \"st0x-liquidity-client --env production debug position set --target-net [--price-usdc ] --reason \\\"\\\"\", \"description\": \"Override a position\\u2019s net exposure after a manual correction. The target is signed: negative is short, 0 is flat.\", \"whenToUse\": \"The recorded net exposure has drifted from reality and must be set explicitly.\", \"appliesTo\": \"Any position\", \"mode\": \"requires-bot\"}]}, {\"object\": \"view\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug view rebuild (--id | --all)\", \"description\": \"Replay all events to reconstruct a corrupted materialized view.\", \"whenToUse\": \"A view became corrupted (e.g. lost updates from optimistic-lock conflicts).\", \"appliesTo\": \"Position / offchain-order / vault-registry views\", \"mode\": \"requires-bot\"}]}, {\"object\": \"cctp\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug cctp complete-mint --burn-tx --source-chain \", \"description\": \"Complete the destination-chain mint of a stuck CCTP transfer. A burn Circle has not attested yet fails at once as retryable; rerun it later.\", \"whenToUse\": \"A CCTP burn succeeded but attestation polling was interrupted before the mint.\", \"appliesTo\": \"CCTP cross-chain USDC transfer\", \"mode\": \"requires-bot\"}]}, {\"object\": \"trade\", \"commands\": [{\"command\": \"st0x-liquidity-client --env production debug process-tx [--chain ]\", \"description\": \"Re-account a missed onchain fill: record the trade and place the hedge.\", \"whenToUse\": \"The bot missed an onchain fill and the position/hedge was never updated.\", \"appliesTo\": \"Onchain (Raindex) fills\", \"mode\": \"requires-bot\"}]}]};\n\n// Business Text \"afterRender\" code for the Liquidity bot header: the SPA's\n// HeaderBar and SettingsBar (header-bar.svelte, settings-bar.svelte,\n// recovery-guide.svelte in ST0x-Technology/st0x.liquidity) in one row, with\n// the same Config and CLI recovery guide dialogs.\n//\n// Input: context.data, rows with a `k` column naming the value (up, uptime,\n// commit, info, equity_target, ...) plus the string labels some of them\n// carry (git_commit on commit; broker, wallet_kind, ... on info).\n//\n// RECOVERY_GUIDE is not defined here: the generator prepends it from\n// recovery-guide.json (see pills() in the generator), because the template output is\n// sanitized and cannot carry data into this code.\n\nconst rows = Array.isArray(context.data) ? context.data : [];\nconst value = {};\nconst labels = {};\n// The stackdriver datasource runs a range query even for an instant target,\n// so each `k` can arrive at several times; keep the newest.\nconst seen = {};\nfor (const row of rows) {\n const time = Number(row.Time) || 0;\n if (row.k !== 'commit') {\n if (row.k in seen && seen[row.k] > time) continue;\n seen[row.k] = time;\n }\n if (row.k === 'commit') {\n // A range query keeps the previous commit's series for a few minutes\n // after a deploy. The commit row's value is its sample timestamp, so the\n // newest one is the running commit.\n if (!labels.commit || row.Value > labels.commit.at) labels.commit = { sha: row.git_commit, at: row.Value };\n } else if (row.k === 'info') {\n labels.info = row;\n } else {\n value[row.k] = row.Value === null || row.Value === undefined || Number.isNaN(Number(row.Value)) ? null : Number(row.Value);\n }\n}\nconst info = labels.info || {};\n\nconst theme = context.grafana.theme;\nconst root = context.element;\n// The guide's commands target the environment the board shows: the client\n// takes the selector's text, production or staging.\nconst ENV = context.grafana.replaceVariables('${env:text}') === 'staging' ? 'staging' : 'production';\n// The plugin pads each rendered row by 8px, which makes this one-row panel\n// scroll; the row has no room for it.\nroot.style.padding = '0';\nroot.style.height = '100%';\nroot.style.overflow = 'hidden';\nroot.style.setProperty('--hdr-muted', theme.colors.text.secondary);\n// Light theme swaps the mode colours for darker shades (header.css).\nroot.classList.toggle('hdr-light', !theme.isDark);\nroot.style.setProperty('--hdr-border', theme.colors.border.weak);\nroot.style.setProperty('--hdr-pill', theme.colors.background.secondary);\nroot.style.setProperty('--hdr-card', theme.colors.background.primary);\nroot.style.setProperty('--hdr-text', theme.colors.text.primary);\n\nconst escapeHtml = (text) =>\n String(text ?? '').replace(/[&<>\"']/g, (char) => ({ '&': '&', '<': '<', '>': '>', '\"': '"', \"'\": ''' })[char]);\n\n// The SPA's fmtPct and band strings: \"50%\" and \"45%\u201355%\".\nconst pct = (fraction) => `${(fraction * 100).toFixed(0)}%`;\nconst band = (target, deviation) =>\n target === null || target === undefined\n ? ''\n : deviation === null || deviation === undefined\n ? ''\n : ` (${pct(target - deviation)}\u2013${pct(target + deviation)})`;\n\n// The SPA's formatUptime.\nconst uptime = (seconds) => {\n if (seconds === null || seconds === undefined) return '';\n const days = Math.floor(seconds / 86400);\n const hours = Math.floor((seconds % 86400) / 3600);\n const minutes = Math.floor((seconds % 3600) / 60);\n if (days > 0) return `up ${days}d ${hours}h`;\n if (hours > 0) return `up ${hours}h ${minutes}m`;\n return `up ${minutes}m`;\n};\n\nconst clock = () => new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC';\n\nconst pill = (inner) => `${inner}`;\n// 1 up, 0 down; no value means the source is not reporting, which says\n// nothing about the bot.\nconst connection =\n value.up === 1 ? ['hdr-ok', 'Connected'] : value.up === 0 ? ['hdr-bad', 'Disconnected'] : ['hdr-unknown', 'No data'];\n\nconst pills = [\n info.broker ? pill(`${escapeHtml(info.broker)}`) : '',\n value.equity_target !== null && value.equity_target !== undefined\n ? pill(`Equity ${pct(value.equity_target)}${band(value.equity_target, value.equity_deviation)}`)\n : '',\n value.usdc_target !== null && value.usdc_target !== undefined\n ? pill(`USDC ${pct(value.usdc_target)}${band(value.usdc_target, value.usdc_deviation)}`)\n : '',\n value.trigger !== null && value.trigger !== undefined ? pill(`Trigger $${value.trigger}`) : '',\n value.cash_reserved !== null && value.cash_reserved !== undefined\n ? pill(`Reserve $${value.cash_reserved}`)\n : '',\n '',\n].join('');\n\nconst configRow = (name, text) =>\n `
${name}${escapeHtml(text)}
`;\nconst seconds = (number) => (number === null || number === undefined ? '' : `${number}s`);\n\nconst configDialog = `\n\n
Configuration
\n
\n ${\n info.wallet_kind\n ? configRow('Wallet', info.wallet_kind) +\n configRow('Address', info.wallet_address) +\n (info.wallet_kind === 'turnkey' && info.turnkey_organization ? configRow('Turnkey org', info.turnkey_organization) : '') +\n '
'\n : ''\n }\n ${configRow('Exported log level', info.log_level)}\n ${configRow('Server port', info.server_port)}\n ${configRow('Deployment block', value.deployment_block ?? '')}\n ${configRow('Order polling', seconds(value.order_polling))}\n ${configRow('Inventory polling', seconds(value.inventory_polling))}\n ${configRow('Cash reserve', value.cash_reserved !== null && value.cash_reserved !== undefined ? `$${value.cash_reserved}` : 'none')}\n ${configRow('Orderbook', info.orderbook)}\n
\n`;\n\nconst guide = RECOVERY_GUIDE;\nconst modeClass = (mode) =>\n mode === 'requires-bot' ? 'hdr-amber' : 'hdr-red';\nconst guideDialog = `\n\n
CLI recovery guide
\n
\n

Every recovery command, grouped by object. They use st0x-liquidity-client, which signs in with your Google account and calls the running bot's API, so no SSH is needed and the bot must be running. The one exception is failing a Base to Alpaca USDC bridge after its vault withdrawal confirmed: that needs the offline stox fail-usdc-transfer with the bot stopped.

\n ${guide.groups\n .map(\n (group) => `\n
\n

${escapeHtml(group.object)}

\n ${group.commands\n .map(\n (entry) => `\n
\n 
${escapeHtml(entry.command.replace('--env production', `--env ${ENV}`))}
\n
\n What${escapeHtml(entry.description)}\n When${escapeHtml(entry.whenToUse)}\n Applies to${escapeHtml(entry.appliesTo)}\n Mode${escapeHtml(guide.modeLabels[entry.mode] || entry.mode)}\n
\n
`\n )\n .join('')}\n
`\n )\n .join('')}\n
\n`;\n\n// The board refreshes every minute and each render replaces the markup, which\n// would close a dialog the operator is reading. Remember it to reopen it at\n// the same scroll position.\nconst openDialog = root.querySelector('dialog[open]');\nconst reopen = openDialog && {\n name: openDialog.dataset.dialog,\n scroll: openDialog.querySelector('.hdr-dialog-body')?.scrollTop || 0,\n};\n\nroot.innerHTML = `\n
\n
${pills}
\n
\n \n ${clock()}\n ${labels.commit && labels.commit.sha ? `${escapeHtml(labels.commit.sha.slice(0, 7))}` : ''}\n ${uptime(value.uptime)}\n ${connection[1]}\n
\n
\n${configDialog}\n${guideDialog}`;\n\nroot.querySelectorAll('[data-open]').forEach((button) => {\n button.addEventListener('click', () => root.querySelector(`[data-dialog=\"${button.dataset.open}\"]`).showModal());\n});\nroot.querySelectorAll('dialog').forEach((dialog) => {\n dialog.querySelector('[data-close]').addEventListener('click', () => dialog.close());\n // A click on the backdrop closes it, like the SPA.\n dialog.addEventListener('click', (event) => {\n if (event.target === dialog) dialog.close();\n });\n});\nif (reopen) {\n const dialog = root.querySelector(`[data-dialog=\"${reopen.name}\"]`);\n if (dialog) {\n dialog.showModal();\n const body = dialog.querySelector('.hdr-dialog-body');\n if (body) body.scrollTop = reopen.scroll;\n }\n}\n\n// The SPA's ticking UTC clock. One interval per element, replaced on every\n// render so a refresh never stacks them. Grafana can also mount a new\n// element (tab switch, view mode), so an interval whose element left the\n// page stops itself.\nclearInterval(root.__hdrClock);\nroot.__hdrClock = setInterval(() => {\n if (!root.isConnected) {\n clearInterval(root.__hdrClock);\n return;\n }\n const node = root.querySelector('[data-clock]');\n if (node) node.textContent = clock();\n}, 1000);\n", "styles": "/* Business Text \"styles\" for the Liquidity bot header row and its two\n dialogs. Theme colours come in as --hdr-* variables set by header.js. */\n\n.hdr {\n display: flex;\n align-items: center;\n justify-content: space-between;\n gap: 8px;\n height: 100%;\n font-size: 13px;\n white-space: nowrap;\n}\n\n.hdr-left,\n.hdr-right {\n display: flex;\n align-items: center;\n gap: 8px;\n min-width: 0;\n}\n\n.hdr-pill {\n display: inline-flex;\n align-items: center;\n gap: 4px;\n padding: 2px 10px;\n border-radius: 9999px;\n background: var(--hdr-pill);\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n border: 0;\n}\n\n.hdr-button {\n color: var(--hdr-muted);\n cursor: pointer;\n}\n\n.hdr-button:hover,\n.hdr-guide:hover,\n.hdr-close:hover {\n color: var(--hdr-text);\n}\n\n.hdr-guide {\n padding: 2px 8px;\n border: 1px solid var(--hdr-border);\n border-radius: 4px;\n background: transparent;\n color: var(--hdr-text);\n font-size: 13px;\n line-height: 18px;\n cursor: pointer;\n}\n\n.hdr-badge {\n padding: 2px 8px;\n border-radius: 9999px;\n font-size: 13px;\n line-height: 18px;\n font-weight: 600;\n}\n\n.hdr-ok { background: #f5f5f5; color: #171717; }\n.hdr-bad { background: #ef4444; color: #fff; }\n.hdr-unknown { border: 1px solid var(--hdr-border); color: var(--hdr-muted); }\n\n.hdr-mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; }\n.hdr-muted { color: var(--hdr-muted); }\n\n.hdr-dialog {\n padding: 0;\n border: 1px solid var(--hdr-border);\n border-radius: 8px;\n background: var(--hdr-card);\n color: var(--hdr-text);\n white-space: normal;\n}\n\n.hdr-dialog::backdrop { background: rgba(0, 0, 0, 0.5); }\n.hdr-narrow { width: min(28rem, 92vw); }\n.hdr-wide { width: min(42rem, 92vw); }\n\n.hdr-dialog-head {\n display: flex;\n align-items: center;\n justify-content: space-between;\n padding: 12px 20px;\n border-bottom: 1px solid var(--hdr-border);\n font-size: 14px;\n font-weight: 600;\n}\n\n.hdr-close {\n border: 0;\n background: transparent;\n color: var(--hdr-muted);\n font-size: 18px;\n line-height: 1;\n cursor: pointer;\n}\n\n.hdr-dialog-body {\n max-height: 70vh;\n overflow-y: auto;\n padding: 16px 20px;\n font-size: 13px;\n}\n\n.hdr-row {\n display: flex;\n justify-content: space-between;\n gap: 16px;\n padding: 4px 0;\n}\n\n.hdr-trunc {\n overflow: hidden;\n text-overflow: ellipsis;\n white-space: nowrap;\n}\n\n.hdr-rule { border-top: 1px solid var(--hdr-border); margin: 8px 0; }\n.hdr-intro { margin: 0 0 16px; }\n.hdr-group { margin-bottom: 20px; }\n.hdr-group h3 { margin: 0 0 8px; font-size: 13px; text-transform: capitalize; }\n\n.hdr-command {\n margin-bottom: 12px;\n padding: 8px 12px;\n border: 1px solid var(--hdr-border);\n border-radius: 6px;\n background: var(--hdr-pill);\n}\n\n.hdr-command pre {\n margin: 0;\n padding: 8px;\n border-radius: 4px;\n background: var(--hdr-card);\n font-size: 12px;\n white-space: pre-wrap;\n word-break: break-all;\n}\n\n.hdr-grid {\n display: grid;\n grid-template-columns: auto 1fr;\n gap: 4px 12px;\n margin-top: 6px;\n font-size: 12px;\n}\n\n.hdr-amber { color: #f59e0b; }\n.hdr-red { color: #ef4444; }\n\n/* Tailwind 700 on the light theme, like detail.css, for a 4.5:1 contrast\n against the white dialog. */\n.hdr-light .hdr-amber { color: #b45309; }\n.hdr-light .hdr-red { color: #b91c1c; }\n", "wrap": false }, diff --git a/observability/liquidity-panels/recovery-commands.js b/observability/liquidity-panels/recovery-commands.js index 4862cad95..8b40cde27 100644 --- a/observability/liquidity-panels/recovery-commands.js +++ b/observability/liquidity-panels/recovery-commands.js @@ -26,12 +26,16 @@ const tradeCommands = (client, symbol) => [ ]; const USDC_FAIL_ALPACA_TO_BASE = - 'Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal ' + - 'or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum ' + - 'that no CCTP burn left the bot wallet after this transfer started (its Started time) and ' + - 'that the wallet has no pending transaction (pending nonce equals latest nonce); if you ' + - 'are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held ' + - 'until you settle them with reconcile-usdc.'; + 'Alpaca to Base: the bot accepts this only before the burn, from a completed ' + + 'withdrawal or a burn submission with no recorded burn, and it does not check the ' + + 'chain. A burn the bot sent but did not record can still land, so do not fail the ' + + 'transfer until you have done every check and the nonce close on Ethereum in ' + + 'docs/cli-ops.md, "Clearing a pre-burn guard latch", with the bot stopped. Then fail ' + + 'it with the offline stox fail-usdc-transfer before you start the bot again: a restart ' + + 'sends the burn of a completed withdrawal (at a burn submission, this command after a ' + + 'restart works too). If a burn landed or you are not certain, run resume-usdc instead. ' + + 'The funds left Alpaca, so the guard stays held until you settle them with ' + + 'reconcile-usdc.'; const USDC_FAIL_BASE_TO_ALPACA = 'Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, ' + @@ -43,25 +47,43 @@ const USDC_FAIL_BASE_TO_ALPACA = 'sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the ' + 'network accepted but has not mined is not in the logs yet, so also wait until the ' + "transfer's attempt timeout has passed and confirm the wallet has no pending " + - 'transaction (no pending withdraw4, and pending nonce equals latest nonce on more than ' + - 'one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal ' + - 'landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, ' + - 'and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot ' + - 'wallet back to the vault by hand. If nothing landed, the resume keeps redriving: ' + - 'repeat the checks, and once they show that no withdrawal landed and none is pending, ' + - 'take the next step. If none landed, close the nonce before you fail the transfer: ' + - 'stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself ' + - 'at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market ' + - 'fee, wait for its required confirmations, and check the withdrawal logs again. If a ' + - 'withdrawal mined instead, start the bot and run resume-usdc, and do not fail the ' + - 'transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after ' + - 'you restart the bot. If a withdrawal for this transfer mines later anyway, move its ' + - 'USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is ' + - 'still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the ' + - 'bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn ' + - 'left the bot wallet after this transfer started and the wallet has no pending ' + - 'transaction. If any of that is not certain, start the bot and run resume-usdc; else ' + - 'run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.'; + 'transaction (no pending withdraw4, pending nonce equals latest nonce on more than one ' + + 'RPC provider, and the explorer shows no queued transaction from the wallet). A queued ' + + 'transaction waits behind a nonce gap: with the bot stopped, fill each empty nonce ' + + 'below it, in order, with a self-transfer as below, after making sure no send the bot ' + + 'signed and keeps holds that nonce, then repeat the checks. These checks cannot prove ' + + 'that nothing is pending. If a withdrawal landed or you are not certain, run ' + + 'resume-usdc instead: it adopts a withdrawal of the full amount, and any other amount ' + + 'ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to ' + + 'the cash vault by hand. If nothing landed, the resume keeps redriving: repeat the ' + + 'checks, and once every check is clean, take the next step. If none landed, close the ' + + 'nonce before you fail the transfer: stop the bot, check again that the wallet has no ' + + 'pending or queued transaction, make sure no send the bot signed and keeps holds that ' + + 'nonce (if one does, start the bot so it rebroadcasts the send instead), send a 0-value ' + + 'transfer with no calldata from the bot wallet to itself at the latest nonce, with ' + + 'maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required ' + + 'confirmations, and check the withdrawal logs again. The self-transfer closes only ' + + 'that nonce. If a withdrawal mined instead, a transaction is pending or queued, or you ' + + 'cannot tell, start the bot and run resume-usdc, and do not fail the transfer. ' + + 'Otherwise run the offline stox fail-usdc-transfer, or this command after you restart ' + + 'the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the ' + + 'bot wallet back to the cash vault by hand. While a recorded withdrawal is still ' + + 'confirming, run resume-usdc. Once it has confirmed the bot refuses: do every check ' + + 'and the nonce close on Base in docs/cli-ops.md, "Clearing a pre-burn guard latch". If ' + + 'a burn landed, something is pending or queued, or you are not certain, start the bot ' + + 'and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox ' + + 'fail-usdc-transfer and move the wallet USDC back by hand. If ' + + 'a burn for this transfer mines later anyway, resume and reconcile refuse the failed ' + + 'transfer. Keep the bot stopped until that USDC is at Alpaca: the offline commands send ' + + 'from the bot wallet and replace whatever is pending at their nonce, so first run the ' + + 'pending, queued and kept-send checks on Ethereum as docs/cli-ops.md describes, and let ' + + 'any kept send mine. Mint it with stox ' + + "cctp complete-mint --burn-tx --source-chain base (it adopts a relayer's mint; " + + 'do not mint by hand, since the MessageSent log holds a placeholder nonce). Then send ' + + 'the amount received that complete-mint printed, divided by 1,000,000 (it prints USDC ' + + 'base units), to Alpaca with stox alpaca-deposit -a , and do not move that USDC ' + + 'back to the vault. Once Alpaca credits it, convert ' + + 'it with stox alpaca-convert -d to-usd -a , as the bot would have.'; const USDC_BRIDGING_BASE_TO_ALPACA = ' If the burn keeps failing, the running bot refuses fail-usdc-transfer here: see ' + diff --git a/observability/liquidity-panels/recovery-guide.json b/observability/liquidity-panels/recovery-guide.json index 0ad0266e9..ceb04a395 100644 --- a/observability/liquidity-panels/recovery-guide.json +++ b/observability/liquidity-panels/recovery-guide.json @@ -38,7 +38,7 @@ }, { "command": "st0x-liquidity-client --env production debug fail-usdc-transfer --reason \"\"", - "description": "Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn. The burn runs on Ethereum: verify on Ethereum that no CCTP burn left the bot wallet after this transfer started (its Started time) and that the wallet has no pending transaction (pending nonce equals latest nonce); if you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, and pending nonce equals latest nonce on more than one RPC provider). These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a full withdrawal, and a short one ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once they show that no withdrawal landed and none is pending, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. If a withdrawal mined instead, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: stop the bot, then confirm the transfer has no recorded burn, and on Base that no CCTP burn left the bot wallet after this transfer started and the wallet has no pending transaction. If any of that is not certain, start the bot and run resume-usdc; else run the offline stox fail-usdc-transfer and move the wallet USDC back by hand.", + "description": "Force a USDC bridge stuck before its burn into Failed. Alpaca to Base: the bot accepts this only before the burn, from a completed withdrawal or a burn submission with no recorded burn, and it does not check the chain. A burn the bot sent but did not record can still land, so do not fail the transfer until you have done every check and the nonce close on Ethereum in docs/cli-ops.md, \"Clearing a pre-burn guard latch\", with the bot stopped. Then fail it with the offline stox fail-usdc-transfer before you start the bot again: a restart sends the burn of a completed withdrawal (at a burn submission, this command after a restart works too). If a burn landed or you are not certain, run resume-usdc instead. The funds left Alpaca, so the guard stays held until you settle them with reconcile-usdc. Base to Alpaca: the bot accepts this only while the vault withdrawal is unrecorded, and it does not check the chain. First confirm on Base that the bot wallet made no OperatorWithdraw on the inventory after the transfer's from_block, and no WithdrawV2 on the orderbook either (a withdrawal sent before the inventory migration). Only a USDC withdrawal from the cash vault counts: its vaultId equals the cash vault_id in the bot config, and the bot wallet is the OperatorWithdraw operator or the WithdrawV2 sender. A WithdrawV2 with the inventory as sender does not count. A withdraw the network accepted but has not mined is not in the logs yet, so also wait until the transfer's attempt timeout has passed and confirm the wallet has no pending transaction (no pending withdraw4, pending nonce equals latest nonce on more than one RPC provider, and the explorer shows no queued transaction from the wallet). A queued transaction waits behind a nonce gap: with the bot stopped, fill each empty nonce below it, in order, with a self-transfer as below, after making sure no send the bot signed and keeps holds that nonce, then repeat the checks. These checks cannot prove that nothing is pending. If a withdrawal landed or you are not certain, run resume-usdc instead: it adopts a withdrawal of the full amount, and any other amount ends the transfer in WithdrawalFailed, so move that USDC from the bot wallet back to the cash vault by hand. If nothing landed, the resume keeps redriving: repeat the checks, and once every check is clean, take the next step. If none landed, close the nonce before you fail the transfer: stop the bot, check again that the wallet has no pending or queued transaction, make sure no send the bot signed and keeps holds that nonce (if one does, start the bot so it rebroadcasts the send instead), send a 0-value transfer with no calldata from the bot wallet to itself at the latest nonce, with maxFeePerGas and maxPriorityFeePerGas well above the market fee, wait for its required confirmations, and check the withdrawal logs again. The self-transfer closes only that nonce. If a withdrawal mined instead, a transaction is pending or queued, or you cannot tell, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer, or this command after you restart the bot. If a withdrawal for this transfer mines later anyway, move its USDC from the bot wallet back to the cash vault by hand. While a recorded withdrawal is still confirming, run resume-usdc. Once it has confirmed the bot refuses: do every check and the nonce close on Base in docs/cli-ops.md, \"Clearing a pre-burn guard latch\". If a burn landed, something is pending or queued, or you are not certain, start the bot and run resume-usdc, and do not fail the transfer. Otherwise run the offline stox fail-usdc-transfer and move the wallet USDC back by hand. If a burn for this transfer mines later anyway, resume and reconcile refuse the failed transfer. Keep the bot stopped until that USDC is at Alpaca: the offline commands send from the bot wallet and replace whatever is pending at their nonce, so first run the pending, queued and kept-send checks on Ethereum as docs/cli-ops.md describes, and let any kept send mine. Mint it with stox cctp complete-mint --burn-tx --source-chain base (it adopts a relayer's mint; do not mint by hand, since the MessageSent log holds a placeholder nonce). Then send the amount received that complete-mint printed, divided by 1,000,000 (it prints USDC base units), to Alpaca with stox alpaca-deposit -a , and do not move that USDC back to the vault. Once Alpaca credits it, convert it with stox alpaca-convert -d to-usd -a , as the bot would have.", "whenToUse": "A USDC bridge is stuck before its CCTP burn and must be terminalized. Run the check for its direction in the description first; if it is not certain, run resume-usdc.", "appliesTo": "USDC bridge (pre-burn stuck)", "mode": "requires-bot"