fix: complete the recovery steps and parse-check the shown commands [RAI-3132]

crates/cli/src/cli/mod.rs

@@ -528,20 +528,32 @@ pub enum Commands {
         chain: Option<TokenizationNetwork>,
     },
 
-    /// Mark a pre-burn USDC rebalance as failed, clearing the in-progress guard.
+    /// Mark a pre-burn USDC rebalance as failed.
     ///
-    /// Valid only from `BridgingSubmitting` or `WithdrawalComplete`. Refused for
-    /// any state where a CCTP burn transaction has been submitted. Drives the
-    /// aggregate to `BridgingFailed { burn_tx_hash: None }`, which is
-    /// non-guard-holding. Guard clears on the next bot restart.
+    /// Valid from `WithdrawalComplete`, from `BridgingSubmitting` with no
+    /// recorded burn, and from a BaseToAlpaca `WithdrawalSubmitting` whose
+    /// withdrawal never reached the chain. Refused for any state where a CCTP
+    /// burn transaction has been submitted. The first two end in
+    /// `BridgingFailed { burn_tx_hash: None }`, the last in `WithdrawalFailed`.
+    /// A BaseToAlpaca guard clears on the next bot restart; at
+    /// `WithdrawalComplete` or `BridgingSubmitting` its withdrawn USDC sits in
+    /// the market-maker wallet, so move it back to the vault by hand. An
+    /// AlpacaToBase guard stays held until `transfer reconcile --kind usdc`
+    /// settles the withdrawn funds.
     ///
     /// Safety procedure:
     /// - Stop the bot before running to avoid the concurrent-burn race where the
     ///   bot advances the transfer to `Bridging` between the preflight and the
     ///   send.
     /// - For a `BridgingSubmitting` transfer, verify on-chain that no recent
     ///   CCTP burn left the market-maker wallet before running (a crash at this
-    ///   state may have broadcast a burn whose event never persisted).
+    ///   state may have broadcast a burn whose event never persisted), and close
+    ///   the wallet's nonce first as docs/cli-ops.md describes.
+    /// - For a BaseToAlpaca `WithdrawalSubmitting` transfer, confirm on Base
+    ///   that the bot wallet made no `OperatorWithdraw` after the transfer's
+    ///   `from_block` and that none is pending, as docs/cli-ops.md
+    ///   ("Withdrawal that never reached the chain") describes. If one landed,
+    ///   use `transfer resume --kind usdc` instead.
     /// - Post-burn terminal failures (e.g. `DepositFailed`) use
     ///   `transfer reconcile`. In-flight post-burn states (`Bridging`,
     ///   `AwaitingAttestation`, `Attested`, `Bridged`, `DepositInitiated`)
@@ -1039,8 +1051,11 @@ pub enum CctpCommand {
     /// Live RPC only: touches no database state or aggregate. Ensure the bot is
     /// not concurrently driving the same on-chain mint. After completing the
     /// mint, bring a stuck `UsdcRebalance` aggregate back in sync: `transfer
-    /// resume` while it is still non-terminal (it adopts the existing mint), or
-    /// `transfer reconcile` if it already reached a post-burn terminal failure.
+    /// resume` while it is still non-terminal (it adopts the existing mint),
+    /// and `transfer reconcile` only after a terminal failure whose funds leg
+    /// is done and verified at the destination. For a BaseToAlpaca bridge that
+    /// failed after its burn, run this mint only as a step of docs/cli-ops.md,
+    /// "Settling a post-burn Base to Alpaca failure by hand".
     CompleteMint {
         /// Transaction hash of the burn transaction on the source chain
         #[arg(long = "burn-tx")]
@@ -4180,6 +4195,62 @@ mod tests {
         }
     }
 
+    /// Every offline stox command the SPA shows, written by
+    /// `dashboard/src/lib/transfer-board.test.ts`.
+    const SHOWN_STOX_COMMANDS: &str = include_str!("../../testdata/shown-stox-commands.txt");
+
+    /// Each way to fill a shown command's `<a|b>` placeholders, split into
+    /// words.
+    fn expand_shown_stox(line: &str) -> Vec<Vec<String>> {
+        line.split_whitespace()
+            .fold(vec![Vec::new()], |argvs, word| {
+                let choices: Vec<String> = match word
+                    .strip_prefix('<')
+                    .and_then(|rest| rest.strip_suffix('>'))
+                {
+                    Some(inner) if inner.contains('|') => {
+                        inner.split('|').map(str::to_string).collect()
+                    }
+                    Some(inner) => panic!("no test value for the placeholder <{inner}> in {line}"),
+                    None => vec![word.to_string()],
+                };
+                argvs
+                    .iter()
+                    .flat_map(|argv| {
+                        choices.iter().map(move |choice| {
+                            let mut next = argv.clone();
+                            next.push(choice.clone());
+                            next
+                        })
+                    })
+                    .collect()
+            })
+    }
+
+    /// The SPA shows offline stox commands for the steps that run with the
+    /// bot stopped. Each must parse and classify, so a USDC reconcile's id is
+    /// a UUID and its reason one the command accepts.
+    #[test]
+    fn every_shown_stox_command_parses() {
+        let lines: Vec<&str> = SHOWN_STOX_COMMANDS
+            .lines()
+            .filter(|line| !line.starts_with('#') && !line.trim().is_empty())
+            .collect();
+        assert!(!lines.is_empty(), "the shown stox command file is empty");
+
+        for line in lines {
+            for argv in expand_shown_stox(line) {
+                assert_eq!(argv.first().map(String::as_str), Some("stox"), "{line}");
+                let argv = std::iter::once("st0x-cli".to_string()).chain(argv.into_iter().skip(1));
+                let cli = Cli::try_parse_from(argv)
+                    .unwrap_or_else(|error| panic!("{line} does not parse: {error}"));
+                if let Err(error) = classify_command(cli.command) {
+                    panic!("{line} does not classify: {error:#}");
+                }
+            }
+        }
+    }
+
     #[test]
     fn cctp_complete_mint_parses_and_classifies_as_provider() {
         let cli = Cli::try_parse_from([

crates/cli/testdata/shown-stox-commands.txt

@@ -0,0 +1,3 @@
+# Every offline stox command the SPA shows; crates/cli parses each with st0x-cli.
+# Written by dashboard/src/lib/transfer-board.test.ts; do not edit by hand.
+stox transfer reconcile --kind usdc --id 00000000-0000-4000-8000-000000000001 --reason <funds-moved-manually|deposit-credited-offline>

crates/liquidity-client/src/cli.rs

@@ -115,7 +115,10 @@ pub(crate) enum Debug {
         /// USDC rebalance id.
         id: String,
     },
-    /// Reconcile a stuck USDC rebalance to OperatorReconciled.
+    /// Reconcile a stuck USDC rebalance to OperatorReconciled. The bot
+    /// re-runs a post-burn BaseToAlpaca BridgingFailed at every start, so do
+    /// not mint, send or reconcile that one with the bot running: follow
+    /// docs/cli-ops.md, "Settling a post-burn Base to Alpaca failure by hand".
     ReconcileUsdc {
         /// USDC rebalance id.
         id: String,
@@ -168,10 +171,20 @@ pub(crate) enum Debug {
         #[arg(long, value_parser = nonblank_reason)]
         reason: String,
     },
-    /// Mark an AlpacaToBase USDC rebalance failed before its burn; the guard
-    /// stays held until reconcile-usdc settles the withdrawn funds. Refuses
-    /// states after the burn and BaseToAlpaca transfers; verify onchain that
-    /// no burn landed first.
+    /// Mark a USDC rebalance failed before its burn. Accepts an AlpacaToBase
+    /// rebalance at WithdrawalComplete or BridgingSubmitting with no recorded
+    /// burn (the guard stays held until reconcile-usdc settles the withdrawn
+    /// funds; at WithdrawalComplete a restart sends the burn, so stop the bot
+    /// and use the offline stox fail-usdc-transfer before you start it again;
+    /// at BridgingSubmitting, close the wallet's nonce first, and this command
+    /// after a restart works too), and a BaseToAlpaca rebalance whose vault withdrawal was never
+    /// recorded (WithdrawalSubmitting; the guard clears). A BaseToAlpaca
+    /// rebalance at WithdrawalComplete or BridgingSubmitting with no recorded
+    /// burn needs the offline stox fail-usdc-transfer, and its withdrawn USDC
+    /// goes back to the vault by hand. Both refuse every other state. It does
+    /// not check the chain: first verify, for BaseToAlpaca, that no withdrawal
+    /// landed or is pending, and for AlpacaToBase, that no burn landed or is
+    /// pending (see docs/cli-ops.md).
     FailUsdcTransfer {
         /// USDC rebalance id.
         id: String,
@@ -280,8 +293,10 @@ pub(crate) enum Cctp {
     /// Complete the destination mint of a CCTP burn whose mint never landed.
     /// A burn Circle has not attested yet fails at once as retryable; rerun it
     /// later. Rerunning is always safe: a mint that already landed is adopted.
-    /// Afterwards bring the stuck rebalance back in sync with resume-usdc or
-    /// reconcile-usdc.
+    /// Afterwards run resume-usdc while the rebalance is not terminal.
+    /// reconcile-usdc only once a terminal failure's funds leg is done and
+    /// verified. For a post-burn BaseToAlpaca bridge, follow docs/cli-ops.md,
+    /// "Settling a post-burn Base to Alpaca failure by hand", instead.
     CompleteMint {
         /// Transaction hash of the burn on the source chain.
         #[arg(long)]
@@ -1300,6 +1315,196 @@ mod tests {
         }
     }
 
+    /// Every client command the SPA and the board show, written by
+    /// `dashboard/src/lib/transfer-board.test.ts`.
+    const SHOWN_COMMANDS: &str = include_str!("../testdata/shown-commands.txt");
+
+    /// A value for each `<name>` placeholder in a shown command. A new
+    /// placeholder fails the test until it gets a value here.
+    fn placeholder_value(name: &str) -> &'static str {
+        match name {
+            "id" => "abc",
+            "reason" => "operator note",
+            "symbol" => "AAPL",
+            "order-id" => "ord-1",
+            "hash" => "0xabc",
+            "N" => "-5",
+            "USDC_PER_SHARE" => "1.5",
+            other => panic!("no test value for the placeholder <{other}>"),
+        }
+    }
+
+    /// Splits a shown command into words like a shell: double quotes group
+    /// words and are dropped.
+    fn shown_words(line: &str) -> Vec<String> {
+        let mut words = Vec::new();
+        let mut current = String::new();
+        let mut quoted = false;
+        let mut in_word = false;
+        for character in line.chars() {
+            match character {
+                '"' => {
+                    quoted = !quoted;
+                    in_word = true;
+                }
+                ' ' if !quoted => {
+                    if in_word {
+                        words.push(std::mem::take(&mut current));
+                        in_word = false;
+                    }
+                }
+                other => {
+                    current.push(other);
+                    in_word = true;
+                }
+            }
+        }
+        assert!(!quoted, "unmatched quote in {line}");
+        if in_word {
+            words.push(current);
+        }
+        words
+    }
+
+    /// Every way to fill one word's `<...>` placeholders: `<a|b>` gives each
+    /// alternative, `<name>` its test value.
+    fn fill_word(word: &str) -> Vec<String> {
+        let Some(open) = word.find('<') else {
+            return vec![word.to_owned()];
+        };
+        let close = open + word[open..].find('>').expect("unclosed <");
+        let inner = &word[open + 1..close];
+        let choices: Vec<&str> = if inner.contains('|') {
+            inner.split('|').collect()
+        } else {
+            vec![placeholder_value(inner)]
+        };
+        fill_word(&word[close + 1..])
+            .into_iter()
+            .flat_map(|rest| {
+                choices
+                    .iter()
+                    .map(move |choice| format!("{}{choice}{rest}", &word[..open]))
+            })
+            .collect()
+    }
+
+    /// The words of a `[...]` or `(...)` group that starts at the first word,
+    /// without its brackets, and the words after it.
+    fn split_group(words: &[String], open: char, close: char) -> (Vec<String>, &[String]) {
+        let end = words
+            .iter()
+            .position(|word| word.ends_with(close))
+            .unwrap_or_else(|| panic!("unclosed {open} in {words:?}"));
+        let mut group = words[..=end].to_vec();
+        group[0] = group[0].trim_start_matches(open).to_owned();
+        group[end] = group[end].trim_end_matches(close).to_owned();
+        group.retain(|word| !word.is_empty());
+        (group, &words[end + 1..])
+    }
+
+    fn join_each(heads: &[Vec<String>], tails: &[Vec<String>]) -> Vec<Vec<String>> {
+        heads
+            .iter()
+            .flat_map(|head| {
+                tails
+                    .iter()
+                    .map(move |tail| [head.clone(), tail.clone()].concat())
+            })
+            .collect()
+    }
+
+    /// Every argument list a shown command stands for: each optional `[...]`
+    /// with and without, each `(a | b)` alternative, and each placeholder.
+    fn expand_shown(words: &[String]) -> Vec<Vec<String>> {
+        let Some(first) = words.first() else {
+            return vec![Vec::new()];
+        };
+
+        if first.starts_with('[') {
+            let (group, rest) = split_group(words, '[', ']');
+            let tails = expand_shown(rest);
+            let mut both = tails.clone();
+            both.extend(join_each(&expand_shown(&group), &tails));
+            return both;
+        }
+
+        if first.starts_with('(') {
+            let (group, rest) = split_group(words, '(', ')');
+            let heads: Vec<Vec<String>> = group
+                .split(|word| word == "|")
+                .flat_map(expand_shown)
+                .collect();
+            return join_each(&heads, &expand_shown(rest));
+        }
+
+        let heads: Vec<Vec<String>> = fill_word(first)
+            .into_iter()
+            .map(|word| vec![word])
+            .collect();
+        join_each(&heads, &expand_shown(&words[1..]))
+    }
+
+    #[test]
+    fn shown_commands_expand_every_choice() {
+        let words = shown_words(
+            r#"debug view rebuild <position|offchain-order> (--id <id> | --all) [--x "<reason>"]"#,
+        );
+        let expanded: Vec<String> = expand_shown(&words)
+            .into_iter()
+            .map(|argv| argv.join(" "))
+            .collect();
+        assert_eq!(
+            expanded,
+            [
+                "debug view rebuild position --id abc",
+                "debug view rebuild position --id abc --x operator note",
+                "debug view rebuild position --all",
+                "debug view rebuild position --all --x operator note",
+                "debug view rebuild offchain-order --id abc",
+                "debug view rebuild offchain-order --id abc --x operator note",
+                "debug view rebuild offchain-order --all",
+                "debug view rebuild offchain-order --all --x operator note",
+            ]
+        );
+    }
+
+    /// The SPA and the board show recovery commands as text. A command whose
+    /// verb, flag, or value this client no longer accepts fails here, in both
+    /// environments.
+    #[test]
+    fn every_shown_recovery_command_parses() {
+        let lines: Vec<&str> = SHOWN_COMMANDS
+            .lines()
+            .filter(|line| !line.starts_with('#') && !line.trim().is_empty())
+            .collect();
+        assert!(lines.len() > 20, "only {} shown commands", lines.len());
+
+        for line in lines {
+            // Clap never checks the program name, so a renamed binary would
+            // still parse here and fail for the operator.
+            assert_eq!(
+                shown_words(line).first().map(String::as_str),
+                Some(env!("CARGO_BIN_NAME")),
+                "{line}"
+            );
+            for argv in expand_shown(&shown_words(line)) {
+                for env in ["production", "staging"] {
+                    let argv: Vec<String> = argv
+                        .iter()
+                        .map(|word| match word.as_str() {
+                            "production" => env.to_owned(),
+                            _ => word.clone(),
+                        })
+                        .collect();
+                    if let Err(error) = Cli::try_parse_from(&argv) {
+                        panic!("{line}\nas {argv:?}\ndoes not parse: {error}");
+                    }
+                }
+            }
+        }
+    }
+
     #[test]
     fn vault_id_boundaries() {
         let hex64 = "F".repeat(64);

crates/liquidity-client/testdata/shown-commands.txt

@@ -0,0 +1,28 @@
+# Every st0x-liquidity-client command the SPA and the board show.
+# Written by dashboard/src/lib/transfer-board.test.ts; do not edit by hand.
+st0x-liquidity-client --env production debug cctp complete-mint --burn-tx <hash> --source-chain <ethereum|base>
+st0x-liquidity-client --env production debug fail-equity-transfer <mint|redemption> <id> --reason "<reason>"
+st0x-liquidity-client --env production debug fail-equity-transfer mint X1 --reason "<reason>"
+st0x-liquidity-client --env production debug fail-equity-transfer redemption X1 --reason "<reason>"
+st0x-liquidity-client --env production debug fail-usdc-transfer 00000000-0000-4000-8000-000000000001 --reason "<reason>"
+st0x-liquidity-client --env production debug fail-usdc-transfer <id> --reason "<reason>"
+st0x-liquidity-client --env production debug position release-hedge <symbol> --order-id <order-id> --reason "<reason>"
+st0x-liquidity-client --env production debug position release-hedge MSTR --order-id <order-id> --reason "<reason>"
+st0x-liquidity-client --env production debug position set <symbol> --target-net <N> [--price-usdc <USDC_PER_SHARE>] --reason "<reason>"
+st0x-liquidity-client --env production debug position set MSTR --target-net <N> [--price-usdc <USDC_PER_SHARE>] --reason "<reason>"
+st0x-liquidity-client --env production debug process-tx <hash> [--chain <base|ethereum|hyperevm|robinhood>]
+st0x-liquidity-client --env production debug recheck <mint|redemption|usdc> <id>
+st0x-liquidity-client --env production debug recheck mint X1
+st0x-liquidity-client --env production debug recheck redemption X1
+st0x-liquidity-client --env production debug reconcile-equity <mint|redemption> <id> --reason "<reason>"
+st0x-liquidity-client --env production debug reconcile-equity mint X1 --reason "<reason>"
+st0x-liquidity-client --env production debug reconcile-equity redemption X1 --reason "<reason>"
+st0x-liquidity-client --env production debug reconcile-usdc 00000000-0000-4000-8000-000000000001 --reason <funds-moved-manually|deposit-credited-offline>
+st0x-liquidity-client --env production debug reconcile-usdc <id> --reason <funds-moved-manually|deposit-credited-offline>
+st0x-liquidity-client --env production debug resume
+st0x-liquidity-client --env production debug resume-usdc <alpaca-to-base|base-to-alpaca> 00000000-0000-4000-8000-000000000001
+st0x-liquidity-client --env production debug resume-usdc <alpaca-to-base|base-to-alpaca> <id>
+st0x-liquidity-client --env production debug resume-usdc alpaca-to-base 00000000-0000-4000-8000-000000000001
+st0x-liquidity-client --env production debug resume-usdc base-to-alpaca 00000000-0000-4000-8000-000000000001
+st0x-liquidity-client --env production debug view rebuild <position|offchain-order|vault-registry> (--id <id> | --all)
+st0x-liquidity-client --env production debug view rebuild position --id MSTR