From ddc1e4205d172dd6ad0466a2b111eda8ca913e63 Mon Sep 17 00:00:00 2001 From: JuaniRios Date: Thu, 8 Oct 2026 09:39:04 -0300 Subject: [PATCH] [RAI-2998] Add the liq_* parity comparison tool scripts/liq-parity/compare.py compares the bot's liq_* series with the exporter sidecar's, from snapshot pairs taken on a live environment. Only findings present in every pair are reported, matched by kind and series so a mismatch whose values move still counts. Names not ported yet are listed, not reported, and --drop-list / --known-diffs apply the documented exceptions. It exits 2 on a snapshot that is not Prometheus text or has no liq_* series. CI runs its tests next to the board check, and docs/observability.md describes the operator procedure over IAP SSH. --- .github/workflows/ci.yaml | 5 +- docs/ci.md | 15 +- docs/observability.md | 64 +++ flake.nix | 3 +- scripts/liq-parity/compare.py | 349 ++++++++++++++++ scripts/liq-parity/test_compare.py | 383 ++++++++++++++++++ scripts/liq-parity/testdata/bot.prom | 17 + scripts/liq-parity/testdata/degraded.prom | 1 + scripts/liq-parity/testdata/empty.prom | 2 + scripts/liq-parity/testdata/exporter.prom | 10 + .../liq-parity/testdata/not-prometheus.txt | 1 + scripts/liq-parity/testdata/repeated.prom | 2 + 12 files changed, 843 insertions(+), 9 deletions(-) create mode 100644 scripts/liq-parity/compare.py create mode 100644 scripts/liq-parity/test_compare.py create mode 100644 scripts/liq-parity/testdata/bot.prom create mode 100644 scripts/liq-parity/testdata/degraded.prom create mode 100644 scripts/liq-parity/testdata/empty.prom create mode 100644 scripts/liq-parity/testdata/exporter.prom create mode 100644 scripts/liq-parity/testdata/not-prometheus.txt create mode 100644 scripts/liq-parity/testdata/repeated.prom diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a5bbf8f6b0..a9c996a97f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -129,12 +129,13 @@ jobs: save-if: ${{ env.UNTRUSTED_PR != 'true' }} # The board JSON must match its generator, with every liq_ selector - # pinned to the exporter job. - - name: Board generator check + # pinned to the exporter job; compare.py is the staging parity tool. + - name: Board generator and liq-parity checks run: | nix develop .#ci-backend -c bash -c " set -euxo pipefail python3 observability/gen-t0-liquidity.py --check + python3 scripts/liq-parity/test_compare.py " # Build the test binaries once into a nextest archive shared by all four diff --git a/docs/ci.md b/docs/ci.md index 06900ffe36..afe24a5dff 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -69,10 +69,13 @@ Use a pull request event for merge-blocking validation. Reserve `workflow_dispatch` for manual runs whose result does not need to become a pull request status check. -## Board check +## Board and parity checks -The `backend build` job also runs -`python3 observability/gen-t0-liquidity.py --check` in the `ci-backend` shell, -and `nix run .#ci` runs it too. It fails when the committed liquidity board JSON -differs from the generator output, or when a `liq_` selector is not pinned to -the exporter job. +The `backend build` job also runs two Python checks in the `ci-backend` shell, +and `nix run .#ci` runs them too: + +- `python3 observability/gen-t0-liquidity.py --check` fails when the committed + liquidity board JSON differs from the generator output, or when a `liq_` + selector is not pinned to the exporter job. +- `python3 scripts/liq-parity/test_compare.py` tests the staging parity tool + against a checked-in snapshot pair. diff --git a/docs/observability.md b/docs/observability.md index 1c14f3f295..b04d45c7e6 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -114,3 +114,67 @@ reads `job="t0-liquidity-exporter"`. `observability/gen-t0-liquidity.py` adds the matcher and fails if any `liq_` selector is left unpinned. `python3 observability/gen-t0-liquidity.py --check` (run in CI) fails when the committed board JSON differs from the generator output. + +### Checking parity on a live environment + +Run this after each release that ports names, while the exporter sidecar still +runs next to the bot. Run it from a local checkout of this repository: the VMs +have no checkout, and both endpoints listen only on the VM itself, so each +snapshot is read over the IAP SSH access described in `docs/cli-ops.md`. + +1. Check out the commit the VM runs. `compare.py`'s list of ported names must + match the deployed release: a newer checkout reports names the release does + not publish yet as missing. The commit is the `git_commit` label of + `liq_bot_info` in the bot's `/metrics`. +2. Set `VM` and `PROJECT` to the staging or production values from + `docs/cli-ops.md`, then run the whole block from the repository root. It + takes three snapshot pairs about two minutes apart into a new directory, and + compares them only after all three succeed. Each pair is one SSH command that + reads both bodies back to back: + + ```bash + ( + set -euo pipefail + dir=$(mktemp -d) + echo "snapshots in $dir" + for i in 1 2 3; do + gcloud compute ssh "$VM" --project "$PROJECT" --zone europe-west3-b \ + --tunnel-through-iap --command \ + 'curl -fsS --max-time 10 localhost:8001/metrics && + echo "# ---- exporter ----" && + curl -fsS --max-time 10 localhost:9101/metrics' > "$dir/pair-$i.prom" + sed '/^# ---- exporter ----$/,$d' "$dir/pair-$i.prom" > "$dir/bot-$i.prom" + sed '1,/^# ---- exporter ----$/d' "$dir/pair-$i.prom" > "$dir/exporter-$i.prom" + if [ "$i" -lt 3 ]; then sleep 120; fi + done + status=0 + python3 -I scripts/liq-parity/compare.py --drop-list --known-diffs \ + "$dir/bot-1.prom" "$dir/exporter-1.prom" \ + "$dir/bot-2.prom" "$dir/exporter-2.prom" \ + "$dir/bot-3.prom" "$dir/exporter-3.prom" || status=$? + echo "compare.py exit status: $status" + ) + ``` + + `curl -f` fails on an HTTP error and `--max-time` fails a request that + stalls. The `&&` chain makes the SSH command fail with it, so a dead or hung + endpoint stops the block before the comparison. A fresh directory per run + means a failed run can never compare the files of an earlier one. +3. If the block ends without a `compare.py exit status:` line, the capture + failed (expired `gcloud` credentials, wrong VM or project, missing IAP + access, or a dead endpoint); fix that and run it again. Otherwise read that + status. `0` means no series disagreed in all three pairs. `1` lists the + series that disagree in every pair: each one is a bug in the port or a + difference missing from `KNOWN_DIFFS`. `compare.py` also compares the + `# TYPE` of each name both sides publish. The exporter writes no `# TYPE` + lines, so the bot's `gauge` against the exporter's untyped is expected for + every `liq_*` name, and `--known-diffs` accepts it. Any other `type differs:` + line (for example a bot `counter`) is a bug in the port. `2` means a snapshot + is unusable, and `compare.py` prints the file and the reason. For + `no ported liq_* series`, the wrong endpoint was read or a target was + degraded; check the endpoint and run it again. For any other reason + (`not UTF-8 text`, `not a Prometheus text body`, `repeated series`, + `repeated label name`, or `repeated TYPE`) in a `bot-N.prom` file, report it + as a bug in the port, because a rerun gives the same result. + +The block only reads, so it is safe to rerun. diff --git a/flake.nix b/flake.nix index 568714d925..3eec2c8eaa 100644 --- a/flake.nix +++ b/flake.nix @@ -426,6 +426,7 @@ cargo clippy --workspace --all-targets --all-features cargo fmt -- --check python3 observability/gen-t0-liquidity.py --check + python3 scripts/liq-parity/test_compare.py ' nix develop .#ci-audit -c bash -c './scripts/test-rust-audit.sh && ./scripts/audit-rust.sh' @@ -850,7 +851,7 @@ # deploy scripts. abiEnv is needed because build.rs / sol! macros # consume ST0X_*_ABI at compile time. foundry is required because # tests spawn anvil for local EVM simulation. python3 runs the - # board generator check. + # board generator check and the liq-parity script tests. ci-backend = pkgs.mkShell ( { buildInputs = diff --git a/scripts/liq-parity/compare.py b/scripts/liq-parity/compare.py new file mode 100644 index 0000000000..20a70eff6e --- /dev/null +++ b/scripts/liq-parity/compare.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +"""Compares the bot's liq_* series with the exporter sidecar's. + + python3 scripts/liq-parity/compare.py [--drop-list] [--known-diffs] \\ + BOT.prom EXPORTER.prom [BOT.prom EXPORTER.prom ...] + +Each pair is one snapshot of both /metrics bodies taken within the same +second. With several pairs (for example three, two minutes apart) only +findings present in every pair are reported: the bot publishes on events and +the exporter polls, so a one-off difference is timing, not a defect. + +Findings, per series `(name, sorted labels)`: + + missing from bot the exporter has it and its name is ported + extra in bot the bot has it and neither side should + value differs both have it and the values differ by more than 1e-9 + relative + +and, per name both sides publish: + + type differs the `# TYPE` lines disagree; a name without one is + untyped + +Exporter names not ported yet are listed once and are not findings. Lines +that are not liq_* (the bot recorder's own metrics) are ignored. + +--drop-list removes the names nobody reads, which the bot never ports. +--known-diffs applies the documented differences below, including +KNOWN_TYPE_DIFF: the bot types every liq_* name as a gauge and the exporter +writes no `# TYPE` lines, so bot gauge against exporter untyped is expected. +Any other type difference (for example a bot counter) is still a finding. + +A finding is identified by its series alone, not by its kind or the values +it prints, so a series that disagrees in every pair is reported even when +its values move or its kind changes (for example missing in one pair and a +different value in the next). The report shows the last pair's message. + +Exit status: 0 with no findings, 1 with findings, 2 when a snapshot is not +UTF-8 Prometheus text or holds none of the ported liq_* series (a wrong +file, an empty body, an HTML or JSON error page, or a degraded target that +only reports liq_up). + + +Items that port more names extend PORTED, BOT_ONLY and KNOWN_DIFFS. +""" + +import argparse +import math +import re +import sys + +# Exporter names the bot publishes with the same labels and meaning. +PORTED = { + "liq_bot_info", + "liq_bot_start_timestamp_seconds", + "liq_settings_info", + "liq_settings_equity_target", + "liq_settings_equity_deviation", + "liq_settings_usdc_target", + "liq_settings_usdc_deviation", + "liq_settings_cash_reserved", + "liq_settings_execution_threshold_usd", + "liq_settings_order_polling_seconds", + "liq_settings_inventory_poll_seconds", + "liq_settings_deployment_block", + "liq_asset_counter_trading", + "liq_asset_extended_hours", + "liq_asset_rebalancing", +} + +# Names only the bot publishes. +BOT_ONLY = { + "liq_collector_last_success_ts_seconds", +} + +# Exporter names that are not ported, because nothing reads them or the bot +# has a native replacement. +DROP_LIST = { + "liq_up", + "liq_exporter_last_success_ts_seconds", + "liq_exporter_collect_seconds", + "liq_asset_flags", + "liq_asset_operational_limit", + "liq_position_net", + "liq_equity_onchain_inflight", + "liq_equity_offchain_inflight", + "liq_usdc_withdrawable", + "liq_hedge_cycles_24h", + "liq_hedge_fill_count_24h", + "liq_failure_event_last_ts_seconds", + "liq_job_queue_oldest_pending_ts_seconds", + "liq_rebalance_operations", + "liq_rebalance_skipped", + "liq_poll_duration_ms_samples", + "liq_dependency_latency_ms_samples", + "liq_rebalance_stage_ms_samples", + "liq_raindex_order_vault_balance", + "liq_raindex_order_io_ratio", + "liq_raindex_order_created_ts_seconds", +} + +# Documented differences. ("absolute", tolerance): values may differ by up to +# the tolerance. "values": keys are compared, values are not. "ignore": the +# name is not compared at all. The item that ports a name adds its entry, with +# the evidence for it. +KNOWN_DIFFS = { + # The exporter derives the start from integer uptime at poll time. + "liq_bot_start_timestamp_seconds": ("absolute", 2.0), + # The endpoint the exporter reads truncates at 50,000 entries and counts + # an exact window; the bot counts every entry in one-minute buckets. + "liq_reliability_log_count_24h": "values", + "liq_log_target_count_24h": "values", +} + +# (bot type, exporter type) that --known-diffs accepts for every name. The +# bot publishes every liq_* name as a gauge; the exporter is untyped. +KNOWN_TYPE_DIFF = ("gauge", "untyped") + +RELATIVE_TOLERANCE = 1e-9 + +# The Prometheus text format, one sample line at a time. A line that does not +# match the whole grammar makes the snapshot unusable rather than being read +# loosely. +_LABEL = r'[A-Za-z_][A-Za-z0-9_]*\s*=\s*"(?:[^"\\\n]|\\[\\"n])*"' +_LABEL_PAIR = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)\s*=\s*"((?:[^"\\\n]|\\[\\"n])*)"') +# A decimal float, Inf, Infinity, or NaN (any case, as Go reads them). Python's float() also takes forms such as +# `1_0` that Prometheus rejects, so the token is checked before conversion. +_VALUE = (r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?" + r"|[+-]?(?i:inf(?:inity)?)|(?i:nan)") +_SAMPLE_LINE = re.compile( + r"(?P[A-Za-z_:][A-Za-z0-9_:]*)" + r"(?:\{\s*(?:(?P" + _LABEL + r"(?:\s*,\s*" + _LABEL + r")*)\s*,?\s*)?\})?" + r"\s+(?P" + _VALUE + r")" + r"(?:\s+(?P-?[0-9]+))?" +) +_UNESCAPE = {"\\\\": "\\", '\\"': '"', "\\n": "\n"} +# Like Prometheus, blanks after the `#` are optional: `#TYPE` is a TYPE line. +_TYPE_LINE = re.compile(r"#[ \t]*TYPE(?:[ \t]+(?P.*))?") +_TYPE_BODY = re.compile( + r"(?P[A-Za-z_:][A-Za-z0-9_:]*)[ \t]+" + r"(?Pcounter|gauge|histogram|summary|untyped)[ \t]*") + + +def parse_exposition(text): + """`(name, sorted label tuple) -> value` for every sample line.""" + if text and not text.endswith("\n"): + raise ValueError("no line feed after the last line") + series = {} + # Only LF ends a line. splitlines() would also split on characters such as + # U+2028, which HELP text and label values may hold. + for raw in text.split("\n"): + line = raw.strip(" \t") + if not line or line.startswith("#"): + continue + match = _SAMPLE_LINE.fullmatch(line) + if match is None: + raise ValueError(f"not a sample line: {raw!r}") + labels = [(key, re.sub(r"\\[\\\"n]", lambda m: _UNESCAPE[m.group()], value)) + for key, value in _LABEL_PAIR.findall(match.group("labels") or "")] + if len({label for label, _ in labels}) != len(labels): + raise ValueError(f"repeated label name: {raw!r}") + key = (match.group("name"), tuple(sorted(labels))) + if key in series: + raise ValueError(f"repeated series: {raw!r}") + series[key] = float(match.group("value")) + return series + + +def parse_types(text): + """`name -> type` for every `# TYPE` line. Other comments are ignored. + Like Prometheus, a TYPE line must come before the first sample of its + name.""" + types = {} + sampled = set() + for raw in text.split("\n"): + line = raw.strip(" \t") + match = _TYPE_LINE.fullmatch(line) + if match is None: + sample = _SAMPLE_LINE.fullmatch(line) + if sample is not None: + sampled.add(sample.group("name")) + continue + body = _TYPE_BODY.fullmatch(match.group("rest") or "") + if body is None: + raise ValueError(f"not a TYPE line: {raw!r}") + if body.group("name") in types: + raise ValueError(f"repeated TYPE: {raw!r}") + if body.group("name") in sampled: + raise ValueError(f"TYPE after a sample: {raw!r}") + types[body.group("name")] = body.group("type") + return types + + +def escape_label_value(value): + """Prometheus text escaping, so a finding stays on one line.""" + return value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") + + +def format_series(key): + name, labels = key + if not labels: + return name + rendered = ",".join(f'{label}="{escape_label_value(value)}"' for label, value in labels) + return f"{name}{{{rendered}}}" + + +def values_differ(bot, exporter, rule): + if rule == "values": + return False + if math.isnan(bot) or math.isnan(exporter): + return not (math.isnan(bot) and math.isnan(exporter)) + if isinstance(rule, tuple): + return abs(bot - exporter) > rule[1] + return not math.isclose(bot, exporter, rel_tol=RELATIVE_TOLERANCE) + + +def type_findings(bot_text, exporter_text, dropped, known_diffs): + """`{(name, "# TYPE"): message}` for each liq_* name both sides publish + whose types differ. A KNOWN_DIFFS rule only skips values, so a name it + ignores still has its type compared. A string never equals a label tuple, + so the key cannot collide with a series.""" + def names(text): + return {name for name, _ in parse_exposition(text) + if name.startswith("liq_") and name not in dropped} + + bot_types = parse_types(bot_text) + exporter_types = parse_types(exporter_text) + findings = {} + for name in names(bot_text) & names(exporter_text): + types = (bot_types.get(name, "untyped"), exporter_types.get(name, "untyped")) + if types[0] != types[1] and not (known_diffs and types == KNOWN_TYPE_DIFF): + findings[(name, "# TYPE")] = ( + f"type differs: {name} bot={types[0]} exporter={types[1]}") + return findings + + +def compare_pair(bot_text, exporter_text, drop_list, known_diffs): + """Returns ({series: message}, not_ported_names) for one pair.""" + known = KNOWN_DIFFS if known_diffs else {} + dropped = DROP_LIST if drop_list else set() + + def compared(key): + name = key[0] + return (name.startswith("liq_") and name not in dropped + and known.get(name) != "ignore") + + bot = {key: value for key, value in parse_exposition(bot_text).items() + if compared(key)} + exporter = {key: value for key, value in parse_exposition(exporter_text).items() + if compared(key)} + + published = {name for name, _ in bot} + not_ported = {name for name, _ in exporter + if name not in PORTED and name not in published} + exporter = {key: value for key, value in exporter.items() + if key[0] not in not_ported} + + findings = type_findings(bot_text, exporter_text, dropped, known_diffs) + for key in exporter.keys() - bot.keys(): + findings[key] = f"missing from bot: {format_series(key)}" + for key in bot.keys() - exporter.keys(): + if key[0] not in BOT_ONLY: + findings[key] = f"extra in bot: {format_series(key)}" + for key in bot.keys() & exporter.keys(): + if values_differ(bot[key], exporter[key], known.get(key[0])): + findings[key] = ( + f"value differs: {format_series(key)} " + f"bot={bot[key]!r} exporter={exporter[key]!r}") + return findings, not_ported + + +def compare(pairs, drop_list, known_diffs): + """Findings present in every pair, and the union of unported names.""" + persistent = None + latest = {} + not_ported = set() + for bot_text, exporter_text in pairs: + findings, unported = compare_pair(bot_text, exporter_text, drop_list, known_diffs) + series = findings.keys() + persistent = set(series) if persistent is None else persistent & series + latest = findings + not_ported |= unported + return sorted(latest[identity] for identity in persistent or ()), sorted(not_ported) + + +def unusable_snapshots(paths, texts): + """`path: reason` for each body that is not UTF-8, does not parse, or has + no liq_* sample. An undecodable body is passed as None.""" + unusable = [] + for path, text in zip(paths, texts): + if text is None: + unusable.append(f"{path}: not UTF-8 text") + continue + try: + series = parse_exposition(text) + parse_types(text) + except ValueError as error: + message = str(error) + reason = message.split(":", 1)[0] if message.startswith("repeated") \ + else "not a Prometheus text body" + unusable.append(f"{path}: {reason}") + continue + if not any(name in PORTED for name, _ in series): + unusable.append(f"{path}: no ported liq_* series") + return unusable + + +def report(pairs, findings, not_ported): + lines = [f"compared {pairs} snapshot pair(s); findings present in every pair:"] + lines += findings or ["none"] + if not_ported: + lines.append("not yet ported (exporter-only names, ignored): " + + ", ".join(not_ported)) + lines.append(f"{len(findings)} finding(s)") + return "\n".join(lines) + "\n" + + +def main(argv): + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + parser.add_argument("--drop-list", action="store_true", + help="skip exporter names the bot never ports") + parser.add_argument("--known-diffs", action="store_true", + help="apply the documented differences") + parser.add_argument("files", nargs="+", metavar="BOT.prom EXPORTER.prom") + args = parser.parse_args(argv) + if len(args.files) % 2: + parser.error("files come in BOT.prom EXPORTER.prom pairs") + + texts = [] + for path in args.files: + with open(path, "rb") as f: + body = f.read() + try: + texts.append(body.decode("utf-8")) + except UnicodeDecodeError: + texts.append(None) + unusable = unusable_snapshots(args.files, texts) + if unusable: + sys.stderr.write("unusable snapshot(s):\n" + "".join(f" {line}\n" for line in unusable)) + return 2 + pairs = list(zip(texts[::2], texts[1::2])) + + findings, not_ported = compare(pairs, args.drop_list, args.known_diffs) + sys.stdout.write(report(len(pairs), findings, not_ported)) + return 1 if findings else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/liq-parity/test_compare.py b/scripts/liq-parity/test_compare.py new file mode 100644 index 0000000000..f7bb444769 --- /dev/null +++ b/scripts/liq-parity/test_compare.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +"""Tests for compare.py against the checked-in snapshot pair in testdata/. + + python3 scripts/liq-parity/test_compare.py +""" + +import io +import math +import os +import sys +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from unittest import mock + +# Keep the import from leaving a __pycache__ directory in the repository. +sys.dont_write_bytecode = True + +import compare # noqa: E402 + +HERE = os.path.dirname(os.path.abspath(__file__)) +BOT = os.path.join(HERE, "testdata", "bot.prom") +EXPORTER = os.path.join(HERE, "testdata", "exporter.prom") + + +def run(*args): + out = io.StringIO() + with redirect_stdout(out): + status = compare.main(list(args)) + return status, out.getvalue() + + +class CompareTest(unittest.TestCase): + def test_reports_missing_extra_and_value_findings_after_drop_list_and_known_diffs(self): + status, report = run("--drop-list", "--known-diffs", BOT, EXPORTER) + + self.assertEqual(status, 1) + self.assertEqual(report, ( + "compared 1 snapshot pair(s); findings present in every pair:\n" + "extra in bot: liq_settings_cash_reserved\n" + 'missing from bot: liq_asset_rebalancing{symbol="TSLA"}\n' + "value differs: liq_settings_usdc_target bot=0.45 exporter=0.4\n" + "not yet ported (exporter-only names, ignored): " + "liq_block_lag_blocks, liq_equity_total\n" + "3 finding(s)\n" + )) + + def test_without_known_diffs_the_start_time_jitter_and_the_gauge_types_are_findings(self): + status, report = run(BOT, EXPORTER) + + self.assertEqual(status, 1) + self.assertEqual(report, ( + "compared 1 snapshot pair(s); findings present in every pair:\n" + "extra in bot: liq_settings_cash_reserved\n" + 'missing from bot: liq_asset_rebalancing{symbol="TSLA"}\n' + "type differs: liq_asset_rebalancing bot=gauge exporter=untyped\n" + "type differs: liq_bot_info bot=gauge exporter=untyped\n" + "type differs: liq_bot_start_timestamp_seconds bot=gauge exporter=untyped\n" + "type differs: liq_settings_equity_target bot=gauge exporter=untyped\n" + "type differs: liq_settings_usdc_target bot=gauge exporter=untyped\n" + "value differs: liq_bot_start_timestamp_seconds " + "bot=1700000000.0 exporter=1700000001.9\n" + "value differs: liq_settings_usdc_target bot=0.45 exporter=0.4\n" + "not yet ported (exporter-only names, ignored): " + "liq_asset_flags, liq_block_lag_blocks, liq_equity_total, liq_up\n" + "9 finding(s)\n" + )) + + def test_known_diffs_accept_only_a_bot_gauge_against_an_untyped_exporter(self): + exporter = ( + "liq_settings_usdc_target 0.4\n" + "liq_settings_equity_target 0.5\n" + "# TYPE liq_settings_cash_reserved gauge\n" + "liq_settings_cash_reserved 1000\n" + "liq_bot_info 1\n") + bot = ( + "# TYPE liq_settings_usdc_target gauge\n" + "liq_settings_usdc_target 0.4\n" + "# TYPE liq_settings_equity_target counter\n" + "liq_settings_equity_target 0.5\n" + "liq_settings_cash_reserved 1000\n" + "# TYPE liq_bot_info untyped\n" + "liq_bot_info 1\n") + + findings, _ = compare.compare([(bot, exporter)], True, True) + + self.assertEqual(findings, [ + "type differs: liq_settings_cash_reserved bot=untyped exporter=gauge", + "type differs: liq_settings_equity_target bot=counter exporter=untyped", + ]) + + def test_a_type_is_compared_only_for_a_name_both_sides_publish(self): + bot = "# TYPE liq_settings_cash_reserved counter\nliq_settings_cash_reserved 1\n" + + findings, _ = compare.compare([(bot, "liq_bot_info 1\n")], True, True) + + self.assertEqual(findings, [ + "extra in bot: liq_settings_cash_reserved", + "missing from bot: liq_bot_info", + ]) + + def test_a_type_difference_present_in_only_one_pair_is_not_reported(self): + exporter = "liq_bot_info 1\n" + counter = "# TYPE liq_bot_info counter\nliq_bot_info 1\n" + gauge = "# TYPE liq_bot_info gauge\nliq_bot_info 1\n" + + findings, _ = compare.compare([(counter, exporter), (gauge, exporter)], True, True) + + self.assertEqual(findings, []) + + def test_a_name_whose_values_are_ignored_still_has_its_type_compared(self): + bot = "# TYPE liq_bot_info counter\nliq_bot_info 2\n" + + with mock.patch.dict(compare.KNOWN_DIFFS, {"liq_bot_info": "ignore"}): + findings, _ = compare.compare([(bot, "liq_bot_info 1\n")], True, True) + + self.assertEqual(findings, ["type differs: liq_bot_info bot=counter exporter=untyped"]) + + def test_a_type_line_without_a_blank_after_the_hash_is_read(self): + bot = "#TYPE liq_bot_info counter\nliq_bot_info 1\n" + + findings, _ = compare.compare([(bot, "liq_bot_info 1\n")], True, True) + + self.assertEqual(findings, ["type differs: liq_bot_info bot=counter exporter=untyped"]) + self.assertEqual( + compare.unusable_snapshots( + ["a"], ["# TYPE liq_bot_info gauge\n#TYPE liq_bot_info gauge\nliq_bot_info 1\n"]), + ["a: repeated TYPE"]) + + def test_a_malformed_or_repeated_type_line_makes_the_snapshot_unusable(self): + sample = "liq_bot_info 1\n" + bodies = [ + "# TYPE liq_bot_info\n" + sample, + "# TYPE liq_bot_info gauges\n" + sample, + "# TYPE liq_bot_info gauge\n# TYPE liq_bot_info gauge\n" + sample, + ] + + self.assertEqual(compare.unusable_snapshots(["a", "b", "c"], bodies), [ + "a: not a Prometheus text body", + "b: not a Prometheus text body", + "c: repeated TYPE", + ]) + + def test_a_type_line_after_a_sample_of_its_name_makes_the_snapshot_unusable(self): + with tempfile.NamedTemporaryFile("w", suffix=".prom") as late: + late.write("liq_bot_info 1\n#TYPE liq_bot_info gauge\n") + late.flush() + err = io.StringIO() + with redirect_stderr(err): + status, report = run(late.name, EXPORTER) + + self.assertEqual(status, 2) + self.assertEqual(report, "") + self.assertEqual(err.getvalue(), + f"unusable snapshot(s):\n {late.name}: not a Prometheus text body\n") + self.assertEqual( + compare.unusable_snapshots(["a"], ["liq_bot_info 1\n# TYPE liq_bot_info gauge\n"]), + ["a: not a Prometheus text body"]) + + def test_type_lines_parse_and_other_comments_are_ignored(self): + body = ("# HELP liq_bot_info Build.\n" + " # TYPE liq_bot_info\tgauge \n" + "# TYPEWRITER liq_a counter\n" + "# a TYPE comment\n" + "# TYPE hedge_trades_total counter\n") + + self.assertEqual(compare.parse_types(body), + {"liq_bot_info": "gauge", "hedge_trades_total": "counter"}) + + def test_only_findings_present_in_every_pair_are_reported(self): + with open(BOT) as f: + bot = f.read() + with open(EXPORTER) as f: + exporter = f.read() + caught_up = exporter.replace("liq_settings_usdc_target 0.4", "liq_settings_usdc_target 0.45") + + findings, _ = compare.compare([(bot, exporter), (bot, caught_up)], True, True) + + self.assertEqual(findings, [ + "extra in bot: liq_settings_cash_reserved", + 'missing from bot: liq_asset_rebalancing{symbol="TSLA"}', + ]) + + def test_a_mismatch_whose_values_move_between_pairs_is_still_reported(self): + bot = "liq_bot_start_timestamp_seconds 100\n" + pairs = [(bot, f"liq_bot_start_timestamp_seconds {start}\n") + for start in ("103.1", "103.2", "103.3")] + + findings, _ = compare.compare(pairs, True, True) + + self.assertEqual(findings, [ + "value differs: liq_bot_start_timestamp_seconds bot=100.0 exporter=103.3", + ]) + + def test_a_snapshot_without_liq_series_fails_instead_of_passing(self): + empty = os.path.join(HERE, "testdata", "empty.prom") + err = io.StringIO() + with redirect_stderr(err): + status, report = run(BOT, empty) + + self.assertEqual(status, 2) + self.assertEqual(report, "") + self.assertEqual(err.getvalue(), f"unusable snapshot(s):\n {empty}: no ported liq_* series\n") + + def test_a_body_that_is_not_prometheus_text_fails_instead_of_crashing(self): + html = os.path.join(HERE, "testdata", "not-prometheus.txt") + err = io.StringIO() + with redirect_stderr(err): + status, report = run(BOT, html) + + self.assertEqual(status, 2) + self.assertEqual(report, "") + self.assertEqual(err.getvalue(), + f"unusable snapshot(s):\n {html}: not a Prometheus text body\n") + + def test_a_series_that_changes_kind_between_pairs_is_still_reported(self): + exporter = "liq_settings_usdc_target 0.4\nliq_bot_info{git_commit=\"a\"} 1\n" + differs = "liq_settings_usdc_target 0.45\nliq_bot_info{git_commit=\"a\"} 1\n" + missing = "liq_bot_info{git_commit=\"a\"} 1\n" + + findings, _ = compare.compare( + [(differs, exporter), (missing, exporter), (differs, exporter)], True, True) + + self.assertEqual(findings, [ + "value differs: liq_settings_usdc_target bot=0.45 exporter=0.4", + ]) + + def test_a_repeated_series_makes_the_snapshot_unusable(self): + doubled = os.path.join(HERE, "testdata", "repeated.prom") + err = io.StringIO() + with redirect_stderr(err): + status, report = run(doubled, EXPORTER) + + self.assertEqual(status, 2) + self.assertEqual(report, "") + self.assertEqual(err.getvalue(), f"unusable snapshot(s):\n {doubled}: repeated series\n") + + def test_a_repeated_label_name_makes_the_snapshot_unusable(self): + doubled = 'liq_bot_info{git_commit="a",git_commit="b"} 1\n' + + self.assertEqual(compare.unusable_snapshots(["bot.prom"], [doubled]), + ["bot.prom: repeated label name"]) + + def test_a_snapshot_that_is_not_utf8_fails_instead_of_crashing(self): + with tempfile.NamedTemporaryFile(suffix=".prom") as binary: + binary.write(b"liq_bot_info{git_commit=\"\xff\"} 1\n") + binary.flush() + err = io.StringIO() + with redirect_stderr(err): + status, report = run(BOT, binary.name) + + self.assertEqual(status, 2) + self.assertEqual(report, "") + self.assertEqual(err.getvalue(), + f"unusable snapshot(s):\n {binary.name}: not UTF-8 text\n") + + def test_a_body_without_a_final_line_feed_fails_instead_of_passing(self): + with tempfile.NamedTemporaryFile("w", suffix=".prom") as cut: + cut.write("liq_bot_info 1") + cut.flush() + err = io.StringIO() + with redirect_stderr(err): + status, report = run(cut.name, cut.name) + + self.assertEqual(status, 2) + self.assertEqual(report, "") + self.assertEqual(err.getvalue(), ( + "unusable snapshot(s):\n" + f" {cut.name}: not a Prometheus text body\n" + f" {cut.name}: not a Prometheus text body\n")) + + def test_a_comma_needs_a_label_before_it(self): + self.assertEqual(compare.parse_exposition("liq_a{} 1\nliq_b{x=\"1\",} 2\n"), + {("liq_a", ()): 1.0, ("liq_b", (("x", "1"),)): 2.0}) + with self.assertRaises(ValueError): + compare.parse_exposition("liq_bot_info{,} 1\n") + + def test_only_a_line_feed_ends_a_line(self): + body = '# HELP liq_bot_info Build\u2028stamp.\nliq_bot_info{git_commit="a\u2028b"} 1\n' + + self.assertEqual(compare.parse_exposition(body), + {("liq_bot_info", (("git_commit", "a\u2028b"),)): 1.0}) + + def test_matching_snapshots_have_no_findings(self): + status, report = run(BOT, BOT) + + self.assertEqual(status, 0) + self.assertEqual(report, ( + "compared 1 snapshot pair(s); findings present in every pair:\n" + "none\n" + "0 finding(s)\n" + )) + + def test_leading_whitespace_and_tabs_parse(self): + series = compare.parse_exposition( + ' # HELP liq_bot_info Build.\n\tliq_settings_usdc_target\t0.4\n' + ' liq_bot_info{git_commit="abc"} 1\n') + + self.assertEqual(series, { + ("liq_settings_usdc_target", ()): 0.4, + ("liq_bot_info", (("git_commit", "abc"),)): 1.0, + }) + + def test_spaces_after_label_commas_give_the_same_series(self): + spaced = compare.parse_exposition('liq_asset_rebalancing{chain="base", symbol="X"} 1\n') + tight = compare.parse_exposition('liq_asset_rebalancing{chain="base",symbol="X"} 1\n') + + self.assertEqual(spaced, tight) + + def test_a_sample_line_allows_only_a_value_and_an_integer_timestamp(self): + self.assertEqual(compare.parse_exposition("liq_bot_start_timestamp_seconds 5 1700000000000\n"), + {("liq_bot_start_timestamp_seconds", ()): 5.0}) + with self.assertRaises(ValueError): + compare.parse_exposition("liq_bot_start_timestamp_seconds 5 garbage\n") + with self.assertRaises(ValueError): + compare.parse_exposition("liq_bot_start_timestamp_seconds\n") + + def test_a_value_that_prometheus_rejects_fails_instead_of_passing(self): + with tempfile.NamedTemporaryFile("w", suffix=".prom") as underscored: + underscored.write("liq_bot_info 1_0\n") + underscored.flush() + err = io.StringIO() + with redirect_stderr(err): + status, report = run(underscored.name, EXPORTER) + + self.assertEqual(status, 2) + self.assertEqual(report, "") + self.assertEqual(err.getvalue(), ( + "unusable snapshot(s):\n" + f" {underscored.name}: not a Prometheus text body\n")) + for value in ("0x10", "1e", "infin", "--1"): + with self.assertRaises(ValueError): + compare.parse_exposition(f"liq_a {value}\n") + self.assertEqual(compare.parse_exposition("liq_a 1.5e3\nliq_b .5\nliq_c -2.\nliq_d -Infinity\n"), + {("liq_a", ()): 1500.0, ("liq_b", ()): 0.5, ("liq_c", ()): -2.0, ("liq_d", ()): -math.inf}) + + def test_a_degraded_snapshot_with_only_liq_up_is_unusable(self): + degraded = os.path.join(HERE, "testdata", "degraded.prom") + err = io.StringIO() + with redirect_stderr(err): + status, _ = run(BOT, degraded) + + self.assertEqual(status, 2) + self.assertEqual(err.getvalue(), f"unusable snapshot(s):\n {degraded}: no ported liq_* series\n") + + def test_a_trailing_label_comma_parses_and_an_unknown_escape_does_not(self): + self.assertEqual(compare.parse_exposition('liq_asset_rebalancing{symbol="AAPL",} 1\n'), + {("liq_asset_rebalancing", (("symbol", "AAPL"),)): 1.0}) + with self.assertRaises(ValueError): + compare.parse_exposition('liq_bot_info{git_commit="a\\qb"} 1\n') + + def test_labels_without_a_separator_are_rejected(self): + with self.assertRaises(ValueError): + compare.parse_exposition('liq_bot_info{git_commit="a"chain="base"} 1\n') + + def test_special_values_parse(self): + series = compare.parse_exposition("liq_a NaN\nliq_b +Inf\nliq_c -Inf\n") + + self.assertTrue(math.isnan(series[("liq_a", ())])) + self.assertEqual(series[("liq_b", ())], math.inf) + self.assertEqual(series[("liq_c", ())], -math.inf) + + def test_findings_re_escape_label_values(self): + key = ("liq_bot_info", (("git_commit", 'a\\b"c\nd'),)) + + self.assertEqual(compare.format_series(key), 'liq_bot_info{git_commit="a\\\\b\\"c\\nd"}') + + def test_nan_against_a_number_differs_under_every_rule(self): + nan = float("nan") + + self.assertTrue(compare.values_differ(nan, 1.0, ("absolute", 2.0))) + self.assertTrue(compare.values_differ(1.0, nan, None)) + self.assertFalse(compare.values_differ(nan, nan, ("absolute", 2.0))) + self.assertFalse(compare.values_differ(nan, 1.0, "values")) + + def test_escaped_label_values_parse_back(self): + series = compare.parse_exposition('liq_bot_info{git_commit="a\\\\b\\"c\\nd"} 1\n') + + self.assertEqual(series, {("liq_bot_info", (("git_commit", 'a\\b"c\nd'),)): 1.0}) + + +if __name__ == "__main__": + sys.exit(unittest.main()) diff --git a/scripts/liq-parity/testdata/bot.prom b/scripts/liq-parity/testdata/bot.prom new file mode 100644 index 0000000000..ea6412b4ca --- /dev/null +++ b/scripts/liq-parity/testdata/bot.prom @@ -0,0 +1,17 @@ +# HELP liq_bot_info Always 1; git_commit is the first 12 characters of the running build +# TYPE liq_bot_info gauge +liq_bot_info{git_commit="0123456789ab"} 1 +# TYPE liq_bot_start_timestamp_seconds gauge +liq_bot_start_timestamp_seconds 1700000000.0 +# TYPE liq_settings_equity_target gauge +liq_settings_equity_target 0.5 +# TYPE liq_settings_usdc_target gauge +liq_settings_usdc_target 0.45 +# TYPE liq_settings_cash_reserved gauge +liq_settings_cash_reserved 1000 +# TYPE liq_asset_rebalancing gauge +liq_asset_rebalancing{symbol="AAPL"} 1 +# TYPE liq_collector_last_success_ts_seconds gauge +liq_collector_last_success_ts_seconds{collector="settings"} 1700000100 +# TYPE hedge_trades_total counter +hedge_trades_total{direction="buy",symbol="AAPL"} 3 diff --git a/scripts/liq-parity/testdata/degraded.prom b/scripts/liq-parity/testdata/degraded.prom new file mode 100644 index 0000000000..93c56c7302 --- /dev/null +++ b/scripts/liq-parity/testdata/degraded.prom @@ -0,0 +1 @@ +liq_up 0 diff --git a/scripts/liq-parity/testdata/empty.prom b/scripts/liq-parity/testdata/empty.prom new file mode 100644 index 0000000000..d1dbf194c3 --- /dev/null +++ b/scripts/liq-parity/testdata/empty.prom @@ -0,0 +1,2 @@ +# HELP hedge_trades_total Hedge trades placed. +hedge_trades_total 3 diff --git a/scripts/liq-parity/testdata/exporter.prom b/scripts/liq-parity/testdata/exporter.prom new file mode 100644 index 0000000000..4100c012b0 --- /dev/null +++ b/scripts/liq-parity/testdata/exporter.prom @@ -0,0 +1,10 @@ +liq_up 1.0 +liq_bot_info{git_commit="0123456789ab"} 1.0 +liq_bot_start_timestamp_seconds 1700000001.9 +liq_settings_equity_target 0.5 +liq_settings_usdc_target 0.4 +liq_asset_rebalancing{symbol="AAPL"} 1.0 +liq_asset_rebalancing{symbol="TSLA"} 0.0 +liq_asset_flags{counter_trading="enabled",extended_hours="true",rebalancing="true",symbol="AAPL"} 1.0 +liq_equity_total{symbol="AAPL"} 5.0 +liq_block_lag_blocks{chain="base"} 3.0 diff --git a/scripts/liq-parity/testdata/not-prometheus.txt b/scripts/liq-parity/testdata/not-prometheus.txt new file mode 100644 index 0000000000..1f565b4f11 --- /dev/null +++ b/scripts/liq-parity/testdata/not-prometheus.txt @@ -0,0 +1 @@ +Bad Gateway diff --git a/scripts/liq-parity/testdata/repeated.prom b/scripts/liq-parity/testdata/repeated.prom new file mode 100644 index 0000000000..742df18257 --- /dev/null +++ b/scripts/liq-parity/testdata/repeated.prom @@ -0,0 +1,2 @@ +liq_settings_usdc_target 0.9 +liq_settings_usdc_target 0.4