diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 295cce5863..a5bbf8f6b0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -128,6 +128,15 @@ jobs: workspaces: ". -> target/test" save-if: ${{ env.UNTRUSTED_PR != 'true' }} + # The board JSON must match its generator, with every liq_ selector + # pinned to the exporter job. + - name: Board generator check + run: | + nix develop .#ci-backend -c bash -c " + set -euxo pipefail + python3 observability/gen-t0-liquidity.py --check + " + # Build the test binaries once into a nextest archive shared by all four # partitions, and run clippy and rustdoc here since this is the single # full compile. diff --git a/docs/ci.md b/docs/ci.md index 125c6c2881..06900ffe36 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -68,3 +68,11 @@ cannot satisfy pull request branch protection. Use a pull request event for merge-blocking validation. Reserve `workflow_dispatch` for manual runs whose result does not need to become a pull request status check. + +## Board check + +The `backend build` job also runs +`python3 observability/gen-t0-liquidity.py --check` in the `ci-backend` shell, +and `nix run .#ci` runs it too. It fails when the committed liquidity board JSON +differs from the generator output, or when a `liq_` selector is not pinned to +the exporter job. diff --git a/docs/observability.md b/docs/observability.md index f1241d56af..1c14f3f295 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -104,3 +104,13 @@ changes land. A refused token file does not alert: whoever publishes it checks also covers startup failure, alerts through `liquidity-token-file-fallback-production` in `observability/alerting/liquidity.rules.yml`. + +## Metrics + +### Board pins + +Until consumers move to the bot, every `liq_` selector on the liquidity boards +reads `job="t0-liquidity-exporter"`. `observability/gen-t0-liquidity.py` adds +the matcher and fails if any `liq_` selector is left unpinned. +`python3 observability/gen-t0-liquidity.py --check` (run in CI) fails when the +committed board JSON differs from the generator output. diff --git a/flake.nix b/flake.nix index cc29864b7a..568714d925 100644 --- a/flake.nix +++ b/flake.nix @@ -425,6 +425,7 @@ cargo nextest run --workspace --all-features cargo clippy --workspace --all-targets --all-features cargo fmt -- --check + python3 observability/gen-t0-liquidity.py --check ' nix develop .#ci-audit -c bash -c './scripts/test-rust-audit.sh && ./scripts/audit-rust.sh' @@ -848,7 +849,8 @@ # CI: cargo check/nextest/clippy + sqlx db reset. No terraform, no # deploy scripts. abiEnv is needed because build.rs / sol! macros # consume ST0X_*_ABI at compile time. foundry is required because - # tests spawn anvil for local EVM simulation. + # tests spawn anvil for local EVM simulation. python3 runs the + # board generator check. ci-backend = pkgs.mkShell ( { buildInputs = @@ -856,6 +858,7 @@ ++ [ pkgs.sqlx-cli pkgs.cargo-nextest + pkgs.python3 foundryBin ] ++ moldInputs; diff --git a/observability/dashboards/liquidity/t0-liquidity.json b/observability/dashboards/liquidity/t0-liquidity.json index 748ec70cc2..e3fa4d61b7 100644 --- a/observability/dashboards/liquidity/t0-liquidity.json +++ b/observability/dashboards/liquidity/t0-liquidity.json @@ -142,7 +142,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up)", + "expr": "max(liq_up{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -244,7 +244,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", + "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=\"t0-liquidity-exporter\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", "step": "60s" }, "timeSeriesList": { @@ -331,7 +331,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds)", + "expr": "time() - max(liq_bot_start_timestamp_seconds{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -419,7 +419,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info)", + "expr": "sum by (broker) (liq_settings_info{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -506,7 +506,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_equity_target)", + "expr": "max(liq_settings_equity_target{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -594,7 +594,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_usdc_target)", + "expr": "max(liq_settings_usdc_target{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -682,7 +682,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd)", + "expr": "max(liq_settings_execution_threshold_usd{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -770,7 +770,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (row, col) (label_replace(label_replace(liq_usdc_onchain_available, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Raindex\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_inflight_total, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Inflight\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_alpaca_total, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Alpaca Total\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_total, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Total\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_ratio, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Ratio\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_ratio_deviation and liq_usdc_total > 0, \"row\", \"USDC\", \"\", \"\"), \"col\", \"\u0394\", \"\", \"\"))", + "expr": "sum by (row, col) (label_replace(label_replace(liq_usdc_onchain_available{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Raindex\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_inflight_total{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Inflight\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_alpaca_total{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Alpaca Total\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_total{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Total\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_ratio{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Ratio\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_ratio_deviation{job=\"t0-liquidity-exporter\"} and liq_usdc_total{job=\"t0-liquidity-exporter\"} > 0, \"row\", \"USDC\", \"\", \"\"), \"col\", \"\u0394\", \"\", \"\"))", "step": "60s" }, "timeSeriesList": { @@ -1084,7 +1084,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (row, col) (label_replace(label_replace(liq_usdc_alpaca_usdc, \"row\", \"USDC\", \"\", \"\"), \"col\", \"USDC\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_rebalanceable, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Rebalanceable\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_alpaca_total, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Counter-tradeable\", \"\", \"\"))", + "expr": "sum by (row, col) (label_replace(label_replace(liq_usdc_alpaca_usdc{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"USDC\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_rebalanceable{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Rebalanceable\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_alpaca_total{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Counter-tradeable\", \"\", \"\"))", "step": "60s" }, "timeSeriesList": { @@ -1267,7 +1267,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (row, col) (label_replace(label_replace(liq_usdc_inflight_ethereum_wallet, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Eth\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_inflight_base_wallet, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Base\", \"\", \"\"))", + "expr": "sum by (row, col) (label_replace(label_replace(liq_usdc_inflight_ethereum_wallet{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Eth\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_inflight_base_wallet{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Base\", \"\", \"\"))", "step": "60s" }, "timeSeriesList": { @@ -1429,7 +1429,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (symbol, col) (label_replace(liq_asset_counter_trading, \"col\", \"CT\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_asset_rebalancing, \"col\", \"Rebal\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_asset_extended_hours, \"col\", \"Ext\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_onchain_available, \"col\", \"Raindex\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_inflight_total, \"col\", \"Inflight\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_offchain_available, \"col\", \"Alpaca\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_total, \"col\", \"Total\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_ratio, \"col\", \"Ratio\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_ratio_deviation and liq_equity_total > 0, \"col\", \"\u0394\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_exposure_usd or (liq_equity_total * NaN), \"col\", \"Exposure\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_unwrapped, \"col\", \"Unwrapped\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_wrapped, \"col\", \"Wrapped\", \"\", \"\"))", + "expr": "sum by (symbol, col) (label_replace(liq_asset_counter_trading{job=\"t0-liquidity-exporter\"}, \"col\", \"CT\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_asset_rebalancing{job=\"t0-liquidity-exporter\"}, \"col\", \"Rebal\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_asset_extended_hours{job=\"t0-liquidity-exporter\"}, \"col\", \"Ext\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_onchain_available{job=\"t0-liquidity-exporter\"}, \"col\", \"Raindex\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_inflight_total{job=\"t0-liquidity-exporter\"}, \"col\", \"Inflight\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_offchain_available{job=\"t0-liquidity-exporter\"}, \"col\", \"Alpaca\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_total{job=\"t0-liquidity-exporter\"}, \"col\", \"Total\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_ratio{job=\"t0-liquidity-exporter\"}, \"col\", \"Ratio\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_ratio_deviation{job=\"t0-liquidity-exporter\"} and liq_equity_total{job=\"t0-liquidity-exporter\"} > 0, \"col\", \"\u0394\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_exposure_usd{job=\"t0-liquidity-exporter\"} or (liq_equity_total{job=\"t0-liquidity-exporter\"} * NaN), \"col\", \"Exposure\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_unwrapped{job=\"t0-liquidity-exporter\"}, \"col\", \"Unwrapped\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_wrapped{job=\"t0-liquidity-exporter\"}, \"col\", \"Wrapped\", \"\", \"\"))", "step": "60s" }, "timeSeriesList": { diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json b/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json index 4d06b09677..d4e472d9e6 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json @@ -304,7 +304,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up)", + "expr": "max(liq_up{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -406,7 +406,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", + "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=\"t0-liquidity-exporter\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", "step": "60s" }, "timeSeriesList": { @@ -493,7 +493,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds)", + "expr": "time() - max(liq_bot_start_timestamp_seconds{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -581,7 +581,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info)", + "expr": "sum by (broker) (liq_settings_info{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -668,7 +668,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_equity_target)", + "expr": "max(liq_settings_equity_target{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -756,7 +756,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_usdc_target)", + "expr": "max(liq_settings_usdc_target{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -844,7 +844,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd)", + "expr": "max(liq_settings_execution_threshold_usd{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json b/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json index 882e42a193..9cda52ec70 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json @@ -142,7 +142,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up)", + "expr": "max(liq_up{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -244,7 +244,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", + "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=\"t0-liquidity-exporter\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", "step": "60s" }, "timeSeriesList": { @@ -331,7 +331,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds)", + "expr": "time() - max(liq_bot_start_timestamp_seconds{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -419,7 +419,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info)", + "expr": "sum by (broker) (liq_settings_info{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -506,7 +506,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_equity_target)", + "expr": "max(liq_settings_equity_target{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -594,7 +594,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_usdc_target)", + "expr": "max(liq_settings_usdc_target{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -682,7 +682,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd)", + "expr": "max(liq_settings_execution_threshold_usd{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -1055,7 +1055,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pending_orders_total) or vector(0)", + "expr": "max(liq_pending_orders_total{job=\"t0-liquidity-exporter\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1137,7 +1137,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_raindex_orders_total)", + "expr": "max(liq_raindex_orders_total{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -1219,7 +1219,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_raindex_orders_unavailable) or vector(0)", + "expr": "max(liq_raindex_orders_unavailable{job=\"t0-liquidity-exporter\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json b/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json index 393e6bbf38..35c4cb036f 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json @@ -183,7 +183,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up)", + "expr": "max(liq_up{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -285,7 +285,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", + "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=\"t0-liquidity-exporter\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", "step": "60s" }, "timeSeriesList": { @@ -372,7 +372,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds)", + "expr": "time() - max(liq_bot_start_timestamp_seconds{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -460,7 +460,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info)", + "expr": "sum by (broker) (liq_settings_info{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -547,7 +547,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_equity_target)", + "expr": "max(liq_settings_equity_target{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -635,7 +635,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_usdc_target)", + "expr": "max(liq_settings_usdc_target{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -723,7 +723,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd)", + "expr": "max(liq_settings_execution_threshold_usd{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -811,7 +811,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{stage=\"detection\",quantile=\"p95\"})", + "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"detection\",quantile=\"p95\"})", "step": "60s" }, "timeSeriesList": { @@ -900,7 +900,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{stage=\"exposure_window\",quantile=\"p95\"})", + "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"exposure_window\",quantile=\"p95\"})", "step": "60s" }, "timeSeriesList": { @@ -989,7 +989,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_reliability_log_count_24h{level=\"error\"}) or vector(0)", + "expr": "max(liq_reliability_log_count_24h{job=\"t0-liquidity-exporter\",level=\"error\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1078,7 +1078,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - min(liq_open_exposure_oldest_ts_seconds)", + "expr": "time() - min(liq_open_exposure_oldest_ts_seconds{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -1167,7 +1167,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_block_lag_blocks)", + "expr": "max(liq_block_lag_blocks{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -1256,7 +1256,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{stage=\"detection\",quantile=\"p50\"})", + "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"detection\",quantile=\"p50\"})", "step": "60s" }, "timeSeriesList": { @@ -1342,7 +1342,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms_samples{stage=\"detection\"}) or vector(0)", + "expr": "max(liq_hedge_latency_ms_samples{job=\"t0-liquidity-exporter\",stage=\"detection\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1428,7 +1428,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{stage=\"exposure_window\",quantile=\"p50\"})", + "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"exposure_window\",quantile=\"p50\"})", "step": "60s" }, "timeSeriesList": { @@ -1514,7 +1514,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms_samples{stage=\"exposure_window\"}) or vector(0)", + "expr": "max(liq_hedge_latency_ms_samples{job=\"t0-liquidity-exporter\",stage=\"exposure_window\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1600,7 +1600,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_reliability_log_count_24h{level=\"warning\"}) or vector(0)", + "expr": "max(liq_reliability_log_count_24h{job=\"t0-liquidity-exporter\",level=\"warning\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1686,7 +1686,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum(liq_failure_event_count_24h) or vector(0)", + "expr": "sum(liq_failure_event_count_24h{job=\"t0-liquidity-exporter\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1772,7 +1772,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "count(sum by (job_type) (liq_job_queue{state=~\"failed|killed\"}) > 0) or vector(0)", + "expr": "count(sum by (job_type) (liq_job_queue{job=\"t0-liquidity-exporter\",state=~\"failed|killed\"}) > 0) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1862,7 +1862,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum(liq_open_exposure_fill_count) or vector(0)", + "expr": "sum(liq_open_exposure_fill_count{job=\"t0-liquidity-exporter\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1948,7 +1948,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "count(liq_open_exposure_fill_count > 0) or vector(0)", + "expr": "count(liq_open_exposure_fill_count{job=\"t0-liquidity-exporter\"} > 0) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -2044,7 +2044,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - max(liq_block_lag_sampled_ts_seconds)", + "expr": "time() - max(liq_block_lag_sampled_ts_seconds{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -2130,7 +2130,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_poll_skipped_ticks_24h) or vector(0)", + "expr": "max(liq_poll_skipped_ticks_24h{job=\"t0-liquidity-exporter\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -2216,7 +2216,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (stage) (liq_hedge_latency_ms{quantile=\"p95\"})", + "expr": "max by (stage) (liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",quantile=\"p95\"})", "step": "60s" }, "timeSeriesList": { @@ -2294,7 +2294,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{stage=\"$stage\",quantile=\"p50\"})", + "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"$stage\",quantile=\"p50\"})", "step": "60s" }, "timeSeriesList": { @@ -2314,7 +2314,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{stage=\"$stage\",quantile=\"p90\"})", + "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"$stage\",quantile=\"p90\"})", "step": "60s" }, "timeSeriesList": { @@ -2334,7 +2334,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{stage=\"$stage\",quantile=\"p99\"})", + "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"$stage\",quantile=\"p99\"})", "step": "60s" }, "timeSeriesList": { @@ -2450,7 +2450,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "label_replace(liq_log_target_count_24h, \"col\", \"$1\", \"level\", \"(.*)\")", + "expr": "label_replace(liq_log_target_count_24h{job=\"t0-liquidity-exporter\"}, \"col\", \"$1\", \"level\", \"(.*)\")", "step": "60s" }, "timeSeriesList": { @@ -2655,7 +2655,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (stage) (liq_rebalance_stage_ms{kind=\"usdc\",quantile=\"p95\"})", + "expr": "max by (stage) (liq_rebalance_stage_ms{job=\"t0-liquidity-exporter\",kind=\"usdc\",quantile=\"p95\"})", "step": "60s" }, "timeSeriesList": { @@ -2725,7 +2725,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (stage) (liq_rebalance_stage_ms{kind=\"equity\",quantile=\"p95\"})", + "expr": "max by (stage) (liq_rebalance_stage_ms{job=\"t0-liquidity-exporter\",kind=\"equity\",quantile=\"p95\"})", "step": "60s" }, "timeSeriesList": { @@ -2795,7 +2795,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "label_join(label_replace(liq_dependency_calls_24h, \"col\", \"calls\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_errors_24h, \"col\", \"errors\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_latency_ms{quantile=\"p50\"}, \"col\", \"p50 ms\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_latency_ms{quantile=\"p95\"}, \"col\", \"p95 ms\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_latency_ms{quantile=\"max\"}, \"col\", \"max ms\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\")", + "expr": "label_join(label_replace(liq_dependency_calls_24h{job=\"t0-liquidity-exporter\"}, \"col\", \"calls\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_errors_24h{job=\"t0-liquidity-exporter\"}, \"col\", \"errors\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_latency_ms{job=\"t0-liquidity-exporter\",quantile=\"p50\"}, \"col\", \"p50 ms\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_latency_ms{job=\"t0-liquidity-exporter\",quantile=\"p95\"}, \"col\", \"p95 ms\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_latency_ms{job=\"t0-liquidity-exporter\",quantile=\"max\"}, \"col\", \"max ms\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\")", "step": "60s" }, "timeSeriesList": { @@ -2955,7 +2955,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_attestation_last_ms{kind=\"usdc\"})", + "expr": "max(liq_attestation_last_ms{job=\"t0-liquidity-exporter\",kind=\"usdc\"})", "step": "60s" }, "timeSeriesList": { @@ -2975,7 +2975,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_attestation_last_ms{kind=\"equity\"})", + "expr": "max(liq_attestation_last_ms{job=\"t0-liquidity-exporter\",kind=\"equity\"})", "step": "60s" }, "timeSeriesList": { @@ -3076,7 +3076,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_block_lag_blocks)", + "expr": "max(liq_block_lag_blocks{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -3147,7 +3147,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_poll_cycles_24h)", + "expr": "max(liq_poll_cycles_24h{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -3228,7 +3228,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_poll_errors_24h) or vector(0)", + "expr": "max(liq_poll_errors_24h{job=\"t0-liquidity-exporter\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -3313,7 +3313,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_poll_skipped_ticks_24h) or vector(0)", + "expr": "max(liq_poll_skipped_ticks_24h{job=\"t0-liquidity-exporter\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -3398,7 +3398,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (quantile) (liq_poll_duration_ms)", + "expr": "max by (quantile) (liq_poll_duration_ms{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -3468,7 +3468,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "label_replace(liq_job_queue, \"col\", \"$1\", \"state\", \"(.*)\")", + "expr": "label_replace(liq_job_queue{job=\"t0-liquidity-exporter\"}, \"col\", \"$1\", \"state\", \"(.*)\")", "step": "60s" }, "timeSeriesList": { @@ -3907,7 +3907,7 @@ { "id": 92, "type": "timeseries", - "title": "Hedge scans skipped, by cause", + "title": "Hedge scans skipped, by reason", "description": "Extended-hours buys the position scan dropped before enqueueing a hedge job, because reference-price resolution or crossing failed. Each one is an on-chain fill that did not get hedged on that pass.", "datasource": { "type": "stackdriver", @@ -3923,9 +3923,9 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (cause, symbol) (increase(hedge_scan_skipped_total[1h]))", + "expr": "sum by (reason, symbol) (increase(hedge_scan_skipped_total[1h]))", "step": "60s", - "legendFormat": "{{cause}} \u00b7 {{symbol}}" + "legendFormat": "{{reason}} \u00b7 {{symbol}}" }, "timeSeriesList": { "projectName": "", diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json b/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json index 643b2b1e62..8205d2ec9d 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json @@ -189,7 +189,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up)", + "expr": "max(liq_up{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -291,7 +291,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", + "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=\"t0-liquidity-exporter\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", "step": "60s" }, "timeSeriesList": { @@ -378,7 +378,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds)", + "expr": "time() - max(liq_bot_start_timestamp_seconds{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -466,7 +466,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info)", + "expr": "sum by (broker) (liq_settings_info{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -553,7 +553,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_equity_target)", + "expr": "max(liq_settings_equity_target{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -641,7 +641,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_usdc_target)", + "expr": "max(liq_settings_usdc_target{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -729,7 +729,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd)", + "expr": "max(liq_settings_execution_threshold_usd{job=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -817,7 +817,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{stream=\"net_realized\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"net_realized\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -911,7 +911,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{stream=\"gross_realized\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"gross_realized\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1005,7 +1005,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{stream=\"tracked_costs\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"tracked_costs\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1091,7 +1091,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{stream=\"tracked_revenue\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"tracked_revenue\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1185,7 +1185,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{stream=\"counter_trade\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"counter_trade\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1279,7 +1279,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{stream=\"onchain_netting\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"onchain_netting\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1373,7 +1373,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{stream=\"directional_imbalance_excess\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"directional_imbalance_excess\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1467,7 +1467,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{stream=\"directional_inventory_baseline\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"directional_inventory_baseline\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1561,7 +1561,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{stream=\"directional_exposure\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"directional_exposure\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1655,7 +1655,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{stream=\"total\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"total\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1749,7 +1749,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_shares{kind=\"inventory_drift\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_shares{job=\"t0-liquidity-exporter\",kind=\"inventory_drift\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1831,7 +1831,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_sample_total_fills{window=\"$window\"})", + "expr": "max(liq_pnl_sample_total_fills{job=\"t0-liquidity-exporter\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1913,7 +1913,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_capital_avg_deployed_usd{window=\"$window\"})", + "expr": "max(liq_pnl_capital_avg_deployed_usd{job=\"t0-liquidity-exporter\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1995,7 +1995,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_capital_annualized_return_pct{window=\"$window\"})", + "expr": "max(liq_pnl_capital_annualized_return_pct{job=\"t0-liquidity-exporter\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -2085,7 +2085,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_capital_coverage_days{window=\"$window\"})", + "expr": "max(liq_pnl_capital_coverage_days{job=\"t0-liquidity-exporter\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -2167,7 +2167,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{col=\"gross_realized\",window=\"$window\"}, \"col\", \"Gross\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{col=\"tracked_costs\",window=\"$window\"}, \"col\", \"Costs\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{col=\"tracked_revenue\",window=\"$window\"}, \"col\", \"Revenue\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{col=\"net_realized\",window=\"$window\"}, \"col\", \"Net\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{col=\"counter_trade\",window=\"$window\"}, \"col\", \"Counter\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{col=\"onchain_netting\",window=\"$window\"}, \"col\", \"On-chain\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{col=\"directional_imbalance_excess\",window=\"$window\"}, \"col\", \"Directional\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{col=\"directional_inventory_baseline\",window=\"$window\"}, \"col\", \"Baseline Drift\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{col=\"total\",window=\"$window\"}, \"col\", \"Total\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_lots{window=\"$window\"}, \"col\", \"Lots\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_volume_shares{window=\"$window\"}, \"col\", \"Volume\", \"\", \"\"))", + "expr": "sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"gross_realized\",window=\"$window\"}, \"col\", \"Gross\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"tracked_costs\",window=\"$window\"}, \"col\", \"Costs\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"tracked_revenue\",window=\"$window\"}, \"col\", \"Revenue\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"net_realized\",window=\"$window\"}, \"col\", \"Net\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"counter_trade\",window=\"$window\"}, \"col\", \"Counter\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"onchain_netting\",window=\"$window\"}, \"col\", \"On-chain\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"directional_imbalance_excess\",window=\"$window\"}, \"col\", \"Directional\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"directional_inventory_baseline\",window=\"$window\"}, \"col\", \"Baseline Drift\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"total\",window=\"$window\"}, \"col\", \"Total\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_lots{job=\"t0-liquidity-exporter\",window=\"$window\"}, \"col\", \"Lots\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_volume_shares{job=\"t0-liquidity-exporter\",window=\"$window\"}, \"col\", \"Volume\", \"\", \"\"))", "step": "60s" }, "timeSeriesList": { @@ -2697,7 +2697,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (day, symbol) (liq_pnl_day_usd{window=\"$window\"})", + "expr": "sum by (day, symbol) (liq_pnl_day_usd{job=\"t0-liquidity-exporter\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -2844,7 +2844,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (day, stream) (liq_pnl_day_stream_usd{window=\"$window\"})", + "expr": "sum by (day, stream) (liq_pnl_day_stream_usd{job=\"t0-liquidity-exporter\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -3052,7 +3052,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (day, symbol) (liq_pnl_day_cum_usd{window=\"$window\"})", + "expr": "sum by (day, symbol) (liq_pnl_day_cum_usd{job=\"t0-liquidity-exporter\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -3199,7 +3199,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (day, stream) (liq_pnl_day_cum_stream_usd{window=\"$window\"})", + "expr": "sum by (day, stream) (liq_pnl_day_cum_stream_usd{job=\"t0-liquidity-exporter\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { diff --git a/observability/gen-t0-liquidity.py b/observability/gen-t0-liquidity.py index 66314fa30c..448bfeda62 100644 --- a/observability/gen-t0-liquidity.py +++ b/observability/gen-t0-liquidity.py @@ -6,6 +6,10 @@ python3 observability/gen-t0-liquidity.py +Check that the committed JSON matches the generator (CI runs this): + + python3 observability/gen-t0-liquidity.py --check + Data comes from the exporter sidecar on the liquidity VM (terraform/staging-liquidity/exporter/exporter.py): @@ -41,12 +45,24 @@ maxDataPoints itself (see latest_status_table). """ +import filecmp import json import os +import re +import sys +import tempfile HERE = os.path.dirname(os.path.abspath(__file__)) NATIVE_ROWS = os.path.join(HERE, "t0-liquidity-native-rows.json") +# The bot serves liq_* on its own /metrics too, so an unpinned liq_ selector +# would sum the exporter and the bot together. Every liq_ selector on these +# boards reads the exporter's job until the board learns a source switch. +LIQ_JOB = "t0-liquidity-exporter" +LIQ_JOB_MATCHER = f'job="{LIQ_JOB}"' +LIQ_NAME = re.compile(r"(? `liq_x{job="..."}`) and names that already + carry matchers (`liq_x{a="b"}` -> `liq_x{job="...",a="b"}`). A selector + that already names a job is left for `unpinned_liq_selectors` to judge. + """ + out = [] + pos = 0 + for match in LIQ_NAME.finditer(expr): + out.append(expr[pos:match.end()]) + pos = match.end() + if not expr.startswith("{", pos): + out.append("{" + LIQ_JOB_MATCHER + "}") + continue + close = expr.find("}", pos) + if close == -1: + raise SystemExit(f"unterminated label matcher in PromQL: {expr}") + matchers = expr[pos + 1:close].strip() + if re.search(r"(? Result, InvalidT .collect() } -static STARTED_AT: LazyLock> = LazyLock::new(Utc::now); const DEFAULT_RAINDEX_ORDERS_PAGE_SIZE: u32 = 50; const MAX_RAINDEX_ORDERS_PAGE_SIZE: u32 = 100; /// Bounds the rows SQLite may need to sort and skip for one history request. @@ -121,7 +120,7 @@ const MAX_TRANSFER_HISTORY_OFFSET: usize = 10_000; /// Upper bound on ERROR/WARN log entries aggregated per reliability report. const MAX_RELIABILITY_LOG_ENTRIES: usize = 50_000; -const GIT_COMMIT: &str = match option_env!("ST0X_GIT_COMMIT") { +pub(crate) const GIT_COMMIT: &str = match option_env!("ST0X_GIT_COMMIT") { Some(val) => val, None => "dev", }; @@ -246,7 +245,7 @@ impl Serialize for ProtocolTrades<'_> { } async fn health(State(state): State) -> (StatusCode, Json) { - let uptime = Utc::now() - *STARTED_AT; + let uptime = Utc::now() - DateTime::::from(*PROCESS_START); // Gated on the startup barrier: every essential run loop has // acknowledged before this reports healthy, and the conductor diff --git a/src/bin/server.rs b/src/bin/server.rs index d69c570f35..e21a8cd332 100644 --- a/src/bin/server.rs +++ b/src/bin/server.rs @@ -1,13 +1,15 @@ use clap::Parser; +use std::sync::LazyLock; use st0x_config::{Ctx, Env, TokenSource, claim_boot_tokens}; use st0x_hedge::{ - apalis_board_tracing_layer, install_tls_crypto_provider, report_registry_boot, + PROCESS_START, apalis_board_tracing_layer, install_tls_crypto_provider, report_registry_boot, run_server_bot_session, setup_tracing, }; #[tokio::main] async fn main() -> anyhow::Result<()> { + LazyLock::force(&PROCESS_START); install_tls_crypto_provider(); let Env { diff --git a/src/lib.rs b/src/lib.rs index 403ecf3ccd..cbeeb4c624 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -94,6 +94,7 @@ pub use registry_boot::report_boot as report_registry_boot; pub use st0x_config::{ ExtraLayer, FileLogGuard, TelemetryError, TelemetryGuard, mk_env_filter, setup_tracing, }; +pub use startup::PROCESS_START; pub use tls::install_tls_crypto_provider; #[cfg(any(test, feature = "test-support"))] diff --git a/src/startup.rs b/src/startup.rs index 900f6f830a..dac16eb064 100644 --- a/src/startup.rs +++ b/src/startup.rs @@ -12,14 +12,22 @@ use std::future::Future; use std::io; use std::path::PathBuf; use std::pin::Pin; -use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, LazyLock}; use std::task::{Context as TaskContext, Poll}; +use std::time::SystemTime; use task_supervisor::{SupervisedTask, TaskResult}; use tokio::sync::Notify; const READY_FILE_ENV: &str = "ST0X_STARTUP_READY_FILE"; +/// When this process started. +/// +/// The server binary forces it before loading the config, so `/health` uptime +/// and `liq_bot_start_timestamp_seconds` count from process start, not from the +/// first request that reads them. +pub static PROCESS_START: LazyLock = LazyLock::new(SystemTime::now); + pub(crate) trait StartupNotifier: Send + Sync { fn notify_ready(&self) -> io::Result<()>; }