diff --git a/docs/observability.md b/docs/observability.md index a8fca50be..4f7ed0f92 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -277,11 +277,34 @@ its `pnl_` collector time stops advancing. compares samples only. Extend `PORTED` and, where needed, `KNOWN_DIFFS` in `scripts/liq-parity/compare.py`. -### Board pins - -Until consumers move to the bot, every `liq_` selector on the liquidity boards -reads `job="t0-liquidity-exporter"`. `observability/gen-t0-liquidity.py` adds -the matcher and fails if any `liq_` selector is left unpinned. +### Board source + +Both the exporter sidecar and the bot serve `liq_*`, so a selector without a job +would add the two together. The liquidity boards have a `Source` variable, +`$source`, and every `liq_` selector on them reads `job=~"$source"`: + +| Source | Job matcher | +| -------------------- | ------------------------------------ | +| `exporter` (default) | `t0-liquidity-exporter` | +| `bot` | `t0-liquidity\|t0-liquidity-staging` | + +The bot's job is `t0-liquidity` in production and `t0-liquidity-staging` in +staging. `$env` already picks the project, so one regex covers both. PromQL +anchors a regex matcher, so `bot` never matches the exporter job. + +The bot never emits `liq_up`, so the header's Bot pill reads the exporter's +`liq_up` on the exporter source and the bot scrape's own `up` on the bot source: +`max(liq_up{job=~"$source"}) or max(up{job=~"$source",job!="t0-liquidity-exporter"})`. +A stopped bot shows red on either source. A stopped exporter shows no data, not +red, because the exporter's own `up` says nothing about the bot. + +Pick `bot` to compare the two side by side. Stage 4 of the migration flips the +default to `bot` in `SOURCE_VAR`, a one-line change; a later stage removes the +variable with the exporter. The log tables (Trades, Rebalances, Logs) read the +exporter's log names on either source until the bot writes its own. + +`observability/gen-t0-liquidity.py` adds the matcher and fails if any `liq_` +selector is left without exactly `job=~"$source"`. `python3 observability/gen-t0-liquidity.py --check` (run in CI) fails when the committed board JSON differs from the generator output. diff --git a/observability/README.md b/observability/README.md index 7c2a94c87..a73f57d10 100644 --- a/observability/README.md +++ b/observability/README.md @@ -27,6 +27,14 @@ The Liquidity bot board and its four tab boards are generated: edit Do not edit those files by hand or replace them with a Grafana export; the next run of the generator overwrites them. +The liquidity boards read `liq_*` from one source at a time, picked by the +`Source` variable: the exporter sidecar (`job="t0-liquidity-exporter"`, the +default) or the bot's own `/metrics` (`t0-liquidity` in production, +`t0-liquidity-staging` in staging). The generator writes `job=~"$source"` into +every `liq_` selector and fails on one without it, because an unpinned selector +adds both sources together. Stage 4 of the migration makes `bot` the default +(`SOURCE_VAR` in the generator). See `docs/observability.md`, "Board source". + `alerting/liquidity.rules.yml` is the alert rules, in the `Alerts` folder. Each rule's `uid` is permanent and must be unique across every repo's file. Removing a rule from the file does not remove it from Grafana: add its `uid` to diff --git a/observability/dashboards/liquidity/t0-liquidity.json b/observability/dashboards/liquidity/t0-liquidity.json index e3fa4d61b..a48ee0e25 100644 --- a/observability/dashboards/liquidity/t0-liquidity.json +++ b/observability/dashboards/liquidity/t0-liquidity.json @@ -44,6 +44,33 @@ "value": "t0-liquidity-staging" } ] + }, + { + "name": "source", + "type": "custom", + "label": "Source", + "description": "Which process serves the liq_* metrics: the exporter sidecar (the default for now) or the bot's own /metrics. Pick bot to compare the two.", + "query": "exporter : t0-liquidity-exporter, bot : t0-liquidity|t0-liquidity-staging", + "includeAll": false, + "multi": false, + "hide": 0, + "current": { + "selected": true, + "text": "exporter", + "value": "t0-liquidity-exporter" + }, + "options": [ + { + "selected": true, + "text": "exporter", + "value": "t0-liquidity-exporter" + }, + { + "selected": false, + "text": "bot", + "value": "t0-liquidity|t0-liquidity-staging" + } + ] } ] }, @@ -127,7 +154,7 @@ "id": 1, "type": "stat", "title": "", - "description": "The SPA header's connection light: is the bot process up and answering /health? '\u2014' means the exporter itself is not reporting.", + "description": "The SPA header's connection light: is the bot process up and answering? The exporter source reads liq_up, which is 0 when the exporter cannot reach the bot's /health. The bot never emits liq_up, so the bot source reads the scrape's own up. '\u2014' means the source is not reporting.", "datasource": { "type": "stackdriver", "uid": "cloudmon" @@ -142,7 +169,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -244,7 +271,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=\"t0-liquidity-exporter\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", + "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=~\"$source\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", "step": "60s" }, "timeSeriesList": { @@ -331,7 +358,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds{job=\"t0-liquidity-exporter\"})", + "expr": "time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -419,7 +446,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info{job=\"t0-liquidity-exporter\"})", + "expr": "sum by (broker) (liq_settings_info{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -506,7 +533,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_equity_target{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_equity_target{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -594,7 +621,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_usdc_target{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_usdc_target{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -682,7 +709,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_execution_threshold_usd{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -770,7 +797,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (row, col) (label_replace(label_replace(liq_usdc_onchain_available{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Raindex\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_inflight_total{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Inflight\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_alpaca_total{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Alpaca Total\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_total{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Total\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_ratio{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Ratio\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_ratio_deviation{job=\"t0-liquidity-exporter\"} and liq_usdc_total{job=\"t0-liquidity-exporter\"} > 0, \"row\", \"USDC\", \"\", \"\"), \"col\", \"\u0394\", \"\", \"\"))", + "expr": "sum by (row, col) (label_replace(label_replace(liq_usdc_onchain_available{job=~\"$source\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Raindex\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_inflight_total{job=~\"$source\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Inflight\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_alpaca_total{job=~\"$source\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Alpaca Total\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_total{job=~\"$source\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Total\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_ratio{job=~\"$source\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Ratio\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_ratio_deviation{job=~\"$source\"} and liq_usdc_total{job=~\"$source\"} > 0, \"row\", \"USDC\", \"\", \"\"), \"col\", \"\u0394\", \"\", \"\"))", "step": "60s" }, "timeSeriesList": { @@ -1084,7 +1111,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (row, col) (label_replace(label_replace(liq_usdc_alpaca_usdc{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"USDC\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_rebalanceable{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Rebalanceable\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_alpaca_total{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Counter-tradeable\", \"\", \"\"))", + "expr": "sum by (row, col) (label_replace(label_replace(liq_usdc_alpaca_usdc{job=~\"$source\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"USDC\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_rebalanceable{job=~\"$source\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Rebalanceable\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_alpaca_total{job=~\"$source\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Counter-tradeable\", \"\", \"\"))", "step": "60s" }, "timeSeriesList": { @@ -1267,7 +1294,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (row, col) (label_replace(label_replace(liq_usdc_inflight_ethereum_wallet{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Eth\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_inflight_base_wallet{job=\"t0-liquidity-exporter\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Base\", \"\", \"\"))", + "expr": "sum by (row, col) (label_replace(label_replace(liq_usdc_inflight_ethereum_wallet{job=~\"$source\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Eth\", \"\", \"\")) or sum by (row, col) (label_replace(label_replace(liq_usdc_inflight_base_wallet{job=~\"$source\"}, \"row\", \"USDC\", \"\", \"\"), \"col\", \"Base\", \"\", \"\"))", "step": "60s" }, "timeSeriesList": { @@ -1429,7 +1456,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (symbol, col) (label_replace(liq_asset_counter_trading{job=\"t0-liquidity-exporter\"}, \"col\", \"CT\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_asset_rebalancing{job=\"t0-liquidity-exporter\"}, \"col\", \"Rebal\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_asset_extended_hours{job=\"t0-liquidity-exporter\"}, \"col\", \"Ext\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_onchain_available{job=\"t0-liquidity-exporter\"}, \"col\", \"Raindex\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_inflight_total{job=\"t0-liquidity-exporter\"}, \"col\", \"Inflight\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_offchain_available{job=\"t0-liquidity-exporter\"}, \"col\", \"Alpaca\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_total{job=\"t0-liquidity-exporter\"}, \"col\", \"Total\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_ratio{job=\"t0-liquidity-exporter\"}, \"col\", \"Ratio\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_ratio_deviation{job=\"t0-liquidity-exporter\"} and liq_equity_total{job=\"t0-liquidity-exporter\"} > 0, \"col\", \"\u0394\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_exposure_usd{job=\"t0-liquidity-exporter\"} or (liq_equity_total{job=\"t0-liquidity-exporter\"} * NaN), \"col\", \"Exposure\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_unwrapped{job=\"t0-liquidity-exporter\"}, \"col\", \"Unwrapped\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_wrapped{job=\"t0-liquidity-exporter\"}, \"col\", \"Wrapped\", \"\", \"\"))", + "expr": "sum by (symbol, col) (label_replace(liq_asset_counter_trading{job=~\"$source\"}, \"col\", \"CT\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_asset_rebalancing{job=~\"$source\"}, \"col\", \"Rebal\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_asset_extended_hours{job=~\"$source\"}, \"col\", \"Ext\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_onchain_available{job=~\"$source\"}, \"col\", \"Raindex\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_inflight_total{job=~\"$source\"}, \"col\", \"Inflight\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_offchain_available{job=~\"$source\"}, \"col\", \"Alpaca\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_total{job=~\"$source\"}, \"col\", \"Total\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_ratio{job=~\"$source\"}, \"col\", \"Ratio\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_ratio_deviation{job=~\"$source\"} and liq_equity_total{job=~\"$source\"} > 0, \"col\", \"\u0394\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_exposure_usd{job=~\"$source\"} or (liq_equity_total{job=~\"$source\"} * NaN), \"col\", \"Exposure\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_unwrapped{job=~\"$source\"}, \"col\", \"Unwrapped\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_equity_wrapped{job=~\"$source\"}, \"col\", \"Wrapped\", \"\", \"\"))", "step": "60s" }, "timeSeriesList": { diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json b/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json index d4e472d9e..615f155ec 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-logs.json @@ -45,6 +45,33 @@ } ] }, + { + "name": "source", + "type": "custom", + "label": "Source", + "description": "Which process serves the liq_* metrics: the exporter sidecar (the default for now) or the bot's own /metrics. Pick bot to compare the two.", + "query": "exporter : t0-liquidity-exporter, bot : t0-liquidity|t0-liquidity-staging", + "includeAll": false, + "multi": false, + "hide": 0, + "current": { + "selected": true, + "text": "exporter", + "value": "t0-liquidity-exporter" + }, + "options": [ + { + "selected": true, + "text": "exporter", + "value": "t0-liquidity-exporter" + }, + { + "selected": false, + "text": "bot", + "value": "t0-liquidity|t0-liquidity-staging" + } + ] + }, { "name": "level", "type": "custom", @@ -289,7 +316,7 @@ "id": 108, "type": "stat", "title": "", - "description": "The SPA header's connection light: is the bot process up and answering /health? '\u2014' means the exporter itself is not reporting.", + "description": "The SPA header's connection light: is the bot process up and answering? The exporter source reads liq_up, which is 0 when the exporter cannot reach the bot's /health. The bot never emits liq_up, so the bot source reads the scrape's own up. '\u2014' means the source is not reporting.", "datasource": { "type": "stackdriver", "uid": "cloudmon" @@ -304,7 +331,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -406,7 +433,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=\"t0-liquidity-exporter\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", + "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=~\"$source\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", "step": "60s" }, "timeSeriesList": { @@ -493,7 +520,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds{job=\"t0-liquidity-exporter\"})", + "expr": "time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -581,7 +608,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info{job=\"t0-liquidity-exporter\"})", + "expr": "sum by (broker) (liq_settings_info{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -668,7 +695,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_equity_target{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_equity_target{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -756,7 +783,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_usdc_target{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_usdc_target{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -844,7 +871,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_execution_threshold_usd{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json b/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json index 9cda52ec7..8057dc6d6 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-orders.json @@ -44,6 +44,33 @@ "value": "t0-liquidity-staging" } ] + }, + { + "name": "source", + "type": "custom", + "label": "Source", + "description": "Which process serves the liq_* metrics: the exporter sidecar (the default for now) or the bot's own /metrics. Pick bot to compare the two.", + "query": "exporter : t0-liquidity-exporter, bot : t0-liquidity|t0-liquidity-staging", + "includeAll": false, + "multi": false, + "hide": 0, + "current": { + "selected": true, + "text": "exporter", + "value": "t0-liquidity-exporter" + }, + "options": [ + { + "selected": true, + "text": "exporter", + "value": "t0-liquidity-exporter" + }, + { + "selected": false, + "text": "bot", + "value": "t0-liquidity|t0-liquidity-staging" + } + ] } ] }, @@ -127,7 +154,7 @@ "id": 14, "type": "stat", "title": "", - "description": "The SPA header's connection light: is the bot process up and answering /health? '\u2014' means the exporter itself is not reporting.", + "description": "The SPA header's connection light: is the bot process up and answering? The exporter source reads liq_up, which is 0 when the exporter cannot reach the bot's /health. The bot never emits liq_up, so the bot source reads the scrape's own up. '\u2014' means the source is not reporting.", "datasource": { "type": "stackdriver", "uid": "cloudmon" @@ -142,7 +169,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -244,7 +271,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=\"t0-liquidity-exporter\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", + "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=~\"$source\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", "step": "60s" }, "timeSeriesList": { @@ -331,7 +358,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds{job=\"t0-liquidity-exporter\"})", + "expr": "time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -419,7 +446,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info{job=\"t0-liquidity-exporter\"})", + "expr": "sum by (broker) (liq_settings_info{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -506,7 +533,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_equity_target{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_equity_target{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -594,7 +621,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_usdc_target{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_usdc_target{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -682,7 +709,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_execution_threshold_usd{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -1055,7 +1082,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pending_orders_total{job=\"t0-liquidity-exporter\"}) or vector(0)", + "expr": "max(liq_pending_orders_total{job=~\"$source\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1137,7 +1164,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_raindex_orders_total{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_raindex_orders_total{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -1219,7 +1246,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_raindex_orders_unavailable{job=\"t0-liquidity-exporter\"}) or vector(0)", + "expr": "max(liq_raindex_orders_unavailable{job=~\"$source\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json b/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json index 35c4cb036..f6c6754dd 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-performance.json @@ -45,6 +45,33 @@ } ] }, + { + "name": "source", + "type": "custom", + "label": "Source", + "description": "Which process serves the liq_* metrics: the exporter sidecar (the default for now) or the bot's own /metrics. Pick bot to compare the two.", + "query": "exporter : t0-liquidity-exporter, bot : t0-liquidity|t0-liquidity-staging", + "includeAll": false, + "multi": false, + "hide": 0, + "current": { + "selected": true, + "text": "exporter", + "value": "t0-liquidity-exporter" + }, + "options": [ + { + "selected": true, + "text": "exporter", + "value": "t0-liquidity-exporter" + }, + { + "selected": false, + "text": "bot", + "value": "t0-liquidity|t0-liquidity-staging" + } + ] + }, { "name": "stage", "type": "custom", @@ -168,7 +195,7 @@ "id": 52, "type": "stat", "title": "", - "description": "The SPA header's connection light: is the bot process up and answering /health? '\u2014' means the exporter itself is not reporting.", + "description": "The SPA header's connection light: is the bot process up and answering? The exporter source reads liq_up, which is 0 when the exporter cannot reach the bot's /health. The bot never emits liq_up, so the bot source reads the scrape's own up. '\u2014' means the source is not reporting.", "datasource": { "type": "stackdriver", "uid": "cloudmon" @@ -183,7 +210,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -285,7 +312,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=\"t0-liquidity-exporter\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", + "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=~\"$source\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", "step": "60s" }, "timeSeriesList": { @@ -372,7 +399,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds{job=\"t0-liquidity-exporter\"})", + "expr": "time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -460,7 +487,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info{job=\"t0-liquidity-exporter\"})", + "expr": "sum by (broker) (liq_settings_info{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -547,7 +574,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_equity_target{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_equity_target{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -635,7 +662,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_usdc_target{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_usdc_target{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -723,7 +750,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_execution_threshold_usd{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -811,7 +838,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"detection\",quantile=\"p95\"})", + "expr": "max(liq_hedge_latency_ms{job=~\"$source\",stage=\"detection\",quantile=\"p95\"})", "step": "60s" }, "timeSeriesList": { @@ -900,7 +927,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"exposure_window\",quantile=\"p95\"})", + "expr": "max(liq_hedge_latency_ms{job=~\"$source\",stage=\"exposure_window\",quantile=\"p95\"})", "step": "60s" }, "timeSeriesList": { @@ -989,7 +1016,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_reliability_log_count_24h{job=\"t0-liquidity-exporter\",level=\"error\"}) or vector(0)", + "expr": "max(liq_reliability_log_count_24h{job=~\"$source\",level=\"error\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1078,7 +1105,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - min(liq_open_exposure_oldest_ts_seconds{job=\"t0-liquidity-exporter\"})", + "expr": "time() - min(liq_open_exposure_oldest_ts_seconds{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -1167,7 +1194,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_block_lag_blocks{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_block_lag_blocks{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -1256,7 +1283,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"detection\",quantile=\"p50\"})", + "expr": "max(liq_hedge_latency_ms{job=~\"$source\",stage=\"detection\",quantile=\"p50\"})", "step": "60s" }, "timeSeriesList": { @@ -1342,7 +1369,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms_samples{job=\"t0-liquidity-exporter\",stage=\"detection\"}) or vector(0)", + "expr": "max(liq_hedge_latency_ms_samples{job=~\"$source\",stage=\"detection\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1428,7 +1455,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"exposure_window\",quantile=\"p50\"})", + "expr": "max(liq_hedge_latency_ms{job=~\"$source\",stage=\"exposure_window\",quantile=\"p50\"})", "step": "60s" }, "timeSeriesList": { @@ -1514,7 +1541,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms_samples{job=\"t0-liquidity-exporter\",stage=\"exposure_window\"}) or vector(0)", + "expr": "max(liq_hedge_latency_ms_samples{job=~\"$source\",stage=\"exposure_window\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1600,7 +1627,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_reliability_log_count_24h{job=\"t0-liquidity-exporter\",level=\"warning\"}) or vector(0)", + "expr": "max(liq_reliability_log_count_24h{job=~\"$source\",level=\"warning\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1686,7 +1713,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum(liq_failure_event_count_24h{job=\"t0-liquidity-exporter\"}) or vector(0)", + "expr": "sum(liq_failure_event_count_24h{job=~\"$source\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1772,7 +1799,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "count(sum by (job_type) (liq_job_queue{job=\"t0-liquidity-exporter\",state=~\"failed|killed\"}) > 0) or vector(0)", + "expr": "count(sum by (job_type) (liq_job_queue{job=~\"$source\",state=~\"failed|killed\"}) > 0) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1862,7 +1889,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum(liq_open_exposure_fill_count{job=\"t0-liquidity-exporter\"}) or vector(0)", + "expr": "sum(liq_open_exposure_fill_count{job=~\"$source\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -1948,7 +1975,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "count(liq_open_exposure_fill_count{job=\"t0-liquidity-exporter\"} > 0) or vector(0)", + "expr": "count(liq_open_exposure_fill_count{job=~\"$source\"} > 0) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -2044,7 +2071,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - max(liq_block_lag_sampled_ts_seconds{job=\"t0-liquidity-exporter\"})", + "expr": "time() - max(liq_block_lag_sampled_ts_seconds{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -2130,7 +2157,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_poll_skipped_ticks_24h{job=\"t0-liquidity-exporter\"}) or vector(0)", + "expr": "max(liq_poll_skipped_ticks_24h{job=~\"$source\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -2216,7 +2243,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (stage) (liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",quantile=\"p95\"})", + "expr": "max by (stage) (liq_hedge_latency_ms{job=~\"$source\",quantile=\"p95\"})", "step": "60s" }, "timeSeriesList": { @@ -2294,7 +2321,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"$stage\",quantile=\"p50\"})", + "expr": "max(liq_hedge_latency_ms{job=~\"$source\",stage=\"$stage\",quantile=\"p50\"})", "step": "60s" }, "timeSeriesList": { @@ -2314,7 +2341,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"$stage\",quantile=\"p90\"})", + "expr": "max(liq_hedge_latency_ms{job=~\"$source\",stage=\"$stage\",quantile=\"p90\"})", "step": "60s" }, "timeSeriesList": { @@ -2334,7 +2361,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_hedge_latency_ms{job=\"t0-liquidity-exporter\",stage=\"$stage\",quantile=\"p99\"})", + "expr": "max(liq_hedge_latency_ms{job=~\"$source\",stage=\"$stage\",quantile=\"p99\"})", "step": "60s" }, "timeSeriesList": { @@ -2450,7 +2477,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "label_replace(liq_log_target_count_24h{job=\"t0-liquidity-exporter\"}, \"col\", \"$1\", \"level\", \"(.*)\")", + "expr": "label_replace(liq_log_target_count_24h{job=~\"$source\"}, \"col\", \"$1\", \"level\", \"(.*)\")", "step": "60s" }, "timeSeriesList": { @@ -2655,7 +2682,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (stage) (liq_rebalance_stage_ms{job=\"t0-liquidity-exporter\",kind=\"usdc\",quantile=\"p95\"})", + "expr": "max by (stage) (liq_rebalance_stage_ms{job=~\"$source\",kind=\"usdc\",quantile=\"p95\"})", "step": "60s" }, "timeSeriesList": { @@ -2725,7 +2752,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (stage) (liq_rebalance_stage_ms{job=\"t0-liquidity-exporter\",kind=\"equity\",quantile=\"p95\"})", + "expr": "max by (stage) (liq_rebalance_stage_ms{job=~\"$source\",kind=\"equity\",quantile=\"p95\"})", "step": "60s" }, "timeSeriesList": { @@ -2795,7 +2822,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "label_join(label_replace(liq_dependency_calls_24h{job=\"t0-liquidity-exporter\"}, \"col\", \"calls\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_errors_24h{job=\"t0-liquidity-exporter\"}, \"col\", \"errors\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_latency_ms{job=\"t0-liquidity-exporter\",quantile=\"p50\"}, \"col\", \"p50 ms\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_latency_ms{job=\"t0-liquidity-exporter\",quantile=\"p95\"}, \"col\", \"p95 ms\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_latency_ms{job=\"t0-liquidity-exporter\",quantile=\"max\"}, \"col\", \"max ms\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\")", + "expr": "label_join(label_replace(liq_dependency_calls_24h{job=~\"$source\"}, \"col\", \"calls\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_errors_24h{job=~\"$source\"}, \"col\", \"errors\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_latency_ms{job=~\"$source\",quantile=\"p50\"}, \"col\", \"p50 ms\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_latency_ms{job=~\"$source\",quantile=\"p95\"}, \"col\", \"p95 ms\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\") or label_join(label_replace(liq_dependency_latency_ms{job=~\"$source\",quantile=\"max\"}, \"col\", \"max ms\", \"\", \"\"), \"dep\", \" \u00b7 \", \"dependency\", \"operation\")", "step": "60s" }, "timeSeriesList": { @@ -2955,7 +2982,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_attestation_last_ms{job=\"t0-liquidity-exporter\",kind=\"usdc\"})", + "expr": "max(liq_attestation_last_ms{job=~\"$source\",kind=\"usdc\"})", "step": "60s" }, "timeSeriesList": { @@ -2975,7 +3002,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_attestation_last_ms{job=\"t0-liquidity-exporter\",kind=\"equity\"})", + "expr": "max(liq_attestation_last_ms{job=~\"$source\",kind=\"equity\"})", "step": "60s" }, "timeSeriesList": { @@ -3076,7 +3103,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_block_lag_blocks{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_block_lag_blocks{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -3147,7 +3174,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_poll_cycles_24h{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_poll_cycles_24h{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -3228,7 +3255,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_poll_errors_24h{job=\"t0-liquidity-exporter\"}) or vector(0)", + "expr": "max(liq_poll_errors_24h{job=~\"$source\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -3313,7 +3340,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_poll_skipped_ticks_24h{job=\"t0-liquidity-exporter\"}) or vector(0)", + "expr": "max(liq_poll_skipped_ticks_24h{job=~\"$source\"}) or vector(0)", "step": "60s" }, "timeSeriesList": { @@ -3398,7 +3425,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (quantile) (liq_poll_duration_ms{job=\"t0-liquidity-exporter\"})", + "expr": "max by (quantile) (liq_poll_duration_ms{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -3468,7 +3495,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "label_replace(liq_job_queue{job=\"t0-liquidity-exporter\"}, \"col\", \"$1\", \"state\", \"(.*)\")", + "expr": "label_replace(liq_job_queue{job=~\"$source\"}, \"col\", \"$1\", \"state\", \"(.*)\")", "step": "60s" }, "timeSeriesList": { diff --git a/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json b/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json index 8205d2ec9..71fadf3b5 100644 --- a/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json +++ b/observability/dashboards/liquidity/tabs/t0-liquidity-pnl.json @@ -45,6 +45,33 @@ } ] }, + { + "name": "source", + "type": "custom", + "label": "Source", + "description": "Which process serves the liq_* metrics: the exporter sidecar (the default for now) or the bot's own /metrics. Pick bot to compare the two.", + "query": "exporter : t0-liquidity-exporter, bot : t0-liquidity|t0-liquidity-staging", + "includeAll": false, + "multi": false, + "hide": 0, + "current": { + "selected": true, + "text": "exporter", + "value": "t0-liquidity-exporter" + }, + "options": [ + { + "selected": true, + "text": "exporter", + "value": "t0-liquidity-exporter" + }, + { + "selected": false, + "text": "bot", + "value": "t0-liquidity|t0-liquidity-staging" + } + ] + }, { "name": "window", "type": "custom", @@ -174,7 +201,7 @@ "id": 25, "type": "stat", "title": "", - "description": "The SPA header's connection light: is the bot process up and answering /health? '\u2014' means the exporter itself is not reporting.", + "description": "The SPA header's connection light: is the bot process up and answering? The exporter source reads liq_up, which is 0 when the exporter cannot reach the bot's /health. The bot never emits liq_up, so the bot source reads the scrape's own up. '\u2014' means the source is not reporting.", "datasource": { "type": "stackdriver", "uid": "cloudmon" @@ -189,7 +216,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_up{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_up{job=~\"$source\"}) or max(up{job=~\"$source\",job!=\"t0-liquidity-exporter\"})", "step": "60s" }, "timeSeriesList": { @@ -291,7 +318,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=\"t0-liquidity-exporter\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", + "expr": "max by (git_commit) (topk(1, label_replace(timestamp(liq_bot_info{job=~\"$source\"}), \"git_commit\", \"$1\", \"git_commit\", \"(.{7}).*\")))", "step": "60s" }, "timeSeriesList": { @@ -378,7 +405,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "time() - max(liq_bot_start_timestamp_seconds{job=\"t0-liquidity-exporter\"})", + "expr": "time() - max(liq_bot_start_timestamp_seconds{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -466,7 +493,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (broker) (liq_settings_info{job=\"t0-liquidity-exporter\"})", + "expr": "sum by (broker) (liq_settings_info{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -553,7 +580,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_equity_target{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_equity_target{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -641,7 +668,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_usdc_target{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_usdc_target{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -729,7 +756,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_settings_execution_threshold_usd{job=\"t0-liquidity-exporter\"})", + "expr": "max(liq_settings_execution_threshold_usd{job=~\"$source\"})", "step": "60s" }, "timeSeriesList": { @@ -817,7 +844,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"net_realized\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=~\"$source\",stream=\"net_realized\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -911,7 +938,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"gross_realized\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=~\"$source\",stream=\"gross_realized\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1005,7 +1032,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"tracked_costs\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=~\"$source\",stream=\"tracked_costs\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1091,7 +1118,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"tracked_revenue\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=~\"$source\",stream=\"tracked_revenue\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1185,7 +1212,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"counter_trade\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=~\"$source\",stream=\"counter_trade\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1279,7 +1306,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"onchain_netting\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=~\"$source\",stream=\"onchain_netting\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1373,7 +1400,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"directional_imbalance_excess\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=~\"$source\",stream=\"directional_imbalance_excess\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1467,7 +1494,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"directional_inventory_baseline\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=~\"$source\",stream=\"directional_inventory_baseline\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1561,7 +1588,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"directional_exposure\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=~\"$source\",stream=\"directional_exposure\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1655,7 +1682,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_usd{job=\"t0-liquidity-exporter\",stream=\"total\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_usd{job=~\"$source\",stream=\"total\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1749,7 +1776,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_summary_shares{job=\"t0-liquidity-exporter\",kind=\"inventory_drift\",window=\"$window\"})", + "expr": "max(liq_pnl_summary_shares{job=~\"$source\",kind=\"inventory_drift\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1831,7 +1858,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_sample_total_fills{job=\"t0-liquidity-exporter\",window=\"$window\"})", + "expr": "max(liq_pnl_sample_total_fills{job=~\"$source\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1913,7 +1940,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_capital_avg_deployed_usd{job=\"t0-liquidity-exporter\",window=\"$window\"})", + "expr": "max(liq_pnl_capital_avg_deployed_usd{job=~\"$source\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -1995,7 +2022,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_capital_annualized_return_pct{job=\"t0-liquidity-exporter\",window=\"$window\"})", + "expr": "max(liq_pnl_capital_annualized_return_pct{job=~\"$source\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -2085,7 +2112,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "max(liq_pnl_capital_coverage_days{job=\"t0-liquidity-exporter\",window=\"$window\"})", + "expr": "max(liq_pnl_capital_coverage_days{job=~\"$source\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -2167,7 +2194,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"gross_realized\",window=\"$window\"}, \"col\", \"Gross\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"tracked_costs\",window=\"$window\"}, \"col\", \"Costs\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"tracked_revenue\",window=\"$window\"}, \"col\", \"Revenue\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"net_realized\",window=\"$window\"}, \"col\", \"Net\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"counter_trade\",window=\"$window\"}, \"col\", \"Counter\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"onchain_netting\",window=\"$window\"}, \"col\", \"On-chain\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"directional_imbalance_excess\",window=\"$window\"}, \"col\", \"Directional\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"directional_inventory_baseline\",window=\"$window\"}, \"col\", \"Baseline Drift\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=\"t0-liquidity-exporter\",col=\"total\",window=\"$window\"}, \"col\", \"Total\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_lots{job=\"t0-liquidity-exporter\",window=\"$window\"}, \"col\", \"Lots\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_volume_shares{job=\"t0-liquidity-exporter\",window=\"$window\"}, \"col\", \"Volume\", \"\", \"\"))", + "expr": "sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=~\"$source\",col=\"gross_realized\",window=\"$window\"}, \"col\", \"Gross\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=~\"$source\",col=\"tracked_costs\",window=\"$window\"}, \"col\", \"Costs\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=~\"$source\",col=\"tracked_revenue\",window=\"$window\"}, \"col\", \"Revenue\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=~\"$source\",col=\"net_realized\",window=\"$window\"}, \"col\", \"Net\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=~\"$source\",col=\"counter_trade\",window=\"$window\"}, \"col\", \"Counter\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=~\"$source\",col=\"onchain_netting\",window=\"$window\"}, \"col\", \"On-chain\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=~\"$source\",col=\"directional_imbalance_excess\",window=\"$window\"}, \"col\", \"Directional\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=~\"$source\",col=\"directional_inventory_baseline\",window=\"$window\"}, \"col\", \"Baseline Drift\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_usd{job=~\"$source\",col=\"total\",window=\"$window\"}, \"col\", \"Total\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_lots{job=~\"$source\",window=\"$window\"}, \"col\", \"Lots\", \"\", \"\")) or sum by (symbol, col) (label_replace(liq_pnl_symbol_volume_shares{job=~\"$source\",window=\"$window\"}, \"col\", \"Volume\", \"\", \"\"))", "step": "60s" }, "timeSeriesList": { @@ -2697,7 +2724,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (day, symbol) (liq_pnl_day_usd{job=\"t0-liquidity-exporter\",window=\"$window\"})", + "expr": "sum by (day, symbol) (liq_pnl_day_usd{job=~\"$source\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -2844,7 +2871,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (day, stream) (liq_pnl_day_stream_usd{job=\"t0-liquidity-exporter\",window=\"$window\"})", + "expr": "sum by (day, stream) (liq_pnl_day_stream_usd{job=~\"$source\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -3052,7 +3079,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (day, symbol) (liq_pnl_day_cum_usd{job=\"t0-liquidity-exporter\",window=\"$window\"})", + "expr": "sum by (day, symbol) (liq_pnl_day_cum_usd{job=~\"$source\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { @@ -3199,7 +3226,7 @@ "queryType": "promQL", "promQLQuery": { "projectName": "$env", - "expr": "sum by (day, stream) (liq_pnl_day_cum_stream_usd{job=\"t0-liquidity-exporter\",window=\"$window\"})", + "expr": "sum by (day, stream) (liq_pnl_day_cum_stream_usd{job=~\"$source\",window=\"$window\"})", "step": "60s" }, "timeSeriesList": { diff --git a/observability/gen-t0-liquidity.py b/observability/gen-t0-liquidity.py index 448bfeda6..034ae3e9b 100644 --- a/observability/gen-t0-liquidity.py +++ b/observability/gen-t0-liquidity.py @@ -11,7 +11,8 @@ python3 observability/gen-t0-liquidity.py --check Data comes from the exporter sidecar on the liquidity VM -(terraform/staging-liquidity/exporter/exporter.py): +(terraform/liquidity-exporter/exporter.py in T0Trade/t0.devops) or, for the +liq_* gauges, from the bot's own /metrics, as the `source` variable picks: cloudmon liq_* gauges via Ops Agent scrape -> managed prometheus cloudlog logNames liquidity-trades / liquidity-transfers / @@ -57,9 +58,15 @@ # The bot serves liq_* on its own /metrics too, so an unpinned liq_ selector # would sum the exporter and the bot together. Every liq_ selector on these -# boards reads the exporter's job until the board learns a source switch. -LIQ_JOB = "t0-liquidity-exporter" -LIQ_JOB_MATCHER = f'job="{LIQ_JOB}"' +# boards reads the one job the `source` variable picks (SOURCE_VAR). PromQL +# anchors a regex matcher, so the bot's value never matches the exporter job. +LIQ_JOB_MATCHER = 'job=~"$source"' +# The header's Bot light. The bot source has no liq_up and reads its scrape's +# up. The exporter source never falls back to up: the exporter's own up says +# nothing about the bot, so a stopped exporter shows no data, not red. +# pin_liq leaves both selectors alone, since each already names its job. +LIQ_UP = ('max(liq_up{job=~"$source"}) or ' + 'max(up{job=~"$source",job!="t0-liquidity-exporter"})') LIQ_NAME = re.compile(r"(? `liq_x{job="..."}`) and names that already carry matchers (`liq_x{a="b"}` -> `liq_x{job="...",a="b"}`). A selector @@ -108,7 +115,7 @@ def pin_liq(expr): def unpinned_liq_selectors(expr): """Every liq_ occurrence in `expr` that is not a selector pinned to - exactly LIQ_JOB. A templated name such as `liq_$col` is never a valid + exactly LIQ_JOB_MATCHER. A templated name such as `liq_$col` is never a valid selector, so it is reported too: the check fails closed.""" pinned = {match.start() for match in LIQ_NAME.finditer(expr) if _selector_is_pinned(expr, match.end())} @@ -725,9 +732,12 @@ def pill(title, desc, expr, display=None, unit="short", decimals=None, return [ pill("", "The SPA header's connection light: is the bot process up " - "and answering /health? '—' means the exporter itself is not " - "reporting.", - "max(liq_up)", display="Bot", + "and answering? The exporter source reads liq_up, which is 0 " + "when the exporter cannot reach the bot's /health. The bot " + "never emits liq_up, so the bot source reads the scrape's own " + "up. '—' means the source is not reporting.", + LIQ_UP, + display="Bot", mappings=[{"type": "value", "options": { "1": {"text": "Connected", "color": "green", "index": 0}, "0": {"text": "Disconnected", "color": "red", "index": 1}}}], @@ -791,6 +801,26 @@ def pill(title, desc, expr, display=None, unit="short", decimals=None, ], } +# Which job's liq_* series the boards read. The exporter sidecar is the +# default until Stage 4 of the migration flips it to the bot; the bot's job is +# t0-liquidity in production and t0-liquidity-staging in staging, and `env` +# already picks the project, so one regex covers both. +SOURCE_VAR = { + "name": "source", "type": "custom", "label": "Source", + "description": "Which process serves the liq_* metrics: the exporter " + "sidecar (the default for now) or the bot's own " + "/metrics. Pick bot to compare the two.", + "query": "exporter : t0-liquidity-exporter, " + "bot : t0-liquidity|t0-liquidity-staging", + "includeAll": False, "multi": False, "hide": 0, + "current": {"selected": True, "text": "exporter", + "value": "t0-liquidity-exporter"}, + "options": [{"selected": True, "text": "exporter", + "value": "t0-liquidity-exporter"}, + {"selected": False, "text": "bot", + "value": "t0-liquidity|t0-liquidity-staging"}], +} + ENV_VAR = { "name": "env", "type": "custom", "label": "Environment", "description": "GCP project id — both the Cloud Monitoring and Cloud " @@ -1134,7 +1164,7 @@ def column(name, px, mappings=None, color_text=False): "compute start-iap-tunnel 8080 " "--local-host-port=localhost:8080 --zone europe-west3-b " "--project $env (VM name = project id)", - panels, tab_links("Dashboard"), [ENV_VAR])) + panels, tab_links("Dashboard"), [ENV_VAR, SOURCE_VAR])) # ========================================================================== # Tab 2: Orders — the SPA's Raindex Orders table (+ ops extras). @@ -1188,7 +1218,7 @@ def column(name, px, mappings=None, color_text=False): dashboards.append(make_dashboard( "t0-liquidity-orders", "Liquidity bot: Orders", "The SPA's Orders tab.", - panels, tab_links("Orders"), [ENV_VAR])) + panels, tab_links("Orders"), [ENV_VAR, SOURCE_VAR])) # ========================================================================== # Tab 3: PnL — tiles, per-asset table, the four charts in SPA order. @@ -1360,7 +1390,7 @@ def column(name, px, mappings=None, color_text=False): "t0-liquidity-pnl", "Liquidity bot: PnL", "The SPA's PnL tab, tiles, table and charts all driven by the range " "pill; closed-lot and cost-ledger drilldowns live in the SPA.", - panels, tab_links("PnL"), [ENV_VAR, WINDOW_VAR])) + panels, tab_links("PnL"), [ENV_VAR, SOURCE_VAR, WINDOW_VAR])) # ========================================================================== # Tab 4: Performance — SLO cards, stage charts, health tables + the @@ -1691,7 +1721,7 @@ def column(name, px, mappings=None, color_text=False): "t0-liquidity-performance", "Liquidity bot: Performance", "The SPA's Performance tab, plus the pre-exporter native bot metrics " "as collapsed rows at the bottom.", - panels, tab_links("Performance"), [ENV_VAR, STAGE_VAR])) + panels, tab_links("Performance"), [ENV_VAR, SOURCE_VAR, STAGE_VAR])) # ========================================================================== # Tab 5: Logs. @@ -1817,7 +1847,7 @@ def column(name, px, mappings=None, color_text=False): "The SPA's Logs tab: level/category/target/search-filterable " "structured log lines.", panels, tab_links("Logs"), - [ENV_VAR, LEVEL_VAR, CATEGORY_VAR, TARGET_VAR, SEARCH_VAR])) + [ENV_VAR, SOURCE_VAR, LEVEL_VAR, CATEGORY_VAR, TARGET_VAR, SEARCH_VAR])) # ========================================================================== # Emit.