fix(telemetry): bound Prometheus distribution buffer via fork pin

When ELECTRIC_PROMETHEUS_PORT is set but the /metrics endpoint is scraped
infrequently or never (e.g. OTel-only deployments), telemetry_metrics_prometheus_core
buffers one ETS row per distribution observation and only drains on scrape, so
the dist table grows without bound (the per-transaction receive_lag metric
dominates). An 8GB+ ETS table and eventual OOM was observed in the field.

Pin telemetry_metrics_prometheus_core to a fork that bounds the buffer by
aggregating automatically on a size threshold (default 10k samples) and a time
fallback (default 60s), in addition to on scrape. Only affects the
MIX_TARGET=application build (the standalone sync-service / Docker image); the
telemetry deps are target-gated out of the Hex package, so this git dep does not
affect publishing of `electric`.

Upstream PR: beam-telemetry/telemetry_metrics_prometheus_core#77
Revert to the Hex release once it lands.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YW7Njz5ZpBDaoGviW1eVR8

Author: robacourt

.changeset/prometheus-buffer-bound.md

@@ -0,0 +1,5 @@
+---
+'@core/sync-service': patch
+---
+
+Fix unbounded memory growth in the Prometheus metrics reporter. When `ELECTRIC_PROMETHEUS_PORT` was set but the `/metrics` endpoint was scraped infrequently (or never — e.g. on OpenTelemetry-only deployments), distribution metrics such as `receive_lag` buffered one ETS row per observation without limit, eventually exhausting memory. Pins `telemetry_metrics_prometheus_core` to a fork that bounds the buffer by aggregating automatically on a size threshold and time fallback (upstream PR: https://github.com/beam-telemetry/telemetry_metrics_prometheus_core/pull/77).

packages/electric-telemetry/mix.exs

@@ -23,7 +23,13 @@ defmodule ElectricTelemetry.MixProject do
         {:req, "~> 0.5"},
         {:telemetry, "~> 1.3"},
         {:telemetry_metrics, "~> 1.1"},
-        {:telemetry_metrics_prometheus_core, "~> 1.2"},
+        # Temporary fork pin: bounds the distribution sample buffer so an enabled-but-unscraped
+        # Prometheus endpoint can't grow ETS without limit. Upstream PR:
+        # https://github.com/beam-telemetry/telemetry_metrics_prometheus_core/pull/77
+        # Revert to `{:telemetry_metrics_prometheus_core, "~> 1.2"}` once that lands on Hex.
+        {:telemetry_metrics_prometheus_core,
+         github: "robacourt/telemetry_metrics_prometheus_core",
+         branch: "feat/auto-flush-distributions"},
         {:telemetry_metrics_statsd, "~> 0.7"},
         {:telemetry_poller, "~> 1.3"}
       ],

packages/electric-telemetry/mix.lock

@@ -30,9 +30,9 @@
   "ranch": {:hex, :ranch, "1.8.1", "208169e65292ac5d333d6cdbad49388c1ae198136e4697ae2f474697140f201c", [:make, :rebar3], [], "hexpm", "aed58910f4e21deea992a67bf51632b6d60114895eb03bb392bb733064594dd0"},
   "req": {:hex, :req, "0.5.17", "0096ddd5b0ed6f576a03dde4b158a0c727215b15d2795e59e0916c6971066ede", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0b8bc6ffdfebbc07968e59d3ff96d52f2202d0536f10fef4dc11dc02a2a43e39"},
   "retry": {:hex, :retry, "0.19.0", "aeb326d87f62295d950f41e1255fe6f43280a1b390d36e280b7c9b00601ccbc2", [:mix], [], "hexpm", "85ef376aa60007e7bff565c366310966ec1bd38078765a0e7f20ec8a220d02ca"},
-  "telemetry": {:hex, :telemetry, "1.4.1", "ab6de178e2b29b58e8256b92b382ea3f590a47152ca3651ea857a6cae05ac423", [:rebar3], [], "hexpm", "2172e05a27531d3d31dd9782841065c50dd5c3c7699d95266b2edd54c2dafa1c"},
+  "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
   "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"},
-  "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"},
+  "telemetry_metrics_prometheus_core": {:git, "https://github.com/robacourt/telemetry_metrics_prometheus_core.git", "39539d83570a957caf4711b5dad85e86dfbee437", [branch: "feat/auto-flush-distributions"]},
   "telemetry_metrics_statsd": {:hex, :telemetry_metrics_statsd, "0.7.2", "a70cfaf821cb2f3ac2e767988461179add44762d1db752e74dfa0c93449b2857", [:mix], [{:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "933bb8b176b95d5088404b7137d94926b8dea9a74ef2c95d616f2740f1571c13"},
   "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},
 }

packages/sync-service/mix.lock

@@ -60,9 +60,9 @@
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
   "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"},
   "stream_split": {:hex, :stream_split, "0.1.7", "2d3fd1fd21697da7f91926768d65f79409086052c9ec7ae593987388f52425f8", [:mix], [], "hexpm", "1dc072ff507a64404a0ad7af90df97096183fee8eeac7b300320cea7c4679147"},
-  "telemetry": {:hex, :telemetry, "1.4.1", "ab6de178e2b29b58e8256b92b382ea3f590a47152ca3651ea857a6cae05ac423", [:rebar3], [], "hexpm", "2172e05a27531d3d31dd9782841065c50dd5c3c7699d95266b2edd54c2dafa1c"},
+  "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
   "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"},
-  "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"},
+  "telemetry_metrics_prometheus_core": {:git, "https://github.com/robacourt/telemetry_metrics_prometheus_core.git", "39539d83570a957caf4711b5dad85e86dfbee437", [branch: "feat/auto-flush-distributions"]},
   "telemetry_metrics_statsd": {:hex, :telemetry_metrics_statsd, "0.7.2", "a70cfaf821cb2f3ac2e767988461179add44762d1db752e74dfa0c93449b2857", [:mix], [{:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "933bb8b176b95d5088404b7137d94926b8dea9a74ef2c95d616f2740f1571c13"},
   "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},
   "thousand_island": {:hex, :thousand_island, "1.4.3", "2158209580f633be38d43ec4e3ce0a01079592b9657afff9080d5d8ca149a3af", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6e4ce09b0fd761a58594d02814d40f77daff460c48a7354a15ab353bb998ea0b"},