Alloy

.github/workflows/lint.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
-          python-version: 3.13
+          python-version: 3.14
           cache: 'pip'
       - name: Lint sources
         uses: pre-commit/action@v3.0.1

.github/workflows/test-grafana.yml

@@ -46,8 +46,8 @@ jobs:
         run: ./.github/check-service.sh json-exporter
       - name: Test cadvisor
         run: ./.github/check-service.sh cadvisor
-      - name: Test promtail
-        run: ./.github/check-service.sh promtail
+      - name: Test alloy
+        run: ./.github/check-service.sh alloy
       - name: Test Loki
         run: ./.github/check-service.sh loki
       - name: Test Grafana

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 ---
 default_language_version:
-  python: python3.13
+  python: python3.14
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks

alloy-obol/.gitignore

@@ -0,0 +1,4 @@
+*
+!*.sample
+!docker-entrypoint.sh
+!.gitignore

alloy-obol/config.alloy.sample

@@ -0,0 +1,120 @@
+discovery.docker "docker" {
+  host = "unix:///var/run/docker.sock"
+}
+
+// Logs
+loki.process "docker" {
+  forward_to = [loki.write.default.receiver]
+
+  stage.docker { }
+}
+
+discovery.relabel "docker" {
+  targets = discovery.docker.docker.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_label_obol_alloy_monitored"]
+    regex         = "true"
+    action        = "keep"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container"
+  }
+
+  rule {
+    source_labels = ["container"]
+    regex         = ".*charon.*"
+    target_label  = "job"
+    replacement   = "lido-charon"
+  }
+
+  rule {
+    source_labels = ["container"]
+    regex         = ".*execution.*"
+    target_label  = "job"
+    replacement   = "lido-execution"
+  }
+
+  rule {
+    source_labels = ["container"]
+    regex         = ".*consensus.*"
+    target_label  = "job"
+    replacement   = "lido-consensus"
+  }
+
+  rule {
+    source_labels = ["container"]
+    regex         = ".*validator.*"
+    target_label  = "job"
+    replacement   = "lido-validator"
+  }
+
+  rule {
+    source_labels = ["container"]
+    regex         = ".*mev-boost.*"
+    target_label  = "job"
+    replacement   = "lido-mev-boost"
+  }
+
+  rule {
+    target_label = "cluster_name"
+    replacement  = "$CLUSTER_NAME"
+  }
+
+  rule {
+    target_label = "cluster_peer"
+    replacement  = "$CLUSTER_PEER"
+  }
+}
+
+loki.source.docker "docker" {
+  host          = "unix:///var/run/docker.sock"
+  targets       = discovery.docker.docker.targets
+  forward_to    = [loki.process.docker.receiver]
+  relabel_rules = discovery.relabel.docker.rules
+}
+
+loki.write "default" {
+  endpoint {
+    url = "$CHARON_LOKI_ADDRESSES"
+  }
+  external_labels = {
+    cluster_name = "$CLUSTER_NAME",
+    cluster_peer = "$CLUSTER_PEER",
+  }
+}
+
+// Metrics
+discovery.relabel "docker_metrics" {
+  targets = discovery.docker.docker.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "job"
+    regex         = "charon"
+    action        = "keep"
+  }
+}
+
+prometheus.scrape "docker_metrics" {
+  targets         = discovery.relabel.docker_metrics.output
+  forward_to      = [
+    prometheus.remote_write.default.receiver,
+  ]
+  scrape_interval = "15s"
+  scrape_timeout  = "12s"
+}
+
+prometheus.remote_write "default" {
+  endpoint {
+    url = "https://vm.monitoring.gcp.obol.tech/write"
+
+    authorization {
+      type        = "Bearer"
+      credentials = "$OBOL_PROM_REMOTE_WRITE_TOKEN"
+    }
+  }
+}

alloy-obol/docker-entrypoint.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+if [ -z "${CHARON_LOKI_ADDRESSES:-}" ]; then
+  echo "\$CHARON_LOKI_ADDRESSES variable is empty - using local Loki"
+  CHARON_LOKI_ADDRESSES=http://loki:3100/loki/api/v1/push
+fi
+
+if [ -z "${CLUSTER_NAME:-}" ]; then
+  echo "Error: \$CLUSTER_NAME variable is empty" >&2
+  CLUSTER_NAME=dummy-cluster
+fi
+
+if [ -z "${CLUSTER_PEER:-}" ]; then
+  echo "Error: \$CLUSTER_PEER variable is empty" >&2
+  CLUSTER_PEER=dummy-peer
+fi
+
+if [ -z "${OBOL_PROM_REMOTE_WRITE_TOKEN:-}" ]; then
+  echo "Error: \$OBOL_PROM_REMOTE_WRITE_TOKEN variable is empty" >&2
+  OBOL_PROM_REMOTE_WRITE_TOKEN=dummy-token
+fi
+
+
+SRC="/etc/alloy/config.alloy.sample"
+DST="/etc/alloy/config.alloy"
+
+echo "Rendering template: $SRC -> $DST"
+
+sed -e "s|\$CHARON_LOKI_ADDRESSES|${CHARON_LOKI_ADDRESSES}|g" \
+    -e "s|\$CLUSTER_NAME|${CLUSTER_NAME}|g" \
+    -e "s|\$CLUSTER_PEER|${CLUSTER_PEER}|g" \
+    -e "s|\$OBOL_PROM_REMOTE_WRITE_TOKEN|${OBOL_PROM_REMOTE_WRITE_TOKEN}|g" \
+    "$SRC" > "$DST"
+
+echo "Config rendered to $DST"
+
+# Execute the command passed as arguments if any
+if [ $# -gt 0 ]; then
+  echo "Executing:" "$@"
+  exec "$@"
+fi

alloy-shared.yml

@@ -0,0 +1,5 @@
+# Expose Alloy port for UI access
+services:
+  alloy:
+    ports:
+      - ${SHARE_IP:-}:${ALLOY_PORT:-12345}:12345/tcp

promtail-obol/.gitignore → alloy/.gitignore

@@ -1,4 +1,4 @@
 *
+!config.alloy
 !*.sample
-!entrypoint.sh
 !.gitignore

alloy/alloy-logs.alloy.sample

@@ -0,0 +1,20 @@
+// Adjust alloy-logs.alloy, but not .sample, if you want to adjust log level or trace target
+logging {
+  level  = "info"
+  format = "logfmt"
+}
+
+tracing {
+  sampling_fraction = 0.1
+
+  write_to = [otelcol.exporter.otlp.tempo.input]
+}
+
+otelcol.exporter.otlp "tempo" {
+  client {
+    endpoint = "tempo:4317"
+    tls {
+      insecure = true
+    }
+  }
+}

alloy/config.alloy

@@ -0,0 +1,100 @@
+// This file is not meant to be edited
+// Create your own config.d/<something.alloy> file instead for additional config
+
+// Discover docker containers to collect logs and metrics from
+discovery.docker "docker_containers" {
+  host = "unix:///var/run/docker.sock"
+}
+
+// Log config
+// Extract container name from __meta_docker_container_name label and add as label
+discovery.relabel "docker_logs" {
+  targets = discovery.docker.docker_containers.targets
+
+ rule {
+    source_labels = ["__meta_docker_container_label_logs_collect"]
+    regex         = "true"
+    action        = "keep"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    target_label  = "container_name"
+  }
+}
+
+// Scrape logs from docker containers and send to be processed
+loki.source.docker "docker_logs" {
+  host    = "unix:///var/run/docker.sock"
+  targets = discovery.relabel.docker_logs.output
+  forward_to = [loki.process.process_logs.receiver]
+}
+
+// Logs write targets are in loki-write.alloy
+
+
+// Metrics config
+discovery.relabel "docker_metrics" {
+  targets = discovery.docker.docker_containers.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_label_metrics_scrape"]
+    regex         = "true"
+    action        = "keep"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "job"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
+    target_label  = "project"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_metrics_path"]
+    regex         = "(.+)"
+    target_label  = "__metrics_path__"
+  }
+
+  rule {
+    source_labels = ["__address__", "__meta_docker_container_label_metrics_port"]
+    regex         = "([^:]+)(?::\\d+)?;(\\d+)"
+    target_label  = "__address__"
+    replacement   = "$1:$2"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_metrics_host", "__meta_docker_container_label_metrics_port"]
+    regex         = "(.+);(\\d+)"
+    target_label  = "__address__"
+    replacement   = "$1:$2"
+  }
+
+  rule {
+  source_labels = ["__meta_docker_container_label_metrics_job"]
+  regex         = "(.+)"
+  target_label  = "job"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_metrics_interval"]
+    regex         = "(.+)"
+    target_label  = "__scrape_interval__"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_metrics_instance"]
+    regex         = "(.+)"
+    target_label  = "instance"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_metrics_network"]
+    target_label  = "network"
+  }
+}
+
+// Metrics write targets are in prometheus-write.alloy

alloy/host-scrape.alloy.sample

@@ -0,0 +1,10 @@
+prometheus.scrape "myjob" {  // Give the job a name
+  targets = [{
+    // Scrape something directly on the host. For a docker service, do "<service-name>:<port>" instead
+    __address__ = "host.docker.internal:9090",
+  }]
+  forward_to      = [prometheus.remote_write.local_prometheus.receiver]
+  // Override the global default and scrape targets from this job every 5 seconds.
+  scrape_interval = "5s"
+  scrape_timeout  = "5s"
+}

alloy/loki-write.alloy.sample

@@ -0,0 +1,35 @@
+// Adjust loki-write.alloy, but not .sample, if you have a remote Loki
+// Process logs and send to Loki
+loki.process "process_logs" {
+    stage.docker { }
+
+    forward_to = [
+      loki.write.local_loki.receiver,  // Comment out if there is no local Loki
+//      loki.write.remote_loki.receiver,  // Uncomment to also send to remote Loki
+    ]
+}
+
+loki.write "local_loki" {
+  endpoint {
+    url = "http://loki:3100/loki/api/v1/push"
+  }
+}
+
+loki.write "remote_loki" {
+  external_labels = {
+    server = "<server-name>",
+  }
+
+  endpoint {
+    url = "https://my-loki.example.com/loki/api/v1/push"  // Adjust to your Loki URL
+    http_headers = {
+      "X-Scope-OrgID" = ["primary"],
+      // "CF-Access-Client-Id" = ["<the-CF-id>"],
+      // "CF-Access-Client-Secret" = ["<the-CF-secret>"],
+    }
+    // basic_auth {
+    //  username = "<USERNAME>"
+    //  password = "<PASSWORD>"
+    // }
+  }
+}