Support custom otel attributes

charts/logfire/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-version: 0.13.5
+version: 0.13.6
 name: logfire
 description: Helm chart for self-hosted Pydantic Logfire
 

charts/logfire/README.md

@@ -1,6 +1,6 @@
 # logfire
 
-![Version: 0.13.5](https://img.shields.io/badge/Version-0.13.5-informational?style=flat-square) ![AppVersion: 80bc3daa](https://img.shields.io/badge/AppVersion-80bc3daa-informational?style=flat-square)
+![Version: 0.13.6](https://img.shields.io/badge/Version-0.13.6-informational?style=flat-square) ![AppVersion: 80bc3daa](https://img.shields.io/badge/AppVersion-80bc3daa-informational?style=flat-square)
 
 Helm chart for self-hosted Pydantic Logfire
 
@@ -718,6 +718,7 @@ Before diving deeper, verify these common configuration issues:
 | objectStore.uri | string | `nil` | URI for object storage (e.g., `s3://bucket`) |
 | objectStore.volumeMounts | list | `[]` | Volume mounts for object store credentials |
 | objectStore.volumes | list | `[]` | Volumes for object store credentials |
+| otelResourceAttributes | object | `{}` | Additional OTEL resource attributes to stamp onto internal telemetry emitted by Logfire workloads. These are merged on top of the chart defaults and can override them. Example:   deployment.environment.name: prod   service.namespace: logfire |
 | otel_collector | object | `{"exporter":{"endpoint":"http://logfire-ff-ingest:8012","headers":{},"tls":{"insecure":true}},"image":{"pullPolicy":"IfNotPresent","repository":"ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib","tag":"0.139.0"},"prometheus":{"add_metric_suffixes":false,"enable_open_metrics":true,"enabled":false,"endpoint":"0.0.0.0","metric_expiration":"180m","port":9090,"resource_to_telemetry_conversion":{"enabled":true},"send_timestamp":true}}` | otel-collector configuration |
 | otel_collector.exporter | object | `{"endpoint":"http://logfire-ff-ingest:8012","headers":{},"tls":{"insecure":true}}` | exporter configuration for the otlphttp exporter Override these to send telemetry data to a different OTLP-compatible destination. |
 | podSecurityContext | object | `{}` | Pod SecurityContext (https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) See: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context for details |

charts/logfire/templates/_helpers-fusionfire.tpl

@@ -0,0 +1,53 @@
+{{/*
+================================================================================
+Fusionfire Helpers
+================================================================================
+Helpers specific to Fusionfire workloads and configuration.
+*/}}
+
+{{- define "logfire.ffCompactionTiersValue" -}}
+{{- if (get (get .Values "logfire-ff-maintenance-worker" | default  dict) "compactionTiers") -}}
+{{- with (get (get .Values "logfire-ff-maintenance-worker" | default  dict) "compactionTiers") -}}
+{{ . | toJson }}
+{{- end -}}
+{{- else -}}
+[{"count_threshold":10,"size_threshold_bytes":"1KB"},{"count_threshold":10,"size_threshold_bytes":"10KB"},{"count_threshold":10,"size_threshold_bytes":"100KB"},{"count_threshold":10,"size_threshold_bytes":"1MB"},{"count_threshold":10,"size_threshold_bytes":"10MB"},{"count_threshold":10,"size_threshold_bytes":"100MB"}]
+{{- end -}}
+{{- end -}}
+
+{{/*
+Derive FF service CPU core count and DataFusion thread count from the effective CPU request.
+*/}}
+{{- define "logfire.ffThreadSettings" -}}
+{{- $effectiveResources := include "logfire.effectiveResources" . | fromJson -}}
+{{- $cpu := get $effectiveResources "cpuRequest" -}}
+{{- $cpuCores := int (include "logfire.cpuCores" $cpu) -}}
+{{- $dataFusionThreads := max 1 (sub $cpuCores 1) -}}
+{{- dict "cpuCores" $cpuCores "dataFusionThreads" $dataFusionThreads | toJson -}}
+{{- end -}}
+
+{{/*
+Conservative default DataFusion memory cap for query-worker execution.
+Leaves headroom for the HTTP process/runtime, scales up with queryParallelism,
+and caps at 3Gi unless explicitly overridden.
+*/}}
+{{- define "logfire.ffQueryDatafusionMemoryDefault" -}}
+{{- $effectiveResources := include "logfire.effectiveResources" . | fromJson -}}
+{{- $effectiveServiceValues := include "logfire.effectiveServiceValues" . | fromJson -}}
+{{- $memory := get $effectiveResources "memoryRequest" -}}
+{{- $memoryMi := int (include "logfire.memoryToMi" $memory) -}}
+{{- $queryParallelism := int (get $effectiveServiceValues "queryParallelism" | default "4") -}}
+{{- $parallelismFloorMi := mul $queryParallelism 256 -}}
+{{- $baselineMi := max 512 (max (div $memoryMi 2) $parallelismFloorMi) -}}
+{{- $headroomMi := max 512 (div (mul $memoryMi 3) 4) -}}
+{{- $datafusionMemoryMi := min 3072 (min $baselineMi $headroomMi) -}}
+{{- printf "%dMB" $datafusionMemoryMi -}}
+{{- end -}}
+
+{{- define "logfire.ffMigrations.name" -}}
+{{- if .Values.dev.deployPostgres -}}
+"logfire-ff-migrations-{{ .Release.Revision }}"
+{{- else -}}
+"logfire-ff-migrations"
+{{- end -}}
+{{- end -}}

charts/logfire/templates/_helpers-resources.tpl

@@ -57,17 +57,6 @@ Supports both legacy flat keys and nested requests/limits.
 {{- dict "cpuRequest" $cpuRequest "memoryRequest" $memoryRequest "ephemeralStorageRequest" $ephemeralStorageRequest "cpuLimit" $cpuLimit "memoryLimit" $memoryLimit "ephemeralStorageLimit" $ephemeralStorageLimit | toJson -}}
 {{- end -}}
 
-{{/*
-Derive FF service CPU core count and DataFusion thread count from the effective CPU request.
-*/}}
-{{- define "logfire.ffThreadSettings" -}}
-{{- $effectiveResources := include "logfire.effectiveResources" . | fromJson -}}
-{{- $cpu := get $effectiveResources "cpuRequest" -}}
-{{- $cpuCores := int (include "logfire.cpuCores" $cpu) -}}
-{{- $dataFusionThreads := max 1 (sub $cpuCores 1) -}}
-{{- dict "cpuCores" $cpuCores "dataFusionThreads" $dataFusionThreads | toJson -}}
-{{- end -}}
-
 {{/*
 Convert Kubernetes memory quantity to mebibytes (Mi).
 Supports common binary and decimal suffixes plus plain bytes.

charts/logfire/templates/_helpers.tpl

@@ -173,18 +173,6 @@ spec:
 {{- end }}
 {{- end }}
 
-{{- define "logfire.ffCompactionTiers" -}}
-{{- if (get (get .Values "logfire-ff-maintenance-worker" | default  dict) "compactionTiers") -}}
-{{- with (get (get .Values "logfire-ff-maintenance-worker" | default  dict) "compactionTiers") -}}
-- name: FF_COMPACTION_TIERS
-  value: {{ . | toJson | squote }}
-{{- end -}}
-{{- else -}}
-- name: FF_COMPACTION_TIERS
-  value: '[{"count_threshold":10,"size_threshold_bytes":"1KB"},{"count_threshold":10,"size_threshold_bytes":"10KB"},{"count_threshold":10,"size_threshold_bytes":"100KB"},{"count_threshold":10,"size_threshold_bytes":"1MB"},{"count_threshold":10,"size_threshold_bytes":"10MB"},{"count_threshold":10,"size_threshold_bytes":"100MB"}]'
-{{- end -}}
-{{- end -}}
-
 {{- define "logfire.resources" -}}
 {{- $serviceValues := include "logfire.effectiveServiceValues" . | fromJson -}}
 {{- $resources := index $serviceValues "resources" | default dict -}}
@@ -419,15 +407,29 @@ Usage: {{ include "logfire.otelResourceAttributes" (dict "root" . "serviceName"
 */}}
 {{- define "logfire.otelResourceAttributes" -}}
 {{- $version := include "logfire.serviceVersion" . -}}
-{{- $attrs := list
-  "vcs.repository.url.full=https://github.com/pydantic/platform"
-  (printf "vcs.repository.ref.revision=%s" $version)
-  (printf "service.version=%s" $version)
+{{- $attrs := dict
+  "vcs.repository.url.full" "https://github.com/pydantic/platform"
+  "vcs.repository.ref.revision" $version
+  "service.version" $version
 -}}
 {{- with .codeWorkDir }}
-  {{- $attrs = append $attrs (printf "logfire.code.work_dir=%s" .) -}}
+  {{- $_ := set $attrs "logfire.code.work_dir" . -}}
 {{- end }}
-{{- join "," $attrs -}}
+{{- $attrs = mergeOverwrite $attrs ((get .root.Values "otelResourceAttributes") | default dict) -}}
+{{- include "logfire.renderOtelResourceAttributes" $attrs -}}
+{{- end -}}
+
+{{/*
+Render OTEL resource attributes from a map into OTEL_RESOURCE_ATTRIBUTES format.
+Usage: {{ include "logfire.renderOtelResourceAttributes" (dict "service.name" "my-service") }}
+*/}}
+{{- define "logfire.renderOtelResourceAttributes" -}}
+{{- $resourceAttributes := . | default dict -}}
+{{- $pairs := list -}}
+{{- range $key := keys $resourceAttributes | sortAlpha }}
+  {{- $pairs = append $pairs (printf "%s=%v" $key (get $resourceAttributes $key)) -}}
+{{- end -}}
+{{- join "," $pairs -}}
 {{- end -}}
 
 {{/*
@@ -854,14 +856,6 @@ default-checksum
 {{- end -}}
 {{- end -}}
 
-{{- define "logfire.ffMigrations.name" -}}
-{{- if .Values.dev.deployPostgres -}}
-"logfire-ff-migrations-{{ .Release.Revision }}"
-{{- else -}}
-"logfire-ff-migrations"
-{{- end -}}
-{{- end -}}
-
 {{- define "logfire.podScheduling" -}}
 {{- $serviceValues := index .Values .serviceName | default dict -}}
 {{- $nodeSelector := merge (deepCopy ($serviceValues.nodeSelector | default dict)) (.Values.nodeSelector | default dict) -}}

charts/logfire/templates/logfire-ff-compaction-worker.yaml

@@ -68,33 +68,15 @@ spec:
                 name: logfire-ff-base-config
             - configMapRef:
                 name: logfire-ff-write-config
+            - configMapRef:
+                name: logfire-ff-maintenance-config
           env:
             - name: FF_IO_THREADS
               value: {{ $cpuCores | quote }}
             - name: FF_DATAFUSION_THREADS
               value: {{ $dataFusionThreads | quote }}
-            - name: FF_ENABLE_SCHEMA_UPGRADES
-              value: "true"
-            - name: FF_UPGRADE_MAX_DAYS
-              value: "7"
-            - name: FF_CACHE_OBJECT_STORE_HTTP_HEADERS
-              value: x-ff-no-warm-cache=1
-            - name: FF_COMPACTION_MAX_DAYS
-              value: "3"
-            - name: FF_MAX_PARTITIONS_PER_COMPACTION_SCAN
-              value: "5000"
-            - name: FF_COMPACTION_LOOP_BACKOFF_DELAY
-              value: "1s"
-            - name: FF_DELETION_LOOP_DELAY
-              value: "30s"
-            - name: FF_COMPACTION_GATHER_DELAY
-              value: "1s"
-            - name: FF_MAINTENANCE_SCHEDULING_CONCURRENCY
-              value: "8"
             - name: FF_DATAFUSION_MEMORY_LIMIT
               value: {{ (get $effectiveServiceValues "datafusionMemory" | default "512MB" | quote) }}
-            - name: FF_MAX_FILES_PER_COMPACTION_JOB
-              value: "10000"
             - name: FF_ENABLE_SPILL_TO_DISK
               value: "true"
             - name: FF_TEMP_DIR
@@ -113,7 +95,6 @@ spec:
                 secretKeyRef:
                   name: {{ include "logfire.postgresSecretName" . }}
                   key: postgresFFDsn
-            {{- include "logfire.ffCompactionTiers" . | nindent 12 -}}
             {{- include "logfire.objectStoreEnv" . | nindent 12 -}}
             {{- with (index .Values $serviceName | default dict).env }}
               {{- . | toYaml | nindent 12 }}

charts/logfire/templates/logfire-ff-config.yaml

@@ -34,7 +34,6 @@ metadata:
     {{- include "logfire.labels" . | nindent 4 }}
     app.kubernetes.io/component: logfire-ff-config
 data:
-  FF_FUSIONFIRE_COLLECT_STATISTICS: "0.0"
   FF_IPC_CACHE_URI: "{{ include "logfire.scheme" . }}://logfire-ff-cache-ipc:{{ include "logfire.port" (dict "port" 9001 "root" .) }}"
   FF_IPC_CACHE_HOT_FILE_AGE_MINUTES: "30"
   FF_IPC_CACHE_SMALL_FILE_SIZE_MB: "5"
@@ -57,6 +56,20 @@ data:
 
 ---
 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logfire-ff-maintenance-config
+  labels:
+    {{- include "logfire.labels" . | nindent 4 }}
+    app.kubernetes.io/component: logfire-ff-config
+data:
+  FF_CACHE_OBJECT_STORE_HTTP_HEADERS: x-ff-no-warm-cache=1
+  FF_COMPACTION_LOOP_BACKOFF_DELAY: "1s"
+  FF_COMPACTION_TIERS: {{ include "logfire.ffCompactionTiersValue" . | squote }}
+
+---
+
 apiVersion: v1
 kind: ConfigMap
 metadata:

charts/logfire/templates/logfire-ff-maintenance-scheduler.yaml

@@ -68,6 +68,8 @@ spec:
                 name: logfire-ff-base-config
             - configMapRef:
                 name: logfire-ff-write-config
+            - configMapRef:
+                name: logfire-ff-maintenance-config
           env:
             - name: FF_IO_THREADS
               value: {{ $cpuCores | quote }}
@@ -77,35 +79,26 @@ spec:
               value: "true"
             - name: FF_UPGRADE_MAX_DAYS
               value: "7"
-            - name: FF_CACHE_OBJECT_STORE_HTTP_HEADERS
-              value: x-ff-no-warm-cache=1
             - name: FF_COMPACTION_MAX_DAYS
               value: "3"
             - name: FF_MAX_PARTITIONS_PER_COMPACTION_SCAN
               value: "5000"
-            - name: FF_COMPACTION_LOOP_BACKOFF_DELAY
-              value: "1s"
             - name: FF_DELETION_LOOP_DELAY
               value: "30s"
             - name: FF_COMPACTION_GATHER_DELAY
               value: "1s"
             - name: FF_MAINTENANCE_SCHEDULING_CONCURRENCY
-              value: "8"
+              value: "1"
             - name: FF_DATAFUSION_MEMORY_LIMIT
               value: {{ (get $effectiveServiceValues "datafusionMemory" | default "512MB" | quote) }}
             - name: FF_MAX_FILES_PER_COMPACTION_JOB
               value: "10000"
-            - name: FF_COMPACTION_DOWNLOAD_PARALLELISM
-              value: {{ (get $effectiveServiceValues "downloadParallelism" | default "10" | quote) }}
-            - name: FF_COMPACTION_JOB_PARALLELISM
-              value: {{ (get $effectiveServiceValues "jobParallelism" | default "3" | quote) }}
             {{- include "logfire.otlpExporterEnv" (dict "root" . "serviceName" $serviceName "codeWorkDir" "/app") | nindent 12 }}
             - name: PG_DSN
               valueFrom:
                 secretKeyRef:
                   name: {{ include "logfire.postgresSecretName" . }}
                   key: postgresFFDsn
-            {{- include "logfire.ffCompactionTiers" . | nindent 12 -}}
             {{- include "logfire.objectStoreEnv" . | nindent 12 -}}
             {{- with (index .Values $serviceName | default dict).env }}
               {{- . | toYaml | nindent 12 }}

charts/logfire/templates/logfire-ff-maintenance-worker.yaml

@@ -68,6 +68,8 @@ spec:
                 name: logfire-ff-base-config
             - configMapRef:
                 name: logfire-ff-write-config
+            - configMapRef:
+                name: logfire-ff-maintenance-config
           env:
             - name: FF_IO_THREADS
               value: {{ $cpuCores | quote }}
@@ -77,24 +79,12 @@ spec:
               value: "true"
             - name: FF_UPGRADE_MAX_DAYS
               value: "7"
-            - name: FF_CACHE_OBJECT_STORE_HTTP_HEADERS
-              value: x-ff-no-warm-cache=1
-            - name: FF_COMPACTION_MAX_DAYS
-              value: "3"
-            - name: FF_MAX_PARTITIONS_PER_COMPACTION_SCAN
-              value: "5000"
-            - name: FF_COMPACTION_LOOP_BACKOFF_DELAY
-              value: "1s"
             - name: FF_DELETION_LOOP_DELAY
               value: "30s"
-            - name: FF_COMPACTION_GATHER_DELAY
-              value: "1s"
-            - name: FF_MAINTENANCE_SCHEDULING_CONCURRENCY
-              value: "8"
             - name: FF_DATAFUSION_MEMORY_LIMIT
               value: {{ (get $effectiveServiceValues "datafusionMemory" | default "512MB" | quote) }}
-            - name: FF_MAX_FILES_PER_COMPACTION_JOB
-              value: "10000"
+            - name: FF_DATAFUSION_TARGET_PARTITIONS
+              value: "1"
             - name: FF_ENABLE_SPILL_TO_DISK
               value: "true"
             - name: FF_TEMP_DIR
@@ -113,7 +103,6 @@ spec:
                 secretKeyRef:
                   name: {{ include "logfire.postgresSecretName" . }}
                   key: postgresFFDsn
-            {{- include "logfire.ffCompactionTiers" . | nindent 12 -}}
             {{- include "logfire.objectStoreEnv" . | nindent 12 -}}
             {{- with (index .Values $serviceName | default dict).env }}
               {{- . | toYaml | nindent 12 }}

charts/logfire/templates/logfire-ff-query-api.yaml

@@ -1,5 +1,9 @@
 {{- $serviceName := "logfire-ff-query-api" }}
 {{- $effectiveServiceValues := include "logfire.effectiveServiceValues" (dict "Values" .Values "serviceName" $serviceName) | fromJson -}}
+{{- $threadSettings := include "logfire.ffThreadSettings" (dict "Values" .Values "serviceName" $serviceName) | fromJson -}}
+{{- $defaultDatafusionMemory := include "logfire.ffQueryDatafusionMemoryDefault" (dict "Values" .Values "serviceName" $serviceName) -}}
+{{- $cpuCores := int (get $threadSettings "cpuCores") -}}
+{{- $dataFusionThreads := int (get $threadSettings "dataFusionThreads") -}}
 apiVersion: v1
 kind: Service
 metadata:
@@ -97,10 +101,18 @@ spec:
               value: /scratch/fusionfire
             - name: FF_QUERY_PARALLELISM
               value: {{ (get $effectiveServiceValues "queryParallelism" | default "4" | quote) }}
+            {{- if not (get (get .Values "logfire-ff-query-worker" | default  dict) "enabled") }}
+            - name: FF_IO_THREADS
+              value: {{ $cpuCores | quote }}
+            - name: FF_DATAFUSION_THREADS
+              value: {{ $dataFusionThreads | quote }}
+            - name: FF_DATAFUSION_MEMORY_LIMIT
+              value: {{ (get $effectiveServiceValues "datafusionMemory" | default $defaultDatafusionMemory | quote) }}
+            - name: FF_IO_THREAD_STACK_SIZE
+              value: "8MB"
+            {{- end }}
             - name: FF_DATAFUSION_THREAD_STACK_SIZE
               value: "8MB"
-            - name: FF_USE_PHYSICAL_PLAN_ENDPOINT_CHANCE
-              value: "0.05"
             - name: FF_CACHE_ARROW_IPC_CHANCE
               value: "1.0"
             - name: FF_REDIS_TAIL_DSN

charts/logfire/templates/logfire-ff-query-worker.yaml

@@ -1,6 +1,7 @@
 {{- $serviceName := "logfire-ff-query-worker" }}
 {{- $effectiveServiceValues := include "logfire.effectiveServiceValues" (dict "Values" .Values "serviceName" $serviceName) | fromJson -}}
 {{- $threadSettings := include "logfire.ffThreadSettings" (dict "Values" .Values "serviceName" $serviceName) | fromJson -}}
+{{- $defaultDatafusionMemory := include "logfire.ffQueryDatafusionMemoryDefault" (dict "Values" .Values "serviceName" $serviceName) -}}
 {{- $cpuCores := int (get $threadSettings "cpuCores") -}}
 {{- $dataFusionThreads := int (get $threadSettings "dataFusionThreads") -}}
 {{- if (index .Values $serviceName | default dict).enabled }}
@@ -86,7 +87,7 @@ spec:
             - name: FF_DATAFUSION_THREADS
               value: {{ $dataFusionThreads | quote }}
             - name: FF_DATAFUSION_MEMORY_LIMIT
-              value: {{ (get $effectiveServiceValues "datafusionMemory" | default "512MB" | quote) }}
+              value: {{ (get $effectiveServiceValues "datafusionMemory" | default $defaultDatafusionMemory | quote) }}
             - name: FF_DATAFUSION_THREAD_STACK_SIZE
               value: "8MB"
             - name: FF_IO_THREAD_STACK_SIZE