[amazon-cloudwatch-observability] feat: bundle ServiceMonitor/PodMonitor CRDs (zero-step install)

charts/amazon-cloudwatch-observability/crds/cloudwatch.aws.amazon.com_amazoncloudwatchagents.yaml

@@ -6405,9 +6405,10 @@ spec:
                   allocationStrategy:
                     description: |-
                       AllocationStrategy determines which strategy the target allocator should use for allocation.
-                      The current option is consistent-hashing.
+                      The options are consistent-hashing and per-node.
                     enum:
                     - consistent-hashing
+                    - per-node
                     type: string
                   enabled:
                     description: Enabled indicates whether to use a target allocation

charts/amazon-cloudwatch-observability/templates/_helpers.tpl

@@ -5,6 +5,29 @@ Expand the name of the chart.
 {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
+{{/*
+Whether to bundle the community ServiceMonitor/PodMonitor CRDs. Honours
+.Values.prometheusCRDs.install: "always" => true; "never" => empty;
+"auto" (default) => true only when otelContainerInsights.enabled is true.
+Returns the string "true" when CRDs should be rendered, empty otherwise.
+*/}}
+{{- define "amazon-cloudwatch-observability.prometheusCRDsEnabled" -}}
+{{- $install := "auto" -}}
+{{- if hasKey .Values "prometheusCRDs" -}}
+{{- $install = (.Values.prometheusCRDs.install | default "auto") -}}
+{{- end -}}
+{{- if eq $install "always" -}}
+true
+{{- else if eq $install "never" -}}
+{{- else if eq $install "auto" -}}
+{{- if .Values.otelContainerInsights.enabled -}}
+true
+{{- end -}}
+{{- else -}}
+{{- fail (printf "prometheusCRDs.install must be one of \"auto\", \"always\", or \"never\", got: %s" $install) -}}
+{{- end -}}
+{{- end -}}
+
 {{- define "amazon-cloudwatch-observability.common.tolerations" -}}
 {{- $tolerations := .context.Values.tolerations }}
 {{- if .component }}
@@ -191,6 +214,20 @@ Logic:
 {{- end -}}
 {{- end -}}
 
+{{/*
+Returns "true" when otelContainerInsights-driven ServiceMonitor/PodMonitor scraping
+applies to the given agent. True when otelContainerInsights is enabled, the agent is
+the configured targetAgent, and at least one of serviceMonitor/podMonitor is enabled.
+Accepts a dict with "agentName" (string) and "context" (root context $).
+*/}}
+{{- define "cloudwatch-agent.otelCIScrapeEnabled" -}}
+{{- $ctx := .context -}}
+{{- $agentName := .agentName -}}
+{{- if and $ctx.Values.otelContainerInsights.enabled (eq $agentName $ctx.Values.otelContainerInsights.targetAgent) (or $ctx.Values.otelContainerInsights.serviceMonitor.enabled $ctx.Values.otelContainerInsights.podMonitor.enabled) -}}
+true
+{{- end -}}
+{{- end -}}
+
 {{/*
 Helper function to modify cloudwatch-agent config
 */}}

charts/amazon-cloudwatch-observability/templates/linux/_otel-container-insights-cluster-scraper-config.tpl

@@ -231,7 +231,8 @@ processors:
         - k8s.job.name
         - k8s.cronjob.name
       labels:
-        # $$$1 is Helm escaping: $$$ → $$ (Helm) → $ (OTel env resolver) → literal $1 backreference
+        # $$$1 -> literal $1 backreference (group 1 = label key). The agent's OTel confmap
+        # resolves it twice (expandconverter + resolver), each collapsing $$->$; Helm leaves it as-is.
         - tag_name: "k8s.pod.label.$$$1"
           key_regex: "(.*)"
           from: pod

charts/amazon-cloudwatch-observability/templates/linux/_otel-container-insights-config.tpl

@@ -43,6 +43,23 @@ receivers:
             - targets:
                 - ${env:HOST_IP}:10250
 
+{{- if or .Values.otelContainerInsights.serviceMonitor.enabled .Values.otelContainerInsights.podMonitor.enabled }}
+  # ServiceMonitor/PodMonitor scraping via the Target Allocator (prometheusCR discovery).
+  # The Target Allocator deployed for the targetAgent serves the scrape jobs derived from
+  # ServiceMonitor/PodMonitor CRs; this receiver pulls this collector's assigned shard and
+  # routes the series into the v2 OTLP pipeline (-> CloudWatch/Zeus). Requires the Target
+  # Allocator + prometheusCR to be enabled for the targetAgent and the POD_NAME env (set below).
+  prometheus/cw_k8s_ci_v0_prometheuscr:
+    target_allocator:
+      endpoint: https://{{ .Values.otelContainerInsights.targetAgent }}-target-allocator-service:80
+      interval: {{ .Values.otelContainerInsights.metricResolution }}
+      collector_id: ${env:POD_NAME}
+      tls:
+        ca_file: /etc/amazon-cloudwatch-observability-agent-cert/tls-ca.crt
+        cert_file: /etc/amazon-cloudwatch-observability-agent-ta-client-cert/client.crt
+        key_file: /etc/amazon-cloudwatch-observability-agent-ta-client-cert/client.key
+{{- end }}
+
   {{- if .Values.dcgmExporter.enabled }}
   prometheus/cw_k8s_ci_v0_dcgm:
     config:
@@ -329,6 +346,18 @@ processors:
           - set(attributes["cloudwatch.solution"], "k8s-otel-container-insights")
           - set(attributes["cloudwatch.pipeline"], "efa")
 
+{{- if or .Values.otelContainerInsights.serviceMonitor.enabled .Values.otelContainerInsights.podMonitor.enabled }}
+  transform/cw_k8s_ci_v0_set_scope_prometheuscr:
+    error_mode: ignore
+    metric_statements:
+      - context: scope
+        statements:
+          - set(scope.schema_url, "")
+          - set(attributes["cloudwatch.source"], "cloudwatch-agent")
+          - set(attributes["cloudwatch.solution"], "k8s-otel-container-insights")
+          - set(attributes["cloudwatch.pipeline"], "prometheus-cr")
+{{- end }}
+
   transform/cw_k8s_ci_v0_set_scope_ebs_csi:
     error_mode: ignore
     metric_statements:
@@ -427,6 +456,8 @@ processors:
         - k8s.job.name
         - k8s.cronjob.name
       labels:
+        # $$$1 -> literal $1 backreference (group 1 = label key). The agent's OTel confmap
+        # resolves it twice (expandconverter + resolver), each collapsing $$->$; Helm leaves it as-is.
         - tag_name: "k8s.pod.label.$$$1"
           key_regex: "(.*)"
           from: pod
@@ -829,6 +860,20 @@ service:
       exporters:
         - otlphttp/cw_k8s_ci_v0_metrics_dest
 
+{{- if or .Values.otelContainerInsights.serviceMonitor.enabled .Values.otelContainerInsights.podMonitor.enabled }}
+    metrics/cw_k8s_ci_v0_prometheuscr:
+      receivers: [prometheus/cw_k8s_ci_v0_prometheuscr]
+      processors:
+        - filter/cw_k8s_ci_v0_scrape_metadata
+        - metricstarttime/cw_k8s_ci_v0
+        - transform/cw_k8s_ci_v0_set_cluster_name
+        - transform/cw_k8s_ci_v0_set_scope_prometheuscr
+        - resourcedetection/cw_k8s_ci_v0
+        - batch/cw_k8s_ci_v0_metrics_dest
+      exporters:
+        - otlphttp/cw_k8s_ci_v0_metrics_dest
+{{- end }}
+
     {{- if .Values.dcgmExporter.enabled }}
     metrics/cw_k8s_ci_v0_dcgm:
       receivers: [prometheus/cw_k8s_ci_v0_dcgm]

charts/amazon-cloudwatch-observability/templates/linux/cloudwatch-agent-custom-resource.yaml

@@ -110,21 +110,46 @@ spec:
   {{- else if ne (trimAll " \n\t" $generatedOtelConfig) "{}" }}
   otelConfig: {{ include "cloudwatch-agent.modify-otel-config" (merge (dict "OtelConfig" $generatedOtelConfig) $) }}
   {{- end }}
+  {{- $otelCIScrape := eq (include "cloudwatch-agent.otelCIScrapeEnabled" (dict "agentName" $agent.name "context" $)) "true" }}
+  {{- $taEnabled := or $agent.prometheus.targetAllocator.enabled $otelCIScrape }}
   {{- if $agent.prometheus.config }}
   prometheus:
     {{- with $agent.prometheus.config }}
     config:
       {{- toYaml . | nindent 6 }}
     {{- end }}
+  {{- else if $taEnabled }}
+  # The operator builds the Target Allocator config from spec.prometheus and requires
+  # a prometheus.config containing a scrape_configs key (see operator
+  # targetallocator/adapters GetPromConfig). With prometheusCR (ServiceMonitor/
+  # PodMonitor) discovery the static scrape list is empty -- the Target Allocator
+  # discovers targets from ServiceMonitor/PodMonitor CRs.
+  prometheus:
+    config:
+      scrape_configs: []
   {{- end }}
-  {{- if $agent.prometheus.targetAllocator.enabled }}
+  {{- if $taEnabled }}
   targetAllocator:
-    enabled: {{ $agent.prometheus.targetAllocator.enabled | default false }}
+    enabled: true
     image: {{ template "target-allocator.image" (merge $agent.prometheus.targetAllocator.image (dict "region" $.Values.region)) }}
-    allocationStrategy: "consistent-hashing"
-    {{- if $agent.prometheus.targetAllocator.prometheusCR.enabled }}
+    allocationStrategy: {{ $agent.prometheus.targetAllocator.allocationStrategy | default (ternary "per-node" "consistent-hashing" $otelCIScrape) | quote }}
+    {{- if or $agent.prometheus.targetAllocator.prometheusCR.enabled $otelCIScrape }}
     prometheusCR:
-      enabled: {{ $agent.prometheus.targetAllocator.prometheusCR.enabled | default false }}
+      enabled: true
+      {{- if $otelCIScrape }}
+      scrapeInterval: {{ $.Values.otelContainerInsights.metricResolution | quote }}
+      {{- if not $.Values.otelContainerInsights.serviceMonitor.enabled }}
+      # serviceMonitor.enabled=false: select a label no real ServiceMonitor carries,
+      # so the Target Allocator discovers no ServiceMonitors.
+      serviceMonitorSelector:
+        amazon-cloudwatch-observability.aws/otel-ci-scrape: disabled
+      {{- end }}
+      {{- if not $.Values.otelContainerInsights.podMonitor.enabled }}
+      # podMonitor.enabled=false: select a label no real PodMonitor carries.
+      podMonitorSelector:
+        amazon-cloudwatch-observability.aws/otel-ci-scrape: disabled
+      {{- end }}
+      {{- end }}
     {{- end }}
   {{- end }}
   {{- with $agent.resources }}
@@ -281,6 +306,12 @@ spec:
   {{- if $.Values.otelContainerInsights.enabled }}
   - name: OTEL_CI_VERSION
     value: "1.0.0"
+  # POD_NAME is used as the Target Allocator collector_id by the prometheusCR
+  # receiver (otel-container-insights.config) so each agent pulls its own shard.
+  - name: POD_NAME
+    valueFrom:
+      fieldRef:
+        fieldPath: metadata.name
   {{- end }}
   {{- with $agent.env }}
   {{- . | toYaml | nindent 2 }}