From 70bb9872f7bd6b843b517f22b85449ccb18f8136 Mon Sep 17 00:00:00 2001 From: "Zach N." Date: Mon, 20 Jul 2026 07:40:49 -0700 Subject: [PATCH 1/9] Add easy GPU driver setup --- gremlin/templates/_helpers.tpl | 125 ++++++++++++++++ gremlin/templates/daemonset.yaml | 12 ++ gremlin/tests/daemonset_gpu_test.yaml | 202 ++++++++++++++++++++++++++ gremlin/values.yaml | 76 ++++++++++ 4 files changed, 415 insertions(+) create mode 100644 gremlin/tests/daemonset_gpu_test.yaml diff --git a/gremlin/templates/_helpers.tpl b/gremlin/templates/_helpers.tpl index 5aebf27..3053bf8 100644 --- a/gremlin/templates/_helpers.tpl +++ b/gremlin/templates/_helpers.tpl @@ -244,6 +244,131 @@ When createSecret or existingSecret are configured {{- end -}} {{- end -}} +{{/* +gremlinGpuVendorPreset returns a YAML dict of the "easy default" GPU configuration for a +given vendor. The context passed in is the vendor string (gremlin.gpu.vendor). Supported +values are "nvidia" and "amd"; any other value (including "") yields an empty preset so the +chart falls back to whatever the advanced options specify. + + nvidia - relies on the NVIDIA container toolkit: run under the "nvidia" RuntimeClass and set + NVIDIA_VISIBLE_DEVICES / NVIDIA_DRIVER_CAPABILITIES so the runtime injects the driver + libraries and device nodes (no hostMounts required). + amd - relies on the ROCm/amdgpu driver on the host: mount the /dev/kfd and /dev/dri device + nodes plus the OpenCL ICD registry so the container can reach the GPUs directly. +*/}} +{{- define "gremlinGpuVendorPreset" -}} +{{- $vendor := . -}} +{{- if eq $vendor "nvidia" -}} +runtimeClassName: nvidia +env: + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_DRIVER_CAPABILITIES + value: "all" +hostMounts: [] +{{- else if eq $vendor "amd" -}} +runtimeClassName: "" +env: [] +hostMounts: + - name: kfd + hostPath: /dev/kfd + type: CharDevice + readOnly: false + - name: dri + hostPath: /dev/dri + type: Directory + readOnly: false + - name: opencl-vendors + hostPath: /etc/OpenCL/vendors + mountPath: /etc/OpenCL/vendors + type: DirectoryOrCreate + readOnly: true +{{- else -}} +runtimeClassName: "" +env: [] +hostMounts: [] +{{- end -}} +{{- end -}} + +{{/* +gremlinGpuEffective returns a YAML dict with the effective GPU configuration: the vendor +preset (see gremlinGpuVendorPreset) with each advanced option overriding the preset when set. +A blank runtimeClassName, an empty env list, or an empty hostMounts list means "use the preset". +Keys: runtimeClassName (string), env (list), hostMounts (list). +*/}} +{{- define "gremlinGpuEffective" -}} +{{- $gpu := .Values.gremlin.gpu -}} +{{- $preset := fromYaml (include "gremlinGpuVendorPreset" (default "" $gpu.vendor)) -}} +{{- $runtimeClassName := $preset.runtimeClassName -}} +{{- if $gpu.runtimeClassName -}}{{- $runtimeClassName = $gpu.runtimeClassName -}}{{- end -}} +{{- $env := $preset.env -}} +{{- if $gpu.env -}}{{- $env = $gpu.env -}}{{- end -}} +{{- $hostMounts := $preset.hostMounts -}} +{{- if $gpu.hostMounts -}}{{- $hostMounts = $gpu.hostMounts -}}{{- end -}} +{{- dict "runtimeClassName" $runtimeClassName "env" $env "hostMounts" $hostMounts | toYaml -}} +{{- end -}} + +{{/* +gremlinGpuRuntimeClassName returns the effective RuntimeClass name for the Gremlin pods, or +nothing when GPU access is disabled or no RuntimeClass applies. +*/}} +{{- define "gremlinGpuRuntimeClassName" -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} +{{- $eff.runtimeClassName -}} +{{- end -}} +{{- end -}} + +{{/* +gremlinGpuEnv returns the environment variables that enable GPU/OpenCL access. +When gremlin.gpu.enabled is true, the effective env entries (vendor preset or the gremlin.gpu.env +override) are emitted. For the NVIDIA container toolkit these include NVIDIA_VISIBLE_DEVICES and +NVIDIA_DRIVER_CAPABILITIES (which must include `compute` or `all` for OpenCL). +*/}} +{{- define "gremlinGpuEnv" -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} +{{- with $eff.env -}} +{{- toYaml . -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +gremlinGpuVolumeMounts returns the volumeMounts that expose host GPU/OpenCL drivers. +Each effective hostMounts entry (vendor preset or the gremlin.gpu.hostMounts override) becomes a +volumeMount. mountPath defaults to hostPath and readOnly defaults to true when not specified. +*/}} +{{- define "gremlinGpuVolumeMounts" -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} +{{- range $eff.hostMounts }} +- name: {{ .name }} + mountPath: {{ default .hostPath .mountPath }} + readOnly: {{ if hasKey . "readOnly" }}{{ .readOnly }}{{ else }}true{{ end }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +gremlinGpuVolumes returns the hostPath volumes that expose host GPU/OpenCL drivers. +Each effective hostMounts entry (vendor preset or the gremlin.gpu.hostMounts override) becomes a +hostPath volume. An optional `type` (e.g. DirectoryOrCreate, CharDevice) is passed through when present. +*/}} +{{- define "gremlinGpuVolumes" -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} +{{- range $eff.hostMounts }} +- name: {{ .name }} + hostPath: + path: {{ .hostPath }} + {{- if .type }} + type: {{ .type }} + {{- end }} +{{- end -}} +{{- end -}} +{{- end -}} + {{/* chaoTlsIdentityArgs returns the chao cli arguments needed to configure TLS client identity When remoteSecret is configured diff --git a/gremlin/templates/daemonset.yaml b/gremlin/templates/daemonset.yaml index 6a3de46..f113c9f 100644 --- a/gremlin/templates/daemonset.yaml +++ b/gremlin/templates/daemonset.yaml @@ -50,6 +50,9 @@ spec: {{- end }} spec: serviceAccountName: gremlin + {{- if include "gremlinGpuRuntimeClassName" . }} + runtimeClassName: {{ include "gremlinGpuRuntimeClassName" . }} + {{- end }} {{- if .Values.affinity }} affinity: {{ toYaml .Values.affinity | trimSuffix "\n" | nindent 8 }} {{- end }} @@ -163,6 +166,9 @@ spec: {{- if include "gremlinTlsIdentityEnv" . }} {{- include "gremlinTlsIdentityEnv" . | nindent 10 }} {{- end }} + {{- if include "gremlinGpuEnv" . }} + {{- include "gremlinGpuEnv" . | nindent 10 }} + {{- end }} {{- with .Values.gremlin.extraEnv }} {{- toYaml . | nindent 10 }} {{- end }} @@ -195,6 +201,9 @@ spec: {{- if include "gremlinTlsIdentityVolumeMounts" . }} {{- include "gremlinTlsIdentityVolumeMounts" . | nindent 10 }} {{- end }} + {{- if include "gremlinGpuVolumeMounts" . }} + {{- include "gremlinGpuVolumeMounts" . | nindent 10 }} + {{- end }} volumes: - name: cgroup-root hostPath: @@ -235,6 +244,9 @@ spec: {{- if include "gremlinTlsIdentityVolumes" . }} {{- include "gremlinTlsIdentityVolumes" . | nindent 8 }} {{- end }} + {{- if include "gremlinGpuVolumes" . }} + {{- include "gremlinGpuVolumes" . | nindent 8 }} + {{- end }} {{- if .Values.gremlin.priorityClassName }} priorityClassName: {{ .Values.gremlin.priorityClassName }} {{- end }} diff --git a/gremlin/tests/daemonset_gpu_test.yaml b/gremlin/tests/daemonset_gpu_test.yaml new file mode 100644 index 0000000..f992c72 --- /dev/null +++ b/gremlin/tests/daemonset_gpu_test.yaml @@ -0,0 +1,202 @@ +suite: Test GPU / OpenCL access options +templates: + - daemonset.yaml +release: + name: my-release + namespace: my-namespace + revision: 1 + upgrade: true +tests: + - it: should not add any GPU configuration by default + asserts: + - notExists: + path: spec.template.spec.runtimeClassName + - notContains: + path: spec.template.spec.volumes + content: + name: opencl-vendors + hostPath: + path: /etc/OpenCL/vendors + type: DirectoryOrCreate + + - it: should not add GPU configuration when a vendor is set but gpu is disabled + set: + gremlin.gpu.enabled: false + gremlin.gpu.vendor: nvidia + asserts: + - notExists: + path: spec.template.spec.runtimeClassName + - notContains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_VISIBLE_DEVICES + value: "all" + + # --- Easy default: nvidia preset --------------------------------------------------------- + - it: should apply the nvidia preset when enabled with the default vendor + set: + gremlin.gpu.enabled: true + asserts: + - equal: + path: spec.template.spec.runtimeClassName + value: nvidia + - contains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_VISIBLE_DEVICES + value: "all" + - contains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_DRIVER_CAPABILITIES + value: "all" + # nvidia relies on the container toolkit, so no GPU hostPath mounts are added + - notContains: + path: spec.template.spec.volumes + content: + name: opencl-vendors + hostPath: + path: /etc/OpenCL/vendors + type: DirectoryOrCreate + + # --- Easy default: amd preset ------------------------------------------------------------ + - it: should apply the amd preset when vendor is amd + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: amd + asserts: + # amd does not use a RuntimeClass + - notExists: + path: spec.template.spec.runtimeClassName + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: kfd + mountPath: /dev/kfd + readOnly: false + - contains: + path: spec.template.spec.volumes + content: + name: kfd + hostPath: + path: /dev/kfd + type: CharDevice + - contains: + path: spec.template.spec.volumes + content: + name: dri + hostPath: + path: /dev/dri + type: Directory + - contains: + path: spec.template.spec.volumes + content: + name: opencl-vendors + hostPath: + path: /etc/OpenCL/vendors + type: DirectoryOrCreate + + - it: should apply no preset when vendor is blank + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: "" + asserts: + - notExists: + path: spec.template.spec.runtimeClassName + - notContains: + path: spec.template.spec.volumes + content: + name: kfd + hostPath: + path: /dev/kfd + type: CharDevice + + # --- Advanced overrides ------------------------------------------------------------------ + - it: should let runtimeClassName override the vendor preset + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: nvidia + gremlin.gpu.runtimeClassName: nvidia-custom + asserts: + - equal: + path: spec.template.spec.runtimeClassName + value: nvidia-custom + + - it: should let env override the vendor preset + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: nvidia + gremlin.gpu.env: + - name: NVIDIA_VISIBLE_DEVICES + value: "0,1" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_VISIBLE_DEVICES + value: "0,1" + # the preset's NVIDIA_DRIVER_CAPABILITIES is replaced, not merged + - notContains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_DRIVER_CAPABILITIES + value: "all" + + - it: should let hostMounts override the vendor preset + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: amd + gremlin.gpu.hostMounts: + - name: opencl-vendors + hostPath: /etc/OpenCL/vendors + mountPath: /etc/OpenCL/vendors + type: DirectoryOrCreate + readOnly: true + - name: nvidia0 + hostPath: /dev/nvidia0 + type: CharDevice + readOnly: false + asserts: + # amd preset device nodes are replaced by the custom mounts + - notContains: + path: spec.template.spec.volumes + content: + name: kfd + hostPath: + path: /dev/kfd + type: CharDevice + # mountPath defaults to hostPath and readOnly is respected + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: nvidia0 + mountPath: /dev/nvidia0 + readOnly: false + - contains: + path: spec.template.spec.volumes + content: + name: nvidia0 + hostPath: + path: /dev/nvidia0 + type: CharDevice + + - it: should default readOnly to true and mountPath to hostPath when omitted + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: "" + gremlin.gpu.hostMounts: + - name: opencl-vendors + hostPath: /etc/OpenCL/vendors + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: opencl-vendors + mountPath: /etc/OpenCL/vendors + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: opencl-vendors + hostPath: + path: /etc/OpenCL/vendors diff --git a/gremlin/values.yaml b/gremlin/values.yaml index c6f7cf0..f0172c9 100644 --- a/gremlin/values.yaml +++ b/gremlin/values.yaml @@ -243,6 +243,82 @@ gremlin: # Attacks targeting other Kubernetes pods will use the resource specification of their target. resources: {} + # gremlin.gpu - + # Exposes the host's GPU / OpenCL drivers to the Gremlin agent so that the GPU attack can + # enumerate and target GPUs from inside the container. + # + # The easy path is to just set `enabled: true` and pick a `vendor` (e.g. "nvidia" or "amd"). + # Each vendor is a preset that fills in all the typical options + # for that hardware, so most users never need to touch the advanced options. + # + # The advanced options (runtimeClassName, env, hostMounts) remain available for custom setups. + # Any advanced option you set explicitly OVERRIDES the corresponding value from the vendor + # preset, so you can start from a preset and tweak a single field, or set vendor to "" and + # configure everything by hand. + gpu: + # gremlin.gpu.enabled - + # When true, applies the GPU configuration (vendor preset plus any advanced overrides below) + # to the Gremlin Daemonset. When false, none of the GPU settings have any effect. + enabled: false + + # gremlin.gpu.vendor - + # The easy default. Selects a preset that configures all the normal GPU options for you: + # nvidia - run under the "nvidia" RuntimeClass and set NVIDIA_VISIBLE_DEVICES / + # NVIDIA_DRIVER_CAPABILITIES=all so the NVIDIA container toolkit injects the driver + # libraries and device nodes. No hostMounts needed. + # amd - mount the /dev/kfd and /dev/dri device nodes and the OpenCL ICD registry + # (/etc/OpenCL/vendors) so the ROCm/amdgpu driver on the host is reachable. + # "" - no preset; configure everything via the advanced options below. + vendor: nvidia + + # --- Advanced options ------------------------------------------------------------------- + # Everything below overrides the vendor preset. Leave a field at its empty default to keep + # the preset's value for that field. + + # gremlin.gpu.runtimeClassName - + # The Kubernetes RuntimeClass to run the Gremlin Daemonset pods under. Overrides the vendor + # preset when set. Ignored when blank (the preset value, if any, is used). + runtimeClassName: "" + + # gremlin.gpu.env - + # Extra environment variables applied to the Gremlin Daemonset container when GPU access is + # enabled. Overrides the vendor preset's env when set to a non-empty list. For the NVIDIA + # container toolkit the capabilities value MUST include `compute` (or `all`) for OpenCL. + env: [] + # - name: NVIDIA_VISIBLE_DEVICES + # value: "all" + # - name: NVIDIA_DRIVER_CAPABILITIES + # value: "all" + + # gremlin.gpu.hostMounts - + # A list of hostPath volumes mounted into the Gremlin container so the OpenCL ICD loader can + # find vendor drivers and access GPU device nodes. Overrides the vendor preset's hostMounts + # when set to a non-empty list. Each entry accepts: + # - name: a unique volume name (required) + # - hostPath: the path on the Kubernetes node (required) + # - mountPath: the path inside the container (defaults to hostPath when omitted) + # - type: an optional hostPath type, e.g. Directory, DirectoryOrCreate, CharDevice + # - readOnly: whether the mount is read-only (defaults to true; set false for device nodes) + # + # NOTE: GPU device nodes (/dev/nvidia*, /dev/kfd, /dev/dri) require the container to have + # device access. This generally means running privileged (gremlin.podSecurity.privileged: + # true) or otherwise granting the pod cgroup device permissions. Device access is not + # required when the NVIDIA container toolkit injects the devices for you. + hostMounts: [] + # - name: opencl-vendors + # hostPath: /etc/OpenCL/vendors + # mountPath: /etc/OpenCL/vendors + # type: DirectoryOrCreate + # readOnly: true + # - name: nvidia0 + # hostPath: /dev/nvidia0 + # type: CharDevice + # readOnly: false + # # NVIDIA OpenCL driver library (path/version varies by distro and driver release) + # - name: nvidia-opencl-lib + # hostPath: /usr/lib/x86_64-linux-gnu/libnvidia-opencl.so.1 + # readOnly: true + secret: # Gremlin supports both `certificate` and `secret` types # To manage secrets with helm, set `managed=true` and fill in either the certificate auth or secret auth sections From 0281768b953c50dc75ddb45f5a55230bdea177af Mon Sep 17 00:00:00 2001 From: "Zach N." Date: Mon, 20 Jul 2026 08:18:44 -0700 Subject: [PATCH 2/9] Add runtimeclass creation --- gremlin/templates/runtimeclass.yaml | 24 ++++++++++++++++++++++++ gremlin/values.yaml | 8 ++++++++ 2 files changed, 32 insertions(+) create mode 100644 gremlin/templates/runtimeclass.yaml diff --git a/gremlin/templates/runtimeclass.yaml b/gremlin/templates/runtimeclass.yaml new file mode 100644 index 0000000..9163a72 --- /dev/null +++ b/gremlin/templates/runtimeclass.yaml @@ -0,0 +1,24 @@ +{{- /* +Creates the RuntimeClass referenced by the GPU configuration when it does not already exist. + +NOTE: `lookup` only sees the cluster during `helm install`/`upgrade`. Under `helm template` or a +GitOps renderer with no cluster access, `lookup` returns empty and the RuntimeClass is rendered. +Set gremlin.gpu.createRuntimeClass to false if you manage the RuntimeClass yourself. +*/ -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- $runtimeClassName := include "gremlinGpuRuntimeClassName" . -}} +{{- if and $runtimeClassName .Values.gremlin.gpu.createRuntimeClass -}} +{{- if not (lookup "node.k8s.io/v1" "RuntimeClass" "" $runtimeClassName) }} +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: {{ $runtimeClassName }} + labels: + app.kubernetes.io/name: {{ include "gremlin.name" . }} + helm.sh/chart: {{ include "gremlin.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +handler: {{ default $runtimeClassName .Values.gremlin.gpu.runtimeClassHandler | quote }} +{{- end }} +{{- end }} +{{- end }} diff --git a/gremlin/values.yaml b/gremlin/values.yaml index f0172c9..78848f0 100644 --- a/gremlin/values.yaml +++ b/gremlin/values.yaml @@ -271,6 +271,14 @@ gremlin: # "" - no preset; configure everything via the advanced options below. vendor: nvidia + # gremlin.gpu.createRuntimeClass - + # When true, and the effective runtimeClassName is set, the chart creates that RuntimeClass + # object IF it does not already exist in the cluster. + # + # An existing RuntimeClass of the same name is detected and left untouched, so this will not + # interfere with a RuntimeClass already managed by the GPU Operator or your platform. + createRuntimeClass: false + # --- Advanced options ------------------------------------------------------------------- # Everything below overrides the vendor preset. Leave a field at its empty default to keep # the preset's value for that field. From c5e802daeb7981b8f2f4fb8bee3a896e6e387709 Mon Sep 17 00:00:00 2001 From: "Zach N." Date: Tue, 21 Jul 2026 09:42:01 -0700 Subject: [PATCH 3/9] Fix ICD not mounting --- gremlin/templates/_helpers.tpl | 33 +++++++++++ gremlin/templates/opencl-icd-configmap.yaml | 26 +++++++++ gremlin/tests/daemonset_gpu_test.yaml | 35 ++++++++++++ gremlin/tests/opencl_icd_configmap_test.yaml | 59 ++++++++++++++++++++ gremlin/values.yaml | 19 +++++++ 5 files changed, 172 insertions(+) create mode 100644 gremlin/templates/opencl-icd-configmap.yaml create mode 100644 gremlin/tests/opencl_icd_configmap_test.yaml diff --git a/gremlin/templates/_helpers.tpl b/gremlin/templates/_helpers.tpl index 3053bf8..c746fb8 100644 --- a/gremlin/templates/_helpers.tpl +++ b/gremlin/templates/_helpers.tpl @@ -334,10 +334,32 @@ NVIDIA_DRIVER_CAPABILITIES (which must include `compute` or `all` for OpenCL). {{- end -}} {{- end -}} +{{/* +gremlinGpuOpenclIcdActive returns "true" when the chart should project an OpenCL ICD registry +file into the container, and nothing otherwise. This is the fix for NVIDIA runtimes that inject +the OpenCL driver library (libnvidia-opencl.so.1) but do NOT create /etc/OpenCL/vendors/nvidia.icd, +leaving clGetPlatformIDs with no platforms to enumerate. + +The ICD is projected automatically whenever GPU access is enabled and gremlin.gpu.openclIcd.library +is set, UNLESS an effective hostMount already provides /etc/OpenCL/vendors (e.g. the amd preset or +a custom mount), in which case we defer to that mount to avoid overlapping mount paths. +*/}} +{{- define "gremlinGpuOpenclIcdActive" -}} +{{- if and .Values.gremlin.gpu.enabled .Values.gremlin.gpu.openclIcd.library -}} +{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} +{{- $dirMounted := false -}} +{{- range $eff.hostMounts }} +{{- if eq (default .hostPath .mountPath) "/etc/OpenCL/vendors" }}{{- $dirMounted = true -}}{{- end }} +{{- end -}} +{{- if not $dirMounted -}}true{{- end -}} +{{- end -}} +{{- end -}} + {{/* gremlinGpuVolumeMounts returns the volumeMounts that expose host GPU/OpenCL drivers. Each effective hostMounts entry (vendor preset or the gremlin.gpu.hostMounts override) becomes a volumeMount. mountPath defaults to hostPath and readOnly defaults to true when not specified. +When gremlinGpuOpenclIcdActive, the projected OpenCL ICD file is mounted as well. */}} {{- define "gremlinGpuVolumeMounts" -}} {{- if .Values.gremlin.gpu.enabled -}} @@ -347,6 +369,12 @@ volumeMount. mountPath defaults to hostPath and readOnly defaults to true when n mountPath: {{ default .hostPath .mountPath }} readOnly: {{ if hasKey . "readOnly" }}{{ .readOnly }}{{ else }}true{{ end }} {{- end -}} +{{- if include "gremlinGpuOpenclIcdActive" . }} +- name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/{{ .Values.gremlin.gpu.openclIcd.filename }} + subPath: {{ .Values.gremlin.gpu.openclIcd.filename }} + readOnly: true +{{- end -}} {{- end -}} {{- end -}} @@ -366,6 +394,11 @@ hostPath volume. An optional `type` (e.g. DirectoryOrCreate, CharDevice) is pass type: {{ .type }} {{- end }} {{- end -}} +{{- if include "gremlinGpuOpenclIcdActive" . }} +- name: gremlin-opencl-icd + configMap: + name: {{ include "gremlin.fullname" . }}-opencl-icd +{{- end -}} {{- end -}} {{- end -}} diff --git a/gremlin/templates/opencl-icd-configmap.yaml b/gremlin/templates/opencl-icd-configmap.yaml new file mode 100644 index 0000000..89d17d8 --- /dev/null +++ b/gremlin/templates/opencl-icd-configmap.yaml @@ -0,0 +1,26 @@ +{{- /* +Projects an OpenCL ICD registry file into the Gremlin container. + +Some NVIDIA container runtimes inject the OpenCL driver library (libnvidia-opencl.so.1) but do +NOT create the /etc/OpenCL/vendors/.icd registry file that the ICD loader reads to +discover the driver. Without it, clGetPlatformIDs returns no platforms (surfacing in the GPU +attack as "GetPlatformIdsPlatformListUnavailable"). The ICD file is a one-line text file +containing the soname of the vendor's OpenCL library, which the loader dlopen()s. + +See gremlinGpuOpenclIcdActive for when this is rendered. +*/ -}} +{{- if include "gremlinGpuOpenclIcdActive" . }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "gremlin.fullname" . }}-opencl-icd + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ include "gremlin.chart" . }} + app.kubernetes.io/name: {{ include "gremlin.fullname" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + {{ .Values.gremlin.gpu.openclIcd.filename }}: | + {{ .Values.gremlin.gpu.openclIcd.library }} +{{- end }} diff --git a/gremlin/tests/daemonset_gpu_test.yaml b/gremlin/tests/daemonset_gpu_test.yaml index f992c72..7728bb6 100644 --- a/gremlin/tests/daemonset_gpu_test.yaml +++ b/gremlin/tests/daemonset_gpu_test.yaml @@ -58,6 +58,33 @@ tests: hostPath: path: /etc/OpenCL/vendors type: DirectoryOrCreate + # the OpenCL ICD registry file is projected so the injected driver is discoverable + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/nvidia.icd + subPath: nvidia.icd + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: gremlin-opencl-icd + configMap: + name: my-release-gremlin-opencl-icd + + - it: should not project the OpenCL ICD when the library is blank + set: + gremlin.gpu.enabled: true + gremlin.gpu.openclIcd.library: "" + asserts: + - notContains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/nvidia.icd + subPath: nvidia.icd + readOnly: true # --- Easy default: amd preset ------------------------------------------------------------ - it: should apply the amd preset when vendor is amd @@ -95,6 +122,14 @@ tests: hostPath: path: /etc/OpenCL/vendors type: DirectoryOrCreate + # amd already mounts /etc/OpenCL/vendors from the host, so the chart defers and does not + # project its own ICD file there + - notContains: + path: spec.template.spec.volumes + content: + name: gremlin-opencl-icd + configMap: + name: my-release-gremlin-opencl-icd - it: should apply no preset when vendor is blank set: diff --git a/gremlin/tests/opencl_icd_configmap_test.yaml b/gremlin/tests/opencl_icd_configmap_test.yaml new file mode 100644 index 0000000..7063ce8 --- /dev/null +++ b/gremlin/tests/opencl_icd_configmap_test.yaml @@ -0,0 +1,59 @@ +suite: Test OpenCL ICD registry ConfigMap +templates: + - opencl-icd-configmap.yaml +release: + name: my-release + namespace: my-namespace + revision: 1 + upgrade: true +tests: + - it: should not render when gpu is disabled + asserts: + - hasDocuments: + count: 0 + + - it: should render the nvidia ICD ConfigMap when gpu is enabled + set: + gremlin.gpu.enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: my-release-gremlin-opencl-icd + - equal: + path: data["nvidia.icd"] + value: | + libnvidia-opencl.so.1 + - matchRegex: + path: data["nvidia.icd"] + pattern: libnvidia-opencl\.so\.1 + + - it: should honor a custom filename and library + set: + gremlin.gpu.enabled: true + gremlin.gpu.openclIcd.filename: custom.icd + gremlin.gpu.openclIcd.library: libcustom-opencl.so.1 + asserts: + - equal: + path: data["custom.icd"] + value: | + libcustom-opencl.so.1 + + - it: should not render when the library is blank + set: + gremlin.gpu.enabled: true + gremlin.gpu.openclIcd.library: "" + asserts: + - hasDocuments: + count: 0 + + - it: should not render for the amd preset (host registry is mounted instead) + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: amd + asserts: + - hasDocuments: + count: 0 diff --git a/gremlin/values.yaml b/gremlin/values.yaml index 78848f0..52dcf05 100644 --- a/gremlin/values.yaml +++ b/gremlin/values.yaml @@ -279,6 +279,25 @@ gremlin: # interfere with a RuntimeClass already managed by the GPU Operator or your platform. createRuntimeClass: false + # gremlin.gpu.openclIcd - + # Projects an OpenCL ICD registry file into the container. Some NVIDIA container runtimes + # inject the OpenCL driver library (libnvidia-opencl.so.1) but do NOT create the + # /etc/OpenCL/vendors/nvidia.icd registry file the ICD loader reads to discover it. Without + # that file the GPU attack fails with "GetPlatformIdsPlatformListUnavailable" (no OpenCL + # platforms found) even though `nvidia-smi` works inside the pod. + # + # This is applied automatically whenever GPU access is enabled, EXCEPT when an effective + # hostMount already provides /etc/OpenCL/vendors (e.g. the amd preset mounts the host's + # registry), in which case the chart defers to that mount. + openclIcd: + # gremlin.gpu.openclIcd.filename - + # The ICD registry filename created under /etc/OpenCL/vendors/. + filename: nvidia.icd + # gremlin.gpu.openclIcd.library - + # The OpenCL vendor library soname written into the ICD file; the ICD loader dlopen()s it. + # Defaults to the NVIDIA driver library. Set to "" to disable ICD projection entirely. + library: libnvidia-opencl.so.1 + # --- Advanced options ------------------------------------------------------------------- # Everything below overrides the vendor preset. Leave a field at its empty default to keep # the preset's value for that field. From a5526c8ceb6e7f46714dfa1f75d3bbf1d685d89e Mon Sep 17 00:00:00 2001 From: "Zach N." Date: Tue, 21 Jul 2026 10:06:35 -0700 Subject: [PATCH 4/9] Simplify ICD loader config --- gremlin/templates/_helpers.tpl | 30 ++++++++++++++------ gremlin/templates/opencl-icd-configmap.yaml | 6 ++-- gremlin/tests/daemonset_gpu_test.yaml | 4 +-- gremlin/tests/opencl_icd_configmap_test.yaml | 15 ++-------- gremlin/values.yaml | 22 ++------------ 5 files changed, 31 insertions(+), 46 deletions(-) diff --git a/gremlin/templates/_helpers.tpl b/gremlin/templates/_helpers.tpl index c746fb8..94c4336 100644 --- a/gremlin/templates/_helpers.tpl +++ b/gremlin/templates/_helpers.tpl @@ -252,9 +252,14 @@ chart falls back to whatever the advanced options specify. nvidia - relies on the NVIDIA container toolkit: run under the "nvidia" RuntimeClass and set NVIDIA_VISIBLE_DEVICES / NVIDIA_DRIVER_CAPABILITIES so the runtime injects the driver - libraries and device nodes (no hostMounts required). + libraries and device nodes (no hostMounts required). Also projects the nvidia OpenCL + ICD registry file, since some runtimes inject libnvidia-opencl.so.1 but not the + /etc/OpenCL/vendors/nvidia.icd file the loader needs to discover it. amd - relies on the ROCm/amdgpu driver on the host: mount the /dev/kfd and /dev/dri device nodes plus the OpenCL ICD registry so the container can reach the GPUs directly. + +The optional `openclIcd` key (filename + library) tells the chart to project an OpenCL ICD +registry file; vendors that already expose /etc/OpenCL/vendors (amd) omit it. */}} {{- define "gremlinGpuVendorPreset" -}} {{- $vendor := . -}} @@ -266,6 +271,9 @@ env: - name: NVIDIA_DRIVER_CAPABILITIES value: "all" hostMounts: [] +openclIcd: + filename: nvidia.icd + library: libnvidia-opencl.so.1 {{- else if eq $vendor "amd" -}} runtimeClassName: "" env: [] @@ -294,7 +302,7 @@ hostMounts: [] gremlinGpuEffective returns a YAML dict with the effective GPU configuration: the vendor preset (see gremlinGpuVendorPreset) with each advanced option overriding the preset when set. A blank runtimeClassName, an empty env list, or an empty hostMounts list means "use the preset". -Keys: runtimeClassName (string), env (list), hostMounts (list). +Keys: runtimeClassName (string), env (list), hostMounts (list), openclIcd (dict, from the preset). */}} {{- define "gremlinGpuEffective" -}} {{- $gpu := .Values.gremlin.gpu -}} @@ -305,7 +313,9 @@ Keys: runtimeClassName (string), env (list), hostMounts (list). {{- if $gpu.env -}}{{- $env = $gpu.env -}}{{- end -}} {{- $hostMounts := $preset.hostMounts -}} {{- if $gpu.hostMounts -}}{{- $hostMounts = $gpu.hostMounts -}}{{- end -}} -{{- dict "runtimeClassName" $runtimeClassName "env" $env "hostMounts" $hostMounts | toYaml -}} +{{- $effective := dict "runtimeClassName" $runtimeClassName "env" $env "hostMounts" $hostMounts -}} +{{- if $preset.openclIcd -}}{{- $_ := set $effective "openclIcd" $preset.openclIcd -}}{{- end -}} +{{- $effective | toYaml -}} {{- end -}} {{/* @@ -340,13 +350,14 @@ file into the container, and nothing otherwise. This is the fix for NVIDIA runti the OpenCL driver library (libnvidia-opencl.so.1) but do NOT create /etc/OpenCL/vendors/nvidia.icd, leaving clGetPlatformIDs with no platforms to enumerate. -The ICD is projected automatically whenever GPU access is enabled and gremlin.gpu.openclIcd.library -is set, UNLESS an effective hostMount already provides /etc/OpenCL/vendors (e.g. the amd preset or -a custom mount), in which case we defer to that mount to avoid overlapping mount paths. +The ICD is projected automatically whenever the selected vendor preset defines one (nvidia does), +UNLESS an effective hostMount already provides /etc/OpenCL/vendors (e.g. the amd preset or a custom +mount), in which case we defer to that mount to avoid overlapping mount paths. */}} {{- define "gremlinGpuOpenclIcdActive" -}} -{{- if and .Values.gremlin.gpu.enabled .Values.gremlin.gpu.openclIcd.library -}} +{{- if .Values.gremlin.gpu.enabled -}} {{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} +{{- if and $eff.openclIcd $eff.openclIcd.library -}} {{- $dirMounted := false -}} {{- range $eff.hostMounts }} {{- if eq (default .hostPath .mountPath) "/etc/OpenCL/vendors" }}{{- $dirMounted = true -}}{{- end }} @@ -354,6 +365,7 @@ a custom mount), in which case we defer to that mount to avoid overlapping mount {{- if not $dirMounted -}}true{{- end -}} {{- end -}} {{- end -}} +{{- end -}} {{/* gremlinGpuVolumeMounts returns the volumeMounts that expose host GPU/OpenCL drivers. @@ -371,8 +383,8 @@ When gremlinGpuOpenclIcdActive, the projected OpenCL ICD file is mounted as well {{- end -}} {{- if include "gremlinGpuOpenclIcdActive" . }} - name: gremlin-opencl-icd - mountPath: /etc/OpenCL/vendors/{{ .Values.gremlin.gpu.openclIcd.filename }} - subPath: {{ .Values.gremlin.gpu.openclIcd.filename }} + mountPath: /etc/OpenCL/vendors/{{ $eff.openclIcd.filename }} + subPath: {{ $eff.openclIcd.filename }} readOnly: true {{- end -}} {{- end -}} diff --git a/gremlin/templates/opencl-icd-configmap.yaml b/gremlin/templates/opencl-icd-configmap.yaml index 89d17d8..ae13bd9 100644 --- a/gremlin/templates/opencl-icd-configmap.yaml +++ b/gremlin/templates/opencl-icd-configmap.yaml @@ -7,9 +7,11 @@ discover the driver. Without it, clGetPlatformIDs returns no platforms (surfacin attack as "GetPlatformIdsPlatformListUnavailable"). The ICD file is a one-line text file containing the soname of the vendor's OpenCL library, which the loader dlopen()s. +The ICD filename and library come from the selected vendor preset (see gremlinGpuVendorPreset). See gremlinGpuOpenclIcdActive for when this is rendered. */ -}} {{- if include "gremlinGpuOpenclIcdActive" . }} +{{- $eff := fromYaml (include "gremlinGpuEffective" .) }} apiVersion: v1 kind: ConfigMap metadata: @@ -21,6 +23,6 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: - {{ .Values.gremlin.gpu.openclIcd.filename }}: | - {{ .Values.gremlin.gpu.openclIcd.library }} + {{ $eff.openclIcd.filename }}: | + {{ $eff.openclIcd.library }} {{- end }} diff --git a/gremlin/tests/daemonset_gpu_test.yaml b/gremlin/tests/daemonset_gpu_test.yaml index 7728bb6..cace05c 100644 --- a/gremlin/tests/daemonset_gpu_test.yaml +++ b/gremlin/tests/daemonset_gpu_test.yaml @@ -73,10 +73,10 @@ tests: configMap: name: my-release-gremlin-opencl-icd - - it: should not project the OpenCL ICD when the library is blank + - it: should not project the OpenCL ICD when no vendor preset defines one set: gremlin.gpu.enabled: true - gremlin.gpu.openclIcd.library: "" + gremlin.gpu.vendor: "" asserts: - notContains: path: spec.template.spec.containers[0].volumeMounts diff --git a/gremlin/tests/opencl_icd_configmap_test.yaml b/gremlin/tests/opencl_icd_configmap_test.yaml index 7063ce8..96e7977 100644 --- a/gremlin/tests/opencl_icd_configmap_test.yaml +++ b/gremlin/tests/opencl_icd_configmap_test.yaml @@ -31,21 +31,10 @@ tests: path: data["nvidia.icd"] pattern: libnvidia-opencl\.so\.1 - - it: should honor a custom filename and library + - it: should not render when no vendor preset defines an ICD set: gremlin.gpu.enabled: true - gremlin.gpu.openclIcd.filename: custom.icd - gremlin.gpu.openclIcd.library: libcustom-opencl.so.1 - asserts: - - equal: - path: data["custom.icd"] - value: | - libcustom-opencl.so.1 - - - it: should not render when the library is blank - set: - gremlin.gpu.enabled: true - gremlin.gpu.openclIcd.library: "" + gremlin.gpu.vendor: "" asserts: - hasDocuments: count: 0 diff --git a/gremlin/values.yaml b/gremlin/values.yaml index 52dcf05..5617282 100644 --- a/gremlin/values.yaml +++ b/gremlin/values.yaml @@ -265,7 +265,8 @@ gremlin: # The easy default. Selects a preset that configures all the normal GPU options for you: # nvidia - run under the "nvidia" RuntimeClass and set NVIDIA_VISIBLE_DEVICES / # NVIDIA_DRIVER_CAPABILITIES=all so the NVIDIA container toolkit injects the driver - # libraries and device nodes. No hostMounts needed. + # libraries and device nodes. Also projects the nvidia OpenCL ICD registry file so + # the injected driver is discoverable. No hostMounts needed. # amd - mount the /dev/kfd and /dev/dri device nodes and the OpenCL ICD registry # (/etc/OpenCL/vendors) so the ROCm/amdgpu driver on the host is reachable. # "" - no preset; configure everything via the advanced options below. @@ -279,25 +280,6 @@ gremlin: # interfere with a RuntimeClass already managed by the GPU Operator or your platform. createRuntimeClass: false - # gremlin.gpu.openclIcd - - # Projects an OpenCL ICD registry file into the container. Some NVIDIA container runtimes - # inject the OpenCL driver library (libnvidia-opencl.so.1) but do NOT create the - # /etc/OpenCL/vendors/nvidia.icd registry file the ICD loader reads to discover it. Without - # that file the GPU attack fails with "GetPlatformIdsPlatformListUnavailable" (no OpenCL - # platforms found) even though `nvidia-smi` works inside the pod. - # - # This is applied automatically whenever GPU access is enabled, EXCEPT when an effective - # hostMount already provides /etc/OpenCL/vendors (e.g. the amd preset mounts the host's - # registry), in which case the chart defers to that mount. - openclIcd: - # gremlin.gpu.openclIcd.filename - - # The ICD registry filename created under /etc/OpenCL/vendors/. - filename: nvidia.icd - # gremlin.gpu.openclIcd.library - - # The OpenCL vendor library soname written into the ICD file; the ICD loader dlopen()s it. - # Defaults to the NVIDIA driver library. Set to "" to disable ICD projection entirely. - library: libnvidia-opencl.so.1 - # --- Advanced options ------------------------------------------------------------------- # Everything below overrides the vendor preset. Leave a field at its empty default to keep # the preset's value for that field. From ed9b656fbd1a82d8955ee0d257ce13661f4de13d Mon Sep 17 00:00:00 2001 From: "Zach N." Date: Tue, 21 Jul 2026 12:10:22 -0700 Subject: [PATCH 5/9] Support Nvidia CDI config, clean up docs --- gremlin/templates/_helpers.tpl | 37 +++++++++------- gremlin/templates/daemonset.yaml | 5 ++- gremlin/templates/opencl-icd-configmap.yaml | 4 +- gremlin/tests/daemonset_gpu_test.yaml | 45 ++++++++++++++++++++ gremlin/tests/opencl_icd_configmap_test.yaml | 8 ++++ gremlin/values.yaml | 38 +++++++++++++---- 6 files changed, 109 insertions(+), 28 deletions(-) diff --git a/gremlin/templates/_helpers.tpl b/gremlin/templates/_helpers.tpl index 94c4336..551caa0 100644 --- a/gremlin/templates/_helpers.tpl +++ b/gremlin/templates/_helpers.tpl @@ -245,10 +245,9 @@ When createSecret or existingSecret are configured {{- end -}} {{/* -gremlinGpuVendorPreset returns a YAML dict of the "easy default" GPU configuration for a -given vendor. The context passed in is the vendor string (gremlin.gpu.vendor). Supported -values are "nvidia" and "amd"; any other value (including "") yields an empty preset so the -chart falls back to whatever the advanced options specify. +gremlinGpuVendorPreset returns the GPU configuration for a given vendor. The context passed in +is the vendor string (gremlin.gpu.vendor). Supported values are "nvidia" and "amd"; any other value +(including "") yields an empty preset so the chart falls back to whatever the advanced options specify. nvidia - relies on the NVIDIA container toolkit: run under the "nvidia" RuntimeClass and set NVIDIA_VISIBLE_DEVICES / NVIDIA_DRIVER_CAPABILITIES so the runtime injects the driver @@ -299,10 +298,8 @@ hostMounts: [] {{- end -}} {{/* -gremlinGpuEffective returns a YAML dict with the effective GPU configuration: the vendor -preset (see gremlinGpuVendorPreset) with each advanced option overriding the preset when set. -A blank runtimeClassName, an empty env list, or an empty hostMounts list means "use the preset". -Keys: runtimeClassName (string), env (list), hostMounts (list), openclIcd (dict, from the preset). +gremlinGpuEffective returns the vendor preset (see gremlinGpuVendorPreset) with each advanced +option overriding the preset when set. */}} {{- define "gremlinGpuEffective" -}} {{- $gpu := .Values.gremlin.gpu -}} @@ -344,18 +341,28 @@ NVIDIA_DRIVER_CAPABILITIES (which must include `compute` or `all` for OpenCL). {{- end -}} {{- end -}} +{{/* +gremlinGpuCdiAnnotation returns the CDI device pod annotation when gremlin.gpu.cdiDevice is set, +and nothing otherwise. This targets the annotation-based CDI injection path supported by +containerd >= 1.7 / CRI-O, which does not consume a schedulable GPU resource. +*/}} +{{- define "gremlinGpuCdiAnnotation" -}} +{{- if and .Values.gremlin.gpu.enabled .Values.gremlin.gpu.cdiDevice -}} +cdi.k8s.io/gremlin-gpu: {{ .Values.gremlin.gpu.cdiDevice | quote }} +{{- end -}} +{{- end -}} + {{/* gremlinGpuOpenclIcdActive returns "true" when the chart should project an OpenCL ICD registry -file into the container, and nothing otherwise. This is the fix for NVIDIA runtimes that inject -the OpenCL driver library (libnvidia-opencl.so.1) but do NOT create /etc/OpenCL/vendors/nvidia.icd, -leaving clGetPlatformIDs with no platforms to enumerate. +file into the container, and nothing otherwise. -The ICD is projected automatically whenever the selected vendor preset defines one (nvidia does), -UNLESS an effective hostMount already provides /etc/OpenCL/vendors (e.g. the amd preset or a custom -mount), in which case we defer to that mount to avoid overlapping mount paths. +The ICD is projected automatically whenever gremlin.gpu.projectOpenclIcd is true and the selected +vendor preset defines one (nvidia does), UNLESS an effective hostMount already provides +/etc/OpenCL/vendors (e.g. the amd preset or a custom mount), in which case we defer to that mount +to avoid overlapping mount paths. */}} {{- define "gremlinGpuOpenclIcdActive" -}} -{{- if .Values.gremlin.gpu.enabled -}} +{{- if and .Values.gremlin.gpu.enabled .Values.gremlin.gpu.projectOpenclIcd -}} {{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} {{- if and $eff.openclIcd $eff.openclIcd.library -}} {{- $dirMounted := false -}} diff --git a/gremlin/templates/daemonset.yaml b/gremlin/templates/daemonset.yaml index f113c9f..7a8a6bf 100644 --- a/gremlin/templates/daemonset.yaml +++ b/gremlin/templates/daemonset.yaml @@ -31,7 +31,7 @@ spec: {{- if .Values.gremlin.podLabels }} {{- toYaml .Values.gremlin.podLabels | nindent 8 }} {{- end }} - {{- if or .Values.gremlin.apparmor .Values.gremlin.installApparmorProfile .Values.gremlin.podSecurity.seccomp.enabled .Values.gremlin.podSecurity.securityContextConstraints.create .Values.gremlin.podAnnotations }} + {{- if or .Values.gremlin.apparmor .Values.gremlin.installApparmorProfile .Values.gremlin.podSecurity.seccomp.enabled .Values.gremlin.podSecurity.securityContextConstraints.create .Values.gremlin.podAnnotations (include "gremlinGpuCdiAnnotation" .) }} annotations: {{- if .Values.gremlin.apparmor }} container.apparmor.security.beta.kubernetes.io/{{ .Chart.Name }}: {{ .Values.gremlin.apparmor }} @@ -44,6 +44,9 @@ spec: {{- if .Values.gremlin.podSecurity.securityContextConstraints.create }} openshift.io/required-scc: "gremlin" {{- end }} + {{- if include "gremlinGpuCdiAnnotation" . }} + {{- include "gremlinGpuCdiAnnotation" . | nindent 8 }} + {{- end }} {{- if .Values.gremlin.podAnnotations }} {{- toYaml .Values.gremlin.podAnnotations | nindent 8 }} {{- end }} diff --git a/gremlin/templates/opencl-icd-configmap.yaml b/gremlin/templates/opencl-icd-configmap.yaml index ae13bd9..13651f1 100644 --- a/gremlin/templates/opencl-icd-configmap.yaml +++ b/gremlin/templates/opencl-icd-configmap.yaml @@ -3,9 +3,7 @@ Projects an OpenCL ICD registry file into the Gremlin container. Some NVIDIA container runtimes inject the OpenCL driver library (libnvidia-opencl.so.1) but do NOT create the /etc/OpenCL/vendors/.icd registry file that the ICD loader reads to -discover the driver. Without it, clGetPlatformIDs returns no platforms (surfacing in the GPU -attack as "GetPlatformIdsPlatformListUnavailable"). The ICD file is a one-line text file -containing the soname of the vendor's OpenCL library, which the loader dlopen()s. +discover the driver. Without it, the OpenCL platforms cannot be listed. The ICD filename and library come from the selected vendor preset (see gremlinGpuVendorPreset). See gremlinGpuOpenclIcdActive for when this is rendered. diff --git a/gremlin/tests/daemonset_gpu_test.yaml b/gremlin/tests/daemonset_gpu_test.yaml index cace05c..d95d784 100644 --- a/gremlin/tests/daemonset_gpu_test.yaml +++ b/gremlin/tests/daemonset_gpu_test.yaml @@ -235,3 +235,48 @@ tests: name: opencl-vendors hostPath: path: /etc/OpenCL/vendors + + # --- CDI (Container Device Interface) ---------------------------------------------------- + - it: should not add a CDI annotation by default + set: + gremlin.gpu.enabled: true + asserts: + - notExists: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + + - it: should add the CDI device annotation when cdiDevice is set + set: + gremlin.gpu.enabled: true + gremlin.gpu.cdiDevice: nvidia.com/gpu=all + asserts: + - equal: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + value: nvidia.com/gpu=all + + - it: should not add a CDI annotation when gpu is disabled + set: + gremlin.gpu.enabled: false + gremlin.gpu.cdiDevice: nvidia.com/gpu=all + asserts: + - notExists: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + + # --- ICD projection toggle --------------------------------------------------------------- + - it: should not project the OpenCL ICD when projectOpenclIcd is false + set: + gremlin.gpu.enabled: true + gremlin.gpu.projectOpenclIcd: false + asserts: + - notContains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/nvidia.icd + subPath: nvidia.icd + readOnly: true + - notContains: + path: spec.template.spec.volumes + content: + name: gremlin-opencl-icd + configMap: + name: my-release-gremlin-opencl-icd diff --git a/gremlin/tests/opencl_icd_configmap_test.yaml b/gremlin/tests/opencl_icd_configmap_test.yaml index 96e7977..8f62a32 100644 --- a/gremlin/tests/opencl_icd_configmap_test.yaml +++ b/gremlin/tests/opencl_icd_configmap_test.yaml @@ -39,6 +39,14 @@ tests: - hasDocuments: count: 0 + - it: should not render when projectOpenclIcd is false + set: + gremlin.gpu.enabled: true + gremlin.gpu.projectOpenclIcd: false + asserts: + - hasDocuments: + count: 0 + - it: should not render for the amd preset (host registry is mounted instead) set: gremlin.gpu.enabled: true diff --git a/gremlin/values.yaml b/gremlin/values.yaml index 5617282..0c09835 100644 --- a/gremlin/values.yaml +++ b/gremlin/values.yaml @@ -248,8 +248,8 @@ gremlin: # enumerate and target GPUs from inside the container. # # The easy path is to just set `enabled: true` and pick a `vendor` (e.g. "nvidia" or "amd"). - # Each vendor is a preset that fills in all the typical options - # for that hardware, so most users never need to touch the advanced options. + # Each vendor is a preset that fills in all the typical options for that hardware, + # so most users never need to touch the advanced options. # # The advanced options (runtimeClassName, env, hostMounts) remain available for custom setups. # Any advanced option you set explicitly OVERRIDES the corresponding value from the vendor @@ -274,25 +274,45 @@ gremlin: # gremlin.gpu.createRuntimeClass - # When true, and the effective runtimeClassName is set, the chart creates that RuntimeClass - # object IF it does not already exist in the cluster. + # object IF it does not already exist in the cluster. A non-existent RuntimeClass will cause + # the gremlin pod to fail to start. # # An existing RuntimeClass of the same name is detected and left untouched, so this will not # interfere with a RuntimeClass already managed by the GPU Operator or your platform. createRuntimeClass: false + # gremlin.gpu.cdiDevice - + # For nodes using the Container Device Interface (CDI) rather than the container-toolkit + # runtime hook. When set, the chart adds a `cdi.k8s.io/gremlin-gpu: ` pod annotation + # so the CRI injects the named CDI device(s). Use this when NVIDIA_VISIBLE_DEVICES is NOT + # honored by your runtime (pure Kubernetes-native CDI). + # + # Requires containerd >= 1.7 or CRI-O with CDI enabled. + # + # NOTE: in pure CDI mode NVIDIA_DRIVER_CAPABILITIES is ignored (capabilities come from the CDI + # device), and you likely do not need a RuntimeClass — set runtimeClassName: "" below if the + # "nvidia" RuntimeClass does not exist on your cluster. + cdiDevice: "" + + # gremlin.gpu.projectOpenclIcd - + # Whether to project the vendor preset's OpenCL ICD registry file (e.g. /etc/OpenCL/vendors/ + # nvidia.icd) into the container. Defaults to true, which fixes runtimes that inject the + # OpenCL driver library but not the ICD file. Set to false when your CDI spec (or another + # mechanism) already creates the ICD file, to avoid a conflicting mount at the same path. + projectOpenclIcd: true + # --- Advanced options ------------------------------------------------------------------- # Everything below overrides the vendor preset. Leave a field at its empty default to keep # the preset's value for that field. # gremlin.gpu.runtimeClassName - - # The Kubernetes RuntimeClass to run the Gremlin Daemonset pods under. Overrides the vendor - # preset when set. Ignored when blank (the preset value, if any, is used). + # The Kubernetes RuntimeClass to run the Gremlin Daemonset pods under. runtimeClassName: "" # gremlin.gpu.env - # Extra environment variables applied to the Gremlin Daemonset container when GPU access is - # enabled. Overrides the vendor preset's env when set to a non-empty list. For the NVIDIA - # container toolkit the capabilities value MUST include `compute` (or `all`) for OpenCL. + # enabled. For the NVIDIA container toolkit the capabilities value MUST include `compute` + # (or `all`) for OpenCL. env: [] # - name: NVIDIA_VISIBLE_DEVICES # value: "all" @@ -301,8 +321,8 @@ gremlin: # gremlin.gpu.hostMounts - # A list of hostPath volumes mounted into the Gremlin container so the OpenCL ICD loader can - # find vendor drivers and access GPU device nodes. Overrides the vendor preset's hostMounts - # when set to a non-empty list. Each entry accepts: + # find vendor drivers and access GPU device nodes. + # Each entry accepts: # - name: a unique volume name (required) # - hostPath: the path on the Kubernetes node (required) # - mountPath: the path inside the container (defaults to hostPath when omitted) From cd043d55e9be40b6a40de4cbda456d3ee95669a5 Mon Sep 17 00:00:00 2001 From: "Zach N." Date: Wed, 22 Jul 2026 04:43:41 -0700 Subject: [PATCH 6/9] Remove orphaned variable --- gremlin/templates/runtimeclass.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gremlin/templates/runtimeclass.yaml b/gremlin/templates/runtimeclass.yaml index 9163a72..bc88697 100644 --- a/gremlin/templates/runtimeclass.yaml +++ b/gremlin/templates/runtimeclass.yaml @@ -18,7 +18,7 @@ metadata: helm.sh/chart: {{ include "gremlin.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} -handler: {{ default $runtimeClassName .Values.gremlin.gpu.runtimeClassHandler | quote }} +handler: {{ $runtimeClassName | quote }} {{- end }} {{- end }} {{- end }} From 9048fe0beb4efdf9fa047a46c9412dddc00567a4 Mon Sep 17 00:00:00 2001 From: "Zach N." Date: Wed, 22 Jul 2026 12:18:15 -0700 Subject: [PATCH 7/9] Add README changes --- gremlin/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/gremlin/README.md b/gremlin/README.md index f9b786b..fb67cc9 100644 --- a/gremlin/README.md +++ b/gremlin/README.md @@ -74,6 +74,14 @@ their default values. See values.yaml for all available options. | `gremlin.proxy.url` | Specifies the http proxy the agent should use to communicate with api.gremlin.com. | `""` (ignored) | | | `gremlin.extraEnv` | Specify any arbitrary environment variables to pass to the Gremlin Agent daemonset. | `[]` | | `gremlin.features.discoverDestinationService.enabled` | Enable discovery of a destination service in a service mesh to resolve hostnames | `false` | +| `gremlin.gpu.enabled` | Expose host GPU/OpenCL drivers to the agent for the GPU attack | `false` | +| `gremlin.gpu.vendor` | GPU vendor preset (`nvidia`, `amd`, or `""` for manual config) | `nvidia` | +| `gremlin.gpu.createRuntimeClass` | Create the effective RuntimeClass if it doesn't already exist | `false` | +| `gremlin.gpu.cdiDevice` | CDI device to inject via pod annotation (for CDI-based runtimes) | `""` | +| `gremlin.gpu.projectOpenclIcd` | Project the vendor's OpenCL ICD registry file into the container | `true` | +| `gremlin.gpu.runtimeClassName` | Override the RuntimeClass to run the agent under | `""` (uses vendor preset) | +| `gremlin.gpu.env` | Override GPU environment variables on the agent container | `[]` (uses vendor preset) | +| `gremlin.gpu.hostMounts` | Override hostPath mounts for GPU drivers and device nodes | `[]` (uses vendor preset) | | `ssl.certFile` | Add a certificate file to Gremlin's set of certificate authorities. This argument expects a file containing the certificate(s) you wish to add. When set, this chart creates secret (`ssl-cert-file`) with the contents and passes it to both agents. This value is ignored when blank or absent. | `""` (ignored) | | `ssl.certDir` | sets the SSL_CERT_DIR environment variable on the both agents. Unlike ssl.certFile, this value accepts only a path to an existing directory on the Kubernetes nodes. This value is ignored when blank or absent. | `""` (ignored) | @@ -213,6 +221,24 @@ helm install gremlin gremlin/gremlin \ --set-file ssl.certFile=$HOME/Workspace/proxy/ca.pem ``` +### With GPU Support + +To let the GPU attack enumerate and target GPUs, enable `gremlin.gpu` and pick a vendor preset. + +```shell +helm install gremlin gremlin/gremlin \ + --namespace gremlin \ + --set gremlin.secret.managed=true \ + --set gremlin.secret.teamID=$GREMLIN_TEAM_ID \ + --set gremlin.secret.clusterID=$GREMLIN_CLUSTER_ID \ + --set-file gremlin.secret.certificate=/path/to/gremlin.cert \ + --set-file gremlin.secret.key=/path/to/gremlin.key \ + --set gremlin.gpu.enabled=true \ + --set gremlin.gpu.vendor=nvidia +``` + +_note_: The `nvidia` preset runs the agent under the `nvidia` RuntimeClass. If the Gremlin pod fails to start (for example, a `RuntimeClass not found` error), the RuntimeClass likely doesn't exist on your cluster. Have the chart create it by also setting `--set gremlin.gpu.createRuntimeClass=true`. + ## Uninstallation ```shell From 7be99b5a2e5445bd846e1181672eda2c30282e00 Mon Sep 17 00:00:00 2001 From: "Zach N." Date: Wed, 22 Jul 2026 14:56:41 -0700 Subject: [PATCH 8/9] bump chart version --- gremlin/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gremlin/Chart.yaml b/gremlin/Chart.yaml index 65e4e37..be142c2 100644 --- a/gremlin/Chart.yaml +++ b/gremlin/Chart.yaml @@ -1,5 +1,5 @@ name: gremlin -version: 0.26.0 +version: 0.27.0 description: The Gremlin Inc client application apiVersion: v1 home: https://www.gremlin.com From 9bbff57ffca48498944f5fa9f2cb9420bc08618d Mon Sep 17 00:00:00 2001 From: "Zach N." Date: Wed, 29 Jul 2026 00:26:34 -0700 Subject: [PATCH 9/9] Address comments, add multi-daemon mode for GPU --- gremlin/README.md | 27 +- gremlin/templates/NOTES.txt | 11 + gremlin/templates/_daemonset.tpl | 299 ++++++++++++++++++++ gremlin/templates/_helpers.tpl | 219 +++++--------- gremlin/templates/daemonset.yaml | 295 +++---------------- gremlin/templates/opencl-icd-configmap.yaml | 25 +- gremlin/templates/runtimeclass.yaml | 24 -- gremlin/tests/daemonset_gpu_test.yaml | 97 +++---- gremlin/tests/notes_test.yaml | 14 + gremlin/values.yaml | 192 +++++++------ 10 files changed, 621 insertions(+), 582 deletions(-) create mode 100644 gremlin/templates/_daemonset.tpl delete mode 100644 gremlin/templates/runtimeclass.yaml diff --git a/gremlin/README.md b/gremlin/README.md index fb67cc9..57448bb 100644 --- a/gremlin/README.md +++ b/gremlin/README.md @@ -75,13 +75,11 @@ their default values. See values.yaml for all available options. | `gremlin.extraEnv` | Specify any arbitrary environment variables to pass to the Gremlin Agent daemonset. | `[]` | | `gremlin.features.discoverDestinationService.enabled` | Enable discovery of a destination service in a service mesh to resolve hostnames | `false` | | `gremlin.gpu.enabled` | Expose host GPU/OpenCL drivers to the agent for the GPU attack | `false` | -| `gremlin.gpu.vendor` | GPU vendor preset (`nvidia`, `amd`, or `""` for manual config) | `nvidia` | -| `gremlin.gpu.createRuntimeClass` | Create the effective RuntimeClass if it doesn't already exist | `false` | +| `gremlin.gpu.vendor` | Fallback vendor block used when the chart cannot inspect the cluster's Nodes (`nvidia`, `amd`, `custom`, ...) | `nvidia` | | `gremlin.gpu.cdiDevice` | CDI device to inject via pod annotation (for CDI-based runtimes) | `""` | -| `gremlin.gpu.projectOpenclIcd` | Project the vendor's OpenCL ICD registry file into the container | `true` | -| `gremlin.gpu.runtimeClassName` | Override the RuntimeClass to run the agent under | `""` (uses vendor preset) | -| `gremlin.gpu.env` | Override GPU environment variables on the agent container | `[]` (uses vendor preset) | -| `gremlin.gpu.hostMounts` | Override hostPath mounts for GPU drivers and device nodes | `[]` (uses vendor preset) | +| `gremlin.gpu.projectOpenclIcd` | Project the selected vendor's OpenCL ICD registry file into the container | `true` | +| `gremlin.gpu.vendors` | Vendor blocks to detect & target on a hybrid (mixed) cluster | `[nvidia, amd]` | +| `gremlin.gpu.` | Per-vendor config block: `nodeSelector`, `resources`, `runtimeClassName`, `env`, `volumes`, `volumeMounts`, `openclIcd` | see `values.yaml` | | `ssl.certFile` | Add a certificate file to Gremlin's set of certificate authorities. This argument expects a file containing the certificate(s) you wish to add. When set, this chart creates secret (`ssl-cert-file`) with the contents and passes it to both agents. This value is ignored when blank or absent. | `""` (ignored) | | `ssl.certDir` | sets the SSL_CERT_DIR environment variable on the both agents. Unlike ssl.certFile, this value accepts only a path to an existing directory on the Kubernetes nodes. This value is ignored when blank or absent. | `""` (ignored) | @@ -237,7 +235,22 @@ helm install gremlin gremlin/gremlin \ --set gremlin.gpu.vendor=nvidia ``` -_note_: The `nvidia` preset runs the agent under the `nvidia` RuntimeClass. If the Gremlin pod fails to start (for example, a `RuntimeClass not found` error), the RuntimeClass likely doesn't exist on your cluster. Have the chart create it by also setting `--set gremlin.gpu.createRuntimeClass=true`. +_note_: The `nvidia` preset runs the agent under the `nvidia` RuntimeClass. If the Gremlin pod fails to start (for example, a `RuntimeClass not found` error), the RuntimeClass likely doesn't exist on your cluster. This chart does not create RuntimeClass objects. Ensure the RuntimeClass named by the vendor block exists (it is normally provided by the NVIDIA GPU Operator or your platform), or set the vendor's `runtimeClassName` to `""`. + +#### Hybrid (mixed) clusters + +On a cluster where only some nodes have GPUs and different nodes may have different GPU vendors, a single GPU DaemonSet cannot run cluster-wide: nodes lacking the vendor's RuntimeClass or device mounts would fail to start the Gremlin pod. So whenever `gremlin.gpu.enabled` is set, the chart inspects the cluster's Nodes at install/upgrade time and renders: + +- one GPU DaemonSet per detected vendor (`-gremlin-gpu-`), scheduled via node affinity onto that vendor's nodes and carrying that vendor's GPU configuration, and +- one plain DaemonSet (`-gremlin`) for every remaining non-GPU node. + +The vendors to look for come from `gremlin.gpu.vendors`. Detection and scheduling use each vendor block's `nodeSelector` (node labels such as `nvidia.com/gpu.present` / `amd.com/gpu.present`, set by the NVIDIA GPU Operator / Node Feature Discovery and the AMD GPU labeller), plus optional `resources` (node capacity) for detection. GPU nodes must carry the vendor's `nodeSelector` labels for its DaemonSet to schedule. + +Because node inspection relies on Helm's `lookup`, it only runs during `helm install`/`upgrade`. Under `helm template` or a GitOps renderer with no cluster access, detection finds no nodes and the chart falls back to a single DaemonSet using `gremlin.gpu.vendor`. If that fallback happens during an actual `helm install`/`upgrade` (for example, the account running Helm cannot list Nodes), the chart prints a warning to the console noting that per-node GPU DaemonSets could not be created. + +This chart does not create RuntimeClass objects; any RuntimeClass named by a vendor block must already exist on the cluster. + +_note_: The per-node DaemonSets carry an extra `gremlin.com/gpu` pod-selector label so each only manages its own pods. Because a DaemonSet's selector is immutable, upgrading an existing GPU-less Gremlin install to a GPU one may require deleting the old DaemonSet first (or `helm upgrade --force`). ## Uninstallation diff --git a/gremlin/templates/NOTES.txt b/gremlin/templates/NOTES.txt index 53d227b..e5d7d2b 100644 --- a/gremlin/templates/NOTES.txt +++ b/gremlin/templates/NOTES.txt @@ -1 +1,12 @@ {{- include "gremlin.validateValues" $}} +{{- if and .Values.gremlin.gpu.enabled (not (include "gremlinGpuNodesVisible" .)) }} + +WARNING: GPU support is enabled, but this chart could not list the cluster's Nodes so the Kubernetes +`lookup` returned nothing. Per-node GPU DaemonSets could NOT be created, so a single GPU DaemonSet +using gremlin.gpu.vendor ({{ .Values.gremlin.gpu.vendor }}) was rendered instead. On a mixed cluster +that DaemonSet will fail to start on any node that lacks the GPU RuntimeClass or device mounts. + +This usually means the account running Helm cannot list Nodes (RBAC), or the cluster genuinely has +no Nodes yet. To have the agent split per node, re-run `helm upgrade` with an account that can list +Nodes. +{{- end }} diff --git a/gremlin/templates/_daemonset.tpl b/gremlin/templates/_daemonset.tpl new file mode 100644 index 0000000..00b52b8 --- /dev/null +++ b/gremlin/templates/_daemonset.tpl @@ -0,0 +1,299 @@ +{{/* +gremlin.daemonset renders one Gremlin DaemonSet from an already-resolved context. Deciding how many +DaemonSets exist and what goes in each is the caller's job (see daemonset.yaml); this template only +renders what it is handed. + +Context: dict + "root" $ # the chart root context + "name" # DaemonSet name + "selectorLabels" # extra selector/pod labels, so DaemonSets sharing a cluster do not + # fight over each other's pods (optional; default none) + "gpu" # a gremlin.gpu. config block (optional; default no GPU config) + "affinity" # rendered affinity (optional; defaults to .Values.affinity) +*/}} +{{- define "gremlin.daemonset" -}} +{{- $root := .root -}} +{{- $name := .name -}} +{{- $selectorLabels := default (dict) .selectorLabels -}} +{{- $gpu := default (dict) .gpu -}} +{{- $affinity := .affinity -}} +{{- if and (not $affinity) $root.Values.affinity -}}{{- $affinity = toYaml $root.Values.affinity -}}{{- end -}} +{{- /* CDI device: global gremlin.gpu.cdiDevice, optionally overridden per vendor block */ -}} +{{- $cdiDevice := "" -}} +{{- with $gpu -}}{{- $cdiDevice = default $root.Values.gremlin.gpu.cdiDevice .cdiDevice -}}{{- end -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "gremlin.name" $root }} + helm.sh/chart: {{ include "gremlin.chart" $root }} + app.kubernetes.io/instance: {{ $root.Release.Name }} + app.kubernetes.io/managed-by: {{ $root.Release.Service }} + version: v1 + {{- with $selectorLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if $root.Values.gremlin.podLabels }} + {{- toYaml $root.Values.gremlin.podLabels | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "gremlin.name" $root }} + {{- with $selectorLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if $root.Values.gremlin.updateStrategy }} + updateStrategy: + {{- toYaml $root.Values.gremlin.updateStrategy | nindent 4 }} + {{- end }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "gremlin.name" $root }} + helm.sh/chart: {{ include "gremlin.chart" $root }} + app.kubernetes.io/instance: {{ $root.Release.Name }} + app.kubernetes.io/managed-by: {{ $root.Release.Service }} + version: v1 + {{- with $selectorLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if $root.Values.gremlin.podLabels }} + {{- toYaml $root.Values.gremlin.podLabels | nindent 8 }} + {{- end }} + {{- if or $root.Values.gremlin.apparmor $root.Values.gremlin.installApparmorProfile $root.Values.gremlin.podSecurity.seccomp.enabled $root.Values.gremlin.podSecurity.securityContextConstraints.create $root.Values.gremlin.podAnnotations $cdiDevice }} + annotations: + {{- if $root.Values.gremlin.apparmor }} + container.apparmor.security.beta.kubernetes.io/{{ $root.Chart.Name }}: {{ $root.Values.gremlin.apparmor }} + {{- else if $root.Values.gremlin.installApparmorProfile }} + container.apparmor.security.beta.kubernetes.io/{{ $root.Chart.Name }}: {{ "localhost/gremlin-agent" }} + {{- end }} + {{- if $root.Values.gremlin.podSecurity.seccomp.enabled }} + container.seccomp.security.alpha.kubernetes.io/{{ $root.Chart.Name }}: {{ $root.Values.gremlin.podSecurity.seccomp.profile }} + {{- end }} + {{- if $root.Values.gremlin.podSecurity.securityContextConstraints.create }} + openshift.io/required-scc: "gremlin" + {{- end }} + {{- if $cdiDevice }} + cdi.k8s.io/gremlin-gpu: {{ $cdiDevice | quote }} + {{- end }} + {{- if $root.Values.gremlin.podAnnotations }} + {{- toYaml $root.Values.gremlin.podAnnotations | nindent 8 }} + {{- end }} + {{- end }} + spec: + serviceAccountName: gremlin + {{- with $gpu.runtimeClassName }} + runtimeClassName: {{ . }} + {{- end }} + {{- with $affinity }} + affinity: {{ . | trimSuffix "\n" | nindent 8 }} + {{- end }} + {{- if $root.Values.nodeSelector }} + nodeSelector: {{ toYaml $root.Values.nodeSelector | trimSuffix "\n" | nindent 8 }} + {{- end }} + {{- if $root.Values.tolerations }} + tolerations: {{ toYaml $root.Values.tolerations | trimSuffix "\n" | nindent 8 }} + {{- end }} + dnsPolicy: {{ $root.Values.gremlin.dnsPolicy }} + hostPID: {{ $root.Values.gremlin.hostPID }} + hostNetwork: {{ $root.Values.gremlin.hostNetwork }} + {{- if $root.Values.image.pullSecret }} + imagePullSecrets: + - name: {{ $root.Values.image.pullSecret }} + {{- end }} + {{- if and $root.Values.gremlin.podSecurity.seccomp.enabled (eq "localhost/gremlin" $root.Values.gremlin.podSecurity.seccomp.profile) }} + initContainers: + - name: seccomp-init + image: {{ $root.Values.image.repository }}:{{ $root.Values.image.tag }} + imagePullPolicy: {{ $root.Values.image.pullPolicy }} + volumeMounts: + - mountPath: {{ $root.Values.gremlin.podSecurity.seccomp.root }} + name: seccomp-root + - mountPath: /gremlin + name: seccomp-profile + command: + - cp + - /gremlin/seccomp.json + - {{ $root.Values.gremlin.podSecurity.seccomp.root }}/gremlin + {{- end }} + containers: + - name: {{ $root.Chart.Name }} + image: {{ $root.Values.image.repository }}:{{ $root.Values.image.tag }} + args: [ "daemon" ] + imagePullPolicy: {{ $root.Values.image.pullPolicy }} + {{- if $root.Values.gremlin.resources }} + resources: {{ toYaml $root.Values.gremlin.resources | nindent 10 }} + {{- end }} + securityContext: + privileged: {{ $root.Values.gremlin.podSecurity.privileged }} + allowPrivilegeEscalation: {{ $root.Values.gremlin.podSecurity.allowPrivilegeEscalation }} + capabilities: + add: {{ toYaml $root.Values.gremlin.podSecurity.capabilities | nindent 14 }} + {{- if $root.Values.gremlin.podSecurity.seLinuxOptions }} + seLinuxOptions: {{ toYaml $root.Values.gremlin.podSecurity.seLinuxOptions | nindent 12 }} + {{- end }} + readOnlyRootFilesystem: {{ $root.Values.gremlin.podSecurity.readOnlyRootFilesystem }} + env: + - name: GREMLIN_TEAM_ID + {{- /* If we aren't managing this secret and a teamID was supplied, assume teamID is not in the external secret */}} + {{- if (and (not $root.Values.gremlin.secret.managed) (default $root.Values.gremlin.teamID $root.Values.gremlin.secret.teamID)) }} + value: {{ default $root.Values.gremlin.teamID $root.Values.gremlin.secret.teamID | quote }} + {{- else }} + valueFrom: + secretKeyRef: + name: {{ include "gremlin.secretName" $root }} + key: GREMLIN_TEAM_ID + {{- end }} + + {{- if (eq (include "gremlin.secretType" $root) "secret") }} + - name: GREMLIN_TEAM_SECRET + valueFrom: + secretKeyRef: + name: {{ include "gremlin.secretName" $root }} + key: GREMLIN_TEAM_SECRET + {{- else }} + - name: GREMLIN_TEAM_CERTIFICATE_OR_FILE + {{- /* If managed outside of this chart, or if the value is a literal, reference the secret as a file */}} + {{- if or (not $root.Values.gremlin.secret.managed) (hasPrefix "-----BEGIN" $root.Values.gremlin.secret.certificate) }} + value: file:///var/lib/gremlin/cert/gremlin.cert + {{- else }} + value: {{ $root.Values.gremlin.secret.certificate }} + {{- end }} + - name: GREMLIN_TEAM_PRIVATE_KEY_OR_FILE + {{- /* If managed outside of this chart, or if the value is a literal, reference the secret as a file */}} + {{- if or (not $root.Values.gremlin.secret.managed) (hasPrefix "-----BEGIN" $root.Values.gremlin.secret.certificate) }} + value: file:///var/lib/gremlin/cert/gremlin.key + {{- else }} + value: {{ $root.Values.gremlin.secret.key }} + {{- end }} + {{- end }} + - name: GREMLIN_IDENTIFIER + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: GREMLIN_CLIENT_TAGS + value: {{ $root.Values.gremlin.client.tags }} + - name: GREMLIN_COLLECT_DNS + value: {{ $root.Values.gremlin.collect.dns | quote }} + - name: GREMLIN_SERVICE_URL + value: {{ include "gremlinServiceUrl" $root }} + {{- if not $root.Values.gremlin.features.pushCIDRTags.enabled }} + - name: GREMLIN_PUSH_POD_CIDR_TAGS + value: "false" + - name: GREMLIN_PUSH_ZONE_CIDR_TAGS + value: "false" + {{- end }} + {{- if $root.Values.gremlin.proxy.url }} + - name: https_proxy + value: {{ $root.Values.gremlin.proxy.url }} + {{- end }} + {{- if $root.Values.ssl.certFile }} + - name: SSL_CERT_FILE + value: /etc/gremlin/ssl/certfile.pem + {{- end }} + {{- if $root.Values.ssl.certDir }} + - name: SSL_CERT_DIR + value: {{ $root.Values.ssl.certDir }} + {{- end }} + {{- if include "gremlinTlsIdentityEnv" $root }} + {{- include "gremlinTlsIdentityEnv" $root | nindent 10 }} + {{- end }} + {{- with $gpu.env }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with $root.Values.gremlin.extraEnv }} + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + - name: gremlin-state + mountPath: /var/lib/gremlin + readOnly: false + - name: gremlin-executions + mountPath: /var/lib/gremlin/executions + readOnly: false + - name: gremlin-logs + mountPath: /var/log/gremlin + readOnly: false + - name: cgroup-root + mountPath: /sys/fs/cgroup + readOnly: false + {{- if include "containerMounts" $root }} + {{- include "containerMounts" $root | nindent 10 }} + {{- end }} + {{- if (eq (include "gremlin.secretType" $root) "certificate") }} + - name: gremlin-cert + mountPath: /var/lib/gremlin/cert + readOnly: true + {{- end }} + {{- if $root.Values.ssl.certFile }} + - name: ssl-cert-file + mountPath: /etc/gremlin/ssl + readOnly: true + {{- end }} + {{- if include "gremlinTlsIdentityVolumeMounts" $root }} + {{- include "gremlinTlsIdentityVolumeMounts" $root | nindent 10 }} + {{- end }} + {{- with $gpu.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if include "gremlinGpuOpenclIcdActiveFromSpec" (dict "root" $root "gpu" $gpu) }} + - name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/{{ $gpu.openclIcd.filename }} + subPath: {{ $gpu.openclIcd.filename }} + readOnly: true + {{- end }} + volumes: + - name: cgroup-root + hostPath: + path: {{ $root.Values.gremlin.cgroup.root }} + {{- if include "containerVolumes" $root }} + {{- include "containerVolumes" $root | nindent 8 }} + {{- end }} + # The Gremlin daemon communicates with Gremlin sidecars via its state directory. + - name: gremlin-state + emptyDir: + medium: Memory + - name: gremlin-executions + hostPath: + path: /var/lib/gremlin/executions + # The Gremlin daemon forwards logs from the Gremlin sidecars to the Gremlin control plane + # These logs should be shared with the host + - name: gremlin-logs + hostPath: + path: /var/log/gremlin + {{- if (eq (include "gremlin.secretType" $root) "certificate") }} + - name: gremlin-cert + secret: + secretName: {{ include "gremlin.secretName" $root }} + {{- end }} + {{- if and $root.Values.gremlin.podSecurity.seccomp.enabled (eq "localhost/gremlin" $root.Values.gremlin.podSecurity.seccomp.profile) }} + - name: seccomp-root + hostPath: + path: {{ $root.Values.gremlin.podSecurity.seccomp.root }} + - name: seccomp-profile + configMap: + name: {{ template "gremlin.fullname" $root }}-seccomp + {{- end }} + {{- if $root.Values.ssl.certFile }} + - name: ssl-cert-file + secret: + secretName: ssl-cert-file + {{- end }} + {{- if include "gremlinTlsIdentityVolumes" $root }} + {{- include "gremlinTlsIdentityVolumes" $root | nindent 8 }} + {{- end }} + {{- with $gpu.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if include "gremlinGpuOpenclIcdActiveFromSpec" (dict "root" $root "gpu" $gpu) }} + - name: gremlin-opencl-icd + configMap: + name: {{ include "gremlin.fullname" $root }}-opencl-icd + {{- end }} +{{- if $root.Values.gremlin.priorityClassName }} + priorityClassName: {{ $root.Values.gremlin.priorityClassName }} +{{- end }} +{{- end -}} diff --git a/gremlin/templates/_helpers.tpl b/gremlin/templates/_helpers.tpl index 551caa0..2fa35c1 100644 --- a/gremlin/templates/_helpers.tpl +++ b/gremlin/templates/_helpers.tpl @@ -245,179 +245,92 @@ When createSecret or existingSecret are configured {{- end -}} {{/* -gremlinGpuVendorPreset returns the GPU configuration for a given vendor. The context passed in -is the vendor string (gremlin.gpu.vendor). Supported values are "nvidia" and "amd"; any other value -(including "") yields an empty preset so the chart falls back to whatever the advanced options specify. - - nvidia - relies on the NVIDIA container toolkit: run under the "nvidia" RuntimeClass and set - NVIDIA_VISIBLE_DEVICES / NVIDIA_DRIVER_CAPABILITIES so the runtime injects the driver - libraries and device nodes (no hostMounts required). Also projects the nvidia OpenCL - ICD registry file, since some runtimes inject libnvidia-opencl.so.1 but not the - /etc/OpenCL/vendors/nvidia.icd file the loader needs to discover it. - amd - relies on the ROCm/amdgpu driver on the host: mount the /dev/kfd and /dev/dri device - nodes plus the OpenCL ICD registry so the container can reach the GPUs directly. - -The optional `openclIcd` key (filename + library) tells the chart to project an OpenCL ICD -registry file; vendors that already expose /etc/OpenCL/vendors (amd) omit it. -*/}} -{{- define "gremlinGpuVendorPreset" -}} -{{- $vendor := . -}} -{{- if eq $vendor "nvidia" -}} -runtimeClassName: nvidia -env: - - name: NVIDIA_VISIBLE_DEVICES - value: "all" - - name: NVIDIA_DRIVER_CAPABILITIES - value: "all" -hostMounts: [] -openclIcd: - filename: nvidia.icd - library: libnvidia-opencl.so.1 -{{- else if eq $vendor "amd" -}} -runtimeClassName: "" -env: [] -hostMounts: - - name: kfd - hostPath: /dev/kfd - type: CharDevice - readOnly: false - - name: dri - hostPath: /dev/dri - type: Directory - readOnly: false - - name: opencl-vendors - hostPath: /etc/OpenCL/vendors - mountPath: /etc/OpenCL/vendors - type: DirectoryOrCreate - readOnly: true -{{- else -}} -runtimeClassName: "" -env: [] -hostMounts: [] -{{- end -}} -{{- end -}} - -{{/* -gremlinGpuEffective returns the vendor preset (see gremlinGpuVendorPreset) with each advanced -option overriding the preset when set. +gremlinGpuOpenclIcdActiveFromSpec returns "true" when the OpenCL ICD file should be projected: +gremlin.gpu.projectOpenclIcd is set and the vendor block defines openclIcd (filename + library). +Context: dict "root" $ "gpu" */}} -{{- define "gremlinGpuEffective" -}} -{{- $gpu := .Values.gremlin.gpu -}} -{{- $preset := fromYaml (include "gremlinGpuVendorPreset" (default "" $gpu.vendor)) -}} -{{- $runtimeClassName := $preset.runtimeClassName -}} -{{- if $gpu.runtimeClassName -}}{{- $runtimeClassName = $gpu.runtimeClassName -}}{{- end -}} -{{- $env := $preset.env -}} -{{- if $gpu.env -}}{{- $env = $gpu.env -}}{{- end -}} -{{- $hostMounts := $preset.hostMounts -}} -{{- if $gpu.hostMounts -}}{{- $hostMounts = $gpu.hostMounts -}}{{- end -}} -{{- $effective := dict "runtimeClassName" $runtimeClassName "env" $env "hostMounts" $hostMounts -}} -{{- if $preset.openclIcd -}}{{- $_ := set $effective "openclIcd" $preset.openclIcd -}}{{- end -}} -{{- $effective | toYaml -}} +{{- define "gremlinGpuOpenclIcdActiveFromSpec" -}} +{{- $icd := .gpu.openclIcd -}} +{{- if and .root.Values.gremlin.gpu.projectOpenclIcd $icd $icd.filename $icd.library -}}true{{- end -}} {{- end -}} {{/* -gremlinGpuRuntimeClassName returns the effective RuntimeClass name for the Gremlin pods, or -nothing when GPU access is disabled or no RuntimeClass applies. +gremlinGpuNodesVisible returns "true" when GPU access is on AND the cluster's Nodes are visible via +`lookup`, which is what enables the per-node (mixed cluster) DaemonSets. lookup is empty under +`helm template`/GitOps, so the chart then falls back to a single DaemonSet driven by +gremlin.gpu.vendor. When GPU is on and this is empty, NOTES.txt warns that the split could not be +made. */}} -{{- define "gremlinGpuRuntimeClassName" -}} +{{- define "gremlinGpuNodesVisible" -}} {{- if .Values.gremlin.gpu.enabled -}} -{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} -{{- $eff.runtimeClassName -}} -{{- end -}} -{{- end -}} - -{{/* -gremlinGpuEnv returns the environment variables that enable GPU/OpenCL access. -When gremlin.gpu.enabled is true, the effective env entries (vendor preset or the gremlin.gpu.env -override) are emitted. For the NVIDIA container toolkit these include NVIDIA_VISIBLE_DEVICES and -NVIDIA_DRIVER_CAPABILITIES (which must include `compute` or `all` for OpenCL). -*/}} -{{- define "gremlinGpuEnv" -}} -{{- if .Values.gremlin.gpu.enabled -}} -{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} -{{- with $eff.env -}} -{{- toYaml . -}} -{{- end -}} -{{- end -}} +{{- if gt (len (default (list) (lookup "v1" "Node" "" "").items)) 0 -}} +{{- true -}} {{- end -}} - -{{/* -gremlinGpuCdiAnnotation returns the CDI device pod annotation when gremlin.gpu.cdiDevice is set, -and nothing otherwise. This targets the annotation-based CDI injection path supported by -containerd >= 1.7 / CRI-O, which does not consume a schedulable GPU resource. -*/}} -{{- define "gremlinGpuCdiAnnotation" -}} -{{- if and .Values.gremlin.gpu.enabled .Values.gremlin.gpu.cdiDevice -}} -cdi.k8s.io/gremlin-gpu: {{ .Values.gremlin.gpu.cdiDevice | quote }} {{- end -}} {{- end -}} {{/* -gremlinGpuOpenclIcdActive returns "true" when the chart should project an OpenCL ICD registry -file into the container, and nothing otherwise. - -The ICD is projected automatically whenever gremlin.gpu.projectOpenclIcd is true and the selected -vendor preset defines one (nvidia does), UNLESS an effective hostMount already provides -/etc/OpenCL/vendors (e.g. the amd preset or a custom mount), in which case we defer to that mount -to avoid overlapping mount paths. +gremlinGpuVendorsInUse returns the vendor blocks the install needs, comma-separated: those of +gremlin.gpu.vendors whose nodes are present in the cluster, or just gremlin.gpu.vendor when the Nodes +are not visible (the single-DaemonSet fallback). Empty when GPU access is disabled. A node counts for +a vendor when it carries all of the vendor block's nodeSelector labels, or advertises any of its +`resources` in the node's capacity. */}} -{{- define "gremlinGpuOpenclIcdActive" -}} -{{- if and .Values.gremlin.gpu.enabled .Values.gremlin.gpu.projectOpenclIcd -}} -{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} -{{- if and $eff.openclIcd $eff.openclIcd.library -}} -{{- $dirMounted := false -}} -{{- range $eff.hostMounts }} -{{- if eq (default .hostPath .mountPath) "/etc/OpenCL/vendors" }}{{- $dirMounted = true -}}{{- end }} -{{- end -}} -{{- if not $dirMounted -}}true{{- end -}} -{{- end -}} +{{- define "gremlinGpuVendorsInUse" -}} +{{- if not (include "gremlinGpuNodesVisible" .) -}} +{{- if .Values.gremlin.gpu.enabled -}}{{- .Values.gremlin.gpu.vendor -}}{{- end -}} +{{- else -}} +{{- $root := . -}} +{{- $nodes := default (list) (lookup "v1" "Node" "" "").items -}} +{{- $active := list -}} +{{- range $name := (default (list) $root.Values.gremlin.gpu.vendors) -}} +{{- $vspec := default (dict) (index $root.Values.gremlin.gpu $name) -}} +{{- $found := false -}} +{{- range $node := $nodes -}} +{{- $labels := default (dict) $node.metadata.labels -}} +{{- if $vspec.nodeSelector -}} +{{- $match := true -}} +{{- range $lk, $lv := $vspec.nodeSelector -}}{{- if ne (toString (index $labels $lk)) (toString $lv) -}}{{- $match = false -}}{{- end -}}{{- end -}} +{{- if $match -}}{{- $found = true -}}{{- end -}} {{- end -}} +{{- range $r := (default (list) $vspec.resources) -}}{{- if hasKey (default (dict) $node.status.capacity) $r -}}{{- $found = true -}}{{- end -}}{{- end -}} {{- end -}} - -{{/* -gremlinGpuVolumeMounts returns the volumeMounts that expose host GPU/OpenCL drivers. -Each effective hostMounts entry (vendor preset or the gremlin.gpu.hostMounts override) becomes a -volumeMount. mountPath defaults to hostPath and readOnly defaults to true when not specified. -When gremlinGpuOpenclIcdActive, the projected OpenCL ICD file is mounted as well. -*/}} -{{- define "gremlinGpuVolumeMounts" -}} -{{- if .Values.gremlin.gpu.enabled -}} -{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} -{{- range $eff.hostMounts }} -- name: {{ .name }} - mountPath: {{ default .hostPath .mountPath }} - readOnly: {{ if hasKey . "readOnly" }}{{ .readOnly }}{{ else }}true{{ end }} -{{- end -}} -{{- if include "gremlinGpuOpenclIcdActive" . }} -- name: gremlin-opencl-icd - mountPath: /etc/OpenCL/vendors/{{ $eff.openclIcd.filename }} - subPath: {{ $eff.openclIcd.filename }} - readOnly: true +{{- if $found -}}{{- $active = append $active $name -}}{{- end -}} {{- end -}} +{{- join "," $active -}} {{- end -}} {{- end -}} {{/* -gremlinGpuVolumes returns the hostPath volumes that expose host GPU/OpenCL drivers. -Each effective hostMounts entry (vendor preset or the gremlin.gpu.hostMounts override) becomes a -hostPath volume. An optional `type` (e.g. DirectoryOrCreate, CharDevice) is passed through when present. +gremlinGpuNodeAffinity returns the user's .Values.affinity (as YAML) with an added requirement that a +node carry ("In") or lack ("NotIn") the nodeSelector labels of the given vendor blocks. The +requirement is ANDed into every nodeSelectorTerm the user supplied (or a new term when they supplied +none), so GPU node targeting and a user's affinity are both honored. Renders nothing when those +vendor blocks define no nodeSelector, rather than an empty (API-invalid) nodeSelectorTerm. +Context: dict "root" $ "vendors" "operator" "In" | "NotIn" */}} -{{- define "gremlinGpuVolumes" -}} -{{- if .Values.gremlin.gpu.enabled -}} -{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} -{{- range $eff.hostMounts }} -- name: {{ .name }} - hostPath: - path: {{ .hostPath }} - {{- if .type }} - type: {{ .type }} - {{- end }} -{{- end -}} -{{- if include "gremlinGpuOpenclIcdActive" . }} -- name: gremlin-opencl-icd - configMap: - name: {{ include "gremlin.fullname" . }}-opencl-icd -{{- end -}} +{{- define "gremlinGpuNodeAffinity" -}} +{{- $root := .root -}} +{{- $operator := .operator -}} +{{- $exprs := list -}} +{{- range $vendor := .vendors -}} +{{- range $k, $v := (default (dict) (index $root.Values.gremlin.gpu $vendor)).nodeSelector -}} +{{- $exprs = append $exprs (dict "key" $k "operator" $operator "values" (list (toString $v))) -}} +{{- end -}} +{{- end -}} +{{- if $exprs -}} +{{- $affinity := deepCopy (default (dict) $root.Values.affinity) -}} +{{- $nodeAffinity := default (dict) $affinity.nodeAffinity -}} +{{- $required := default (dict) $nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution -}} +{{- /* the terms are dicts, so `set` updates them in place inside $terms */ -}} +{{- $terms := default (list (dict)) $required.nodeSelectorTerms -}} +{{- range $term := $terms -}} +{{- $_ := set $term "matchExpressions" (concat (default (list) $term.matchExpressions) $exprs) -}} +{{- end -}} +{{- $_ := set $required "nodeSelectorTerms" $terms -}} +{{- $_ := set $nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" $required -}} +{{- $_ := set $affinity "nodeAffinity" $nodeAffinity -}} +{{- $affinity | toYaml -}} {{- end -}} {{- end -}} diff --git a/gremlin/templates/daemonset.yaml b/gremlin/templates/daemonset.yaml index 7a8a6bf..ff3d3f4 100644 --- a/gremlin/templates/daemonset.yaml +++ b/gremlin/templates/daemonset.yaml @@ -1,255 +1,44 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ include "gremlin.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "gremlin.name" . }} - helm.sh/chart: {{ include "gremlin.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - version: v1 - {{- if .Values.gremlin.podLabels }} - {{- toYaml .Values.gremlin.podLabels | nindent 4 }} - {{- end }} -spec: - selector: - matchLabels: - app.kubernetes.io/name: {{ include "gremlin.name" . }} - {{- if .Values.gremlin.updateStrategy }} - updateStrategy: - {{- toYaml .Values.gremlin.updateStrategy | nindent 4 }} - {{- end }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ include "gremlin.name" . }} - helm.sh/chart: {{ include "gremlin.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - version: v1 - {{- if .Values.gremlin.podLabels }} - {{- toYaml .Values.gremlin.podLabels | nindent 8 }} - {{- end }} - {{- if or .Values.gremlin.apparmor .Values.gremlin.installApparmorProfile .Values.gremlin.podSecurity.seccomp.enabled .Values.gremlin.podSecurity.securityContextConstraints.create .Values.gremlin.podAnnotations (include "gremlinGpuCdiAnnotation" .) }} - annotations: - {{- if .Values.gremlin.apparmor }} - container.apparmor.security.beta.kubernetes.io/{{ .Chart.Name }}: {{ .Values.gremlin.apparmor }} - {{- else if .Values.gremlin.installApparmorProfile }} - container.apparmor.security.beta.kubernetes.io/{{ .Chart.Name }}: {{ "localhost/gremlin-agent" }} - {{- end }} - {{- if .Values.gremlin.podSecurity.seccomp.enabled }} - container.seccomp.security.alpha.kubernetes.io/{{ .Chart.Name }}: {{ .Values.gremlin.podSecurity.seccomp.profile }} - {{- end }} - {{- if .Values.gremlin.podSecurity.securityContextConstraints.create }} - openshift.io/required-scc: "gremlin" - {{- end }} - {{- if include "gremlinGpuCdiAnnotation" . }} - {{- include "gremlinGpuCdiAnnotation" . | nindent 8 }} - {{- end }} - {{- if .Values.gremlin.podAnnotations }} - {{- toYaml .Values.gremlin.podAnnotations | nindent 8 }} - {{- end }} - {{- end }} - spec: - serviceAccountName: gremlin - {{- if include "gremlinGpuRuntimeClassName" . }} - runtimeClassName: {{ include "gremlinGpuRuntimeClassName" . }} - {{- end }} - {{- if .Values.affinity }} - affinity: {{ toYaml .Values.affinity | trimSuffix "\n" | nindent 8 }} - {{- end }} - {{- if .Values.nodeSelector }} - nodeSelector: {{ toYaml .Values.nodeSelector | trimSuffix "\n" | nindent 8 }} - {{- end }} - {{- if .Values.tolerations }} - tolerations: {{ toYaml .Values.tolerations | trimSuffix "\n" | nindent 8 }} - {{- end }} - dnsPolicy: {{ .Values.gremlin.dnsPolicy }} - hostPID: {{ .Values.gremlin.hostPID }} - hostNetwork: {{ .Values.gremlin.hostNetwork }} - {{- if .Values.image.pullSecret }} - imagePullSecrets: - - name: {{ .Values.image.pullSecret }} - {{- end }} - {{- if and .Values.gremlin.podSecurity.seccomp.enabled (eq "localhost/gremlin" .Values.gremlin.podSecurity.seccomp.profile) }} - initContainers: - - name: seccomp-init - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - volumeMounts: - - mountPath: {{ .Values.gremlin.podSecurity.seccomp.root }} - name: seccomp-root - - mountPath: /gremlin - name: seccomp-profile - command: - - cp - - /gremlin/seccomp.json - - {{ .Values.gremlin.podSecurity.seccomp.root }}/gremlin - {{- end }} - containers: - - name: {{ .Chart.Name }} - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - args: [ "daemon" ] - imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if .Values.gremlin.resources }} - resources: {{ toYaml .Values.gremlin.resources | nindent 10 }} - {{- end }} - securityContext: - privileged: {{ .Values.gremlin.podSecurity.privileged }} - allowPrivilegeEscalation: {{ .Values.gremlin.podSecurity.allowPrivilegeEscalation }} - capabilities: - add: {{ toYaml .Values.gremlin.podSecurity.capabilities | nindent 14 }} - {{- if .Values.gremlin.podSecurity.seLinuxOptions }} - seLinuxOptions: {{ toYaml .Values.gremlin.podSecurity.seLinuxOptions | nindent 12 }} - {{- end }} - readOnlyRootFilesystem: {{ .Values.gremlin.podSecurity.readOnlyRootFilesystem }} - env: - - name: GREMLIN_TEAM_ID - {{- /* If we aren't managing this secret and a teamID was supplied, assume teamID is not in the external secret */}} - {{- if (and (not .Values.gremlin.secret.managed) (default .Values.gremlin.teamID .Values.gremlin.secret.teamID)) }} - value: {{ default .Values.gremlin.teamID .Values.gremlin.secret.teamID | quote }} - {{- else }} - valueFrom: - secretKeyRef: - name: {{ include "gremlin.secretName" . }} - key: GREMLIN_TEAM_ID - {{- end }} +{{- /* +Decides which Gremlin DaemonSet(s) exist and resolves each one's name, pod-selector labels, GPU +vendor config and node affinity. The DaemonSet body itself is rendered by the "gremlin.daemonset" +template (see _daemonset.tpl), which knows nothing about GPUs or node mixes. - {{- if (eq (include "gremlin.secretType" .) "secret") }} - - name: GREMLIN_TEAM_SECRET - valueFrom: - secretKeyRef: - name: {{ include "gremlin.secretName" . }} - key: GREMLIN_TEAM_SECRET - {{- else }} - - name: GREMLIN_TEAM_CERTIFICATE_OR_FILE - {{- /* If managed outside of this chart, or if the value is a literal, reference the secret as a file */}} - {{- if or (not .Values.gremlin.secret.managed) (hasPrefix "-----BEGIN" .Values.gremlin.secret.certificate) }} - value: file:///var/lib/gremlin/cert/gremlin.cert - {{- else }} - value: {{ .Values.gremlin.secret.certificate }} - {{- end }} - - name: GREMLIN_TEAM_PRIVATE_KEY_OR_FILE - {{- /* If managed outside of this chart, or if the value is a literal, reference the secret as a file */}} - {{- if or (not .Values.gremlin.secret.managed) (hasPrefix "-----BEGIN" .Values.gremlin.secret.certificate) }} - value: file:///var/lib/gremlin/cert/gremlin.key - {{- else }} - value: {{ .Values.gremlin.secret.key }} - {{- end }} - {{- end }} - - name: GREMLIN_IDENTIFIER - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: GREMLIN_CLIENT_TAGS - value: {{ .Values.gremlin.client.tags }} - - name: GREMLIN_COLLECT_DNS - value: {{ .Values.gremlin.collect.dns | quote }} - - name: GREMLIN_SERVICE_URL - value: {{ include "gremlinServiceUrl" . }} - {{- if not .Values.gremlin.features.pushCIDRTags.enabled }} - - name: GREMLIN_PUSH_POD_CIDR_TAGS - value: "false" - - name: GREMLIN_PUSH_ZONE_CIDR_TAGS - value: "false" - {{- end }} - {{- if .Values.gremlin.proxy.url }} - - name: https_proxy - value: {{ .Values.gremlin.proxy.url }} - {{- end }} - {{- if .Values.ssl.certFile }} - - name: SSL_CERT_FILE - value: /etc/gremlin/ssl/certfile.pem - {{- end }} - {{- if .Values.ssl.certDir }} - - name: SSL_CERT_DIR - value: {{ .Values.ssl.certDir }} - {{- end }} - {{- if include "gremlinTlsIdentityEnv" . }} - {{- include "gremlinTlsIdentityEnv" . | nindent 10 }} - {{- end }} - {{- if include "gremlinGpuEnv" . }} - {{- include "gremlinGpuEnv" . | nindent 10 }} - {{- end }} - {{- with .Values.gremlin.extraEnv }} - {{- toYaml . | nindent 10 }} - {{- end }} - volumeMounts: - - name: gremlin-state - mountPath: /var/lib/gremlin - readOnly: false - - name: gremlin-executions - mountPath: /var/lib/gremlin/executions - readOnly: false - - name: gremlin-logs - mountPath: /var/log/gremlin - readOnly: false - - name: cgroup-root - mountPath: /sys/fs/cgroup - readOnly: false - {{- if include "containerMounts" . }} - {{- include "containerMounts" . | nindent 10 }} - {{- end }} - {{- if (eq (include "gremlin.secretType" .) "certificate") }} - - name: gremlin-cert - mountPath: /var/lib/gremlin/cert - readOnly: true - {{- end }} - {{- if .Values.ssl.certFile }} - - name: ssl-cert-file - mountPath: /etc/gremlin/ssl - readOnly: true - {{- end }} - {{- if include "gremlinTlsIdentityVolumeMounts" . }} - {{- include "gremlinTlsIdentityVolumeMounts" . | nindent 10 }} - {{- end }} - {{- if include "gremlinGpuVolumeMounts" . }} - {{- include "gremlinGpuVolumeMounts" . | nindent 10 }} - {{- end }} - volumes: - - name: cgroup-root - hostPath: - path: {{ .Values.gremlin.cgroup.root }} - {{- if include "containerVolumes" . }} - {{- include "containerVolumes" . | nindent 8 }} - {{- end }} - # The Gremlin daemon communicates with Gremlin sidecars via its state directory. - - name: gremlin-state - emptyDir: - medium: Memory - - name: gremlin-executions - hostPath: - path: /var/lib/gremlin/executions - # The Gremlin daemon forwards logs from the Gremlin sidecars to the Gremlin control plane - # These logs should be shared with the host - - name: gremlin-logs - hostPath: - path: /var/log/gremlin - {{- if (eq (include "gremlin.secretType" .) "certificate") }} - - name: gremlin-cert - secret: - secretName: {{ include "gremlin.secretName" . }} - {{- end }} - {{- if and .Values.gremlin.podSecurity.seccomp.enabled (eq "localhost/gremlin" .Values.gremlin.podSecurity.seccomp.profile) }} - - name: seccomp-root - hostPath: - path: {{ .Values.gremlin.podSecurity.seccomp.root }} - - name: seccomp-profile - configMap: - name: {{ template "gremlin.fullname" . }}-seccomp - {{- end }} - {{- if .Values.ssl.certFile }} - - name: ssl-cert-file - secret: - secretName: ssl-cert-file - {{- end }} - {{- if include "gremlinTlsIdentityVolumes" . }} - {{- include "gremlinTlsIdentityVolumes" . | nindent 8 }} - {{- end }} - {{- if include "gremlinGpuVolumes" . }} - {{- include "gremlinGpuVolumes" . | nindent 8 }} - {{- end }} -{{- if .Values.gremlin.priorityClassName }} - priorityClassName: {{ .Values.gremlin.priorityClassName }} + - GPU disabled -> a single DaemonSet. + - GPU enabled, nodes visible -> one DaemonSet per detected GPU vendor, pinned to that vendor's + nodes, plus one DaemonSet for every remaining non-GPU node. See + gremlinGpuNodesVisible / gremlinGpuVendorsInUse for how the + cluster is inspected. + - GPU enabled, otherwise -> a single GPU DaemonSet using gremlin.gpu.vendor. This is the + fallback when the cluster's nodes are not visible. +*/ -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- if include "gremlinGpuNodesVisible" . -}} +{{- $active := compact (splitList "," (include "gremlinGpuVendorsInUse" .)) -}} +{{- range $i, $vendor := $active }} +{{- if $i }} +--- +{{ end }} +{{- include "gremlin.daemonset" (dict + "root" $ + "name" (printf "%s-gpu-%s" (include "gremlin.fullname" $) $vendor) + "selectorLabels" (dict "gremlin.com/gpu" $vendor) + "gpu" (default (dict) (index $.Values.gremlin.gpu $vendor)) + "affinity" (include "gremlinGpuNodeAffinity" (dict "root" $ "vendors" (list $vendor) "operator" "In"))) }} {{- end }} +{{- if $active }} +--- +{{ end }} +{{- include "gremlin.daemonset" (dict + "root" $ + "name" (include "gremlin.fullname" $) + "selectorLabels" (dict "gremlin.com/gpu" "none") + "affinity" (include "gremlinGpuNodeAffinity" (dict "root" $ "vendors" $active "operator" "NotIn"))) }} +{{- else -}} +{{- include "gremlin.daemonset" (dict + "root" $ + "name" (include "gremlin.fullname" $) + "gpu" (default (dict) (index .Values.gremlin.gpu .Values.gremlin.gpu.vendor))) }} +{{- end -}} +{{- else -}} +{{- include "gremlin.daemonset" (dict "root" $ "name" (include "gremlin.fullname" $)) }} +{{- end -}} diff --git a/gremlin/templates/opencl-icd-configmap.yaml b/gremlin/templates/opencl-icd-configmap.yaml index 13651f1..615f25b 100644 --- a/gremlin/templates/opencl-icd-configmap.yaml +++ b/gremlin/templates/opencl-icd-configmap.yaml @@ -1,15 +1,24 @@ {{- /* -Projects an OpenCL ICD registry file into the Gremlin container. +Projects an OpenCL ICD registry file into the Gremlin container(s). Some NVIDIA container runtimes inject the OpenCL driver library (libnvidia-opencl.so.1) but do NOT create the /etc/OpenCL/vendors/.icd registry file that the ICD loader reads to discover the driver. Without it, the OpenCL platforms cannot be listed. -The ICD filename and library come from the selected vendor preset (see gremlinGpuVendorPreset). -See gremlinGpuOpenclIcdActive for when this is rendered. +One ConfigMap holds an entry per GPU vendor in use that needs an ICD file (see gremlinGpuVendorsInUse +and gremlinGpuOpenclIcdActiveFromSpec). On a hybrid cluster each vendor DaemonSet subPath-mounts its +own entry from this shared ConfigMap. The ICD filename and library come from the vendor block's +`openclIcd` field in values.yaml (gremlin.gpu..openclIcd). */ -}} -{{- if include "gremlinGpuOpenclIcdActive" . }} -{{- $eff := fromYaml (include "gremlinGpuEffective" .) }} +{{- $root := . -}} +{{- $entries := dict -}} +{{- range $vendor := (compact (splitList "," (include "gremlinGpuVendorsInUse" .))) -}} +{{- $spec := default (dict) (index $root.Values.gremlin.gpu $vendor) -}} +{{- if include "gremlinGpuOpenclIcdActiveFromSpec" (dict "root" $root "gpu" $spec) -}} +{{- $_ := set $entries $spec.openclIcd.filename $spec.openclIcd.library -}} +{{- end -}} +{{- end -}} +{{- if $entries }} apiVersion: v1 kind: ConfigMap metadata: @@ -21,6 +30,8 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: - {{ $eff.openclIcd.filename }}: | - {{ $eff.openclIcd.library }} +{{- range $filename, $library := $entries }} + {{ $filename }}: | + {{ $library }} +{{- end }} {{- end }} diff --git a/gremlin/templates/runtimeclass.yaml b/gremlin/templates/runtimeclass.yaml deleted file mode 100644 index bc88697..0000000 --- a/gremlin/templates/runtimeclass.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{- /* -Creates the RuntimeClass referenced by the GPU configuration when it does not already exist. - -NOTE: `lookup` only sees the cluster during `helm install`/`upgrade`. Under `helm template` or a -GitOps renderer with no cluster access, `lookup` returns empty and the RuntimeClass is rendered. -Set gremlin.gpu.createRuntimeClass to false if you manage the RuntimeClass yourself. -*/ -}} -{{- if .Values.gremlin.gpu.enabled -}} -{{- $runtimeClassName := include "gremlinGpuRuntimeClassName" . -}} -{{- if and $runtimeClassName .Values.gremlin.gpu.createRuntimeClass -}} -{{- if not (lookup "node.k8s.io/v1" "RuntimeClass" "" $runtimeClassName) }} -apiVersion: node.k8s.io/v1 -kind: RuntimeClass -metadata: - name: {{ $runtimeClassName }} - labels: - app.kubernetes.io/name: {{ include "gremlin.name" . }} - helm.sh/chart: {{ include "gremlin.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -handler: {{ $runtimeClassName | quote }} -{{- end }} -{{- end }} -{{- end }} diff --git a/gremlin/tests/daemonset_gpu_test.yaml b/gremlin/tests/daemonset_gpu_test.yaml index d95d784..9e949f5 100644 --- a/gremlin/tests/daemonset_gpu_test.yaml +++ b/gremlin/tests/daemonset_gpu_test.yaml @@ -32,8 +32,8 @@ tests: name: NVIDIA_VISIBLE_DEVICES value: "all" - # --- Easy default: nvidia preset --------------------------------------------------------- - - it: should apply the nvidia preset when enabled with the default vendor + # --- Easy default: nvidia block ---------------------------------------------------------- + - it: should apply the nvidia block when enabled with the default vendor set: gremlin.gpu.enabled: true asserts: @@ -50,7 +50,7 @@ tests: content: name: NVIDIA_DRIVER_CAPABILITIES value: "all" - # nvidia relies on the container toolkit, so no GPU hostPath mounts are added + # nvidia relies on the container toolkit, so no GPU hostPath volumes are added - notContains: path: spec.template.spec.volumes content: @@ -73,7 +73,7 @@ tests: configMap: name: my-release-gremlin-opencl-icd - - it: should not project the OpenCL ICD when no vendor preset defines one + - it: should not project the OpenCL ICD when the vendor block defines none set: gremlin.gpu.enabled: true gremlin.gpu.vendor: "" @@ -86,8 +86,8 @@ tests: subPath: nvidia.icd readOnly: true - # --- Easy default: amd preset ------------------------------------------------------------ - - it: should apply the amd preset when vendor is amd + # --- Easy default: amd block ------------------------------------------------------------- + - it: should apply the amd block when vendor is amd set: gremlin.gpu.enabled: true gremlin.gpu.vendor: amd @@ -122,8 +122,8 @@ tests: hostPath: path: /etc/OpenCL/vendors type: DirectoryOrCreate - # amd already mounts /etc/OpenCL/vendors from the host, so the chart defers and does not - # project its own ICD file there + # amd already mounts /etc/OpenCL/vendors from the host and defines no openclIcd, so the + # chart does not project its own ICD file there - notContains: path: spec.template.spec.volumes content: @@ -131,7 +131,7 @@ tests: configMap: name: my-release-gremlin-opencl-icd - - it: should apply no preset when vendor is blank + - it: should apply no GPU config when vendor is blank set: gremlin.gpu.enabled: true gremlin.gpu.vendor: "" @@ -146,22 +146,22 @@ tests: path: /dev/kfd type: CharDevice - # --- Advanced overrides ------------------------------------------------------------------ - - it: should let runtimeClassName override the vendor preset + # --- Editing / overriding a vendor block ------------------------------------------------- + - it: should honor a runtimeClassName set on the vendor block set: gremlin.gpu.enabled: true gremlin.gpu.vendor: nvidia - gremlin.gpu.runtimeClassName: nvidia-custom + gremlin.gpu.nvidia.runtimeClassName: nvidia-custom asserts: - equal: path: spec.template.spec.runtimeClassName value: nvidia-custom - - it: should let env override the vendor preset + - it: should honor env set on the vendor block set: gremlin.gpu.enabled: true gremlin.gpu.vendor: nvidia - gremlin.gpu.env: + gremlin.gpu.nvidia.env: - name: NVIDIA_VISIBLE_DEVICES value: "0,1" asserts: @@ -170,37 +170,27 @@ tests: content: name: NVIDIA_VISIBLE_DEVICES value: "0,1" - # the preset's NVIDIA_DRIVER_CAPABILITIES is replaced, not merged + # the block's env is the whole list, so the default NVIDIA_DRIVER_CAPABILITIES is gone - notContains: path: spec.template.spec.containers[0].env content: name: NVIDIA_DRIVER_CAPABILITIES value: "all" - - it: should let hostMounts override the vendor preset + - it: should render native volumes and volumeMounts from a custom vendor block set: gremlin.gpu.enabled: true - gremlin.gpu.vendor: amd - gremlin.gpu.hostMounts: - - name: opencl-vendors - hostPath: /etc/OpenCL/vendors - mountPath: /etc/OpenCL/vendors - type: DirectoryOrCreate - readOnly: true + gremlin.gpu.vendor: custom + gremlin.gpu.custom.volumes: + - name: nvidia0 + hostPath: + path: /dev/nvidia0 + type: CharDevice + gremlin.gpu.custom.volumeMounts: - name: nvidia0 - hostPath: /dev/nvidia0 - type: CharDevice + mountPath: /dev/nvidia0 readOnly: false asserts: - # amd preset device nodes are replaced by the custom mounts - - notContains: - path: spec.template.spec.volumes - content: - name: kfd - hostPath: - path: /dev/kfd - type: CharDevice - # mountPath defaults to hostPath and readOnly is respected - contains: path: spec.template.spec.containers[0].volumeMounts content: @@ -215,27 +205,6 @@ tests: path: /dev/nvidia0 type: CharDevice - - it: should default readOnly to true and mountPath to hostPath when omitted - set: - gremlin.gpu.enabled: true - gremlin.gpu.vendor: "" - gremlin.gpu.hostMounts: - - name: opencl-vendors - hostPath: /etc/OpenCL/vendors - asserts: - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: opencl-vendors - mountPath: /etc/OpenCL/vendors - readOnly: true - - contains: - path: spec.template.spec.volumes - content: - name: opencl-vendors - hostPath: - path: /etc/OpenCL/vendors - # --- CDI (Container Device Interface) ---------------------------------------------------- - it: should not add a CDI annotation by default set: @@ -253,6 +222,16 @@ tests: path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] value: nvidia.com/gpu=all + - it: should let a vendor block override the CDI device + set: + gremlin.gpu.enabled: true + gremlin.gpu.cdiDevice: nvidia.com/gpu=all + gremlin.gpu.nvidia.cdiDevice: nvidia.com/gpu=0 + asserts: + - equal: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + value: nvidia.com/gpu=0 + - it: should not add a CDI annotation when gpu is disabled set: gremlin.gpu.enabled: false @@ -280,3 +259,11 @@ tests: name: gremlin-opencl-icd configMap: name: my-release-gremlin-opencl-icd + + # --- Single DaemonSet under helm template (lookup finds no nodes) ------------------------ + - it: should render exactly one DaemonSet when GPU is enabled (falls back without a cluster) + set: + gremlin.gpu.enabled: true + asserts: + - hasDocuments: + count: 1 diff --git a/gremlin/tests/notes_test.yaml b/gremlin/tests/notes_test.yaml index 248e778..35b5f34 100644 --- a/gremlin/tests/notes_test.yaml +++ b/gremlin/tests/notes_test.yaml @@ -11,3 +11,17 @@ tests: asserts: - failedTemplate: errorPattern: "When using a managed certificate, both the certificate and key must be provided." + + # Under helm unittest the Node `lookup` returns nothing, so GPU always triggers the console + # warning that per-node GPU DaemonSets could not be created. + - it: should warn when GPU is enabled but node lookup finds nothing + set: + gremlin.gpu.enabled: true + asserts: + - matchRegexRaw: + pattern: "DaemonSets could NOT be created" + + - it: should not warn when GPU is disabled + asserts: + - notMatchRegexRaw: + pattern: "DaemonSets could NOT be created" diff --git a/gremlin/values.yaml b/gremlin/values.yaml index 0c09835..24827b5 100644 --- a/gremlin/values.yaml +++ b/gremlin/values.yaml @@ -247,106 +247,132 @@ gremlin: # Exposes the host's GPU / OpenCL drivers to the Gremlin agent so that the GPU attack can # enumerate and target GPUs from inside the container. # - # The easy path is to just set `enabled: true` and pick a `vendor` (e.g. "nvidia" or "amd"). - # Each vendor is a preset that fills in all the typical options for that hardware, - # so most users never need to touch the advanced options. - # - # The advanced options (runtimeClassName, env, hostMounts) remain available for custom setups. - # Any advanced option you set explicitly OVERRIDES the corresponding value from the vendor - # preset, so you can start from a preset and tweak a single field, or set vendor to "" and - # configure everything by hand. + # The easy path is to set `enabled: true` and pick a `vendor` (e.g. "nvidia" or "amd"). Each + # vendor's full configuration lives in a same-named block below. + # To customize, edit that vendor's block, or copy it to `custom` and set vendor: custom. gpu: # gremlin.gpu.enabled - - # When true, applies the GPU configuration (vendor preset plus any advanced overrides below) - # to the Gremlin Daemonset. When false, none of the GPU settings have any effect. + # When true, applies the selected vendor's GPU configuration to the Gremlin DaemonSet(s). When + # false, none of the GPU settings have any effect. enabled: false # gremlin.gpu.vendor - - # The easy default. Selects a preset that configures all the normal GPU options for you: - # nvidia - run under the "nvidia" RuntimeClass and set NVIDIA_VISIBLE_DEVICES / - # NVIDIA_DRIVER_CAPABILITIES=all so the NVIDIA container toolkit injects the driver - # libraries and device nodes. Also projects the nvidia OpenCL ICD registry file so - # the injected driver is discoverable. No hostMounts needed. - # amd - mount the /dev/kfd and /dev/dri device nodes and the OpenCL ICD registry - # (/etc/OpenCL/vendors) so the ROCm/amdgpu driver on the host is reachable. - # "" - no preset; configure everything via the advanced options below. - vendor: nvidia - - # gremlin.gpu.createRuntimeClass - - # When true, and the effective runtimeClassName is set, the chart creates that RuntimeClass - # object IF it does not already exist in the cluster. A non-existent RuntimeClass will cause - # the gremlin pod to fail to start. - # - # An existing RuntimeClass of the same name is detected and left untouched, so this will not - # interfere with a RuntimeClass already managed by the GPU Operator or your platform. - createRuntimeClass: false + # Which vendor block below to use when the chart cannot inspect the cluster's Nodes (see + # gremlin.gpu.vendors): "nvidia", "amd", "custom", or any other block you define. Whenever the + # Nodes are visible this is unused, the per-node vendor is detected automatically. + vendor: nvidia # nvidia | amd | custom + + # NOTE: when a selected vendor sets runtimeClassName, that RuntimeClass must already exist on the + # cluster (typically created by the GPU Operator or your platform). This chart does NOT create + # RuntimeClass objects; a missing RuntimeClass will cause the Gremlin pod to fail to start. # gremlin.gpu.cdiDevice - - # For nodes using the Container Device Interface (CDI) rather than the container-toolkit + # For nodes using the Nvidia Container Device Interface (CDI) rather than the Nvidia container-toolkit # runtime hook. When set, the chart adds a `cdi.k8s.io/gremlin-gpu: ` pod annotation # so the CRI injects the named CDI device(s). Use this when NVIDIA_VISIBLE_DEVICES is NOT - # honored by your runtime (pure Kubernetes-native CDI). + # honored by your runtime (pure Kubernetes-native CDI). A vendor block may override this with + # its own `cdiDevice`. # # Requires containerd >= 1.7 or CRI-O with CDI enabled. - # - # NOTE: in pure CDI mode NVIDIA_DRIVER_CAPABILITIES is ignored (capabilities come from the CDI - # device), and you likely do not need a RuntimeClass — set runtimeClassName: "" below if the - # "nvidia" RuntimeClass does not exist on your cluster. cdiDevice: "" # gremlin.gpu.projectOpenclIcd - - # Whether to project the vendor preset's OpenCL ICD registry file (e.g. /etc/OpenCL/vendors/ - # nvidia.icd) into the container. Defaults to true, which fixes runtimes that inject the - # OpenCL driver library but not the ICD file. Set to false when your CDI spec (or another - # mechanism) already creates the ICD file, to avoid a conflicting mount at the same path. + # Whether to project the selected vendor's `openclIcd` file (e.g. /etc/OpenCL/vendors/nvidia.icd) + # into the container, when that vendor block defines one. Defaults to true, which fixes runtimes + # that inject the OpenCL driver library but not the ICD file. Set to false when your CDI spec + # (or another mechanism) already creates the ICD file, to avoid a conflicting mount. projectOpenclIcd: true - # --- Advanced options ------------------------------------------------------------------- - # Everything below overrides the vendor preset. Leave a field at its empty default to keep - # the preset's value for that field. - - # gremlin.gpu.runtimeClassName - - # The Kubernetes RuntimeClass to run the Gremlin Daemonset pods under. - runtimeClassName: "" - - # gremlin.gpu.env - - # Extra environment variables applied to the Gremlin Daemonset container when GPU access is - # enabled. For the NVIDIA container toolkit the capabilities value MUST include `compute` - # (or `all`) for OpenCL. - env: [] - # - name: NVIDIA_VISIBLE_DEVICES - # value: "all" - # - name: NVIDIA_DRIVER_CAPABILITIES - # value: "all" - - # gremlin.gpu.hostMounts - - # A list of hostPath volumes mounted into the Gremlin container so the OpenCL ICD loader can - # find vendor drivers and access GPU device nodes. - # Each entry accepts: - # - name: a unique volume name (required) - # - hostPath: the path on the Kubernetes node (required) - # - mountPath: the path inside the container (defaults to hostPath when omitted) - # - type: an optional hostPath type, e.g. Directory, DirectoryOrCreate, CharDevice - # - readOnly: whether the mount is read-only (defaults to true; set false for device nodes) + # gremlin.gpu.vendors - + # The vendor blocks to detect and target on a hybrid / mixed cluster, where only some nodes have + # GPUs and different nodes may have different vendors. Whenever GPU access is enabled the chart + # inspects the cluster's Nodes and renders a separate DaemonSet per detected vendor; scheduled + # only onto that vendor's nodes, carrying that vendor's GPU config, plus one plain DaemonSet + # for every remaining node, so each node runs exactly one agent matching its capabilities. + # + # Each name must reference a vendor block below defining a `nodeSelector`, which both detects and + # schedules, so GPU nodes must carry those labels (the NVIDIA GPU Operator / Node Feature + # Discovery and the AMD GPU labeller set them). A block's optional `resources` also detects. + vendors: + - nvidia + - amd + + # --- Vendor configuration blocks -------------------------------------------------------- + # Each block is the full, native-shaped GPU configuration for one vendor. Fields: + # nodeSelector - node label(s) that identify AND schedule this vendor's nodes + # resources - (optional) node capacity resource names that also mark the vendor + # runtimeClassName - the RuntimeClass the Gremlin pods run under ("" for none) + # env - environment variables added to the Gremlin container (native env entries). + # For the NVIDIA container toolkit the capabilities value MUST include + # `compute` (or `all`) for OpenCL. + # volumes - pod volumes added to the Gremlin DaemonSet (native volume entries) + # volumeMounts - container volumeMounts added to the Gremlin container (native mount entries) + # openclIcd - optional {filename, library}. When set (and projectOpenclIcd is true) the + # chart projects an /etc/OpenCL/vendors/ file containing , + # for runtimes that inject the driver library but not its ICD registry file. + # cdiDevice - optional per-vendor override of gremlin.gpu.cdiDevice # - # NOTE: GPU device nodes (/dev/nvidia*, /dev/kfd, /dev/dri) require the container to have - # device access. This generally means running privileged (gremlin.podSecurity.privileged: - # true) or otherwise granting the pod cgroup device permissions. Device access is not - # required when the NVIDIA container toolkit injects the devices for you. - hostMounts: [] - # - name: opencl-vendors - # hostPath: /etc/OpenCL/vendors - # mountPath: /etc/OpenCL/vendors - # type: DirectoryOrCreate - # readOnly: true - # - name: nvidia0 - # hostPath: /dev/nvidia0 - # type: CharDevice - # readOnly: false - # # NVIDIA OpenCL driver library (path/version varies by distro and driver release) - # - name: nvidia-opencl-lib - # hostPath: /usr/lib/x86_64-linux-gnu/libnvidia-opencl.so.1 - # readOnly: true + # NOTE: GPU device nodes (/dev/nvidia*, /dev/kfd, /dev/dri) require the container to have device + # access. This generally means running privileged (gremlin.podSecurity.privileged: true) or + # otherwise granting the pod cgroup device permissions. Device access is not required when the + # NVIDIA container toolkit injects the devices for you. + + nvidia: + # Relies on the NVIDIA container toolkit: run under the "nvidia" RuntimeClass and set + # NVIDIA_VISIBLE_DEVICES / NVIDIA_DRIVER_CAPABILITIES so the runtime injects the driver + # libraries and device nodes (no host volumes required). Projects the nvidia OpenCL ICD + # registry file so the injected driver is discoverable. + nodeSelector: + nvidia.com/gpu.present: "true" + resources: + - nvidia.com/gpu + runtimeClassName: nvidia + env: + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_DRIVER_CAPABILITIES + value: "all" + volumes: [] + volumeMounts: [] + openclIcd: + filename: nvidia.icd + library: libnvidia-opencl.so.1 + + amd: + # Relies on the ROCm/amdgpu driver on the host: mount the /dev/kfd and /dev/dri device nodes + # plus the host's OpenCL ICD registry (/etc/OpenCL/vendors) so the container reaches the GPUs + # directly. + nodeSelector: + amd.com/gpu.present: "true" + resources: + - amd.com/gpu + runtimeClassName: "" + env: [] + volumes: + - name: kfd + hostPath: { path: /dev/kfd, type: CharDevice } + - name: dri + hostPath: { path: /dev/dri, type: Directory } + - name: opencl-vendors + hostPath: { path: /etc/OpenCL/vendors, type: DirectoryOrCreate } + volumeMounts: + - name: kfd + mountPath: /dev/kfd + readOnly: false + - name: dri + mountPath: /dev/dri + readOnly: false + - name: opencl-vendors + mountPath: /etc/OpenCL/vendors + readOnly: true + + custom: + # A blank block to configure a GPU vendor by hand. Add "custom" to gremlin.gpu.vendors along + # with a nodeSelector (and/or set gremlin.gpu.vendor: custom) and fill in the fields. + runtimeClassName: "" + env: [] + volumes: [] + volumeMounts: [] secret: # Gremlin supports both `certificate` and `secret` types