Add simple options for enabling Gremlin GPU access

gremlin/README.md

@@ -74,6 +74,14 @@ their default values. See values.yaml for all available options.
 | `gremlin.proxy.url`                    | Specifies the http proxy the agent should use to communicate with api.gremlin.com. | `""` (ignored)                                                                              |                                       |
 | `gremlin.extraEnv`                     | Specify any arbitrary environment variables to pass to the Gremlin Agent daemonset. | `[]`                                                                                        |
 | `gremlin.features.discoverDestinationService.enabled` | Enable discovery of a destination service in a service mesh to resolve hostnames | `false`                                                                |
+| `gremlin.gpu.enabled`                  | Expose host GPU/OpenCL drivers to the agent for the GPU attack | `false`                                                                                     |
+| `gremlin.gpu.vendor`                   | GPU vendor preset (`nvidia`, `amd`, or `""` for manual config) | `nvidia`                                                                                    |
+| `gremlin.gpu.createRuntimeClass`       | Create the effective RuntimeClass if it doesn't already exist  | `false`                                                                                     |
+| `gremlin.gpu.cdiDevice`                | CDI device to inject via pod annotation (for CDI-based runtimes) | `""`                                                                                        |
+| `gremlin.gpu.projectOpenclIcd`         | Project the vendor's OpenCL ICD registry file into the container | `true`                                                                                      |
+| `gremlin.gpu.runtimeClassName`         | Override the RuntimeClass to run the agent under               | `""` (uses vendor preset)                                                                   |
+| `gremlin.gpu.env`                      | Override GPU environment variables on the agent container      | `[]` (uses vendor preset)                                                                   |
+| `gremlin.gpu.hostMounts`               | Override hostPath mounts for GPU drivers and device nodes      | `[]` (uses vendor preset)                                                                   |
 | `ssl.certFile`                         | Add a certificate file to Gremlin's set of certificate authorities. This argument expects a file containing the certificate(s) you wish to add. When set, this chart creates secret (`ssl-cert-file`) with the contents and passes it to both agents. This value is ignored when blank or absent. | `""` (ignored)                                                                              |
 | `ssl.certDir`                          | sets the SSL_CERT_DIR environment variable on the both agents. Unlike ssl.certFile, this value accepts only a path to an existing directory on the Kubernetes nodes. This value is ignored when blank or absent. | `""` (ignored)                                                                              |
 
@@ -213,6 +221,24 @@ helm install gremlin gremlin/gremlin \
     --set-file ssl.certFile=$HOME/Workspace/proxy/ca.pem
 ```
 
+### With GPU Support
+
+To let the GPU attack enumerate and target GPUs, enable `gremlin.gpu` and pick a vendor preset.
+
+```shell
+helm install gremlin gremlin/gremlin \
+    --namespace gremlin \
+    --set      gremlin.secret.managed=true \
+    --set      gremlin.secret.teamID=$GREMLIN_TEAM_ID \
+    --set      gremlin.secret.clusterID=$GREMLIN_CLUSTER_ID \
+    --set-file gremlin.secret.certificate=/path/to/gremlin.cert \
+    --set-file gremlin.secret.key=/path/to/gremlin.key \
+    --set      gremlin.gpu.enabled=true \
+    --set      gremlin.gpu.vendor=nvidia
+```
+
+_note_: The `nvidia` preset runs the agent under the `nvidia` RuntimeClass. If the Gremlin pod fails to start (for example, a `RuntimeClass not found` error), the RuntimeClass likely doesn't exist on your cluster. Have the chart create it by also setting `--set gremlin.gpu.createRuntimeClass=true`.
+
 ## Uninstallation
 
 ```shell

gremlin/templates/_helpers.tpl

@@ -244,6 +244,183 @@ When createSecret or existingSecret are configured
 {{- end -}}
 {{- end -}}
 
+{{/*
+gremlinGpuVendorPreset returns the GPU configuration for a given vendor. The context passed in
+is the vendor string (gremlin.gpu.vendor). Supported values are "nvidia" and "amd"; any other value
+(including "") yields an empty preset so the chart falls back to whatever the advanced options specify.
+
+  nvidia - relies on the NVIDIA container toolkit: run under the "nvidia" RuntimeClass and set
+           NVIDIA_VISIBLE_DEVICES / NVIDIA_DRIVER_CAPABILITIES so the runtime injects the driver
+           libraries and device nodes (no hostMounts required). Also projects the nvidia OpenCL
+           ICD registry file, since some runtimes inject libnvidia-opencl.so.1 but not the
+           /etc/OpenCL/vendors/nvidia.icd file the loader needs to discover it.
+  amd    - relies on the ROCm/amdgpu driver on the host: mount the /dev/kfd and /dev/dri device
+           nodes plus the OpenCL ICD registry so the container can reach the GPUs directly.
+
+The optional `openclIcd` key (filename + library) tells the chart to project an OpenCL ICD
+registry file; vendors that already expose /etc/OpenCL/vendors (amd) omit it.
+*/}}
+{{- define "gremlinGpuVendorPreset" -}}
+{{- $vendor := . -}}
+{{- if eq $vendor "nvidia" -}}
+runtimeClassName: nvidia
+env:
+  - name: NVIDIA_VISIBLE_DEVICES
+    value: "all"
+  - name: NVIDIA_DRIVER_CAPABILITIES
+    value: "all"
+hostMounts: []
+openclIcd:
+  filename: nvidia.icd
+  library: libnvidia-opencl.so.1
+{{- else if eq $vendor "amd" -}}
+runtimeClassName: ""
+env: []
+hostMounts:
+  - name: kfd
+    hostPath: /dev/kfd
+    type: CharDevice
+    readOnly: false
+  - name: dri
+    hostPath: /dev/dri
+    type: Directory
+    readOnly: false
+  - name: opencl-vendors
+    hostPath: /etc/OpenCL/vendors
+    mountPath: /etc/OpenCL/vendors
+    type: DirectoryOrCreate
+    readOnly: true
+{{- else -}}
+runtimeClassName: ""
+env: []
+hostMounts: []
+{{- end -}}
+{{- end -}}
+
+{{/*
+gremlinGpuEffective returns the vendor preset (see gremlinGpuVendorPreset) with each advanced
+option overriding the preset when set.
+*/}}
+{{- define "gremlinGpuEffective" -}}
+{{- $gpu := .Values.gremlin.gpu -}}
+{{- $preset := fromYaml (include "gremlinGpuVendorPreset" (default "" $gpu.vendor)) -}}
+{{- $runtimeClassName := $preset.runtimeClassName -}}
+{{- if $gpu.runtimeClassName -}}{{- $runtimeClassName = $gpu.runtimeClassName -}}{{- end -}}
+{{- $env := $preset.env -}}
+{{- if $gpu.env -}}{{- $env = $gpu.env -}}{{- end -}}
+{{- $hostMounts := $preset.hostMounts -}}
+{{- if $gpu.hostMounts -}}{{- $hostMounts = $gpu.hostMounts -}}{{- end -}}
+{{- $effective := dict "runtimeClassName" $runtimeClassName "env" $env "hostMounts" $hostMounts -}}
+{{- if $preset.openclIcd -}}{{- $_ := set $effective "openclIcd" $preset.openclIcd -}}{{- end -}}
+{{- $effective | toYaml -}}
+{{- end -}}
+
+{{/*
+gremlinGpuRuntimeClassName returns the effective RuntimeClass name for the Gremlin pods, or
+nothing when GPU access is disabled or no RuntimeClass applies.
+*/}}
+{{- define "gremlinGpuRuntimeClassName" -}}
+{{- if .Values.gremlin.gpu.enabled -}}
+{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}}
+{{- $eff.runtimeClassName -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+gremlinGpuEnv returns the environment variables that enable GPU/OpenCL access.
+When gremlin.gpu.enabled is true, the effective env entries (vendor preset or the gremlin.gpu.env
+override) are emitted. For the NVIDIA container toolkit these include NVIDIA_VISIBLE_DEVICES and
+NVIDIA_DRIVER_CAPABILITIES (which must include `compute` or `all` for OpenCL).
+*/}}
+{{- define "gremlinGpuEnv" -}}
+{{- if .Values.gremlin.gpu.enabled -}}
+{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}}
+{{- with $eff.env -}}
+{{- toYaml . -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+gremlinGpuCdiAnnotation returns the CDI device pod annotation when gremlin.gpu.cdiDevice is set,
+and nothing otherwise. This targets the annotation-based CDI injection path supported by
+containerd >= 1.7 / CRI-O, which does not consume a schedulable GPU resource.
+*/}}
+{{- define "gremlinGpuCdiAnnotation" -}}
+{{- if and .Values.gremlin.gpu.enabled .Values.gremlin.gpu.cdiDevice -}}
+cdi.k8s.io/gremlin-gpu: {{ .Values.gremlin.gpu.cdiDevice | quote }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+gremlinGpuOpenclIcdActive returns "true" when the chart should project an OpenCL ICD registry
+file into the container, and nothing otherwise.
+
+The ICD is projected automatically whenever gremlin.gpu.projectOpenclIcd is true and the selected
+vendor preset defines one (nvidia does), UNLESS an effective hostMount already provides
+/etc/OpenCL/vendors (e.g. the amd preset or a custom mount), in which case we defer to that mount
+to avoid overlapping mount paths.
+*/}}
+{{- define "gremlinGpuOpenclIcdActive" -}}
+{{- if and .Values.gremlin.gpu.enabled .Values.gremlin.gpu.projectOpenclIcd -}}
+{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}}
+{{- if and $eff.openclIcd $eff.openclIcd.library -}}
+{{- $dirMounted := false -}}
+{{- range $eff.hostMounts }}
+{{- if eq (default .hostPath .mountPath) "/etc/OpenCL/vendors" }}{{- $dirMounted = true -}}{{- end }}
+{{- end -}}
+{{- if not $dirMounted -}}true{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+gremlinGpuVolumeMounts returns the volumeMounts that expose host GPU/OpenCL drivers.
+Each effective hostMounts entry (vendor preset or the gremlin.gpu.hostMounts override) becomes a
+volumeMount. mountPath defaults to hostPath and readOnly defaults to true when not specified.
+When gremlinGpuOpenclIcdActive, the projected OpenCL ICD file is mounted as well.
+*/}}
+{{- define "gremlinGpuVolumeMounts" -}}
+{{- if .Values.gremlin.gpu.enabled -}}
+{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}}
+{{- range $eff.hostMounts }}
+- name: {{ .name }}
+  mountPath: {{ default .hostPath .mountPath }}
+  readOnly: {{ if hasKey . "readOnly" }}{{ .readOnly }}{{ else }}true{{ end }}
+{{- end -}}
+{{- if include "gremlinGpuOpenclIcdActive" . }}
+- name: gremlin-opencl-icd
+  mountPath: /etc/OpenCL/vendors/{{ $eff.openclIcd.filename }}
+  subPath: {{ $eff.openclIcd.filename }}
+  readOnly: true
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+gremlinGpuVolumes returns the hostPath volumes that expose host GPU/OpenCL drivers.
+Each effective hostMounts entry (vendor preset or the gremlin.gpu.hostMounts override) becomes a
+hostPath volume. An optional `type` (e.g. DirectoryOrCreate, CharDevice) is passed through when present.
+*/}}
+{{- define "gremlinGpuVolumes" -}}
+{{- if .Values.gremlin.gpu.enabled -}}
+{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}}
+{{- range $eff.hostMounts }}
+- name: {{ .name }}
+  hostPath:
+    path: {{ .hostPath }}
+    {{- if .type }}
+    type: {{ .type }}
+    {{- end }}
+{{- end -}}
+{{- if include "gremlinGpuOpenclIcdActive" . }}
+- name: gremlin-opencl-icd
+  configMap:
+    name: {{ include "gremlin.fullname" . }}-opencl-icd
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 chaoTlsIdentityArgs returns the chao cli arguments needed to configure TLS client identity
 When remoteSecret is configured

gremlin/templates/daemonset.yaml

@@ -31,7 +31,7 @@ spec:
         {{- if .Values.gremlin.podLabels }}
         {{- toYaml .Values.gremlin.podLabels | nindent 8 }}
         {{- end }}
-      {{- if or .Values.gremlin.apparmor .Values.gremlin.installApparmorProfile .Values.gremlin.podSecurity.seccomp.enabled .Values.gremlin.podSecurity.securityContextConstraints.create .Values.gremlin.podAnnotations }}
+      {{- if or .Values.gremlin.apparmor .Values.gremlin.installApparmorProfile .Values.gremlin.podSecurity.seccomp.enabled .Values.gremlin.podSecurity.securityContextConstraints.create .Values.gremlin.podAnnotations (include "gremlinGpuCdiAnnotation" .) }}
       annotations:
         {{- if .Values.gremlin.apparmor }}
         container.apparmor.security.beta.kubernetes.io/{{ .Chart.Name }}: {{ .Values.gremlin.apparmor }}
@@ -44,12 +44,18 @@ spec:
         {{- if .Values.gremlin.podSecurity.securityContextConstraints.create }}
         openshift.io/required-scc: "gremlin"
         {{- end }}
+        {{- if include "gremlinGpuCdiAnnotation" . }}
+        {{- include "gremlinGpuCdiAnnotation" . | nindent 8 }}
+        {{- end }}
         {{- if .Values.gremlin.podAnnotations }}
         {{- toYaml .Values.gremlin.podAnnotations | nindent 8 }}
         {{- end }}
       {{- end }}
     spec:
       serviceAccountName: gremlin
+      {{- if include "gremlinGpuRuntimeClassName" . }}
+      runtimeClassName: {{ include "gremlinGpuRuntimeClassName" . }}
+      {{- end }}
       {{- if .Values.affinity }}
       affinity: {{ toYaml .Values.affinity | trimSuffix "\n" | nindent 8 }}
       {{- end }}
@@ -163,6 +169,9 @@ spec:
           {{- if include "gremlinTlsIdentityEnv" . }}
           {{- include "gremlinTlsIdentityEnv" . | nindent 10 }}
           {{- end }}
+          {{- if include "gremlinGpuEnv" . }}
+          {{- include "gremlinGpuEnv" . | nindent 10 }}
+          {{- end }}
           {{- with .Values.gremlin.extraEnv }}
             {{- toYaml . | nindent 10 }}
           {{- end }}
@@ -195,6 +204,9 @@ spec:
           {{- if include "gremlinTlsIdentityVolumeMounts" . }}
           {{- include "gremlinTlsIdentityVolumeMounts" . | nindent 10 }}
           {{- end }}
+          {{- if include "gremlinGpuVolumeMounts" . }}
+          {{- include "gremlinGpuVolumeMounts" . | nindent 10 }}
+          {{- end }}
       volumes:
         - name: cgroup-root
           hostPath:
@@ -235,6 +247,9 @@ spec:
         {{- if include "gremlinTlsIdentityVolumes" . }}
         {{- include "gremlinTlsIdentityVolumes" . | nindent 8 }}
         {{- end }}
+        {{- if include "gremlinGpuVolumes" . }}
+        {{- include "gremlinGpuVolumes" . | nindent 8 }}
+        {{- end }}
 {{- if .Values.gremlin.priorityClassName }}
       priorityClassName: {{ .Values.gremlin.priorityClassName }}
 {{- end }}

gremlin/templates/opencl-icd-configmap.yaml

@@ -0,0 +1,26 @@
+{{- /*
+Projects an OpenCL ICD registry file into the Gremlin container.
+
+Some NVIDIA container runtimes inject the OpenCL driver library (libnvidia-opencl.so.1) but do
+NOT create the /etc/OpenCL/vendors/<vendor>.icd registry file that the ICD loader reads to
+discover the driver. Without it, the OpenCL platforms cannot be listed.
+
+The ICD filename and library come from the selected vendor preset (see gremlinGpuVendorPreset).
+See gremlinGpuOpenclIcdActive for when this is rendered.
+*/ -}}
+{{- if include "gremlinGpuOpenclIcdActive" . }}
+{{- $eff := fromYaml (include "gremlinGpuEffective" .) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "gremlin.fullname" . }}-opencl-icd
+  namespace: {{ .Release.Namespace }}
+  labels:
+    helm.sh/chart: {{ include "gremlin.chart" . }}
+    app.kubernetes.io/name: {{ include "gremlin.fullname" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+  {{ $eff.openclIcd.filename }}: |
+    {{ $eff.openclIcd.library }}
+{{- end }}

gremlin/templates/runtimeclass.yaml

@@ -0,0 +1,24 @@
+{{- /*
+Creates the RuntimeClass referenced by the GPU configuration when it does not already exist.
+
+NOTE: `lookup` only sees the cluster during `helm install`/`upgrade`. Under `helm template` or a
+GitOps renderer with no cluster access, `lookup` returns empty and the RuntimeClass is rendered.
+Set gremlin.gpu.createRuntimeClass to false if you manage the RuntimeClass yourself.
+*/ -}}
+{{- if .Values.gremlin.gpu.enabled -}}
+{{- $runtimeClassName := include "gremlinGpuRuntimeClassName" . -}}
+{{- if and $runtimeClassName .Values.gremlin.gpu.createRuntimeClass -}}
+{{- if not (lookup "node.k8s.io/v1" "RuntimeClass" "" $runtimeClassName) }}
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+  name: {{ $runtimeClassName }}
+  labels:
+    app.kubernetes.io/name: {{ include "gremlin.name" . }}
+    helm.sh/chart: {{ include "gremlin.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+handler: {{ $runtimeClassName | quote }}
+{{- end }}
+{{- end }}
+{{- end }}