diff --git a/gremlin/Chart.yaml b/gremlin/Chart.yaml index 65e4e37..be142c2 100644 --- a/gremlin/Chart.yaml +++ b/gremlin/Chart.yaml @@ -1,5 +1,5 @@ name: gremlin -version: 0.26.0 +version: 0.27.0 description: The Gremlin Inc client application apiVersion: v1 home: https://www.gremlin.com diff --git a/gremlin/README.md b/gremlin/README.md index f9b786b..fb67cc9 100644 --- a/gremlin/README.md +++ b/gremlin/README.md @@ -74,6 +74,14 @@ their default values. See values.yaml for all available options. | `gremlin.proxy.url` | Specifies the http proxy the agent should use to communicate with api.gremlin.com. | `""` (ignored) | | | `gremlin.extraEnv` | Specify any arbitrary environment variables to pass to the Gremlin Agent daemonset. | `[]` | | `gremlin.features.discoverDestinationService.enabled` | Enable discovery of a destination service in a service mesh to resolve hostnames | `false` | +| `gremlin.gpu.enabled` | Expose host GPU/OpenCL drivers to the agent for the GPU attack | `false` | +| `gremlin.gpu.vendor` | GPU vendor preset (`nvidia`, `amd`, or `""` for manual config) | `nvidia` | +| `gremlin.gpu.createRuntimeClass` | Create the effective RuntimeClass if it doesn't already exist | `false` | +| `gremlin.gpu.cdiDevice` | CDI device to inject via pod annotation (for CDI-based runtimes) | `""` | +| `gremlin.gpu.projectOpenclIcd` | Project the vendor's OpenCL ICD registry file into the container | `true` | +| `gremlin.gpu.runtimeClassName` | Override the RuntimeClass to run the agent under | `""` (uses vendor preset) | +| `gremlin.gpu.env` | Override GPU environment variables on the agent container | `[]` (uses vendor preset) | +| `gremlin.gpu.hostMounts` | Override hostPath mounts for GPU drivers and device nodes | `[]` (uses vendor preset) | | `ssl.certFile` | Add a certificate file to Gremlin's set of certificate authorities. This argument expects a file containing the certificate(s) you wish to add. When set, this chart creates secret (`ssl-cert-file`) with the contents and passes it to both agents. This value is ignored when blank or absent. | `""` (ignored) | | `ssl.certDir` | sets the SSL_CERT_DIR environment variable on the both agents. Unlike ssl.certFile, this value accepts only a path to an existing directory on the Kubernetes nodes. This value is ignored when blank or absent. | `""` (ignored) | @@ -213,6 +221,24 @@ helm install gremlin gremlin/gremlin \ --set-file ssl.certFile=$HOME/Workspace/proxy/ca.pem ``` +### With GPU Support + +To let the GPU attack enumerate and target GPUs, enable `gremlin.gpu` and pick a vendor preset. + +```shell +helm install gremlin gremlin/gremlin \ + --namespace gremlin \ + --set gremlin.secret.managed=true \ + --set gremlin.secret.teamID=$GREMLIN_TEAM_ID \ + --set gremlin.secret.clusterID=$GREMLIN_CLUSTER_ID \ + --set-file gremlin.secret.certificate=/path/to/gremlin.cert \ + --set-file gremlin.secret.key=/path/to/gremlin.key \ + --set gremlin.gpu.enabled=true \ + --set gremlin.gpu.vendor=nvidia +``` + +_note_: The `nvidia` preset runs the agent under the `nvidia` RuntimeClass. If the Gremlin pod fails to start (for example, a `RuntimeClass not found` error), the RuntimeClass likely doesn't exist on your cluster. Have the chart create it by also setting `--set gremlin.gpu.createRuntimeClass=true`. + ## Uninstallation ```shell diff --git a/gremlin/templates/_helpers.tpl b/gremlin/templates/_helpers.tpl index 5aebf27..551caa0 100644 --- a/gremlin/templates/_helpers.tpl +++ b/gremlin/templates/_helpers.tpl @@ -244,6 +244,183 @@ When createSecret or existingSecret are configured {{- end -}} {{- end -}} +{{/* +gremlinGpuVendorPreset returns the GPU configuration for a given vendor. The context passed in +is the vendor string (gremlin.gpu.vendor). Supported values are "nvidia" and "amd"; any other value +(including "") yields an empty preset so the chart falls back to whatever the advanced options specify. + + nvidia - relies on the NVIDIA container toolkit: run under the "nvidia" RuntimeClass and set + NVIDIA_VISIBLE_DEVICES / NVIDIA_DRIVER_CAPABILITIES so the runtime injects the driver + libraries and device nodes (no hostMounts required). Also projects the nvidia OpenCL + ICD registry file, since some runtimes inject libnvidia-opencl.so.1 but not the + /etc/OpenCL/vendors/nvidia.icd file the loader needs to discover it. + amd - relies on the ROCm/amdgpu driver on the host: mount the /dev/kfd and /dev/dri device + nodes plus the OpenCL ICD registry so the container can reach the GPUs directly. + +The optional `openclIcd` key (filename + library) tells the chart to project an OpenCL ICD +registry file; vendors that already expose /etc/OpenCL/vendors (amd) omit it. +*/}} +{{- define "gremlinGpuVendorPreset" -}} +{{- $vendor := . -}} +{{- if eq $vendor "nvidia" -}} +runtimeClassName: nvidia +env: + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_DRIVER_CAPABILITIES + value: "all" +hostMounts: [] +openclIcd: + filename: nvidia.icd + library: libnvidia-opencl.so.1 +{{- else if eq $vendor "amd" -}} +runtimeClassName: "" +env: [] +hostMounts: + - name: kfd + hostPath: /dev/kfd + type: CharDevice + readOnly: false + - name: dri + hostPath: /dev/dri + type: Directory + readOnly: false + - name: opencl-vendors + hostPath: /etc/OpenCL/vendors + mountPath: /etc/OpenCL/vendors + type: DirectoryOrCreate + readOnly: true +{{- else -}} +runtimeClassName: "" +env: [] +hostMounts: [] +{{- end -}} +{{- end -}} + +{{/* +gremlinGpuEffective returns the vendor preset (see gremlinGpuVendorPreset) with each advanced +option overriding the preset when set. +*/}} +{{- define "gremlinGpuEffective" -}} +{{- $gpu := .Values.gremlin.gpu -}} +{{- $preset := fromYaml (include "gremlinGpuVendorPreset" (default "" $gpu.vendor)) -}} +{{- $runtimeClassName := $preset.runtimeClassName -}} +{{- if $gpu.runtimeClassName -}}{{- $runtimeClassName = $gpu.runtimeClassName -}}{{- end -}} +{{- $env := $preset.env -}} +{{- if $gpu.env -}}{{- $env = $gpu.env -}}{{- end -}} +{{- $hostMounts := $preset.hostMounts -}} +{{- if $gpu.hostMounts -}}{{- $hostMounts = $gpu.hostMounts -}}{{- end -}} +{{- $effective := dict "runtimeClassName" $runtimeClassName "env" $env "hostMounts" $hostMounts -}} +{{- if $preset.openclIcd -}}{{- $_ := set $effective "openclIcd" $preset.openclIcd -}}{{- end -}} +{{- $effective | toYaml -}} +{{- end -}} + +{{/* +gremlinGpuRuntimeClassName returns the effective RuntimeClass name for the Gremlin pods, or +nothing when GPU access is disabled or no RuntimeClass applies. +*/}} +{{- define "gremlinGpuRuntimeClassName" -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} +{{- $eff.runtimeClassName -}} +{{- end -}} +{{- end -}} + +{{/* +gremlinGpuEnv returns the environment variables that enable GPU/OpenCL access. +When gremlin.gpu.enabled is true, the effective env entries (vendor preset or the gremlin.gpu.env +override) are emitted. For the NVIDIA container toolkit these include NVIDIA_VISIBLE_DEVICES and +NVIDIA_DRIVER_CAPABILITIES (which must include `compute` or `all` for OpenCL). +*/}} +{{- define "gremlinGpuEnv" -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} +{{- with $eff.env -}} +{{- toYaml . -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +gremlinGpuCdiAnnotation returns the CDI device pod annotation when gremlin.gpu.cdiDevice is set, +and nothing otherwise. This targets the annotation-based CDI injection path supported by +containerd >= 1.7 / CRI-O, which does not consume a schedulable GPU resource. +*/}} +{{- define "gremlinGpuCdiAnnotation" -}} +{{- if and .Values.gremlin.gpu.enabled .Values.gremlin.gpu.cdiDevice -}} +cdi.k8s.io/gremlin-gpu: {{ .Values.gremlin.gpu.cdiDevice | quote }} +{{- end -}} +{{- end -}} + +{{/* +gremlinGpuOpenclIcdActive returns "true" when the chart should project an OpenCL ICD registry +file into the container, and nothing otherwise. + +The ICD is projected automatically whenever gremlin.gpu.projectOpenclIcd is true and the selected +vendor preset defines one (nvidia does), UNLESS an effective hostMount already provides +/etc/OpenCL/vendors (e.g. the amd preset or a custom mount), in which case we defer to that mount +to avoid overlapping mount paths. +*/}} +{{- define "gremlinGpuOpenclIcdActive" -}} +{{- if and .Values.gremlin.gpu.enabled .Values.gremlin.gpu.projectOpenclIcd -}} +{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} +{{- if and $eff.openclIcd $eff.openclIcd.library -}} +{{- $dirMounted := false -}} +{{- range $eff.hostMounts }} +{{- if eq (default .hostPath .mountPath) "/etc/OpenCL/vendors" }}{{- $dirMounted = true -}}{{- end }} +{{- end -}} +{{- if not $dirMounted -}}true{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +gremlinGpuVolumeMounts returns the volumeMounts that expose host GPU/OpenCL drivers. +Each effective hostMounts entry (vendor preset or the gremlin.gpu.hostMounts override) becomes a +volumeMount. mountPath defaults to hostPath and readOnly defaults to true when not specified. +When gremlinGpuOpenclIcdActive, the projected OpenCL ICD file is mounted as well. +*/}} +{{- define "gremlinGpuVolumeMounts" -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} +{{- range $eff.hostMounts }} +- name: {{ .name }} + mountPath: {{ default .hostPath .mountPath }} + readOnly: {{ if hasKey . "readOnly" }}{{ .readOnly }}{{ else }}true{{ end }} +{{- end -}} +{{- if include "gremlinGpuOpenclIcdActive" . }} +- name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/{{ $eff.openclIcd.filename }} + subPath: {{ $eff.openclIcd.filename }} + readOnly: true +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +gremlinGpuVolumes returns the hostPath volumes that expose host GPU/OpenCL drivers. +Each effective hostMounts entry (vendor preset or the gremlin.gpu.hostMounts override) becomes a +hostPath volume. An optional `type` (e.g. DirectoryOrCreate, CharDevice) is passed through when present. +*/}} +{{- define "gremlinGpuVolumes" -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- $eff := fromYaml (include "gremlinGpuEffective" .) -}} +{{- range $eff.hostMounts }} +- name: {{ .name }} + hostPath: + path: {{ .hostPath }} + {{- if .type }} + type: {{ .type }} + {{- end }} +{{- end -}} +{{- if include "gremlinGpuOpenclIcdActive" . }} +- name: gremlin-opencl-icd + configMap: + name: {{ include "gremlin.fullname" . }}-opencl-icd +{{- end -}} +{{- end -}} +{{- end -}} + {{/* chaoTlsIdentityArgs returns the chao cli arguments needed to configure TLS client identity When remoteSecret is configured diff --git a/gremlin/templates/daemonset.yaml b/gremlin/templates/daemonset.yaml index 6a3de46..7a8a6bf 100644 --- a/gremlin/templates/daemonset.yaml +++ b/gremlin/templates/daemonset.yaml @@ -31,7 +31,7 @@ spec: {{- if .Values.gremlin.podLabels }} {{- toYaml .Values.gremlin.podLabels | nindent 8 }} {{- end }} - {{- if or .Values.gremlin.apparmor .Values.gremlin.installApparmorProfile .Values.gremlin.podSecurity.seccomp.enabled .Values.gremlin.podSecurity.securityContextConstraints.create .Values.gremlin.podAnnotations }} + {{- if or .Values.gremlin.apparmor .Values.gremlin.installApparmorProfile .Values.gremlin.podSecurity.seccomp.enabled .Values.gremlin.podSecurity.securityContextConstraints.create .Values.gremlin.podAnnotations (include "gremlinGpuCdiAnnotation" .) }} annotations: {{- if .Values.gremlin.apparmor }} container.apparmor.security.beta.kubernetes.io/{{ .Chart.Name }}: {{ .Values.gremlin.apparmor }} @@ -44,12 +44,18 @@ spec: {{- if .Values.gremlin.podSecurity.securityContextConstraints.create }} openshift.io/required-scc: "gremlin" {{- end }} + {{- if include "gremlinGpuCdiAnnotation" . }} + {{- include "gremlinGpuCdiAnnotation" . | nindent 8 }} + {{- end }} {{- if .Values.gremlin.podAnnotations }} {{- toYaml .Values.gremlin.podAnnotations | nindent 8 }} {{- end }} {{- end }} spec: serviceAccountName: gremlin + {{- if include "gremlinGpuRuntimeClassName" . }} + runtimeClassName: {{ include "gremlinGpuRuntimeClassName" . }} + {{- end }} {{- if .Values.affinity }} affinity: {{ toYaml .Values.affinity | trimSuffix "\n" | nindent 8 }} {{- end }} @@ -163,6 +169,9 @@ spec: {{- if include "gremlinTlsIdentityEnv" . }} {{- include "gremlinTlsIdentityEnv" . | nindent 10 }} {{- end }} + {{- if include "gremlinGpuEnv" . }} + {{- include "gremlinGpuEnv" . | nindent 10 }} + {{- end }} {{- with .Values.gremlin.extraEnv }} {{- toYaml . | nindent 10 }} {{- end }} @@ -195,6 +204,9 @@ spec: {{- if include "gremlinTlsIdentityVolumeMounts" . }} {{- include "gremlinTlsIdentityVolumeMounts" . | nindent 10 }} {{- end }} + {{- if include "gremlinGpuVolumeMounts" . }} + {{- include "gremlinGpuVolumeMounts" . | nindent 10 }} + {{- end }} volumes: - name: cgroup-root hostPath: @@ -235,6 +247,9 @@ spec: {{- if include "gremlinTlsIdentityVolumes" . }} {{- include "gremlinTlsIdentityVolumes" . | nindent 8 }} {{- end }} + {{- if include "gremlinGpuVolumes" . }} + {{- include "gremlinGpuVolumes" . | nindent 8 }} + {{- end }} {{- if .Values.gremlin.priorityClassName }} priorityClassName: {{ .Values.gremlin.priorityClassName }} {{- end }} diff --git a/gremlin/templates/opencl-icd-configmap.yaml b/gremlin/templates/opencl-icd-configmap.yaml new file mode 100644 index 0000000..13651f1 --- /dev/null +++ b/gremlin/templates/opencl-icd-configmap.yaml @@ -0,0 +1,26 @@ +{{- /* +Projects an OpenCL ICD registry file into the Gremlin container. + +Some NVIDIA container runtimes inject the OpenCL driver library (libnvidia-opencl.so.1) but do +NOT create the /etc/OpenCL/vendors/.icd registry file that the ICD loader reads to +discover the driver. Without it, the OpenCL platforms cannot be listed. + +The ICD filename and library come from the selected vendor preset (see gremlinGpuVendorPreset). +See gremlinGpuOpenclIcdActive for when this is rendered. +*/ -}} +{{- if include "gremlinGpuOpenclIcdActive" . }} +{{- $eff := fromYaml (include "gremlinGpuEffective" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "gremlin.fullname" . }}-opencl-icd + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ include "gremlin.chart" . }} + app.kubernetes.io/name: {{ include "gremlin.fullname" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + {{ $eff.openclIcd.filename }}: | + {{ $eff.openclIcd.library }} +{{- end }} diff --git a/gremlin/templates/runtimeclass.yaml b/gremlin/templates/runtimeclass.yaml new file mode 100644 index 0000000..bc88697 --- /dev/null +++ b/gremlin/templates/runtimeclass.yaml @@ -0,0 +1,24 @@ +{{- /* +Creates the RuntimeClass referenced by the GPU configuration when it does not already exist. + +NOTE: `lookup` only sees the cluster during `helm install`/`upgrade`. Under `helm template` or a +GitOps renderer with no cluster access, `lookup` returns empty and the RuntimeClass is rendered. +Set gremlin.gpu.createRuntimeClass to false if you manage the RuntimeClass yourself. +*/ -}} +{{- if .Values.gremlin.gpu.enabled -}} +{{- $runtimeClassName := include "gremlinGpuRuntimeClassName" . -}} +{{- if and $runtimeClassName .Values.gremlin.gpu.createRuntimeClass -}} +{{- if not (lookup "node.k8s.io/v1" "RuntimeClass" "" $runtimeClassName) }} +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: {{ $runtimeClassName }} + labels: + app.kubernetes.io/name: {{ include "gremlin.name" . }} + helm.sh/chart: {{ include "gremlin.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +handler: {{ $runtimeClassName | quote }} +{{- end }} +{{- end }} +{{- end }} diff --git a/gremlin/tests/daemonset_gpu_test.yaml b/gremlin/tests/daemonset_gpu_test.yaml new file mode 100644 index 0000000..d95d784 --- /dev/null +++ b/gremlin/tests/daemonset_gpu_test.yaml @@ -0,0 +1,282 @@ +suite: Test GPU / OpenCL access options +templates: + - daemonset.yaml +release: + name: my-release + namespace: my-namespace + revision: 1 + upgrade: true +tests: + - it: should not add any GPU configuration by default + asserts: + - notExists: + path: spec.template.spec.runtimeClassName + - notContains: + path: spec.template.spec.volumes + content: + name: opencl-vendors + hostPath: + path: /etc/OpenCL/vendors + type: DirectoryOrCreate + + - it: should not add GPU configuration when a vendor is set but gpu is disabled + set: + gremlin.gpu.enabled: false + gremlin.gpu.vendor: nvidia + asserts: + - notExists: + path: spec.template.spec.runtimeClassName + - notContains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_VISIBLE_DEVICES + value: "all" + + # --- Easy default: nvidia preset --------------------------------------------------------- + - it: should apply the nvidia preset when enabled with the default vendor + set: + gremlin.gpu.enabled: true + asserts: + - equal: + path: spec.template.spec.runtimeClassName + value: nvidia + - contains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_VISIBLE_DEVICES + value: "all" + - contains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_DRIVER_CAPABILITIES + value: "all" + # nvidia relies on the container toolkit, so no GPU hostPath mounts are added + - notContains: + path: spec.template.spec.volumes + content: + name: opencl-vendors + hostPath: + path: /etc/OpenCL/vendors + type: DirectoryOrCreate + # the OpenCL ICD registry file is projected so the injected driver is discoverable + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/nvidia.icd + subPath: nvidia.icd + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: gremlin-opencl-icd + configMap: + name: my-release-gremlin-opencl-icd + + - it: should not project the OpenCL ICD when no vendor preset defines one + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: "" + asserts: + - notContains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/nvidia.icd + subPath: nvidia.icd + readOnly: true + + # --- Easy default: amd preset ------------------------------------------------------------ + - it: should apply the amd preset when vendor is amd + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: amd + asserts: + # amd does not use a RuntimeClass + - notExists: + path: spec.template.spec.runtimeClassName + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: kfd + mountPath: /dev/kfd + readOnly: false + - contains: + path: spec.template.spec.volumes + content: + name: kfd + hostPath: + path: /dev/kfd + type: CharDevice + - contains: + path: spec.template.spec.volumes + content: + name: dri + hostPath: + path: /dev/dri + type: Directory + - contains: + path: spec.template.spec.volumes + content: + name: opencl-vendors + hostPath: + path: /etc/OpenCL/vendors + type: DirectoryOrCreate + # amd already mounts /etc/OpenCL/vendors from the host, so the chart defers and does not + # project its own ICD file there + - notContains: + path: spec.template.spec.volumes + content: + name: gremlin-opencl-icd + configMap: + name: my-release-gremlin-opencl-icd + + - it: should apply no preset when vendor is blank + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: "" + asserts: + - notExists: + path: spec.template.spec.runtimeClassName + - notContains: + path: spec.template.spec.volumes + content: + name: kfd + hostPath: + path: /dev/kfd + type: CharDevice + + # --- Advanced overrides ------------------------------------------------------------------ + - it: should let runtimeClassName override the vendor preset + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: nvidia + gremlin.gpu.runtimeClassName: nvidia-custom + asserts: + - equal: + path: spec.template.spec.runtimeClassName + value: nvidia-custom + + - it: should let env override the vendor preset + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: nvidia + gremlin.gpu.env: + - name: NVIDIA_VISIBLE_DEVICES + value: "0,1" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_VISIBLE_DEVICES + value: "0,1" + # the preset's NVIDIA_DRIVER_CAPABILITIES is replaced, not merged + - notContains: + path: spec.template.spec.containers[0].env + content: + name: NVIDIA_DRIVER_CAPABILITIES + value: "all" + + - it: should let hostMounts override the vendor preset + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: amd + gremlin.gpu.hostMounts: + - name: opencl-vendors + hostPath: /etc/OpenCL/vendors + mountPath: /etc/OpenCL/vendors + type: DirectoryOrCreate + readOnly: true + - name: nvidia0 + hostPath: /dev/nvidia0 + type: CharDevice + readOnly: false + asserts: + # amd preset device nodes are replaced by the custom mounts + - notContains: + path: spec.template.spec.volumes + content: + name: kfd + hostPath: + path: /dev/kfd + type: CharDevice + # mountPath defaults to hostPath and readOnly is respected + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: nvidia0 + mountPath: /dev/nvidia0 + readOnly: false + - contains: + path: spec.template.spec.volumes + content: + name: nvidia0 + hostPath: + path: /dev/nvidia0 + type: CharDevice + + - it: should default readOnly to true and mountPath to hostPath when omitted + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: "" + gremlin.gpu.hostMounts: + - name: opencl-vendors + hostPath: /etc/OpenCL/vendors + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: opencl-vendors + mountPath: /etc/OpenCL/vendors + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: opencl-vendors + hostPath: + path: /etc/OpenCL/vendors + + # --- CDI (Container Device Interface) ---------------------------------------------------- + - it: should not add a CDI annotation by default + set: + gremlin.gpu.enabled: true + asserts: + - notExists: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + + - it: should add the CDI device annotation when cdiDevice is set + set: + gremlin.gpu.enabled: true + gremlin.gpu.cdiDevice: nvidia.com/gpu=all + asserts: + - equal: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + value: nvidia.com/gpu=all + + - it: should not add a CDI annotation when gpu is disabled + set: + gremlin.gpu.enabled: false + gremlin.gpu.cdiDevice: nvidia.com/gpu=all + asserts: + - notExists: + path: spec.template.metadata.annotations["cdi.k8s.io/gremlin-gpu"] + + # --- ICD projection toggle --------------------------------------------------------------- + - it: should not project the OpenCL ICD when projectOpenclIcd is false + set: + gremlin.gpu.enabled: true + gremlin.gpu.projectOpenclIcd: false + asserts: + - notContains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gremlin-opencl-icd + mountPath: /etc/OpenCL/vendors/nvidia.icd + subPath: nvidia.icd + readOnly: true + - notContains: + path: spec.template.spec.volumes + content: + name: gremlin-opencl-icd + configMap: + name: my-release-gremlin-opencl-icd diff --git a/gremlin/tests/opencl_icd_configmap_test.yaml b/gremlin/tests/opencl_icd_configmap_test.yaml new file mode 100644 index 0000000..8f62a32 --- /dev/null +++ b/gremlin/tests/opencl_icd_configmap_test.yaml @@ -0,0 +1,56 @@ +suite: Test OpenCL ICD registry ConfigMap +templates: + - opencl-icd-configmap.yaml +release: + name: my-release + namespace: my-namespace + revision: 1 + upgrade: true +tests: + - it: should not render when gpu is disabled + asserts: + - hasDocuments: + count: 0 + + - it: should render the nvidia ICD ConfigMap when gpu is enabled + set: + gremlin.gpu.enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: my-release-gremlin-opencl-icd + - equal: + path: data["nvidia.icd"] + value: | + libnvidia-opencl.so.1 + - matchRegex: + path: data["nvidia.icd"] + pattern: libnvidia-opencl\.so\.1 + + - it: should not render when no vendor preset defines an ICD + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: "" + asserts: + - hasDocuments: + count: 0 + + - it: should not render when projectOpenclIcd is false + set: + gremlin.gpu.enabled: true + gremlin.gpu.projectOpenclIcd: false + asserts: + - hasDocuments: + count: 0 + + - it: should not render for the amd preset (host registry is mounted instead) + set: + gremlin.gpu.enabled: true + gremlin.gpu.vendor: amd + asserts: + - hasDocuments: + count: 0 diff --git a/gremlin/values.yaml b/gremlin/values.yaml index c6f7cf0..0c09835 100644 --- a/gremlin/values.yaml +++ b/gremlin/values.yaml @@ -243,6 +243,111 @@ gremlin: # Attacks targeting other Kubernetes pods will use the resource specification of their target. resources: {} + # gremlin.gpu - + # Exposes the host's GPU / OpenCL drivers to the Gremlin agent so that the GPU attack can + # enumerate and target GPUs from inside the container. + # + # The easy path is to just set `enabled: true` and pick a `vendor` (e.g. "nvidia" or "amd"). + # Each vendor is a preset that fills in all the typical options for that hardware, + # so most users never need to touch the advanced options. + # + # The advanced options (runtimeClassName, env, hostMounts) remain available for custom setups. + # Any advanced option you set explicitly OVERRIDES the corresponding value from the vendor + # preset, so you can start from a preset and tweak a single field, or set vendor to "" and + # configure everything by hand. + gpu: + # gremlin.gpu.enabled - + # When true, applies the GPU configuration (vendor preset plus any advanced overrides below) + # to the Gremlin Daemonset. When false, none of the GPU settings have any effect. + enabled: false + + # gremlin.gpu.vendor - + # The easy default. Selects a preset that configures all the normal GPU options for you: + # nvidia - run under the "nvidia" RuntimeClass and set NVIDIA_VISIBLE_DEVICES / + # NVIDIA_DRIVER_CAPABILITIES=all so the NVIDIA container toolkit injects the driver + # libraries and device nodes. Also projects the nvidia OpenCL ICD registry file so + # the injected driver is discoverable. No hostMounts needed. + # amd - mount the /dev/kfd and /dev/dri device nodes and the OpenCL ICD registry + # (/etc/OpenCL/vendors) so the ROCm/amdgpu driver on the host is reachable. + # "" - no preset; configure everything via the advanced options below. + vendor: nvidia + + # gremlin.gpu.createRuntimeClass - + # When true, and the effective runtimeClassName is set, the chart creates that RuntimeClass + # object IF it does not already exist in the cluster. A non-existent RuntimeClass will cause + # the gremlin pod to fail to start. + # + # An existing RuntimeClass of the same name is detected and left untouched, so this will not + # interfere with a RuntimeClass already managed by the GPU Operator or your platform. + createRuntimeClass: false + + # gremlin.gpu.cdiDevice - + # For nodes using the Container Device Interface (CDI) rather than the container-toolkit + # runtime hook. When set, the chart adds a `cdi.k8s.io/gremlin-gpu: ` pod annotation + # so the CRI injects the named CDI device(s). Use this when NVIDIA_VISIBLE_DEVICES is NOT + # honored by your runtime (pure Kubernetes-native CDI). + # + # Requires containerd >= 1.7 or CRI-O with CDI enabled. + # + # NOTE: in pure CDI mode NVIDIA_DRIVER_CAPABILITIES is ignored (capabilities come from the CDI + # device), and you likely do not need a RuntimeClass — set runtimeClassName: "" below if the + # "nvidia" RuntimeClass does not exist on your cluster. + cdiDevice: "" + + # gremlin.gpu.projectOpenclIcd - + # Whether to project the vendor preset's OpenCL ICD registry file (e.g. /etc/OpenCL/vendors/ + # nvidia.icd) into the container. Defaults to true, which fixes runtimes that inject the + # OpenCL driver library but not the ICD file. Set to false when your CDI spec (or another + # mechanism) already creates the ICD file, to avoid a conflicting mount at the same path. + projectOpenclIcd: true + + # --- Advanced options ------------------------------------------------------------------- + # Everything below overrides the vendor preset. Leave a field at its empty default to keep + # the preset's value for that field. + + # gremlin.gpu.runtimeClassName - + # The Kubernetes RuntimeClass to run the Gremlin Daemonset pods under. + runtimeClassName: "" + + # gremlin.gpu.env - + # Extra environment variables applied to the Gremlin Daemonset container when GPU access is + # enabled. For the NVIDIA container toolkit the capabilities value MUST include `compute` + # (or `all`) for OpenCL. + env: [] + # - name: NVIDIA_VISIBLE_DEVICES + # value: "all" + # - name: NVIDIA_DRIVER_CAPABILITIES + # value: "all" + + # gremlin.gpu.hostMounts - + # A list of hostPath volumes mounted into the Gremlin container so the OpenCL ICD loader can + # find vendor drivers and access GPU device nodes. + # Each entry accepts: + # - name: a unique volume name (required) + # - hostPath: the path on the Kubernetes node (required) + # - mountPath: the path inside the container (defaults to hostPath when omitted) + # - type: an optional hostPath type, e.g. Directory, DirectoryOrCreate, CharDevice + # - readOnly: whether the mount is read-only (defaults to true; set false for device nodes) + # + # NOTE: GPU device nodes (/dev/nvidia*, /dev/kfd, /dev/dri) require the container to have + # device access. This generally means running privileged (gremlin.podSecurity.privileged: + # true) or otherwise granting the pod cgroup device permissions. Device access is not + # required when the NVIDIA container toolkit injects the devices for you. + hostMounts: [] + # - name: opencl-vendors + # hostPath: /etc/OpenCL/vendors + # mountPath: /etc/OpenCL/vendors + # type: DirectoryOrCreate + # readOnly: true + # - name: nvidia0 + # hostPath: /dev/nvidia0 + # type: CharDevice + # readOnly: false + # # NVIDIA OpenCL driver library (path/version varies by distro and driver release) + # - name: nvidia-opencl-lib + # hostPath: /usr/lib/x86_64-linux-gnu/libnvidia-opencl.so.1 + # readOnly: true + secret: # Gremlin supports both `certificate` and `secret` types # To manage secrets with helm, set `managed=true` and fill in either the certificate auth or secret auth sections