diff --git a/.gitignore b/.gitignore index 38de891c..91ced1e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Ignore Visual Studio Code settings settings.json +.vscode/ .idea/ Chart.lock \ No newline at end of file diff --git a/charts/mini-runtime/templates/deployment.yaml b/charts/mini-runtime/templates/deployment.yaml index 79a58a57..890528a7 100644 --- a/charts/mini-runtime/templates/deployment.yaml +++ b/charts/mini-runtime/templates/deployment.yaml @@ -49,9 +49,23 @@ spec: - name: AKTO_INSTANCE_TYPE value: {{ quote .Values.mini_runtime.aktoApiSecurityRuntime.env.aktoInstanceType }} - name: AKTO_KAFKA_BROKER_MAL - value: {{ if .Values.mini_runtime.useExternalKafka }}{{ quote .Values.mini_runtime.externalKafka.brokerUrl }}{{ else }}{{ quote .Values.mini_runtime.aktoApiSecurityRuntime.env.aktoKafkaBrokerMal }}{{ end }} + value: >- + {{- if .Values.mini_runtime.useExternalKafka }} + {{ .Values.mini_runtime.externalKafka.brokerUrl }} + {{- else if .Values.mini_runtime.useSeparateKafkaDeployment }} + {{ include "akto.fullname" . }}-kafka.{{ .Release.Namespace }}.svc.{{ .Values.kubernetesClusterDomain }}:9092 + {{- else }} + {{ .Values.mini_runtime.aktoApiSecurityRuntime.env.aktoKafkaBrokerMal }} + {{- end }} - name: AKTO_KAFKA_BROKER_URL - value: {{ if .Values.mini_runtime.useExternalKafka }}{{ quote .Values.mini_runtime.externalKafka.brokerUrl }}{{ else }}{{ quote .Values.mini_runtime.aktoApiSecurityRuntime.env.aktoKafkaBrokerUrl }}{{ end }} + value: >- + {{- if .Values.mini_runtime.useExternalKafka }} + {{ .Values.mini_runtime.externalKafka.brokerUrl }} + {{- else if .Values.mini_runtime.useSeparateKafkaDeployment }} + {{ include "akto.fullname" . }}-kafka.{{ .Release.Namespace }}.svc.{{ .Values.kubernetesClusterDomain }}:9092 + {{- else }} + {{ .Values.mini_runtime.aktoApiSecurityRuntime.env.aktoKafkaBrokerUrl }} + {{- end }} - name: AKTO_KAFKA_GROUP_ID_CONFIG value: {{ quote .Values.mini_runtime.aktoApiSecurityRuntime.env.aktoKafkaGroupIdConfig }} @@ -118,7 +132,7 @@ spec: name: akto-api-security-runtime resources: {{- toYaml .Values.mini_runtime.aktoApiSecurityRuntime.resources | nindent 10 }} - {{- if not .Values.mini_runtime.useExternalKafka }} + {{- if and (not .Values.mini_runtime.useExternalKafka) (not .Values.mini_runtime.useSeparateKafkaDeployment) }} - env: - name: MY_POD_NAMESPACE valueFrom: @@ -258,7 +272,7 @@ spec: {{- end }} restartPolicy: Always - {{- if and (not .Values.mini_runtime.useExternalKafka) (or .Values.mini_runtime.kafka1.useTls .Values.mini_runtime.kafka1.useSasl) }} + {{- if and (not .Values.mini_runtime.useExternalKafka) (not .Values.mini_runtime.useSeparateKafkaDeployment) (or .Values.mini_runtime.kafka1.useTls .Values.mini_runtime.kafka1.useSasl) }} volumes: {{- if .Values.mini_runtime.kafka1.useTls }} - name: kafka-certs @@ -407,9 +421,23 @@ spec: value: {{ quote .Values.threat_client.aktoApiSecurityThreatClient.env.postgresPassword }} - name: AKTO_TRAFFIC_KAFKA_BOOTSTRAP_SERVER - value: {{ if .Values.mini_runtime.useExternalKafka }}{{ quote .Values.mini_runtime.externalKafka.brokerUrl }}{{ else }}akto-mini-runtime-mini-runtime.{{ .Release.Namespace }}.svc.cluster.local:9092{{ end }} + value: >- + {{- if .Values.mini_runtime.useExternalKafka }} + {{ .Values.mini_runtime.externalKafka.brokerUrl }} + {{- else if .Values.mini_runtime.useSeparateKafkaDeployment }} + {{ include "akto.fullname" . }}-kafka.{{ .Release.Namespace }}.svc.{{ .Values.kubernetesClusterDomain }}:9092 + {{- else }} + {{ .Release.Name }}-mini-runtime.{{ .Release.Namespace }}.svc.{{ .Values.kubernetesClusterDomain }}:9092 + {{- end }} - name: AKTO_INTERNAL_KAFKA_BOOTSTRAP_SERVER - value: {{ if .Values.mini_runtime.useExternalKafka }}{{ quote .Values.mini_runtime.externalKafka.brokerUrl }}{{ else }}akto-mini-runtime-mini-runtime.{{ .Release.Namespace }}.svc.cluster.local:9092{{ end }} + value: >- + {{- if .Values.mini_runtime.useExternalKafka }} + {{ .Values.mini_runtime.externalKafka.brokerUrl }} + {{- else if .Values.mini_runtime.useSeparateKafkaDeployment }} + {{ include "akto.fullname" . }}-kafka.{{ .Release.Namespace }}.svc.{{ .Values.kubernetesClusterDomain }}:9092 + {{- else }} + {{ .Release.Name }}-mini-runtime.{{ .Release.Namespace }}.svc.{{ .Values.kubernetesClusterDomain }}:9092 + {{- end }} - name: AKTO_THREAT_DETECTION_LOCAL_REDIS_URI value: "redis://{{ include "akto.fullname" . }}-redis:6379" diff --git a/charts/mini-runtime/templates/kafka-service.yaml b/charts/mini-runtime/templates/kafka-service.yaml new file mode 100644 index 00000000..54fa4d7d --- /dev/null +++ b/charts/mini-runtime/templates/kafka-service.yaml @@ -0,0 +1,50 @@ +{{- if and (not .Values.mini_runtime.useExternalKafka) .Values.mini_runtime.useSeparateKafkaDeployment }} +--- +# Headless Service for StatefulSet pod DNS +apiVersion: v1 +kind: Service +metadata: + name: {{ include "akto.fullname" . }}-kafka-headless + labels: + {{- include "akto.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka +spec: + clusterIP: None + selector: + app: {{ include "akto.fullname" . }}-kafka + {{- include "akto.selectorLabels" . | nindent 4 }} + ports: + {{- if .Values.mini_runtime.kafka1.useSasl }} + {{- .Values.ports.sasl | toYaml | nindent 4 -}} + {{- else if .Values.mini_runtime.kafka1.useTls }} + {{- .Values.ports.tls | toYaml | nindent 4 -}} + {{- else }} + {{- .Values.ports.default | toYaml | nindent 4 -}} + {{- end }} +--- +# Regular Service for client connections +apiVersion: v1 +kind: Service +metadata: + name: {{ include "akto.fullname" . }}-kafka + labels: + {{- include "akto.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + annotations: + {{- with .Values.mini_runtime.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.mini_runtime.type }} + selector: + app: {{ include "akto.fullname" . }}-kafka + {{- include "akto.selectorLabels" . | nindent 4 }} + ports: + {{- if .Values.mini_runtime.kafka1.useSasl }} + {{- .Values.ports.sasl | toYaml | nindent 4 -}} + {{- else if .Values.mini_runtime.kafka1.useTls }} + {{- .Values.ports.tls | toYaml | nindent 4 -}} + {{- else }} + {{- .Values.ports.default | toYaml | nindent 4 -}} + {{- end }} +{{- end }} diff --git a/charts/mini-runtime/templates/kafka-statefulset.yaml b/charts/mini-runtime/templates/kafka-statefulset.yaml new file mode 100644 index 00000000..1deb1184 --- /dev/null +++ b/charts/mini-runtime/templates/kafka-statefulset.yaml @@ -0,0 +1,207 @@ +{{- if and (not .Values.mini_runtime.useExternalKafka) .Values.mini_runtime.useSeparateKafkaDeployment }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "akto.fullname" . }}-kafka + labels: + {{- include "akto.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka +spec: + serviceName: {{ include "akto.fullname" . }}-kafka-headless + replicas: {{ .Values.mini_runtime.kafka1.replicas }} + selector: + matchLabels: + app: {{ include "akto.fullname" . }}-kafka + {{- include "akto.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: {{ include "akto.fullname" . }}-kafka + {{- include "akto.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: kafka + spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: kafka + image: {{ .Values.mini_runtime.kafka1.image.repository }}:{{ .Values.mini_runtime.kafka1.image.tag | default .Chart.AppVersion }} + command: + - bash + - -c + - | + # Extract pod ordinal from hostname + export POD_ORDINAL=${HOSTNAME##*-} + export KAFKA_NODE_ID=${POD_ORDINAL} + export KAFKA_BROKER_ID=${POD_ORDINAL} + + # Build controller quorum voters list for KRaft + QUORUM_VOTERS="" + for i in $(seq 0 $(({{ .Values.mini_runtime.kafka1.replicas }} - 1))); do + if [ -n "$QUORUM_VOTERS" ]; then + QUORUM_VOTERS="${QUORUM_VOTERS}," + fi + QUORUM_VOTERS="${QUORUM_VOTERS}${i}@{{ include "akto.fullname" . }}-kafka-${i}.{{ include "akto.fullname" . }}-kafka-headless.${MY_POD_NAMESPACE}.svc.{{ .Values.kubernetesClusterDomain }}:9094" + done + export KAFKA_CONTROLLER_QUORUM_VOTERS=$QUORUM_VOTERS + + echo "Starting Kafka with NODE_ID=${KAFKA_NODE_ID}" + echo "Controller Quorum: ${KAFKA_CONTROLLER_QUORUM_VOTERS}" + + # Start Kafka + exec /etc/confluent/docker/run + ports: + - containerPort: 9092 + name: kafka + - containerPort: 9093 + name: kafka-ssl + - containerPort: 9094 + name: controller + - containerPort: 9999 + name: jmx + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KAFKA_ADVERTISED_LISTENERS + value: >- + {{- if .Values.mini_runtime.kafka1.env.kafkaAdvertisedListeners }} + {{ .Values.mini_runtime.kafka1.env.kafkaAdvertisedListeners }} + {{- else if and .Values.mini_runtime.kafka1.useSasl .Values.mini_runtime.kafka1.useTls }} + LISTENER_DOCKER_EXTERNAL_DIFFHOST_SASL://$(MY_POD_NAME).{{ include "akto.fullname" . }}-kafka-headless.$(MY_POD_NAMESPACE).svc.{{ .Values.kubernetesClusterDomain }}:9093 + {{- else if .Values.mini_runtime.kafka1.useSasl }} + LISTENER_DOCKER_EXTERNAL_DIFFHOST_SASL://$(MY_POD_NAME).{{ include "akto.fullname" . }}-kafka-headless.$(MY_POD_NAMESPACE).svc.{{ .Values.kubernetesClusterDomain }}:9092 + {{- else if .Values.mini_runtime.kafka1.useTls }} + LISTENER_DOCKER_EXTERNAL_DIFFHOST://$(MY_POD_NAME).{{ include "akto.fullname" . }}-kafka-headless.$(MY_POD_NAMESPACE).svc.{{ .Values.kubernetesClusterDomain }}:9092,LISTENER_DOCKER_EXTERNAL_DIFFHOST_ENCRYPTED://$(MY_POD_NAME).{{ include "akto.fullname" . }}-kafka-headless.$(MY_POD_NAMESPACE).svc.{{ .Values.kubernetesClusterDomain }}:9093 + {{- else }} + LISTENER_DOCKER_EXTERNAL_DIFFHOST://$(MY_POD_NAME).{{ include "akto.fullname" . }}-kafka-headless.$(MY_POD_NAMESPACE).svc.{{ .Values.kubernetesClusterDomain }}:9092 + {{- end }} + - name: KAFKA_CLEANUP_POLICY + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaCleanupPolicy }} + - name: KAFKA_CREATE_TOPICS + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaCreateTopics }} + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: >- + {{- if .Values.mini_runtime.kafka1.env.kafkaAdvertisedListeners }} + {{ .Values.mini_runtime.kafka1.env.kafkaInterBrokerListenerName }} + {{- else if .Values.mini_runtime.kafka1.useSasl }} + LISTENER_DOCKER_EXTERNAL_DIFFHOST_SASL + {{- else if .Values.mini_runtime.kafka1.useTls }} + LISTENER_DOCKER_EXTERNAL_DIFFHOST + {{- else }} + LISTENER_DOCKER_EXTERNAL_DIFFHOST + {{- end }} + - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP + value: >- + {{- if and .Values.mini_runtime.kafka1.useSasl .Values.mini_runtime.kafka1.useTls }} + {{ .Values.mini_runtime.kafka1.env.kafkaListenerSecurityProtocolMapSaslTls }} + {{- else if .Values.mini_runtime.kafka1.useSasl }} + {{ .Values.mini_runtime.kafka1.env.kafkaListenerSecurityProtocolMapSasl }} + {{- else if .Values.mini_runtime.kafka1.useTls }} + {{ .Values.mini_runtime.kafka1.env.kafkaListenerSecurityProtocolMapSsl }} + {{- else }} + CONTROLLER:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT + {{- end }} + - name: KAFKA_PROCESS_ROLES + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaProcessRoles }} + - name: KAFKA_CONTROLLER_LISTENER_NAMES + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaControllerListenerNames }} + - name: KAFKA_LISTENERS + value: >- + {{- if .Values.mini_runtime.kafka1.env.kafkaListeners }} + {{ .Values.mini_runtime.kafka1.env.kafkaListeners }} + {{- else if and .Values.mini_runtime.kafka1.useSasl .Values.mini_runtime.kafka1.useTls }} + CONTROLLER://0.0.0.0:9094,LISTENER_DOCKER_EXTERNAL_DIFFHOST_SASL://0.0.0.0:9093 + {{- else if .Values.mini_runtime.kafka1.useSasl }} + CONTROLLER://0.0.0.0:9094,LISTENER_DOCKER_EXTERNAL_DIFFHOST_SASL://0.0.0.0:9092 + {{- else if .Values.mini_runtime.kafka1.useTls }} + CONTROLLER://0.0.0.0:9094,LISTENER_DOCKER_EXTERNAL_DIFFHOST_ENCRYPTED://0.0.0.0:9093 + {{- else }} + CONTROLLER://0.0.0.0:9094,LISTENER_DOCKER_EXTERNAL_DIFFHOST://0.0.0.0:9092 + {{- end }} + - name: KAFKA_LOG_CLEANER_ENABLE + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaLogCleanerEnable }} + - name: KAFKA_LOG_RETENTION_BYTES + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaLogRetentionBytes }} + - name: KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaLogRetentionCheckIntervalMs }} + - name: KAFKA_LOG_RETENTION_HOURS + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaLogRetentionHours }} + - name: KAFKA_LOG_SEGMENT_BYTES + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaLogSegmentBytes }} + - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaOffsetsTopicReplicationFactor }} + - name: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaTransactionStateLogMinIsr }} + - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaTransactionStateLogReplicationFactor }} + - name: CLUSTER_ID + value: {{ quote .Values.mini_runtime.kafka1.env.kafkaClusterId }} + {{- if .Values.mini_runtime.kafka1.useTls }} + - name: KAFKA_SSL_KEYSTORE_LOCATION + value: {{ quote .Values.mini_runtime.kafka1.env.sslKeystoreLocation }} + - name: KAFKA_SSL_KEYSTORE_PASSWORD + value: {{ quote .Values.mini_runtime.kafka1.env.sslKeystorePassword }} + - name: KAFKA_SSL_KEY_PASSWORD + value: {{ quote .Values.mini_runtime.kafka1.env.sslKeyPassword }} + - name: KAFKA_SSL_TRUSTSTORE_LOCATION + value: {{ quote .Values.mini_runtime.kafka1.env.sslTruststoreLocation }} + - name: KAFKA_SSL_TRUSTSTORE_PASSWORD + value: {{ quote .Values.mini_runtime.kafka1.env.sslTruststorePassword }} + {{- end }} + {{- if .Values.mini_runtime.kafka1.useSasl }} + - name: KAFKA_OPTS + value: "-Djava.security.auth.login.config=/etc/kafka/config/kafka_server_jaas.conf" + - name: KAFKA_SASL_ENABLED_MECHANISMS + value: "PLAIN" + - name: KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL + value: "PLAIN" + {{- end }} + - name: KUBERNETES_CLUSTER_DOMAIN + value: {{ quote .Values.kubernetesClusterDomain }} + {{- if or .Values.mini_runtime.kafka1.useTls .Values.mini_runtime.kafka1.useSasl }} + volumeMounts: + {{- if .Values.mini_runtime.kafka1.useTls }} + - name: kafka-certs + mountPath: {{ quote .Values.mini_runtime.kafka1.env.sslBaseMountPath }} + {{- end }} + {{- if .Values.mini_runtime.kafka1.useSasl }} + - name: kafka-jaas-config + mountPath: /etc/kafka/config + readOnly: true + {{- end }} + {{- end }} + resources: {{- toYaml .Values.mini_runtime.kafka1.resources | nindent 10 }} + restartPolicy: Always + {{- if or .Values.mini_runtime.kafka1.useTls .Values.mini_runtime.kafka1.useSasl }} + volumes: + {{- if .Values.mini_runtime.kafka1.useTls }} + - name: kafka-certs + secret: + secretName: {{ quote .Values.mini_runtime.kafka1.env.sslSecretName }} + {{- end }} + {{- if .Values.mini_runtime.kafka1.useSasl }} + - name: kafka-jaas-config + configMap: + name: {{ include "akto.fullname" . }}-kafka-jaas-config + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/mini-runtime/templates/mini-runtime.yaml b/charts/mini-runtime/templates/mini-runtime.yaml index 815d40a9..ad494d47 100644 --- a/charts/mini-runtime/templates/mini-runtime.yaml +++ b/charts/mini-runtime/templates/mini-runtime.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.mini_runtime.useExternalKafka }} +{{- if and (not .Values.mini_runtime.useExternalKafka) (not .Values.mini_runtime.useSeparateKafkaDeployment) }} apiVersion: v1 kind: Service metadata: diff --git a/charts/mini-runtime/values.yaml b/charts/mini-runtime/values.yaml index 29259d85..0f9f92f0 100644 --- a/charts/mini-runtime/values.yaml +++ b/charts/mini-runtime/values.yaml @@ -27,6 +27,12 @@ affinity: {} # - coreservices mini_runtime: + # Kafka Deployment Mode Configuration + # useSeparateKafkaDeployment: Set to true to deploy Kafka as a separate StatefulSet (recommended for production) + # When true, Kafka will be deployed as an independent, horizontally scalable cluster with FQDN-based communication + # When false, Kafka will run as a sidecar container in the mini-runtime pod (legacy mode, localhost communication) + useSeparateKafkaDeployment: true + # External Kafka Configuration # Set useExternalKafka to true to use an existing Kafka cluster # When enabled, the internal Kafka container will not be deployed @@ -76,12 +82,14 @@ mini_runtime: cpu: 2 memory: "8Gi" kafka1: + # Number of Kafka replicas (only applies when useSeparateKafkaDeployment is true) + replicas: 2 env: # add advertised listeners if you want to override the default listeners kafkaAdvertisedListeners: "" kafkaBrokerId: "1" kafkaCleanupPolicy: delete - kafkaCreateTopics: akto.api.logs:3:3 + kafkaCreateTopics: akto.api.logs:12:2 kafkaInterBrokerListenerName: LISTENER_DOCKER_EXTERNAL_LOCALHOST kafkaListenerSecurityProtocolMap: CONTROLLER:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT kafkaListenerSecurityProtocolMapSsl: CONTROLLER:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_DIFFHOST_ENCRYPTED:SSL @@ -97,9 +105,9 @@ mini_runtime: kafkaLogRetentionCheckIntervalMs: "60000" kafkaLogRetentionHours: "5" kafkaLogSegmentBytes: "104857600" - kafkaOffsetsTopicReplicationFactor: "1" + kafkaOffsetsTopicReplicationFactor: "2" kafkaTransactionStateLogMinIsr: "1" - kafkaTransactionStateLogReplicationFactor: "1" + kafkaTransactionStateLogReplicationFactor: "2" kafkaClusterId: "c6a1b8e2-4f2a-4b2a-9c3f-1a2b3c4d5e6f" sslKeystoreLocation: "/etc/kafka/ssl/secrets/server.keystore.jks" sslKeystorePassword: "password"