Skip to content

Commit fc4f677

Browse files
author
Cameron Wright
committed
preparing kafka 4.0 for deployment
1 parent 95138e0 commit fc4f677

6 files changed

Lines changed: 89 additions & 12 deletions

File tree

‎.github/workflows/publish-kafka-docker.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,23 @@ jobs:
5353
run: |
5454
KAFKA_VERSION=$(echo "${{ needs.prepare.outputs.tag }}" | sed 's/kafka-//')
5555
echo "kafka_version=$KAFKA_VERSION" >> $GITHUB_OUTPUT
56+
- name: Validate tagged Kafka distribution
57+
env:
58+
IMAGE_VERSION: ${{ steps.kafka_version.outputs.kafka_version }}
59+
run: |
60+
set -euo pipefail
61+
if [[ ! "$IMAGE_VERSION" =~ ^([0-9]+\.[0-9]+)-([0-9]+\.[0-9]+\.[0-9]+)(-jdk[0-9]+\.[0-9]+\.[0-9]+)?$ ]]; then
62+
echo "::error::Expected kafka-<scala>-<kafka>[-jdk<version>] release tag"
63+
exit 1
64+
fi
65+
TAG_SCALA="${BASH_REMATCH[1]}"
66+
TAG_KAFKA="${BASH_REMATCH[2]}"
67+
DIST_SCALA=$(sed -nE 's/^ARG scala_version=([^[:space:]]+).*$/\1/p' docker/kafka/Dockerfile)
68+
DIST_KAFKA=$(sed -nE 's/^ARG KAFKA_VERSION=([^[:space:]]+).*$/\1/p' docker/kafka/Dockerfile)
69+
if [[ "$TAG_SCALA" != "$DIST_SCALA" || "$TAG_KAFKA" != "$DIST_KAFKA" ]]; then
70+
echo "::error::Tag $IMAGE_VERSION does not match Dockerfile distribution $DIST_SCALA-$DIST_KAFKA"
71+
exit 1
72+
fi
5673
- name: Check if Kafka image already exists
5774
id: image_exists
5875
run: |

‎docker/kafka/Dockerfile‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
ARG scala_version=2.13
2-
# Renovate is pinned to the 3.x line (see the apache/kafka packageRule in renovate.json).
3-
# Kafka 4.x removes ZooKeeper support entirely (KRaft-only) and needs an operator-side
4-
# migration; when this repo is ready for that, drop the allowedVersions constraint too.
5-
# Also bump java_version to 25 (next LTS after 21) as part of that migration.
6-
ARG KAFKA_VERSION=3.9.2 # renovate: datasource=github-tags depName=apache/kafka
2+
# Kafka 4 is KRaft-only.
3+
ARG KAFKA_VERSION=4.0.2 # renovate: datasource=github-tags depName=apache/kafka
74
ARG java_version=21
85

96
FROM alpine:latest@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a AS kafka_dist
@@ -139,7 +136,9 @@ RUN apt-get update && \
139136
mkdir -p ${KAFKA_HOME}/libs/extensions
140137

141138
COPY --from=kafka_dist /var/tmp/kafka_$scala_version-$KAFKA_VERSION ${KAFKA_HOME}
142-
COPY opt/kafka/config/log4j.properties ${KAFKA_HOME}/config/log4j.properties
139+
COPY opt/kafka/config/log4j2.properties ${KAFKA_HOME}/config/log4j2.properties
140+
141+
ENV KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=/opt/kafka/config/log4j2.properties"
143142

144143
# Optimize Kafka installation: remove unnecessary files and set permissions
145144
RUN chmod a+x ${KAFKA_HOME}/bin/*.sh && \

‎docker/kafka/opt/kafka/config/log4j2.properties‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
status=warn
2+
name=KafkaConsole
3+
appender.console.type=Console
4+
appender.console.name=STDOUT
5+
appender.console.target=SYSTEM_OUT
6+
appender.console.layout.type=PatternLayout
7+
appender.console.layout.pattern=[%d] %p %m (%c)%n
8+
rootLogger.level=info
9+
rootLogger.appenderRef.stdout.ref=STDOUT
10+
logger.requests.name=kafka.request.logger
11+
logger.requests.level=warn
12+
logger.authorizer.name=kafka.authorizer.logger
13+
logger.authorizer.level=info

‎pkg/resources/kafka/pod.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -656,9 +656,11 @@ func generateEnvConfig(brokerConfig *v1beta1.BrokerConfig, defaultEnvVars []core
656656
}
657657

658658
if brokerConfig.Log4jConfig != "" {
659-
envs["KAFKA_LOG4J_OPTS"] = corev1.EnvVar{
660-
Name: "KAFKA_LOG4J_OPTS",
661-
Value: "-Dlog4j.configuration=file:/config/log4j.properties",
659+
if _, configured := envs["KAFKA_LOG4J_OPTS"]; !configured {
660+
envs["KAFKA_LOG4J_OPTS"] = corev1.EnvVar{
661+
Name: "KAFKA_LOG4J_OPTS",
662+
Value: "-Dlog4j.configuration=file:/config/log4j.properties",
663+
}
662664
}
663665
}
664666

‎pkg/resources/kafka/pod_test.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,52 @@ func Test_generateEnvConfig(t *testing.T) {
188188
}
189189
}
190190

191+
func TestGenerateEnvConfigLogging(t *testing.T) {
192+
for _, test := range []struct {
193+
name string
194+
config string
195+
envs []corev1.EnvVar
196+
expected corev1.EnvVar
197+
expectFound bool
198+
}{
199+
{
200+
name: "legacy logging default",
201+
config: "log4j.rootLogger=INFO, stdout",
202+
expected: corev1.EnvVar{Name: "KAFKA_LOG4J_OPTS", Value: "-Dlog4j.configuration=file:/config/log4j.properties"},
203+
expectFound: true,
204+
},
205+
{
206+
name: "explicit Log4j 2 option",
207+
config: "rootLogger.level=info",
208+
envs: []corev1.EnvVar{{Name: "KAFKA_LOG4J_OPTS", Value: "-Dlog4j2.configurationFile=/config/log4j.properties"}},
209+
expected: corev1.EnvVar{Name: "KAFKA_LOG4J_OPTS", Value: "-Dlog4j2.configurationFile=/config/log4j.properties"},
210+
expectFound: true,
211+
},
212+
{
213+
name: "explicit option from secret",
214+
config: "rootLogger.level=info",
215+
envs: []corev1.EnvVar{{Name: "KAFKA_LOG4J_OPTS", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "logging"}, Key: "opts"}}}},
216+
expected: corev1.EnvVar{Name: "KAFKA_LOG4J_OPTS", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "logging"}, Key: "opts"}}},
217+
expectFound: true,
218+
},
219+
{
220+
name: "image logging default without custom config",
221+
},
222+
} {
223+
t.Run(test.name, func(t *testing.T) {
224+
result := generateEnvConfig(&v1beta1.BrokerConfig{Log4jConfig: test.config, Envs: test.envs}, nil)
225+
found := false
226+
for _, env := range result {
227+
if env.Name == "KAFKA_LOG4J_OPTS" {
228+
found = true
229+
assert.DeepEqual(t, env, test.expected)
230+
}
231+
}
232+
assert.Equal(t, found, test.expectFound)
233+
})
234+
}
235+
}
236+
191237
func TestAddDynamicKRaftQuorumFormatEnv(t *testing.T) {
192238
brokers := []v1beta1.Broker{
193239
{Id: 0, BrokerConfig: &v1beta1.BrokerConfig{Roles: []string{"controller"}}},

‎renovate.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -171,8 +171,8 @@
171171
"matchPackageNames": [
172172
"apache/kafka"
173173
],
174-
"allowedVersions": "<4.0.0",
175-
"description": "Kafka 4.x drops ZooKeeper support (KRaft-only) and needs an operator-side migration. Keep Renovate on the 3.x line until docker/kafka/Dockerfile is ready for that jump, then remove this rule."
174+
"allowedVersions": ">=4.0.2 <4.1.0",
175+
"description": "Kafka 4.0.2 is the OR2 dev upgrade target. Keep distribution updates on the 4.0 line; later minor/major versions require separate validation."
176176
},
177177
{
178178
"matchCurrentValue": "/^\\d+\\.\\d+\\.\\d+-ado?be-\\d+$/",
@@ -195,7 +195,7 @@
195195
],
196196
"matchCurrentValue": "/^\\d+\\.\\d+-\\d+\\.\\d+\\.\\d+(-jdk\\d+\\.\\d+\\.\\d+)?$/",
197197
"versioning": "regex:^(?<compatibility>\\d+\\.\\d+)-(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)(?:-jdk(?<build>\\d+)\\.\\d+\\.(?<revision>\\d+))?$",
198-
"description": "Tag format is <scala_version>-<kafka_version>[-jdk<jdk_version>], e.g. 2.13-3.9.1 or 2.13-3.9.2-jdk21.0.11 (the latter shape shipped starting with the 3.9.2 release tag). Verified live against ghcr.io: default docker versioning parses currentVersion as just '2.13' and silently drops everything after the first hyphen, so Renovate can never see a real 3.9.1->3.9.2 bump (only digest-pin updates fire, via the docker pinDigests rule below) - this affects every '# renovate:' annotated occurrence of this image. Pinning the scala prefix as 'compatibility' (never auto-changed) and parsing the kafka release as major.minor.patch fixes real version comparison. The optional '-jdk<major>.<interim>.<update>' suffix maps jdk major to 'build' and jdk update to 'revision' (Renovate's regex versioning caps out at 5 numeric levels: major/minor/patch/build/revision, each required to be purely numeric, so the always-0 jdk interim segment is matched but not captured); tags without the suffix default build/revision to 0 and still compare correctly against suffixed ones since a differing kafka patch alone already resolves ordering. This does NOT gate jdk-major jumps (e.g. jdk21.x -> jdk25.x) the way 'compatibility' gates scala - that's covered by the same operator-side migration as the apache/kafka 4.x bump below (see docker/kafka/Dockerfile's java_version comment); revisit if jdk25 lands before that migration."
198+
"description": "Tag format is <scala_version>-<kafka_version>[-jdk<jdk_version>], e.g. 2.13-3.9.2-jdk21.0.11 or 2.13-4.0.2. Pinning the scala prefix as compatibility and parsing the Kafka release as major.minor.patch permits real Kafka version comparisons. The optional JDK suffix maps its major version to build and update to revision. JDK major upgrades require separate validation; the Kafka 4.0.2 image retains Java 21."
199199
},
200200
{
201201
"groupName": "kafka docker images (major)",

0 commit comments

Comments
 (0)