Skip to content

Commit d2d9e61

Browse files
authored
feat(kafka): Add 3.9.2 and 4.2.1 (#1483)
* add kafka 4.2.0 * disable cyclone dx * update cyclone plugin and java versions * replace 4.2.0 with 4.2.1 * fix jackson dependency * add kafka 3.9.2 * update changelog * document sources for jvm compatibility
1 parent 99ea108 commit d2d9e61

11 files changed

Lines changed: 616 additions & 0 deletions

CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file.
1212
- nifi: Backport NIFI-15901 to 2.x versions ([#1481]).
1313
- testing-tools: Added grpcurl utility ([#1493]).
1414
- opa: Add `1.16.2` ([#1509]).
15+
- kafka: Add `3.9.2` and `4.2.1` ([#1483])
1516

1617
### Changed
1718

@@ -41,6 +42,7 @@ All notable changes to this project will be documented in this file.
4142
[#1474]: https://github.com/stackabletech/docker-images/pull/1474
4243
[#1476]: https://github.com/stackabletech/docker-images/pull/1476
4344
[#1481]: https://github.com/stackabletech/docker-images/pull/1481
45+
[#1483]: https://github.com/stackabletech/docker-images/pull/1483
4446
[#1493]: https://github.com/stackabletech/docker-images/pull/1493
4547
[#1507]: https://github.com/stackabletech/docker-images/pull/1507
4648
[#1509]: https://github.com/stackabletech/docker-images/pull/1509

kafka/boil-config.toml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,17 @@ java-devel = "21"
1212
scala-version = "2.13"
1313
jmx-exporter-version = "1.3.0"
1414

15+
[versions."3.9.2".local-images]
16+
java-base = "21"
17+
java-devel = "21"
18+
"kafka/kcat" = "1.7.0"
19+
"kafka/kafka-opa-plugin" = "1.5.1"
20+
"shared/reload4j" = "1.2.25"
21+
22+
[versions."3.9.2".build-arguments]
23+
scala-version = "2.13"
24+
jmx-exporter-version = "1.3.0"
25+
1526
[versions."4.1.1".local-images]
1627
java-base = "24"
1728
java-devel = "24"
@@ -25,3 +36,18 @@ java-devel = "24"
2536
[versions."4.1.1".build-arguments]
2637
scala-version = "2.13"
2738
jmx-exporter-version = "1.3.0"
39+
40+
[versions."4.2.1".local-images]
41+
# For JVM compatibility see README.md and docs/getting-started/compatibility.md in the Kafka repository
42+
java-base = "25"
43+
java-devel = "25"
44+
"kafka/kcat" = "1.7.0"
45+
"kafka/kafka-opa-plugin" = "1.5.1"
46+
# TODO: this is not used in this version but it's added
47+
# to avoid major changes to the Kafka image build on short notice.
48+
# Building this image is quick and in CI should not even be noticed.
49+
"shared/reload4j" = "1.2.25"
50+
51+
[versions."4.2.1".build-arguments]
52+
scala-version = "2.13"
53+
jmx-exporter-version = "1.3.0"
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
From c19817ee82f7b90aac0a8be66e3efe17f268eb35 Mon Sep 17 00:00:00 2001
2+
From: Lukas Voetmand <lukas.voetmand@stackable.tech>
3+
Date: Thu, 17 Oct 2024 11:01:40 +0200
4+
Subject: Add CycloneDX plugin
5+
6+
---
7+
build.gradle | 42 ++++++++++++++++++++++++++++++++++++++++++
8+
1 file changed, 42 insertions(+)
9+
10+
diff --git a/build.gradle b/build.gradle
11+
index 381e964d13..f51f410858 100644
12+
--- a/build.gradle
13+
+++ b/build.gradle
14+
@@ -48,6 +48,48 @@ plugins {
15+
// We are going to drop JDK8 support. Hence, the spotless is upgrade to newest version and be applied only if the build env is compatible with JDK 11.
16+
// spotless 6.15.0+ has issue in runtime with JDK8 even through we define it with `apply:false`. see https://github.com/diffplug/spotless/issues/2156 for more details
17+
id 'com.diffplug.spotless' version "6.14.0" apply false
18+
+ id 'org.cyclonedx.bom' version '1.10.0'
19+
+}
20+
+
21+
+cyclonedxBom {
22+
+ // Specified the type of project being built. Defaults to 'library'
23+
+ projectType = "application"
24+
+ // Specified the version of the CycloneDX specification to use. Defaults to '1.5'
25+
+ schemaVersion = "1.5"
26+
+ // Boms destination directory. Defaults to 'build/reports'
27+
+ destination = file("build/reports")
28+
+ // The file name for the generated BOMs (before the file format suffix). Defaults to 'bom'
29+
+ outputName = "bom"
30+
+ // The file format generated, can be xml, json or all for generating both. Defaults to 'all'
31+
+ outputFormat = "json"
32+
+ includeConfigs = ["runtimeClasspath"]
33+
+ // Exclude test components. This list needs to be checked and, if it changed, updated for every new Kafka version.
34+
+ // The list can be obtained by running `gradle projects | grep upgrade-system-tests`
35+
+ skipProjects = [
36+
+ 'upgrade-system-tests-0100',
37+
+ 'upgrade-system-tests-0101',
38+
+ 'upgrade-system-tests-0102',
39+
+ 'upgrade-system-tests-0110',
40+
+ 'upgrade-system-tests-10',
41+
+ 'upgrade-system-tests-11',
42+
+ 'upgrade-system-tests-20',
43+
+ 'upgrade-system-tests-21',
44+
+ 'upgrade-system-tests-22',
45+
+ 'upgrade-system-tests-23',
46+
+ 'upgrade-system-tests-24',
47+
+ 'upgrade-system-tests-25',
48+
+ 'upgrade-system-tests-26',
49+
+ 'upgrade-system-tests-27',
50+
+ 'upgrade-system-tests-28',
51+
+ 'upgrade-system-tests-30',
52+
+ 'upgrade-system-tests-31',
53+
+ 'upgrade-system-tests-32',
54+
+ 'upgrade-system-tests-33',
55+
+ 'upgrade-system-tests-34',
56+
+ 'upgrade-system-tests-35',
57+
+ 'upgrade-system-tests-36',
58+
+ 'upgrade-system-tests-37'
59+
+ ]
60+
}
61+
62+
ext {
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
From 343021661d7ed166a7555e6bce9da4805340f6cb Mon Sep 17 00:00:00 2001
2+
From: Lars Francke <git@lars-francke.de>
3+
Date: Thu, 12 Dec 2024 10:09:47 +0100
4+
Subject: Change Gradle to use the Nexus Build Repo
5+
6+
---
7+
build.gradle | 8 ++++++--
8+
1 file changed, 6 insertions(+), 2 deletions(-)
9+
10+
diff --git a/build.gradle b/build.gradle
11+
index f51f410858..5ccd99e024 100644
12+
--- a/build.gradle
13+
+++ b/build.gradle
14+
@@ -20,7 +20,9 @@ import java.nio.charset.StandardCharsets
15+
16+
buildscript {
17+
repositories {
18+
- mavenCentral()
19+
+ maven {
20+
+ url 'https://build-repo.stackable.tech/repository/maven-public/'
21+
+ }
22+
}
23+
apply from: "$rootDir/gradle/dependencies.gradle"
24+
25+
@@ -168,7 +170,9 @@ ext {
26+
allprojects {
27+
28+
repositories {
29+
- mavenCentral()
30+
+ maven {
31+
+ url 'https://build-repo.stackable.tech/repository/maven-public/'
32+
+ }
33+
}
34+
35+
dependencyUpdates {
Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
From 944ef05a7468a0437c0da923898aecae983eb5f6 Mon Sep 17 00:00:00 2001
2+
From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
3+
Date: Mon, 1 Jun 2026 08:52:19 +0200
4+
Subject: Build custom Stackable version
5+
6+
How it was done:
7+
8+
Replace in files in root dir (do not garble .git):
9+
10+
for d in $(ls -p|grep -v /); do find $d -type f -exec sed -i 's/3\.9\.2/3.9.2-stackable0.0.0-dev/g' {} +; done
11+
12+
Replace in all subdirs:
13+
14+
for d in $(ls -d */); do find $d -type f -exec sed -i 's/3\.9\.2/3.9.2-stackable0.0.0-dev/g' {} +; done
15+
---
16+
docs/getting-started/upgrade.md | 2 +-
17+
docs/operations/kraft.md | 2 +-
18+
gradle.properties | 2 +-
19+
kafka-merge-pr.py | 2 +-
20+
streams/quickstart/java/pom.xml | 2 +-
21+
.../java/src/main/resources/archetype-resources/pom.xml | 2 +-
22+
streams/quickstart/pom.xml | 2 +-
23+
tests/kafkatest/__init__.py | 2 +-
24+
tests/kafkatest/version.py | 2 +-
25+
9 files changed, 9 insertions(+), 9 deletions(-)
26+
27+
diff --git a/docs/getting-started/upgrade.md b/docs/getting-started/upgrade.md
28+
index d7347b3ee8..7f888fc90a 100644
29+
--- a/docs/getting-started/upgrade.md
30+
+++ b/docs/getting-started/upgrade.md
31+
@@ -58,7 +58,7 @@ If you are upgrading from version 2.4.0 or above, and you have not overridden th
32+
33+
34+
35+
-### Notable changes in 3.9.2
36+
+### Notable changes in 3.9.2-stackable0.0.0-dev
37+
38+
* There is a new server configuration, `alter.config.policy.kraft.compatibility.enable`, which ensures that an AlterConfigPolicy is invoked consistently between KRaft and ZooKeeper modes. To enable this behavior, set the property to `true` in `server.properties`. For backward compatibility, the default value is `false`. See [KIP-1252](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=399279475) for more details.
39+
40+
diff --git a/docs/operations/kraft.md b/docs/operations/kraft.md
41+
index a63ce05d4e..a8caef2d2b 100644
42+
--- a/docs/operations/kraft.md
43+
+++ b/docs/operations/kraft.md
44+
@@ -275,7 +275,7 @@ In general, the migration process passes through several phases.
45+
* After the migration has been finalized, it is not possible to revert back to ZooKeeper mode.
46+
* During the migration, if a ZK broker is running with multiple log directories, any directory failure will cause the broker to shutdown. Brokers with broken log directories will only be able to migrate to KRaft once the directories are repaired. For further details refer to [KAFKA-16431](https://issues.apache.org/jira/browse/KAFKA-16431).
47+
* As noted above, some features are not fully implemented in KRaft mode. If you are using one of those features, you will not be able to migrate to KRaft yet.
48+
- * There is a known inconsistency between ZK and KRaft modes in the arguments passed to an `AlterConfigPolicy`, when operations of type `SUBTRACT`, `DELETE` or `APPEND` are processed. This has been addressed with a compatibility flag in version 3.9.2. For further details see [KIP-1252](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=399279475).
49+
+ * There is a known inconsistency between ZK and KRaft modes in the arguments passed to an `AlterConfigPolicy`, when operations of type `SUBTRACT`, `DELETE` or `APPEND` are processed. This has been addressed with a compatibility flag in version 3.9.2-stackable0.0.0-dev. For further details see [KIP-1252](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=399279475).
50+
51+
52+
53+
diff --git a/gradle.properties b/gradle.properties
54+
index 3c674c89c1..24bf3d2259 100644
55+
--- a/gradle.properties
56+
+++ b/gradle.properties
57+
@@ -22,7 +22,7 @@ group=org.apache.kafka
58+
# - streams/quickstart/pom.xml
59+
# - streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
60+
# - streams/quickstart/java/pom.xml
61+
-version=3.9.2
62+
+version=3.9.2-stackable0.0.0-dev
63+
scalaVersion=2.13.15
64+
# Adding swaggerVersion in gradle.properties to have a single version in place for swagger
65+
# New version of Swagger 2.2.14 requires minimum JDK 11.
66+
diff --git a/kafka-merge-pr.py b/kafka-merge-pr.py
67+
index 4ac7434065..4f5f69d872 100755
68+
--- a/kafka-merge-pr.py
69+
+++ b/kafka-merge-pr.py
70+
@@ -70,7 +70,7 @@ TEMP_BRANCH_PREFIX = "PR_TOOL"
71+
72+
DEV_BRANCH_NAME = "trunk"
73+
74+
-DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.9.2")
75+
+DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.9.2-stackable0.0.0-dev")
76+
77+
ORIGINAL_HEAD = ""
78+
79+
diff --git a/streams/quickstart/java/pom.xml b/streams/quickstart/java/pom.xml
80+
index f8d1bd9e54..2ee0861269 100644
81+
--- a/streams/quickstart/java/pom.xml
82+
+++ b/streams/quickstart/java/pom.xml
83+
@@ -26,7 +26,7 @@
84+
<parent>
85+
<groupId>org.apache.kafka</groupId>
86+
<artifactId>streams-quickstart</artifactId>
87+
- <version>3.9.2</version>
88+
+ <version>3.9.2-stackable0.0.0-dev</version>
89+
<relativePath>..</relativePath>
90+
</parent>
91+
92+
diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
93+
index b62ab64acf..3d3d881096 100644
94+
--- a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
95+
+++ b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
96+
@@ -29,7 +29,7 @@
97+
98+
<properties>
99+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
100+
- <kafka.version>3.9.2</kafka.version>
101+
+ <kafka.version>3.9.2-stackable0.0.0-dev</kafka.version>
102+
<slf4j.version>1.7.36</slf4j.version>
103+
</properties>
104+
105+
diff --git a/streams/quickstart/pom.xml b/streams/quickstart/pom.xml
106+
index ac81ea06bb..1686b2bd56 100644
107+
--- a/streams/quickstart/pom.xml
108+
+++ b/streams/quickstart/pom.xml
109+
@@ -22,7 +22,7 @@
110+
<groupId>org.apache.kafka</groupId>
111+
<artifactId>streams-quickstart</artifactId>
112+
<packaging>pom</packaging>
113+
- <version>3.9.2</version>
114+
+ <version>3.9.2-stackable0.0.0-dev</version>
115+
116+
<name>Kafka Streams :: Quickstart</name>
117+
118+
diff --git a/tests/kafkatest/__init__.py b/tests/kafkatest/__init__.py
119+
index 1362dad9ca..e20c655bbf 100644
120+
--- a/tests/kafkatest/__init__.py
121+
+++ b/tests/kafkatest/__init__.py
122+
@@ -22,4 +22,4 @@
123+
# Instead, in development branches, the version should have a suffix of the form ".devN"
124+
#
125+
# For example, when Kafka is at version 1.0.0-SNAPSHOT, this should be something like "1.0.0.dev0"
126+
-__version__ = '3.9.2'
127+
+__version__ = '3.9.2-stackable0.0.0-dev'
128+
diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py
129+
index fa301e6a8d..1980c35d11 100644
130+
--- a/tests/kafkatest/version.py
131+
+++ b/tests/kafkatest/version.py
132+
@@ -122,7 +122,7 @@ def get_version(node=None):
133+
return DEV_BRANCH
134+
135+
DEV_BRANCH = KafkaVersion("dev")
136+
-DEV_VERSION = KafkaVersion("3.9.2-SNAPSHOT")
137+
+DEV_VERSION = KafkaVersion("3.9.2-stackable0.0.0-dev-SNAPSHOT")
138+
139+
# This should match the LATEST_PRODUCTION version defined in MetadataVersion.java
140+
LATEST_STABLE_METADATA_VERSION = "3.9-IV0"
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
base = "5e9866f43ab8e7e41ef39e5584ac50019381328d"
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
From 5ef351ca945bcaa8346ef066ba00fd9288a4f443 Mon Sep 17 00:00:00 2001
2+
From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
3+
Date: Wed, 13 May 2026 16:30:33 +0300
4+
Subject: Add CycloneDX plugin
5+
6+
---
7+
build.gradle | 45 +++++++++++++++++++++++++++++++++++++++++++++
8+
1 file changed, 45 insertions(+)
9+
10+
diff --git a/build.gradle b/build.gradle
11+
index c07cd3b047..effbe0f65d 100644
12+
--- a/build.gradle
13+
+++ b/build.gradle
14+
@@ -41,8 +41,53 @@ plugins {
15+
id 'org.scoverage' version '8.1' apply false
16+
id 'com.gradleup.shadow' version '8.3.9' apply false
17+
id 'com.diffplug.spotless' version "8.0.0"
18+
+ id 'org.cyclonedx.bom' version '3.2.4'
19+
}
20+
21+
+cyclonedxBom {
22+
+ // Specified the type of project being built. Defaults to 'library'
23+
+ projectType = org.cyclonedx.model.Component.Type.APPLICATION
24+
+ // Specified the version of the CycloneDX specification to use. Defaults to VERSION_16.
25+
+ schemaVersion = org.cyclonedx.Version.VERSION_15
26+
+ jsonOutput = file("build/reports/bom.json")
27+
+ xmlOutput.unsetConvention()
28+
+}
29+
+
30+
+// Exclude test components. This list needs to be checked and, if it changed, updated for every new Kafka version.
31+
+// The list can be obtained by running `gradle projects | grep upgrade-system-tests`
32+
+def skipCyclonedxProjects = [
33+
+ ':streams:upgrade-system-tests-0110',
34+
+ ':streams:upgrade-system-tests-10',
35+
+ ':streams:upgrade-system-tests-11',
36+
+ ':streams:upgrade-system-tests-20',
37+
+ ':streams:upgrade-system-tests-21',
38+
+ ':streams:upgrade-system-tests-22',
39+
+ ':streams:upgrade-system-tests-23',
40+
+ ':streams:upgrade-system-tests-24',
41+
+ ':streams:upgrade-system-tests-25',
42+
+ ':streams:upgrade-system-tests-26',
43+
+ ':streams:upgrade-system-tests-27',
44+
+ ':streams:upgrade-system-tests-28',
45+
+ ':streams:upgrade-system-tests-30',
46+
+ ':streams:upgrade-system-tests-31',
47+
+ ':streams:upgrade-system-tests-32',
48+
+ ':streams:upgrade-system-tests-33',
49+
+ ':streams:upgrade-system-tests-34',
50+
+ ':streams:upgrade-system-tests-35',
51+
+ ':streams:upgrade-system-tests-36',
52+
+ ':streams:upgrade-system-tests-37',
53+
+ ':streams:upgrade-system-tests-38',
54+
+ ':streams:upgrade-system-tests-39',
55+
+ ':streams:upgrade-system-tests-40',
56+
+ ':streams:upgrade-system-tests-41',
57+
+]
58+
+
59+
+configure(subprojects.findAll { skipCyclonedxProjects.contains(it.path) }) {
60+
+ tasks.named('cyclonedxDirectBom') {
61+
+ enabled = false
62+
+ }
63+
+}
64+
+
65+
ext {
66+
minClientJavaVersion = 11
67+
minNonClientJavaVersion = 17
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
From e4e184668e99d71e2cbdebfd55ef4ba3322a6508 Mon Sep 17 00:00:00 2001
2+
From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
3+
Date: Wed, 13 May 2026 16:31:44 +0300
4+
Subject: Change Gradle to use the Nexus Build Repo
5+
6+
---
7+
build.gradle | 8 ++++++--
8+
1 file changed, 6 insertions(+), 2 deletions(-)
9+
10+
diff --git a/build.gradle b/build.gradle
11+
index effbe0f65d..bd840210bd 100644
12+
--- a/build.gradle
13+
+++ b/build.gradle
14+
@@ -18,7 +18,9 @@ import java.nio.charset.StandardCharsets
15+
16+
buildscript {
17+
repositories {
18+
- mavenCentral()
19+
+ maven {
20+
+ url 'https://build-repo.stackable.tech/repository/maven-public/'
21+
+ }
22+
}
23+
apply from: "$rootDir/gradle/dependencies.gradle"
24+
25+
@@ -212,7 +214,9 @@ ext {
26+
allprojects {
27+
28+
repositories {
29+
- mavenCentral()
30+
+ maven {
31+
+ url 'https://build-repo.stackable.tech/repository/maven-public/'
32+
+ }
33+
}
34+
35+
dependencyUpdates {

0 commit comments

Comments
 (0)