feat: custom version for Trino (#1168)

* feat: custom version for Trino

* chore: changelog

* fix: directory names / storage plugin location

Author: dervoeti

CHANGELOG.md

@@ -82,6 +82,7 @@ All notable changes to this project will be documented in this file.
 - vector: Bump to `0.47.0` ([#1152]).
 - zookeeper: backport ZOOKEEPER-4846, ZOOKEEPER-4921, ZOOKEEPER-4925 into Zookeeper 3.9.3 ([#1150]).
 - testing-tools: Update base image ([#1165]).
+- trino: Enable custom versions ([#1168]).
 - opa: Enable custom versions ([#1170]).
 
 ### Fixed
@@ -187,6 +188,7 @@ All notable changes to this project will be documented in this file.
 [#1156]: https://github.com/stackabletech/docker-images/pull/1156
 [#1163]: https://github.com/stackabletech/docker-images/pull/1163
 [#1165]: https://github.com/stackabletech/docker-images/pull/1165
+[#1168]: https://github.com/stackabletech/docker-images/pull/1168
 [#1170]: https://github.com/stackabletech/docker-images/pull/1170
 
 ## [25.3.0] - 2025-03-21

conf.py

@@ -33,6 +33,7 @@
 superset = importlib.import_module("superset.versions")
 trino_cli = importlib.import_module("trino-cli.versions")
 trino = importlib.import_module("trino.versions")
+trino_jars = importlib.import_module("trino.trino.versions")
 trino_storage_connector = importlib.import_module("trino.storage-connector.versions")
 kafka_testing_tools = importlib.import_module("kafka-testing-tools.versions")
 kcat = importlib.import_module("kafka.kcat.versions")
@@ -66,6 +67,7 @@
     {"name": "superset", "versions": superset.versions},
     {"name": "trino-cli", "versions": trino_cli.versions},
     {"name": "trino", "versions": trino.versions},
+    {"name": "trino/trino", "versions": trino_jars.versions},
     {"name": "trino/storage-connector", "versions": trino_storage_connector.versions},
     {"name": "kafka-testing-tools", "versions": kafka_testing_tools.versions},
     {"name": "kafka/kcat", "versions": kcat.versions},

trino/Dockerfile

@@ -3,89 +3,14 @@
 
 FROM stackable/image/trino/storage-connector AS trino-storage-connector-image
 
-FROM stackable/image/java-devel AS trino-builder
-
-ARG PRODUCT
-ARG STACKABLE_USER_UID
-ARG JMX_EXPORTER
-
-WORKDIR /stackable
-
-COPY --chown=${STACKABLE_USER_UID}:0 trino/stackable/patches/patchable.toml /stackable/src/trino/stackable/patches/patchable.toml
-COPY --chown=${STACKABLE_USER_UID}:0 trino/stackable/patches/${PRODUCT} /stackable/src/trino/stackable/patches/${PRODUCT}
-
-COPY --from=trino-storage-connector-image /stackable/src/trino-storage-connector/patchable-work/worktree/${PRODUCT}/target/trino-storage-${PRODUCT} /stackable/trino-server-${PRODUCT}/plugin/trino-storage-${PRODUCT}
-COPY --from=trino-storage-connector-image /stackable/src/trino-storage-connector/patchable-work/worktree/${PRODUCT}/target/bom.json /stackable/trino-server-${PRODUCT}/plugin/trino-storage-${PRODUCT}/trino-storage-${PRODUCT}.cdx.json
-COPY --from=trino-storage-connector-image /stackable/trino-storage-connector-${PRODUCT}-src.tar.gz /stackable
-COPY --chown=${STACKABLE_USER_UID}:0 trino/stackable/jmx /stackable/jmx
-
-# adding a hadolint ignore for SC2215, due to https://github.com/hadolint/hadolint/issues/980
-# hadolint ignore=SC2215
-RUN --mount=type=cache,id=maven-${PRODUCT},target=/root/.m2/repository <<EOF
-cd "$(/stackable/patchable --images-repo-root=src checkout trino ${PRODUCT})"
-
-# Create snapshot of the source code including custom patches
-tar -czf /stackable/trino-${PRODUCT}-src.tar.gz .
-
-# Trino is using something (git-commit-id-plugin in the past, maybe something else now) that is
-# reading the Git history and searches for a tag to pull the version from. It sounds weird to me
-# why someone would do that over just picking the version from the pom.xml, but they propably
-# have their reasons. See e.g. https://github.com/trinodb/trino/discussions/18963.
-# So we fake it till we make it and create a Git repo and the correct tag. The trino-operator
-# smoke test checks that "select version()" is working.
-# Also, we need to initialize a new Git repo because `git-commit-id-maven-plugin` has a problem with worktrees, see https://github.com/git-commit-id/git-commit-id-maven-plugin/issues/215
-rm .git
-git init
-git config user.email "[email protected]"
-git config user.name "Fake commiter"
-git commit --allow-empty --message "Fake commit, so that we can create a tag"
-git tag ${PRODUCT}
-
-SKIP_PROJECTS="!docs"
-if [ "$PRODUCT" = "470" ] || [ "$PRODUCT" = "451" ]; then
-  SKIP_PROJECTS="$SKIP_PROJECTS,!core/trino-server-rpm"
-fi
-
-# We need to use ./mvnw instead of mvn to get a recent maven version (which is required to build Trino)
-./mvnw \
-  --batch-mode \
-  --no-transfer-progress \
-  package \
-  `# -Dmaven.test.skip # Unable to skip test compilation without an unused dependency error for software.amazon.awssdk:identity-spi` \
-  -DskipTests `# Skip test execution` \
-  -Dcheckstyle.skip `# Skip checkstyle checks. We dont care if the code is properly formatted, it just wastes time` \
-  -Dmaven.javadoc.skip=true `# Dont generate javadoc` \
-  --projects="$SKIP_PROJECTS"
-
-# Delete intermediate build products to free some space and keep runners happy
-rm -r plugin/*/target core/trino-server/target/trino-server-${PRODUCT}
-
-# Extract from tarball to save space; the tarball deduplicates jars (replacing them with symlinks),
-# while the raw output folder does not
-tar -xzf core/trino-server/target/trino-server-${PRODUCT}.tar.gz -C /stackable
-mv core/trino-server/target/bom.json /stackable/trino-server-${PRODUCT}/trino-server-${PRODUCT}.cdx.json
-
-# Delete intermediate build products to free some space and keep runners happy
-(cd .. && rm -r ${PRODUCT})
-rm -r /stackable/.m2
-
-# JMX Exporter
-curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar
-chmod +x /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar
-
-# Softlinks
-ln -s /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar /stackable/jmx/jmx_prometheus_javaagent.jar
-ln -s /stackable/trino-server-${PRODUCT} /stackable/trino-server
-
-# Set correct permissions
-chmod --recursive g=u /stackable
-EOF
+FROM stackable/image/trino/trino AS trino-builder
 
 FROM stackable/image/java-base
 
 ARG PRODUCT
 ARG RELEASE
 ARG STACKABLE_USER_UID
+ARG JMX_EXPORTER
 
 LABEL \
     name="Trino" \
@@ -110,9 +35,27 @@ microdnf clean all
 rm -rf /var/cache/yum
 EOF
 
-COPY --from=trino-builder --chown=${STACKABLE_USER_UID}:0 /stackable /stackable
+COPY --from=trino-builder --chown=${STACKABLE_USER_UID}:0 /stackable/trino-server /stackable/trino-server-${PRODUCT}-stackable${RELEASE}
 COPY --chown=${STACKABLE_USER_UID}:0 trino/licenses /licenses
 
+COPY --from=trino-storage-connector-image --chown=${STACKABLE_USER_UID}:0 /stackable/src/trino-storage-connector/patchable-work/worktree/${PRODUCT}/target/trino-storage-${PRODUCT}-stackable${RELEASE} /stackable/trino-server-${PRODUCT}-stackable${RELEASE}/plugin/trino-storage-${PRODUCT}-stackable${RELEASE}
+COPY --from=trino-storage-connector-image --chown=${STACKABLE_USER_UID}:0 /stackable/src/trino-storage-connector/patchable-work/worktree/${PRODUCT}/target/bom.json /stackable/trino-server-${PRODUCT}-stackable${RELEASE}/plugin/trino-storage-${PRODUCT}-stackable${RELEASE}/trino-storage-${PRODUCT}-stackable${RELEASE}.cdx.json
+COPY --from=trino-storage-connector-image --chown=${STACKABLE_USER_UID}:0 /stackable/trino-storage-connector-${PRODUCT}-stackable${RELEASE}-src.tar.gz /stackable
+
+COPY --chown=${STACKABLE_USER_UID}:0 trino/stackable/jmx /stackable/jmx
+
+RUN <<EOF
+# JMX Exporter
+curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar
+chmod +x /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar
+ln -s /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar /stackable/jmx/jmx_prometheus_javaagent.jar
+ln -s /stackable/trino-server-${PRODUCT}-stackable${RELEASE} /stackable/trino-server
+
+# Set correct permissions
+chown -R ${STACKABLE_USER_UID}:0 /stackable/jmx /stackable/trino-server
+chmod --recursive g=u /stackable
+EOF
+
 # ----------------------------------------
 # Checks
 # This section is to run final checks to ensure the created final images

trino/storage-connector/Dockerfile

@@ -1,35 +1,49 @@
 # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
 # check=error=true
 
+FROM stackable/image/trino/trino AS trino-builder
+
 FROM stackable/image/java-devel AS storage-connector-builder
 
 ARG PRODUCT
+ARG RELEASE
 ARG STACKABLE_USER_UID
 
 WORKDIR /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 trino/storage-connector/stackable/patches/patchable.toml /stackable/src/trino-storage-connector/stackable/patches/patchable.toml
 COPY --chown=${STACKABLE_USER_UID}:0 trino/storage-connector/stackable/patches/${PRODUCT} /stackable/src/trino-storage-connector/stackable/patches/${PRODUCT}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=trino-builder /stackable/patched-libs /stackable/patched-libs
+USER ${STACKABLE_USER_UID}
 
 # adding a hadolint ignore for SC2215, due to https://github.com/hadolint/hadolint/issues/980
 # hadolint ignore=SC2215
-RUN --mount=type=cache,id=maven-${PRODUCT},target=/root/.m2/repository <<EOF
+RUN --mount=type=cache,id=maven-${PRODUCT},target=/stackable/.m2/repository,uid=${STACKABLE_USER_UID} <<EOF
 cd "$(/stackable/patchable --images-repo-root=src checkout trino-storage-connector ${PRODUCT})"
 
+# Make Maven aware of custom Stackable libraries
+cp -r /stackable/patched-libs/maven/* /stackable/.m2/repository
+
+ORIGINAL_VERSION="${PRODUCT}"
+NEW_VERSION="${PRODUCT}-stackable${RELEASE}"
+
 # Create snapshot of the source code including custom patches
-tar -czf /stackable/trino-storage-connector-${PRODUCT}-src.tar.gz .
+tar -czf /stackable/trino-storage-connector-${NEW_VERSION}-src.tar.gz .
 
-# Upstream builds are marked as -SNAPSHOT, even for release builds
-mvn versions:set -DnewVersion=${PRODUCT}
+mvn versions:set -DnewVersion=${NEW_VERSION}
 
 # We need to use ./mvnw instead of mvn to get a recent maven version (which is required to build Trino)
 ./mvnw \
   --batch-mode \
   --no-transfer-progress \
   package \
-  -Dmaven.test.skip `# Skip test compilation` \
   -DskipTests `# Skip test execution` \
+  -Ddep.trino.version=${NEW_VERSION} `# Use custom Stackable Trino version in tests` \
   -Dcheckstyle.skip `# Skip checkstyle checks. We dont care if the code is properly formatted, it just wastes time` \
   -Dmaven.javadoc.skip=true `# Dont generate javadoc` \
   -Dmaven.gitcommitid.skip=true `# The gitcommitid plugin cannot work with git workspaces (ie: patchable)`
+sed -i "s/${NEW_VERSION}/${ORIGINAL_VERSION}/g" target/bom.json
+
+# Set correct permissions
+chmod --recursive g=u target /stackable/trino-storage-connector-${NEW_VERSION}-src.tar.gz
 EOF

trino/storage-connector/versions.py

@@ -1,14 +1,17 @@
 versions = [
     {
         "product": "451",
+        "trino/trino": "451",
         "java-devel": "22",
     },
     {
         "product": "470",
+        "trino/trino": "470",
         "java-devel": "23",
     },
     {
         "product": "476",
+        "trino/trino": "476",
         "java-devel": "24",
     },
 ]

trino/trino/Dockerfile

@@ -0,0 +1,76 @@
+# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
+# check=error=true
+FROM stackable/image/java-devel AS trino-builder
+
+ARG PRODUCT
+ARG RELEASE
+ARG STACKABLE_USER_UID
+
+WORKDIR /stackable
+
+COPY --chown=${STACKABLE_USER_UID}:0 trino/trino/stackable/patches/patchable.toml /stackable/src/trino/trino/stackable/patches/patchable.toml
+COPY --chown=${STACKABLE_USER_UID}:0 trino/trino/stackable/patches/${PRODUCT} /stackable/src/trino/trino/stackable/patches/${PRODUCT}
+
+# adding a hadolint ignore for SC2215, due to https://github.com/hadolint/hadolint/issues/980
+# hadolint ignore=SC2215
+RUN --mount=type=cache,id=maven-${PRODUCT},target=/root/.m2/repository <<EOF
+cd "$(/stackable/patchable --images-repo-root=src checkout trino/trino ${PRODUCT})"
+
+NEW_VERSION="${PRODUCT}-stackable${RELEASE}"
+
+mvn versions:set -DnewVersion=$NEW_VERSION
+
+# Create snapshot of the source code including custom patches
+tar -czf /stackable/trino-${NEW_VERSION}-src.tar.gz .
+
+# Trino is using something (git-commit-id-plugin in the past, maybe something else now) that is
+# reading the Git history and searches for a tag to pull the version from. It sounds weird to me
+# why someone would do that over just picking the version from the pom.xml, but they propably
+# have their reasons. See e.g. https://github.com/trinodb/trino/discussions/18963.
+# So we fake it till we make it and create a Git repo and the correct tag. The trino-operator
+# smoke test checks that "select version()" is working.
+# Also, we need to initialize a new Git repo because `git-commit-id-maven-plugin` has a problem with worktrees, see https://github.com/git-commit-id/git-commit-id-maven-plugin/issues/215
+rm .git
+git init
+git config user.email "[email protected]"
+git config user.name "Fake commiter"
+git commit --allow-empty --message "Fake commit, so that we can create a tag"
+git tag ${NEW_VERSION}
+
+SKIP_PROJECTS="!docs"
+if [ "$PRODUCT" = "470" ] || [ "$PRODUCT" = "451" ]; then
+  SKIP_PROJECTS="$SKIP_PROJECTS,!core/trino-server-rpm"
+fi
+
+# We need to use ./mvnw instead of mvn to get a recent maven version (which is required to build Trino)
+./mvnw \
+  --batch-mode \
+  --no-transfer-progress \
+  install \
+  `# -Dmaven.test.skip # Unable to skip test compilation without an unused dependency error for software.amazon.awssdk:identity-spi` \
+  -DskipTests `# Skip test execution` \
+  -Dcheckstyle.skip `# Skip checkstyle checks. We dont care if the code is properly formatted, it just wastes time` \
+  -Dmaven.javadoc.skip=true `# Dont generate javadoc` \
+  -Ddep.presto-jdbc-under-test=${NEW_VERSION} \
+  --projects="$SKIP_PROJECTS"
+
+mkdir -p /stackable/patched-libs/maven/io
+cp -r /root/.m2/repository/io/trino /stackable/patched-libs/maven/io
+
+# Delete intermediate build products to free some space and keep runners happy
+rm -r plugin/*/target core/trino-server/target/trino-server-${NEW_VERSION}
+
+# Extract from tarball to save space; the tarball deduplicates jars (replacing them with symlinks),
+# while the raw output folder does not
+tar -xzf core/trino-server/target/trino-server-${NEW_VERSION}.tar.gz -C /stackable
+mv core/trino-server/target/bom.json /stackable/trino-server-${NEW_VERSION}/trino-server-${NEW_VERSION}.cdx.json
+
+# Delete intermediate build products to free some space and keep runners happy
+(cd .. && rm -r ${PRODUCT})
+rm -r /stackable/.m2
+
+ln -s /stackable/trino-server-${NEW_VERSION} /stackable/trino-server
+
+# Set correct permissions
+chmod --recursive g=u /stackable
+EOF

trino/stackable/patches/451/0001-Add-CycloneDX-plugin.patch renamed to trino/trino/stackable/patches/451/0001-Add-CycloneDX-plugin.patch

@@ -0,0 +1,14 @@
+versions = [
+    {
+        "product": "451",
+        "java-devel": "22",
+    },
+    {
+        "product": "470",
+        "java-devel": "23",
+    },
+    {
+        "product": "476",
+        "java-devel": "24",
+    },
+]

trino/stackable/patches/451/patchable.toml renamed to trino/trino/stackable/patches/451/patchable.toml

@@ -2,21 +2,21 @@
     {
         "product": "451",
         "java-base": "22",
-        "java-devel": "22",
+        "trino/trino": "451",
         "jmx_exporter": "1.3.0",
         "trino/storage-connector": "451",
     },
     {
         "product": "470",
         "java-base": "23",
-        "java-devel": "23",
+        "trino/trino": "470",
         "jmx_exporter": "1.3.0",
         "trino/storage-connector": "470",
     },
     {
         "product": "476",
         "java-base": "24",
-        "java-devel": "24",
+        "trino/trino": "476",
         "jmx_exporter": "1.3.0",
         "trino/storage-connector": "476",
     },