diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e295030e..d4cfc6400 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ All notable changes to this project will be documented in this file. - omid: bump version to 1.1.3 ([#1105]) - hbase: add 2.6.2 and upgrade dependencies ([#1101]) - kafka: Add `4.0.0` ([#1117]) +- Include `.tar.gz` snapshots of the product source code in container images ([#1126]) ### Changed @@ -145,6 +146,7 @@ All notable changes to this project will be documented in this file. [#1122]: https://github.com/stackabletech/docker-images/pull/1122 [#1124]: https://github.com/stackabletech/docker-images/pull/1124 [#1125]: https://github.com/stackabletech/docker-images/pull/1125 +[#1126]: https://github.com/stackabletech/docker-images/pull/1126 [#1128]: https://github.com/stackabletech/docker-images/pull/1128 ## [25.3.0] - 2025-03-21 diff --git a/druid/Dockerfile b/druid/Dockerfile index e4be588f1..6605210ed 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -54,6 +54,9 @@ RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/sta cd "$(cat /tmp/DRUID_SOURCE_DIR)" || exit 1 rm /tmp/DRUID_SOURCE_DIR +# Create snapshot of the source code including custom patches +tar -czf /stackable/druid-${PRODUCT}-src.tar.gz . + mvn \ --batch-mode \ --no-transfer-progress \ @@ -120,6 +123,7 @@ LABEL io.k8s.display-name="${NAME}" COPY --chown=${STACKABLE_USER_UID}:0 --from=druid-builder /stackable/apache-druid-${PRODUCT} /stackable/apache-druid-${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 --from=druid-builder /stackable/druid-${PRODUCT}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/bin /stackable/bin COPY --chown=${STACKABLE_USER_UID}:0 druid/licenses /licenses diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index b3c6eb99a..c8d0f6877 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -11,32 +11,40 @@ ARG TARGETARCH ARG TARGETOS ARG STACKABLE_USER_UID +WORKDIR /stackable + +COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/patchable.toml /stackable/src/shared/protobuf/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBUF} /stackable/src/shared/protobuf/stackable/patches/${PROTOBUF} + +RUN <<EOF +rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm +microdnf update +# boost is a build dependency starting in Hadoop 3.4.0 if compiling native code +# automake and libtool are required to build protobuf +microdnf install boost1.78-devel automake libtool +microdnf clean all +rm -rf /var/cache/yum +EOF + # This Protobuf version is the exact version as used in the Hadoop Dockerfile # See https://github.com/apache/hadoop/blob/trunk/dev-support/docker/pkg-resolver/install-protobuf.sh # (this was hardcoded in the Dockerfile in earlier versions of Hadoop, make sure to look at the exact version in Github) -WORKDIR /opt/protobuf-src RUN <<EOF -curl https://repo.stackable.tech/repository/packages/protobuf/protobuf-java-${PROTOBUF}.tar.gz -o /opt/protobuf.tar.gz - tar xzf /opt/protobuf.tar.gz --strip-components 1 --no-same-owner + cd "$(/stackable/patchable --images-repo-root=src checkout shared/protobuf ${PROTOBUF})" + + # Create snapshot of the source code including custom patches + tar -czf /stackable/protobuf-${PROTOBUF}-src.tar.gz . + + ./autogen.sh ./configure --prefix=/opt/protobuf make "-j$(nproc)" make install - rm -rf /opt/protobuf-src + (cd .. && rm -r ${PROTOBUF}) EOF ENV PROTOBUF_HOME=/opt/protobuf ENV PATH="${PATH}:/opt/protobuf/bin" -RUN <<EOF -rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm -microdnf update -# boost is a build dependency starting in Hadoop 3.4.0 if compiling native code -microdnf install boost1.78-devel -microdnf clean all -rm -rf /var/cache/yum -EOF - -WORKDIR /stackable RUN <<EOF # async-profiler ARCH="${TARGETARCH/amd64/x64}" @@ -63,6 +71,9 @@ COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/jmx /stackable/jmx RUN <<EOF cd "$(/stackable/patchable --images-repo-root=src checkout hadoop ${PRODUCT})" +# Create snapshot of the source code including custom patches +tar -czf /stackable/hadoop-${PRODUCT}-src.tar.gz . + mvn \ --batch-mode \ --no-transfer-progress \ @@ -125,14 +136,20 @@ EOF ENV JAVA_HOME="/usr/lib/jvm/temurin-17-jdk" USER ${STACKABLE_USER_UID} -WORKDIR /build +WORKDIR /stackable + +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/patchable.toml /stackable/src/hadoop/hdfs-utils/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS} /stackable/src/hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS} + # The Stackable HDFS utils contain an OPA authorizer, group mapper & topology provider. # The topology provider provides rack awareness functionality for HDFS by allowing users to specify Kubernetes # labels to build a rackID from. # Starting with hdfs-utils version 0.3.0 the topology provider is not a standalone jar anymore and included in hdfs-utils. RUN <<EOF -curl "https://github.com/stackabletech/hdfs-utils/archive/refs/tags/v${HDFS_UTILS}.tar.gz" | tar -xzC . -cd hdfs-utils-${HDFS_UTILS} +cd "$(/stackable/patchable --images-repo-root=src checkout hadoop/hdfs-utils ${HDFS_UTILS})" + +# Create snapshot of the source code including custom patches +tar -czf /stackable/hdfs-utils-${HDFS_UTILS}-src.tar.gz . mvn \ --batch-mode \ @@ -147,7 +164,7 @@ cp target/hdfs-utils-$HDFS_UTILS.jar /stackable/hdfs-utils-${HDFS_UTILS}.jar rm -rf hdfs-utils-main # Set correct groups -chmod g=u /stackable/hdfs-utils-${HDFS_UTILS}.jar +chmod g=u /stackable/hdfs-utils-${HDFS_UTILS}.jar /stackable/hdfs-utils-${HDFS_UTILS}-src.tar.gz EOF FROM stackable/image/java-base AS final @@ -168,6 +185,7 @@ LABEL \ COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hdfs-utils-${HDFS_UTILS}.jar /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar +COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hdfs-utils-${HDFS_UTILS}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 hadoop/licenses /licenses diff --git a/hadoop/hdfs-utils/stackable/patches/0.4.0/patchable.toml b/hadoop/hdfs-utils/stackable/patches/0.4.0/patchable.toml new file mode 100644 index 000000000..3e4d63e77 --- /dev/null +++ b/hadoop/hdfs-utils/stackable/patches/0.4.0/patchable.toml @@ -0,0 +1 @@ +base = "cf24929fad3da02e530151d5ac66802cd5fa4109" diff --git a/hadoop/hdfs-utils/stackable/patches/0.4.1/patchable.toml b/hadoop/hdfs-utils/stackable/patches/0.4.1/patchable.toml new file mode 100644 index 000000000..232b86be3 --- /dev/null +++ b/hadoop/hdfs-utils/stackable/patches/0.4.1/patchable.toml @@ -0,0 +1 @@ +base = "fb9bd978a3c855984e93f1a6407c57d62fa34733" diff --git a/hadoop/hdfs-utils/stackable/patches/patchable.toml b/hadoop/hdfs-utils/stackable/patches/patchable.toml new file mode 100644 index 000000000..6b46ecc8b --- /dev/null +++ b/hadoop/hdfs-utils/stackable/patches/patchable.toml @@ -0,0 +1 @@ +upstream = "https://github.com/stackabletech/hdfs-utils.git" diff --git a/hbase/Dockerfile b/hbase/Dockerfile index b699066e2..5ac6ddf22 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -49,6 +49,9 @@ RUN --mount=type=cache,id=maven-hbase-${PRODUCT},uid=${STACKABLE_USER_UID},targe ### cd "$(/stackable/patchable --images-repo-root=src checkout hbase ${PRODUCT})" +# Create snapshot of the source code including custom patches +tar -czf /stackable/hbase-${PRODUCT}-src.tar.gz . + # The release scripts of HBase also run the build twice (three times in fact, once again to build the site which we skip here). # I chose to replicate that exact behavior for consistency so please don't merge the two mvn runs into one unless you really know what you're doing! # Cannot skip building tests here because the assembly plugin needs a shell script from the test directory. @@ -146,6 +149,8 @@ ARG RELEASE ARG HADOOP ARG HBASE_PROFILE ARG HBASE_HBASE_OPERATOR_TOOLS +ARG HBASE_HBASE_OPA_AUTHORIZER +ARG HBASE_PHOENIX ARG STACKABLE_USER_UID ARG NAME="Apache HBase" @@ -171,13 +176,16 @@ LABEL io.k8s.description="${DESCRIPTION}" LABEL io.k8s.display-name="${NAME}" COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${PRODUCT} /stackable/hbase-${PRODUCT}/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${PRODUCT}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/async-profiler /stackable/async-profiler/ COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS} /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS}/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/bin/hbck2 /stackable/bin/hbck2 COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/bin/hbase-entrypoint.sh /stackable/hbase-${PRODUCT}/bin/hbase-entrypoint.sh COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix /stackable/phoenix/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix-${HBASE_PHOENIX}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/bin/export-snapshot-to-s3 /stackable/bin/export-snapshot-to-s3 COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/hadoop/share/hadoop/tools/lib/ /stackable/hadoop/share/hadoop/tools/lib/ @@ -190,6 +198,7 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder \ /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar \ /stackable/hbase-${PRODUCT}/lib/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer-${HBASE_HBASE_OPA_AUTHORIZER}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer/target/hbase-opa-authorizer*.jar /stackable/hbase-${PRODUCT}/lib RUN <<EOF @@ -226,6 +235,7 @@ chmod g=u "/stackable/hbase/lib/phoenix-server-hbase-${HBASE_PROFILE}.jar" chmod g=u /stackable/async-profiler chmod g=u /stackable/bin chmod g=u /stackable/phoenix +chmod g=u /stackable/*-src.tar.gz # the whole directory tree /stackable/hadoop/share/hadoop/tools/lib/ must be adapted find /stackable/hadoop -type d -exec chmod g=u {} + EOF diff --git a/hbase/hbase-opa-authorizer/Dockerfile b/hbase/hbase-opa-authorizer/Dockerfile index b2e5e723c..776f578c8 100644 --- a/hbase/hbase-opa-authorizer/Dockerfile +++ b/hbase/hbase-opa-authorizer/Dockerfile @@ -7,25 +7,34 @@ ARG STACKABLE_USER_UID USER ${STACKABLE_USER_UID} WORKDIR /stackable +COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-opa-authorizer/stackable/patches/patchable.toml /stackable/src/hbase/hbase-opa-authorizer/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-opa-authorizer/stackable/patches/${PRODUCT} /stackable/src/hbase/hbase-opa-authorizer/stackable/patches/${PRODUCT} + RUN --mount=type=cache,id=maven-opa,uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF ### ### OPA Authorizer (only for 2.6 upwards) ### if [[ -n "$PRODUCT" ]]; then - git clone --depth 1 --branch "$PRODUCT" https://github.com/stackabletech/hbase-opa-authorizer.git + cd "$(/stackable/patchable --images-repo-root=src checkout hbase/hbase-opa-authorizer ${PRODUCT})" + + # Create snapshot of the source code including custom patches + tar -czf /stackable/hbase-opa-authorizer-${PRODUCT}-src.tar.gz . mvn \ --batch-mode \ --no-transfer-progress \ -DskipTests \ -Dmaven.test.skip=true \ - -fhbase-opa-authorizer \ package else - # Create a dummy jar to avoid errors when copying it the final image - mkdir -p hbase-opa-authorizer/target - touch hbase-opa-authorizer/target/hbase-opa-authorizer.jar + # Create a dummy jar to avoid errors when copying it to the final image + mkdir -p target + touch target/hbase-opa-authorizer.jar + touch /stackable/hbase-opa-authorizer-${PRODUCT}-src.tar.gz fi +mkdir /stackable/hbase-opa-authorizer +mv target /stackable/hbase-opa-authorizer + if [ "${DELETE_CACHES}" = "true" ] ; then rm -rf /stackable/.m2/repository/* fi diff --git a/hbase/hbase-opa-authorizer/stackable/patches/0.1.0/patchable.toml b/hbase/hbase-opa-authorizer/stackable/patches/0.1.0/patchable.toml new file mode 100644 index 000000000..1eb0f3230 --- /dev/null +++ b/hbase/hbase-opa-authorizer/stackable/patches/0.1.0/patchable.toml @@ -0,0 +1 @@ +base = "c7eb27ca6a162bbfdb98262ba69e40d109f1fdd4" diff --git a/hbase/hbase-opa-authorizer/stackable/patches/patchable.toml b/hbase/hbase-opa-authorizer/stackable/patches/patchable.toml new file mode 100644 index 000000000..8a386e3bc --- /dev/null +++ b/hbase/hbase-opa-authorizer/stackable/patches/patchable.toml @@ -0,0 +1 @@ +upstream = "https://github.com/stackabletech/hbase-opa-authorizer.git" diff --git a/hbase/hbase-operator-tools/Dockerfile b/hbase/hbase-operator-tools/Dockerfile index 80676e0ea..d4806849f 100644 --- a/hbase/hbase-operator-tools/Dockerfile +++ b/hbase/hbase-operator-tools/Dockerfile @@ -14,8 +14,8 @@ ARG DELETE_CACHES="true" # so that they are not expanded. Disabling ShellCheck rules in a Dockerfile # does not work, so please ignore the according warning (SC2016). COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbck2.env /stackable/bin/ -COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/patchable.toml /stackable/src/hbase-operator-tools/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/${PRODUCT} /stackable/src/hbase-operator-tools/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/patchable.toml /stackable/src/hbase/hbase-operator-tools/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-operator-tools/stackable/patches/${PRODUCT} /stackable/src/hbase/hbase-operator-tools/stackable/patches/${PRODUCT} COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbase-entrypoint.sh /stackable/bin/ USER ${STACKABLE_USER_UID} @@ -24,7 +24,10 @@ WORKDIR /stackable # Cache mounts are owned by root by default # We need to explicitly give the uid to use RUN --mount=type=cache,id=maven-hbase-operator-tools-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF -cd "$(/stackable/patchable --images-repo-root=src checkout hbase-operator-tools ${PRODUCT})" +cd "$(/stackable/patchable --images-repo-root=src checkout hbase/hbase-operator-tools ${PRODUCT})" + +# Create snapshot of the source code including custom patches +tar -czf /stackable/hbase-operator-tools-$PRODUCT-src.tar.gz . mvn \ --batch-mode \ diff --git a/hbase/phoenix/Dockerfile b/hbase/phoenix/Dockerfile index 2d9273b3c..2cf9ca87b 100644 --- a/hbase/phoenix/Dockerfile +++ b/hbase/phoenix/Dockerfile @@ -19,6 +19,9 @@ WORKDIR /stackable RUN --mount=type=cache,id=maven-phoenix-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF cd "$(/stackable/patchable --images-repo-root=src checkout phoenix ${PRODUCT})" +# Create snapshot of the source code including custom patches +tar -czf /stackable/phoenix-$PRODUCT-src.tar.gz . + # The Maven command can be found inside of the scripts in the create-release folder (release-util.sh as of Phoenix 5.2.0) # https://github.com/apache/phoenix/tree/5.2.0/dev/create-release mvn \ diff --git a/hive/Dockerfile b/hive/Dockerfile index 2e3e49723..9c907d628 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -39,6 +39,9 @@ RUN --mount=type=cache,id=maven-hive-${PRODUCT},uid=${STACKABLE_USER_UID},target BUILD_SRC_DIR="$(/stackable/patchable --images-repo-root=src checkout hive ${PRODUCT})" cd "$BUILD_SRC_DIR" +# Create snapshot of the source code including custom patches +tar -czf /stackable/hive-${PRODUCT}-src.tar.gz . + if [[ "${PRODUCT}" == "3.1.3" ]] ; then mvn --batch-mode --no-transfer-progress clean package -DskipTests --projects standalone-metastore mv standalone-metastore/target/apache-hive-metastore-${PRODUCT}-bin/apache-hive-metastore-${PRODUCT}-bin /stackable @@ -127,7 +130,9 @@ LABEL io.k8s.display-name="${NAME}" WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/apache-hive-metastore-${PRODUCT}-bin +COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/hive-${PRODUCT}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/hadoop-${HADOOP} /stackable/hadoop-${HADOOP} +COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-${HADOOP}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/jmx /stackable/jmx COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/jmx /stackable/jmx COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/bin/start-metastore /stackable/apache-hive-metastore-${PRODUCT}-bin/bin @@ -150,6 +155,7 @@ chmod g=u /stackable/hive-metastore ln -s /stackable/hadoop-${HADOOP} /stackable/hadoop chown -h ${STACKABLE_USER_UID}:0 /stackable/hadoop chmod g=u /stackable/hadoop +chmod g=u /stackable/*-src.tar.gz # fix missing permissions chmod --recursive g=u /stackable/jmx diff --git a/kafka-testing-tools/Dockerfile b/kafka-testing-tools/Dockerfile index 2cc0636f4..d1fe4c321 100644 --- a/kafka-testing-tools/Dockerfile +++ b/kafka-testing-tools/Dockerfile @@ -30,7 +30,8 @@ RUN microdnf install \ && rm -rf /var/cache/yum # Store kcat version with binary name and add softlink -COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KAFKA_KCAT}/kcat /stackable/kcat-${KAFKA_KCAT} +COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat /stackable/kcat-${KAFKA_KCAT} +COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KAFKA_KCAT}-src.tar.gz /stackable RUN ln -s /stackable/kcat-${KAFKA_KCAT} /stackable/kcat COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses diff --git a/kafka/Dockerfile b/kafka/Dockerfile index 1d8f17b32..8a685147c 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -21,6 +21,9 @@ COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackab RUN <<EOF cd "$(/stackable/patchable --images-repo-root=src checkout kafka ${PRODUCT})" +# Create snapshot of the source code including custom patches +tar -czf /stackable/kafka-${PRODUCT}-src.tar.gz . + # TODO: Try to install gradle via package manager (if possible) instead of fetching it from the internet # We don't specify "-x test" to skip the tests, as we might bump some Kafka internal dependencies in the future and # it's a good idea to run the tests in this case. @@ -64,8 +67,10 @@ LABEL \ COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT} COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json /stackable/kafka_${SCALA}-${PRODUCT}/kafka_${SCALA}-${PRODUCT}.cdx.json +COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka-${PRODUCT}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/jmx/ /stackable/jmx/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KAFKA_KCAT}/kcat /stackable/bin/kcat-${KAFKA_KCAT} +COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat /stackable/bin/kcat-${KAFKA_KCAT} +COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KAFKA_KCAT}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses COPY --chown=${STACKABLE_USER_UID}:0 kafka/licenses /licenses @@ -96,6 +101,7 @@ chown -h ${STACKABLE_USER_UID}:0 /stackable/kafka chmod g=u /stackable/bin chmod g=u /stackable/jmx chmod g=u /stackable/kafka_${SCALA}-${PRODUCT} +chmod g=u /stackable/*-src.tar.gz EOF # ---------------------------------------- diff --git a/kafka/kcat/Dockerfile b/kafka/kcat/Dockerfile index e67437888..88812047a 100644 --- a/kafka/kcat/Dockerfile +++ b/kafka/kcat/Dockerfile @@ -4,7 +4,7 @@ # Normally we would use stackable/image/stackable-base here, *but* we fail to link kcat at the end with error messages # shown in Snippet 1, which we were not able to solve. -FROM stackable/image/java-base AS builder +FROM stackable/image/java-devel AS builder ARG PRODUCT ARG STACKABLE_USER_UID @@ -30,13 +30,20 @@ EOF WORKDIR /stackable +COPY --chown=${STACKABLE_USER_UID}:0 kafka/kcat/stackable/patches/patchable.toml /stackable/src/kafka/kcat/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 kafka/kcat/stackable/patches/${PRODUCT} /stackable/src/kafka/kcat/stackable/patches/${PRODUCT} + RUN <<EOF -curl -O https://repo.stackable.tech/repository/packages/kcat/kcat-${PRODUCT}.tar.gz -tar xvfz kcat-${PRODUCT}.tar.gz -cd kcat-${PRODUCT} +cd "$(/stackable/patchable --images-repo-root=src checkout kafka/kcat ${PRODUCT})" + +# Create snapshot of the source code including custom patches +tar -czf /stackable/kcat-${PRODUCT}-src.tar.gz . + ./bootstrap.sh +mv kcat /stackable/kcat # set correct permissions -chmod --recursive g=u /stackable/kcat-${PRODUCT} +chmod --recursive g=u /stackable/kcat +chmod g=u /stackable/kcat-${PRODUCT}-src.tar.gz EOF COPY --chown=${STACKABLE_USER_UID}:0 kafka/kcat/licenses /licenses diff --git a/kafka/kcat/stackable/patches/1.7.0/patchable.toml b/kafka/kcat/stackable/patches/1.7.0/patchable.toml new file mode 100644 index 000000000..f86cf1d25 --- /dev/null +++ b/kafka/kcat/stackable/patches/1.7.0/patchable.toml @@ -0,0 +1,2 @@ +mirror = "https://github.com/stackabletech/kcat.git" +base = "f2236ae5d985b9f31631b076df24ca6c33542e61" diff --git a/kafka/kcat/stackable/patches/patchable.toml b/kafka/kcat/stackable/patches/patchable.toml new file mode 100644 index 000000000..7f01bdb45 --- /dev/null +++ b/kafka/kcat/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/edenhill/kcat.git" +default-mirror = "https://github.com/stackabletech/kcat.git" diff --git a/kafka/kcat/versions.py b/kafka/kcat/versions.py index 82a6fba3b..80f708acd 100644 --- a/kafka/kcat/versions.py +++ b/kafka/kcat/versions.py @@ -1,7 +1,7 @@ versions = [ { "product": "1.7.0", - "java-base": "11", - "stackable-base": "1.0.0", + "java-devel": "11", + "stackable-devel": "1.0.0", } ] diff --git a/nifi/Dockerfile b/nifi/Dockerfile index 636d59ac1..0748e102e 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -45,6 +45,9 @@ curl 'https://repo.stackable.tech/repository/m2/tech/stackable/nifi/stackable-bc cd "$(/stackable/patchable --images-repo-root=src checkout nifi ${PRODUCT})" +# Create snapshot of the source code including custom patches +tar -czf /stackable/nifi-${PRODUCT}-src.tar.gz . + # NOTE: Since NiFi 2.0.0 PutIceberg Processor and services were removed, so including the `include-iceberg` profile does nothing. # Additionally some modules were moved to optional build profiles, so we need to add `include-hadoop` to get `nifi-parquet-nar` for example. if [[ "${PRODUCT}" != 1.* ]] ; then @@ -90,15 +93,20 @@ ARG PRODUCT ARG STACKABLE_USER_UID USER ${STACKABLE_USER_UID} -WORKDIR /build +WORKDIR /stackable + +COPY --chown=${STACKABLE_USER_UID}:0 nifi/nifi-iceberg-bundle/stackable/patches/patchable.toml /stackable/src/nifi/nifi-iceberg-bundle/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 nifi/nifi-iceberg-bundle/stackable/patches/${NIFI_ICEBERG_BUNDLE} /stackable/src/nifi/nifi-iceberg-bundle/stackable/patches/${NIFI_ICEBERG_BUNDLE} RUN <<EOF mkdir -p /stackable # NiFI 1.x natively supports Iceberg, no need to build an iceberg-bundle for it if [[ "${PRODUCT}" != 1.* ]] ; then - curl -L "https://github.com/stackabletech/nifi-iceberg-bundle/archive/refs/tags/${NIFI_ICEBERG_BUNDLE}.tar.gz" | tar -xzC . - cd nifi-iceberg-bundle-${NIFI_ICEBERG_BUNDLE} || exit + cd "$(/stackable/patchable --images-repo-root=src checkout nifi/nifi-iceberg-bundle ${NIFI_ICEBERG_BUNDLE})" + + # Create snapshot of the source code including custom patches + tar -czf /stackable/nifi-iceberg-bundle-${NIFI_ICEBERG_BUNDLE}-src.tar.gz . sed -i -e "s/{{ NIFI_VERSION }}/${PRODUCT}/g" pom.xml @@ -114,11 +122,10 @@ if [[ "${PRODUCT}" != 1.* ]] ; then cp ./nifi-iceberg-services-api-nar/target/nifi-iceberg-services-api-nar-${NIFI_ICEBERG_BUNDLE}.nar /stackable cp ./nifi-iceberg-services-nar/target/nifi-iceberg-services-nar-${NIFI_ICEBERG_BUNDLE}.nar /stackable cp ./nifi-iceberg-processors-nar/target/nifi-iceberg-processors-nar-${NIFI_ICEBERG_BUNDLE}.nar /stackable - cp ./target/bom.json /stackable/nifi-iceberg-bundle.cdx.json + cp ./target/bom.json /stackable/nifi-iceberg-bundle-${NIFI_ICEBERG_BUNDLE}.cdx.json - cd .. # Save disk space, even for intermediate images - rm -rf nifi-iceberg-bundle-${NIFI_ICEBERG_BUNDLE} + (cd .. && rm -r ${NIFI_ICEBERG_BUNDLE}) # Set correct groups chmod g=u /stackable/*.nar @@ -133,13 +140,18 @@ ARG STACKABLE_USER_UID ARG PRODUCT USER ${STACKABLE_USER_UID} -WORKDIR /build +WORKDIR /stackable + +COPY --chown=${STACKABLE_USER_UID}:0 nifi/opa-plugin/stackable/patches/patchable.toml /stackable/src/nifi/opa-plugin/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 nifi/opa-plugin/stackable/patches/${NIFI_OPA_AUTHORIZER_PLUGIN} /stackable/src/nifi/opa-plugin/stackable/patches/${NIFI_OPA_AUTHORIZER_PLUGIN} RUN <<EOF mkdir -p /stackable -curl -L "https://github.com/DavidGitter/nifi-opa-plugin/archive/refs/tags/v${NIFI_OPA_AUTHORIZER_PLUGIN}.tar.gz" | tar -xzC . -cd nifi-opa-plugin-${NIFI_OPA_AUTHORIZER_PLUGIN}/authorizer || exit +cd "$(/stackable/patchable --images-repo-root=src checkout nifi/opa-plugin ${NIFI_OPA_AUTHORIZER_PLUGIN})/authorizer" + +# Create snapshot of the source code including custom patches +tar -czf /stackable/nifi-opa-plugin-${NIFI_OPA_AUTHORIZER_PLUGIN}-src.tar.gz . mvn \ --batch-mode \ @@ -160,6 +172,7 @@ FROM stackable/image/java-base AS final ARG PRODUCT ARG RELEASE ARG STACKABLE_USER_UID +ARG NIFI_OPA_AUTHORIZER_PLUGIN LABEL name="Apache NiFi" \ maintainer="info@stackable.tech" \ @@ -170,12 +183,15 @@ LABEL name="Apache NiFi" \ description="This image is deployed by the Stackable Operator for Apache NiFi." COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/nifi-${PRODUCT} /stackable/nifi-${PRODUCT}/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/nifi-${PRODUCT}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/stackable-bcrypt.jar /stackable/stackable-bcrypt.jar COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-iceberg-bundle-builder /stackable/*.nar /stackable/nifi-${PRODUCT}/lib/ COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-iceberg-bundle-builder /stackable/*.cdx.json /stackable/nifi-${PRODUCT}/lib/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-iceberg-bundle-builder /stackable/*-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/git-sync /stackable/git-sync COPY --chown=${STACKABLE_USER_UID}:0 --from=opa-authorizer-builder /stackable/opa-authorizer.nar /stackable/nifi-${PRODUCT}/extensions/opa-authorizer.nar +COPY --chown=${STACKABLE_USER_UID}:0 --from=opa-authorizer-builder /stackable/nifi-opa-plugin-${NIFI_OPA_AUTHORIZER_PLUGIN}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=opa-authorizer-builder /stackable/LICENSE /licenses/NIFI_OPA_PLUGIN_LICENSE COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/bin /stackable/bin COPY --chown=${STACKABLE_USER_UID}:0 nifi/licenses /licenses @@ -210,6 +226,7 @@ chown --no-dereference ${STACKABLE_USER_UID}:0 /stackable/nifi chmod --recursive g=u /stackable/python chmod --recursive g=u /stackable/bin chmod g=u /stackable/nifi-${PRODUCT} +chmod g=u /stackable/*-src.tar.gz EOF # ---------------------------------------- diff --git a/nifi/nifi-iceberg-bundle/stackable/patches/0.0.4/patchable.toml b/nifi/nifi-iceberg-bundle/stackable/patches/0.0.4/patchable.toml new file mode 100644 index 000000000..abb686119 --- /dev/null +++ b/nifi/nifi-iceberg-bundle/stackable/patches/0.0.4/patchable.toml @@ -0,0 +1 @@ +base = "6515c2850bf03f110fc6798bfb1bfcd2a13d24ea" diff --git a/nifi/nifi-iceberg-bundle/stackable/patches/patchable.toml b/nifi/nifi-iceberg-bundle/stackable/patches/patchable.toml new file mode 100644 index 000000000..914580162 --- /dev/null +++ b/nifi/nifi-iceberg-bundle/stackable/patches/patchable.toml @@ -0,0 +1 @@ +upstream = "https://github.com/stackabletech/nifi-iceberg-bundle.git" diff --git a/nifi/opa-plugin/stackable/patches/0.1.0/patchable.toml b/nifi/opa-plugin/stackable/patches/0.1.0/patchable.toml new file mode 100644 index 000000000..dde67184a --- /dev/null +++ b/nifi/opa-plugin/stackable/patches/0.1.0/patchable.toml @@ -0,0 +1,2 @@ +mirror = "https://github.com/stackabletech/nifi-opa-plugin.git" +base = "42b7372e7c60fe2e68c73cf148e4b968f8595456" diff --git a/nifi/opa-plugin/stackable/patches/patchable.toml b/nifi/opa-plugin/stackable/patches/patchable.toml new file mode 100644 index 000000000..067a30a6a --- /dev/null +++ b/nifi/opa-plugin/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/DavidGitter/nifi-opa-plugin.git" +default-mirror = "https://github.com/stackabletech/nifi-opa-plugin.git" diff --git a/omid/Dockerfile b/omid/Dockerfile index f494cbf84..d65769632 100644 --- a/omid/Dockerfile +++ b/omid/Dockerfile @@ -27,6 +27,8 @@ COPY --chown=${STACKABLE_USER_UID}:0 omid/stackable/patches/${PRODUCT} /stackabl RUN --mount=type=cache,id=maven-omid-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF set -x cd "$(/stackable/patchable --images-repo-root=src checkout omid ${PRODUCT})" + # Create snapshot of the source code including custom patches + tar -czf /stackable/omid-${PRODUCT}-src.tar.gz . mvn --batch-mode --no-transfer-progress package -Phbase-2 -DskipTests tar -xf tso-server/target/omid-tso-server-${PRODUCT}-bin.tar.gz -C /stackable mv tso-server/target/bom.json /stackable/omid-tso-server-${PRODUCT}/omid-tso-server-${PRODUCT}.cdx.json @@ -76,6 +78,7 @@ COPY omid/licenses /licenses COPY --chown=${STACKABLE_USER_UID}:0 omid/stackable /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/omid-tso-server-${PRODUCT} /stackable/omid-tso-server-${PRODUCT} COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/omid-examples-${PRODUCT} /stackable/omid-examples-${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/omid-${PRODUCT}-src.tar.gz /stackable RUN <<EOF microdnf update diff --git a/opa/Dockerfile b/opa/Dockerfile index 5f21aebf8..214c57869 100644 --- a/opa/Dockerfile +++ b/opa/Dockerfile @@ -5,7 +5,7 @@ ARG GOLANG FROM oci.stackable.tech/sdp/library/golang:${GOLANG} AS golang-image -FROM stackable/image/stackable-base AS multilog-builder +FROM stackable/image/stackable-devel AS multilog-builder ARG DAEMONTOOLS_VERSION=0.76 @@ -38,7 +38,7 @@ package/install chmod g=u /daemontools/admin/daemontools/command/multilog EOF -FROM stackable/image/stackable-base AS opa-builder +FROM stackable/image/stackable-devel AS opa-builder ARG PRODUCT ARG RELEASE @@ -66,18 +66,22 @@ COPY --chown=${STACKABLE_USER_UID}:0 opa/stackable/bin /stackable/opa/bin COPY --from=golang-image /usr/local/go/ /usr/local/go/ ENV PATH="/usr/local/go/bin:${PATH}" +COPY --chown=${STACKABLE_USER_UID}:0 opa/stackable/patches/patchable.toml /stackable/src/opa/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 opa/stackable/patches/${PRODUCT} /stackable/src/opa/stackable/patches/${PRODUCT} + +WORKDIR /stackable + RUN <<EOF # We use version 1.7.0, since a newer version of cyclonedx-gomod is not compatible with the version of Golang (>= 1.23.1) go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@v1.7.0 -curl "https://repo.stackable.tech/repository/packages/opa/opa_${PRODUCT}.tar.gz" -o opa.tar.gz -tar -zxvf opa.tar.gz -mv "opa-${PRODUCT}" opa -EOF -WORKDIR /opa +cd "$(/stackable/patchable --images-repo-root=src checkout opa ${PRODUCT})" + +# Create snapshot of the source code including custom patches +tar -czf /stackable/opa-${PRODUCT}-src.tar.gz . -RUN <<EOF # Unfortunately, we need to create a dummy Git repository to allow cyclonedx-gomod to determine the version of OPA +rm .git git init git config user.email "fake.commiter@stackable.tech" git config user.name "Fake commiter" @@ -87,9 +91,9 @@ go build -o opa -buildmode=exe # move artifact to /stackable/*/ to copy in final image ~/go/bin/cyclonedx-gomod app -json -output-version 1.5 -output /stackable/opa/"opa_${PRODUCT}.cdx.json" -packages -files # move artifact to /stackable/* to copy in final image -mv /opa/opa /stackable/opa/ +mv opa /stackable/opa/ # set correct groups -chmod -R g=u /stackable/opa +chmod -R g=u /stackable/opa /stackable/opa-${PRODUCT}-src.tar.gz EOF FROM stackable/image/vector @@ -109,6 +113,7 @@ LABEL name="Open Policy Agent" \ COPY --chown=${STACKABLE_USER_UID}:0 opa/licenses /licenses COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /stackable/opa /stackable/opa +COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /stackable/opa-${PRODUCT}-src.tar.gz /stackable/opa-${PRODUCT}-src.tar.gz COPY --from=multilog-builder --chown=${STACKABLE_USER_UID}:0 /daemontools/admin/daemontools/command/multilog /stackable/multilog RUN <<EOF diff --git a/opa/stackable/patches/1.0.1/patchable.toml b/opa/stackable/patches/1.0.1/patchable.toml new file mode 100644 index 000000000..60690c2f2 --- /dev/null +++ b/opa/stackable/patches/1.0.1/patchable.toml @@ -0,0 +1,2 @@ +mirror = "https://github.com/stackabletech/opa.git" +base = "a5afc5ab5074ae6b17fd0b167ff3d5053303a58c" diff --git a/opa/stackable/patches/1.4.2/patchable.toml b/opa/stackable/patches/1.4.2/patchable.toml new file mode 100644 index 000000000..18fbc4ec4 --- /dev/null +++ b/opa/stackable/patches/1.4.2/patchable.toml @@ -0,0 +1,2 @@ +mirror = "https://github.com/stackabletech/opa.git" +base = "5e4582bb951f70641fe9ee85cc46245d079e5037" diff --git a/opa/stackable/patches/patchable.toml b/opa/stackable/patches/patchable.toml new file mode 100644 index 000000000..6b671e132 --- /dev/null +++ b/opa/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/open-policy-agent/opa.git" +default-mirror = "https://github.com/stackabletech/opa.git" diff --git a/opa/versions.py b/opa/versions.py index f0e7c3137..644a0d336 100644 --- a/opa/versions.py +++ b/opa/versions.py @@ -3,12 +3,12 @@ "product": "1.4.2", "vector": "0.46.1", "golang": "1.23.9", - "stackable-base": "1.0.0", + "stackable-devel": "1.0.0", }, { "product": "1.0.1", "vector": "0.46.1", "golang": "1.23.9", - "stackable-base": "1.0.0", + "stackable-devel": "1.0.0", }, ] diff --git a/shared/protobuf/stackable/patches/3.7.1/patchable.toml b/shared/protobuf/stackable/patches/3.7.1/patchable.toml new file mode 100644 index 000000000..edea16ab4 --- /dev/null +++ b/shared/protobuf/stackable/patches/3.7.1/patchable.toml @@ -0,0 +1 @@ +base = "6973c3a5041636c1d8dc5f7f6c8c1f3c15bc63d6" diff --git a/shared/protobuf/stackable/patches/patchable.toml b/shared/protobuf/stackable/patches/patchable.toml new file mode 100644 index 000000000..6b4fd8d14 --- /dev/null +++ b/shared/protobuf/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/protocolbuffers/protobuf.git" +default-mirror = "https://github.com/stackabletech/protobuf.git" diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile index 9d28cc398..45df3f7a9 100644 --- a/spark-k8s/Dockerfile +++ b/spark-k8s/Dockerfile @@ -18,7 +18,13 @@ WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/stackable/patches/patchable.toml /stackable/src/spark-k8s/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/stackable/patches/${PRODUCT} /stackable/src/spark-k8s/stackable/patches/${PRODUCT} -RUN /stackable/patchable --images-repo-root=src checkout spark-k8s ${PRODUCT} +RUN <<EOF +cd "$(/stackable/patchable --images-repo-root=src checkout spark-k8s ${PRODUCT})" + +# Create snapshot of the source code including custom patches +tar -czf /stackable/spark-${PRODUCT}-src.tar.gz . +chmod g=u /stackable/spark-${PRODUCT}-src.tar.gz +EOF # hbase-connectors-builder: Build the Spark HBase connector and copy # required JARs into /stackable/spark/jars @@ -49,6 +55,9 @@ COPY --chown=${STACKABLE_USER_UID}:0 spark-k8s/hbase-connectors/stackable/patche RUN <<EOF cd "$(/stackable/patchable --images-repo-root=src checkout spark-k8s/hbase-connectors ${HBASE_CONNECTOR})/spark" +# Create snapshot of the source code including custom patches +tar -czf /stackable/hbase-connector-${HBASE_CONNECTOR}-src.tar.gz . + # Building the hbase-connectors with JDK 17 is not yet supported, see # https://github.com/apache/hbase-connectors/pull/132. # As there are no JDK profiles, access to the non-public elements must @@ -105,6 +114,7 @@ mvn --quiet --non-recursive --no-transfer-progress --batch-mode --file /stackabl dependency:copy \ -Dartifact=org.apache.logging.log4j:log4j-slf4j-impl:'${log4j.version}' \ -DoutputDirectory=./jars +chmod g=u /stackable/hbase-connector-${HBASE_CONNECTOR}-src.tar.gz . EOF @@ -232,6 +242,7 @@ ARG PRODUCT ARG PYTHON ARG RELEASE ARG JMX_EXPORTER +ARG HBASE_CONNECTOR ARG STACKABLE_USER_UID LABEL name="Apache Spark" \ @@ -253,6 +264,8 @@ ENV PYTHONPATH=$SPARK_HOME/python COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/spark-${PRODUCT}/dist /stackable/spark +COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-source-builder /stackable/spark-${PRODUCT}-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-connectors-builder /stackable/hbase-connector-${HBASE_CONNECTOR}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/spark-${PRODUCT}/assembly/target/bom.json /stackable/spark/spark-${PRODUCT}.cdx.json COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/jmx /stackable/jmx COPY --from=spark-builder /usr/bin/tini /usr/bin/tini diff --git a/trino/Dockerfile b/trino/Dockerfile index c1f0c77ca..bbcf97153 100644 --- a/trino/Dockerfile +++ b/trino/Dockerfile @@ -15,6 +15,7 @@ COPY --chown=${STACKABLE_USER_UID}:0 trino/stackable/patches/patchable.toml /sta COPY --chown=${STACKABLE_USER_UID}:0 trino/stackable/patches/${PRODUCT} /stackable/src/trino/stackable/patches/${PRODUCT} COPY --from=trino-storage-connector-image /stackable/src/trino-storage-connector/patchable-work/worktree/${PRODUCT}/target/trino-storage-${PRODUCT} /stackable/trino-server-${PRODUCT}/plugin/trino-storage-${PRODUCT} +COPY --from=trino-storage-connector-image /stackable/trino-storage-connector-${PRODUCT}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 trino/stackable/jmx /stackable/jmx # adding a hadolint ignore for SC2215, due to https://github.com/hadolint/hadolint/issues/980 @@ -22,6 +23,9 @@ COPY --chown=${STACKABLE_USER_UID}:0 trino/stackable/jmx /stackable/jmx RUN --mount=type=cache,id=maven-${PRODUCT},target=/root/.m2/repository <<EOF cd "$(/stackable/patchable --images-repo-root=src checkout trino ${PRODUCT})" +# Create snapshot of the source code including custom patches +tar -czf /stackable/trino-${PRODUCT}-src.tar.gz . + # Trino is using something (git-commit-id-plugin in the past, maybe something else now) that is # reading the Git history and searches for a tag to pull the version from. It sounds weird to me # why someone would do that over just picking the version from the pom.xml, but they propably diff --git a/trino/storage-connector/Dockerfile b/trino/storage-connector/Dockerfile index 64f76a0ef..de45dc7eb 100644 --- a/trino/storage-connector/Dockerfile +++ b/trino/storage-connector/Dockerfile @@ -16,6 +16,9 @@ COPY --chown=${STACKABLE_USER_UID}:0 trino/storage-connector/stackable/patches/$ RUN --mount=type=cache,id=maven-${PRODUCT},target=/root/.m2/repository <<EOF cd "$(/stackable/patchable --images-repo-root=src checkout trino-storage-connector ${PRODUCT})" +# Create snapshot of the source code including custom patches +tar -czf /stackable/trino-storage-connector-${PRODUCT}-src.tar.gz . + # Upstream builds are marked as -SNAPSHOT, even for release builds mvn versions:set -DnewVersion=${PRODUCT} diff --git a/zookeeper/Dockerfile b/zookeeper/Dockerfile index 40406a1bd..27066b688 100644 --- a/zookeeper/Dockerfile +++ b/zookeeper/Dockerfile @@ -23,6 +23,9 @@ WORKDIR /stackable RUN <<EOF cd "$(/stackable/patchable --images-repo-root=src checkout zookeeper ${PRODUCT})" +# Create snapshot of the source code including custom patches +tar -czf /stackable/zookeeper-${PRODUCT}-src.tar.gz . + # Exclude the `zookeeper-client` submodule, this is not needed and has c parts # that created all kinds of issues for the build container mvn --batch-mode --no-transfer-progress -pl "!zookeeper-client/zookeeper-client-c" clean install checkstyle:check spotbugs:check -DskipTests -Pfull-build @@ -68,6 +71,7 @@ LABEL \ # Copy over the ZooKeeper binary folder COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/apache-zookeeper-${PRODUCT}-bin /stackable/apache-zookeeper-${PRODUCT}-bin/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/zookeeper-${PRODUCT}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/jmx /stackable/jmx/ COPY zookeeper/licenses /licenses