diff --git a/CHANGELOG.md b/CHANGELOG.md index b4f3a43be..7834911e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,7 @@ All notable changes to this project will be documented in this file. - opa: Enable custom versions ([#1170]). - use custom product versions for Hadoop, HBase, Phoenix, hbase-operator-tools, Druid, Hive and Spark ([#1173]). - hbase: Bump dependencies to the latest patch level for HBase `2.6.1` and `2.6.2` ([#1185]). +- hadoop: Separate Dockerfiles for Hadoop build and HDFS image ([#1186]). ### Fixed @@ -211,6 +212,7 @@ All notable changes to this project will be documented in this file. [#1180]: https://github.com/stackabletech/docker-images/pull/1180 [#1184]: https://github.com/stackabletech/docker-images/pull/1184 [#1185]: https://github.com/stackabletech/docker-images/pull/1185 +[#1186]: https://github.com/stackabletech/docker-images/pull/1186 [#1188]: https://github.com/stackabletech/docker-images/pull/1188 [#1189]: https://github.com/stackabletech/docker-images/pull/1189 diff --git a/conf.py b/conf.py index 2f2b07377..4363382b3 100644 --- a/conf.py +++ b/conf.py @@ -13,6 +13,7 @@ airflow = importlib.import_module("airflow.versions") druid = importlib.import_module("druid.versions") hadoop = importlib.import_module("hadoop.versions") +hadoop_jars = importlib.import_module("hadoop.hadoop.versions") hbase = importlib.import_module("hbase.versions") hbase_jars = importlib.import_module("hbase.hbase.versions") hbase_phoenix = importlib.import_module("hbase.phoenix.versions") @@ -49,6 +50,7 @@ {"name": "airflow", "versions": airflow.versions}, {"name": "druid", "versions": druid.versions}, {"name": "hadoop", "versions": hadoop.versions}, + {"name": "hadoop/hadoop", "versions": hadoop_jars.versions}, {"name": "hbase", "versions": hbase.versions}, {"name": "hbase/hbase", "versions": hbase_jars.versions}, {"name": "hbase/phoenix", "versions": hbase_phoenix.versions}, diff --git a/druid/Dockerfile b/druid/Dockerfile index ba5638b9f..7993010ba 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/hadoop AS hadoop-builder +FROM stackable/image/hadoop/hadoop AS hadoop-builder FROM stackable/image/java-devel AS druid-builder @@ -12,7 +12,7 @@ ARG STAX2_API ARG WOODSTOX_CORE ARG AUTHORIZER ARG STACKABLE_USER_UID -ARG HADOOP +ARG HADOOP_HADOOP # Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.) # This can be used to speed up builds when disk space is of no concern. @@ -75,7 +75,7 @@ mvn \ --no-transfer-progress \ clean install \ -Pdist,stackable-bundle-contrib-exts \ - -Dhadoop.compile.version=${HADOOP}-stackable${RELEASE} \ + -Dhadoop.compile.version=${HADOOP_HADOOP}-stackable${RELEASE} \ -DskipTests `# Skip test execution` \ -Dcheckstyle.skip `# Skip checkstyle checks. We dont care if the code is properly formatted, it just wastes time` \ -Dmaven.javadoc.skip=true `# Dont generate javadoc` \ diff --git a/druid/versions.py b/druid/versions.py index 323868f1c..e380f7c81 100644 --- a/druid/versions.py +++ b/druid/versions.py @@ -4,7 +4,7 @@ # https://druid.apache.org/docs/30.0.1/operations/java/ "java-base": "17", "java-devel": "17", - "hadoop": "3.3.6", + "hadoop/hadoop": "3.3.6", "authorizer": "0.7.0", }, { @@ -12,7 +12,7 @@ # https://druid.apache.org/docs/31.0.1/operations/java/ "java-base": "17", "java-devel": "17", - "hadoop": "3.3.6", + "hadoop/hadoop": "3.3.6", "authorizer": "0.7.0", }, { @@ -20,7 +20,7 @@ # https://druid.apache.org/docs/33.0.0/operations/java/ "java-base": "17", "java-devel": "17", - "hadoop": "3.3.6", + "hadoop/hadoop": "3.3.6", "authorizer": "0.7.0", }, ] diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 3997fb1fb..286bec5c1 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -1,144 +1,14 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/java-devel AS hadoop-builder - -ARG PRODUCT -ARG RELEASE -ARG ASYNC_PROFILER -ARG JMX_EXPORTER -ARG PROTOBUF -ARG TARGETARCH -ARG TARGETOS -ARG STACKABLE_USER_UID - -WORKDIR /stackable - -COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/patchable.toml /stackable/src/shared/protobuf/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBUF} /stackable/src/shared/protobuf/stackable/patches/${PROTOBUF} - -RUN <hadoop-pipes<\/artifactId>/,/<\/dependency>/ { s/.*<\/version>/'"$ORIGINAL_VERSION"'<\/version>/ }' -i hadoop-tools/hadoop-tools-dist/pom.xml - -# Create snapshot of the source code including custom patches -tar -czf /stackable/hadoop-${NEW_VERSION}-src.tar.gz . - -mvn \ - --batch-mode \ - --no-transfer-progress \ - clean package install \ - -Pdist,native \ - -pl '!hadoop-tools/hadoop-pipes' \ - -Dhadoop.version=${NEW_VERSION} \ - -Drequire.fuse=true \ - -DskipTests \ - -Dmaven.javadoc.skip=true - -mkdir -p /stackable/patched-libs/maven/org/apache -cp -r /stackable/.m2/repository/org/apache/hadoop /stackable/patched-libs/maven/org/apache - -cp -r hadoop-dist/target/hadoop-${NEW_VERSION} /stackable/hadoop-${NEW_VERSION} -sed -i "s/${NEW_VERSION}/${ORIGINAL_VERSION}/g" hadoop-dist/target/bom.json -mv hadoop-dist/target/bom.json /stackable/hadoop-${NEW_VERSION}/hadoop-${NEW_VERSION}.cdx.json - -# HDFS fuse-dfs is not part of the regular dist output, so we need to copy it in ourselves -cp hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs /stackable/hadoop-${NEW_VERSION}/bin - -# Remove source code -(cd .. && rm -r ${PRODUCT}) - -ln -s /stackable/hadoop-${NEW_VERSION} /stackable/hadoop - -mv /build/fuse_dfs_wrapper /stackable/hadoop/bin - -# Remove unneeded binaries: -# - code sources -# - mapreduce/yarn binaries that were built as cross-project dependencies -# - minicluster (only used for testing) and test .jars -# - json-io: this is a transitive dependency pulled in by cedarsoft/java-utils/json-io and is excluded in 3.4.0. See CVE-2023-34610. -rm -rf /stackable/hadoop/share/hadoop/common/sources/ -rm -rf /stackable/hadoop/share/hadoop/hdfs/sources/ -rm -rf /stackable/hadoop/share/hadoop/tools/sources/ -rm -rf /stackable/hadoop/share/hadoop/tools/lib/json-io-*.jar -rm -rf /stackable/hadoop/share/hadoop/tools/lib/hadoop-mapreduce-client-*.jar -rm -rf /stackable/hadoop/share/hadoop/tools/lib/hadoop-yarn-server*.jar -find /stackable/hadoop -name 'hadoop-minicluster-*.jar' -type f -delete -find /stackable/hadoop -name 'hadoop-client-minicluster-*.jar' -type f -delete -find /stackable/hadoop -name 'hadoop-*tests.jar' -type f -delete -rm -rf /stackable/.m2 - -# Set correct groups; make sure only required artifacts for the final image are located in /stackable -chmod -R g=u /stackable -EOF +FROM stackable/image/hadoop/hadoop AS hadoop-builder FROM stackable/image/java-devel AS hdfs-utils-builder ARG HDFS_UTILS ARG PRODUCT +ARG RELEASE +ARG HADOOP_HADOOP ARG STACKABLE_USER_UID # Starting with hdfs-utils 0.4.0 we need to use Java 17 for compilation. @@ -161,6 +31,8 @@ WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/patchable.toml /stackable/src/hadoop/hdfs-utils/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS} /stackable/src/hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS} +COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 /stackable/patched-libs /stackable/patched-libs + # The Stackable HDFS utils contain an OPA authorizer, group mapper & topology provider. # The topology provider provides rack awareness functionality for HDFS by allowing users to specify Kubernetes # labels to build a rackID from. @@ -168,6 +40,10 @@ COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/${HDFS_ RUN < /etc/fuse.conf + +ln -s "/stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE}" /stackable/hadoop + +# async-profiler +ARCH="${TARGETARCH/amd64/x64}" +curl "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}.tar.gz" | tar -xzC /stackable +ln -s "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" /stackable/async-profiler + +# JMX Exporter +curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" +chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" +ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar + +# Set correct permissions and ownerships +chown --recursive ${STACKABLE_USER_UID}:0 /stackable/hadoop /stackable/jmx /stackable/async-profiler "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" +chmod --recursive g=u /stackable/jmx /stackable/async-profiler "/stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE}" EOF # ---------------------------------------- diff --git a/hadoop/hadoop/Dockerfile b/hadoop/hadoop/Dockerfile new file mode 100644 index 000000000..844f23b5e --- /dev/null +++ b/hadoop/hadoop/Dockerfile @@ -0,0 +1,123 @@ +# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 +# check=error=true + +FROM stackable/image/java-devel AS hadoop-builder + +ARG PRODUCT +ARG RELEASE +ARG PROTOBUF +ARG STACKABLE_USER_UID + +WORKDIR /stackable + +COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/patchable.toml /stackable/src/shared/protobuf/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBUF} /stackable/src/shared/protobuf/stackable/patches/${PROTOBUF} + +RUN <hadoop-pipes<\/artifactId>/,/<\/dependency>/ { s/.*<\/version>/'"$ORIGINAL_VERSION"'<\/version>/ }' -i hadoop-tools/hadoop-tools-dist/pom.xml + +# Create snapshot of the source code including custom patches +tar -czf /stackable/hadoop-${NEW_VERSION}-src.tar.gz . + +mvn \ + --batch-mode \ + --no-transfer-progress \ + clean package install \ + -Pdist,native \ + -pl '!hadoop-tools/hadoop-pipes' \ + -Dhadoop.version=${NEW_VERSION} \ + -Drequire.fuse=true \ + -DskipTests \ + -Dmaven.javadoc.skip=true + +mkdir -p /stackable/patched-libs/maven/org/apache +cp -r /stackable/.m2/repository/org/apache/hadoop /stackable/patched-libs/maven/org/apache + +rm -rf hadoop-dist/target/hadoop-${NEW_VERSION}/share/hadoop/yarn +rm -rf hadoop-dist/target/hadoop-${NEW_VERSION}/share/hadoop/mapreduce +rm hadoop-dist/target/hadoop-${NEW_VERSION}/share/hadoop/client/hadoop-client-minicluster-*.jar +rm hadoop-dist/target/hadoop-${NEW_VERSION}/share/hadoop/tools/lib/hadoop-minicluster-*.jar + +cp -r hadoop-dist/target/hadoop-${NEW_VERSION} /stackable/hadoop-${NEW_VERSION} +sed -i "s/${NEW_VERSION}/${ORIGINAL_VERSION}/g" hadoop-dist/target/bom.json +mv hadoop-dist/target/bom.json /stackable/hadoop-${NEW_VERSION}/hadoop-${NEW_VERSION}.cdx.json + +# HDFS fuse-dfs is not part of the regular dist output, so we need to copy it in ourselves +cp hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs /stackable/hadoop-${NEW_VERSION}/bin + +# Remove source code +(cd .. && rm -r ${PRODUCT}) + +ln -s /stackable/hadoop-${NEW_VERSION} /stackable/hadoop + +mv /build/fuse_dfs_wrapper /stackable/hadoop/bin + +# Remove unneeded binaries: +# - code sources +# - mapreduce/yarn binaries that were built as cross-project dependencies +# - minicluster (only used for testing) and test .jars +# - json-io: this is a transitive dependency pulled in by cedarsoft/java-utils/json-io and is excluded in 3.4.0. See CVE-2023-34610. +rm -rf /stackable/hadoop/share/hadoop/common/sources/ +rm -rf /stackable/hadoop/share/hadoop/hdfs/sources/ +rm -rf /stackable/hadoop/share/hadoop/tools/sources/ +rm -rf /stackable/hadoop/share/hadoop/tools/lib/json-io-*.jar +rm -rf /stackable/hadoop/share/hadoop/tools/lib/hadoop-mapreduce-client-*.jar +rm -rf /stackable/hadoop/share/hadoop/tools/lib/hadoop-yarn-server*.jar +find /stackable/hadoop -name 'hadoop-minicluster-*.jar' -type f -delete +find /stackable/hadoop -name 'hadoop-client-minicluster-*.jar' -type f -delete +find /stackable/hadoop -name 'hadoop-*tests.jar' -type f -delete +rm -rf /stackable/.m2 + +# Set correct groups; make sure only required artifacts for the final image are located in /stackable +chmod -R g=u /stackable +EOF diff --git a/hadoop/stackable/fuse_dfs_wrapper b/hadoop/hadoop/stackable/fuse_dfs_wrapper similarity index 100% rename from hadoop/stackable/fuse_dfs_wrapper rename to hadoop/hadoop/stackable/fuse_dfs_wrapper diff --git a/hadoop/stackable/patches/3.3.6/0001-YARN-11527-Update-node.js.patch b/hadoop/hadoop/stackable/patches/3.3.6/0001-YARN-11527-Update-node.js.patch similarity index 100% rename from hadoop/stackable/patches/3.3.6/0001-YARN-11527-Update-node.js.patch rename to hadoop/hadoop/stackable/patches/3.3.6/0001-YARN-11527-Update-node.js.patch diff --git a/hadoop/stackable/patches/3.3.6/0002-Allow-overriding-datanode-registration-addresses.patch b/hadoop/hadoop/stackable/patches/3.3.6/0002-Allow-overriding-datanode-registration-addresses.patch similarity index 100% rename from hadoop/stackable/patches/3.3.6/0002-Allow-overriding-datanode-registration-addresses.patch rename to hadoop/hadoop/stackable/patches/3.3.6/0002-Allow-overriding-datanode-registration-addresses.patch diff --git a/hadoop/stackable/patches/3.3.6/0003-HADOOP-18055-Add-async-profiler.patch b/hadoop/hadoop/stackable/patches/3.3.6/0003-HADOOP-18055-Add-async-profiler.patch similarity index 100% rename from hadoop/stackable/patches/3.3.6/0003-HADOOP-18055-Add-async-profiler.patch rename to hadoop/hadoop/stackable/patches/3.3.6/0003-HADOOP-18055-Add-async-profiler.patch diff --git a/hadoop/stackable/patches/3.3.6/0004-Backport-HADOOP-18077.patch b/hadoop/hadoop/stackable/patches/3.3.6/0004-Backport-HADOOP-18077.patch similarity index 100% rename from hadoop/stackable/patches/3.3.6/0004-Backport-HADOOP-18077.patch rename to hadoop/hadoop/stackable/patches/3.3.6/0004-Backport-HADOOP-18077.patch diff --git a/hadoop/stackable/patches/3.3.6/0005-Async-profiler-also-grab-itimer-events.patch b/hadoop/hadoop/stackable/patches/3.3.6/0005-Async-profiler-also-grab-itimer-events.patch similarity index 100% rename from hadoop/stackable/patches/3.3.6/0005-Async-profiler-also-grab-itimer-events.patch rename to hadoop/hadoop/stackable/patches/3.3.6/0005-Async-profiler-also-grab-itimer-events.patch diff --git a/hadoop/stackable/patches/3.3.6/0006-HDFS-17378-Fix-missing-operationType-for-some-operat.patch b/hadoop/hadoop/stackable/patches/3.3.6/0006-HDFS-17378-Fix-missing-operationType-for-some-operat.patch similarity index 100% rename from hadoop/stackable/patches/3.3.6/0006-HDFS-17378-Fix-missing-operationType-for-some-operat.patch rename to hadoop/hadoop/stackable/patches/3.3.6/0006-HDFS-17378-Fix-missing-operationType-for-some-operat.patch diff --git a/hadoop/stackable/patches/3.3.6/0007-Bump-Snappy-version-to-fix-CVEs.patch b/hadoop/hadoop/stackable/patches/3.3.6/0007-Bump-Snappy-version-to-fix-CVEs.patch similarity index 100% rename from hadoop/stackable/patches/3.3.6/0007-Bump-Snappy-version-to-fix-CVEs.patch rename to hadoop/hadoop/stackable/patches/3.3.6/0007-Bump-Snappy-version-to-fix-CVEs.patch diff --git a/hadoop/stackable/patches/3.3.6/0008-Update-CycloneDX-plugin.patch b/hadoop/hadoop/stackable/patches/3.3.6/0008-Update-CycloneDX-plugin.patch similarity index 100% rename from hadoop/stackable/patches/3.3.6/0008-Update-CycloneDX-plugin.patch rename to hadoop/hadoop/stackable/patches/3.3.6/0008-Update-CycloneDX-plugin.patch diff --git a/hadoop/stackable/patches/3.3.6/0009-HADOOP-18516-ABFS-Authentication-Support-Fixed-SAS-T.patch b/hadoop/hadoop/stackable/patches/3.3.6/0009-HADOOP-18516-ABFS-Authentication-Support-Fixed-SAS-T.patch similarity index 100% rename from hadoop/stackable/patches/3.3.6/0009-HADOOP-18516-ABFS-Authentication-Support-Fixed-SAS-T.patch rename to hadoop/hadoop/stackable/patches/3.3.6/0009-HADOOP-18516-ABFS-Authentication-Support-Fixed-SAS-T.patch diff --git a/hadoop/stackable/patches/3.3.6/0010-Build-hadoop-client-modules-before-hadoop-dist.patch b/hadoop/hadoop/stackable/patches/3.3.6/0010-Build-hadoop-client-modules-before-hadoop-dist.patch similarity index 100% rename from hadoop/stackable/patches/3.3.6/0010-Build-hadoop-client-modules-before-hadoop-dist.patch rename to hadoop/hadoop/stackable/patches/3.3.6/0010-Build-hadoop-client-modules-before-hadoop-dist.patch diff --git a/hadoop/stackable/patches/3.3.6/0011-Remove-Hadoop-benchmark.patch b/hadoop/hadoop/stackable/patches/3.3.6/0011-Remove-Hadoop-benchmark.patch similarity index 100% rename from hadoop/stackable/patches/3.3.6/0011-Remove-Hadoop-benchmark.patch rename to hadoop/hadoop/stackable/patches/3.3.6/0011-Remove-Hadoop-benchmark.patch diff --git a/hadoop/stackable/patches/3.3.6/patchable.toml b/hadoop/hadoop/stackable/patches/3.3.6/patchable.toml similarity index 100% rename from hadoop/stackable/patches/3.3.6/patchable.toml rename to hadoop/hadoop/stackable/patches/3.3.6/patchable.toml diff --git a/hadoop/stackable/patches/3.4.1/0001-YARN-11527-Update-node.js.patch b/hadoop/hadoop/stackable/patches/3.4.1/0001-YARN-11527-Update-node.js.patch similarity index 100% rename from hadoop/stackable/patches/3.4.1/0001-YARN-11527-Update-node.js.patch rename to hadoop/hadoop/stackable/patches/3.4.1/0001-YARN-11527-Update-node.js.patch diff --git a/hadoop/stackable/patches/3.4.1/0002-Allow-overriding-datanode-registration-addresses.patch b/hadoop/hadoop/stackable/patches/3.4.1/0002-Allow-overriding-datanode-registration-addresses.patch similarity index 100% rename from hadoop/stackable/patches/3.4.1/0002-Allow-overriding-datanode-registration-addresses.patch rename to hadoop/hadoop/stackable/patches/3.4.1/0002-Allow-overriding-datanode-registration-addresses.patch diff --git a/hadoop/stackable/patches/3.4.1/0003-Async-profiler-also-grab-itimer-events.patch b/hadoop/hadoop/stackable/patches/3.4.1/0003-Async-profiler-also-grab-itimer-events.patch similarity index 100% rename from hadoop/stackable/patches/3.4.1/0003-Async-profiler-also-grab-itimer-events.patch rename to hadoop/hadoop/stackable/patches/3.4.1/0003-Async-profiler-also-grab-itimer-events.patch diff --git a/hadoop/stackable/patches/3.4.1/0004-HDFS-17378-Fix-missing-operationType-for-some-operat.patch b/hadoop/hadoop/stackable/patches/3.4.1/0004-HDFS-17378-Fix-missing-operationType-for-some-operat.patch similarity index 100% rename from hadoop/stackable/patches/3.4.1/0004-HDFS-17378-Fix-missing-operationType-for-some-operat.patch rename to hadoop/hadoop/stackable/patches/3.4.1/0004-HDFS-17378-Fix-missing-operationType-for-some-operat.patch diff --git a/hadoop/stackable/patches/3.4.1/0005-Update-CycloneDX-plugin.patch b/hadoop/hadoop/stackable/patches/3.4.1/0005-Update-CycloneDX-plugin.patch similarity index 100% rename from hadoop/stackable/patches/3.4.1/0005-Update-CycloneDX-plugin.patch rename to hadoop/hadoop/stackable/patches/3.4.1/0005-Update-CycloneDX-plugin.patch diff --git a/hadoop/stackable/patches/3.4.1/0006-HADOOP-19352.-Hadoop-OSS-Connector-adds-support-for-.patch b/hadoop/hadoop/stackable/patches/3.4.1/0006-HADOOP-19352.-Hadoop-OSS-Connector-adds-support-for-.patch similarity index 100% rename from hadoop/stackable/patches/3.4.1/0006-HADOOP-19352.-Hadoop-OSS-Connector-adds-support-for-.patch rename to hadoop/hadoop/stackable/patches/3.4.1/0006-HADOOP-19352.-Hadoop-OSS-Connector-adds-support-for-.patch diff --git a/hadoop/stackable/patches/3.4.1/0007-HADOOP-19335.-Bump-netty-to-4.1.116-due-to-CVE-2024-.patch b/hadoop/hadoop/stackable/patches/3.4.1/0007-HADOOP-19335.-Bump-netty-to-4.1.116-due-to-CVE-2024-.patch similarity index 100% rename from hadoop/stackable/patches/3.4.1/0007-HADOOP-19335.-Bump-netty-to-4.1.116-due-to-CVE-2024-.patch rename to hadoop/hadoop/stackable/patches/3.4.1/0007-HADOOP-19335.-Bump-netty-to-4.1.116-due-to-CVE-2024-.patch diff --git a/hadoop/stackable/patches/3.4.1/0008-HADOOP-19465.-Upgrade-to-netty-4.1.118-due-to-CVE-20.patch b/hadoop/hadoop/stackable/patches/3.4.1/0008-HADOOP-19465.-Upgrade-to-netty-4.1.118-due-to-CVE-20.patch similarity index 100% rename from hadoop/stackable/patches/3.4.1/0008-HADOOP-19465.-Upgrade-to-netty-4.1.118-due-to-CVE-20.patch rename to hadoop/hadoop/stackable/patches/3.4.1/0008-HADOOP-19465.-Upgrade-to-netty-4.1.118-due-to-CVE-20.patch diff --git a/hadoop/stackable/patches/3.4.1/0009-HADOOP-19456.-Upgrade-kafka-to-3.9.0-to-fix-CVE-2024.patch b/hadoop/hadoop/stackable/patches/3.4.1/0009-HADOOP-19456.-Upgrade-kafka-to-3.9.0-to-fix-CVE-2024.patch similarity index 100% rename from hadoop/stackable/patches/3.4.1/0009-HADOOP-19456.-Upgrade-kafka-to-3.9.0-to-fix-CVE-2024.patch rename to hadoop/hadoop/stackable/patches/3.4.1/0009-HADOOP-19456.-Upgrade-kafka-to-3.9.0-to-fix-CVE-2024.patch diff --git a/hadoop/stackable/patches/3.4.1/0010-HADOOP-19225.-Upgrade-Jetty-to-9.4.57.v20241219-due-.patch b/hadoop/hadoop/stackable/patches/3.4.1/0010-HADOOP-19225.-Upgrade-Jetty-to-9.4.57.v20241219-due-.patch similarity index 100% rename from hadoop/stackable/patches/3.4.1/0010-HADOOP-19225.-Upgrade-Jetty-to-9.4.57.v20241219-due-.patch rename to hadoop/hadoop/stackable/patches/3.4.1/0010-HADOOP-19225.-Upgrade-Jetty-to-9.4.57.v20241219-due-.patch diff --git a/hadoop/stackable/patches/3.4.1/patchable.toml b/hadoop/hadoop/stackable/patches/3.4.1/patchable.toml similarity index 100% rename from hadoop/stackable/patches/3.4.1/patchable.toml rename to hadoop/hadoop/stackable/patches/3.4.1/patchable.toml diff --git a/hadoop/stackable/patches/patchable.toml b/hadoop/hadoop/stackable/patches/patchable.toml similarity index 100% rename from hadoop/stackable/patches/patchable.toml rename to hadoop/hadoop/stackable/patches/patchable.toml diff --git a/hadoop/hadoop/versions.py b/hadoop/hadoop/versions.py new file mode 100644 index 000000000..d6907a86f --- /dev/null +++ b/hadoop/hadoop/versions.py @@ -0,0 +1,13 @@ +versions = [ + { + # Not part of SDP 25.7.0, but still required for hbase, hive, spark-k8s + "product": "3.3.6", + "java-devel": "11", + "protobuf": "3.7.1", + }, + { + "product": "3.4.1", + "java-devel": "11", + "protobuf": "3.7.1", + }, +] diff --git a/hadoop/versions.py b/hadoop/versions.py index f1653a05f..e88be1aca 100644 --- a/hadoop/versions.py +++ b/hadoop/versions.py @@ -2,20 +2,20 @@ { # Not part of SDP 25.7.0, but still required for hbase, hive, spark-k8s "product": "3.3.6", + "hadoop/hadoop": "3.3.6", "java-base": "11", "java-devel": "11", "async_profiler": "2.9", "jmx_exporter": "1.3.0", - "protobuf": "3.7.1", "hdfs_utils": "0.4.0", }, { "product": "3.4.1", + "hadoop/hadoop": "3.4.1", "java-base": "11", "java-devel": "11", "async_profiler": "2.9", "jmx_exporter": "1.3.0", - "protobuf": "3.7.1", "hdfs_utils": "0.4.1", }, ] diff --git a/hbase/Dockerfile b/hbase/Dockerfile index cf255a29d..cb40ff0b0 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/hadoop AS hadoop-builder +FROM stackable/image/hadoop/hadoop AS hadoop-builder FROM stackable/image/hbase/hbase AS hbase-builder @@ -17,7 +17,7 @@ FROM stackable/image/java-devel AS hadoop-s3-builder ARG PRODUCT ARG RELEASE -ARG HADOOP +ARG HADOOP_HADOOP ARG HBASE_HBASE ARG STACKABLE_USER_UID @@ -31,7 +31,7 @@ COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 \ # So we try to copy both and if one of them doesn't exist buildx will just ignore it :) /stackable/hadoop/share/hadoop/tools/lib/bundle-*.jar \ /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-*.jar \ - /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP}-stackable${RELEASE}.jar \ + /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP_HADOOP}-stackable${RELEASE}.jar \ /stackable/hadoop/share/hadoop/tools/lib/ COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase/stackable/bin/export-snapshot-to-s3.env /stackable/bin/ @@ -56,7 +56,7 @@ FROM stackable/image/java-base AS final ARG PRODUCT ARG RELEASE -ARG HADOOP +ARG HADOOP_HADOOP ARG HBASE_PROFILE ARG HBASE_HBASE ARG HBASE_HBASE_OPERATOR_TOOLS @@ -106,7 +106,7 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/hadoop/ # hadoop-azure-${HADOOP}.jar contains the AzureBlobFileSystem which is required # by hadoop-common-${HADOOP}.jar if the scheme of a file system is "abfs://". COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder \ - /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}-stackable${RELEASE}.jar \ + /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP_HADOOP}-stackable${RELEASE}.jar \ /stackable/hbase-${HBASE_HBASE}-stackable${RELEASE}/lib/ COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer-${HBASE_HBASE_OPA_AUTHORIZER}-src.tar.gz /stackable diff --git a/hbase/hbase-operator-tools/Dockerfile b/hbase/hbase-operator-tools/Dockerfile index af4790254..27ff5dad2 100644 --- a/hbase/hbase-operator-tools/Dockerfile +++ b/hbase/hbase-operator-tools/Dockerfile @@ -1,6 +1,6 @@ FROM stackable/image/hbase/hbase AS hbase-builder -FROM stackable/image/hadoop AS hadoop-builder +FROM stackable/image/hadoop/hadoop AS hadoop-builder FROM stackable/image/java-devel diff --git a/hbase/hbase-operator-tools/versions.py b/hbase/hbase-operator-tools/versions.py index 9fbee3e66..248ecd343 100644 --- a/hbase/hbase-operator-tools/versions.py +++ b/hbase/hbase-operator-tools/versions.py @@ -2,7 +2,7 @@ { "product": "1.3.0-fd5a5fb-hbase2.6.1", "hbase_operator_tools_version": "1.3.0-fd5a5fb", - "hadoop": "3.3.6", + "hadoop/hadoop": "3.3.6", "hbase_thirdparty": "4.1.9", "hbase/hbase": "2.6.1", "java-devel": "11", @@ -11,7 +11,7 @@ { "product": "1.3.0-fd5a5fb-hbase2.6.2", "hbase_operator_tools_version": "1.3.0-fd5a5fb", - "hadoop": "3.4.1", + "hadoop/hadoop": "3.4.1", "hbase_thirdparty": "4.1.9", "hbase/hbase": "2.6.2", "java-devel": "11", diff --git a/hbase/hbase/Dockerfile b/hbase/hbase/Dockerfile index 62e347260..f52a86e1c 100644 --- a/hbase/hbase/Dockerfile +++ b/hbase/hbase/Dockerfile @@ -1,11 +1,11 @@ -FROM stackable/image/hadoop AS hadoop-builder +FROM stackable/image/hadoop/hadoop AS hadoop-builder FROM stackable/image/java-devel AS hbase-builder ARG PRODUCT ARG RELEASE ARG ASYNC_PROFILER -ARG HADOOP +ARG HADOOP_HADOOP ARG TARGETARCH ARG TARGETOS ARG STACKABLE_USER_UID @@ -55,7 +55,7 @@ mvn \ --batch-mode \ --no-transfer-progress \ -Dhadoop.profile=3.0 \ - -Dhadoop-three.version=${HADOOP}-stackable${RELEASE} \ + -Dhadoop-three.version=${HADOOP_HADOOP}-stackable${RELEASE} \ -DskipTests \ clean install @@ -63,7 +63,7 @@ mvn \ --batch-mode \ --no-transfer-progress \ -Dhadoop.profile=3.0 \ - -Dhadoop-three.version=${HADOOP}-stackable${RELEASE} \ + -Dhadoop-three.version=${HADOOP_HADOOP}-stackable${RELEASE} \ -DskipTests \ -Dcheckstyle.skip=true \ -Prelease \ diff --git a/hbase/hbase/versions.py b/hbase/hbase/versions.py index 136179d58..205cedacb 100644 --- a/hbase/hbase/versions.py +++ b/hbase/hbase/versions.py @@ -3,7 +3,7 @@ # hbase-thirdparty is used to build the hbase-operator-tools and should be set to the version defined in the POM of HBase. { "product": "2.6.1", - "hadoop": "3.3.6", + "hadoop/hadoop": "3.3.6", "java-base": "11", "java-devel": "11", "async_profiler": "2.9", @@ -11,7 +11,7 @@ }, { "product": "2.6.2", - "hadoop": "3.4.1", + "hadoop/hadoop": "3.4.1", "java-base": "11", "java-devel": "11", "async_profiler": "2.9", diff --git a/hbase/phoenix/Dockerfile b/hbase/phoenix/Dockerfile index faff4faa6..2eb834699 100644 --- a/hbase/phoenix/Dockerfile +++ b/hbase/phoenix/Dockerfile @@ -1,4 +1,4 @@ -FROM stackable/image/hadoop AS hadoop-builder +FROM stackable/image/hadoop/hadoop AS hadoop-builder FROM stackable/image/hbase/hbase AS hbase-builder @@ -9,7 +9,7 @@ ARG RELEASE ARG ASYNC_PROFILER ARG HBASE_HBASE ARG HBASE_PROFILE -ARG HADOOP +ARG HADOOP_HADOOP ARG STACKABLE_USER_UID ARG PHOENIX_VERSION @@ -47,7 +47,7 @@ mvn \ --no-transfer-progress \ -Dhbase.version=${HBASE_HBASE}-stackable${RELEASE} \ -Dhbase.profile=${HBASE_PROFILE} \ - -Dhadoop.version=${HADOOP}-stackable${RELEASE} \ + -Dhadoop.version=${HADOOP_HADOOP}-stackable${RELEASE} \ -DskipTests \ -Dcheckstyle.skip=true \ clean \ diff --git a/hbase/phoenix/versions.py b/hbase/phoenix/versions.py index e66c11f00..ed0e304e2 100644 --- a/hbase/phoenix/versions.py +++ b/hbase/phoenix/versions.py @@ -3,7 +3,7 @@ "product": "5.2.1-hbase2.6.1", "phoenix_version": "5.2.1", "hbase/hbase": "2.6.1", - "hadoop": "3.3.6", + "hadoop/hadoop": "3.3.6", "java-devel": "11", "hbase_profile": "2.6", "delete_caches": "true", @@ -12,7 +12,7 @@ "product": "5.2.1-hbase2.6.2", "phoenix_version": "5.2.1", "hbase/hbase": "2.6.2", - "hadoop": "3.4.1", + "hadoop/hadoop": "3.4.1", "java-devel": "11", "hbase_profile": "2.6", "delete_caches": "true", diff --git a/hbase/versions.py b/hbase/versions.py index c2f9fdfbf..3ea6dcdbf 100644 --- a/hbase/versions.py +++ b/hbase/versions.py @@ -7,7 +7,7 @@ "hbase/hbase-operator-tools": "1.3.0-fd5a5fb-hbase2.6.1", "hbase/phoenix": "5.2.1-hbase2.6.1", "hbase/hbase-opa-authorizer": "0.1.0", # only for HBase 2.6.1 - "hadoop": "3.3.6", + "hadoop/hadoop": "3.3.6", "java-base": "11", "java-devel": "11", "hbase_profile": "2.6", @@ -19,7 +19,7 @@ "hbase/hbase-operator-tools": "1.3.0-fd5a5fb-hbase2.6.2", "hbase/phoenix": "5.2.1-hbase2.6.2", "hbase/hbase-opa-authorizer": "0.1.0", # only for HBase 2.6.1 - "hadoop": "3.4.1", + "hadoop/hadoop": "3.4.1", "java-base": "11", "java-devel": "11", "hbase_profile": "2.6", diff --git a/hive/Dockerfile b/hive/Dockerfile index 25a863892..cb3e402f5 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/hadoop AS hadoop-builder +FROM stackable/image/hadoop/hadoop AS hadoop-builder FROM stackable/image/java-devel AS hive-builder @@ -13,7 +13,7 @@ FROM stackable/image/java-devel AS hive-builder ARG PRODUCT ARG RELEASE -ARG HADOOP +ARG HADOOP_HADOOP ARG JMX_EXPORTER ARG AWS_JAVA_SDK_BUNDLE ARG AZURE_STORAGE @@ -30,8 +30,8 @@ COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/patches/${PRODUCT} /stackabl # It is useful to see which version of Hadoop is used at a glance # Therefore the use of the full name here # TODO: Do we really need all of Hadoop in here? -COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop /stackable/hadoop-${HADOOP}-stackable${RELEASE} -COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 /stackable/patched-libs /stackable/patched-libs +COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE} /stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE} +COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/patched-libs /stackable/patched-libs USER ${STACKABLE_USER_UID} WORKDIR /stackable @@ -60,7 +60,7 @@ if [[ "${PRODUCT}" == "3.1.3" ]] ; then else ( # https://issues.apache.org/jira/browse/HIVE-20451 switched the metastore server packaging starting with 4.0.0 - mvn --batch-mode --no-transfer-progress clean package -DskipTests -Dhadoop.version=${HADOOP}-stackable${RELEASE} + mvn --batch-mode --no-transfer-progress clean package -DskipTests -Dhadoop.version=${HADOOP_HADOOP}-stackable${RELEASE} # We only seem to get a .tar.gz archive, so let's extract that to the correct location tar --extract --directory=/stackable -f standalone-metastore/metastore-server/target/apache-hive-standalone-metastore-server-${NEW_VERSION}-bin.tar.gz @@ -84,17 +84,17 @@ ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/j # This way the build will fail should one of the files not be available anymore in a later Hadoop version! # Add S3 Support for Hive (support for s3a://) -cp /stackable/hadoop-${HADOOP}-stackable${RELEASE}/share/hadoop/tools/lib/hadoop-aws-${HADOOP}-stackable${RELEASE}.jar /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin/lib/ +cp /stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE}/share/hadoop/tools/lib/hadoop-aws-${HADOOP_HADOOP}-stackable${RELEASE}.jar /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin/lib/ # According to https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/aws_sdk_upgrade.html, the jar filename has changed from # aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE}.jar to bundle-${AWS_JAVA_SDK_BUNDLE}.jar. In future, you might need to do: -# cp /stackable/hadoop-${HADOOP}-stackable${RELEASE}/share/hadoop/tools/lib/bundle-${AWS_JAVA_SDK_BUNDLE}.jar /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin/lib/ -cp /stackable/hadoop-${HADOOP}-stackable${RELEASE}/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE}.jar /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin/lib/ +# cp /stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE}/share/hadoop/tools/lib/bundle-${AWS_JAVA_SDK_BUNDLE}.jar /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin/lib/ +cp /stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE}/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE}.jar /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin/lib/ # Add Azure ABFS support (support for abfs://) -cp /stackable/hadoop-${HADOOP}-stackable${RELEASE}/share/hadoop/tools/lib/hadoop-azure-${HADOOP}-stackable${RELEASE}.jar /stackable/apache-hive-metastore-${NEW_VERSION}-bin/lib/ -cp /stackable/hadoop-${HADOOP}-stackable${RELEASE}/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE}.jar /stackable/apache-hive-metastore-${NEW_VERSION}-bin/lib/ -cp /stackable/hadoop-${HADOOP}-stackable${RELEASE}/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar /stackable/apache-hive-metastore-${NEW_VERSION}-bin/lib/ +cp /stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE}/share/hadoop/tools/lib/hadoop-azure-${HADOOP_HADOOP}-stackable${RELEASE}.jar /stackable/apache-hive-metastore-${NEW_VERSION}-bin/lib/ +cp /stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE}/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE}.jar /stackable/apache-hive-metastore-${NEW_VERSION}-bin/lib/ +cp /stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE}/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar /stackable/apache-hive-metastore-${NEW_VERSION}-bin/lib/ # We're removing these to make the intermediate layer smaller # This can be necessary even though it's only a builder image because the GitHub Action Runners only have very limited space available @@ -114,7 +114,7 @@ EOF FROM stackable/image/java-base AS final ARG PRODUCT -ARG HADOOP +ARG HADOOP_HADOOP ARG RELEASE ARG STACKABLE_USER_UID @@ -145,8 +145,8 @@ WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/hive-${PRODUCT}-stackable${RELEASE}-src.tar.gz /stackable -COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/hadoop-${HADOOP}-stackable${RELEASE} /stackable/hadoop-${HADOOP}-stackable${RELEASE} -COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-${HADOOP}-stackable${RELEASE}-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE} /stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE} +COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/*-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/jmx /stackable/jmx COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/jmx /stackable/jmx COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/bin/start-metastore /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin/bin @@ -166,7 +166,7 @@ chmod g=u /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin/bi ln -s /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin /stackable/hive-metastore chown -h ${STACKABLE_USER_UID}:0 /stackable/hive-metastore chmod g=u /stackable/hive-metastore -ln -s /stackable/hadoop-${HADOOP}-stackable${RELEASE} /stackable/hadoop +ln -s /stackable/hadoop-${HADOOP_HADOOP}-stackable${RELEASE} /stackable/hadoop chown -h ${STACKABLE_USER_UID}:0 /stackable/hadoop chmod g=u /stackable/hadoop chmod g=u /stackable/*-src.tar.gz diff --git a/hive/versions.py b/hive/versions.py index c7459056d..d28fa84be 100644 --- a/hive/versions.py +++ b/hive/versions.py @@ -5,7 +5,7 @@ # Hive 3 must be built with Java 8 but will run on Java 11 "java-base": "11", "java-devel": "8", - "hadoop": "3.3.6", + "hadoop/hadoop": "3.3.6", # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 "aws_java_sdk_bundle": "1.12.367", "azure_storage": "7.0.1", @@ -17,7 +17,7 @@ # Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 "java-base": "11", "java-devel": "8", - "hadoop": "3.3.6", + "hadoop/hadoop": "3.3.6", # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 "aws_java_sdk_bundle": "1.12.367", "azure_storage": "7.0.1", @@ -29,7 +29,7 @@ # Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 "java-base": "11", "java-devel": "8", - "hadoop": "3.3.6", + "hadoop/hadoop": "3.3.6", # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 "aws_java_sdk_bundle": "1.12.367", "azure_storage": "7.0.1", diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile index 9e198757d..006f87cf6 100644 --- a/spark-k8s/Dockerfile +++ b/spark-k8s/Dockerfile @@ -2,7 +2,7 @@ # check=error=true # hadoop-builder: Provides Hadoop libraries -FROM stackable/image/hadoop AS hadoop-builder +FROM stackable/image/hadoop/hadoop AS hadoop-builder # hbase-builder: Provides HBase libraries FROM stackable/image/hbase AS hbase-builder @@ -37,7 +37,7 @@ FROM stackable/image/java-devel AS hbase-connectors-builder ARG PRODUCT ARG RELEASE -ARG HADOOP +ARG HADOOP_HADOOP ARG HBASE ARG HBASE_CONNECTOR ARG STACKABLE_USER_UID @@ -105,7 +105,7 @@ mvn \ --define spark.version="${PRODUCT}" \ --define scala.version="${SCALA_VERSION}" \ --define scala.binary.version="${SCALA_BINARY_VERSION}" \ - --define hadoop-three.version="${HADOOP}" \ + --define hadoop-three.version="${HADOOP_HADOOP}" \ --define hbase.version="${HBASE}" \ --define skipTests \ clean package @@ -133,7 +133,7 @@ EOF FROM stackable/image/java-devel AS spark-builder ARG PRODUCT -ARG HADOOP +ARG HADOOP_HADOOP ARG HBASE ARG AWS_JAVA_SDK_BUNDLE ARG AZURE_STORAGE @@ -179,7 +179,7 @@ RUN <