From 7ccea846897ea6a8209a2238c06933afb4c489bc Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Fri, 26 Sep 2025 14:56:10 +0000 Subject: [PATCH] HADOOP-19709. [JDK17] Add debian:12 and debian:13 as a build platform with JDK-17 as default This commit introduces support for Debian 12 (Bookworm) and Debian 13 (Trixie) as build platforms, following the approach established for Ubuntu 24. Key changes include: - Creation of `Dockerfile_debian_12` and `Dockerfile_debian_13` based on `Dockerfile_ubuntu_24`, with appropriate base images and package resolver arguments. - Updates to `dev-support/docker/pkg-resolver/packages.json` to include package definitions for `debian:12` and `debian:13`. - Addition of `debian:12` and `debian:13` to `dev-support/docker/pkg-resolver/platforms.json`. - Modification of `BUILDING.txt` to list `debian_12` and `debian_13` as supported OS platforms. --- BUILDING.txt | 2 +- dev-support/docker/Dockerfile_debian_12 | 108 +++++++++++++++ dev-support/docker/Dockerfile_debian_13 | 107 +++++++++++++++ dev-support/docker/pkg-resolver/packages.json | 129 +++++++++++++++++- .../docker/pkg-resolver/platforms.json | 4 +- 5 files changed, 347 insertions(+), 3 deletions(-) create mode 100644 dev-support/docker/Dockerfile_debian_12 create mode 100644 dev-support/docker/Dockerfile_debian_13 diff --git a/BUILDING.txt b/BUILDING.txt index f87cc8d11ead2..13519ff624cc7 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -31,7 +31,7 @@ On Linux / Mac: $ ./start-build-env.sh [OS platform] - - [OS Platform] One of [rockylinux_8, debian_11, ubuntu_20, ubuntu_24, windows_10]. + - [OS Platform] One of [rockylinux_8, debian_11, debian_12, debian_13, ubuntu_20, ubuntu_24, windows_10]. Default is 'ubuntu_20'. Note: Currently only default ('ubuntu_20') is supported on arm machine diff --git a/dev-support/docker/Dockerfile_debian_12 b/dev-support/docker/Dockerfile_debian_12 new file mode 100644 index 0000000000000..03a36cc51c6f8 --- /dev/null +++ b/dev-support/docker/Dockerfile_debian_12 @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 ( +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for installing the necessary dependencies for building Hadoop. +# See BUILDING.txt. + +FROM debian:12 + +WORKDIR /root + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +##### +# Disable suggests/recommends +##### +RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/10disableextras +RUN echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/10disableextras + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_TERSE true + +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008,SC2046 +RUN apt-get -q update +RUN apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates +RUN apt-get -q install -y --no-install-recommends python3 +RUN echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list +RUN wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc +RUN apt-get -q update +RUN apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:12) +RUN apt-get clean +RUN update-java-alternatives -s temurin-17-jdk-amd64 +RUN rm -rf /var/lib/apt/lists/* + +RUN locale-gen en_US.UTF-8 +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' +ENV PYTHONIOENCODING=utf-8 + +###### +# Set env vars required to build Hadoop +###### +ENV MAVEN_HOME=/opt/maven +ENV PATH="${PATH}:${MAVEN_HOME}/bin" +# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) +ENV JAVA_HOME /usr/lib/jvm/temurin-17-jdk-amd64 + +####### +# Set env vars for SpotBugs 4.2.2 +####### +ENV SPOTBUGS_HOME /opt/spotbugs + +####### +# Set env vars for Google Protobuf 3.21.12 +####### +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH="${PATH}:/opt/protobuf/bin" + +### +# Avoid out of memory errors in builds +### +ENV MAVEN_OPTS -Xms256m -Xmx3072m + +# Skip gpg verification when downloading Yetus via yetus-wrapper +ENV HADOOP_SKIP_YETUS_VERIFICATION true + +#### +# Install packages +#### +RUN pkg-resolver/install-maven.sh debian:12 +RUN pkg-resolver/install-spotbugs.sh debian:12 +RUN pkg-resolver/install-boost.sh debian:12 +RUN pkg-resolver/install-protobuf.sh debian:12 +RUN pkg-resolver/install-hadolint.sh debian:12 +RUN pkg-resolver/install-intel-isa-l.sh debian:12 + +### +# Everything past this point is either not needed for testing or breaks Yetus. +# So tell Yetus not to read the rest of the file: +# YETUS CUT HERE +### + +# Add a welcome message and environment checks. +COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh +RUN chmod 755 /root/hadoop_env_checks.sh +# hadolint ignore=SC2016 +RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc \ No newline at end of file diff --git a/dev-support/docker/Dockerfile_debian_13 b/dev-support/docker/Dockerfile_debian_13 new file mode 100644 index 0000000000000..21d9bb0fa7981 --- /dev/null +++ b/dev-support/docker/Dockerfile_debian_13 @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 ( +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for installing the necessary dependencies for building Hadoop. +# See BUILDING.txt. + +FROM debian:13 + +WORKDIR /root + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +##### +# Disable suggests/recommends +##### +RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/10disableextras + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_TERSE true + +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008,SC2046 +RUN apt-get -q update +RUN apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates +RUN apt-get -q install -y --no-install-recommends python3 +RUN echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list +RUN wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc +RUN apt-get -q update +RUN apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:13) +RUN apt-get clean +RUN update-java-alternatives -s temurin-17-jdk-amd64 +RUN rm -rf /var/lib/apt/lists/* + +RUN locale-gen en_US.UTF-8 +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' +ENV PYTHONIOENCODING=utf-8 + +###### +# Set env vars required to build Hadoop +###### +ENV MAVEN_HOME=/opt/maven +ENV PATH="${PATH}:${MAVEN_HOME}/bin" +# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) +ENV JAVA_HOME /usr/lib/jvm/temurin-17-jdk-amd64 + +####### +# Set env vars for SpotBugs 4.2.2 +####### +ENV SPOTBUGS_HOME /opt/spotbugs + +####### +# Set env vars for Google Protobuf 3.21.12 +####### +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH="${PATH}:/opt/protobuf/bin" + +### +# Avoid out of memory errors in builds +### +ENV MAVEN_OPTS -Xms256m -Xmx3072m + +# Skip gpg verification when downloading Yetus via yetus-wrapper +ENV HADOOP_SKIP_YETUS_VERIFICATION true + +#### +# Install packages +#### +RUN pkg-resolver/install-maven.sh debian:13 +RUN pkg-resolver/install-spotbugs.sh debian:13 +RUN pkg-resolver/install-boost.sh debian:13 +RUN pkg-resolver/install-protobuf.sh debian:13 +RUN pkg-resolver/install-hadolint.sh debian:13 +RUN pkg-resolver/install-intel-isa-l.sh debian:13 + +### +# Everything past this point is either not needed for testing or breaks Yetus. +# So tell Yetus not to read the rest of the file: +# YETUS CUT HERE +### + +# Add a welcome message and environment checks. +COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh +RUN chmod 755 /root/hadoop_env_checks.sh +# hadolint ignore=SC2016 +RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc \ No newline at end of file diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index 2fe367452d979..7a1acc7b9d42f 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -1,6 +1,8 @@ { "ant": { "debian:11": "ant", + "debian:12": "ant", + "debian:13": "ant", "ubuntu:focal": "ant", "ubuntu:focal::arch64": "ant", "ubuntu:noble": "ant", @@ -8,12 +10,16 @@ }, "apt-utils": { "debian:11": "apt-utils", + "debian:12": "apt-utils", + "debian:13": "apt-utils", "ubuntu:focal": "apt-utils", "ubuntu:noble": "apt-utils", "ubuntu:focal::arch64": "apt-utils" }, "automake": { "debian:11": "automake", + "debian:12": "automake", + "debian:13": "automake", "ubuntu:focal": "automake", "ubuntu:noble": "automake", "ubuntu:focal::arch64": "automake", @@ -21,6 +27,8 @@ }, "bats": { "debian:11": "bats", + "debian:12": "bats", + "debian:13": "bats", "ubuntu:focal": "bats", "ubuntu:noble": "bats", "ubuntu:focal::arch64": "bats", @@ -28,6 +36,8 @@ }, "build-essential": { "debian:11": "build-essential", + "debian:12": "build-essential", + "debian:13": "build-essential", "ubuntu:focal": "build-essential", "ubuntu:noble": "build-essential", "ubuntu:focal::arch64": "build-essential" @@ -37,6 +47,14 @@ "bzip2", "libbz2-dev" ], + "debian:12": [ + "bzip2", + "libbz2-dev" + ], + "debian:13": [ + "bzip2", + "libbz2-dev" + ], "ubuntu:focal": [ "bzip2", "libbz2-dev" @@ -56,12 +74,17 @@ }, "clang": { "debian:11": "clang", + "debian:12": "clang", + "debian:13": "clang", "ubuntu:focal": "clang", "ubuntu:noble": "clang", "ubuntu:focal::arch64": "clang", "rockylinux:8": "clang" }, "cmake": { + "debian:11": "cmake", + "debian:12": "cmake", + "debian:13": "cmake", "ubuntu:focal": "cmake", "ubuntu:noble": "cmake", "ubuntu:focal::arch64": "cmake" @@ -71,6 +94,14 @@ "curl", "libcurl4-openssl-dev" ], + "debian:12": [ + "curl", + "libcurl4-openssl-dev" + ], + "debian:13": [ + "curl", + "libcurl4-openssl-dev" + ], "ubuntu:focal": [ "curl", "libcurl4-openssl-dev" @@ -90,6 +121,8 @@ }, "doxygen": { "debian:11": "doxygen", + "debian:12": "doxygen", + "debian:13": "doxygen", "ubuntu:focal": "doxygen", "ubuntu:noble": "doxygen", "ubuntu:focal::arch64": "doxygen" @@ -102,6 +135,14 @@ "fuse", "libfuse-dev" ], + "debian:12": [ + "fuse3", + "libfuse3-dev" + ], + "debian:13": [ + "fuse3", + "libfuse3-dev" + ], "ubuntu:focal": [ "fuse", "libfuse-dev" @@ -127,6 +168,14 @@ "g++" ] }, + "debian:12": [ + "gcc", + "g++" + ], + "debian:13": [ + "gcc", + "g++" + ], "ubuntu:focal": [ "gcc", "g++" @@ -142,6 +191,8 @@ }, "git": { "debian:11": "git", + "debian:12": "git", + "debian:13": "git", "ubuntu:focal": "git", "ubuntu:noble": "git", "ubuntu:focal::arch64": "git", @@ -149,24 +200,32 @@ }, "gnupg-agent": { "debian:11": "gnupg-agent", + "debian:12": "gnupg-agent", + "debian:13": "gnupg-agent", "ubuntu:focal": "gnupg-agent", "ubuntu:noble": "gnupg-agent", "ubuntu:focal::arch64": "gnupg-agent" }, "hugo": { "debian:11": "hugo", + "debian:12": "hugo", + "debian:13": "hugo", "ubuntu:focal": "hugo", "ubuntu:noble": "hugo", "ubuntu:focal::arch64": "hugo" }, "libbcprov-java": { "debian:11": "libbcprov-java", + "debian:12": "libbcprov-java", + "debian:13": "libbcprov-java", "ubuntu:focal": "libbcprov-java", "ubuntu:noble": "libbcprov-java", "ubuntu:focal::arch64": "libbcprov-java" }, "libtool": { "debian:11": "libtool", + "debian:12": "libtool", + "debian:13": "libtool", "ubuntu:focal": "libtool", "ubuntu:noble": "libtool", "ubuntu:focal::arch64": "libtool", @@ -174,6 +233,8 @@ }, "openssl": { "debian:11": "libssl-dev", + "debian:12": "libssl-dev", + "debian:13": "libssl-dev", "ubuntu:focal": "libssl-dev", "ubuntu:noble": "libssl-dev", "ubuntu:focal::arch64": "libssl-dev", @@ -190,6 +251,14 @@ "libprotobuf-dev", "libprotoc-dev" ], + "debian:12": [ + "libprotobuf-dev", + "libprotoc-dev" + ], + "debian:13": [ + "libprotobuf-dev", + "libprotoc-dev" + ], "ubuntu:focal": [ "libprotobuf-dev", "libprotoc-dev" @@ -204,10 +273,14 @@ ] }, "procps": { - "debian:11": "procps" + "debian:11": "procps", + "debian:12": "procps", + "debian:13": "procps" }, "sasl": { "debian:11": "libsasl2-dev", + "debian:12": "libsasl2-dev", + "debian:13": "libsasl2-dev", "ubuntu:focal": "libsasl2-dev", "ubuntu:noble": "libsasl2-dev", "ubuntu:focal::arch64": "libsasl2-dev", @@ -215,6 +288,8 @@ }, "snappy": { "debian:11": "libsnappy-dev", + "debian:12": "libsnappy-dev", + "debian:13": "libsnappy-dev", "ubuntu:focal": "libsnappy-dev", "ubuntu:noble": "libsnappy-dev", "ubuntu:focal::arch64": "libsnappy-dev" @@ -224,6 +299,14 @@ "libzstd-dev", "zlib1g-dev" ], + "debian:12": [ + "libzstd-dev", + "zlib1g-dev" + ], + "debian:13": [ + "libzstd-dev", + "zlib1g-dev" + ], "ubuntu:focal": [ "libzstd-dev", "zlib1g-dev" @@ -243,16 +326,22 @@ }, "locales": { "debian:11": "locales", + "debian:12": "locales", + "debian:13": "locales", "ubuntu:focal": "locales", "ubuntu:noble": "locales", "ubuntu:focal::arch64": "locales" }, "libtirpc-devel": { + "debian:12": "libtirpc-dev", + "debian:13": "libtirpc-dev", "rockylinux:8": "libtirpc-devel", "ubuntu:noble": "libtirpc-dev" }, "make": { "debian:11": "make", + "debian:12": "make", + "debian:13": "make", "ubuntu:focal": "make", "ubuntu:noble": "make", "ubuntu:focal::arch64": "make", @@ -263,6 +352,14 @@ "openjdk-11-jdk", "openjdk-17-jdk" ], + "debian:12": [ + "temurin-17-jdk", + "temurin-24-jdk" + ], + "debian:13": [ + "temurin-17-jdk", + "temurin-24-jdk" + ], "ubuntu:focal": [ "temurin-24-jdk", "openjdk-8-jdk", @@ -286,6 +383,8 @@ }, "pinentry-curses": { "debian:11": "pinentry-curses", + "debian:12": "pinentry-curses", + "debian:13": "pinentry-curses", "ubuntu:focal": "pinentry-curses", "ubuntu:noble": "pinentry-curses", "ubuntu:focal::arch64": "pinentry-curses", @@ -293,6 +392,8 @@ }, "pkg-config": { "debian:11": "pkg-config", + "debian:12": "pkg-config", + "debian:13": "pkg-config", "ubuntu:focal": "pkg-config", "ubuntu:noble": "pkg-config", "ubuntu:focal::arch64": "pkg-config", @@ -306,6 +407,20 @@ "python3-setuptools", "python3-wheel" ], + "debian:12": [ + "python3", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + "python3-wheel" + ], + "debian:13": [ + "python3", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + "python3-wheel" + ], "ubuntu:focal": [ "python3", "python3-pip", @@ -337,6 +452,8 @@ }, "rsync": { "debian:11": "rsync", + "debian:12": "rsync", + "debian:13": "rsync", "ubuntu:focal": "rsync", "ubuntu:noble": "rsync", "ubuntu:focal::arch64": "rsync", @@ -344,6 +461,8 @@ }, "shellcheck": { "debian:11": "shellcheck", + "debian:12": "shellcheck", + "debian:13": "shellcheck", "ubuntu:focal": "shellcheck", "ubuntu:noble": "shellcheck", "ubuntu:focal::arch64": "shellcheck" @@ -353,12 +472,16 @@ }, "software-properties-common": { "debian:11": "software-properties-common", + + "ubuntu:focal": "software-properties-common", "ubuntu:noble": "software-properties-common", "ubuntu:focal::arch64": "software-properties-common" }, "sudo": { "debian:11": "sudo", + "debian:12": "sudo", + "debian:13": "sudo", "ubuntu:focal": "sudo", "ubuntu:noble": "sudo", "ubuntu:focal::arch64": "sudo", @@ -366,6 +489,8 @@ }, "valgrind": { "debian:11": "valgrind", + "debian:12": "valgrind", + "debian:13": "valgrind", "ubuntu:focal": "valgrind", "ubuntu:noble": "valgrind", "ubuntu:focal::arch64": "valgrind", @@ -373,6 +498,8 @@ }, "yasm": { "debian:11": "yasm", + "debian:12": "yasm", + "debian:13": "yasm", "ubuntu:focal": "yasm", "ubuntu:noble": "yasm", "ubuntu:focal::arch64": "yasm" diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json index 00f6c129b207d..3f556de5ad5b5 100644 --- a/dev-support/docker/pkg-resolver/platforms.json +++ b/dev-support/docker/pkg-resolver/platforms.json @@ -3,5 +3,7 @@ "ubuntu:focal::arch64", "ubuntu:noble", "rockylinux:8", - "debian:11" + "debian:11", + "debian:12", + "debian:13" ]