GH-32994: [Dev][Archery] Fix multi-arch Docker configuration (#50125)

pitrou · web-flow · commit 1c15d136b456 · 2026-06-10T14:37:45.000+02:00
### Rationale for this change Our Archery docker setup had support for running foreign-arch containers (for example a ARM64 container on a x86-64 native host, using emulation), but it failed working at some point. ### Are these changes tested? By existing CI runs, as well as locally (by running a couple ARM64 containers on a x86-64 native host). ### Are there any user-facing changes? No. * GitHub Issue: #32994 Authored-by: Antoine Pitrou <antoine@python.org> Signed-off-by: Antoine Pitrou <antoine@python.org>
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -96,7 +96,7 @@ jobs:
             runs-on: ubuntu-latest
             title: AMD64 Ubuntu 24.04 C++ ASAN UBSAN
             ubuntu: 24.04
-          - arch: arm64v8
+          - arch: arm64/v8
             clang-tools: 18
             image: ubuntu-cpp
             llvm: 18
diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml
@@ -221,7 +221,7 @@ jobs:
         platform:
           - arch: "amd64"
             runs-on: ubuntu-latest
-          - arch: "arm64v8"
+          - arch: "arm64/v8"
             runs-on: ubuntu-24.04-arm
     env:
       ARCH: ${{ matrix.platform.arch }}
@@ -584,7 +584,7 @@ jobs:
       ARROW_BUILD_STATIC: OFF
       ARROW_BUILD_TESTS: ON
       ARROW_BUILD_TYPE: release
-      # Turn Arrow CSV off to disable `find_package(Arrow)` check on MSVC CI. 
+      # Turn Arrow CSV off to disable `find_package(Arrow)` check on MSVC CI.
       # GH-49050 TODO: enable `find_package(Arrow)` check on MSVC CI.
       ARROW_CSV: OFF
       ARROW_DEPENDENCY_SOURCE: VCPKG
diff --git a/.hadolint.yaml b/.hadolint.yaml
@@ -22,4 +22,5 @@ ignored:
   - DL3018
   - DL3028 # Ruby gem version pinning
   - DL3007 # r-sanitizer must use latest
+  - DL3029 # Do not use --platform flag with FROM
   - DL3041 # Specify version with `dnf install -y <package>-<version>`.
diff --git a/ci/docker/almalinux-10-verify-rc.dockerfile b/ci/docker/almalinux-10-verify-rc.dockerfile
@@ -16,7 +16,7 @@
 # under the License.
 
 ARG arch=amd64
-FROM ${arch}/almalinux:10
+FROM --platform=linux/${arch} almalinux:10
 
 COPY dev/release/setup-rhel-rebuilds.sh /
 RUN /setup-rhel-rebuilds.sh && \
diff --git a/ci/docker/almalinux-8-verify-rc.dockerfile b/ci/docker/almalinux-8-verify-rc.dockerfile
@@ -16,7 +16,7 @@
 # under the License.
 
 ARG arch=amd64
-FROM ${arch}/almalinux:8
+FROM --platform=linux/${arch} almalinux:8
 
 COPY dev/release/setup-rhel-rebuilds.sh /
 RUN /setup-rhel-rebuilds.sh && \
diff --git a/ci/docker/alpine-linux-3.22-cpp.dockerfile b/ci/docker/alpine-linux-3.22-cpp.dockerfile
@@ -16,7 +16,7 @@
 # under the License.
 
 ARG arch=amd64
-FROM ${arch}/alpine:3.22
+FROM --platform=linux/${arch} alpine:3.22
 
 RUN apk add \
         apache-orc-dev \
diff --git a/ci/docker/alpine-linux-3.22-r.dockerfile b/ci/docker/alpine-linux-3.22-r.dockerfile
@@ -20,7 +20,7 @@
 # Replicates CRAN's Alpine environment as closely as possible
 
 ARG arch=amd64
-FROM ${arch}/alpine:3.22
+FROM --platform=linux/${arch} alpine:3.22
 
 # Install R and essential build tools
 # Note: bash is needed for Arrow CI scripts, even though CRAN's Alpine uses BusyBox
diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile
@@ -17,7 +17,8 @@
 
 ARG repo
 ARG arch
-FROM ${repo}:${arch}-conda
+ARG arch_short
+FROM --platform=linux/${arch} ${repo}:${arch_short}-conda
 
 COPY ci/scripts/install_minio.sh /arrow/ci/scripts
 RUN /arrow/ci/scripts/install_minio.sh latest /opt/conda
diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile
@@ -16,10 +16,10 @@
 # under the License.
 
 ARG repo
-ARG arch=amd64
-FROM ${repo}:${arch}-conda-cpp
+ARG arch
+ARG arch_short
+FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-cpp
 
-ARG arch=amd64
 # We need to synchronize the following values with the values in .env
 # and services.conda-integration in compose.yaml.
 ARG maven=3.9.9
diff --git a/ci/docker/conda-python-dask.dockerfile b/ci/docker/conda-python-dask.dockerfile
@@ -16,9 +16,10 @@
 # under the License.
 
 ARG repo
-ARG arch=amd64
+ARG arch
+ARG arch_short
 ARG python=3.10
-FROM ${repo}:${arch}-conda-python-${python}
+FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python}
 
 ARG dask=latest
 COPY ci/scripts/install_dask.sh /arrow/ci/scripts/
diff --git a/ci/docker/conda-python-emscripten.dockerfile b/ci/docker/conda-python-emscripten.dockerfile
@@ -17,8 +17,9 @@
 
 ARG repo
 ARG arch
+ARG arch_short
 ARG python="3.12"
-FROM ${repo}:${arch}-conda-python-${python}
+FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python}
 
 ARG selenium_version="4.41.0"
 ARG pyodide_version="0.26.0"
diff --git a/ci/docker/conda-python-hdfs.dockerfile b/ci/docker/conda-python-hdfs.dockerfile
@@ -16,9 +16,10 @@
 # under the License.
 
 ARG repo
-ARG arch=amd64
+ARG arch
+ARG arch_short
 ARG python=3.10
-FROM ${repo}:${arch}-conda-python-${python}
+FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python}
 
 ARG jdk=11
 ARG maven=3.9.9
diff --git a/ci/docker/conda-python-jpype.dockerfile b/ci/docker/conda-python-jpype.dockerfile
@@ -16,9 +16,10 @@
 # under the License.
 
 ARG repo
-ARG arch=amd64
+ARG arch
+ARG arch_short
 ARG python=3.10
-FROM ${repo}:${arch}-conda-python-${python}
+FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python}
 
 ARG jdk=11
 ARG maven=3.9.9
diff --git a/ci/docker/conda-python-pandas.dockerfile b/ci/docker/conda-python-pandas.dockerfile
@@ -16,9 +16,10 @@
 # under the License.
 
 ARG repo
-ARG arch=amd64
+ARG arch
+ARG arch_short
 ARG python=3.10
-FROM ${repo}:${arch}-conda-python-${python}
+FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python}
 
 ARG pandas=latest
 ARG numpy=latest
diff --git a/ci/docker/conda-python-spark.dockerfile b/ci/docker/conda-python-spark.dockerfile
@@ -16,9 +16,10 @@
 # under the License.
 
 ARG repo
-ARG arch=amd64
+ARG arch
+ARG arch_short
 ARG python=3.10
-FROM ${repo}:${arch}-conda-python-${python}
+FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-python-${python}
 
 ARG jdk=11
 ARG maven=3.9.9
diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile
@@ -17,7 +17,8 @@
 
 ARG repo
 ARG arch
-FROM ${repo}:${arch}-conda-cpp
+ARG arch_short
+FROM --platform=linux/${arch} ${repo}:${arch_short}-conda-cpp
 
 # install python specific packages
 ARG python=3.10
diff --git a/ci/docker/conda.dockerfile b/ci/docker/conda.dockerfile
@@ -16,7 +16,7 @@
 # under the License.
 
 ARG arch=amd64
-FROM ${arch}/ubuntu:22.04
+FROM --platform=linux/${arch} ubuntu:24.04
 
 # install build essentials
 RUN export DEBIAN_FRONTEND=noninteractive && \
diff --git a/ci/docker/debian-13-cpp.dockerfile b/ci/docker/debian-13-cpp.dockerfile
@@ -16,8 +16,7 @@
 # under the License.
 
 ARG arch=amd64
-FROM ${arch}/debian:13
-ARG arch
+FROM --platform=linux/${arch} debian:13
 
 ENV DEBIAN_FRONTEND noninteractive
 
diff --git a/ci/docker/debian-experimental-cpp.dockerfile b/ci/docker/debian-experimental-cpp.dockerfile
@@ -16,8 +16,7 @@
 # under the License.
 
 ARG arch=amd64
-FROM ${arch}/debian:experimental
-ARG arch
+FROM --platform=linux/${arch} debian:experimental
 
 ENV DEBIAN_FRONTEND noninteractive
 
diff --git a/ci/docker/fedora-42-cpp.dockerfile b/ci/docker/fedora-42-cpp.dockerfile
@@ -16,8 +16,7 @@
 # under the License.
 
 ARG arch
-FROM ${arch}/fedora:42
-ARG arch
+FROM --platform=linux/${arch} fedora:42
 
 # install dependencies
 RUN dnf update -y && \
diff --git a/ci/docker/fedora-42-r-clang.dockerfile b/ci/docker/fedora-42-r-clang.dockerfile
@@ -20,7 +20,7 @@
 # See: https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang
 
 ARG arch=amd64
-FROM ${arch}/fedora:42
+FROM --platform=linux/${arch} fedora:42
 
 # Install build dependencies
 RUN dnf update -y && \
diff --git a/ci/docker/linux-apt-c-glib.dockerfile b/ci/docker/linux-apt-c-glib.dockerfile
@@ -15,8 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+ARG arch
 ARG base
-FROM ${base}
+FROM --platform=linux/${arch} ${base}
 
 RUN apt-get update -y -q && \
     apt-get install -y -q \
diff --git a/ci/docker/linux-apt-python-3.dockerfile b/ci/docker/linux-apt-python-3.dockerfile
@@ -15,8 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+ARG arch
 ARG base
-FROM ${base}
+FROM --platform=linux/${arch} ${base}
 
 COPY python/requirements-build.txt \
      python/requirements-test.txt \
diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile
@@ -15,8 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+ARG arch
 ARG base
-FROM ${base}
+FROM --platform=linux/${arch} ${base}
 
 ARG tz="UTC"
 ENV TZ=${tz}
diff --git a/ci/docker/linux-apt-ruby.dockerfile b/ci/docker/linux-apt-ruby.dockerfile
@@ -16,8 +16,9 @@
 # under the License.
 
 # depends on a C GLib image
+ARG arch
 ARG base
-FROM ${base}
+FROM --platform=linux/${arch} ${base}
 
 COPY ruby/ /arrow/ruby/
 RUN bundle install --gemfile /arrow/ruby/Gemfile
diff --git a/ci/docker/linux-dnf-python-3.dockerfile b/ci/docker/linux-dnf-python-3.dockerfile
@@ -15,8 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+ARG arch
 ARG base
-FROM ${base}
+FROM --platform=linux/${arch} ${base}
 
 RUN dnf install -y \
         python3 \
diff --git a/ci/docker/python-wheel-manylinux-test.dockerfile b/ci/docker/python-wheel-manylinux-test.dockerfile
@@ -17,7 +17,7 @@
 
 ARG arch
 ARG python_image_tag
-FROM ${arch}/python:${python_image_tag}
+FROM --platform=linux/${arch} python:${python_image_tag}
 
 # pandas doesn't provide wheel for aarch64 yet, so cache the compiled
 # test dependencies in a docker image
diff --git a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile
@@ -15,8 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-ARG base=amd64/ubuntu:22.04
-FROM ${base}
+ARG arch=amd64
+ARG base=ubuntu:22.04
+FROM --platform=linux/${arch} ${base}
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile
@@ -15,8 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-ARG base=amd64/ubuntu:22.04
-FROM ${base}
+ARG arch=amd64
+ARG base=ubuntu:22.04
+FROM --platform=linux/${arch} ${base}
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
diff --git a/ci/docker/ubuntu-22.04-verify-rc.dockerfile b/ci/docker/ubuntu-22.04-verify-rc.dockerfile
@@ -16,7 +16,8 @@
 # under the License.
 
 ARG arch=amd64
-FROM ${arch}/ubuntu:22.04
+ARG base=ubuntu:22.04
+FROM --platform=linux/${arch} ${base}
 
 ENV DEBIAN_FRONTEND=noninteractive
 COPY dev/release/setup-ubuntu.sh /
diff --git a/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-24.04-cpp-minimal.dockerfile
@@ -15,8 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-ARG base=amd64/ubuntu:24.04
-FROM ${base}
+ARG arch=amd64
+ARG base=ubuntu:24.04
+FROM --platform=linux/${arch} ${base}
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile
@@ -15,8 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-ARG base=amd64/ubuntu:24.04
-FROM ${base}
+ARG arch=amd64
+ARG base=ubuntu:24.04
+FROM --platform=linux/${arch} ${base}
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
diff --git a/ci/docker/ubuntu-24.04-verify-rc.dockerfile b/ci/docker/ubuntu-24.04-verify-rc.dockerfile
@@ -16,7 +16,8 @@
 # under the License.
 
 ARG arch=amd64
-FROM ${arch}/ubuntu:24.04
+ARG base=ubuntu:24.04
+FROM --platform=linux/${arch} ${base}
 
 ENV DEBIAN_FRONTEND=noninteractive
 COPY dev/release/setup-ubuntu.sh /
diff --git a/compose.yaml b/compose.yaml
diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py
diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml
