arm64 support

Author: alekitto

.gitignore

@@ -1,3 +1,5 @@
 build
+.kvm-images
 .installed-requirements
+.installed-qemu
 namibase/nami-linux-x64.tar.gz

.travis.yml

@@ -1,22 +1,77 @@
 language: bash
-sudo: required
-script: bash shellcheck && sudo bash buildall
-dist: xenial
+dist: focal
+virt: vm
+group: edge
+os: linux
 services:
   - docker
-before_install:
-  - docker version
-  # Fix for Ubuntu Xenial apt-daily.service triggering
-  # https://unix.stackexchange.com/questions/315502/how-to-disable-apt-daily-service-on-ubuntu-cloud-vm-image
-  - |
-    while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do
-      sleep 1
-    done
-  - sudo apt-get -qq update
-  - sudo apt-get install -y debian-archive-keyring debootstrap shellcheck
-deploy:
-  provider: script
-  script: bash pushall
-  skip_cleanup: true
-  on:
-    branch: master
+
+.build_job: &build_job
+  stage: build
+  before_install:
+    - docker version
+    # Fix for Ubuntu Xenial apt-daily.service triggering
+    # https://unix.stackexchange.com/questions/315502/how-to-disable-apt-daily-service-on-ubuntu-cloud-vm-image
+    - |
+      while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do
+        sleep 1
+      done
+    - sudo rm /usr/local/bin/jq
+  install:
+    - sudo make .installed-requirements
+  script: sudo bash buildone $DIST $PLATFORM
+  after_success:
+    - 'if [[ "$TRAVIS_BRANCH" == "master" && "$DIST" == "buster" ]] ; then sudo docker tag "bitnami/minideb:$DIST-$PLATFORM" "$BASENAME:latest-$PLATFORM" ; fi'
+    - 'if [[ "$TRAVIS_BRANCH" == "master" ]] ; then sudo bash pushone $DIST $PLATFORM ; fi'
+    - 'if [[ "$TRAVIS_BRANCH" == "master" && "$DIST" == "buster" ]] ; then sudo bash pushone latest $PLATFORM ; fi'
+
+jobs:
+  include:
+    - stage: shellcheck
+      install:
+        - sudo apt-get -qq update
+        - sudo apt-get install -y shellcheck
+      script: bash shellcheck
+    - <<: *build_job
+      arch: amd64
+      env:
+        - DIST=jessie PLATFORM=amd64
+    - <<: *build_job
+      arch: amd64
+      env:
+        - DIST=stretch PLATFORM=amd64
+    - <<: *build_job
+      arch: amd64
+      env:
+        - DIST=buster PLATFORM=amd64
+    - <<: *build_job
+      arch: arm64-graviton2
+      env:
+        - DIST=stretch PLATFORM=arm64
+    - <<: *build_job
+      arch: arm64-graviton2
+      env:
+        - DIST=buster PLATFORM=arm64
+    - stage: deploy
+      if: branch = master AND type = push
+      env:
+        - DISTS="stretch buster latest"
+      before_install: mkdir $HOME/.docker
+      install: 'echo "{ \"experimental\": \"enabled\" }" > $HOME/.docker/config.json'
+      script:
+        - |
+          if [ -n "${DOCKER_PASSWORD:-}" ]; then
+              docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"
+          fi
+
+          # Create and merge a PR to update minideb-extras
+          CIRCLE_CI_FUNCTIONS_URL=${CIRCLE_CI_FUNCTIONS_URL:-https://raw.githubusercontent.com/bitnami/test-infra/master/circle/functions}
+          source <(curl -sSL "$CIRCLE_CI_FUNCTIONS_URL")
+          for DIST in $DISTS; do
+            sudo docker manifest create bitnami/minideb:$DIST bitnami/minideb:$DIST-amd64 bitnami/minideb:$DIST-arm64
+            sudo docker manifest push bitnami/minideb:$DIST
+
+            # Use '.RepoDigests 0' for getting Dockerhub repo digest as it was the first pushed
+            DIST_REPO_DIGEST=$(docker image inspect --format '{{index .RepoDigests 0}}' "${BASENAME}:${DIST}")
+            update_minideb_derived "https://github.com/bitnami/minideb-runtimes" "$DIST" "$DIST_REPO_DIGEST"
+          done

Makefile

@@ -12,6 +12,11 @@ clean:
 clobber: clean
 	@${RM} .installed-requirements
 
+.installed-qemu:
+	@echo "Installing QEMU and required packages..."
+	@./install-qemu.sh
+	@touch $@
+
 .installed-requirements:
 	@echo "Installing required packages..."
 	@./pre-build.sh

README.md

@@ -53,7 +53,7 @@ We provide a Makefile to help you build Minideb locally. It should be run on a D
 $ sudo make
 ```
 
-To build an individual release (stretch, jessie or unstable)
+To build an individual release (stretch, buster or unstable)
 ```
 $ sudo make stretch
 ```

buildall

@@ -6,22 +6,32 @@ set -e
 set -u
 set -o pipefail
 
+arch=${1:-"amd64 arm64"}
+
 dist="jessie
 stretch
 buster
 "
 dist_with_snapshot="buster"
 
-for i in $dist; do
-    ./buildone "$i"
+for a in $arch; do
+  for i in $dist; do
+    if [[ "$a" != "amd64" && "$i" == "jessie" ]]; then
+      continue
+    fi
+
+    ./buildone "$i" "$a"
+  done
 done
 
 snapshot_id=$(./snapshot_id)
 if [ -n "$snapshot_id" ]; then
+  for a in $arch; do
     for i in $dist_with_snapshot; do
-        ./buildone_snapshot "$i" "$snapshot_id"
+        ./buildone_snapshot "$i" "$snapshot_id" "$a"
     done
 
-    mkdir -p build
-    echo "$snapshot_id" > build/snapshot_id
+    mkdir -p "build/$a"
+    echo "$snapshot_id" > "build/$a/snapshot_id"
+  done
 fi

buildone

@@ -43,8 +43,9 @@ log() {
 
 build() {
     DIST=$1
+    PLATFORM=${2:-amd64}
 
-    debian_snapshot_id=${2:-}
+    debian_snapshot_id=${3:-}
     if [ -n "$debian_snapshot_id" ]; then
         TAG="${DIST}-snapshot-${debian_snapshot_id}"
     else
@@ -64,7 +65,7 @@ build() {
     log "Building $BASENAME:$TAG"
     log "============================================"
     ./mkimage "build/$TAG.tar" "$DIST" "${debian_snapshot_id:-}"
-    built_image_id=$(./import "build/$TAG.tar" "$target_ts")
+    built_image_id=$(./import "build/$TAG.tar" "$target_ts" "$PLATFORM")
     log "============================================"
     log "Running tests for $BASENAME:$TAG"
     log "============================================"
@@ -73,7 +74,7 @@ build() {
     log "Rebuilding $BASENAME:$TAG to test reproducibility"
     log "============================================"
     ./mkimage "build/${TAG}-repro.tar" "$DIST" "${debian_snapshot_id:-}"
-    repro_image_id=$(./import "build/${TAG}-repro.tar" "$target_ts")
+    repro_image_id=$(./import "build/${TAG}-repro.tar" "$target_ts" "$PLATFORM")
     if [ "$repro_image_id" != "$built_image_id" ]; then
         log "$BASENAME:$TAG differs after a rebuild. Examine $built_image_id and $repro_image_id"
         log "to find the differences and fix the build to be reproducible again."
@@ -89,19 +90,19 @@ build() {
             ./dockerdiff "$pulled_image_id" "$built_image_id" || true
             # Re-import with the current timestamp so that the image shows
             # as new
-            built_image_id="$(./import "build/$TAG.tar" "$current_ts")"
+            built_image_id="$(./import "build/$TAG.tar" "$current_ts" "$PLATFORM")"
         else
             log "Image didn't change"
             return
         fi
     fi
-    docker tag "$built_image_id" "$BASENAME:$TAG"
-    log "Tagged $built_image_id as $BASENAME:$TAG"
+    docker tag "$built_image_id" "$BASENAME:$TAG-$PLATFORM"
+    log "Tagged $built_image_id as $BASENAME:$TAG-$PLATFORM"
 }
 
 if [ -z "$1" ]; then
     echo "You must specify the dist to build"
     exit 1
 fi
 
-build "${1}" "${2:-}"
+build "$@"

buildone_snapshot

@@ -6,5 +6,6 @@ set -o pipefail
 
 dist=${1:?dist arg is required}
 snapshot_id=${2:-$(./snapshot_id)}
+platform=${3:-amd64}
 
-./buildone "$dist" "$snapshot_id"
+./buildone "$dist" "$platform" "$snapshot_id"

import

@@ -12,19 +12,20 @@ set -e
 set -u
 set -o pipefail
 
-CONF_TEMPLATE='{"architecture":"amd64","comment":"from Bitnami with love","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":["/bin/bash"],"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"%TIMESTAMP%","docker_version":"1.13.0","history":[{"created":"%TIMESTAMP%","comment":"from Bitnami with love"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:%LAYERSUM%"]}}'
-MANIFEST_TEMPLATE='[{"Config":"%CONF_SHA%.json","RepoTags":null,"Layers":["%LAYERSUM%/layer.tar"]}]'
-
 SOURCE=${1:?Specify the tarball to import}
 TIMESTAMP=${2:?Specify the timestamp to use}
+PLATFORM=${3:?Specify the target platform}
+
+CONF_TEMPLATE='{"architecture":"%PLATFORM%","comment":"from Bitnami with love","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":["/bin/bash"],"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"%TIMESTAMP%","docker_version":"1.13.0","history":[{"created":"%TIMESTAMP%","comment":"from Bitnami with love"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:%LAYERSUM%"]}}'
+MANIFEST_TEMPLATE='[{"Config":"%CONF_SHA%.json","RepoTags":null,"Layers":["%LAYERSUM%/layer.tar"]}]'
 
 import() {
     local TDIR="$(mktemp -d)"
     local LAYERSUM="$(sha256sum $SOURCE | awk '{print $1}')"
     mkdir $TDIR/$LAYERSUM
     cp $SOURCE $TDIR/$LAYERSUM/layer.tar
     echo -n '1.0' > $TDIR/$LAYERSUM/VERSION
-    local CONF="$(echo -n "$CONF_TEMPLATE" | sed -e "s/%TIMESTAMP%/$TIMESTAMP/g" -e "s/%LAYERSUM%/$LAYERSUM/g")"
+    local CONF="$(echo -n "$CONF_TEMPLATE" | sed -e "s/%PLATFORM%/$PLATFORM/g" -e "s/%TIMESTAMP%/$TIMESTAMP/g" -e "s/%LAYERSUM%/$LAYERSUM/g")"
     local CONF_SHA="$(echo -n "$CONF" | sha256sum | awk '{print $1}')"
     echo -n "$CONF" > "$TDIR/${CONF_SHA}.json"
     local MANIFEST="$(echo -n "$MANIFEST_TEMPLATE" | sed -e "s/%CONF_SHA%/$CONF_SHA/g" -e "s/%LAYERSUM%/$LAYERSUM/g")"

install-qemu.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eu
+
+do_sudo() {
+  if [[ "0" == "$(id --user)" ]]; then
+    "$@"
+  else
+    sudo "$@"
+  fi
+}
+
+while do_sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do
+    sleep 1
+done
+
+do_sudo apt-get update
+do_sudo apt-get install -y qemu-kvm libvirt-bin qemu-utils genisoimage virtinst curl rsync qemu-system-x86 qemu-system-arm cloud-image-utils
+

pushone

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+DIST=${1:?Specify the distrubution name}
+PLATFORM=${2:-amd64}
+
+BASENAME=bitnami/minideb
+GCR_BASENAME=gcr.io/bitnami-containers/minideb
+QUAY_BASENAME=quay.io/bitnami/minideb
+
+if [ -n "${DOCKER_PASSWORD:-}" ]; then
+    docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"
+fi
+
+if [ -n "${QUAY_PASSWORD:-}" ]; then
+    docker login -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD" quay.io
+fi
+
+if [ -n "${GCR_KEY:-}" ]; then
+    gcloud auth activate-service-account "$GCR_EMAIL" --key-file <(echo "$GCR_KEY")
+fi
+
+ENABLE_DOCKER_CONTENT_TRUST=0
+if [ -n "${DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE:-}" ] && [ -n "${DOCKER_CONTENT_TRUST_REPOSITORY_KEY:-}" ]; then
+    tmpdir=$(mktemp -d)
+    (cd "${tmpdir}" && bash -c 'echo -n "${DOCKER_CONTENT_TRUST_REPOSITORY_KEY}" | base64 -d > key')
+    chmod 400 "${tmpdir}/key"
+    docker trust key load "${tmpdir}/key"
+    rm -rf "${tmpdir}"
+    export ENABLE_DOCKER_CONTENT_TRUST=1
+fi
+
+push() {
+    local dist="$1"
+    DOCKER_CONTENT_TRUST=${ENABLE_DOCKER_CONTENT_TRUST} docker push "${BASENAME}:${dist}"
+    docker push "${QUAY_BASENAME}:${dist}"
+    gcloud docker -- push "${GCR_BASENAME}:${dist}"
+}
+
+docker tag "${BASENAME}:${DIST}" "${QUAY_BASENAME}:${DIST}-${PLATFORM}"
+docker tag "${BASENAME}:${DIST}" "${GCR_BASENAME}:${DIST}-${PLATFORM}"
+push "$DIST-${PLATFORM}"
+