From e1255e5d84112b2a4448a65ed9f9686f8bb5f26f Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Thu, 2 Jan 2025 18:22:26 +0100 Subject: [PATCH 1/4] Move description to create a dual host builder instance to CONTRIBUTING.md And add a buildkit config which disables the garbage collection of the cache. Our images are too many and too fat, so that it gets deleted half way otherwise. --- CONTRIBUTING.md | 43 +++++++++++++++++++++++++++++++++++++++++ build/buildkitd.toml | 2 ++ mingw64-ucrt/README.md | 44 ------------------------------------------ 3 files changed, 45 insertions(+), 44 deletions(-) create mode 100644 build/buildkitd.toml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1618c8f..a7972ad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,3 +53,46 @@ docker buildx create --use --driver=docker-container bundle exec rake build ``` + +### Create builder instance for two architectures + +Building with qemu emulation fails currently with a segfault, so that it must be built by a builder instance with at least one remote node for the other architecture. +Building on native hardware is also much faster (~45 minutes) than on qemu. +A two-nodes builder requires obviously a ARM and a Intel/AMD device. +It can be created like this: + +```sh +# Make sure the remote instance can be connected +$ docker -H ssh://isa info + +# Create a new builder with the local instance +# Disable the garbage collector by the config file +$ docker buildx create --name isayoga --config build/buildkitd.toml + +# Add the remote instance +$ docker buildx create --name isayoga --config build/buildkitd.toml --append ssh://isa + +# They are inactive from the start +$ docker buildx ls +NAME/NODE DRIVER/ENDPOINT STATUS BUILDKIT PLATFORMS +isayoga docker-container + \_ isayoga0 \_ unix:///var/run/docker.sock inactive + \_ isayoga1 \_ ssh://isa inactive +default* docker + \_ default \_ default running v0.13.2 linux/arm64 + +# Bootstrap the instances +$ docker buildx inspect --bootstrap --builder isayoga + +# Set the new builder as default +$ docker buildx use isayoga + +# Now it should be default and in state "running" +$ docker buildx ls +NAME/NODE DRIVER/ENDPOINT STATUS BUILDKIT PLATFORMS +isayoga* docker-container + \_ isayoga0 \_ unix:///var/run/docker.sock running v0.18.2 linux/arm64 + \_ isayoga1 \_ ssh://isa running v0.18.2 linux/amd64, linux/amd64/v2, linux/amd64/v3, linux/386 +default docker + \_ default \_ default running v0.13.2 linux/arm64 +``` diff --git a/build/buildkitd.toml b/build/buildkitd.toml new file mode 100644 index 0000000..b1770cd --- /dev/null +++ b/build/buildkitd.toml @@ -0,0 +1,2 @@ +[worker.oci] + gc = false diff --git a/mingw64-ucrt/README.md b/mingw64-ucrt/README.md index c62673e..efd2d55 100644 --- a/mingw64-ucrt/README.md +++ b/mingw64-ucrt/README.md @@ -12,47 +12,3 @@ They are built by the following command: ```sh docker buildx build . -t larskanis/mingw64-ucrt:20.04 --platform linux/arm64,linux/amd64 --push ``` - - -Create builder instance for two architectures ------------------- - -Building with qemu emulation fails currently with a segfault, so that it must be built by a builder instance with at least one remote node for the other architecture. -Building on native hardware is also much faster (~30 minutes) than on qemu. -A two-nodes builder requires obviously a ARM and a Intel/AMD device. -It can be created like this: - -```sh -# Make sure the remote instance can be connected -$ docker -H ssh://isa info - -# Create a new builder with the local instance -$ docker buildx create --name isayoga - -# Add the remote instance -$ docker buildx create --name isayoga --append ssh://isa - -# They are inactive from the start -$ docker buildx ls -NAME/NODE DRIVER/ENDPOINT STATUS BUILDKIT PLATFORMS -isayoga docker-container - \_ isayoga0 \_ unix:///var/run/docker.sock inactive - \_ isayoga1 \_ ssh://isa inactive -default* docker - \_ default \_ default running v0.13.2 linux/arm64 - -# Bootstrap the instances -$ docker buildx inspect --bootstrap --builder isayoga - -# Set the new builder as default -$ docker buildx use isayoga - -# Now it should be default and in state "running" -$ docker buildx ls -NAME/NODE DRIVER/ENDPOINT STATUS BUILDKIT PLATFORMS -isayoga* docker-container - \_ isayoga0 \_ unix:///var/run/docker.sock running v0.18.2 linux/arm64 - \_ isayoga1 \_ ssh://isa running v0.18.2 linux/amd64, linux/amd64/v2, linux/amd64/v3, linux/386 -default docker - \_ default \_ default running v0.13.2 linux/arm64 -``` From 6998c5e3ba8cc0af41a981e8dc5c2d380d02afce Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Thu, 2 Jan 2025 18:00:03 +0100 Subject: [PATCH 2/4] Add support for multiple host achitectures This way we can provide images for Intel/AMD based as well as ARM based systems. Image building is now prefixed by the host architecture "arm" or "x86" and the local host arch is loaded after build: ``` rake build:arm:arm-linux-musl # Build and load image for platform arm-linux-musl on linux/arm64 rake build:arm:arm64-darwin # Build and load image for platform arm64-darwin on linux/arm64 rake build:arm:jruby # Build and load image for JRuby on linux/arm64 ``` The foreign architecture is only built: ``` rake build:x86:arm-linux-musl # Build image for platform arm-linux-musl on linux/amd64 rake build:x86:arm64-darwin # Build image for platform arm64-darwin on linux/amd64 rake build:x86:jruby # Build image for JRuby on linux/amd64 ``` Only `docker buildx build` supports multiple platforms, so that it is used now, instead of the classic `docker build`. The number of images doubles with this patch, so that I get errors building all in parallel. Therefore the number of parallel tasks should be limited like: ``` rake -j10 build:all ``` --- Rakefile | 129 +++++++++++++++++++++++---------- build/parallel_docker_build.rb | 28 ++++--- 2 files changed, 105 insertions(+), 52 deletions(-) diff --git a/Rakefile b/Rakefile index 65f3bb8..2718270 100644 --- a/Rakefile +++ b/Rakefile @@ -8,6 +8,27 @@ CLEAN.include("tmp") RakeCompilerDock::GemHelper.install_tasks +def build_mri_images(platforms, host_platforms, output: ) + plats = host_platforms.map(&:first).join(",") + platforms.each do |platform, _| + sdf = "tmp/docker/Dockerfile.mri.#{platform}.#{host_platforms.first[1]}" + image_name = RakeCompilerDock::Starter.container_image_name(platform: platform) + + RakeCompilerDock.docker_build(sdf, tag: image_name, platform: plats, output: output) + + if image_name.include?("linux-gnu") + RakeCompilerDock.docker_build(sdf, tag: image_name.sub("linux-gnu", "linux"), platform: plats, output: output) + end + end +end + +def build_jruby_images(host_platforms, output: ) + image_name = RakeCompilerDock::Starter.container_image_name(rubyvm: "jruby") + plats = host_platforms.map(&:first).join(",") + sdf = "tmp/docker/Dockerfile.jruby.#{host_platforms.first[1]}" + RakeCompilerDock.docker_build(sdf, tag: image_name, platform: plats, output: output) +end + platforms = [ # tuple is [platform, target] ["aarch64-linux-gnu", "aarch64-linux-gnu"], @@ -25,47 +46,84 @@ platforms = [ ["x86_64-linux-musl", "x86_64-unknown-linux-musl"], ] +host_platforms = [ + # tuple is [docker platform, rake task, RUBY_PLATFORM matcher] + ["linux/amd64", "x86", /^x86_64|^x64|^amd64/], + ["linux/arm64", "arm", /^aarch64|arm64/], +] +local_platform = host_platforms.find { |_,_,reg| reg =~ RUBY_PLATFORM } or + raise("RUBY_PLATFORM #{RUBY_PLATFORM} is not supported as host") + namespace :build do - platforms.each do |platform, target| - sdf = "Dockerfile.mri.#{platform}" + mkdir_p "tmp/docker" + + host_platforms.each do |docker_platform, rake_platform| + namespace rake_platform do - desc "Build image for platform #{platform}" - task platform => sdf - task sdf do - image_name = RakeCompilerDock::Starter.container_image_name(platform: platform) - sh(*RakeCompilerDock.docker_build_cmd(platform), "-t", image_name, "-f", "Dockerfile.mri.#{platform}", ".") - if image_name.include?("linux-gnu") - sh("docker", "tag", image_name, image_name.sub("linux-gnu", "linux")) + platforms.each do |platform, target| + sdf = "tmp/docker/Dockerfile.mri.#{platform}.#{rake_platform}" + df = ERB.new(File.read("Dockerfile.mri.erb"), trim_mode: ">").result(binding) + File.write(sdf, df) + CLEAN.include(sdf) + end + sdf = "tmp/docker/Dockerfile.jruby.#{rake_platform}" + df = File.read("Dockerfile.jruby") + File.write(sdf, df) + + builder = RakeCompilerDock::ParallelDockerBuild.new(platforms.map{|pl, _| "tmp/docker/Dockerfile.mri.#{pl}.#{rake_platform}" } + ["tmp/docker/Dockerfile.jruby.#{rake_platform}"], workdir: "tmp/docker", task_prefix: "common-#{rake_platform}-", platform: docker_platform) + + platforms.each do |platform, target| + sdf = "tmp/docker/Dockerfile.mri.#{platform}.#{rake_platform}" + + if docker_platform == local_platform[0] + # Load image after build on local platform only + desc "Build and load image for platform #{platform} on #{docker_platform}" + task platform => sdf do + build_mri_images([platform], [local_platform], output: 'load') + end + else + desc "Build image for platform #{platform} on #{docker_platform}" + task platform => sdf + end + multitask :all => platform end - end - df = ERB.new(File.read("Dockerfile.mri.erb"), trim_mode: ">").result(binding) - File.write(sdf, df) - CLEAN.include(sdf) + sdf = "tmp/docker/Dockerfile.jruby.#{rake_platform}" + if docker_platform == local_platform[0] + # Load image after build on local platform only + desc "Build and load image for JRuby on #{docker_platform}" + task :jruby => sdf do + build_jruby_images([local_platform], output: 'load') + end + else + desc "Build image for JRuby on #{docker_platform}" + task :jruby => sdf + end + multitask :all => :jruby + end + desc "Build all images on #{docker_platform} in parallel" + task rake_platform => "#{rake_platform}:all" end - desc "Build image for JRuby" - task :jruby => "Dockerfile.jruby" - task "Dockerfile.jruby" do - image_name = RakeCompilerDock::Starter.container_image_name(rubyvm: "jruby") - sh(*RakeCompilerDock.docker_build_cmd("jruby"), "-t", image_name, "-f", "Dockerfile.jruby", ".") + all_mri_images = host_platforms.flat_map do |_, rake_platform| + platforms.map do |platform, | + "#{rake_platform}:#{platform}" + end end - - RakeCompilerDock::ParallelDockerBuild.new(platforms.map{|pl, _| "Dockerfile.mri.#{pl}" } + ["Dockerfile.jruby"], workdir: "tmp/docker") - - desc "Build images for all MRI platforms in parallel" + desc "Build images for all MRI platforms and hosts in parallel" if ENV['RCD_USE_BUILDX_CACHE'] - task :mri => platforms.map(&:first) + task :mri => all_mri_images else - multitask :mri => platforms.map(&:first) + multitask :mri => all_mri_images end - desc "Build images for all platforms in parallel" + all_images = all_mri_images + host_platforms.map { |_, pl| "#{pl}:jruby" } + desc "Build images for all platforms and hosts in parallel" if ENV['RCD_USE_BUILDX_CACHE'] - task :all => platforms.map(&:first) + ["jruby"] + task :all => all_images else - multitask :all => platforms.map(&:first) + ["jruby"] + multitask :all => all_images end end @@ -115,18 +173,9 @@ task :update_lists do end namespace :release do - desc "push all docker images" - task :images do - image_name = RakeCompilerDock::Starter.container_image_name(rubyvm: "jruby") - sh("docker", "push", image_name) - - platforms.each do |platform, _| - image_name = RakeCompilerDock::Starter.container_image_name(platform: platform) - sh("docker", "push", image_name) - - if image_name.include?("linux-gnu") - sh("docker", "push", image_name.sub("linux-gnu", "linux")) - end - end + desc "Push all docker images on #{host_platforms.map(&:first).join(",")}" + task :images => "build:all" do + build_jruby_images(host_platforms, output: 'push') + build_mri_images(platforms, host_platforms, output: 'push') end end diff --git a/build/parallel_docker_build.rb b/build/parallel_docker_build.rb index 3e5c922..1854287 100644 --- a/build/parallel_docker_build.rb +++ b/build/parallel_docker_build.rb @@ -14,17 +14,29 @@ def docker_build_cmd(platform=nil) return nil end else - ENV['RCD_DOCKER_BUILD'] || "docker build" + ENV['RCD_DOCKER_BUILD'] || "docker buildx build" end Shellwords.split(cmd) end + + # Run an intermediate dockerfile without tag + # + # The layers will be reused in subsequent builds, even if they run in parallel. + def docker_build(filename, tag: nil, output: false, platform: ) + cmd = docker_build_cmd + return if cmd.nil? + tag_args = ["-t", tag] if tag + push_args = ["--push"] if output == 'push' + push_args = ["--load"] if output == 'load' + Class.new.extend(FileUtils).sh(*cmd, "-f", filename, ".", "--platform", platform, *tag_args, *push_args) + end end # Run docker builds in parallel, but ensure that common docker layers are reused class ParallelDockerBuild include Rake::DSL - def initialize(dockerfiles, workdir: "tmp/docker", inputdir: ".", task_prefix: "common-") + def initialize(dockerfiles, workdir: "tmp/docker", inputdir: ".", task_prefix: "common-", platform: "local") FileUtils.mkdir_p(workdir) files = parse_dockerfiles(dockerfiles, inputdir) @@ -34,6 +46,7 @@ def initialize(dockerfiles, workdir: "tmp/docker", inputdir: ".", task_prefix: " # pp vcs define_common_tasks(vcs, workdir, task_prefix) + @platform = platform end # Read given dockerfiles from inputdir and split into a list of commands. @@ -96,7 +109,7 @@ def define_common_tasks(vcs, workdir, task_prefix, plines=[]) fn = "#{task_prefix}#{Digest::SHA1.hexdigest(files.join)}" File.write(File.join(workdir, fn), (plines + lines).join) task fn do - docker_build(fn, workdir) + RakeCompilerDock.docker_build(File.join(workdir, fn), platform: @platform) end nfn = define_common_tasks(nvcs, workdir, task_prefix, plines + lines) @@ -109,14 +122,5 @@ def define_common_tasks(vcs, workdir, task_prefix, plines=[]) fn end end - - # Run an intermediate dockerfile without tag - # - # The layers will be reused in subsequent builds, even if they run in parallel. - def docker_build(filename, workdir) - cmd = RakeCompilerDock.docker_build_cmd - return if cmd.nil? - sh(*RakeCompilerDock.docker_build_cmd, "-f", File.join(workdir, filename), ".") - end end end From 6bbb52c94ea17e76cf9c604421f69b764da8ae5f Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 3 Jan 2025 18:36:14 +0100 Subject: [PATCH 3/4] CI: Adjust rake tasks regarding host architecture --- .github/workflows/ci.yml | 2 +- .github/workflows/publish-images.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 765bc86..b48ef23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: - name: Build docker image run: | docker buildx create --driver docker-container --use - bundle exec rake build:${{ matrix.platform }} RCD_DOCKER_BUILD="docker buildx build --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new --load" + bundle exec rake build:x86:${{ matrix.platform }} RCD_DOCKER_BUILD="docker buildx build --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new" docker images - name: Update and prune docker buildx layer cache run: | diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index 535e587..85ec8f7 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -57,10 +57,10 @@ jobs: ' | tee -a $GITHUB_OUTPUT - name: Build docker image env: - RCD_DOCKER_BUILD: docker buildx build --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new --load + RCD_DOCKER_BUILD: docker buildx build --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new run: | docker buildx create --driver docker-container --use - bundle exec rake build:${{matrix.platform}} + bundle exec rake build:x86:${{matrix.platform}} # move build cache and remove outdated layers rm -rf tmp/build-cache mv tmp/build-cache-new tmp/build-cache From a6259b1ca75482865bfc3ec9bf9ccd2175e5aea8 Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Sat, 4 Jan 2025 22:03:14 +0100 Subject: [PATCH 4/4] CI: Try to use macos-latest on arm64 to build arm images --- .github/workflows/ci.yml | 85 ++++++---------------------------------- 1 file changed, 12 insertions(+), 73 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b48ef23..762b162 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,8 +4,6 @@ concurrency: cancel-in-progress: true on: workflow_dispatch: - schedule: - - cron: "0 5 * * 3" # At 05:00 on Wednesday # https://crontab.guru/#0_5_*_*_3 push: branches: - main @@ -17,28 +15,6 @@ on: - "*" jobs: - build_source_gem: - name: build source - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.3" - working-directory: test/rcd_test - bundler-cache: true - - - name: Build source gem - run: | - cd test/rcd_test/ - bundle exec rake gem - - - name: Upload source gem - uses: actions/upload-artifact@v4 - with: - name: gem-ruby - path: test/rcd_test/pkg/rcd_test-?.?.?.gem # e.g. rcd_test-1.0.0.gem - build_native_gem: name: build native strategy: @@ -46,33 +22,21 @@ jobs: matrix: include: - platform: aarch64-linux-gnu - alias: aarch64-linux - - platform: aarch64-linux-musl - - platform: arm-linux-gnu - alias: arm-linux - - platform: arm-linux-musl - - platform: arm64-darwin - - platform: jruby - - platform: x64-mingw-ucrt - static: true - - platform: x64-mingw32 - static: true - - platform: x86-linux-gnu - alias: x86-linux - - platform: x86-linux-musl - - platform: x86-mingw32 - - platform: x86_64-darwin - - platform: x86_64-linux-gnu - alias: x86_64-linux - - platform: x86_64-linux-musl - runs-on: ubuntu-latest + runs-on: macos-latest steps: - uses: actions/checkout@v4 - - uses: ruby/setup-ruby@v1 + - uses: ruby/setup-ruby-pkgs@v1 with: ruby-version: "3.3" - bundler-cache: true - + brew: "docker docker-buildx colima" + - name: enable buildx command + run: | + mkdir -p ~/.docker + echo '{"cliPluginsExtraDirs": ["/opt/homebrew/lib/docker/cli-plugins"]}' > ~/.docker/config.json + cat ~/.docker/config.json + - name: start colima + run: | + colima start - name: Fetch docker buildx layer cache uses: actions/cache@v4 with: @@ -82,7 +46,7 @@ jobs: - name: Build docker image run: | docker buildx create --driver docker-container --use - bundle exec rake build:x86:${{ matrix.platform }} RCD_DOCKER_BUILD="docker buildx build --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new" + bundle exec rake build:arm:${{ matrix.platform }} RCD_DOCKER_BUILD="docker buildx build --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new" docker images - name: Update and prune docker buildx layer cache run: | @@ -137,31 +101,6 @@ jobs: name: gem-${{ matrix.alias }} path: test/rcd_test/pkg/*-*-*.gem - test_source_gem: - name: source gem - needs: build_source_gem - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - ruby: ["3.4.0-rc1", "3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby }} - - name: Download source gem - uses: actions/download-artifact@v4 - with: - name: gem-ruby - - name: Test source gem - run: | - gem install --local *.gem --verbose - cd test/rcd_test/ - bundle install - ruby -rrcd_test -S rake test - test-x86_64-linux-setup-ruby: name: "${{ matrix.platform }} setup-ruby(${{ matrix.ruby }})" needs: build_native_gem