From 4e176488b5081928a56f45907da40827efb3e798 Mon Sep 17 00:00:00 2001 From: James Healy Date: Sun, 10 Nov 2024 18:26:52 +1100 Subject: [PATCH 1/7] Update rails-ci pipeline sto run on Buildkite hosted agents An experiment in changing the rails CI pipeline from "self-hosted" agents to "hosted" agents, a recently release Buildkite feature [1]. The hosted agents linux environment is superficially quite similar to the Elastic Stack for AWS, so the required changes are fairly minimal. Roughly half the changes are to take advantage of some performance optimisations available on hosted agents (like cache volumes, and remote buildkit builders with cache that last across builds). The essential changes: * Read the OCI registry from the environment rather than hard code an ECR registry. The current self-hosted agents run in AWS and can access ECR, but the hosted agent environment has access to its own registry specifically for use cases like this - building an image at the start of the build and then reusing it in later jobs * Changing the queue from `default` or `builder`, to `hosted` Optimisations: * There's no need to use the docker-compose plugins cache_from and image_name shenanigans. The images built at the start of each build use a remote buildkit builder with cache that is s hared between builds. The cache is typically warm, and when it is the image build time drops from ~2 mins to ~18sec * Use plain buildkit to build the images, without the docker compose plugin. This avoids the image being exported from buildkit to docker, and when the buildkit cache is warm the jobs complete in as little as 18s. This bypasses the docker-compse built in support for separating building and running, but the docker-compose.yml already kinda bypasses that by hard coding the image used in the run jobs (using the IMAGE_NAME env var) * Create a cache volume for ruby gems that are installed in docker during the initial step. This shaves ~30s off the build time [1] https://buildkite.com/docs/pipelines/hosted-agents/overview --- lib/buildkite/config/build_context.rb | 2 +- lib/buildkite/config/docker_build.rb | 17 ++--------------- lib/buildkite/config/rake_command.rb | 3 +-- pipelines/rails-ci/initial.yml | 17 +++++++++++++++-- pipelines/rails-ci/pipeline.rb | 2 +- 5 files changed, 20 insertions(+), 21 deletions(-) diff --git a/lib/buildkite/config/build_context.rb b/lib/buildkite/config/build_context.rb index c5f77603..9f6874cf 100644 --- a/lib/buildkite/config/build_context.rb +++ b/lib/buildkite/config/build_context.rb @@ -190,7 +190,7 @@ def min_ruby end def remote_image_base - "973266071021.dkr.ecr.us-east-1.amazonaws.com/#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds" + ENV.fetch("REGISTRY") + "/#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds" end end end diff --git a/lib/buildkite/config/docker_build.rb b/lib/buildkite/config/docker_build.rb index b43f6301..ff64c83d 100644 --- a/lib/buildkite/config/docker_build.rb +++ b/lib/buildkite/config/docker_build.rb @@ -30,11 +30,7 @@ def cache_from(build_context) end def build_push(build_context) - [ - build_context.local_branch =~ /:/ ? - build_context.image_name_for("pr-#{build_context.pull_request}") : - build_context.image_name_for("br-#{build_context.local_branch}"), - ] + build_context.image_name_for(build_context.build_id, prefix: nil) end end @@ -66,20 +62,11 @@ def builder(ruby) compressed: ".buildkite.tgz" } - plugin :docker_compose, { - build: "base", - config: ".buildkite/docker-compose.yml", - env: %w[PRE_STEPS RACK], - "image-name" => build_context.ruby.image_name_for(build_context.build_id), - "cache-from" => cache_from(build_context), - push: build_push(build_context), - "image-repository" => build_context.image_base, - } + command "docker build --push --build-arg RUBY_IMAGE=#{build_context.ruby.ruby_image} --tag #{build_push(build_context)} --file .buildkite/Dockerfile ." env({ BUNDLER: build_context.bundler, RUBYGEMS: build_context.rubygems, - RUBY_IMAGE: build_context.ruby.ruby_image, encrypted_0fb9444d0374_key: nil, encrypted_0fb9444d0374_iv: nil }) diff --git a/lib/buildkite/config/rake_command.rb b/lib/buildkite/config/rake_command.rb index 7c78a9f8..1b33996b 100644 --- a/lib/buildkite/config/rake_command.rb +++ b/lib/buildkite/config/rake_command.rb @@ -52,8 +52,7 @@ def install_plugins(service = "default", env = nil, dir = ".") plugin :docker_compose, { "env" => env, "run" => service, - "pull" => service, - "pull-retries" => 3, + "tty" => "true", "config" => ".buildkite/docker-compose.yml", "shell" => ["runner", *dir], }.compact diff --git a/pipelines/rails-ci/initial.yml b/pipelines/rails-ci/initial.yml index 299245db..92b3b9dc 100644 --- a/pipelines/rails-ci/initial.yml +++ b/pipelines/rails-ci/initial.yml @@ -1,9 +1,15 @@ # This file is never read -- it's just a copy of the pipeline's # configuration in the Buildkite UI. +env: + CONFIG_REPO: "https://github.com/yob/buildkite-config" + CONFIG_BRANCH: "hosted" steps: - name: ":pipeline: rails-initial-pipeline" command: | + echo "Fetching registry details" + export REGISTRY="$$(nsc workspace describe -o json -k registry_url)" + PATH=/bin:/usr/bin set -e @@ -30,17 +36,22 @@ steps: echo "Fetching pull-request metadata:" (docker run --rm \ -v "$$PWD":/app:ro -w /app \ + -v "$$PWD/cache/bundler":/usr/local/bundle \ -e GITHUB_PUBLIC_REPO_TOKEN \ -e BUILDKITE_REPO \ -e BUILDKITE_PULL_REQUEST \ ruby:latest \ .buildkite/bin/fetch-pr > .buildkite/tmp/.pr-meta.json) || true + echo "Generating pipeline:" sh -c "$$PIPELINE_COMMAND" ([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true - + cache: + paths: + - "cache/bundler" + name: "rails-initial-bundler-cache" plugins: - artifacts#v1.9.3: upload: ".dockerignore" @@ -58,6 +69,7 @@ steps: PIPELINE_COMMAND: >- docker run --rm -v "$$PWD":/app:ro -w /app + -v "$$PWD/cache/bundler":/usr/local/bundle -e CI -e BUILDKITE -e BUILDKITE_AGENT_META_DATA_QUEUE @@ -72,9 +84,10 @@ steps: -e DOCKER_IMAGE -e RUN_QUEUE -e QUEUE + -e REGISTRY ruby:latest .buildkite/bin/pipeline-generate rails-ci | buildkite-agent pipeline upload timeout_in_minutes: 5 agents: - queue: "${QUEUE-builder}" + queue: hosted diff --git a/pipelines/rails-ci/pipeline.rb b/pipelines/rails-ci/pipeline.rb index f740dcbb..3e3dd56c 100644 --- a/pipelines/rails-ci/pipeline.rb +++ b/pipelines/rails-ci/pipeline.rb @@ -7,7 +7,7 @@ use Buildkite::Config::RakeCommand use Buildkite::Config::RubyGroup - plugin :docker_compose, "docker-compose#v4.16.0" + plugin :docker_compose, "docker-compose#v5.4.1" plugin :artifacts, "artifacts#v1.9.3" if build_context.nightly? From c018bffb8a1bd11231a1ade7b8d16a31ea9e56d7 Mon Sep 17 00:00:00 2001 From: zzak Date: Fri, 14 Feb 2025 14:47:26 +0900 Subject: [PATCH 2/7] Add bundler cache for pipeline generation steps --- pipelines/docs-preview/initial.yml | 5 +++++ pipelines/rails-ci-nightly/initial.yml | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/pipelines/docs-preview/initial.yml b/pipelines/docs-preview/initial.yml index fc7b702c..6fff9d27 100644 --- a/pipelines/docs-preview/initial.yml +++ b/pipelines/docs-preview/initial.yml @@ -30,6 +30,10 @@ steps: ([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true + cache: + paths: + - "cache/bundler" + name: "docs-preview-initial-bundler-cache" plugins: - artifacts#v1.9.3: upload: ".dockerignore" @@ -48,6 +52,7 @@ steps: PIPELINE_COMMAND: >- docker run --rm -v "$$PWD":/app:ro -w /app + -v "$$PWD/cache/bundler":/usr/local/bundle -e CI -e BUILDKITE -e BUILDKITE_AGENT_META_DATA_QUEUE diff --git a/pipelines/rails-ci-nightly/initial.yml b/pipelines/rails-ci-nightly/initial.yml index 4e585930..0df1e3ba 100644 --- a/pipelines/rails-ci-nightly/initial.yml +++ b/pipelines/rails-ci-nightly/initial.yml @@ -30,6 +30,11 @@ steps: ([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true + cache: + paths: + - "cache/bundler" + name: "rails-initial-bundler-cache" + plugins: - artifacts#v1.9.3: upload: ".dockerignore" @@ -48,6 +53,7 @@ steps: PIPELINE_COMMAND: >- docker run --rm -v "$$PWD":/app:ro -w /app + -v "$$PWD/cache/bundler":/usr/local/bundle -e RAILS_CI_NIGHTLY -e CI -e BUILDKITE From e980bc9eff236d08fc5d7a0c0d61aa9c29776521 Mon Sep 17 00:00:00 2001 From: zzak Date: Fri, 14 Feb 2025 14:52:46 +0900 Subject: [PATCH 3/7] Use cluster-secrets for mainline and docs-preview deploys --- lib/buildkite/config/rake_command.rb | 12 +++++++++--- pipelines/docs-preview/pipeline.rb | 7 +++++++ pipelines/rails-ci/pipeline.rb | 1 + 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/buildkite/config/rake_command.rb b/lib/buildkite/config/rake_command.rb index 1b33996b..0abf79a9 100644 --- a/lib/buildkite/config/rake_command.rb +++ b/lib/buildkite/config/rake_command.rb @@ -33,7 +33,7 @@ def build_env(build_context, pre_steps, env) env end - def install_plugins(service = "default", env = nil, dir = ".") + def install_plugins(service = "default", env = nil, dir = ".", mainline: false) plugin :artifacts, { download: ".dockerignore" } @@ -49,6 +49,12 @@ def install_plugins(service = "default", env = nil, dir = ".") compressed: ".buildkite.tgz" } + if mainline + plugin :secrets, { + env: "main_env" + } + end + plugin :docker_compose, { "env" => env, "run" => service, @@ -72,7 +78,7 @@ def bundle(command, label:, env: nil) depends_on "docker-image-#{build_context.ruby.image_key}" command command - install_plugins + install_plugins(mainline: build_context.mainline) env build_env(build_context, nil, env) @@ -98,7 +104,7 @@ def rake(dir, task: "test", label: nil, service: "default", pre_steps: nil, env: depends_on "docker-image-#{build_context.ruby.image_key}" command "rake #{task}" - install_plugins(service, %w[PRE_STEPS RACK], dir) + install_plugins(service, %w[PRE_STEPS RACK], dir, mainline: build_context.mainline) env build_env(build_context, pre_steps, env) diff --git a/pipelines/docs-preview/pipeline.rb b/pipelines/docs-preview/pipeline.rb index b18c3586..88c7617e 100644 --- a/pipelines/docs-preview/pipeline.rb +++ b/pipelines/docs-preview/pipeline.rb @@ -6,6 +6,7 @@ plugin :docker, "docker#v5.10.0" plugin :artifacts, "artifacts#v1.9.3" + plugin :secrets, "cluster-secrets#v1.0.0" build_context = context.extensions.find(Buildkite::Config::BuildContext) build_context.ruby = Buildkite::Config::RubyConfig.new(prefix: "ruby:", version: Gem::Version.new("3.3")) @@ -50,6 +51,9 @@ key "deploy" depends_on "build" timeout_in_minutes 15 + plugin :secrets, { + env: "docs_preview_env" + } plugin :docker, { environment: [ "BUILDKITE_BRANCH", @@ -83,6 +87,9 @@ download: ".buildkite/bin/docs-preview-annotate", compressed: ".buildkite.tgz" } + plugin :secrets, { + env: "docs_preview_env" + } command "sh -c \"$$ANNOTATE_COMMAND\" | buildkite-agent annotate --style info" # CLOUDFLARE_API_TOKEN is used to fetch preview URL from latest deployment env "ANNOTATE_COMMAND" => <<~ANNOTATE.gsub(/[[:space:]]+/, " ").strip diff --git a/pipelines/rails-ci/pipeline.rb b/pipelines/rails-ci/pipeline.rb index 3e3dd56c..f0a31315 100644 --- a/pipelines/rails-ci/pipeline.rb +++ b/pipelines/rails-ci/pipeline.rb @@ -9,6 +9,7 @@ plugin :docker_compose, "docker-compose#v5.4.1" plugin :artifacts, "artifacts#v1.9.3" + plugin :secrets, "cluster-secrets#v1.0.0" if build_context.nightly? build_context.rubies << Buildkite::Config::RubyConfig.master_ruby From 75232ad951732177cb3993ca47e3373c34322b9f Mon Sep 17 00:00:00 2001 From: zzak Date: Fri, 14 Feb 2025 14:57:51 +0900 Subject: [PATCH 4/7] Always build image for docs-preview steps --- pipelines/docs-preview/pipeline.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pipelines/docs-preview/pipeline.rb b/pipelines/docs-preview/pipeline.rb index 88c7617e..59b2d234 100644 --- a/pipelines/docs-preview/pipeline.rb +++ b/pipelines/docs-preview/pipeline.rb @@ -3,6 +3,7 @@ Buildkite::Builder.pipeline do require "buildkite_config" use Buildkite::Config::BuildContext + use Buildkite::Config::DockerBuild plugin :docker, "docker#v5.10.0" plugin :artifacts, "artifacts#v1.9.3" @@ -23,13 +24,16 @@ next end + builder build_context.ruby + command do label "build", emoji: :rails + depends_on "docker-image-#{build_context.ruby.image_key}" key "build" command "bundle install && bundle exec rake preview_docs" timeout_in_minutes 15 plugin :docker, { - image: build_context.image_name_for("br-main", prefix: nil), + image: build_context.image_name_for(build_context.build_id, prefix: nil), environment: [ "BUILDKITE_BRANCH", "BUILDKITE_BUILD_CREATOR", From e1bb074d420701a68da700b1b0b4e6d55bfd70b0 Mon Sep 17 00:00:00 2001 From: zzak Date: Fri, 14 Feb 2025 14:59:36 +0900 Subject: [PATCH 5/7] [docs-preview] Prefer alias url when available --- bin/docs-preview-annotate | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/docs-preview-annotate b/bin/docs-preview-annotate index c24a3be0..343b4417 100755 --- a/bin/docs-preview-annotate +++ b/bin/docs-preview-annotate @@ -27,12 +27,13 @@ end json = JSON.parse(response.body) result = json["result"].first +url = result["aliases"]&.first || result["url"] plan = <<~PLAN #### :writing_hand: rails/docs-preview: -* :link: API -* :link: Guides +* :link: API +* :link: Guides PLAN puts plan From a9e698c37a412127dad9816cc12772f84f93559d Mon Sep 17 00:00:00 2001 From: zzak Date: Fri, 14 Feb 2025 16:08:14 +0900 Subject: [PATCH 6/7] Make registry optional and fix tests --- lib/buildkite/config/build_context.rb | 10 +++++- test/buildkite_config/test_docker_build.rb | 41 +++++++++------------- test/buildkite_config/test_rake_command.rb | 7 ++-- 3 files changed, 27 insertions(+), 31 deletions(-) diff --git a/lib/buildkite/config/build_context.rb b/lib/buildkite/config/build_context.rb index 9f6874cf..014c09b5 100644 --- a/lib/buildkite/config/build_context.rb +++ b/lib/buildkite/config/build_context.rb @@ -189,8 +189,16 @@ def min_ruby Gem::Version.new($1 || "2.0") end + def registry + ENV["REGISTRY"] || "973266071021.dkr.ecr.us-east-1.amazonaws.com" + end + + def image_name + "#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds" + end + def remote_image_base - ENV.fetch("REGISTRY") + "/#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds" + [registry, image_name].join("/") end end end diff --git a/test/buildkite_config/test_docker_build.rb b/test/buildkite_config/test_docker_build.rb index e97dde3c..4d2ef6c9 100644 --- a/test/buildkite_config/test_docker_build.rb +++ b/test/buildkite_config/test_docker_build.rb @@ -19,7 +19,6 @@ def test_builder_with_ruby_config_using_string_version assert_equal ":docker: builder:3.2", pipeline.to_h["steps"][0]["label"] assert_equal "docker-image-builder-3-2", pipeline.to_h["steps"][0]["key"] - assert_equal "builder:3.2", pipeline.to_h["steps"][0]["env"]["RUBY_IMAGE"] end def test_builder_artifacts @@ -60,25 +59,16 @@ def test_builder_compose_plugin end end - plugins = pipeline.to_h["steps"][0]["plugins"] - - compose = plugins.find { |plugin| - plugin.key?("docker-compose#v1.0") - }.fetch("docker-compose#v1.0") + command = pipeline.to_h["steps"][0]["command"].first - %w[image-name cache-from push build config env image-repository].each do |key| - assert_includes compose, key - end + expected = <<~COMMAND.squish + docker build --push + --build-arg RUBY_IMAGE=3.2 + --tag buildkite-config-base:3-2-local + --file .buildkite/Dockerfile . + COMMAND - assert_equal "3-2-local", compose["image-name"] - assert_equal ["base:buildkite-config-base:3-2-br-main"], compose["cache-from"] - assert_equal ["base:buildkite-config-base:3-2-br-"], compose["push"] - - assert_equal "base", compose["build"] - assert_equal ".buildkite/docker-compose.yml", compose["config"] - assert_includes compose["env"], "PRE_STEPS" - assert_includes compose["env"], "RACK" - assert_equal "buildkite-config-base", compose["image-repository"] + assert_equal expected.strip, command end def test_builder_timeout_default @@ -143,14 +133,15 @@ def test_builder_gem_version end end - plugins = pipeline.to_h["steps"][0]["plugins"] + command = pipeline.to_h["steps"][0]["command"].first - compose = plugins.find { |plugin| - plugin.key?("docker-compose#v1.0") - }.fetch("docker-compose#v1.0") + expected = <<~COMMAND.squish + docker build --push + --build-arg RUBY_IMAGE=ruby:1.9.3 + --tag buildkite-config-base:ruby-1-9-3-local + --file .buildkite/Dockerfile . + COMMAND - assert_equal "ruby-1-9-3-local", compose["image-name"] - assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-main"], compose["cache-from"] - assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-"], compose["push"] + assert_equal expected.strip, command end end diff --git a/test/buildkite_config/test_rake_command.rb b/test/buildkite_config/test_rake_command.rb index 77924533..0e72a909 100644 --- a/test/buildkite_config/test_rake_command.rb +++ b/test/buildkite_config/test_rake_command.rb @@ -207,7 +207,7 @@ def test_compose plugin.key?("docker-compose#v1.0") }.fetch("docker-compose#v1.0") - %w[env run pull config shell].each do |key| + %w[env run config shell].each do |key| assert_includes compose, key end @@ -215,7 +215,6 @@ def test_compose assert_includes compose["env"], "RACK" assert_equal "default", compose["run"] - assert_equal "default", compose["pull"] assert_equal ".buildkite/docker-compose.yml", compose["config"] assert_equal ["runner", "test"], compose["shell"] end @@ -263,12 +262,11 @@ def test_docker_compose_plugin_service plugin.key?("docker-compose#v1.0") }.fetch("docker-compose#v1.0") - %w[run pull].each do |key| + %w[run].each do |key| assert_includes compose, key end assert_equal "myservice", compose["run"] - assert_equal "myservice", compose["pull"] end def test_env_yjit @@ -453,7 +451,6 @@ def test_bundle_command assert_not_includes compose, "env" assert_equal "default", compose["run"] - assert_equal "default", compose["pull"] assert_equal ".buildkite/docker-compose.yml", compose["config"] assert_equal ["runner", "."], compose["shell"] From 1d5083635595f84c1d8361372275dcdc3eff03df Mon Sep 17 00:00:00 2001 From: zzak Date: Fri, 14 Feb 2025 16:10:37 +0900 Subject: [PATCH 7/7] Pause setting queue to hosted until after buildkit+secrets --- pipelines/rails-ci/initial.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pipelines/rails-ci/initial.yml b/pipelines/rails-ci/initial.yml index 92b3b9dc..0dc39656 100644 --- a/pipelines/rails-ci/initial.yml +++ b/pipelines/rails-ci/initial.yml @@ -1,8 +1,5 @@ # This file is never read -- it's just a copy of the pipeline's # configuration in the Buildkite UI. -env: - CONFIG_REPO: "https://github.com/yob/buildkite-config" - CONFIG_BRANCH: "hosted" steps: - name: ":pipeline: rails-initial-pipeline" @@ -43,7 +40,6 @@ steps: ruby:latest \ .buildkite/bin/fetch-pr > .buildkite/tmp/.pr-meta.json) || true - echo "Generating pipeline:" sh -c "$$PIPELINE_COMMAND" @@ -90,4 +86,4 @@ steps: buildkite-agent pipeline upload timeout_in_minutes: 5 agents: - queue: hosted + queue: "${QUEUE-builder}"