Cluster Secrets and Buildkit builds

bin/docs-preview-annotate

@@ -27,12 +27,13 @@ end
 
 json = JSON.parse(response.body)
 result = json["result"].first
+url = result["aliases"]&.first || result["url"]
 
 plan = <<~PLAN
 #### :writing_hand: rails/docs-preview:
 
-* <a href="#{result["url"]}/api">:link: API</a>
-* <a href="#{result["url"]}/guides">:link: Guides</a>
+* <a href="#{url}/api">:link: API</a>
+* <a href="#{url}/guides">:link: Guides</a>
 PLAN
 
 puts plan

lib/buildkite/config/build_context.rb

@@ -189,8 +189,16 @@ def min_ruby
         Gem::Version.new($1 || "2.0")
       end
 
+      def registry
+        ENV["REGISTRY"] || "973266071021.dkr.ecr.us-east-1.amazonaws.com"
+      end
+
+      def image_name
+        "#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds"
+      end
+
       def remote_image_base
-        "973266071021.dkr.ecr.us-east-1.amazonaws.com/#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds"
+        [registry, image_name].join("/")
       end
   end
 end

lib/buildkite/config/docker_build.rb

@@ -30,11 +30,7 @@ def cache_from(build_context)
       end
 
       def build_push(build_context)
-        [
-          build_context.local_branch =~ /:/ ?
-            build_context.image_name_for("pr-#{build_context.pull_request}") :
-            build_context.image_name_for("br-#{build_context.local_branch}"),
-        ]
+        build_context.image_name_for(build_context.build_id, prefix: nil)
       end
     end
 
@@ -66,20 +62,11 @@ def builder(ruby)
             compressed: ".buildkite.tgz"
           }
 
-          plugin :docker_compose, {
-            build: "base",
-            config: ".buildkite/docker-compose.yml",
-            env: %w[PRE_STEPS RACK],
-            "image-name" => build_context.ruby.image_name_for(build_context.build_id),
-            "cache-from" => cache_from(build_context),
-            push: build_push(build_context),
-            "image-repository" => build_context.image_base,
-          }
+          command "docker build --push --build-arg RUBY_IMAGE=#{build_context.ruby.ruby_image} --tag #{build_push(build_context)} --file .buildkite/Dockerfile ."
 
           env({
             BUNDLER: build_context.bundler,
             RUBYGEMS: build_context.rubygems,
-            RUBY_IMAGE: build_context.ruby.ruby_image,
             encrypted_0fb9444d0374_key: nil,
             encrypted_0fb9444d0374_iv: nil
           })

lib/buildkite/config/rake_command.rb

@@ -33,7 +33,7 @@ def build_env(build_context, pre_steps, env)
         env
       end
 
-      def install_plugins(service = "default", env = nil, dir = ".")
+      def install_plugins(service = "default", env = nil, dir = ".", mainline: false)
         plugin :artifacts, {
             download: ".dockerignore"
           }
@@ -49,11 +49,16 @@ def install_plugins(service = "default", env = nil, dir = ".")
           compressed: ".buildkite.tgz"
         }
 
+        if mainline
+          plugin :secrets, {
+            env: "main_env"
+          }
+        end
+
         plugin :docker_compose, {
           "env" => env,
           "run" => service,
-          "pull" => service,
-          "pull-retries" => 3,
+          "tty" => "true",
           "config" => ".buildkite/docker-compose.yml",
           "shell" => ["runner", *dir],
         }.compact
@@ -73,7 +78,7 @@ def bundle(command, label:, env: nil)
           depends_on "docker-image-#{build_context.ruby.image_key}"
           command command
 
-          install_plugins
+          install_plugins(mainline: build_context.mainline)
 
           env build_env(build_context, nil, env)
 
@@ -99,7 +104,7 @@ def rake(dir, task: "test", label: nil, service: "default", pre_steps: nil, env:
           depends_on "docker-image-#{build_context.ruby.image_key}"
           command "rake #{task}"
 
-          install_plugins(service,  %w[PRE_STEPS RACK], dir)
+          install_plugins(service,  %w[PRE_STEPS RACK], dir, mainline: build_context.mainline)
 
           env build_env(build_context, pre_steps, env)
 

pipelines/docs-preview/initial.yml

@@ -30,6 +30,10 @@ steps:
 
       ([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true
 
+    cache:
+      paths:
+        - "cache/bundler"
+      name: "docs-preview-initial-bundler-cache"
     plugins:
       - artifacts#v1.9.3:
           upload: ".dockerignore"
@@ -48,6 +52,7 @@ steps:
       PIPELINE_COMMAND: >-
         docker run --rm
         -v "$$PWD":/app:ro -w /app
+        -v "$$PWD/cache/bundler":/usr/local/bundle
         -e CI
         -e BUILDKITE
         -e BUILDKITE_AGENT_META_DATA_QUEUE

pipelines/docs-preview/pipeline.rb

@@ -3,9 +3,11 @@
 Buildkite::Builder.pipeline do
   require "buildkite_config"
   use Buildkite::Config::BuildContext
+  use Buildkite::Config::DockerBuild
 
   plugin :docker, "docker#v5.10.0"
   plugin :artifacts, "artifacts#v1.9.3"
+  plugin :secrets, "cluster-secrets#v1.0.0"
 
   build_context = context.extensions.find(Buildkite::Config::BuildContext)
   build_context.ruby = Buildkite::Config::RubyConfig.new(prefix: "ruby:", version: Gem::Version.new("3.3"))
@@ -22,13 +24,16 @@
     next
   end
 
+  builder build_context.ruby
+
   command do
     label "build", emoji: :rails
+    depends_on "docker-image-#{build_context.ruby.image_key}"
     key "build"
     command "bundle install && bundle exec rake preview_docs"
     timeout_in_minutes 15
     plugin :docker, {
-      image: build_context.image_name_for("br-main", prefix: nil),
+      image: build_context.image_name_for(build_context.build_id, prefix: nil),
       environment: [
         "BUILDKITE_BRANCH",
         "BUILDKITE_BUILD_CREATOR",
@@ -50,6 +55,9 @@
     key "deploy"
     depends_on "build"
     timeout_in_minutes 15
+    plugin :secrets, {
+      env: "docs_preview_env"
+    }
     plugin :docker, {
       environment: [
         "BUILDKITE_BRANCH",
@@ -83,6 +91,9 @@
       download: ".buildkite/bin/docs-preview-annotate",
       compressed: ".buildkite.tgz"
     }
+    plugin :secrets, {
+      env: "docs_preview_env"
+    }
     command "sh -c \"$$ANNOTATE_COMMAND\" | buildkite-agent annotate --style info"
     # CLOUDFLARE_API_TOKEN is used to fetch preview URL from latest deployment
     env "ANNOTATE_COMMAND" => <<~ANNOTATE.gsub(/[[:space:]]+/, " ").strip

pipelines/rails-ci-nightly/initial.yml

@@ -30,6 +30,11 @@ steps:
 
       ([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true
 
+    cache:
+      paths:
+        - "cache/bundler"
+      name: "rails-initial-bundler-cache"
+
     plugins:
       - artifacts#v1.9.3:
           upload: ".dockerignore"
@@ -48,6 +53,7 @@ steps:
       PIPELINE_COMMAND: >-
         docker run --rm
         -v "$$PWD":/app:ro -w /app
+        -v "$$PWD/cache/bundler":/usr/local/bundle
         -e RAILS_CI_NIGHTLY
         -e CI
         -e BUILDKITE

pipelines/rails-ci/initial.yml

@@ -4,6 +4,9 @@
 steps:
   - name: ":pipeline: rails-initial-pipeline"
     command: |
+      echo "Fetching registry details"
+      export REGISTRY="$$(nsc workspace describe -o json -k registry_url)"
+
       PATH=/bin:/usr/bin
       set -e
 
@@ -30,6 +33,7 @@ steps:
       echo "Fetching pull-request metadata:"
       (docker run --rm \
         -v "$$PWD":/app:ro -w /app \
+        -v "$$PWD/cache/bundler":/usr/local/bundle \
         -e GITHUB_PUBLIC_REPO_TOKEN \
         -e BUILDKITE_REPO \
         -e BUILDKITE_PULL_REQUEST \
@@ -40,7 +44,10 @@ steps:
       sh -c "$$PIPELINE_COMMAND"
 
       ([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true
-
+    cache:
+      paths:
+        - "cache/bundler"
+      name: "rails-initial-bundler-cache"
     plugins:
       - artifacts#v1.9.3:
           upload: ".dockerignore"
@@ -58,6 +65,7 @@ steps:
       PIPELINE_COMMAND: >-
         docker run --rm
         -v "$$PWD":/app:ro -w /app
+        -v "$$PWD/cache/bundler":/usr/local/bundle
         -e CI
         -e BUILDKITE
         -e BUILDKITE_AGENT_META_DATA_QUEUE
@@ -72,6 +80,7 @@ steps:
         -e DOCKER_IMAGE
         -e RUN_QUEUE
         -e QUEUE
+        -e REGISTRY
         ruby:latest
         .buildkite/bin/pipeline-generate rails-ci |
         buildkite-agent pipeline upload

pipelines/rails-ci/pipeline.rb

@@ -7,8 +7,9 @@
   use Buildkite::Config::RakeCommand
   use Buildkite::Config::RubyGroup
 
-  plugin :docker_compose, "docker-compose#v4.16.0"
+  plugin :docker_compose, "docker-compose#v5.4.1"
   plugin :artifacts, "artifacts#v1.9.3"
+  plugin :secrets, "cluster-secrets#v1.0.0"
 
   if build_context.nightly?
     build_context.rubies << Buildkite::Config::RubyConfig.master_ruby

test/buildkite_config/test_docker_build.rb

@@ -19,7 +19,6 @@ def test_builder_with_ruby_config_using_string_version
 
     assert_equal ":docker: builder:3.2", pipeline.to_h["steps"][0]["label"]
     assert_equal "docker-image-builder-3-2", pipeline.to_h["steps"][0]["key"]
-    assert_equal "builder:3.2", pipeline.to_h["steps"][0]["env"]["RUBY_IMAGE"]
   end
 
   def test_builder_artifacts
@@ -60,25 +59,16 @@ def test_builder_compose_plugin
       end
     end
 
-    plugins = pipeline.to_h["steps"][0]["plugins"]
-
-    compose = plugins.find { |plugin|
-      plugin.key?("docker-compose#v1.0")
-    }.fetch("docker-compose#v1.0")
+    command = pipeline.to_h["steps"][0]["command"].first
 
-    %w[image-name cache-from push build config env image-repository].each do |key|
-      assert_includes compose, key
-    end
+    expected = <<~COMMAND.squish
+      docker build --push
+        --build-arg RUBY_IMAGE=3.2
+        --tag buildkite-config-base:3-2-local
+        --file .buildkite/Dockerfile .
+    COMMAND
 
-    assert_equal "3-2-local", compose["image-name"]
-    assert_equal ["base:buildkite-config-base:3-2-br-main"], compose["cache-from"]
-    assert_equal ["base:buildkite-config-base:3-2-br-"], compose["push"]
-
-    assert_equal "base", compose["build"]
-    assert_equal ".buildkite/docker-compose.yml", compose["config"]
-    assert_includes compose["env"], "PRE_STEPS"
-    assert_includes compose["env"], "RACK"
-    assert_equal "buildkite-config-base", compose["image-repository"]
+    assert_equal expected.strip, command
   end
 
   def test_builder_timeout_default
@@ -143,14 +133,15 @@ def test_builder_gem_version
       end
     end
 
-    plugins = pipeline.to_h["steps"][0]["plugins"]
+    command = pipeline.to_h["steps"][0]["command"].first
 
-    compose = plugins.find { |plugin|
-      plugin.key?("docker-compose#v1.0")
-    }.fetch("docker-compose#v1.0")
+    expected = <<~COMMAND.squish
+      docker build --push
+        --build-arg RUBY_IMAGE=ruby:1.9.3
+        --tag buildkite-config-base:ruby-1-9-3-local
+        --file .buildkite/Dockerfile .
+    COMMAND
 
-    assert_equal "ruby-1-9-3-local", compose["image-name"]
-    assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-main"], compose["cache-from"]
-    assert_equal ["base:buildkite-config-base:ruby-1-9-3-br-"], compose["push"]
+    assert_equal expected.strip, command
   end
 end

test/buildkite_config/test_rake_command.rb

@@ -207,15 +207,14 @@ def test_compose
       plugin.key?("docker-compose#v1.0")
     }.fetch("docker-compose#v1.0")
 
-    %w[env run pull config shell].each do |key|
+    %w[env run config shell].each do |key|
       assert_includes compose, key
     end
 
     assert_includes compose["env"], "PRE_STEPS"
     assert_includes compose["env"], "RACK"
 
     assert_equal "default", compose["run"]
-    assert_equal "default", compose["pull"]
     assert_equal ".buildkite/docker-compose.yml", compose["config"]
     assert_equal ["runner", "test"], compose["shell"]
   end
@@ -263,12 +262,11 @@ def test_docker_compose_plugin_service
       plugin.key?("docker-compose#v1.0")
     }.fetch("docker-compose#v1.0")
 
-    %w[run pull].each do |key|
+    %w[run].each do |key|
       assert_includes compose, key
     end
 
     assert_equal "myservice", compose["run"]
-    assert_equal "myservice", compose["pull"]
   end
 
   def test_env_yjit
@@ -453,7 +451,6 @@ def test_bundle_command
 
     assert_not_includes compose, "env"
     assert_equal "default", compose["run"]
-    assert_equal "default", compose["pull"]
     assert_equal ".buildkite/docker-compose.yml", compose["config"]
     assert_equal ["runner", "."], compose["shell"]