[CI] Separating jobs in pipeline for Serverless

Author: picandocodigo

.buildkite/create-serverless.sh

@@ -2,17 +2,14 @@
 
 set -euo pipefail
 
-# Using BUILDKITE_JOB_ID for the name to make it unique:
-export EC_PROJECT_NAME="$EC_PROJECT_PREFIX-$BUILDKITE_JOB_ID"
-
-# ensure serverless instance is deleted even if script errors
-cleanup() {
-  echo -e "--- :elasticsearch: :broom::sparkles: Tear down serverless instance $EC_PROJECT_NAME"
-  qaf elastic-cloud projects delete
-  rm -rf "~/.elastic/cloud.json"
-}
-trap cleanup EXIT
+if [[ -z $EC_PROJECT_PREFIX ]]; then
+  echo -e "\033[31;1mERROR:\033[0m Required environment variable [EC_PROJECT_PREFIX] not set\033[0m"
+  exit 1
+fi
 
+# Using BUILDKITE_JOB_ID for the name to make it unique:
+# export EC_PROJECT_NAME="$EC_PROJECT_PREFIX-$BUILDKITE_JOB_ID"
+export EC_PROJECT_NAME="$EC_PROJECT_PREFIX-test"
 echo -e "--- :elasticsearch: Start serverless instance $EC_PROJECT_NAME"
 
 qaf elastic-cloud projects create --project-type elasticsearch
@@ -21,6 +18,8 @@ deployment=$(qaf elastic-cloud projects describe $EC_PROJECT_NAME --as-json --sh
 # Set ELASTICSEARCH_URL and API_KEY variables
 export ES_API_SECRET_KEY=$(echo "$deployment" | jq -r '.credentials.api_key')
 export ELASTICSEARCH_URL=$(echo "$deployment" | jq -r '.elasticsearch.url')
+buildkite-agent meta-data set "ES_API_SECRET_KEY" $ES_API_SECRET_KEY
+buildkite-agent meta-data set "ELASTICSEARCH_URL" $ELASTICSEARCH_URL
 
 echo -e "--- :computer: Environment variables"
 echo -e "ELASTICSEARCH_URL $ELASTICSEARCH_URL"

.buildkite/pipeline.yml

@@ -37,18 +37,31 @@ steps:
       DEBUG: true
     command: ./.buildkite/run-yaml-tests.sh
     artifact_paths: "elasticsearch-api/tmp/*"
-  - label: ":elasticsearch: Serverless :rspec: API Tests :ruby: {{ matrix.ruby_source }} - {{ matrix.ruby }}"
+  - label: "Create :elasticsearch: Serverless projects"
+    key: "create-serverless"
     agents:
       image: docker.elastic.co/appex-qa/qaf:latest
+    env:
+      EC_PROJECT_PREFIX: ruby
+      EC_REGISTER_BACKEND: buildkite
+      EC_REGION: aws-eu-west-1
+      EC_ENV: qa
+    commands:
+      - mkdir ~/.elastic
+      - touch ~/.elastic/cloud.json
+      - echo "{\"api_key\":{\"qa\":\"$(vault read -field=qa secret/ci/elastic-elasticsearch-ruby/cloud-access)\"}}" > ~/.elastic/cloud.json
+      - ./.buildkite/create-serverless.sh
+  - label: "Run :elasticsearch: Serverless :rspec: Tests :ruby:"
+    key: run-serverless-tests
+    depends_on:
+      - step: create-serverless
+    agents:
+      provider: "gcp"
     env:
       RUBY_VERSION: "{{ matrix.ruby }}"
       RUBY_SOURCE: "{{ matrix.ruby_source }}"
       TEST_SUITE: serverless
       QUIET: false
-      EC_PROJECT_PREFIX: ruby
-      EC_REGISTER_BACKEND: local
-      EC_REGION: aws-eu-west-1
-      EC_ENV: qa
     matrix:
       setup:
         ruby:
@@ -59,11 +72,23 @@ steps:
         - with:
             ruby_source: 'jruby'
             ruby: '9.4'
+    command: ./.buildkite/run-serverless-tests.sh
+  - label: "Destroy :elasticsearch: Serverless projects"
+    depends_on:
+      - step: run-serverless-tests
+        allow_failure: true
+    agents:
+      image: docker.elastic.co/appex-qa/qaf:latest
+    env:
+      EC_REGISTER_BACKEND: buildkite
+      EC_ENV: qa
+      EC_REGION: aws-eu-west-1
+      EC_PROJECT_NAME: ruby-test
     commands:
       - mkdir ~/.elastic
       - touch ~/.elastic/cloud.json
       - echo "{\"api_key\":{\"qa\":\"$(vault read -field=qa secret/ci/elastic-elasticsearch-ruby/cloud-access)\"}}" > ~/.elastic/cloud.json
-      - ./.buildkite/run-serverless-tests.sh
+      - qaf elastic-cloud projects delete
   - wait: ~
     continue_on_failure: true
   - label: "Log Results"

.buildkite/run-serverless-tests.sh

@@ -9,16 +9,8 @@ repo=`pwd`
 export RUBY_VERSION=${RUBY_VERSION:-3.4}
 export BUILDKITE=${BUILDKITE:-false}
 export TRANSPORT_VERSION=${TRANSPORT_VERSION:-8}
-
-if [[ -z $EC_PROJECT_PREFIX ]]; then
-  echo -e "\033[31;1mERROR:\033[0m Required environment variable [EC_PROJECT_PREFIX] not set\033[0m"
-  exit 1
-fi
-
-# Create a serverless project:
-source $script_path/create-serverless.sh
-# Make sure we remove projects:
-trap cleanup EXIT
+ELASTICSEARCH_URL=`buildkite-agent meta-data get "ELASTICSEARCH_URL"`
+ES_API_SECRET_KEY=`buildkite-agent meta-data get "ES_API_SECRET_KEY"`
 
 echo "--- :ruby: Building Docker image"
 docker build \