[Buildkite] Add pipeline to test with Elastic serverless daily (elastic#1807)

Add pipeline to test with an Elastic serverless project daily. It also
adds the required settings to trigger this pipeline from a Pull Request.

Author: mrodm

.buildkite/hooks/pre-command

@@ -6,11 +6,18 @@ set -euo pipefail
 GO_VERSION=$(cat .go-version)
 export GO_VERSION
 
+export SERVERLESS=${SERVERLESS:-"false"}
+export WORKSPACE=$(pwd)
+
+
 GCP_SERVICE_ACCOUNT_SECRET_PATH=secret/ci/elastic-elastic-package/gcp-service-account
 AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth
 GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
 PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
 
+EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
+EC_DATA_PATH=secret/ci/elastic-elastic-package/ec_data
+
 # variables required for terraform
 export ENVIRONMENT="ci"
 REPO=$(repo_name "${BUILDKITE_REPO}")
@@ -46,12 +53,24 @@ export CREATED_DATE
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
+export TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO}"
+export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
+
 is_step_required_to_upload_safe_logs() {
-    if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" ]]; then
+    if [[ "$BUILDKITE_PIPELINE_SLUG" != "elastic-package" && "$BUILDKITE_PIPELINE_SLUG" != "elastic-package-test-serverless" ]]; then
         return 1
     fi
-    if [[ "$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives ]]; then
-        return 0
+
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" ]]; then
+        if [[ "$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives ]]; then
+            return 0
+        fi
+    fi
+
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-serverless" ]]; then
+        if [[ "$BUILDKITE_STEP_KEY" == "test-serverless" ]]; then
+            return 0
+        fi
     fi
     return 1
 }
@@ -139,3 +158,17 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-cloud-cleanup" && "$BUILDKI
     export ELASTIC_PACKAGE_GCP_EMAIL_SECRET
 fi
 
+
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package-test-serverless" ]]; then
+    if [[ "${BUILDKITE_STEP_KEY}" == "test-serverless" ]]; then
+        EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_TOKEN_PATH}")
+        export EC_API_KEY_SECRET
+        EC_HOST_SECRET=$(retry 5 vault kv get -field url "${EC_TOKEN_PATH}")
+        export EC_HOST_SECRET
+        EC_REGION_SECRET=$(retry 5 vault read -field region_qa "${EC_DATA_PATH}")
+        export EC_REGION_SECRET
+
+        GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+        export GITHUB_TOKEN
+    fi
+fi

.buildkite/hooks/pre-exit

@@ -4,6 +4,16 @@ source .buildkite/scripts/tooling.sh
 
 set -euo pipefail
 
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "elastic-package-test-serverless" && "${BUILDKITE_STEP_KEY}" == "test-serverless" ]]; then
+    echo "--- Take down the Elastic stack"
+    # BUILDKITE resets PATH contents in pre-exit hook, but elastic-package
+    # is already installed in the test_serverless pipeline step, accessing
+    # directly to the binary
+    EC_API_KEY="${EC_API_KEY_SECRET}" EC_HOST="${EC_HOST_SECRET}" "${HOME}"/go/bin/elastic-package stack down -v
+fi
+
+echo "--- Cleanup"
+cleanup
 unset_secrets
 
 # integrations-parallel-gcp
@@ -15,4 +25,3 @@ unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
 unset ELASTIC_PACKAGE_AWS_SECRET_KEY
 unset AWS_ACCESS_KEY_ID
 unset AWS_SECRET_ACCESS_KEY
-

.buildkite/pipeline.serverless.yml

@@ -0,0 +1,60 @@
+env:
+  NOTIFY_TO: "[email protected]"
+  SETUP_GVM_VERSION: 'v0.5.2' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
+  DOCKER_COMPOSE_VERSION: "v2.24.1"
+  DOCKER_VERSION: "26.1.0"
+  KIND_VERSION: 'v0.20.0'
+  K8S_VERSION: 'v1.29.0'
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+  GH_CLI_VERSION: "2.29.0"
+  # Elastic package settings
+  # Manage docker output/logs
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"
+  # Default license to use by `elastic-package build`
+  ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
+  # Link definitions path (full path to be set in the corresponding step)
+  ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
+  # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
+  ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
+
+steps:
+  - input: "Input values for the variables"
+    key: "input-variables"
+    fields:
+    - select: "SERVERLESS_PROJECT"
+      key: "SERVERLESS_PROJECT"
+      options:
+        - label: "observability"
+          value: "observability"
+        - label: "security"
+          value: "security"
+      default: "observability"
+    if: "build.source == 'ui'"
+
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
+
+  - label: ":elastic: Serverless Integration tests"
+    key: test-serverless
+    command: ".buildkite/scripts/test_packages_with_serverless.sh"
+    env:
+      SERVERLESS_PROJECT: "${SERVERLESS_PROJECT:-observability}"
+      UPLOAD_SAFE_LOGS: 1
+    artifact_paths:
+      - build/test-results/*.xml
+      - build/test-coverage/coverage-*.xml
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":junit: Junit annotate"
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: "build/test-results/*.xml"
+    agents:
+      provider: "gcp"  # junit plugin requires docker
+
+notify:
+  - email: "$NOTIFY_TO"
+    if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'"

.buildkite/pull-requests.json

@@ -31,6 +31,22 @@
         "skip_target_branches": [ ],
         "skip_ci_on_only_changed": [ ],
         "always_require_ci_on_changed": [ ]
+     },
+     {
+        "enabled": true,
+        "pipelineSlug": "elastic-package-test-serverless",
+        "allow_org_users": true,
+        "allowed_repo_permissions": ["admin", "write"],
+        "allowed_list": [ ],
+        "set_commit_status": false,
+        "build_on_commit": false,
+        "build_on_comment": true,
+        "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:test)\\W+(?:serverless))$",
+        "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:test)\\W+(?:serverless))$",
+        "skip_ci_labels": [ ],
+        "skip_target_branches": [ ],
+        "skip_ci_on_only_changed": [ ],
+        "always_require_ci_on_changed": [ ]
      }
    ]
 }

.buildkite/scripts/integration_tests.sh

@@ -5,10 +5,7 @@ source .buildkite/scripts/tooling.sh
 
 set -euo pipefail
 
-WORKSPACE="$(pwd)"
-TMP_FOLDER_TEMPLATE_BASE="tmp.elastic-package"
-
-cleanup() {
+ensure_logout() {
     local error_code=$?
 
     if [ $error_code != 0 ] ; then
@@ -18,14 +15,9 @@ cleanup() {
         fi
     fi
 
-    echo "Deleting temporal files..."
-    cd "${WORKSPACE}"
-    rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*"
-    echo "Done."
-
     exit $error_code
 }
-trap cleanup EXIT
+trap ensure_logout EXIT
 
 usage() {
     echo "$0 [-t <target>] [-h]"
@@ -39,22 +31,25 @@ PARALLEL_TARGET="test-check-packages-parallel"
 FALSE_POSITIVES_TARGET="test-check-packages-false-positives"
 KIND_TARGET="test-check-packages-with-kind"
 SYSTEM_TEST_FLAGS_TARGET="test-system-test-flags"
-TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT=${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT:-"false"}
 
 REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
-REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
+export REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
 TARGET=""
 PACKAGE=""
-while getopts ":t:p:h" o; do
+SERVERLESS="false"
+while getopts ":t:p:sh" o; do
     case "${o}" in
         t)
             TARGET=${OPTARG}
             ;;
         p)
             PACKAGE=${OPTARG}
             ;;
+        s)
+            SERVERLESS="true"
+            ;;
         h)
             usage
             exit 0
@@ -78,57 +73,37 @@ if [[ "${TARGET}" == "" ]]; then
     exit 1
 fi
 
-google_cloud_auth_safe_logs() {
-    local gsUtilLocation=""
-    gsUtilLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE}")
-
-    local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
-
-    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > "${secretFileLocation}"
-
-    google_cloud_auth "${secretFileLocation}"
-}
-
-upload_safe_logs() {
-    local bucket="$1"
-    local source="$2"
-    local target="$3"
-
-    if ! ls ${source} 2>&1 > /dev/null ; then
-        echo "upload_safe_logs: artifacts files not found, nothing will be archived"
-        return
-    fi
-
-    google_cloud_auth_safe_logs
-
-    gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
-
-    google_cloud_logout_active_account
-}
-
 add_bin_path
 
-echo "--- install go"
-with_go
+if [[ "$SERVERLESS" == "false" ]]; then
+    # If packages are tested with Serverless, these action are already performed
+    # here: .buildkite/scripts/test_packages_with_serverless.sh
+    echo "--- install go"
+    with_go
 
-echo "--- install docker"
-with_docker
+    echo "--- install docker"
+    with_docker
 
-echo "--- install docker-compose plugin"
-with_docker_compose_plugin
+    echo "--- install docker-compose plugin"
+    with_docker_compose_plugin
+fi
 
 if [[ "${TARGET}" == "${KIND_TARGET}" || "${TARGET}" == "${SYSTEM_TEST_FLAGS_TARGET}" ]]; then
     echo "--- install kubectl & kind"
     with_kubernetes
 fi
 
-echo "--- Run integration test ${TARGET}"
+label="${TARGET}"
+if [ -n "${PACKAGE}" ]; then
+    label="${label} - ${PACKAGE}"
+fi
+echo "--- Run integration test ${label}"
 if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then
     make install
 
     # allow to fail this command, to be able to upload safe logs
     set +e
-    make PACKAGE_UNDER_TEST="${PACKAGE}" "${TARGET}"
+    make SERVERLESS="${SERVERLESS}" PACKAGE_UNDER_TEST="${PACKAGE}" "${TARGET}"
     testReturnCode=$?
     set -e
 
@@ -162,7 +137,7 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT
     fi
 
     if [ $testReturnCode != 0 ]; then
-        echo "make PACKAGE_UDER_TEST=${PACKAGE} ${TARGET} failed with ${testReturnCode}"
+        echo "make SERVERLESS=${SERVERLESS} PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET} failed with ${testReturnCode}"
         exit ${testReturnCode}
     fi
 

.buildkite/scripts/release.sh

@@ -1,17 +1,18 @@
 #!/bin/bash
 
+source .buildkite/scripts/install_deps.sh
+source .buildkite/scripts/tooling.sh
+
 set -euo pipefail
 
 cleanup() {
     rm -rf "${WORKSPACE}"
 }
 trap cleanup exit
 
-WORKSPACE="/tmp/bin-buildkite/"
+export WORKSPACE="/tmp/bin-buildkite/"
 
 VERSION=""
-source .buildkite/scripts/install_deps.sh
-source .buildkite/scripts/tooling.sh
 
 add_bin_path
 with_go

.buildkite/scripts/test-with-integrations.sh

@@ -3,20 +3,6 @@ source .buildkite/scripts/install_deps.sh
 
 set -euo pipefail
 
-WORKSPACE="$(pwd)"
-
-TMP_FOLDER_TEMPLATE_BASE="tmp.${GITHUB_PR_BASE_REPO}"
-TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
-
-cleanup() {
-    echo "Deleting temporal files..."
-    cd "${WORKSPACE}"
-    rm -rf "${TMP_FOLDER_TEMPLATE_BASE}.*"
-    echo "Done."
-}
-
-trap cleanup EXIT
-
 add_bin_path
 
 echo "--- install gh cli"
@@ -179,10 +165,10 @@ add_pr_comment() {
     local source_pr_number="$1"
     local integrations_pr_link="$2"
 
-    retry 3 \
-        gh pr comment "${source_pr_number}" \
-        --body "Created or updated PR in integrations repository to test this version. Check ${integrations_pr_link}" \
-        --repo "${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}"
+    add_github_comment \
+        "${GITHUB_PR_BASE_OWNER}/${GITHUB_PR_BASE_REPO}" \
+        "${source_pr_number}" \
+        "Created or updated PR in integrations repository to test this version. Check ${integrations_pr_link}"
 }