From 195158cf3b6a3d532d8231a635c7d6e1f14ec06d Mon Sep 17 00:00:00 2001 From: Ian Maia Date: Thu, 6 Feb 2025 18:03:34 +0100 Subject: [PATCH 01/10] Skip running UI tests if there are no relevant changes --- .buildkite/pipeline.yml | 10 ++++++---- .buildkite/shared-pipeline-vars | 5 ++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ce734252301..c24e3d117c6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -117,8 +117,9 @@ steps: - label: ":microscope: UI Tests (iPhone)" command: .buildkite/commands/run-ui-tests.sh UITests 'iPhone 16' depends_on: build - # Only run on `trunk` and `release/*` -- See p91TBi-cBM-p2#comment-13736 - if: build.branch == 'trunk' || build.branch =~ /^release\// + # Only run on `trunk` and `release/*` (see p91TBi-cBM-p2#comment-13736) and when code changes are detected + if: | + (build.branch == 'trunk' || build.branch =~ /^release\//) && $SKIP_UI_TESTS != 'true' plugins: [$CI_TOOLKIT] artifact_paths: - fastlane/test_output/* @@ -129,8 +130,9 @@ steps: - label: ":microscope: UI Tests (iPad)" command: .buildkite/commands/run-ui-tests.sh UITests "iPad (10th generation)" depends_on: build - # Only run on `trunk` and `release/*` -- See p91TBi-cBM-p2#comment-13736 - if: build.branch == 'trunk' || build.branch =~ /^release\// + # Only run on `trunk` and `release/*` (see p91TBi-cBM-p2#comment-13736) and when code changes are detected + if: | + (build.branch == 'trunk' || build.branch =~ /^release\//) && $SKIP_UI_TESTS != 'true' plugins: [$CI_TOOLKIT] artifact_paths: - fastlane/test_output/* diff --git a/.buildkite/shared-pipeline-vars b/.buildkite/shared-pipeline-vars index 2a7cb971dda..4eedf93176d 100644 --- a/.buildkite/shared-pipeline-vars +++ b/.buildkite/shared-pipeline-vars @@ -4,8 +4,11 @@ # to set up some variables that will be interpolated in the `.yml` pipeline before uploading it. XCODE_VERSION=$(sed 's/^~> *//' .xcode-version) -CI_TOOLKIT_PLUGIN_VERSION="3.9.1" +CI_TOOLKIT_PLUGIN_VERSION="iangmaia/pr-changed-files" # Note: `-v4` suffix was added to use xcode-16.1-v4 image; remember to remove that suffix during the next Xcode update export IMAGE_ID="xcode-$XCODE_VERSION-v4" export CI_TOOLKIT="automattic/a8c-ci-toolkit#$CI_TOOLKIT_PLUGIN_VERSION" + +# Skip UI tests if changes are limited to docs, assets, or other non-code files +export SKIP_UI_TESTS=$(pr_changed_files --all-match "*.md" "docs/**" "*.txt" "*.pot" "*.strings" "fastlane/**" ".github/**" ".buildkite/**") From d38ed58f7eb3f163ccd0787ad0f63c1aff0647e1 Mon Sep 17 00:00:00 2001 From: Ian Maia Date: Thu, 6 Feb 2025 18:59:36 +0100 Subject: [PATCH 02/10] [to revert] Test running on this branch --- .buildkite/pipeline.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c24e3d117c6..caa154d757e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -116,10 +116,9 @@ steps: ################# - label: ":microscope: UI Tests (iPhone)" command: .buildkite/commands/run-ui-tests.sh UITests 'iPhone 16' - depends_on: build # Only run on `trunk` and `release/*` (see p91TBi-cBM-p2#comment-13736) and when code changes are detected if: | - (build.branch == 'trunk' || build.branch =~ /^release\//) && $SKIP_UI_TESTS != 'true' + (build.branch == 'trunk' || build.branch =~ /^release\// || build.branch =~ /^iangmaia\//) && $SKIP_UI_TESTS != 'true' plugins: [$CI_TOOLKIT] artifact_paths: - fastlane/test_output/* @@ -129,10 +128,9 @@ steps: - label: ":microscope: UI Tests (iPad)" command: .buildkite/commands/run-ui-tests.sh UITests "iPad (10th generation)" - depends_on: build # Only run on `trunk` and `release/*` (see p91TBi-cBM-p2#comment-13736) and when code changes are detected if: | - (build.branch == 'trunk' || build.branch =~ /^release\//) && $SKIP_UI_TESTS != 'true' + (build.branch == 'trunk' || build.branch =~ /^release\// || build.branch =~ /^iangmaia\//) && $SKIP_UI_TESTS != 'true' plugins: [$CI_TOOLKIT] artifact_paths: - fastlane/test_output/* From 4b9ddfe80d80a71ed24651a580deebbd1177ee41 Mon Sep 17 00:00:00 2001 From: Ian Maia Date: Fri, 7 Feb 2025 13:19:34 +0100 Subject: [PATCH 03/10] Install CI_TOOLKIT before running pr_changed_files --- .buildkite/shared-pipeline-vars | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/shared-pipeline-vars b/.buildkite/shared-pipeline-vars index 4eedf93176d..41fc613056b 100644 --- a/.buildkite/shared-pipeline-vars +++ b/.buildkite/shared-pipeline-vars @@ -10,5 +10,8 @@ CI_TOOLKIT_PLUGIN_VERSION="iangmaia/pr-changed-files" export IMAGE_ID="xcode-$XCODE_VERSION-v4" export CI_TOOLKIT="automattic/a8c-ci-toolkit#$CI_TOOLKIT_PLUGIN_VERSION" +# Install/update the plugin first so we can use pr_changed_files +buildkite-agent plugin install $CI_TOOLKIT + # Skip UI tests if changes are limited to docs, assets, or other non-code files export SKIP_UI_TESTS=$(pr_changed_files --all-match "*.md" "docs/**" "*.txt" "*.pot" "*.strings" "fastlane/**" ".github/**" ".buildkite/**") From b3bb96bc6f45b0ece35f9e5ebcacd162aa441944 Mon Sep 17 00:00:00 2001 From: Ian Maia Date: Fri, 7 Feb 2025 19:27:50 +0100 Subject: [PATCH 04/10] Move call to `pr_changed_files` to a separate file --- .buildkite/commands/run-ui-tests.sh | 5 +++++ .buildkite/commands/should-skip-ui-tests.sh | 4 ++++ .buildkite/pipeline.yml | 8 ++------ .buildkite/shared-pipeline-vars | 6 ------ 4 files changed, 11 insertions(+), 12 deletions(-) create mode 100755 .buildkite/commands/should-skip-ui-tests.sh diff --git a/.buildkite/commands/run-ui-tests.sh b/.buildkite/commands/run-ui-tests.sh index 6e46300596d..407557610e8 100755 --- a/.buildkite/commands/run-ui-tests.sh +++ b/.buildkite/commands/run-ui-tests.sh @@ -1,5 +1,10 @@ #!/bin/bash -eu +if .buildkite/commands/should-skip-ui-tests.sh > /dev/null 2>&1; then + echo "Skipping UI Tests as only documentation, tooling and non-code files were changed" + exit 0 +fi + TEST_NAME=$1 DEVICE=$2 diff --git a/.buildkite/commands/should-skip-ui-tests.sh b/.buildkite/commands/should-skip-ui-tests.sh new file mode 100755 index 00000000000..1b4ac2391ba --- /dev/null +++ b/.buildkite/commands/should-skip-ui-tests.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# Check if changes are limited to documentation, tooling and non-code files +pr_changed_files --all-match "*.md" "docs/**" "*.txt" "*.pot" "fastlane/**" ".github/**" ".buildkite/**" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index caa154d757e..a997770da7f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -116,9 +116,7 @@ steps: ################# - label: ":microscope: UI Tests (iPhone)" command: .buildkite/commands/run-ui-tests.sh UITests 'iPhone 16' - # Only run on `trunk` and `release/*` (see p91TBi-cBM-p2#comment-13736) and when code changes are detected - if: | - (build.branch == 'trunk' || build.branch =~ /^release\// || build.branch =~ /^iangmaia\//) && $SKIP_UI_TESTS != 'true' + if: build.branch == 'trunk' || build.branch =~ /^release\// || build.branch =~ /^iangmaia\// plugins: [$CI_TOOLKIT] artifact_paths: - fastlane/test_output/* @@ -128,9 +126,7 @@ steps: - label: ":microscope: UI Tests (iPad)" command: .buildkite/commands/run-ui-tests.sh UITests "iPad (10th generation)" - # Only run on `trunk` and `release/*` (see p91TBi-cBM-p2#comment-13736) and when code changes are detected - if: | - (build.branch == 'trunk' || build.branch =~ /^release\// || build.branch =~ /^iangmaia\//) && $SKIP_UI_TESTS != 'true' + if: build.branch == 'trunk' || build.branch =~ /^release\// || build.branch =~ /^iangmaia\// plugins: [$CI_TOOLKIT] artifact_paths: - fastlane/test_output/* diff --git a/.buildkite/shared-pipeline-vars b/.buildkite/shared-pipeline-vars index 41fc613056b..be34828d27d 100644 --- a/.buildkite/shared-pipeline-vars +++ b/.buildkite/shared-pipeline-vars @@ -9,9 +9,3 @@ CI_TOOLKIT_PLUGIN_VERSION="iangmaia/pr-changed-files" # Note: `-v4` suffix was added to use xcode-16.1-v4 image; remember to remove that suffix during the next Xcode update export IMAGE_ID="xcode-$XCODE_VERSION-v4" export CI_TOOLKIT="automattic/a8c-ci-toolkit#$CI_TOOLKIT_PLUGIN_VERSION" - -# Install/update the plugin first so we can use pr_changed_files -buildkite-agent plugin install $CI_TOOLKIT - -# Skip UI tests if changes are limited to docs, assets, or other non-code files -export SKIP_UI_TESTS=$(pr_changed_files --all-match "*.md" "docs/**" "*.txt" "*.pot" "*.strings" "fastlane/**" ".github/**" ".buildkite/**") From ce6ea7716b161415b785d174e24b67a471f0ac4f Mon Sep 17 00:00:00 2001 From: "Ian G. Maia" Date: Wed, 12 Feb 2025 17:08:49 +0100 Subject: [PATCH 05/10] Annotate Buildkite build when UITests were skipped Co-authored-by: Olivier Halligon --- .buildkite/commands/run-ui-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/commands/run-ui-tests.sh b/.buildkite/commands/run-ui-tests.sh index 407557610e8..2e7df8d6230 100755 --- a/.buildkite/commands/run-ui-tests.sh +++ b/.buildkite/commands/run-ui-tests.sh @@ -1,7 +1,7 @@ #!/bin/bash -eu if .buildkite/commands/should-skip-ui-tests.sh > /dev/null 2>&1; then - echo "Skipping UI Tests as only documentation, tooling and non-code files were changed" + echo "Skipping UI Tests as only documentation, tooling and/or non-code files were changed" | tee >(buildkite-agent annotate) exit 0 fi From 3c1b30c4ebb59d468b92beb62a62ce9e31158f50 Mon Sep 17 00:00:00 2001 From: "Ian G. Maia" Date: Wed, 12 Feb 2025 17:09:54 +0100 Subject: [PATCH 06/10] Update a8c-ci-toolkit version --- .buildkite/shared-pipeline-vars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/shared-pipeline-vars b/.buildkite/shared-pipeline-vars index be34828d27d..df703fd1efb 100644 --- a/.buildkite/shared-pipeline-vars +++ b/.buildkite/shared-pipeline-vars @@ -4,7 +4,7 @@ # to set up some variables that will be interpolated in the `.yml` pipeline before uploading it. XCODE_VERSION=$(sed 's/^~> *//' .xcode-version) -CI_TOOLKIT_PLUGIN_VERSION="iangmaia/pr-changed-files" +CI_TOOLKIT_PLUGIN_VERSION="3.11.0" # Note: `-v4` suffix was added to use xcode-16.1-v4 image; remember to remove that suffix during the next Xcode update export IMAGE_ID="xcode-$XCODE_VERSION-v4" From 131f7b0351167575c5f33e77b2a4bdd9a300d168 Mon Sep 17 00:00:00 2001 From: Ian Maia Date: Wed, 12 Feb 2025 19:37:25 +0100 Subject: [PATCH 07/10] Adjust `pr_changed_files` usage --- .buildkite/commands/run-ui-tests.sh | 2 +- .buildkite/commands/should-skip-ui-tests.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.buildkite/commands/run-ui-tests.sh b/.buildkite/commands/run-ui-tests.sh index 2e7df8d6230..dc31c17f192 100755 --- a/.buildkite/commands/run-ui-tests.sh +++ b/.buildkite/commands/run-ui-tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -eu -if .buildkite/commands/should-skip-ui-tests.sh > /dev/null 2>&1; then +if .buildkite/commands/should-skip-ui-tests.sh; then echo "Skipping UI Tests as only documentation, tooling and/or non-code files were changed" | tee >(buildkite-agent annotate) exit 0 fi diff --git a/.buildkite/commands/should-skip-ui-tests.sh b/.buildkite/commands/should-skip-ui-tests.sh index 1b4ac2391ba..3daf5364982 100755 --- a/.buildkite/commands/should-skip-ui-tests.sh +++ b/.buildkite/commands/should-skip-ui-tests.sh @@ -1,4 +1,5 @@ #!/bin/bash # Check if changes are limited to documentation, tooling and non-code files -pr_changed_files --all-match "*.md" "docs/**" "*.txt" "*.pot" "fastlane/**" ".github/**" ".buildkite/**" +result=$(pr_changed_files --all-match "*.md" "docs/**" "*.txt" "*.pot" "fastlane/**" ".github/**" ".buildkite/**") +[[ "$result" == "true" ]] From 168e65f1e85f2ef2aacb1a8a7c062e2be9b021d0 Mon Sep 17 00:00:00 2001 From: Ian Maia Date: Wed, 12 Feb 2025 19:47:29 +0100 Subject: [PATCH 08/10] Split `buildkite-agent annotate` to a separate statement --- .buildkite/commands/run-ui-tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/commands/run-ui-tests.sh b/.buildkite/commands/run-ui-tests.sh index dc31c17f192..197664a3314 100755 --- a/.buildkite/commands/run-ui-tests.sh +++ b/.buildkite/commands/run-ui-tests.sh @@ -1,7 +1,9 @@ #!/bin/bash -eu if .buildkite/commands/should-skip-ui-tests.sh; then - echo "Skipping UI Tests as only documentation, tooling and/or non-code files were changed" | tee >(buildkite-agent annotate) + message="Skipping UI Tests as only documentation, tooling and/or non-code files were changed" + echo "$message" | buildkite-agent annotate --style "info" --context "skip-ui-tests" + echo "$message" exit 0 fi From d73af98d392cfa04ee507ba176386a020d178c30 Mon Sep 17 00:00:00 2001 From: Ian Maia Date: Wed, 12 Feb 2025 20:37:32 +0100 Subject: [PATCH 09/10] [to revert] Add test code change to trigger ui tests --- WooCommerce/Classes/Analytics/AppStartupWaitingTimeTracker.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/WooCommerce/Classes/Analytics/AppStartupWaitingTimeTracker.swift b/WooCommerce/Classes/Analytics/AppStartupWaitingTimeTracker.swift index 13b32c43fdd..9e866501329 100644 --- a/WooCommerce/Classes/Analytics/AppStartupWaitingTimeTracker.swift +++ b/WooCommerce/Classes/Analytics/AppStartupWaitingTimeTracker.swift @@ -2,6 +2,7 @@ import Foundation import Yosemite import protocol WooFoundation.Analytics +// test code change -- should trigger ui tests /// Tracks the waiting time for app startup, allowing to evaluate as analytics /// how much time in seconds it took between the init and the final `end(action:)` function call. /// From e5a038d06a036b6672f44d68cc14515e2a17c89b Mon Sep 17 00:00:00 2001 From: Ian Maia Date: Thu, 13 Feb 2025 12:33:52 +0100 Subject: [PATCH 10/10] Revert "[to revert] Add test code change to trigger ui tests" This reverts commit 8214c53c48740ad1f46f08b53a6e88cb4bbcafe4. --- WooCommerce/Classes/Analytics/AppStartupWaitingTimeTracker.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/WooCommerce/Classes/Analytics/AppStartupWaitingTimeTracker.swift b/WooCommerce/Classes/Analytics/AppStartupWaitingTimeTracker.swift index 9e866501329..13b32c43fdd 100644 --- a/WooCommerce/Classes/Analytics/AppStartupWaitingTimeTracker.swift +++ b/WooCommerce/Classes/Analytics/AppStartupWaitingTimeTracker.swift @@ -2,7 +2,6 @@ import Foundation import Yosemite import protocol WooFoundation.Analytics -// test code change -- should trigger ui tests /// Tracks the waiting time for app startup, allowing to evaluate as analytics /// how much time in seconds it took between the init and the final `end(action:)` function call. ///