Protect: Integrate ThreatsDataViews Component #84321
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| push: | |
| branches: | |
| - 'trunk' | |
| - 'prerelease' | |
| # The `**/*/` works around the fact that GitHub considers a leading `**/` as meaning "zero or more path components" where we want "one or more". | |
| - '**/*/branch-**' | |
| pull_request: | |
| concurrency: | |
| # Cancel concurrent jobs on pull_request but not push, by including the run_id in the concurrency group for the latter. | |
| group: build-${{ github.event_name == 'push' && github.run_id || 'pr' }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| COMPOSER_ROOT_VERSION: "dev-trunk" | |
| jobs: | |
| build: | |
| name: Build all projects | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 # 2023-05-25: Build times have crept up to ~15–25+ minutes as we've added more projects, bump to 30. | |
| env: | |
| # Hard-code a specific directory to avoid paths in vendor/composer/installed.json changing every build. | |
| BUILD_BASE: /tmp/jetpack-build | |
| # This string is used as a unique identifier of test reminder comments on PRs. | |
| TEST_COMMENT_INDICATOR: "<!-- wpcom-reminder-comment -->" | |
| outputs: | |
| any_plugins: ${{ steps.plugins.outputs.any }} | |
| changed_projects: ${{ steps.changed.outputs.projects }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # For pull requests, list-changed-projects.sh needs the merge base. | |
| # But it doesn't have to be checked out. | |
| - name: Deepen to merge base | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/deepen-to-merge-base | |
| with: | |
| checkout: false | |
| - name: Setup tools | |
| uses: ./.github/actions/tool-setup | |
| - name: Pnpm install | |
| run: pnpm install | |
| - name: Detect changed projects | |
| id: changed | |
| run: | | |
| CHANGED="$(EXTRA=build .github/files/list-changed-projects.sh)" | |
| echo "projects=${CHANGED}" >> "$GITHUB_OUTPUT" | |
| - name: Check if a WordPress.com test reminder comment is needed. | |
| id: check-test-reminder-comment | |
| uses: actions/github-script@v7 | |
| if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name }} | |
| env: | |
| CHANGED: ${{ steps.changed.outputs.projects }} | |
| with: | |
| script: | | |
| const checkTestReminderComment = require('.github/files/build-reminder-comment/check-test-reminder-comment.js') | |
| const data = await checkTestReminderComment( github, context, core ); | |
| return data; | |
| # We need the tree (but not the blob) for packages that will have -alpha version numbers so the timestamp appending works right. | |
| - name: Deepen tree for packages | |
| env: | |
| CHANGED: ${{ steps.changed.outputs.projects }} | |
| run: | | |
| mapfile -t PROJECTS < <(jq -r 'to_entries[] | select( .value ) | .key' <<<"$CHANGED") | |
| if [[ ${#PROJECTS[@]} -gt 0 ]]; then | |
| depth=$( git rev-list --count --first-parent HEAD ) | |
| [[ "$depth" -lt 1000 ]] && depth=1000 | |
| BASE=$PWD | |
| REF=$(git rev-parse HEAD) | |
| for SLUG in $(pnpm jetpack dependencies list --add-dependencies --extra="build" --ignore-root "${PROJECTS[@]}"); do | |
| [[ "$SLUG" == packages/* ]] || continue | |
| cd "$BASE/projects/$SLUG/" | |
| CHANGES_DIR="$(jq -r '.extra.changelogger["changes-dir"] // "changelog"' composer.json)" | |
| [[ -d "$CHANGES_DIR" && -n "$(ls -- "$CHANGES_DIR")" ]] || continue | |
| echo "Checking depth for $SLUG" | |
| while git log --format='%h, %D,' -1 . | grep ', grafted,'; do | |
| depth=$((depth * 2)) | |
| echo "::group::Deepen to $depth" | |
| echo "/usr/bin/git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=$depth --filter=blob:none origin $REF" | |
| /usr/bin/git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=$depth --filter=blob:none origin "$REF" | |
| echo "::endgroup::" | |
| done | |
| done | |
| fi | |
| - name: Build changed projects | |
| id: build | |
| env: | |
| CHANGED: ${{ steps.changed.outputs.projects }} | |
| run: | | |
| mapfile -t PROJECTS < <(jq -r 'to_entries[] | select( .value ) | .key' <<<"$CHANGED") | |
| if [[ ${#PROJECTS[@]} -eq 0 ]]; then | |
| echo "Nothing to build. Generating empty artifact." | |
| mkdir "$BUILD_BASE" | |
| touch "$BUILD_BASE/mirrors.txt" | |
| else | |
| pnpm jetpack build -v --no-pnpm-install --for-mirrors="$BUILD_BASE" "${PROJECTS[@]}" | |
| fi | |
| - name: Filter mirror list for release branch | |
| if: github.ref == 'refs/heads/prerelease' || contains( github.ref, '/branch-' ) | |
| run: .github/files/filter-mirrors-for-release-branch.sh | |
| - name: Determine plugins to publish | |
| id: plugins | |
| run: | | |
| jq -r 'if .extra["mirror-repo"] and ( .extra["beta-plugin-slug"] // .extra["wp-plugin-slug"] ) then [ ( input_filename | sub( "/composer\\.json$"; "" ) ), .extra["mirror-repo"], .extra["beta-plugin-slug"] // .extra["wp-plugin-slug"] ] else empty end | @tsv' projects/plugins/*/composer.json | while IFS=$'\t' read -r SRC MIRROR SLUG; do | |
| if [[ -e "$BUILD_BASE/$MIRROR" ]] && grep -q --fixed-strings --line-regexp "$MIRROR" "$BUILD_BASE/mirrors.txt"; then | |
| printf '%s\t%s\t%s\n' "$SRC" "$MIRROR" "$SLUG" | |
| fi | |
| done > "$BUILD_BASE/plugins.tsv" | |
| if [[ -s "$BUILD_BASE/plugins.tsv" ]]; then | |
| cat "$BUILD_BASE/plugins.tsv" | |
| echo "any=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "No plugins were built" | |
| echo "any=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| # GitHub's artifact stuff doesn't preserve permissions or file case. Sigh. | |
| # This is the official workaround: https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files | |
| # It'll also make it faster to upload and download though, so maybe it's a win anyway. | |
| - name: Create archive | |
| run: tar --owner=0 --group=0 --xz -cvvf build.tar.xz -C "$BUILD_BASE" --transform 's,^\.,build,' . | |
| - name: Store build as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: jetpack-build | |
| path: build.tar.xz | |
| # Only need to retain for a day since the beta builder slurps it up to distribute. | |
| retention-days: 1 | |
| # Already compressed. | |
| compression-level: 0 | |
| - name: Store plugins.tsv as artifact | |
| if: steps.plugins.outputs.any == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: plugins.tsv | |
| path: ${{ env.BUILD_BASE }}/plugins.tsv | |
| # We don't really care about this artifact, its presence is a flag to the post-build job. | |
| retention-days: 1 | |
| - name: Update reminder with testing instructions | |
| id: update-reminder-comment | |
| uses: actions/github-script@v7 | |
| if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name && fromJSON(steps.check-test-reminder-comment.outputs.result)['commentId'] != 0 }} | |
| env: | |
| BRANCH_NAME: ${{ github.head_ref }} | |
| DATA: ${{ steps.check-test-reminder-comment.outputs.result }} | |
| with: | |
| script: | | |
| const { BRANCH_NAME, TEST_COMMENT_INDICATOR } = process.env; | |
| const data = JSON.parse( process.env.DATA ); | |
| const commands = data.projects.reduce( ( acc, cur ) => { | |
| return acc += ` | |
| \`\`\` | |
| bin/jetpack-downloader test ${ cur } ${ BRANCH_NAME } | |
| \`\`\` | |
| `; | |
| }, '' ); | |
| let jetpackMuWPcomLine = ''; | |
| if ( data.projects.includes( 'jetpack-mu-wpcom-plugin' ) ) { | |
| jetpackMuWPcomLine = " - For \`jetpack-mu-wpcom\` changes, also add \`define( 'JETPACK_MU_WPCOM_LOAD_VIA_BETA_PLUGIN', true );\` to your \`wp-config.php\` file."; | |
| } | |
| const commentBody = `${ TEST_COMMENT_INDICATOR } | |
| Are you an Automattician? Please test your changes on all WordPress.com environments to help mitigate accidental explosions. | |
| - To test on WoA, go to the Plugins menu on a WordPress.com Simple site. Click on the "Upload" button and follow the upgrade flow to be able to upload, install, and activate [the Jetpack Beta plugin](https://jetpack.com/download-jetpack-beta/). Once the plugin is active, go to Jetpack > Jetpack Beta, select your plugin, and enable the \`${ BRANCH_NAME }\` branch. | |
| ${ jetpackMuWPcomLine } | |
| - To test on Simple, run the following command on your sandbox: | |
| ${ commands } | |
| *Interested in more tips and information?* | |
| - In your local development environment, use the \`jetpack rsync\` command to sync your changes to a WoA dev blog. | |
| - Read more about our development workflow here: PCYsg-eg0-p2 | |
| - Figure out **when your changes will be shipped to customers** here: PCYsg-eg5-p2`; | |
| await github.rest.issues.updateComment( { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: commentBody, | |
| comment_id: +data.commentId, | |
| } ); | |
| jetpack_beta: | |
| name: Create artifact for Jetpack Beta plugin | |
| runs-on: ubuntu-latest | |
| needs: build | |
| if: needs.build.outputs.any_plugins == 'true' | |
| timeout-minutes: 10 # 2021-06-24: Successful runs should take just a few seconds now. But sometimes the upload is slow. | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: monorepo | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: jetpack-build | |
| - name: Extract build archive | |
| run: tar --xz -xvvf build.tar.xz build | |
| - name: Prepare plugin zips | |
| id: prepare | |
| env: | |
| SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| run: | | |
| mkdir work | |
| mkdir zips | |
| # Current version must compare greather than any previously used current version for this PR. | |
| # Assume GH run IDs are monotonic. | |
| VERSUFFIX="${GITHUB_RUN_ID}-g${SHA:0:8}" | |
| ANY_BUILT=false | |
| while IFS=$'\t' read -r SRC MIRROR SLUG; do | |
| echo "::group::$MIRROR (src=$SRC slug=$SLUG)" | |
| if [[ ! -e "build/$MIRROR" ]]; then | |
| echo "Plugin was not built, skipping." | |
| echo "::endgroup::" | |
| continue | |
| fi | |
| if ! grep -q --fixed-strings --line-regexp "$MIRROR" build/mirrors.txt; then | |
| echo "Plugin is not being mirrored in this build, skipping." | |
| echo "::endgroup::" | |
| continue | |
| fi | |
| # The Jetpack Beta Tester plugin needs the base directory name to be like "${SLUG}-dev", so copy it over. | |
| mv "build/$MIRROR" "work/${SLUG}-dev" | |
| # Copy testing docs that are not included in the mirror. | |
| if [[ -e "$SRC/to-test.md" ]]; then | |
| cp "$SRC/to-test.md" "work/${SLUG}-dev/" | |
| fi | |
| # Extract and update version. | |
| CURRENT_VERSION=$(monorepo/tools/plugin-version.sh "work/${SLUG}-dev/")-$VERSUFFIX | |
| echo "Using version $CURRENT_VERSION" | |
| echo "$CURRENT_VERSION" > "work/${SLUG}-dev/version.txt" | |
| # Don't use plugin-version.sh here, updating JETPACK__VERSION would clutter stats. | |
| sed -i -e 's/Version: .*$/Version: '"$CURRENT_VERSION"'/' "work/${SLUG}-dev"/*.php | |
| # Remove .github directory. | |
| rm -rf "work/${SLUG}-dev/.github" | |
| # Zip the plugin | |
| ( cd work && zip -9 -r "../zips/${SLUG}-dev.zip" "${SLUG}-dev" ) | |
| ANY_BUILT=true | |
| echo "::endgroup::" | |
| done < build/plugins.tsv | |
| if ! $ANY_BUILT; then | |
| echo "No plugins were built" | |
| fi | |
| echo "any-built=$ANY_BUILT" >> "$GITHUB_OUTPUT" | |
| - name: Create plugins artifact | |
| uses: actions/upload-artifact@v4 | |
| if: steps.prepare.outputs.any-built == 'true' | |
| with: | |
| name: plugins | |
| path: zips | |
| # Only need to retain for a day since the beta builder slurps it up to distribute. | |
| retention-days: 1 | |
| # Already compressed. | |
| compression-level: 0 | |
| update_mirrors: | |
| name: Push to mirror repos | |
| runs-on: ubuntu-latest | |
| needs: build | |
| if: github.event_name == 'push' && github.repository == 'Automattic/jetpack' | |
| # Not setting a job-level timeout because it would be kind of pointless with the blocking step. Set a step timeout for all other steps instead. | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: monorepo | |
| timeout-minutes: 1 # 2021-01-18: Successful runs seem to take a few seconds | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: jetpack-build | |
| timeout-minutes: 2 # 2022-03-15: Successful runs normally take a few seconds, but on occasion they've been taking 60+ recently. | |
| - name: Extract build archive | |
| run: tar --xz -xvvf build.tar.xz build | |
| timeout-minutes: 1 # 2021-01-18: Successful runs seem to take a few seconds | |
| - name: Wait for prior instances of the workflow to finish | |
| uses: ./monorepo/.github/actions/turnstile | |
| - name: Push changed projects | |
| uses: ./monorepo/projects/github-actions/push-to-mirrors | |
| with: | |
| source-directory: ${{ github.workspace }}/monorepo | |
| token: ${{ secrets.API_TOKEN_GITHUB }} | |
| upstream-ref-since: '2024-04-10' # No point in checking 12 years of earlier commits from before we started adding "Upstream-Ref". | |
| username: matticbot | |
| working-directory: ${{ github.workspace }}/build | |
| timeout-minutes: 10 # 2024-04-11: Successful runs seem to take about a minute. |