Skip to content

Commit 6487543

Browse files
whoweswhowes
authored
chore: restore nightly protobuf compatibility check (#13884)
Part of Issue #13867 - restore/adapt previous GitHub Action config (see [diff against last revision before deletion](https://gist.github.com/whowes/c426aa4e4c65e125aa3a8391744382ff)) - adjust logic and nomenclature for using modules of the monorepo v. previous separate repos - add showcase testing - address findings from Zizmor static analysis and Gemini code review Passing manual workflow run: https://github.com/googleapis/google-cloud-java/actions/runs/31056514739
1 parent 3fd0de2 commit 6487543

5 files changed

Lines changed: 198 additions & 80 deletions

File tree

‎.github/workflows/sdk-platform-java-downstream_protobuf_compatibility_check_nightly.yaml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
permissions:
2+
contents: read
3+
4+
on:
5+
pull_request:
6+
# Runs on PRs targeting main, but will be filtered for Release PRs
7+
branches:
8+
- 'main'
9+
workflow_dispatch:
10+
inputs:
11+
protobuf_runtime_versions:
12+
description: 'Comma separated list of Protobuf-Java versions (i.e. "3.25.x","4.x.y"). Leave empty to use version from pom-parent.'
13+
required: false
14+
schedule:
15+
- cron: '0 1 * * *' # Nightly at 1am
16+
17+
# This job intends to test the compatibility of Protobuf runtime version against modules in the monorepo.
18+
name: sdk-platform-java Downstream Protobuf Compatibility Check Nightly
19+
jobs:
20+
setup:
21+
if: github.head_ref == 'release-please--branches--main' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
22+
runs-on: ubuntu-22.04
23+
permissions:
24+
contents: read
25+
outputs:
26+
protobuf-versions: ${{ steps.resolve-versions.outputs.protobuf-versions }}
27+
steps:
28+
- name: Checkout monorepo
29+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
30+
with:
31+
persist-credentials: false
32+
- uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
33+
with:
34+
java-version: 11
35+
distribution: temurin
36+
cache: maven
37+
- id: resolve-versions
38+
name: Resolve Protobuf versions
39+
env:
40+
INPUT_VERSIONS: ${{ inputs.protobuf_runtime_versions }}
41+
run: |
42+
if [ -n "${INPUT_VERSIONS}" ]; then
43+
echo "protobuf-versions=[${INPUT_VERSIONS}]" >> "$GITHUB_OUTPUT"
44+
else
45+
VERSION=$(mvn -f sdk-platform-java/gapic-generator-java-pom-parent/pom.xml help:evaluate -Dexpression=protobuf.version -q -DforceStdout)
46+
echo "protobuf-versions=[\"${VERSION}\"]" >> "$GITHUB_OUTPUT"
47+
fi
48+
49+
downstream-protobuf-test:
50+
needs: setup
51+
# This job runs if any of the three conditions match:
52+
# 1. PR is raised from Release-Please (PR comes from branch: release-please--branches-main)
53+
# 2. Job is invoked by the nightly job (scheduled event)
54+
# 3. Job is manually invoked via Github UI (workflow_dispatch event)
55+
if: github.head_ref == 'release-please--branches--main' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
56+
runs-on: ubuntu-22.04
57+
permissions:
58+
contents: read
59+
strategy:
60+
fail-fast: false
61+
matrix:
62+
module:
63+
- java-bigtable
64+
- java-bigquery
65+
- java-bigquerystorage
66+
- java-datastore
67+
- java-firestore
68+
- java-grafeas
69+
- java-logging
70+
- java-logging-logback
71+
- java-pubsub
72+
- java-resourcemanager
73+
- java-showcase
74+
- java-spanner
75+
- java-spanner-jdbc
76+
- java-storage
77+
- java-storage-nio
78+
- java-translate
79+
protobuf-version: ${{ fromJSON(needs.setup.outputs.protobuf-versions) }}
80+
steps:
81+
- name: Checkout monorepo
82+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
83+
with:
84+
persist-credentials: false
85+
- uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
86+
with:
87+
java-version: 8
88+
distribution: temurin
89+
- run: echo "JAVA8_HOME=${JAVA_HOME}" >> $GITHUB_ENV
90+
# Java Client Libraries are compiled with Java 11 and target Java 8. Java 11 is required because GraalVM
91+
# minimum support is for Java 11.
92+
- uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
93+
with:
94+
java-version: 11
95+
distribution: temurin
96+
cache: maven
97+
- name: Install monorepo modules
98+
shell: bash
99+
run: .kokoro/build.sh
100+
env:
101+
BUILD_SUBDIR: ${{ matrix.module }}
102+
JOB_TYPE: install
103+
- name: Print Protobuf-Java testing version
104+
run: echo "Testing with Protobuf-Java v${PROTOBUF_RUNTIME_VERSION}"
105+
env:
106+
PROTOBUF_RUNTIME_VERSION: ${{ matrix.protobuf-version }}
107+
- name: Perform downstream source compatibility testing
108+
run: ./sdk-platform-java/.kokoro/nightly/downstream-protobuf-source-compatibility.sh
109+
env:
110+
MODULES_UNDER_TEST: ${{ matrix.module }}
111+
PROTOBUF_RUNTIME_VERSION: ${{ matrix.protobuf-version }}
112+
GOOGLE_CLOUD_PROJECT: placeholder-test-project
113+
- name: Perform downstream binary compatibility testing
114+
run: ./sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh
115+
env:
116+
MODULES_UNDER_TEST: ${{ matrix.module }}
117+
PROTOBUF_RUNTIME_VERSION: ${{ matrix.protobuf-version }}

‎.kokoro/common.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,9 @@ function install_modules() {
418418
always_install_deps_list=(
419419
'java-monitoring/google-cloud-monitoring'
420420
'java-monitoring/google-cloud-monitoring-bom'
421+
'java-kms/google-cloud-kms'
422+
'java-kms/proto-google-cloud-kms-v1'
423+
'java-kms/grpc-google-cloud-kms-v1'
421424
'google-auth-library-java/appengine'
422425
'google-auth-library-java/bom'
423426
'google-auth-library-java/cab-token-generator'

‎sdk-platform-java/.kokoro/nightly/common.sh‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,12 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
# For google-cloud-java, only test specific handwritten libraries included in the monorepo. This is to
17-
# help speed up the execution as building the entire repo is an expensive operation. Specify the nested
18-
# `google-cloud-*` path (except for grafeas as it doesn't have one) because maven -pl will only build the
19-
# specified folder (i.e. parent folder) and ignore all the related sub-modules inside
20-
google_cloud_java_handwritten_maven_args="java-grafeas,java-vertexai/google-cloud-vertexai,java-resourcemanager/google-cloud-resourcemanager,java-translate/google-cloud-translate"
21-
2216
# Checks that the protobuf compatibility scripts provide non-empty input
2317
function validate_protobuf_compatibility_script_inputs {
24-
# Comma-delimited list of repos to test
25-
if [ -z "${REPOS_UNDER_TEST}" ]; then
26-
echo "REPOS_UNDER_TEST Env Var must be set. This script expects a"
27-
echo "comma-delimited list: i.e REPOS_UNDER_TEST=\"java-bigtable,java-bigquery\""
18+
# Comma-delimited list of modules to test
19+
if [ -z "${MODULES_UNDER_TEST}" ]; then
20+
echo "MODULES_UNDER_TEST Env Var must be set. This script expects a"
21+
echo "comma-delimited list: i.e MODULES_UNDER_TEST=\"java-bigtable,java-bigquery\""
2822
exit 1
2923
fi
3024

‎sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh‎

Lines changed: 43 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -20,44 +20,53 @@ source "${scriptDir}/common.sh"
2020

2121
validate_protobuf_compatibility_script_inputs
2222

23+
monorepoRoot=$(realpath "${scriptDir}/../../..")
24+
2325
# Declare a map of downstream handwritten libraries and the relevant artifacts to test. The map stores a
24-
# K/V pairing of (Key: repo name, Value: comma separate list of Group ID:Artifact ID pairings). Note: The
25-
# value list doesn't hold the version and this needs to be parsed from the repo's versions.txt file
26-
declare -A repo_linkage_checker_arguments
27-
repo_linkage_checker_arguments["google-cloud-java"]="io.grafeas:grafeas,com.google.cloud:google-cloud-vertexai,com.google.cloud:google-cloud-resourcemanager,com.google.cloud:google-cloud-translate,com.google.api.grpc:grpc-google-cloud-vertexai-v1,com.google.api.grpc:grpc-google-cloud-vertexai-v1beta1,com.google.api.grpc:grpc-google-cloud-resourcemanager-v3,com.google.api.grpc:grpc-google-cloud-translate-v3,com.google.api.grpc:grpc-google-cloud-translate-v3beta1"
28-
repo_linkage_checker_arguments["java-bigtable"]="com.google.cloud:google-cloud-bigtable,com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2,com.google.api.grpc:grpc-google-cloud-bigtable-v2"
29-
repo_linkage_checker_arguments["java-bigquery"]="com.google.cloud:google-cloud-bigquery"
30-
repo_linkage_checker_arguments["java-bigquerystorage"]="com.google.cloud:google-cloud-bigquerystorage,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1alpha"
31-
repo_linkage_checker_arguments["java-datastore"]="com.google.cloud:google-cloud-datastore,com.google.cloud.datastre:datastore-v1-proto-client,com.google.api.grpc:grpc-google-cloud-datastore-admin-v1"
32-
repo_linkage_checker_arguments["java-firestore"]="com.google.cloud:google-cloud-firestore,com.google.cloud:google-cloud-firestore-admin,com.google.api.grpc:grpc-google-cloud-firestore-admin-v1,com.google.api.grpc:grpc-google-cloud-firestore-v1"
33-
repo_linkage_checker_arguments["java-logging"]="com.google.cloud:google-cloud-logging,com.google.api.grpc:grpc-google-cloud-logging-v2"
34-
repo_linkage_checker_arguments["java-logging-logback"]="com.google.cloud:google-cloud-logging-logback"
35-
repo_linkage_checker_arguments["java-pubsub"]="com.google.cloud:google-cloud-pubsub,com.google.api.grpc:grpc-google-cloud-pubsub-v1"
36-
repo_linkage_checker_arguments["java-pubsublite"]="com.google.cloud:google-cloud-pubsublite,com.google.api.grpc:grpc-google-cloud-pubsublite-v1"
37-
repo_linkage_checker_arguments["java-spanner-jdbc"]="com.google.cloud:google-cloud-spanner-jdbc"
38-
repo_linkage_checker_arguments["java-spanner"]="com.google.cloud:google-cloud-spanner,com.google.cloud:google-cloud-spanner-executor,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,com.google.api.grpc:grpc-google-cloud-spanner-executor-v1"
39-
repo_linkage_checker_arguments["java-storage"]="com.google.cloud:google-cloud-storage,com.google.api.grpc:gapic-google-cloud-storage-v2,com.google.api.grpc:grpc-google-cloud-storage-v2,com.google.cloud:google-cloud-storage-control,com.google.api.grpc:grpc-google-cloud-storage-control-v2"
40-
repo_linkage_checker_arguments["java-storage-nio"]="com.google.cloud:google-cloud-nio"
26+
# K/V pairing of (Key: module name, Value: comma separate list of Group ID:Artifact ID pairings). Note: The
27+
# value list doesn't hold the version and this needs to be parsed from the monorepo's versions.txt file
28+
declare -A module_linkage_checker_arguments
29+
module_linkage_checker_arguments["java-grafeas"]="io.grafeas:grafeas"
30+
module_linkage_checker_arguments["java-resourcemanager"]="com.google.cloud:google-cloud-resourcemanager,com.google.api.grpc:grpc-google-cloud-resourcemanager-v3"
31+
module_linkage_checker_arguments["java-translate"]="com.google.cloud:google-cloud-translate,com.google.api.grpc:grpc-google-cloud-translate-v3,com.google.api.grpc:grpc-google-cloud-translate-v3beta1"
32+
module_linkage_checker_arguments["java-bigtable"]="com.google.cloud:google-cloud-bigtable,com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2,com.google.api.grpc:grpc-google-cloud-bigtable-v2"
33+
module_linkage_checker_arguments["java-bigquery"]="com.google.cloud:google-cloud-bigquery"
34+
module_linkage_checker_arguments["java-bigquerystorage"]="com.google.cloud:google-cloud-bigquerystorage,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1alpha"
35+
module_linkage_checker_arguments["java-datastore"]="com.google.cloud:google-cloud-datastore,com.google.cloud.datastre:datastore-v1-proto-client,com.google.api.grpc:grpc-google-cloud-datastore-admin-v1"
36+
module_linkage_checker_arguments["java-firestore"]="com.google.cloud:google-cloud-firestore,com.google.cloud:google-cloud-firestore-admin,com.google.api.grpc:grpc-google-cloud-firestore-admin-v1,com.google.api.grpc:grpc-google-cloud-firestore-v1"
37+
module_linkage_checker_arguments["java-logging"]="com.google.cloud:google-cloud-logging,com.google.api.grpc:grpc-google-cloud-logging-v2"
38+
module_linkage_checker_arguments["java-logging-logback"]="com.google.cloud:google-cloud-logging-logback"
39+
module_linkage_checker_arguments["java-pubsub"]="com.google.cloud:google-cloud-pubsub,com.google.api.grpc:grpc-google-cloud-pubsub-v1"
40+
module_linkage_checker_arguments["java-showcase"]="com.google.cloud:gapic-showcase,com.google.api.grpc:grpc-gapic-showcase-v1beta1"
41+
module_linkage_checker_arguments["java-spanner-jdbc"]="com.google.cloud:google-cloud-spanner-jdbc"
42+
module_linkage_checker_arguments["java-spanner"]="com.google.cloud:google-cloud-spanner,com.google.cloud:google-cloud-spanner-executor,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,com.google.api.grpc:grpc-google-cloud-spanner-executor-v1"
43+
module_linkage_checker_arguments["java-storage"]="com.google.cloud:google-cloud-storage,com.google.api.grpc:gapic-google-cloud-storage-v2,com.google.api.grpc:grpc-google-cloud-storage-v2,com.google.cloud:google-cloud-storage-control,com.google.api.grpc:grpc-google-cloud-storage-control-v2"
44+
module_linkage_checker_arguments["java-storage-nio"]="com.google.cloud:google-cloud-nio"
4145

4246
# This function requires access to the versions.txt to retrieve the versions for the artifacts
4347
# It will try to match the artifact_id in the versions.txt file and attach it to form the GAV
4448
# The GAV list is required by Linkage Checker as program arguments
4549
function build_program_arguments() {
46-
artifact_list="${repo_linkage_checker_arguments[$1]}"
50+
artifact_list="${module_linkage_checker_arguments[$1]}"
4751

4852
for artifact in ${artifact_list//,/ }; do # Split on comma
4953
artifact_id=$(echo "${artifact}" | cut -d ':' -f2)
5054

5155
# The grep query tries to match `{artifact_id}:{released_version}:{current_version}`.
5256
# The artifact_id must be exact otherwise multiple entries may match
53-
version=$(cat "versions.txt" | grep -E "^${artifact_id}:.*:.*$" | cut -d ':' -f3)
54-
repo_gav_coordinate="${artifact}:${version}"
57+
version=$(cat "${monorepoRoot}/versions.txt" | grep -E "^${artifact_id}:.*:.*$" | cut -d ':' -f3 || true)
58+
# Unreleased internal test modules like java-showcase are not tracked in versions.txt,
59+
# so fallback to 0.0.1-SNAPSHOT for linkage checking.
60+
if [ -z "${version}" ]; then
61+
version="0.0.1-SNAPSHOT"
62+
fi
63+
module_gav_coordinate="${artifact}:${version}"
5564

5665
# The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}`
5766
if [ -z "${linkage_checker_arguments}" ]; then
58-
linkage_checker_arguments="${repo_gav_coordinate}"
67+
linkage_checker_arguments="${module_gav_coordinate}"
5968
else
60-
linkage_checker_arguments="${linkage_checker_arguments},${repo_gav_coordinate}"
69+
linkage_checker_arguments="${linkage_checker_arguments},${module_gav_coordinate}"
6170
fi
6271
done
6372
}
@@ -70,34 +79,29 @@ mvn -B -ntp clean compile -T 1C
7079
# Linkage Checker tool resides in the /dependencies subfolder
7180
pushd dependencies
7281

73-
# REPOS_UNDER_TEST Env Var accepts a comma separated list of googleapis repos to test. For Github CI,
74-
# this will be a single repo as Github will build a matrix of repos with each repo being tested in parallel.
75-
# For local invocation, you can pass a list of repos to test multiple repos together.
76-
for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma
77-
# Perform testing on main (with latest changes). Shallow copy as history is not important
78-
git clone "https://github.com/googleapis/${repo}.git" --depth=1
79-
pushd "${repo}"
80-
81-
if [ "${repo}" == "google-cloud-java" ]; then
82-
# The `-am` command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules)
83-
mvn clean install -B -V -ntp -T 1C -DskipTests -Dclirr.skip -Denforcer.skip -Dmaven.javadoc.skip \
84-
-pl "${google_cloud_java_handwritten_maven_args}" -am
85-
else
86-
# Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control)
87-
mvn clean install -B -V -ntp -T 1C -DskipTests -Dclirr.skip -Denforcer.skip -Dmaven.javadoc.skip
82+
# MODULES_UNDER_TEST Env Var accepts a comma separated list of monorepo submodules to test. For Github CI,
83+
# this will be a single module as Github will build a matrix of modules with each being tested in parallel.
84+
# For local invocation, you can pass a list of modules to test multiple modules together.
85+
for module in ${MODULES_UNDER_TEST//,/ }; do # Split on comma
86+
module_dir="${monorepoRoot}/${module}"
87+
if [ ! -d "${module_dir}" ]; then
88+
echo "Directory ${module_dir} does not exist. Skipping or failed." >&2
89+
exit 1
8890
fi
8991

92+
pushd "${module_dir}"
93+
# Module artifacts are installed to ~/.m2 prior to running linkage checker (e.g., via .kokoro/build.sh)
9094

9195
linkage_checker_arguments=""
92-
build_program_arguments "${repo}"
96+
build_program_arguments "${module}"
9397

9498
# Linkage Checker /dependencies
9599
popd
96100

97101
echo "Artifact List: ${linkage_checker_arguments}"
98102
# The `-s` argument filters the linkage check problems that stem from the artifact
99103
program_args="-r --artifacts ${linkage_checker_arguments},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${linkage_checker_arguments}"
100-
echo "Running Linkage Checker on the repo's handwritten modules"
104+
echo "Running Linkage Checker on the module's handwritten artifacts"
101105
echo "Linkage Checker Program Arguments: ${program_args}"
102106
mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker
103107
done

0 commit comments

Comments
 (0)