Merge branch 'googleapis:main' into feature/cert-rotation-streaming

Author: agrawalradhika-cell

.github/workflows/unittest.yml

@@ -69,11 +69,11 @@ jobs:
       uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
       with:
         python-version: "3.10"
-    - name: Set number of files changes in packages directory
+    - name: Set number of files changes in packages and preview-packages directories
       id: packages
       run: |
-        git diff HEAD~1 -- packages > /dev/null
-        num_files_changed=$(git diff HEAD~1 -- packages | wc -l | tr -d ' ')
+        git diff HEAD~1 -- packages preview-packages > /dev/null
+        num_files_changed=$(git diff HEAD~1 -- packages preview-packages | wc -l | tr -d ' ')
         echo "num_files_changed=${num_files_changed}" >> "$GITHUB_OUTPUT"
     - name: Install coverage
       if: ${{ steps.packages.outputs.num_files_changed > 0 }}
@@ -107,7 +107,7 @@ jobs:
           fi
 
           # Find all modified packages
-          modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
+          modified_packages=$(git diff --name-only HEAD~1 -- packages preview-packages | cut -d/ -f1,2 | sort -u)
 
           failed_packages=()
           passed_packages=()

.kokoro/system.sh

@@ -35,17 +35,39 @@ RETVAL=0
 
 pwd
 
+echo "=== KOKORO VM SCOUTING ==="
+echo "CPU Count:"
+nproc
+echo "--------------------------"
+echo "Detailed CPU Info:"
+lscpu | grep -E "^(Model name|CPU\(s\)|Thread\(s\) per core|Core\(s\) per socket)"
+echo "--------------------------"
+echo "Memory Status:"
+free -h
+echo "--------------------------"
+echo "Disk Usage:"
+df -h /
+echo "=========================="
+
 run_package_test() {
   local package_name=$1
   local package_path="packages/${package_name}"
-  
+
   # Declare local overrides to prevent bleeding into the next loop iteration
   local PROJECT_ID
   local GOOGLE_APPLICATION_CREDENTIALS
   local NOX_FILE
   # Inherit NOX_SESSION from environment to allow configs (like prerelease.cfg) to pass it in
   local NOX_SESSION="${NOX_SESSION}"
 
+  # ISOLATION: Create a unique gcloud config dir for this run
+  local gcloud_config_dir=$(mktemp -d -t "gcloud-config-${package_name}-XXXXXX")
+  local CLOUDSDK_CONFIG="${gcloud_config_dir}"
+
+  # 🪤 TRAP: Ensure cleanup of THIS specific temp dir on exit of this subshell
+  trap 'rm -rf "$gcloud_config_dir"' EXIT
+
+
   echo "------------------------------------------------------------"
   echo "Configuring environment for: ${package_name}"
   echo "------------------------------------------------------------"
@@ -80,11 +102,13 @@ run_package_test() {
   esac
 
   # Export variables for the duration of this function's sub-processes
-  export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION
+  export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION CLOUDSDK_CONFIG
   export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}"
 
-  gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS"
-  gcloud config set project "$PROJECT_ID"
+  # 🛡️ Explicit check: Fail early if auth fails
+  gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" || return 1
+  export CLOUDSDK_CORE_PROJECT="${PROJECT_ID}"
+
 
   # Run the actual test
   pushd "${package_path}" > /dev/null
@@ -93,13 +117,81 @@ run_package_test() {
   local res=$?
   set -e
   popd > /dev/null
-  
+
   return $res
 }
 
+# Analyzes results of parallel test runs, prints summary, and dumps failed logs
+reap_parallel_results() {
+  local retval=0
+  local failed_count=0
+  local passed_count=0
+
+  if [ -z "$LOG_DIR" ]; then
+    echo "Error: LOG_DIR is not set."
+    return 1
+  fi
+
+  # Count failed packages by checking for .failed marker files
+  for failed in "$LOG_DIR"/*.failed; do
+    if [ -f "$failed" ]; then
+      failed_count=$((failed_count + 1))
+    fi
+  done
+
+  local total_tested=${#PACKAGES_TO_TEST[@]}
+  passed_count=$((total_tested - failed_count))
+
+  echo ""
+  echo "=================================================="
+  echo "               TEST RUN SUMMARY                   "
+  echo "=================================================="
+  echo "Total tested: $total_tested"
+  echo "Passed:       $passed_count"
+  echo "Failed:       $failed_count"
+  echo "=================================================="
+
+  local passed_packages=()
+  for pkg in "${PACKAGES_TO_TEST[@]}"; do
+    if [ ! -f "$LOG_DIR/$pkg.failed" ]; then
+      passed_packages+=("$pkg")
+    fi
+  done
+
+  if [ ${#passed_packages[@]} -gt 0 ]; then
+    echo ""
+    echo "PASSED PACKAGES:"
+    printf "%s\n" "${passed_packages[@]}" | sort | sed 's/^/- /'
+  fi
+
+  if [ "$failed_count" -gt 0 ]; then
+    echo ""
+    echo "!!! DETAILED LOGS FOR FAILED PACKAGES !!!"
+    for failed in "$LOG_DIR"/*.failed; do
+      if [ -f "$failed" ]; then
+        local pkg=$(basename "$failed" .failed)
+        echo "--------------------------------------------------"
+        echo "LOGS FOR: $pkg"
+        echo "--------------------------------------------------"
+        if [ -n "$KOKORO_ARTIFACTS_DIR" ] && [ -f "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log" ]; then
+          cat "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log"
+        else
+          cat "$LOG_DIR/$pkg.log"
+        fi
+        echo ""
+      fi
+    done
+    retval=1
+  fi
+  return $retval
+}
+
+
 # A file for running system tests
 system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh"
 
+PACKAGES_TO_TEST=()
+
 # Run system tests for each package with directory packages/*/tests/system
 for path in `find 'packages' \
   \( -type d -wholename 'packages/*/tests/system' \) -o \
@@ -113,6 +205,7 @@ for path in `find 'packages' \
   package_name=${package_name%%/*}
   package_path="packages/${package_name}"
 
+
   # Determine if we should skip based on git diff
   # We always check for changes in these specific versioning/config files
   files_to_check=(
@@ -147,10 +240,54 @@ for path in `find 'packages' \
   set -e
 
   if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then
-      # Call the function - its internal exports won't affect the next loop
-      run_package_test "$package_name" || RETVAL=$?
+      PACKAGES_TO_TEST+=("$package_name")
   else
       echo "No changes in ${package_name} and not a continuous build, skipping."
   fi
 done
+
+# Parallel Execution Logic
+MAX_JOBS=${MAX_JOBS:-4}
+
+# Temporary directory for clean log segregation
+LOG_DIR=$(mktemp -d -t test-logs-XXXXXX)
+# Clean up logs on exit
+trap 'rm -rf "$LOG_DIR"' EXIT
+
+if [ ${#PACKAGES_TO_TEST[@]} -eq 0 ]; then
+  echo "No packages to test."
+  exit 0
+fi
+
+echo "=================================================="
+echo "Starting parallel test execution for ${#PACKAGES_TO_TEST[@]} packages"
+echo "Concurrency limit: ${MAX_JOBS}"
+echo "=================================================="
+
+export LOG_DIR
+export -f run_package_test
+export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR
+
+# Stream package names to xargs for parallel execution
+# -P "$MAX_JOBS" controls concurrency
+# -I {} replaces {} with the package name
+printf '%s\n' "${PACKAGES_TO_TEST[@]}" \
+  | xargs -P "$MAX_JOBS" -I {} \
+    bash -c '
+      pkg="$1"
+      # Determine log location: prefer Sponge artifacts directory if available
+      if [ -n "$KOKORO_ARTIFACTS_DIR" ]; then
+        pkg_log_dir="$KOKORO_ARTIFACTS_DIR/$pkg"
+        mkdir -p "$pkg_log_dir" || { touch "$LOG_DIR/$pkg.failed"; exit 1; }
+        log_file="$pkg_log_dir/sponge_log.log"
+      else
+        log_file="$LOG_DIR/$pkg.log"
+      fi
+
+      # Run test; if it fails, create a .failed file to signal failure to the reaper
+      run_package_test "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed"
+    ' _ "{}"
+
+reap_parallel_results || RETVAL=1
+
 exit ${RETVAL}

.librarian/generator-input/client-post-processing/bigtable-integration.yaml

@@ -200,7 +200,7 @@ replacements:
       "protobuf >= 6.33.5, < 8.0.0",
           "google-cloud-core >= 2.0.0, <3.0.0",
           "grpc-google-iam-v1 >= 0.14.2, <1.0.0",
-          "google-crc32c>=1.6.0, <2.0.0dev",
+          "google-crc32c>=1.6.0, < 2.0.0",
       ]
     count: 1
   - paths: [

.librarian/generator-input/client-post-processing/pubsub-integration.yaml

@@ -374,7 +374,8 @@ replacements:
       - "packages/google-cloud-pubsub/setup.py"
     before: '([ \t]+)("grpc-google-iam-v1.*",\n)\]\nextras = \{\}'
     after: |-
-      \g<1>\g<2>\g<1>"grpcio-status >= 1.51.3",
+      \g<1>\g<2>\g<1>"grpcio-status >= 1.59.0, < 2.0.0",
+      \g<1>"grpcio-status >= 1.75.1, < 2.0.0; python_version >= '3.14'",
       \g<1>"opentelemetry-api >= 1.27.0",
       \g<1>"opentelemetry-sdk >= 1.27.0",
       ]
@@ -519,21 +520,7 @@ replacements:
     before: 'grpc-google-iam-v1==0\.14\.2\n(?!grpcio-status)'
     after: |-
       grpc-google-iam-v1==0.14.2
-      grpcio-status==1.51.3
+      grpcio-status==1.59.0
       opentelemetry-api==1.27.0
       opentelemetry-sdk==1.27.0
     count: 1
-
-  - paths:
-      - "packages/google-cloud-pubsub/testing/constraints-3.10.txt"
-    before: 'grpcio==1.59.0'
-    after: 'grpcio==1.51.3'
-    count: 1
-
-  - paths:
-      - "packages/google-cloud-pubsub/setup.py"
-    before: '([ \t]+)"grpcio >= [0-9\.]+, < 2\.0\.0",\n(?:[ \t]*)"grpcio >= 1\.75\.1, < 2\.0\.0; python_version >= ''3\.14''",\n'
-    after: |-
-      \g<1>"grpcio >= 1.51.3, < 2.0.0; python_version < '3.14'",
-      \g<1>"grpcio >= 1.75.1, < 2.0.0; python_version >= '3.14'",
-    count: 1

.librarian/generator-input/client-post-processing/remove-unused-imports.yaml

@@ -0,0 +1,33 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description: Remove unused imports
+url: https://github.com/googleapis/gapic-generator-python/issues/1833
+replacements:
+  - paths: [
+      packages/google-cloud-gke-hub/tests/unit/gapic/gkehub_v1/test_gke_hub.py
+    ]
+    before: |
+      import google.cloud.gkehub.configmanagement.v1.configmanagement_pb2 as configmanagement_pb2  # type: ignore
+      import google.cloud.gkehub.multiclusteringress.v1.multiclusteringress_pb2 as multiclusteringress_pb2  # type: ignore
+      import google.cloud.gkehub.rbacrolebindingactuation.v1.rbacrolebindingactuation_pb2 as rbacrolebindingactuation_pb2  # type: ignore
+    after:  ""
+    count: 1
+  - paths: [
+      packages/google-cloud-binary-authorization/tests/unit/gapic/binaryauthorization_v1/test_validation_helper_v1.py
+    ]
+    before: |
+      import grafeas.v1.attestation_pb2 as attestation_pb2  # type: ignore
+      import grafeas.v1.common_pb2 as common_pb2  # type: ignore
+    after:  ""
+    count: 1

.librarian/generator-input/client-post-processing/spanner-integration.yaml

@@ -165,7 +165,7 @@ replacements:
           # See https://github.com/googleapis/google-cloud-python/issues/12364
           "google-auth >= 2.14.1, <3.0.0,!=2.24.0,!=2.25.0",
           "google-cloud-core >= 2.0.0, < 3.0.0",
-          "grpcio >= 1.49.1, < 2.0.0",
+          "grpcio >= 1.59.0, < 2.0.0",
           "grpcio >= 1.75.1, < 2.0.0; python_version >= '3.14'",
           "grpc-google-iam-v1 >= 0.14.2, <1.0.0",
           "proto-plus >= 1.26.1, <2.0.0",
@@ -1397,6 +1397,7 @@ replacements:
       protobuf==6.33.5
       google-cloud-core==2.0.0
       grpc-google-iam-v1==0.14.2
+      grpcio-status==1.59.0
       sqlparse==0.4.4
       grpc-interceptor==0.15.4
       opentelemetry-api==1.22.0
@@ -1427,12 +1428,6 @@ replacements:
                 "auto",
                 "tests/unit",
     count: 1
-  - paths: [packages/google-cloud-spanner/testing/constraints-3.10.txt]
-    before: 'grpcio==1.59.0\n(?!grpcio-status)'
-    after: |
-      grpcio==1.49.1
-      grpcio-status==1.49.1
-    count: 1
   - paths: [packages/google-cloud-spanner/.coveragerc]
     before: |
       \[report\]

.librarian/generator-input/client-post-processing/storage-integration.yaml

@@ -597,9 +597,9 @@ replacements:
           # experimental in this SDK. More info in b/465352227
           "grpc": [
               "google-api-core[grpc] >= 2.27.0, < 3.0.0",
-              "grpcio >= 1.59.0, < 2.0.0; python_version < '3.14'",
+              "grpcio >= 1.59.0, < 2.0.0",
               "grpcio >= 1.75.1, < 2.0.0; python_version >= '3.14'",
-              "grpcio-status >= 1.59.0, < 2.0.0; python_version < '3.14'",
+              "grpcio-status >= 1.59.0, < 2.0.0",
               "grpcio-status >= 1.75.1, < 2.0.0; python_version >= '3.14'",
               "proto-plus >= 1.22.3, <2.0.0; python_version < '3.13'",
               "proto-plus >= 1.25.0, <2.0.0; python_version >= '3.13'",

.librarian/generator-input/client-post-processing/temporary-post-processing.yaml

@@ -0,0 +1,27 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description: Temporary post processing until https://github.com/googleapis/google-cloud-python/pull/17914 is released
+url: https://github.com/googleapis/google-cloud-python/pull/17914
+replacements:
+  - paths: [
+      "packages/google-cloud-ces/google/cloud/ces_v1/services/session_service/async_client.py",
+      "packages/google-cloud-ces/google/cloud/ces_v1/services/session_service/client.py",
+      "packages/google-cloud-ces/google/cloud/ces_v1beta/services/session_service/async_client.py",
+      "packages/google-cloud-ces/google/cloud/ces_v1beta/services/session_service/client.py",
+    ]
+    before: |
+      header_params = \{\}
+    after: |
+      header_params: dict[str, str] = {}
+    count: 4

ci/run_conditional_tests.sh

@@ -80,6 +80,7 @@ set -e
 
 subdirs=(
     packages
+    preview-packages
 )
 
 RETVAL=0