Merge branch 'googleapis:main' into grpc-cert-rotation

Author: agrawalradhika-cell

.github/workflows/import-profiler.yml

@@ -16,11 +16,16 @@ jobs:
   import-profile:
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [0, 1, 2, 3, 4, 5, 6, 7] # 8 parallel shards
+    name: import-profile (Shard ${{ matrix.shard }})
     steps:
     - name: Checkout
       uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
       with:
-        fetch-depth: 2
+        fetch-depth: 0 # Fetch git history to find changed packages
         persist-credentials: false
     - name: Setup Python
       uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
@@ -41,5 +46,44 @@ jobs:
         PY_VERSION: "3.15"
         # Workaround: Allows libcst to compile on Python 3.15+ while PyO3 catches up
         PYO3_USE_ABI3_FORWARD_COMPATIBILITY: "1"
+        SHARD_INDEX: ${{ matrix.shard }}
+        TOTAL_SHARDS: 8
       run: |
-        ci/run_conditional_tests.sh
+        TARGET_BRANCH=${TARGET_BRANCH:-main}
+        git fetch origin "${TARGET_BRANCH}" --deepen=200 || true
+        
+        # Get unique list of modified packages under packages/
+        modified_packages=$(git diff --name-only origin/"${TARGET_BRANCH}"... | grep '^packages/' | cut -d/ -f1,2 | sort -u)
+        
+        # Filter packages assigned to this specific shard index
+        idx=0
+        packages_to_test=""
+        for pkg in $modified_packages; do
+          if [ -d "$pkg" ]; then
+            if [ "$((idx % TOTAL_SHARDS))" -eq "${SHARD_INDEX}" ]; then
+              packages_to_test="$packages_to_test $pkg"
+            fi
+            idx=$((idx + 1))
+          fi
+        done
+        
+        # Run tests on the assigned packages
+        if [ -n "$packages_to_test" ]; then
+          echo "Shard ${{ matrix.shard }} running packages: $packages_to_test"
+          PACKAGE_LIST="$packages_to_test" ci/run_conditional_tests.sh
+        else
+          echo "No packages assigned to Shard ${{ matrix.shard }}."
+        fi
+
+  all-import-profiles:
+    needs: import-profile
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check import profile results
+        run: |
+          if [[ "${{ needs.import-profile.result }}" != "success" && "${{ needs.import-profile.result }}" != "skipped" ]]; then
+            echo "Import profiles failed"
+            exit 1
+          fi
+          echo "All import profiles passed or were skipped"

.github/workflows/unittest.yml

@@ -18,6 +18,7 @@ env:
   NOX_ENVDIR: "/tmp/shared_nox_envs"
   NOX_DEFAULT_VENV_BACKEND: "uv"
   UV_VENV_SEED: "1"
+  UV_PRERELEASE: "allow"
   # Note: PARALLEL_WORKERS must remain "1" for unit tests because running pytest concurrently
   # on the same runner VM can cause socket/IPC/process deadlocks across packages.
   PARALLEL_WORKERS: "1"
@@ -83,7 +84,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python: ['3.10', "3.11", "3.12", "3.13", "3.14"]
+        python: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
         package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }}
     name: ${{ matrix.package_shard.is_sharded && format('unit ({0}, {1})', matrix.python, matrix.package_shard.name) || format('unit ({0})', matrix.python) }}
     steps:
@@ -100,10 +101,40 @@ jobs:
       with:
         python-version: ${{ matrix.python }}
         cache: 'pip'
-    - name: Install nox
+        allow-prereleases: true
+    - name: Setup uv
+      uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
+      with:
+        enable-cache: true
+        cache-dependency-glob: 'packages/**/testing/constraints*.txt'
+    - name: Install nox and nox-uv
       run: |
-        python -m pip install --upgrade setuptools pip wheel
-        python -m pip install nox uv
+        uv pip install --system nox nox-uv
+    # Caches local wheels for Python 3.15+ to avoid repeatedly downloading or building heavy libraries.
+    # Follow https://github.com/grpc/grpc/issues/41010 for updates.
+    - name: Cache Built Python 3.15 Wheels
+      if: matrix.python == '3.15'
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+      with:
+        path: .uv-wheel-cache
+        key: ${{ runner.os }}-uv-wheels-3.15-${{ hashFiles('packages/**/testing/constraints*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-uv-wheels-3.15-
+    # TODO: Remove this step once official Python 3.15 wheels for grpcio are published to PyPI.
+    # See https://github.com/grpc/grpc/issues/41010#issuecomment-4848264910 for details.
+    - name: Install grpcio wheel for Python 3.15
+      if: matrix.python == '3.15'
+      run: |
+        mkdir -p .uv-wheel-cache
+        WHEEL_FILE=".uv-wheel-cache/grpcio-1.83.0.dev0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl"
+        if [ ! -f "$WHEEL_FILE" ]; then
+          curl -L "https://packages.grpc.io/archive/2026/06/5ccdab32010f762634e1082d19be53cf60458456-b37e49c7-5e62-4cbe-870a-c599bb907b04/python/grpcio/grpcio-1.83.0.dev0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" -o "$WHEEL_FILE"
+        fi
+        echo "d7706633cd5d97a8b8d3c6ed36c5940722a9b5cbb47f0d10bc433ee27af36ff0  $WHEEL_FILE" | sha256sum -c
+        uv pip install --system "$WHEEL_FILE"
+        echo "UV_FIND_LINKS=${{ github.workspace }}/.uv-wheel-cache" >> $GITHUB_ENV
+        echo "UV_CACHE_DIR=${{ github.workspace }}/.uv-wheel-cache" >> $GITHUB_ENV
+        echo "UV_PRERELEASE=allow" >> $GITHUB_ENV
     - name: Run unit tests for ${{ matrix.package_shard.description }}
       env:
         BUILD_TYPE: presubmit

.kokoro/system.sh

@@ -263,16 +263,25 @@ for path in `find 'packages' \
     files_to_check=("${package_path}")
   fi
 
-  echo "checking changes with 'git diff ${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT} -- ${files_to_check[*]}'"
   set +e
-  # Passing the array expanded as arguments to git diff
+  # Passing the array expanded as arguments to git diff.
   package_modified=$(git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- "${files_to_check[@]}" | wc -l)
   set -e
 
-  if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then
+  states=()
+  [[ "${package_modified}" -gt 0 ]] && states+=("changed")
+  [[ "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]] && states+=("continuous")
+
+  # Join states with a comma
+  state_str=$(IFS=, ; echo "${states[*]}")
+
+  commit_hash="${KOKORO_GITHUB_PULL_REQUEST_COMMIT:-HEAD}"
+
+  if [[ ${#states[@]} -gt 0 ]]; then
+      printf "TEST %-20s %-40s %s\n" "[${state_str}]" "${package_name}" "${commit_hash}"
       PACKAGES_TO_TEST+=("$package_name")
   else
-      echo "No changes in ${package_name} and not a continuous build, skipping."
+      printf "SKIP %-20s %-40s %s\n" "[no_changes]" "${package_name}" "${commit_hash}"
   fi
 done
 

.librarian/generator-input/client-post-processing/bigquery-storage-integration.yaml

@@ -520,3 +520,17 @@ replacements:
           bigquery_storage_v1alpha/types_\n\n
       Changelog
     count: 1
+  - paths: [
+      packages/google-cloud-bigquery-storage/noxfile.py,
+    ]
+    before: |
+      # Install all test dependencies, then install this package in-place.\n
+          constraints_path = str\(
+    after: |
+      # Install all test dependencies, then install this package in-place.
+
+          if session.python == "3.15":
+              session.skip("Skipping 3.15 until wheels are available for pyarrow.")
+
+          constraints_path = str(
+    count: 1

.librarian/generator-input/client-post-processing/storage-integration.yaml

@@ -819,3 +819,18 @@ replacements:
               *session.posargs,
           )
     count: 1
+  - paths: [
+      packages/google-cloud-storage/noxfile.py,
+    ]
+    before: |
+      # Install all test dependencies, then install this package in-place.\n
+          constraints_path = str\(
+    after: |
+      # Install all test dependencies, then install this package in-place.
+
+          if session.python == "3.15":
+              session.skip(
+                  "Skipping 3.15 until compatible wheels are available for google-crc32c"
+              )
+          constraints_path = str(
+    count: 1

ci/report_coverage.sh

@@ -26,14 +26,6 @@ MAX_JOBS=$(nproc)
 
 mkdir -p "${LOG_DIR}"
 
-if [ ! -d "${RESULTS_DIR}" ]; then
-    echo "Error: No coverage results found in ${RESULTS_DIR}."
-    exit 1
-fi
-
-# Unzip any zipped coverage results
-find "$RESULTS_DIR" -type f -name '*.zip' -print0 | xargs -0 -P "${MAX_JOBS}" -I {} unzip -q -o {} -d "$RESULTS_DIR"
-
 # Identify modified packages
 BUILD_TYPE="${BUILD_TYPE:-presubmit}"
 TARGET_BRANCH="${TARGET_BRANCH:-main}"
@@ -51,13 +43,36 @@ else
     modified_packages=$(git diff --name-only HEAD~1 -- ${PACKAGE_DIRS} 2>/dev/null | cut -d/ -f1,2 | sort -u)
 fi
 
+# Check if any modified package requires coverage
+requires_coverage=false
+for pkg in ${modified_packages}; do
+    if [ -f "${pkg}/.coveragerc" ] && grep -Eq "fail_under\s*=\s*0" "${pkg}/.coveragerc"; then
+        continue
+    fi
+    requires_coverage=true
+    break
+done
+
+if [ ! -d "${RESULTS_DIR}" ] || [ -z "$(ls -A "${RESULTS_DIR}" 2>/dev/null)" ]; then
+    if [ "${requires_coverage}" = "true" ]; then
+        echo "Error: No coverage results found in ${RESULTS_DIR}, but some modified packages require coverage."
+        exit 1
+    else
+        echo "No coverage results found, but no modified packages require coverage."
+        exit 0
+    fi
+fi
+
+# Unzip any zipped coverage results
+find "$RESULTS_DIR" -type f -name '*.zip' -print0 | xargs -0 -P "${MAX_JOBS}" -I {} unzip -q -o {} -d "$RESULTS_DIR"
+
 # Function to report coverage for a single package
 report_package_coverage() {
     local pkg=$1
     local pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
     local pkg_log="${LOG_DIR}/${pkg_name_clean}.log"
     local pkg_status="${LOG_DIR}/${pkg_name_clean}.status"
-    
+
     # Use /dev/shm for the combined coverage file to reduce disk I/O
     local pkg_coverage_db="/dev/shm/.coverage.${pkg_name_clean}"
 
@@ -104,7 +119,7 @@ report_package_coverage() {
             echo "No .coveragerc found for ${pkg}, enforcing default" >> "${pkg_log}"
             COVERAGE_FILE="${pkg_coverage_db}" coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
         fi
-        
+
         echo $? > "${pkg_status}"
         popd > /dev/null
     )

ci/run_single_test.sh

@@ -90,6 +90,11 @@ case ${TEST_TYPE} in
             nox -s unit-3.14
             retval=$?
             ;;
+        "3.15")
+            # This is needed to speed up builds
+            nox --force-venv-backend uv -s unit-3.15
+            retval=$?
+            ;;
         *)
             echo "unsupported PY_VERSION"
             exit 1
@@ -98,13 +103,35 @@ case ${TEST_TYPE} in
         ;;
     import_profile)
         if [ -f setup.py ] || [ -f pyproject.toml ]; then
+            PACKAGE_NAME=$(basename $(pwd))
+
+            # TODO(https://github.com/googleapis/google-cloud-python/issues/18035):
+            # Remove this skip once Python 3.15 is officially released and upstream binary wheels
+            # (e.g. numpy, pyarrow, pandas, geopandas, pikepdf) are published on PyPI.
+            # Packages with heavy C/Rust dependencies attempt full source compilation on pre-release Python,
+            # taking 5-10+ minutes before failing due to unreleased CPython 3.15 C-API changes.
+            if [[ "${PY_VERSION}" == "3.15"* ]]; then
+                UNSUPPORTED_PRE_RELEASE_PACKAGES=(
+                    "bigframes"
+                    "pandas-gbq"
+                    "google-cloud-documentai-toolbox"
+                    "db-dtypes"
+                    "bigquery-magics"
+                )
+                for unsupported in "${UNSUPPORTED_PRE_RELEASE_PACKAGES[@]}"; do
+                    if [ "${PACKAGE_NAME}" = "${unsupported}" ]; then
+                        echo "WARNING: Skipping import_profile for ${PACKAGE_NAME}: package has heavy C/Rust dependencies not yet supported on pre-release Python ${PY_VERSION}."
+                        exit 0
+                    fi
+                done
+            fi
+
             echo "Creating temporary virtualenv for import profile..."
             python3 -m venv .venv-profiler
             source .venv-profiler/bin/activate
             export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1
             python -m pip install --upgrade pip setuptools
 
-            PACKAGE_NAME=$(basename $(pwd))
             PROFILER_TEMP_DIR=$(mktemp -d)
             cp ../../scripts/import_profiler/profiler.py "${PROFILER_TEMP_DIR}/profiler.py"
             PROFILER_SCRIPT="${PROFILER_TEMP_DIR}/profiler.py"
@@ -121,13 +148,26 @@ case ${TEST_TYPE} in
                     WORKTREE_DIR=$(mktemp -d)
                     rmdir "${WORKTREE_DIR}"
                     if git worktree add "${WORKTREE_DIR}" "${BASELINE_COMMIT}" 2>/dev/null; then
-                        (
+                        if ! (
                             cd "${WORKTREE_DIR}/${REPO_PREFIX}"
                             if [ -f setup.py ] || [ -f pyproject.toml ]; then
-                                pip install -e .
-                                python "${PROFILER_SCRIPT}" --package "${PACKAGE_NAME}" --iterations 11 --csv "${BASELINE_CSV}"
+                                if pip install -e . ; then
+                                    echo "INFO: Successfully installed baseline dependencies for ${PACKAGE_NAME}."
+                                    python "${PROFILER_SCRIPT}" --package "${PACKAGE_NAME}" --iterations 11 --csv "${BASELINE_CSV}"
+                                    if [ $? -eq 0 ]; then
+                                        echo "INFO: Successfully ran baseline profiler for ${PACKAGE_NAME}."
+                                    fi
+                                elif [[ "${PY_VERSION}" != "3.15"* ]]; then
+                                    exit 1
+                                fi
                             fi
-                        )
+                        ); then
+                            git worktree remove -f "${WORKTREE_DIR}"
+                            deactivate
+                            rm -rf .venv-profiler
+                            rm -rf "${PROFILER_TEMP_DIR}"
+                            exit 1
+                        fi
                         git worktree remove -f "${WORKTREE_DIR}"
                     else
                         echo "Failed to create git worktree for baseline. Skipping baseline generation."
@@ -137,14 +177,28 @@ case ${TEST_TYPE} in
                 fi
             fi
 
-            pip install -e .
-            
-            if [ -f "${BASELINE_CSV}" ]; then
-                python ${PROFILER_SCRIPT} --package ${PACKAGE_NAME} --iterations 11 --fail-threshold 5000 --diff-baseline "${BASELINE_CSV}" --diff-threshold 100
+            # TODO(https://github.com/googleapis/google-cloud-python/issues/18035):
+            # Clean up this fallback once Python 3.15 is officially released and upstream binary wheels are available on PyPI.
+            # On pre-release Python versions, packages with complex C/Rust dependencies (e.g. bigframes) fail during pip install due to missing pre-built wheels.
+            if ! pip install -e . ; then
+                if [[ "${PY_VERSION}" == "3.15"* ]]; then
+                    echo "WARNING: Could not install dependencies for ${PACKAGE_NAME} on Python ${PY_VERSION} (missing pre-built binary wheels for pre-release Python). Skipping import_profile."
+                    retval=0
+                else
+                    retval=1
+                fi
             else
-                python ${PROFILER_SCRIPT} --package ${PACKAGE_NAME} --iterations 11 --fail-threshold 5000
+                echo "INFO: Successfully installed dependencies for ${PACKAGE_NAME} on Python ${PY_VERSION}."
+                if [ -f "${BASELINE_CSV}" ]; then
+                    python ${PROFILER_SCRIPT} --package ${PACKAGE_NAME} --iterations 11 --fail-threshold 5000 --diff-baseline "${BASELINE_CSV}" --diff-threshold 100
+                else
+                    python ${PROFILER_SCRIPT} --package ${PACKAGE_NAME} --iterations 11 --fail-threshold 5000
+                fi
+                retval=$?
+                if [ $retval -eq 0 ]; then
+                    echo "INFO: Successfully completed import_profile for ${PACKAGE_NAME}."
+                fi
             fi
-            retval=$?
             deactivate
             rm -rf .venv-profiler
             rm -rf "${PROFILER_TEMP_DIR}"

librarian.yaml

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 language: python
-version: v0.31.0
+version: v0.35.0
 repo: googleapis/google-cloud-python
 sources:
   googleapis: