From 622f8ac4a75f220056d81801d1f274f25daf956f Mon Sep 17 00:00:00 2001 From: Shahin Saadati Date: Wed, 29 Jul 2026 13:15:47 -0700 Subject: [PATCH 1/5] feat: add retail product search skill with Vector Search integration and supporting scripts --- skills/retail/retail-product-search/EVAL.yaml | 209 +++++++ skills/retail/retail-product-search/README.md | 80 +++ skills/retail/retail-product-search/SKILL.md | 271 +++++++++ .../assets/design-spec.md | 37 ++ .../assets/sample-products.csv | 6 + .../retail-product-search/pyproject.toml | 39 ++ .../references/agent-example.md | 71 +++ .../references/architecture.md | 49 ++ .../references/dependencies.md | 27 + .../references/ingestion-scripts.md | 165 ++++++ .../references/install-paths.md | 44 ++ .../references/troubleshooting.md | 39 ++ .../retail-product-search/scripts/_paths.py | 35 ++ .../scripts/_setup_utils.py | 59 ++ .../retail-product-search/scripts/agent.py | 144 +++++ .../scripts/bootstrap.sh | 110 ++++ .../retail-product-search/scripts/cleanup.py | 533 ++++++++++++++++++ .../scripts/ingest_bigquery.py | 467 +++++++++++++++ .../scripts/ingest_vertex_search.py | 467 +++++++++++++++ .../scripts/retrievers.py | 163 ++++++ .../retail-product-search/scripts/setup.py | 246 ++++++++ .../scripts/validate_schema.py | 281 +++++++++ 22 files changed, 3542 insertions(+) create mode 100644 skills/retail/retail-product-search/EVAL.yaml create mode 100644 skills/retail/retail-product-search/README.md create mode 100644 skills/retail/retail-product-search/SKILL.md create mode 100644 skills/retail/retail-product-search/assets/design-spec.md create mode 100644 skills/retail/retail-product-search/assets/sample-products.csv create mode 100644 skills/retail/retail-product-search/pyproject.toml create mode 100644 skills/retail/retail-product-search/references/agent-example.md create mode 100644 skills/retail/retail-product-search/references/architecture.md create mode 100644 skills/retail/retail-product-search/references/dependencies.md create mode 100644 skills/retail/retail-product-search/references/ingestion-scripts.md create mode 100644 skills/retail/retail-product-search/references/install-paths.md create mode 100644 skills/retail/retail-product-search/references/troubleshooting.md create mode 100644 skills/retail/retail-product-search/scripts/_paths.py create mode 100644 skills/retail/retail-product-search/scripts/_setup_utils.py create mode 100644 skills/retail/retail-product-search/scripts/agent.py create mode 100755 skills/retail/retail-product-search/scripts/bootstrap.sh create mode 100644 skills/retail/retail-product-search/scripts/cleanup.py create mode 100644 skills/retail/retail-product-search/scripts/ingest_bigquery.py create mode 100644 skills/retail/retail-product-search/scripts/ingest_vertex_search.py create mode 100644 skills/retail/retail-product-search/scripts/retrievers.py create mode 100644 skills/retail/retail-product-search/scripts/setup.py create mode 100644 skills/retail/retail-product-search/scripts/validate_schema.py diff --git a/skills/retail/retail-product-search/EVAL.yaml b/skills/retail/retail-product-search/EVAL.yaml new file mode 100644 index 0000000000..552f49928a --- /dev/null +++ b/skills/retail/retail-product-search/EVAL.yaml @@ -0,0 +1,209 @@ +# uses_catalog: true +cases: +- id: headphones_under_200 + prompt: "I'm looking for wireless headphones under $200, what do you have?" + metadata: + skill_name: "retail-product-search" + products: + - "Vertex AI Vector Search" + - "BigQuery" + rubric: + - "The response MUST return product results that include wireless headphones from the catalog." + - "The response MUST only include products priced under $200." + - "The response MUST include key product details: name, price, and description for each result." + - "The response MUST NOT hallucinate product names, brands, or prices not present in the catalog." + - "The response SHOULD use semantic search to match the user intent, not just keyword matching." + assertions: + - type: contains_product + field: name + value: Wireless Headphones + - type: price_under + value: 200 + - type: tool_called + tool: retrieve_docs + - type: no_hallucination + forbidden: + - Bose + - AirPods + - Sennheiser + +- id: cheapest_item + prompt: "What is the cheapest product you have available right now?" + metadata: + skill_name: "retail-product-search" + products: + - "Vertex AI Vector Search" + - "BigQuery" + rubric: + - "The response MUST identify the lowest-priced product in the catalog." + - "The response MUST include the exact price as a dollar amount." + - "The response MUST include the product name and a brief description." + - "The response MUST NOT fabricate prices or products not present in the catalog." + assertions: + - type: tool_called + tool: retrieve_docs + - type: mentions_price + value: true + - type: response_not_empty + +- id: keyboard_search + prompt: "I need a mechanical keyboard for coding, what do you recommend?" + metadata: + skill_name: "retail-product-search" + products: + - "Vertex AI Vector Search" + - "BigQuery" + rubric: + - "The response MUST return at least one mechanical keyboard product from the catalog." + - "The response MUST include the brand and price for each returned product." + - "The response MUST provide details relevant to coding use (e.g., key switch type, ergonomics)." + - "The response MUST NOT recommend products outside the catalog inventory." + - "The response SHOULD explain why the recommended product suits coding workflows." + assertions: + - type: contains_product + field: name + value: Mechanical Keyboard + - type: mentions_field + field: brand + - type: mentions_field + field: price + - type: tool_called + tool: retrieve_docs + +- id: category_filter + prompt: "Show me all audio products you carry." + metadata: + skill_name: "retail-product-search" + products: + - "Vertex AI Vector Search" + - "BigQuery" + rubric: + - "The response MUST return products categorized under Audio from the catalog." + - "The response MUST include the name and price for each audio product listed." + - "The response MUST NOT include products from non-audio categories in the results." + - "The response MUST NOT hallucinate audio products not present in the catalog (e.g., speaker systems, turntables)." + assertions: + - type: tool_called + tool: retrieve_docs + - type: contains_product + field: category + value: Audio + - type: no_hallucination + forbidden: + - speaker system + - turntable + +- id: brand_search + prompt: "Do you have any Sony products in your store?" + metadata: + skill_name: "retail-product-search" + products: + - "Vertex AI Vector Search" + - "BigQuery" + rubric: + - "The response MUST return products from the Sony brand present in the catalog." + - "The response MUST include the product name, category, and price for each Sony item." + - "The response MUST NOT fabricate Sony products that are not in the catalog." + - "The response MUST NOT include products from other brands unless explicitly relevant for comparison." + assertions: + - type: tool_called + tool: retrieve_docs + - type: contains_product + field: brand + value: Sony + - type: response_not_empty + +- id: no_results_handling + prompt: "Do you have any cars for sale?" + metadata: + skill_name: "retail-product-search" + products: + - "Vertex AI Vector Search" + - "BigQuery" + rubric: + - "The response MUST clearly state that no matching products were found in the catalog." + - "The response MUST NOT fabricate or hallucinate car products." + - "The response SHOULD suggest alternative actions (e.g., browsing other categories or refining the query)." + - "The response MUST remain helpful and polite even when no results are available." + assertions: + - type: tool_called + tool: retrieve_docs + - type: graceful_no_results + +- id: comparison_query + prompt: "Can you compare the keyboard and the headphones you have?" + metadata: + skill_name: "retail-product-search" + products: + - "Vertex AI Vector Search" + - "BigQuery" + rubric: + - "The response MUST include a side-by-side comparison covering at least name, price, and category." + - "The response MUST reference actual products from the catalog for both the keyboard and headphones." + - "The response MUST NOT invent specifications or features not listed in the catalog." + - "The response SHOULD highlight key differences that help the user make a purchase decision." + assertions: + - type: tool_called + tool: retrieve_docs + - type: mentions_field + field: price + - type: response_min_length + value: 50 + +- id: price_range + prompt: "Show me products between $50 and $150." + metadata: + skill_name: "retail-product-search" + products: + - "Vertex AI Vector Search" + - "BigQuery" + rubric: + - "The response MUST only include products with prices between $50 and $150 inclusive." + - "The response MUST display the price for each returned product." + - "The response MUST NOT return products priced outside the specified range." + - "The response MUST NOT hallucinate products or prices not present in the catalog." + assertions: + - type: tool_called + tool: retrieve_docs + - type: mentions_price + value: true + - type: response_not_empty + +- id: vague_query_handling + prompt: "I need something for my desk, any suggestions?" + metadata: + skill_name: "retail-product-search" + products: + - "Vertex AI Vector Search" + - "BigQuery" + rubric: + - "The response MUST return relevant desk-related products from the catalog (e.g., keyboards, monitors)." + - "The response MUST include product name and price for each suggestion." + - "The response MUST NOT hallucinate products not in the catalog." + - "The response SHOULD interpret the vague query using semantic search rather than exact keyword matching." + assertions: + - type: tool_called + tool: retrieve_docs + - type: response_not_empty + - type: response_min_length + value: 30 + +- id: skill_covers_vector_search + prompt: "How does the product search system work under the hood?" + metadata: + skill_name: "retail-product-search" + products: + - "Vertex AI Vector Search" + - "BigQuery" + - "Cloud Storage" + rubric: + - "The response MUST mention vector search as part of the product retrieval mechanism." + - "The response MUST reference BigQuery as the data storage or query layer." + - "The response MUST NOT claim the system uses technologies or services it does not employ." + - "The response SHOULD explain how semantic embeddings power product matching." + assertions: + - type: skill_covers + topic: vector search + - type: skill_covers + topic: bigquery + - type: response_not_empty diff --git a/skills/retail/retail-product-search/README.md b/skills/retail/retail-product-search/README.md new file mode 100644 index 0000000000..ae730f31ab --- /dev/null +++ b/skills/retail/retail-product-search/README.md @@ -0,0 +1,80 @@ +# Retail Product Search + +Semantic product search agent on Google Cloud (Vertex AI Vector Search, +BigQuery, embeddings). Use to build e-commerce search, catalog discovery, or +shopping assistant agents. + +## Install + +```bash +npx skills add tanvisinghal-0105/solution_skills --skill retail-product-search +``` + +Hosts that support the skills spec (Claude Code, Gemini CLI, Codex, ...) +auto-detect and drop `SKILL.md` into their skills directory +(e.g. `~/.claude/skills/`, `~/.agents/skills/`). + +## Prerequisites + +- Python 3.10+ +- [`gcloud` CLI](https://cloud.google.com/sdk/docs/install) with ADC + configured (`gcloud auth application-default login`) +- A GCP project with billing enabled and BigQuery + Vertex AI APIs on: + ```bash + gcloud services enable bigquery.googleapis.com aiplatform.googleapis.com + ``` + +## Run + +In a fresh workspace, launch your AI coding agent and trigger the skill: + +``` +Use the retail-product-search skill to set up a product search agent on Google Cloud. +``` + +The agent walks Q-MODE (2 questions Quick / 4 questions Full), runs +`scripts/bootstrap.sh` to create the venv, then `scripts/setup.py` to validate +the catalog, ingest to BigQuery, create the Vector Search collection, and +launch the ADK web UI. + +## Use your own catalog + +Choose Full setup and point Q-B at a local CSV or `gs://` URI. CSV needs +`product_id`, `name`, `price`, `description`. Optional: `category`, `brand`, +`image_url`, `rating`, `stock`. + +## Cleanup + +In the agent chat: + +``` +clean up the GCP resources +``` + +Runs `cleanup.py --confirm` to delete the BigQuery dataset and Vector Search +collection. + +## Troubleshooting + +| Error | Fix | +|---|---| +| `MethodNotImplemented: 501` from Vector Search | `VECTOR_SEARCH_COLLECTION` has a newline. Re-export on one line | +| `ModuleNotFoundError: google.adk` | `bash -c "pip install -e '${SKILL_DIR}[adk]'"` (the `bash -c` matters in zsh) | +| `Package requires Python: 3.9.X` | Recreate venv with `python3.12 -m venv .venv` | +| `BILLING_DISABLED` / `PERMISSION_DENIED` | GCP project setup — see [references/troubleshooting.md](references/troubleshooting.md) | + +Full table: [references/troubleshooting.md](references/troubleshooting.md). + +## What gets built + +- BigQuery dataset `retail_skill_products.products` +- Vertex AI Vector Search collection `retail-skill-products-collection` in + `us-central1`, with auto-embeddings via `gemini-embedding-001` +- Workspace venv with the skill installed editable + a `design-spec.md` + +The skill's source code stays in the install directory — nothing is copied +to your workspace. + +## License + +Apache 2.0 diff --git a/skills/retail/retail-product-search/SKILL.md b/skills/retail/retail-product-search/SKILL.md new file mode 100644 index 0000000000..8c7718dc1f --- /dev/null +++ b/skills/retail/retail-product-search/SKILL.md @@ -0,0 +1,271 @@ +--- +name: retail-product-search +description: >- + Creates product search agents with semantic search and RAG on Google Cloud + (Vertex AI Vector Search, BigQuery, embeddings). Use when the user wants to + "build a product search agent", "create an e-commerce search", "make a + shopping assistant", "set up semantic catalog discovery", "ingest products + into Vector Search", or "deploy a retail RAG agent". Handles the full + pipeline: catalog data ingestion to BigQuery, Vertex AI Vector Search + collection setup, ADK agent scaffolding, evaluation, and Cloud Run deployment. +metadata: + author: Google + license: Apache-2.0 + version: 0.1.0 +--- + +# Product Search Agent + +Creates product search agents with semantic search and RAG on Google Cloud. + +## STOP — Q-MODE FIRST + +**If a catalog is already loaded** (system context says "DEPLOYED search agent" +or provides a `` block), skip Q-MODE and answer product queries +directly using the catalog. + +**Otherwise**, your first message MUST be exactly this: + +``` +[skill: retail-product-search] active. +Q-MODE: Pick a setup mode? [default: 1] + 1. Quick start -- 2 questions, smart defaults, ~60s. Best for demos and first-timers. + 2. Full setup -- 4 questions, ~2 min. Best for real builds. +``` + +Then stop and wait. Accept `1`, `quick`, empty/Enter (Quick), or `2`, `full` (Full). + +## Execution Rules + +1. Q-MODE first. No preamble, no plan-proposing. +2. One question at a time. Show `[default: ...]`. Empty input = default. +3. Save answers to `./design-spec.md` in the workspace as you collect them. +4. After interview, run `scripts/setup.py` (see Workspace Setup below). +5. User can say "configure more" mid-Quick-Start to switch to Full. + +## Workspace Setup + +The skill has two locations: +- **Install dir** — where SKILL.md and scripts live (varies by host) +- **Workspace** — the agent's cwd; design-spec.md, .venv, and per-run state live here + +By the end of this section the workspace must have `.venv/` (with the skill +installed editable + `[adk]` extras), `design-spec.md`, and `SKILL_DIR` +exported in the shell. + +Run this as ONE shell command — splitting it across tool calls loses state: + +```bash +SKILL_DIR=$(for d in ~/.claude/skills ~/.agents/skills ~/.gemini/skills ~/.cursor/skills; do + [ -f "$d/retail-product-search/SKILL.md" ] && echo "$d/retail-product-search" && break +done) +bash "$SKILL_DIR/scripts/bootstrap.sh" +``` + +`bootstrap.sh` finds a Python 3.10+ interpreter (with absolute-path fallback +for sandboxed shells), creates `.venv`, installs the skill editable, and +copies `design-spec.md` into the workspace. + +All scripts run from the install dir against the workspace config. **Use +`.venv/bin/python`, not bare `python`** — bare `python` may resolve to a +Python without the skill's editable install on sys.path. + +```bash +.venv/bin/python "$SKILL_DIR/scripts/setup.py" --config ./design-spec.md +.venv/bin/python "$SKILL_DIR/scripts/cleanup.py" --config ./design-spec.md --confirm +``` + +Details in [references/install-paths.md](references/install-paths.md). + +## Skill Dependencies + +This skill works in conjunction with the following core Google Cloud skills: +- `bigquery-basics` (for database configuration guidelines) +- `gemini-api` (for Vertex AI API / Google Gen AI SDK best practices) + +Verify if these skills are installed in your active skills directory. If they are missing, recommend the developer to install them by running: +`npx skills add google/skills --skill bigquery-basics gemini-api` + +## Mode 1: Quick Start (2 questions) + +| Q | Question | Default | +|---|---|---| +| Q-A | GCP project ID? | `$GOOGLE_CLOUD_PROJECT` or `gcloud config get-value project` | +| Q-B | Where's your product data? | `assets/sample-products.csv` (bundled) | + +Accepted for Q-B: empty / `default` (bundled), `/path/to/file.csv`, or `gs://...`. + +Silent defaults: Extended fields, `us-central1`. + +After Q-A and Q-B, do this automatically (don't ask the user to copy/paste). +**Run these steps SEQUENTIALLY — do not parallelize.** Steps 2-3 modify the +file bootstrap copies in step 1; running them concurrently is a race. + +1. **Run bootstrap first and wait for completion.** `bash "$SKILL_DIR/scripts/bootstrap.sh"` + copies the YAML-frontmatter design-spec template into the workspace at + `./design-spec.md`. Do NOT touch `./design-spec.md` until bootstrap exits. +2. **Mutate the existing `./design-spec.md`** — do NOT rewrite it from scratch. + `setup.py` parses YAML frontmatter via `_setup_utils.py`. A Markdown-only + file fails with `'NoneType' object has no attribute 'get'`. Use Edit / sed + to replace specific lines: + - `gcp_project_id: ""` → `gcp_project_id: ""` + - `data_source: assets/sample-products.csv` → `data_source: ` (only if user gave a non-default) +3. Say: "Taking defaults for the rest. Running setup — this takes 2-5 min to + create a BigQuery dataset and Vector Search collection. Say 'configure + more' to switch to Full setup." +4. Run `.venv/bin/python "$SKILL_DIR/scripts/setup.py" --config ./design-spec.md` +5. Stream output. On non-zero exit, surface the error and check + [references/troubleshooting.md](references/troubleshooting.md) +6. On success, set `VECTOR_SEARCH_COLLECTION` and proceed to Test + +## Mode 2: Full Setup + +Adds two more questions: product fields level and GCP region. + +| Q | Question | Default | Notes | +|---|---|---|---| +| Q-fields | Product fields level | `Extended` | `Basic` / `Standard` / `Extended` / `Full`. Match this to your CSV's columns. Don't offer "Custom" — `validate_schema.py` rejects it. | +| Q-region | GCP region | `us-central1` | **Only confirmed-working region for Vector Search 2.0.** Other regions return `501 MethodNotImplemented`. | + +Otherwise identical to Quick Start. + +## When to Use + +- E-commerce product search, shopping assistants, semantic catalog discovery + +Don't use for generic document search, simple keyword search, or non-retail. + +## Project Tree + +``` +retail-product-search/ + assets/ + design-spec.md # Source of truth -- filled by Q-MODE + sample-products.csv # Bundled 5-product demo catalog + references/ # Deep-dive docs (load on demand) + scripts/ + agent.py # Reference ADK agent + retrievers.py # Vector Search retrieval logic + setup.py # Pipeline driver (reads design-spec.md) + bootstrap.sh # Workspace bootstrap (called from Workspace Setup) + validate_schema.py + ingest_bigquery.py + ingest_vertex_search.py + cleanup.py +``` + +Customize: rewrite `scripts/agent.py` (see +[references/agent-example.md](references/agent-example.md)) and +`scripts/retrievers.py` with your product-specific fields. + +## Test + +After `setup.py` succeeds, set the collection env var (one line, no newlines): + +```bash +export VECTOR_SEARCH_COLLECTION="projects/$GOOGLE_CLOUD_PROJECT/locations/us-central1/collections/retail-skill-products-collection" +``` + +Then either: + +**With ADK** (interactive UI): +```bash +# Use the WORKSPACE VENV's adk (not bare `adk`) so the skill's editable +# install is on sys.path. Bare `adk` may resolve to a global Python (pyenv, +# brew, etc.) whose ADK can't find the skill and reports an empty app list. +.venv/bin/adk web "$SKILL_DIR/scripts" --port 8765 +``` +Open http://127.0.0.1:8765, click `scripts`, query. + +⚠️ Two things must be right: +- **Point `adk web` at `$SKILL_DIR/scripts`, not at `.`** — agent code lives + in the install dir, not the workspace. `adk web .` fails with "No agents + found in current folder". +- **Use `.venv/bin/adk`, not bare `adk`** — bare `adk` may launch the wrong + Python and silently fail to load the agent (UI loads, but `/list-apps` + returns `[]` and queries time out). + +**Without ADK** (direct smoke test): +```bash +.venv/bin/python -c "from scripts.retrievers import search; print(search('laptop for video editing', top_k=3))" +``` + +Semantic-only retrieval — no structured filters on price, stock, or rating. +For demo queries and how to add structured filtering, see +[references/architecture.md](references/architecture.md). + +## Evaluate + +```bash +cd +./vs eval retail-product-search --project-id $PROJECT +``` + +`EVAL.yaml` declares `rubric` (LLM-as-judge) + `assertions` (deterministic +checks). Target: 80%+ passing. + +## Deploy + +**Never deploy without explicit human approval.** + +Cloud Run service account needs `roles/bigquery.dataViewer` on the dataset and +`roles/aiplatform.user` on the project. Deploy via `gcloud run deploy` or your +org's existing tooling. + +## Gotchas + +- **No results**: collection empty or `VECTOR_SEARCH_COLLECTION` not set +- **Slow search**: check region and `top_k` +- **No structured filters**: `search()` is pure semantic similarity. Price / + stock / currency filters happen client-side in the LLM, so results may + include items outside the constraint. Don't promise hard filters +- **ADK session memory**: if the retriever errored in earlier turns, the + model "learns" the tool is broken. Click "New Session" in `adk web` after + fixing the underlying issue + +## Troubleshooting + +Most-common failures inline; full table in +[references/troubleshooting.md](references/troubleshooting.md). + +| Error | Fix | +|---|---| +| `setup.py` exits with `'NoneType' object has no attribute 'get'` | `design-spec.md` was written as plain Markdown instead of mutating the YAML-frontmatter template bootstrap copied | Wait for bootstrap to finish, then **edit** (not rewrite) `./design-spec.md` — only change the field values inside the existing `---...---` frontmatter | +| `adk web` starts but `/list-apps` returns `[]` / browser shows "No agents found" | Bare `adk` resolved to a global Python that lacks the editable install. Kill it and restart with `.venv/bin/adk web "$SKILL_DIR/scripts" --port 8765` | +| `MethodNotImplemented: 501` from Vector Search | `VECTOR_SEARCH_COLLECTION` has a newline. Re-export on one line | +| `ModuleNotFoundError: google.adk` | `bash -c "pip install -e '${SKILL_DIR}[adk]'"` (the `bash -c` matters in zsh) | +| `Package requires Python: 3.9.X` | venv used system Python 3.9. Recreate with `python3.12 -m venv .venv` | +| `BILLING_DISABLED` / `PERMISSION_DENIED` / `API has not been used` | GCP project setup — see troubleshooting.md | + +## MCP Migration + +This skill uses `gcloud` CLI + Python SDKs (`google-genai`, +`google-cloud-bigquery`, `google-cloud-aiplatform`). Per +[Phase 2 Skills guidelines](https://github.com/google/skills), 1p skills +should prefer remote MCP tools when available. Migration map: + +| Service | Where | Future MCP | +|---|---|---| +| BigQuery | `ingest_bigquery.py`, `validate_schema.py` | BigQuery MCP | +| Vertex AI Vector Search | `ingest_vertex_search.py`, `setup.py` | Vertex AI MCP | +| Vertex AI Embeddings | `retrievers.py` | Vertex AI MCP | +| Cloud Run | `gcloud run deploy` | Cloud Run MCP | + +## Completion Checklist + +- [ ] Product fields level and data source confirmed +- [ ] Data ingestion ran; Vector Search populated +- [ ] `retrieve_docs` returns results in ADK web UI +- [ ] Evaluation passes success criteria +- [ ] Deployed (if beyond prototype) + +## References + +Load on demand: + +- [references/install-paths.md](references/install-paths.md) — host install dirs, Python fallback, `bash -c` rationale +- [references/dependencies.md](references/dependencies.md) — pip deps and install quirks +- [references/architecture.md](references/architecture.md) — what retrieval does (semantic-only), demo queries, structured-filter strategies +- [references/troubleshooting.md](references/troubleshooting.md) — full error table +- [references/agent-example.md](references/agent-example.md) — how agent.py + retrievers.py fit together +- [references/ingestion-scripts.md](references/ingestion-scripts.md) — per-script CLI reference diff --git a/skills/retail/retail-product-search/assets/design-spec.md b/skills/retail/retail-product-search/assets/design-spec.md new file mode 100644 index 0000000000..57542ed5f5 --- /dev/null +++ b/skills/retail/retail-product-search/assets/design-spec.md @@ -0,0 +1,37 @@ +--- +# Product Search Agent - Design Spec +# Fill in your values and use this as input for project generation. + +# --- Required --- +gcp_project_id: "" # REQUIRED: your GCP project ID + +# --- Data --- +data_source: assets/sample-products.csv # local CSV path or gs://bucket/path/products.csv +product_fields: Extended # Basic | Standard | Extended | Full + +# --- GCP --- +gcp_region: us-central1 # Vector Search 2.0 region (us-central1 is the only confirmed-working region today) +dataset_id: retail_skill_products +table_id: products + +# --- Optional warnings --- +catalog_size: "1K-50K" # only used to trigger a >500K Dataflow hint +--- + +# Product Search Agent + +## Overview + +This design spec captures the configuration decisions for a retail product search +agent built on Google Cloud. It is generated during the SKILL.md interview and +used by ingestion scripts and agent scaffolding. + +## How to Use + +1. Fill in `gcp_project_id` above (or let the coding agent do it conversationally). +2. Pass this file to any script: `python scripts/ingest_bigquery.py --config assets/design-spec.md`. +3. CLI args always override values from this file. + +## Design Decisions + +Document any non-obvious choices here so future contributors understand the "why". diff --git a/skills/retail/retail-product-search/assets/sample-products.csv b/skills/retail/retail-product-search/assets/sample-products.csv new file mode 100644 index 0000000000..f75d833758 --- /dev/null +++ b/skills/retail/retail-product-search/assets/sample-products.csv @@ -0,0 +1,6 @@ +product_id,name,price,description,category,brand,image_url,rating,stock +prod-001,Wireless Headphones,199.99,Premium noise-cancelling over-ear headphones with 30-hour battery,Audio,Sony,,4.5,50 +prod-002,Laptop Stand,49.99,Aluminum adjustable laptop stand with cable management,Accessories,Generic,,4.2,100 +prod-003,USB-C Cable,19.99,6ft braided USB-C to USB-C cable with 100W charging,Cables,Anker,,4.7,200 +prod-004,Mechanical Keyboard,129.99,Compact 75% mechanical keyboard with hot-swappable switches,Peripherals,Keychron,,4.6,30 +prod-005,Monitor Light Bar,59.99,Screen-mounted LED light bar with adjustable color temperature,Lighting,BenQ,,4.4,75 diff --git a/skills/retail/retail-product-search/pyproject.toml b/skills/retail/retail-product-search/pyproject.toml new file mode 100644 index 0000000000..8c129d94d0 --- /dev/null +++ b/skills/retail/retail-product-search/pyproject.toml @@ -0,0 +1,39 @@ +[project] +name = "retail-product-search" +version = "0.1.0" +description = "Semantic product search agent using Vertex AI Vector Search and BigQuery" +requires-python = ">=3.10" +license = {text = "Apache-2.0"} + +dependencies = [ + "google-cloud-bigquery>=3.0", + "google-cloud-storage>=2.0", + "google-cloud-vectorsearch>=0.5,<1.0", # preview/beta -- pin to known-working range + "google-cloud-aiplatform>=1.30", + "google-genai>=1.0", + "pyyaml>=6.0", + "requests>=2.28", + "pyOpenSSL>=23.0", # mTLS during BigQuery / Vector Search auth in some envs +] + +[project.optional-dependencies] +adk = ["google-adk>=2.2.0"] # agent runtime for `adk web` and the ADK framework wrapper + +[tool.setuptools.packages.find] +include = ["scripts*"] + +[dependency-groups] +dev = [ + "pytest>=8.0", + "pytest-mock>=3.14", + "pytest-cov>=4.1", +] + +[tool.pytest.ini_options] +testpaths = ["tests/unit", "tests/integration"] +addopts = "-s -v" +log_cli = true +log_cli_level = "INFO" +markers = [ + "live: integration test that hits real GCP. Requires LIVE_EVAL=1 and a fresh GCP project to ingest into.", +] diff --git a/skills/retail/retail-product-search/references/agent-example.md b/skills/retail/retail-product-search/references/agent-example.md new file mode 100644 index 0000000000..3fe74238f8 --- /dev/null +++ b/skills/retail/retail-product-search/references/agent-example.md @@ -0,0 +1,71 @@ +# Example Agent Code + +The shipping implementation is in `scripts/agent.py` and `scripts/retrievers.py`. +Read those files directly -- this doc summarizes the shape so you know what +to look at when adapting it. + +## Files + +- **`scripts/agent.py`** -- defines `root_agent`, a single ADK `Agent` with one + tool (`retrieve_docs`) and a static instruction string focused on product + search. +- **`scripts/retrievers.py`** -- defines `search_collection`, the Vector Search + 2.0 query that backs `retrieve_docs`, plus a `search()` convenience + wrapper used by the Step 9 smoke test. + +## Shape + +``` +User query + | +root_agent (Gemini, ADK) + | instruction = "search the catalog for every query, present results..." + | +retrieve_docs(query) + | +search_collection(query, collection_path, top_k=10) + | +vectorsearch.DataObjectSearchServiceClient.search_data_objects( + SemanticSearch(search_text=query, search_field="text_embedding", + task_type="QUESTION_ANSWERING", top_k=10, + output_fields=[product_id, name, price, description, + category, brand, rating, stock]) +) + | +Formatted "Product 1: name, $price, by brand, rated X/5, " string +``` + +## Configuration + +The agent reads these env vars at runtime: + +| Variable | Default | Purpose | +|---|---|---| +| `GEMINI_MODEL` | `gemini-3.5-flash` | LLM used by `root_agent` | +| `GOOGLE_CLOUD_PROJECT` | from ADC | GCP project for Vertex AI | +| `GOOGLE_CLOUD_LOCATION` | `global` | LLM region | +| `VECTOR_SEARCH_LOCATION` | `us-central1` | Vector Search region (used to build the default collection path) | +| `VECTOR_SEARCH_COLLECTION` | `projects//locations//collections/retail-skill-products-collection` | Full collection resource path | + +`VECTOR_SEARCH_COLLECTION` must match the regex +`projects/

/locations//collections/` with no whitespace. A +newline embedded mid-path (common from multi-line shell pastes) causes the +SDK to return a confusing 501 -- `scripts/agent.py` validates this up front. + +## Extending + +To add a tool (e.g. cart actions, inventory checks, price-history lookups): + +1. Define a function in `scripts/agent.py` with a docstring describing when to + use it -- ADK feeds the docstring to the LLM as the tool description. +2. Append it to the `tools=[...]` list on `root_agent`. +3. Update the `instruction` string so the LLM knows when to pick the new + tool over `retrieve_docs`. + +To change the LLM, set `GEMINI_MODEL` -- no code changes needed. + +To change the retrieval contract (different fields, different `top_k`, +filtering), edit `search_collection` in `scripts/retrievers.py`. Note that VS +2.0 semantic search has no built-in structured filters; any +price/stock/category gating must happen in the agent's prompt or in a +client-side post-filter. diff --git a/skills/retail/retail-product-search/references/architecture.md b/skills/retail/retail-product-search/references/architecture.md new file mode 100644 index 0000000000..7ff0de9a2f --- /dev/null +++ b/skills/retail/retail-product-search/references/architecture.md @@ -0,0 +1,49 @@ +# What the retrieval layer actually does + +Read this when demoing the skill or extending it — covers what's +semantic vs what isn't, and where structured filtering would have to +live. + +## Semantic search, not filtered search + +`search(query, top_k)` runs Vertex AI Vector Search semantic similarity +over the embedding fields configured in `design-spec.md` (default: +`name, description, category, brand`). **It does NOT apply structured +filters on price, currency, stock, or rating** -- those words in a +query are just text the embedder sees. + +Implications when you demo: + +- `"headphones under 100 EUR"` returns headphones ranked by semantic + match; any price filtering happens in the agent's LLM, not the + retriever, so results may include items above the threshold (the LLM + may then narrate "no matches" if it filters client-side). +- The bundled `sample-products.csv` has USD prices. If the user asks in + EUR, the agent silently treats them as equivalent. For a real + multi-currency demo, add a `price_eur` or `currency` column to the + catalog and prompt the agent to use it. + +## Test queries + +| Query | What you're testing | +|---|---| +| `"laptop for video editing"` | Pure semantic match (works as documented) | +| `"I need a gift"` | Vague-query clarification flow (agent prompt-driven) | +| `"Which one has the best battery life?"` | RAG-style follow-up | +| `"wireless headphones under $100"` | Semantic + LLM-side price filtering; agent will narrate the price filter even though the retriever doesn't apply it | + +## Where to add structured filtering + +If a real catalog needs `price < X` to actually constrain results +(not just narrate around them), there are three places to add it: + +1. **Inside the retriever** — pre-filter the BigQuery dataset + before embedding ingest, or post-filter the Vector Search results + in `scripts/retrievers.py:search()`. +2. **As a separate tool on the agent** — add a `filter_by_price` tool + alongside `retrieve_docs` and let the LLM compose them. +3. **In the agent's system prompt** — give the LLM filter rules and + trust it to apply them. Cheapest, least reliable. + +Option 1 is the right answer for production; options 2 and 3 are +useful for demos that need quick wins. diff --git a/skills/retail/retail-product-search/references/dependencies.md b/skills/retail/retail-product-search/references/dependencies.md new file mode 100644 index 0000000000..22c38fddff --- /dev/null +++ b/skills/retail/retail-product-search/references/dependencies.md @@ -0,0 +1,27 @@ +# Dependencies + +`pip install -e .` from the install dir resolves everything from +`pyproject.toml`. You don't install these manually. + +## Required + +- `google-cloud-bigquery>=3.0` +- `google-cloud-storage>=2.0` +- `google-cloud-vectorsearch>=0.5,<1.0` (preview, pinned) +- `google-cloud-aiplatform>=1.30` +- `google-genai>=1.0` +- `pyyaml>=6.0`, `requests>=2.28`, `pyOpenSSL` (mTLS for BigQuery) + +## Optional `[adk]` extra + +- `google-adk>=2.2.0` — for the ADK web UI + +## Install quirks (handled by bootstrap.sh) + +1. **zsh expands `[adk]` as a glob** and silently drops extras. Workaround: + `bash -c "pip install -e '${SKILL_DIR}[adk]'"`. +2. **Agent shell tools reset state between calls.** Run the workspace setup + block as a single shell invocation. +3. **Stripped PATH** in sandboxed terminals hides brew/pyenv Python. The + bootstrap script falls back to absolute paths + (`/opt/homebrew/bin/python3.13`, `~/.pyenv/shims/python3.12`, etc.). diff --git a/skills/retail/retail-product-search/references/ingestion-scripts.md b/skills/retail/retail-product-search/references/ingestion-scripts.md new file mode 100644 index 0000000000..ad9b10e451 --- /dev/null +++ b/skills/retail/retail-product-search/references/ingestion-scripts.md @@ -0,0 +1,165 @@ +# Product Search - Data Ingestion Scripts Reference + +Reference documentation for the scripts in `scripts/`. These are standalone +Python scripts that handle product data ingestion and vector search indexing. + +All scripts support `--config design-spec.md` to load defaults from your +project configuration. + +--- + +## Script 1: ingest_bigquery.py + +**Purpose**: Load product catalog from CSV/JSON into BigQuery +**Location**: `scripts/ingest_bigquery.py` + +### Usage + +```bash +# From GCS +python scripts/ingest_bigquery.py \ + --project-id my-project \ + --gcs-bucket my-project-products \ + --gcs-path products.csv + +# From local file +python scripts/ingest_bigquery.py \ + --project-id my-project \ + --local-file data/products.json \ + --format json + +# Using design-spec.md for defaults +python scripts/ingest_bigquery.py \ + --config design-spec.md \ + --local-file data/products.csv +``` + +### Schema + +The script uses a fixed schema matching the Extended product field level: + +| Field | Type | Mode | Notes | +|-------|------|------|-------| +| product_id | STRING | REQUIRED | Unique identifier | +| name | STRING | REQUIRED | Product name | +| price | FLOAT64 | REQUIRED | Price in configured currency | +| description | STRING | REQUIRED | Product description | +| category | STRING | NULLABLE | Product category | +| brand | STRING | NULLABLE | Brand name | +| image_url | STRING | NULLABLE | Product image URL | +| rating | FLOAT64 | NULLABLE | Rating (0-5) | +| stock | INT64 | NULLABLE | Stock quantity | + +To customize the schema (e.g. for Basic or Full field levels), edit +`REQUIRED_FIELDS`, `OPTIONAL_FIELDS`, and `SCHEMA` at the top of the script. + +### Validation + +The script validates each row before loading: +- Required fields must be present and non-empty +- `price` must be numeric +- `stock` must be an integer +- Invalid rows are skipped with warnings (not fatal) + +--- + +## Script 2: ingest_vertex_search.py + +**Purpose**: Create a Vector Search 2.0 Collection and ingest products +**Location**: `scripts/ingest_vertex_search.py` + +Uses Vector Search 2.0 Collections with auto-embeddings. No manual +embedding generation or GCS bucket needed -- the Collection's configured +embedding model generates embeddings automatically when data objects +are inserted. + +### Usage + +```bash +# Basic usage +python scripts/ingest_vertex_search.py \ + --project-id my-project \ + --collection-id retail-skill-products-collection + +# Using design-spec.md for defaults +python scripts/ingest_vertex_search.py --config design-spec.md +``` + +### Pipeline + +1. Fetch products from BigQuery (`retail_skill_products.products`) +2. Create Vector Search 2.0 Collection if it doesn't exist (with auto-embedding config) +3. Insert products as data objects one at a time, catching `AlreadyExists` for idempotent re-runs +4. VS 2.0 auto-generates embeddings from the configured text template + +### Configuration + +| Flag | Default | Config key | +|------|---------|------------| +| `--collection-id` | retail-skill-products-collection | `collection_id` | +| `--embedding-model` | gemini-embedding-001 | `embedding_model` | +| `--embedding-fields` | name,description,category,brand | `embedding_fields` | +| `--location` | us-central1 | `gcp_region` | + +### Output + +After ingestion, set the `VECTOR_SEARCH_COLLECTION` environment variable +in your agent to the collection path printed by the script: + +```bash +export VECTOR_SEARCH_COLLECTION="projects/PROJECT_ID/locations/us-central1/collections/retail-skill-products-collection" +``` + +--- + +## Script 3: validate_schema.py + +**Purpose**: Validate product data files before ingestion +**Location**: `scripts/validate_schema.py` + +### Usage + +```bash +# Validate CSV at Extended level (matches sample-products.csv) +python scripts/validate_schema.py \ + --file data/products.csv \ + --fields-level Extended + +# Validate JSON at Standard level (format auto-detected from .json/.jsonl suffix) +python scripts/validate_schema.py \ + --file data/products.json \ + --fields-level Standard +``` + +### Field Levels + +| Level | Required | Optional | +|-------|----------|----------| +| Basic | product_id, name, price, description | (none) | +| Standard | Basic | category, brand, image_url | +| Extended | Basic | Standard + rating, stock, manufacturer | +| Full | Basic | Extended + variants, tags, specifications, reviews | + +--- + +## Ingestion Order + +Run scripts in this order: + +1. `validate_schema.py` -- validate your data file +2. `ingest_bigquery.py` -- load to BigQuery +3. `ingest_vertex_search.py` -- create collection + ingest products + +**Always use these retail skill scripts** -- generic document ingestion +pipelines are incompatible with product catalogs. + +--- + +## Customizing the Schema + +To change the product schema: + +1. Edit `REQUIRED_FIELDS`, `OPTIONAL_FIELDS`, and `SCHEMA` in `ingest_bigquery.py` +2. Edit `DEFAULT_EMBEDDING_FIELDS` and `PRODUCT_DATA_FIELDS` in `ingest_vertex_search.py` +3. Update `validate_schema.py` field level definitions if needed +4. Update `design-spec.md` with the new field level and embedding fields diff --git a/skills/retail/retail-product-search/references/install-paths.md b/skills/retail/retail-product-search/references/install-paths.md new file mode 100644 index 0000000000..f2fca63ffd --- /dev/null +++ b/skills/retail/retail-product-search/references/install-paths.md @@ -0,0 +1,44 @@ +# Install Paths and Environment Quirks + +## Known install dirs per host + +| Host | Install dir | +|---|---| +| Claude Code | `~/.claude/skills//` | +| `npx skills add` (Codex, Gemini CLI, GitHub Copilot, others) | `~/.agents/skills//` | +| Gemini CLI standalone | `~/.gemini/skills//` | +| Cursor | `~/.cursor/skills//` | + +`bootstrap.sh` tries these paths in order. If your host installs elsewhere, +set `SKILL_DIR` manually: + +```bash +export SKILL_DIR=/path/to/your/install/dir +``` + +## Python interpreter fallback + +`bootstrap.sh` tries `command -v python3.{13,12,11,10}` first, then falls +back to absolute paths (`/opt/homebrew/bin/`, `/usr/local/bin/`, +`~/.pyenv/shims/`). The fallback handles sandboxed shells with stripped PATH. + +For conda/asdf/other layouts, set `PYTHON_BIN` and skip the loop. + +## Why `bash -c` wraps the pip command + +`pip install -e "$SKILL_DIR[adk]"` looks correct but breaks in zsh: zsh treats +`[adk]` as a glob character class, silently expands to empty, and the +`[adk]` extras are dropped. `bash -c "pip install -e '${SKILL_DIR}[adk]'"` +sidesteps this — pip runs under bash, which doesn't glob-expand `[adk]`. + +## Why the workspace setup must run as one shell command + +Agent shell tools reset cwd and clear variables between calls. If you set +`SKILL_DIR` in call 1 and run pip in call 2, `$SKILL_DIR` is empty in call 2 +and the install becomes `pip install -e [adk]` (invalid). + +## Description tuning for triggering + +If your skill is registered but the agent doesn't pick it for relevant +prompts, the `description` field is what hosts' routers match against. +Include concrete trigger phrases alongside the abstract description. diff --git a/skills/retail/retail-product-search/references/troubleshooting.md b/skills/retail/retail-product-search/references/troubleshooting.md new file mode 100644 index 0000000000..3af752deb4 --- /dev/null +++ b/skills/retail/retail-product-search/references/troubleshooting.md @@ -0,0 +1,39 @@ +# Troubleshooting + +Read this when `setup.py` or a per-step script exits non-zero. Match +the error message against the table below before guessing. + +## Error table + +| Error pattern | Likely cause | Fix | +|---|---|---| +| `BILLING_DISABLED` / `Billing must be enabled` | GCP project has no billing account | Link a billing account in Cloud Console, then re-run | +| `PERMISSION_DENIED` on BigQuery | Service account missing IAM | `gcloud projects add-iam-policy-binding $PROJECT_ID --member=user:$EMAIL --role=roles/bigquery.dataEditor` | +| `PERMISSION_DENIED` on Vertex AI | Missing `aiplatform.user` role | `gcloud projects add-iam-policy-binding $PROJECT_ID --member=user:$EMAIL --role=roles/aiplatform.user` | +| `API has not been used` / `is disabled` | Required API not enabled | `gcloud services enable bigquery.googleapis.com aiplatform.googleapis.com` | +| `Collection not found` at query time | Vector Search collection never created or wrong path | Re-run `ingest_vertex_search.py`; verify `VECTOR_SEARCH_COLLECTION` env var | +| `setup.py` exits with `AttributeError: 'NoneType' object has no attribute 'get'` from `_setup_utils.py` | `design-spec.md` is not in YAML-frontmatter form. The agent wrote it as plain Markdown instead of editing the YAML template bootstrap copied. `_setup_utils.load_config()` returns `None` for non-YAML files, then `setup.py` calls `.get(...)` on it | Wait for `bootstrap.sh` to finish before touching `./design-spec.md`. Then **edit** the file (don't rewrite) — modify only the values inside the existing `---...---` frontmatter block. The canonical template is at `/assets/design-spec.md` | +| `adk web` boots but `/list-apps` returns `[]`, browser shows "No agents found in current folder" | Bare `adk` resolved through `PATH` to a global Python (pyenv, brew, etc.) that doesn't have the skill's editable install on `sys.path`. The server runs but can't import the agent module | Kill it (`lsof -ti :8765 \| xargs kill -9`) and restart with the workspace venv's adk explicitly: `.venv/bin/adk web "$SKILL_DIR/scripts" --port 8765`. Same root cause if `python "$SKILL_DIR/scripts/setup.py"` succeeds but the smoke test fails — use `.venv/bin/python` instead of bare `python` | +| `VECTOR_SEARCH_COLLECTION is malformed` | env var has embedded whitespace (usually a newline from a wrapped paste) | Re-`export` the value on a single line; prefer the `$PROJECT_ID` short form | +| `MethodNotImplemented: 501` from Vector Search in `us-central1` | Almost always a malformed `VECTOR_SEARCH_COLLECTION` (see above) -- not a real region/service issue | Echo `$VECTOR_SEARCH_COLLECTION` and confirm it has no `\n` or whitespace | +| `ModuleNotFoundError: google.cloud.aiplatform` | Python deps not installed | `pip install -e .` from the skill dir, or `pip install google-cloud-aiplatform google-cloud-bigquery google-genai` | +| `Package 'retail-product-search' requires a different Python: 3.9.X not in '>=3.10'` | venv was created with system Python 3.9 (macOS default) | Recreate the venv with a 3.10+ interpreter: `rm -rf .venv && python3.12 -m venv .venv && source .venv/bin/activate && bash -c "pip install -e '${SKILL_DIR}[adk]'"` (substitute the Python version you have installed). Use the interpreter-detection loop from the Workspace Setup section to auto-find one. | +| `ModuleNotFoundError: google.adk` | Optional ADK dep missing | Use the standalone test path in Step 9, or `pip install -e '.[adk]'` from the skill dir (single-quote the `.[adk]` so zsh doesn't glob-expand the brackets) | +| `Schema mismatch` from `validate_schema.py` | CSV missing required fields (`product_id`, `name`, `price`) | Add the missing columns and re-run; or pass `--fields-level Standard` if your data is sparse | +| `Quota exceeded` on embedding requests | Free-tier embedding quota hit | Wait an hour, re-run with a smaller catalog slice, or request a quota increase. The script catches per-row failures and continues, so a partial run is fine to resume | +| Setup script hangs on Vector Search create | Collection creation is async and takes 2-5 min | Wait. If >10 min with no progress, check Cloud Console > Vertex AI > Vector Search for collection status | +| `projects//locations/...` in error path | `$GOOGLE_CLOUD_PROJECT` was empty when you ran the `export VECTOR_SEARCH_COLLECTION=...` -- substitution produced an empty project ID | `export GOOGLE_CLOUD_PROJECT=` first, then re-run the `export VECTOR_SEARCH_COLLECTION` line, then restart `adk web` | +| Agent in `adk web` keeps saying "tool encountered an error" / "I am still unable to search" even after env fixed | ADK session memory -- model learned the tool is broken from earlier turns | Click "New Session" in the ADK web UI; the fresh session will retry the tool | +| Agent ignores price/currency filters in user queries | Retriever is pure semantic similarity; no structured filters at retrieval time | Expected behavior. Document the limit in your demo; or add a `currency`/`price_eur` column to the catalog and prompt the agent to filter client-side | + +## ADK web UI: "tool encountered an error" loop + +If `adk web` returns "tool encountered an error" repeatedly even after +the env var is correct: ADK persists session history in +`scripts/.adk/session.db`. If the first few queries in a session got a +tool error (typically a malformed env var, since fixed), the model +"learns" the tool is broken and starts skipping it, apologizing without +retrying. + +Click "New Session" in the ADK web UI after fixing the underlying +issue -- a fresh session will retry the tool from scratch. diff --git a/skills/retail/retail-product-search/scripts/_paths.py b/skills/retail/retail-product-search/scripts/_paths.py new file mode 100644 index 0000000000..1640839cdb --- /dev/null +++ b/skills/retail/retail-product-search/scripts/_paths.py @@ -0,0 +1,35 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Install-dir resolution for the retail-product-search skill. + +INSTALL_DIR is the directory containing SKILL.md -- i.e. the root of the +installed skill, regardless of where the user invokes a script from. +Computed from this file's own location at import time, so it stays stable +even when the user's cwd changes mid-session. + +Use this for any "where do I find my bundled assets?" lookup. +Do NOT use this for "where should I write user state?" -- user-mutable +state goes to the agent's current working directory. +""" + +from __future__ import annotations + +import pathlib + +INSTALL_DIR: pathlib.Path = pathlib.Path(__file__).resolve().parent.parent + +ASSETS_DIR: pathlib.Path = INSTALL_DIR / "assets" +SAMPLE_PRODUCTS_CSV: pathlib.Path = ASSETS_DIR / "sample-products.csv" +DEFAULT_DESIGN_SPEC: pathlib.Path = ASSETS_DIR / "design-spec.md" diff --git a/skills/retail/retail-product-search/scripts/_setup_utils.py b/skills/retail/retail-product-search/scripts/_setup_utils.py new file mode 100644 index 0000000000..c7847edecc --- /dev/null +++ b/skills/retail/retail-product-search/scripts/_setup_utils.py @@ -0,0 +1,59 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared utilities for skill setup scripts. + +Provides load_config() for reading YAML frontmatter from design-spec.md. + +Vendored locally so this skill ships as a self-contained package -- no +parent-directory _shared/ folder required to run setup. +""" + +import logging +import pathlib +from typing import Any + +import yaml # type: ignore[import-untyped] + +logger = logging.getLogger(__name__) + + +def load_config(config_path: str) -> dict[str, Any]: + """Load design-spec.md YAML frontmatter. + + Args: + config_path: Path to the design-spec markdown file. + + Returns: + Parsed YAML frontmatter as a dict. Empty dict if the file is + missing, has no frontmatter, or the frontmatter is empty. + """ + if not pathlib.Path(config_path).exists(): + return {} + text = pathlib.Path(config_path).read_text(encoding="utf-8") + if text.startswith("---"): + lines = text.split("\n") + yaml_lines = [] + in_frontmatter = False + for line in lines: + if line.strip() == "---": + if not in_frontmatter: + in_frontmatter = True + continue + break + if in_frontmatter: + yaml_lines.append(line) + yaml_text = "\n".join(yaml_lines) + return yaml.safe_load(yaml_text) or {} + return yaml.safe_load(text) or {} diff --git a/skills/retail/retail-product-search/scripts/agent.py b/skills/retail/retail-product-search/scripts/agent.py new file mode 100644 index 0000000000..b0585d94d8 --- /dev/null +++ b/skills/retail/retail-product-search/scripts/agent.py @@ -0,0 +1,144 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ADK agent definition for the retail product search skill. + +Exposes a :data:`root_agent` (and :data:`app`) that uses Vertex AI Vector +Search 2.0 for semantic product retrieval via the :func:`retrieve_docs` tool. +""" + +# pylint: disable=line-too-long +# (error messages and example URLs are intentionally long; breaking them hurts copy-paste UX.) + +import os +import re + +import google +from google.adk import agents +from google.adk import apps +from google.adk import models +import vertexai + +from scripts.retrievers import search_collection + +LLM_LOCATION = os.getenv("GOOGLE_CLOUD_LOCATION", "global") +VECTOR_SEARCH_LOCATION = os.getenv("VECTOR_SEARCH_LOCATION", "us-central1") +LLM = os.getenv("GEMINI_MODEL", "gemini-3.5-flash") + +_, _default_project = google.auth.default() +_project_id = os.getenv("GOOGLE_CLOUD_PROJECT", _default_project) +os.environ.setdefault("GOOGLE_CLOUD_PROJECT", _project_id) +os.environ.setdefault("GOOGLE_CLOUD_LOCATION", LLM_LOCATION) +os.environ.setdefault("GOOGLE_GENAI_USE_VERTEXAI", "True") +vertexai.init(project=_project_id, location=LLM_LOCATION) + + +_COLLECTION_PATH_RE = re.compile( + r"^projects/[^/\s]+/locations/[^/\s]+/collections/[^/\s]+$" +) + + +def _get_vector_search_collection() -> str: + """Return the Vector Search collection resource path. + + Reads from the ``VECTOR_SEARCH_COLLECTION`` env var when set; otherwise + builds the default path from ``GOOGLE_CLOUD_PROJECT`` and + ``VECTOR_SEARCH_LOCATION``. + + Returns: + The fully qualified Vector Search collection path. + + Raises: + ValueError: If ``VECTOR_SEARCH_COLLECTION`` is set to a malformed + path (e.g. contains a newline from a wrapped shell paste, which + silently causes the Vector Search SDK to return a 501). + """ + raw = os.getenv("VECTOR_SEARCH_COLLECTION") + if raw is None: + return ( + f"projects/{_project_id}/locations/{VECTOR_SEARCH_LOCATION}" + "/collections/retail-skill-products-collection" + ) + # Strip whitespace -- multi-line shell pastes can embed a newline mid-path, + # which the Vector Search SDK silently maps to a 501 from the wrong endpoint. + cleaned = raw.strip() + if not _COLLECTION_PATH_RE.match(cleaned): + raise ValueError( + "VECTOR_SEARCH_COLLECTION is malformed -- expected " + "'projects//locations//collections/' with no whitespace. " + f"Got: {raw!r}. " + "Tip: type the export on a single line, e.g. " + "export VECTOR_SEARCH_COLLECTION=" + '"projects/$GOOGLE_CLOUD_PROJECT/locations/us-central1/' + 'collections/retail-skill-products-collection"' + ) + return cleaned + + +def retrieve_docs(query: str) -> str: + """Search the product catalog using semantic similarity. + + Call this for every user query about products -- by name, by attributes + (price/brand/category), by intent ('something for my desk'), or by + comparison ('X vs Y'). Returns a formatted list of matching products + with name, price, brand, rating, and a description snippet. + + Args: + query: The product search query in natural language. + + Returns: + Formatted product list, or a "no matches" message. + """ + try: + return search_collection( + query=query, + collection_path=_get_vector_search_collection(), + ) + except Exception as e: # pylint: disable=broad-exception-caught + return ( + f"Calling retrieval tool with query:\n\n{query}\n\n" + f"raised the following error:\n\n{type(e)}: {e}" + ) + + +INSTRUCTION = """You are a retail product search assistant. +Use the retrieve_docs tool to search the product catalog for every user query. + +IMPORTANT: After receiving tool results, you MUST immediately present the results to the user. +Do not stop after saying "let me search" -- always continue to present the full results. + +When presenting search results: +- Say each product's name, price, brand, and a short description. +- Keep it conversational and concise. +- Never make up products -- only mention products returned by the tool. +- If no products match, say so and suggest broadening the search.""" + + +root_agent = agents.Agent( + name="root_agent", + model=models.Gemini(model=LLM), + instruction=INSTRUCTION, + tools=[retrieve_docs], +) + +app = apps.App( + root_agent=root_agent, + # ADK's `adk web` auto-names the app from the agent module's parent + # directory. The agent lives at scripts/agent.py, so ADK names it + # "scripts". The App name MUST match the auto-discovered name or + # session creation fails with "Session not found: . The runner + # is configured with app name '', but the root agent was loaded + # from '/.../scripts'." + name="scripts", +) diff --git a/skills/retail/retail-product-search/scripts/bootstrap.sh b/skills/retail/retail-product-search/scripts/bootstrap.sh new file mode 100755 index 0000000000..29a14bfc90 --- /dev/null +++ b/skills/retail/retail-product-search/scripts/bootstrap.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Workspace bootstrap for retail-product-search. +# +# Discovers the skill's install dir, finds a Python 3.10+ interpreter, +# creates .venv in the current directory, installs the skill editable +# with the [adk] extras, and copies design-spec.md into the workspace. +# +# Run from the workspace directory as a single shell invocation: +# bash /path/to/scripts/bootstrap.sh +# +# All defensive logic (interpreter fallback, bash -c around pip, +# stripped-PATH workarounds) is documented in +# references/install-paths.md. + +set -e + +# 1. Locate the install dir. Try common spec-compliant locations. +SKILL_DIR="" +for candidate in \ + ~/.claude/skills/retail-product-search \ + ~/.agents/skills/retail-product-search \ + ~/.gemini/skills/retail-product-search \ + ~/.cursor/skills/retail-product-search; do + if [ -f "$candidate/SKILL.md" ]; then + SKILL_DIR="$candidate" + break + fi +done + +# Bootstrap may itself be invoked via absolute path; if so, derive +# SKILL_DIR from $0 as a final fallback. +if [ -z "$SKILL_DIR" ]; then + script_dir="$(cd "$(dirname "$0")" && pwd)" + if [ -f "$script_dir/../SKILL.md" ]; then + SKILL_DIR="$(cd "$script_dir/.." && pwd)" + fi +fi + +if [ -z "$SKILL_DIR" ]; then + echo "ERROR: retail-product-search skill not installed. Install it first." >&2 + exit 1 +fi + +# 2. Pick a Python 3.10+ interpreter. +# Try PATH lookup first; fall back to absolute paths for sandboxed +# shells that launch with a stripped PATH. +PYTHON_BIN="" +for py in python3.13 python3.12 python3.11 python3.10 python3; do + if command -v "$py" >/dev/null 2>&1; then + ver=$("$py" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null) + case "$ver" in 3.10|3.11|3.12|3.13) PYTHON_BIN="$py"; break ;; esac + fi +done + +if [ -z "$PYTHON_BIN" ]; then + for path in \ + /opt/homebrew/bin/python3.13 \ + /opt/homebrew/bin/python3.12 \ + /opt/homebrew/bin/python3.11 \ + /opt/homebrew/bin/python3.10 \ + /usr/local/bin/python3.13 \ + /usr/local/bin/python3.12 \ + /usr/local/bin/python3.11 \ + /usr/local/bin/python3.10 \ + "$HOME/.pyenv/shims/python3.13" \ + "$HOME/.pyenv/shims/python3.12" \ + "$HOME/.pyenv/shims/python3.11" \ + "$HOME/.pyenv/shims/python3.10"; do + if [ -x "$path" ]; then + ver=$("$path" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null) + case "$ver" in 3.10|3.11|3.12|3.13) PYTHON_BIN="$path"; break ;; esac + fi + done +fi + +if [ -z "$PYTHON_BIN" ]; then + echo "ERROR: need Python 3.10+. Install one (brew install python@3.12) and retry." >&2 + exit 1 +fi + +echo "Using PYTHON_BIN=$PYTHON_BIN" +echo "Using SKILL_DIR=$SKILL_DIR" + +# 3. Create venv, activate, install the skill with extras. +# bash -c around pip ensures the [adk] extras aren't glob-expanded by zsh. +if [ ! -d .venv ]; then + "$PYTHON_BIN" -m venv .venv +fi +source .venv/bin/activate +bash -c "pip install -e '${SKILL_DIR}[adk]'" + +# 4. Copy the design-spec template into the workspace. +if [ ! -f ./design-spec.md ]; then + cp "$SKILL_DIR/assets/design-spec.md" ./design-spec.md +fi + +echo "" +echo "READY" +echo " SKILL_DIR=$SKILL_DIR" +echo " workspace=$(pwd)" +echo " venv=$(pwd)/.venv" +echo " design-spec=$(pwd)/design-spec.md" diff --git a/skills/retail/retail-product-search/scripts/cleanup.py b/skills/retail/retail-product-search/scripts/cleanup.py new file mode 100644 index 0000000000..e9c3466fee --- /dev/null +++ b/skills/retail/retail-product-search/scripts/cleanup.py @@ -0,0 +1,533 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +r"""Clean up all GCP resources created by the product search agent. + +Deletes BigQuery datasets, Vector Search 2.0 collections (and legacy +v1 MatchingEngineIndex if present), and Cloud Run services. + +Usage: + # Dry run (show what would be deleted, don't delete) + python scripts/cleanup.py --config design-spec.md --dry-run + + # Delete everything + python scripts/cleanup.py --config design-spec.md --confirm + + # Delete only specific resources + python scripts/cleanup.py --config design-spec.md --confirm \\ + --only bigquery,vectorsearch +""" + +# pylint: disable=line-too-long +# (error messages with resource paths and gcloud commands are intentionally long.) + +import argparse +import logging +import pathlib +import subprocess +import sys +from typing import Any + +from _setup_utils import load_config # noqa: E402 # pylint: disable=wrong-import-position +from google.api_core import exceptions +from google.cloud import aiplatform +from google.cloud import bigquery +from google.cloud import vectorsearch + +# Allow imports from the script's own directory before pip install -e is run. +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +ALL_RESOURCE_TYPES = ["bigquery", "vectorsearch", "cloudrun"] +# Quick Start only creates BigQuery + Vector Search. Cloud Run is only used by +# the deploy extension, so we skip it by default to avoid slow `gcloud describe` +# roundtrips. Users who deployed to Cloud Run can opt in with --only cloudrun. +DEFAULT_RESOURCE_TYPES = ["bigquery", "vectorsearch"] + +# Bound how long we'll wait for a single `gcloud describe` to answer. Without +# this, an unreachable network / wrong project can hang cleanup indefinitely. +GCLOUD_DESCRIBE_TIMEOUT_SEC = 15 + + +def delete_bigquery(project_id: str, dataset_id: str, dry_run: bool) -> bool: + """Delete a BigQuery dataset and all of its tables. + + Args: + project_id: GCP project ID. + dataset_id: Bare dataset name (no project prefix). + dry_run: If True, log what would be deleted but don't delete. + + Returns: + True on successful delete or when the dataset is already absent. + False if the delete call raised an unexpected exception. + """ + client = bigquery.Client(project=project_id) + dataset_ref = f"{project_id}.{dataset_id}" + + try: + client.get_dataset(dataset_ref) + except Exception: # pylint: disable=broad-exception-caught + logger.info("BigQuery dataset %s does not exist, skipping", dataset_ref) + return True + + if dry_run: + logger.info( + "[DRY RUN] Would delete BigQuery dataset: %s (and all tables)", dataset_ref + ) + return True + + try: + client.delete_dataset(dataset_ref, delete_contents=True, not_found_ok=True) + logger.info("Deleted BigQuery dataset: %s", dataset_ref) + return True + except Exception as e: # pylint: disable=broad-exception-caught + logger.error("Failed to delete BigQuery dataset %s: %s", dataset_ref, e) + return False + + +def _drain_data_objects(collection_path: str) -> int: + """Delete all data objects inside a Vector Search collection. + + The API rejects collection deletion while data objects exist, so we + list them via semantic search (broad query) and batch-delete in chunks. + + Args: + collection_path: Fully qualified Vector Search collection path + (``projects/.../locations/.../collections/...``). + + Returns: + The total number of data objects deleted. + """ + search_client = vectorsearch.DataObjectSearchServiceClient() + data_client = vectorsearch.DataObjectServiceClient() + + # Broad-stroke search to enumerate IDs server-side. top_k is capped by the + # API; we loop until a search returns no new IDs (or we hit a sane ceiling). + seen: set[str] = set() + deleted_total = 0 + batch_size = 100 + max_iterations = ( + 50 # 50 * 100 = 5000 objects; raise if you have a bigger collection + ) + + for _ in range(max_iterations): + request = vectorsearch.SearchDataObjectsRequest( + parent=collection_path, + semantic_search=vectorsearch.SemanticSearch( + search_text="product", # broad query -- collection is product catalog + search_field="text_embedding", + task_type="QUESTION_ANSWERING", # required by the API; matches scripts/retrievers.py + top_k=batch_size, + output_fields=vectorsearch.OutputFields(data_fields=["product_id"]), + ), + ) + + batch_ids: list[str] = [] + for result in search_client.search_data_objects(request): + object_name = result.data_object.name + if object_name and object_name not in seen: + seen.add(object_name) + batch_ids.append(object_name) + + if not batch_ids: + break + + delete_requests = [ + vectorsearch.DeleteDataObjectRequest(name=name) for name in batch_ids + ] + data_client.batch_delete_data_objects( + request=vectorsearch.BatchDeleteDataObjectsRequest( + parent=collection_path, + requests=delete_requests, + ) + ) + deleted_total += len(batch_ids) + logger.info(" Drained %d data objects (%d total)", len(batch_ids), deleted_total) + + return deleted_total + + +def delete_vectorsearch_collection( + project_id: str, location: str, collection_id: str, dry_run: bool +) -> bool: + """Delete a Vector Search 2.0 Collection. + + The API requires the collection to be empty before deletion, so we + first drain all contained data objects. + + Args: + project_id: GCP project ID. + location: GCP region of the collection. + collection_id: Bare collection ID (no project/location prefix). + dry_run: If True, log what would be deleted but don't delete. + + Returns: + True on successful delete, when the collection is already absent, + or when Vector Search isn't available in this region. False if the + drain or delete call raised an unexpected exception. + """ + client = vectorsearch.VectorSearchServiceClient() + collection_name = ( + f"projects/{project_id}/locations/{location}/collections/{collection_id}" + ) + + try: + client.get_collection( + request=vectorsearch.GetCollectionRequest(name=collection_name) + ) + except exceptions.NotFound: + logger.info( + "Vector Search collection %s does not exist, skipping", collection_id + ) + return True + except exceptions.MethodNotImplemented: + # Vector Search 2.0 isn't available in this region. Nothing to delete + # because nothing could have been created here in the first place. + logger.info( + "Vector Search not available in %s; no collection to clean up.", location + ) + return True + + if dry_run: + logger.info( + "[DRY RUN] Would drain data objects and delete Vector Search collection: %s", + collection_name, + ) + return True + + try: + logger.info("Draining data objects from %s before deletion...", collection_id) + drained = _drain_data_objects(collection_name) + logger.info("Drained %d data object(s) from %s", drained, collection_id) + except Exception as e: # pylint: disable=broad-exception-caught + logger.error( + "Failed to drain data objects from %s: %s\n" + " Collection deletion will fail until it is empty. Try the manual path:\n" + " gcloud ai vector-search-collections delete %s " + "--region=%s --project=%s --force", + collection_id, + e, + collection_id, + location, + project_id, + ) + return False + + try: + operation = client.delete_collection( + request=vectorsearch.DeleteCollectionRequest(name=collection_name) + ) + operation.result() + logger.info("Deleted Vector Search collection: %s", collection_name) + return True + except Exception as e: # pylint: disable=broad-exception-caught + logger.error( + "Failed to delete Vector Search collection %s: %s\n" + " Data objects were drained, but the delete still failed. Possible causes:\n" + " - An async embedding/index operation is still in flight (wait 2-3 min and retry)\n" + " - Stragglers remain that the drain pass missed; verify in Cloud Console.\n" + " Manual cleanup:\n" + " gcloud ai vector-search-collections delete %s " + "--region=%s --project=%s --force", + collection_id, + e, + collection_id, + location, + project_id, + ) + return False + + +def delete_vectorsearch_v1_index( + project_id: str, location: str, index_name: str, dry_run: bool +) -> bool: + """Delete legacy Vector Search v1 MatchingEngineIndex (if any remain). + + Args: + project_id: GCP project ID. + location: GCP region of the legacy index. + index_name: Display name of the legacy MatchingEngineIndex. + dry_run: If True, log what would be deleted but don't delete. + + Returns: + True on success, or when there's no matching index to delete. + False if a delete call failed. + """ + aiplatform.init(project=project_id, location=location) + + indexes = aiplatform.MatchingEngineIndex.list(filter=f'display_name="{index_name}"') + + if not indexes: + return True + + ok = True + for index in indexes: + if dry_run: + logger.info( + "[DRY RUN] Would delete legacy Vector Search index: %s", + index.resource_name, + ) + continue + + try: + index.delete() # type: ignore[attr-defined] + logger.info("Deleted legacy Vector Search index: %s", index.resource_name) + except Exception as e: # pylint: disable=broad-exception-caught + logger.error( + "Failed to delete legacy index %s: %s", index.resource_name, e + ) + ok = False + + return ok + + +def delete_cloudrun( + project_id: str, location: str, service_name: str, dry_run: bool +) -> bool: + """Delete a Cloud Run service. + + Args: + project_id: GCP project ID. + location: GCP region of the service. + service_name: Cloud Run service name. + dry_run: If True, log what would be deleted but don't delete. + + Returns: + True on successful delete, when the service is already absent, or + when ``gcloud describe`` times out (treated as expected-absent). + False if the delete call raised an unexpected error. + """ + try: + result = subprocess.run( + [ + "gcloud", + "run", + "services", + "describe", + service_name, + "--region", + location, + "--project", + project_id, + "--format", + "value(name)", + ], + capture_output=True, + text=True, + timeout=GCLOUD_DESCRIBE_TIMEOUT_SEC, + check=False, + ) + except subprocess.TimeoutExpired: + logger.warning( + "gcloud timed out checking Cloud Run service %s after %ds. Skipping. " + "If you did deploy to Cloud Run, delete the service manually: " + "gcloud run services delete %s --region=%s --project=%s", + service_name, + GCLOUD_DESCRIBE_TIMEOUT_SEC, + service_name, + location, + project_id, + ) + return True + + if result.returncode != 0: + logger.info("Cloud Run service %s does not exist, skipping", service_name) + return True + + if dry_run: + logger.info("[DRY RUN] Would delete Cloud Run service: %s", service_name) + return True + + try: + subprocess.run( + [ + "gcloud", + "run", + "services", + "delete", + service_name, + "--region", + location, + "--project", + project_id, + "--quiet", + ], + check=True, + ) + logger.info("Deleted Cloud Run service: %s", service_name) + return True + except subprocess.CalledProcessError as e: + logger.error("Failed to delete Cloud Run service %s: %s", service_name, e) + return False + + +def cleanup( + config: dict[str, Any], + dry_run: bool, + only: list[str], + dataset_id: str = "retail_skill_products", +) -> bool: + """Run cleanup for all or selected resource types. + + Args: + config: Parsed design-spec config dict. + dry_run: If True, log what would be deleted but don't delete. + only: Resource types to delete; subset of + ``["bigquery", "vectorsearch", "cloudrun"]``. + dataset_id: BigQuery dataset name to delete. + + Returns: + True if every step (across the resource types in ``only``) + succeeded; False if any step returned failure. + """ + project_id = config.get("gcp_project_id", "") + if not project_id: + logger.error("gcp_project_id not set in config") + sys.exit(1) + + location = config.get("gcp_region", "us-central1") + project_name = config.get("project_name", "product-search") + collection_id = config.get("collection_id", "retail-skill-products-collection") + + mode = "[DRY RUN] " if dry_run else "" + logger.info("%sCleaning up resources for project: %s", mode, project_id) + logger.info("%sResource types: %s", mode, ", ".join(only)) + logger.info("%sBigQuery dataset: %s", mode, dataset_id) + logger.info("") + + failures: list[str] = [] + + def _track(label: str, success: bool) -> None: + if not success: + failures.append(label) + + if "bigquery" in only: + _track("bigquery", delete_bigquery(project_id, dataset_id, dry_run)) + + if "vectorsearch" in only: + _track( + "vectorsearch (collection)", + delete_vectorsearch_collection( + project_id, location, collection_id, dry_run + ), + ) + _track( + "vectorsearch (v1 index)", + delete_vectorsearch_v1_index( + project_id, location, "retail_skill_products_index", dry_run + ), + ) + + if "cloudrun" in only: + _track( + "cloudrun", + delete_cloudrun(project_id, location, project_name, dry_run), + ) + + logger.info("") + if dry_run: + logger.info("Dry run complete. No resources were deleted.") + logger.info("Run with --confirm to actually delete.") + return True + + if failures: + logger.error( + "Cleanup finished with errors. Failed steps: %s", ", ".join(failures) + ) + logger.error("See messages above for the manual cleanup commands.") + return False + + logger.info("Cleanup complete. All selected resources deleted.") + return True + + +def main(): + """Parse CLI arguments and run cleanup.""" + parser = argparse.ArgumentParser( + description="Clean up all GCP resources created by the product search agent" + ) + parser.add_argument("--config", required=True, help="Path to design-spec.md") + parser.add_argument( + "--confirm", + action="store_true", + help="Actually delete resources (without this flag, runs in dry-run mode)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be deleted without deleting", + ) + parser.add_argument( + "--only", + default="", + help=( + f"Comma-separated resource types to delete. Valid: {','.join(ALL_RESOURCE_TYPES)}. " + f"Default if omitted: {','.join(DEFAULT_RESOURCE_TYPES)} " + "(Cloud Run excluded by default; opt in with --only cloudrun if you deployed to Cloud Run)." # noqa: E501 + ), + ) + parser.add_argument( + "--dataset-id", + default="retail_skill_products", + help="BigQuery dataset name to delete (default: retail_skill_products)", + ) + + args = parser.parse_args() + + config = load_config(args.config) + if not config: + logger.error("Config file not found or empty: %s", args.config) + sys.exit(1) + + # Honor dataset_id from design-spec.md unless overridden on the CLI. + # Otherwise per-case suffixes get ignored and the wrong dataset is "deleted" + # (i.e. cleanup looks for the default and reports a false success). + dataset_id = args.dataset_id + if dataset_id == "retail_skill_products" and config.get("dataset_id"): + dataset_id = config["dataset_id"] + + dry_run = not args.confirm or args.dry_run + + if args.only: + only = [r.strip() for r in args.only.split(",")] + invalid = [r for r in only if r not in ALL_RESOURCE_TYPES] + if invalid: + parser.error( + f"Invalid resource types: {invalid}. Valid: {ALL_RESOURCE_TYPES}" + ) + else: + only = DEFAULT_RESOURCE_TYPES + + if not dry_run: + project_id = config.get("gcp_project_id", "unknown") + print( + f"\nYou are about to permanently delete GCP resources in project: {project_id}" + ) + print(f"Resource types: {', '.join(only)}") + try: + answer = input("\nAre you sure? Type 'yes' to confirm: ").strip().lower() + except (EOFError, KeyboardInterrupt): + answer = "" + if answer != "yes": + print("Aborted. No resources were deleted.") + sys.exit(0) + + ok = cleanup(config, dry_run, only, dataset_id=dataset_id) + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/skills/retail/retail-product-search/scripts/ingest_bigquery.py b/skills/retail/retail-product-search/scripts/ingest_bigquery.py new file mode 100644 index 0000000000..3d35a56e5d --- /dev/null +++ b/skills/retail/retail-product-search/scripts/ingest_bigquery.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +r"""Ingest product catalog data into BigQuery. + +Supports CSV and JSON source formats from GCS or local files. +Validates products against the configured schema before loading. + +Usage: + python ingest_bigquery.py \\ + --project-id my-project \\ + --gcs-bucket my-project-products \\ + --gcs-path products.csv + + python ingest_bigquery.py \\ + --project-id my-project \\ + --local-file data/products.json \\ + --format json + + # Or use design-spec.md for defaults: + python ingest_bigquery.py --config design-spec.md \\ + --local-file data/products.csv +""" + +# pylint: disable=line-too-long +# (validation error messages and argparse help strings are intentionally long.) + +import argparse +import csv +import io +import json +import logging +import pathlib +import sys +from typing import Any + +from google.cloud import bigquery, storage + +# Allow imports from the script's own directory before pip install -e is run. +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +from _setup_utils import load_config # noqa: E402 # pylint: disable=wrong-import-position + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Configure these for your product schema. +# Basic: product_id, name, price, description +# Standard: + category, brand, image_url +# Extended: + rating, stock, manufacturer +# Full: + variants, tags, specifications, reviews +REQUIRED_FIELDS = ["product_id", "name", "price", "description"] +OPTIONAL_FIELDS = ["category", "brand", "image_url", "rating", "stock"] + +SCHEMA = [ + bigquery.SchemaField("product_id", "STRING", mode="REQUIRED"), + bigquery.SchemaField("name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("price", "FLOAT64", mode="REQUIRED"), + bigquery.SchemaField("description", "STRING", mode="REQUIRED"), + bigquery.SchemaField("category", "STRING", mode="NULLABLE"), + bigquery.SchemaField("brand", "STRING", mode="NULLABLE"), + bigquery.SchemaField("image_url", "STRING", mode="NULLABLE"), + bigquery.SchemaField("rating", "FLOAT64", mode="NULLABLE"), + bigquery.SchemaField("stock", "INT64", mode="NULLABLE"), +] + + +def validate_product(product: dict[str, Any], row_num: int) -> list[str]: + """Validate a single product record. + + Args: + product: One product as a dict from the source file. + row_num: 1-based row number for error messages. + + Returns: + List of error strings for this row (empty if valid). + """ + errors = [] + for field in REQUIRED_FIELDS: + if field not in product or not product[field]: + errors.append(f"Row {row_num}: missing required field '{field}'") + + if "price" in product and product["price"]: + try: + float(product["price"]) + except (ValueError, TypeError): + errors.append( + f"Row {row_num}: 'price' must be numeric, got '{product['price']}'" + ) + + if "stock" in product and product["stock"]: + try: + int(product["stock"]) + except (ValueError, TypeError): + errors.append( + f"Row {row_num}: 'stock' must be an integer, got '{product['stock']}'" + ) + + return errors + + +def convert_types(product: dict[str, Any]) -> dict[str, Any]: + """Convert string values to proper types for BigQuery. + + Args: + product: Raw product dict (typically with string-typed values + from CSV parsing). + + Returns: + New dict with ``price``/``rating`` cast to float, ``stock`` cast + to int, and string fields passed through. + """ + converted = {} + converted["product_id"] = product.get("product_id", "") + converted["name"] = product.get("name", "") + converted["description"] = product.get("description", "") + + if "price" in product and product["price"]: + converted["price"] = float(product["price"]) + + if "rating" in product and product["rating"]: + converted["rating"] = float(product["rating"]) + + if "stock" in product and product["stock"]: + converted["stock"] = int(product["stock"]) + + for field in ["category", "brand", "image_url"]: + if field in product: + converted[field] = product[field] + + return converted + + +def load_from_csv(source: str) -> list[dict[str, Any]]: + """Load products from CSV (GCS URI or local path). + + Args: + source: Either a ``gs://bucket/path.csv`` URI or a local filesystem + path to a CSV file. + + Returns: + List of validated, type-converted product dicts. + """ + if source.startswith("gs://"): + parts = source.replace("gs://", "").split("/", 1) + client = storage.Client() + blob = client.bucket(parts[0]).blob(parts[1]) + content = blob.download_as_text() + reader = csv.DictReader(content.splitlines()) + else: + with open(source, encoding="utf-8") as f: + reader = csv.DictReader(f) + return _validate_and_convert(list(reader)) + + return _validate_and_convert(list(reader)) + + +def load_from_json(source: str) -> list[dict[str, Any]]: + """Load products from JSON or JSONL (GCS URI or local path). + + Args: + source: Either a ``gs://bucket/path.json[l]`` URI or a local + filesystem path. JSONL is detected from the suffix. + + Returns: + List of validated, type-converted product dicts. + """ + if source.startswith("gs://"): + parts = source.replace("gs://", "").split("/", 1) + client = storage.Client() + blob = client.bucket(parts[0]).blob(parts[1]) + content = blob.download_as_text() + else: + content = pathlib.Path(source).read_text(encoding="utf-8") + + if source.endswith(".jsonl"): + raw = [ + json.loads(line) for line in content.strip().splitlines() if line.strip() + ] + else: + parsed = json.loads(content) + raw = parsed if isinstance(parsed, list) else parsed.get("products", []) + + return _validate_and_convert(raw) + + +def _validate_and_convert( + raw_products: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Validate and type-convert a list of product dicts. + + Logs a summary of skipped rows when validation produces errors. + + Args: + raw_products: Raw dicts straight from the source loader. + + Returns: + List of cleaned, type-converted product dicts (skipped rows are + excluded). + """ + products = [] + all_errors = [] + + for i, product in enumerate(raw_products, start=1): + errors = validate_product(product, i) + if errors: + all_errors.extend(errors) + continue + products.append(convert_types(product)) + + if all_errors: + logger.warning("Skipped rows with %d validation errors:", len(all_errors)) + for err in all_errors[:10]: + logger.warning(" %s", err) + if len(all_errors) > 10: + logger.warning(" ... and %d more", len(all_errors) - 10) + + logger.info("Loaded %d valid products", len(products)) + return products + + +def _ensure_dataset_and_table( + client: bigquery.Client, + dataset_ref: str, + table_ref: str, + table_id: str, + if_exists: str, +) -> bool: + """Ensure the BigQuery dataset and table exist based on if_exists policy. + + Args: + client: An initialized BigQuery client. + dataset_ref: Fully qualified dataset reference (``project.dataset``). + table_ref: Fully qualified table reference (``project.dataset.table``). + table_id: Bare table name (used when creating the table). + if_exists: Behavior when the dataset already exists: ``"error"``, + ``"skip"``, or ``"rename"`` (interactive only). + + Returns: + ``True`` if ingestion should continue (dataset/table ready), + ``False`` if the existing data should be left alone (skip path). + + Raises: + ValueError: When ``if_exists="rename"`` and the user supplied a new + dataset name; the new name is the exception's argument. The + caller must re-derive refs from this name and re-invoke. + """ + try: + client.get_dataset(dataset_ref) + dataset_exists = True + except Exception: # pylint: disable=broad-exception-caught + dataset_exists = False + + if dataset_exists: + if if_exists == "skip": + try: + client.get_table(table_ref) + count_query = f"SELECT COUNT(*) AS n FROM `{table_ref}`" + row_count = next(iter(client.query(count_query).result())).n + if row_count > 0: + logger.info( + "Dataset %s and table %s already exist with %d rows " + "-- skipping ingestion.", + dataset_ref, + table_ref, + row_count, + ) + return False + logger.info("Table %s exists but is empty -- loading rows.", table_ref) + except Exception: # pylint: disable=broad-exception-caught + logger.info( + "Dataset %s exists but table %s missing -- creating table.", + dataset_ref, + table_ref, + ) + table = bigquery.Table(table_ref, schema=SCHEMA) + client.create_table(table) + elif if_exists == "rename" and sys.stdin.isatty(): + logger.warning("Dataset %s already exists.", dataset_ref) + try: + new_name = input( + " Enter a different dataset name (or Ctrl+C to cancel): " + ).strip() + except (EOFError, KeyboardInterrupt): + print() + logger.info("Cancelled.") + sys.exit(0) + if not new_name: + logger.error("No name provided. Exiting.") + sys.exit(1) + # Caller must re-derive refs from new_name; signal by returning None + # We return the new dataset ID via sys.exit to keep signature simple. + # Raise so the caller can re-try with new name. + raise ValueError(new_name) + else: + logger.error( + "Dataset %s already exists. " + "Re-run with --if-exists skip to reuse it, or --dataset-id .", + dataset_ref, + ) + sys.exit(1) + + if not dataset_exists: + logger.info("Creating dataset %s", dataset_ref) + dataset = bigquery.Dataset(dataset_ref) + dataset.location = "US" + client.create_dataset(dataset) + + logger.info("Creating table %s", table_ref) + table = bigquery.Table(table_ref, schema=SCHEMA) + client.create_table(table) + + return True + + +def ingest( + project_id: str, + dataset_id: str, + table_id: str, + source: str, + source_format: str = "csv", + if_exists: str = "error", +): + """Ingest products into BigQuery. + + if_exists controls behavior when the target dataset already exists: + - "error" (default): fail with a clear message + - "skip": no-op if the table already has rows (idempotent re-runs) + - "rename": prompt for a new name (interactive only) + """ + client = bigquery.Client(project=project_id) + + dataset_ref = f"{project_id}.{dataset_id}" + table_ref = f"{project_id}.{dataset_id}.{table_id}" + + try: + should_continue = _ensure_dataset_and_table( + client, dataset_ref, table_ref, table_id, if_exists + ) + except ValueError as new_name: + # "rename" path: recurse with the new dataset name. + dataset_id = str(new_name) + dataset_ref = f"{project_id}.{dataset_id}" + table_ref = f"{project_id}.{dataset_id}.{table_id}" + try: + client.get_dataset(dataset_ref) + logger.error( + "Dataset %s also exists. Re-run with --dataset-id .", + dataset_ref, + ) + sys.exit(1) + except Exception: # pylint: disable=broad-exception-caught + pass + should_continue = _ensure_dataset_and_table( + client, dataset_ref, table_ref, table_id, "error" + ) + + if not should_continue: + return + + # Load products + if source_format == "json": + products = load_from_json(source) + else: + products = load_from_csv(source) + + if not products: + logger.error("No valid products to ingest") + sys.exit(1) + + # Use a load job (batch) instead of streaming insert. Streaming inserts go + # through a buffer that can keep stale dataset-uuid references for ~5-10 min + # after a delete, which causes "Dataset is deleted" errors when the same + # dataset name is recreated quickly (e.g. test re-runs). Load jobs route + # through a different path with no such race. + job_config = bigquery.LoadJobConfig( + schema=SCHEMA, + write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE, + source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON, + ) + ndjson = "\n".join(json.dumps(p) for p in products).encode("utf-8") + load_job = client.load_table_from_file( + io.BytesIO(ndjson), + table_ref, + job_config=job_config, + ) + load_job.result() # Wait for completion; raises on error. + if load_job.errors: + logger.error("Load job errors: %s", load_job.errors) + sys.exit(1) + + logger.info("Successfully ingested %d products to %s", len(products), table_ref) + + +def main(): + """Parse CLI arguments and ingest products to BigQuery.""" + parser = argparse.ArgumentParser(description="Ingest product catalog to BigQuery") + parser.add_argument( + "--config", + default="", + help="Path to design-spec.md (provides defaults for other args)", + ) + parser.add_argument("--project-id", help="GCP project ID") + parser.add_argument( + "--dataset-id", + default="retail_skill_products", + help="BigQuery dataset ID", + ) + parser.add_argument("--table-id", default="products", help="BigQuery table ID") + parser.add_argument("--gcs-bucket", help="GCS bucket name (used with --gcs-path)") + parser.add_argument("--gcs-path", help="Path to data file in GCS bucket") + parser.add_argument("--local-file", help="Path to local data file") + parser.add_argument( + "--format", choices=["csv", "json"], default="csv", help="Source format" + ) + parser.add_argument( + "--if-exists", + choices=["error", "skip", "rename"], + default="error", + help=( + "Behavior when dataset already exists: error (default), " + "skip (idempotent re-run), rename (interactive)" + ), + ) + + args = parser.parse_args() + + # Load design-spec.md defaults -- CLI args override config values + if args.config: + cfg = load_config(args.config) + if not args.project_id: + args.project_id = cfg.get("gcp_project_id", "") + + if not args.project_id: + parser.error( + "--project-id is required (or set gcp_project_id in design-spec.md)" + ) + if args.local_file: + source = args.local_file + elif args.gcs_bucket and args.gcs_path: + source = f"gs://{args.gcs_bucket}/{args.gcs_path}" + else: + parser.error("Provide either --local-file or both --gcs-bucket and --gcs-path") + + ingest( + project_id=args.project_id, + dataset_id=args.dataset_id, + table_id=args.table_id, + source=source, + source_format=args.format, + if_exists=args.if_exists, + ) + + +if __name__ == "__main__": + main() diff --git a/skills/retail/retail-product-search/scripts/ingest_vertex_search.py b/skills/retail/retail-product-search/scripts/ingest_vertex_search.py new file mode 100644 index 0000000000..e3e3345290 --- /dev/null +++ b/skills/retail/retail-product-search/scripts/ingest_vertex_search.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +r"""Ingest products into Vertex AI Vector Search 2.0 Collection. + +Reads products from BigQuery, creates a Vector Search 2.0 Collection +(if it doesn't exist), and inserts products one at a time. `AlreadyExists` +errors are caught so re-runs are idempotent. Embeddings are auto-generated +by the Collection's configured model. + +Usage: + python ingest_vertex_search.py \\ + --project-id my-project \\ + --collection-id my-products + + # Or use design-spec.md for defaults: + python ingest_vertex_search.py --config design-spec.md +""" + +# pylint: disable=line-too-long +# (error messages with collection_id hints and gcloud commands are intentionally long.) + +import argparse +import logging +import pathlib +import sys +from typing import Any + +from google.api_core import exceptions +from google.cloud import bigquery, vectorsearch +from google.protobuf import struct_pb2 + +# Allow imports from the script's own directory before pip install -e is run. +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +from _setup_utils import load_config # noqa: E402 # pylint: disable=wrong-import-position + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +DEFAULT_EMBEDDING_MODEL = "gemini-embedding-001" +DEFAULT_EMBEDDING_FIELDS = ["name", "description", "category", "brand"] + +# All product fields stored as data object fields +PRODUCT_DATA_FIELDS = { + "product_id": "string", + "name": "string", + "price": "number", + "description": "string", + "category": "string", + "brand": "string", + "image_url": "string", + "rating": "number", + "stock": "integer", +} + + +def fetch_products( + project_id: str, dataset_id: str, table_id: str +) -> list[dict[str, Any]]: + """Fetch all products from a BigQuery table. + + Args: + project_id: GCP project ID. + dataset_id: BigQuery dataset name. + table_id: BigQuery table name. + + Returns: + List of product dicts (one per BQ row). + """ + client = bigquery.Client(project=project_id) + query = f"SELECT * FROM `{project_id}.{dataset_id}.{table_id}`" + + logger.info("Fetching products from %s.%s.%s", project_id, dataset_id, table_id) + results = client.query(query).result() + products = [dict(row.items()) for row in results] + logger.info("Fetched %d products", len(products)) + return products + + +def build_text_template(embedding_fields: list[str]) -> str: + """Build the ``text_template`` for auto-embedding from field names. + + The template uses ``{field_name}`` placeholders that VS 2.0 resolves + from the data object's data fields. + + Args: + embedding_fields: Field names to include in the template. + + Returns: + Pipe-separated template string, e.g. + ``"name: {name} | description: {description}"``. + """ + parts = [f"{field}: {{{field}}}" for field in embedding_fields] + return " | ".join(parts) + + +def get_data_schema(products: list[dict[str, Any]]) -> dict[str, Any]: + """Build the data schema from known product fields. + + Only includes fields that exist in the actual data (sampled from the + first 10 products). + + Args: + products: Product dicts loaded from BigQuery. + + Returns: + JSON-Schema-style dict ``{"type": "object", "properties": {...}}`` + with one property per detected field. + """ + properties: dict[str, Any] = {} + sample_keys: set[str] = set() + for p in products[:10]: + sample_keys.update(p.keys()) + + for field, field_type in PRODUCT_DATA_FIELDS.items(): + if field in sample_keys: + properties[field] = {"type": field_type} + + return {"type": "object", "properties": properties} + + +def create_collection_if_needed( # pylint: disable=too-many-arguments + project_id: str, + location: str, + collection_id: str, + embedding_model: str, + embedding_fields: list[str], + data_schema: dict[str, Any], +) -> str: + """Create a Vector Search 2.0 Collection if it doesn't exist. + + Args: + project_id: GCP project ID. + location: GCP region for the collection. + collection_id: Collection ID (RFC1035-compliant, max 63 chars). + embedding_model: Auto-embedding model ID (e.g. + ``"gemini-embedding-001"``). + embedding_fields: Fields used in the embedding text template. + data_schema: JSON-Schema-style dict from :func:`get_data_schema`. + + Returns: + Fully qualified collection resource path. + """ + client = vectorsearch.VectorSearchServiceClient() + parent = f"projects/{project_id}/locations/{location}" + collection_name = f"{parent}/collections/{collection_id}" + + try: + client.get_collection( + request=vectorsearch.GetCollectionRequest(name=collection_name) + ) + logger.info("Collection '%s' already exists", collection_id) + return collection_name + except exceptions.NotFound: + pass + + text_template = build_text_template(embedding_fields) + logger.info( + "Creating collection '%s' with text_template: %s", collection_id, text_template + ) + + request = vectorsearch.CreateCollectionRequest( + parent=parent, + collection_id=collection_id, + collection={ + "data_schema": data_schema, + "vector_schema": { + "text_embedding": { + "dense_vector": { + "dimensions": 768, + "vertex_embedding_config": { + "model_id": embedding_model, + "text_template": text_template, + "task_type": "RETRIEVAL_DOCUMENT", + }, + }, + }, + }, + }, + ) + + try: + operation = client.create_collection(request=request) + logger.info("Waiting for collection creation to complete...") + operation.result() + logger.info("Collection created successfully") + except Exception as e: # pylint: disable=broad-exception-caught + # The SDK often mangles the underlying API error into a confusing gRPC + # StatusCode mismatch. Most common real causes: + # 1. collection_id > 63 chars (RFC1035 limit) + # 2. collection_id has uppercase or invalid characters + # 3. location/region doesn't support Vector Search + # 4. project lacks aiplatform.user IAM + hints = [] + if len(collection_id) > 63: + hints.append( + f"collection_id is {len(collection_id)} chars (max 63 per RFC1035). " + "Shorten it via the 'collection_id' key in design-spec.md." + ) + if not collection_id.islower() or any(c.isupper() for c in collection_id): + hints.append( + "collection_id must be all lowercase letters, digits, and dashes." + ) + hint_text = ( + ("\n Likely cause: " + " ".join(hints)) + if hints + else ( + "\n Possible causes: collection_id too long (>63), uppercase chars, " + "region doesn't support Vector Search, or missing aiplatform.user IAM." + ) + ) + logger.error( + "Error creating collection '%s': %s%s\n" + " Manual cleanup if the collection was partially created:\n" + " gcloud ai vector-search-collections delete %s " + "--region=%s --project=%s", + collection_id, + e, + hint_text, + collection_id, + location, + project_id, + ) + sys.exit(1) + + return collection_name + + +def ingest_products( + collection_path: str, + products: list[dict[str, Any]], +) -> None: + """Insert products into the Vector Search 2.0 Collection. + + Embeddings are auto-generated by the Collection's configured model. + Uses individual inserts for reliability. + """ + client = vectorsearch.DataObjectServiceClient() + + created = 0 + skipped = 0 + errors = 0 + + for i, product in enumerate(products): + product_id = str(product.get("product_id", "")) + # Ensure ID is RFC1035 compliant (lowercase, starts with letter) + safe_id = product_id.lower().replace("_", "-") + if safe_id and not safe_id[0].isalpha(): + safe_id = "p-" + safe_id + + data_struct = struct_pb2.Struct() # pylint: disable=no-member + for field, field_type in PRODUCT_DATA_FIELDS.items(): + value = product.get(field) + if value is None: + continue + # Preserve proper types for the schema + if field_type == "number": + data_struct.update({field: float(value)}) + elif field_type == "integer": + data_struct.update({field: int(value)}) + else: + data_struct.update({field: str(value)}) + + try: + client.create_data_object( + parent=collection_path, + data_object_id=safe_id, + data_object=vectorsearch.DataObject(data=data_struct), + ) + created += 1 + except exceptions.AlreadyExists: + skipped += 1 + except Exception as e: # pylint: disable=broad-exception-caught + errors += 1 + logger.warning("Failed to insert %s: %s", safe_id, e) + + if (i + 1) % 50 == 0 or (i + 1) == len(products): + logger.info("Processed %d/%d products", i + 1, len(products)) + + logger.info( + "Ingestion complete: %d created, %d skipped, %d errors", + created, + skipped, + errors, + ) + + +def ingest_pipeline( # pylint: disable=too-many-arguments + *, + project_id: str, + location: str, + dataset_id: str, + table_id: str, + collection_id: str, + embedding_model: str, + embedding_fields: list[str], +) -> str | None: + """Run the Vector Search ingestion pipeline programmatically. + + Args: + project_id: GCP project ID. + location: GCP region for the collection. + dataset_id: BigQuery dataset to read products from. + table_id: BigQuery table to read products from. + collection_id: Vector Search collection ID. + embedding_model: Auto-embedding model ID. + embedding_fields: Fields used in the embedding text template. + + Returns: + The collection path on success (whether freshly created or + already populated and skipped). ``None`` on error (e.g. no + products found in BigQuery). + """ + collection_path = ( + f"projects/{project_id}/locations/{location}/collections/{collection_id}" + ) + + # Re-run short-circuit: if the collection already exists AND is populated, + # skip the BQ fetch + per-row insert loop. + try: + vs_client = vectorsearch.VectorSearchServiceClient() + vs_client.get_collection( + request=vectorsearch.GetCollectionRequest(name=collection_path) + ) + search_client = vectorsearch.DataObjectSearchServiceClient() + probe = search_client.search_data_objects( + request=vectorsearch.SearchDataObjectsRequest( + parent=collection_path, + semantic_search=vectorsearch.SemanticSearch( + search_text="probe", + search_field="text_embedding", + task_type="QUESTION_ANSWERING", + top_k=1, + output_fields=vectorsearch.OutputFields(data_fields=["product_id"]), + ), + ) + ) + if any(True for _ in probe): + logger.info( + "Collection '%s' already populated -- " + "skipping ingestion (re-run idempotent path).", + collection_id, + ) + logger.info("Collection path: %s", collection_path) + return collection_path + except exceptions.NotFound: + pass + + products = fetch_products(project_id, dataset_id, table_id) + if not products: + logger.error("No products found in BigQuery") + return None + + data_schema = get_data_schema(products) + collection_path = create_collection_if_needed( + project_id, + location, + collection_id, + embedding_model, + embedding_fields, + data_schema, + ) + + ingest_products(collection_path, products) + + logger.info( + "Successfully ingested %d products into collection '%s'", + len(products), + collection_id, + ) + logger.info("Collection path: %s", collection_path) + return collection_path + + +def main(): + """Parse CLI arguments and run the Vector Search ingestion pipeline.""" + parser = argparse.ArgumentParser( + description="Ingest products to Vertex AI Vector Search 2.0" + ) + parser.add_argument( + "--config", + default="", + help="Path to design-spec.md (provides defaults for other args)", + ) + parser.add_argument("--project-id", help="GCP project ID") + parser.add_argument("--location", default="us-central1", help="GCP region") + parser.add_argument( + "--dataset-id", default="retail_skill_products", help="BigQuery dataset" + ) + parser.add_argument("--table-id", default="products", help="BigQuery table") + parser.add_argument( + "--collection-id", + default="retail-skill-products-collection", + help="Vector Search 2.0 Collection ID", + ) + parser.add_argument( + "--embedding-model", + default=DEFAULT_EMBEDDING_MODEL, + help="Auto-embedding model (default: gemini-embedding-001)", + ) + parser.add_argument( + "--embedding-fields", + default=",".join(DEFAULT_EMBEDDING_FIELDS), + help="Comma-separated fields for embedding text template", + ) + + args = parser.parse_args() + + # Load design-spec.md defaults -- CLI args override config values + if args.config: + cfg = load_config(args.config) + if not args.project_id: + args.project_id = cfg.get("gcp_project_id", "") + if args.location == "us-central1" and cfg.get("gcp_region"): + args.location = cfg["gcp_region"] + if cfg.get("dataset_id") and args.dataset_id == "retail_skill_products": + args.dataset_id = cfg["dataset_id"] + if cfg.get("table_id") and args.table_id == "products": + args.table_id = cfg["table_id"] + if ( + cfg.get("collection_id") + and args.collection_id == "retail-skill-products-collection" + ): + args.collection_id = cfg["collection_id"] + if cfg.get("embedding_model"): + model_name = cfg["embedding_model"].split(" ")[0] + if args.embedding_model == DEFAULT_EMBEDDING_MODEL: + args.embedding_model = model_name + if cfg.get("embedding_fields") and args.embedding_fields == ",".join( + DEFAULT_EMBEDDING_FIELDS + ): + args.embedding_fields = cfg["embedding_fields"] + + if not args.project_id: + parser.error( + "--project-id is required (or set gcp_project_id in design-spec.md)" + ) + + embedding_fields = [f.strip() for f in args.embedding_fields.split(",")] + + result_path = ingest_pipeline( + project_id=args.project_id, + location=args.location, + dataset_id=args.dataset_id, + table_id=args.table_id, + collection_id=args.collection_id, + embedding_model=args.embedding_model, + embedding_fields=embedding_fields, + ) + if result_path: + logger.info("Set VECTOR_SEARCH_COLLECTION env var to this path in your agent.") + + +if __name__ == "__main__": + main() diff --git a/skills/retail/retail-product-search/scripts/retrievers.py b/skills/retail/retail-product-search/scripts/retrievers.py new file mode 100644 index 0000000000..89927c9dcb --- /dev/null +++ b/skills/retail/retail-product-search/scripts/retrievers.py @@ -0,0 +1,163 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Retrieval helpers for the retail product search agent. + +Provides :func:`search_collection` for semantic product lookup via +Vertex AI Vector Search 2.0, and :func:`search` as a convenience +wrapper that reads the collection path from the environment. +""" + +# pylint: disable=line-too-long +# (error messages and example URLs are intentionally long; breaking them hurts copy-paste UX.) + +import os +import re + +from google.cloud import vectorsearch + +_COLLECTION_PATH_RE = re.compile( + r"^projects/[^/\s]+/locations/[^/\s]+/collections/[^/\s]+$" +) + + +def _create_search_client(): + return vectorsearch.DataObjectSearchServiceClient() + + +def _format_result(index: int, result) -> str: + """Format a single search result as a compact one-line string. + + Args: + index: 1-based product index used in the formatted output. + result: A Vector Search ``SearchDataObjectsResponse`` entry. + + Returns: + A one-line string of the form + ``"Product N: name, $price, by brand, rated R/5, "``. + """ + data = result.data_object.data + name = data.get("name", "Unknown") + price = data.get("price", "N/A") + brand = data.get("brand", "") + rating = data.get("rating", "") + description = data.get("description", "") + + parts = [f"{name}, ${price}"] + if brand: + parts.append(f"by {brand}") + if rating: + parts.append(f"rated {rating} out of 5") + if description: + parts.append(description[:80]) + + return f"Product {index}: " + ", ".join(parts) + + +def search_collection( + query: str, + collection_path: str, + top_k: int = 10, +) -> str: + """Search a Vector Search 2.0 Collection using semantic search. + + Args: + query: The search query text. + collection_path: Full resource path of the collection. + top_k: Number of results to return. + + Returns: + Formatted string containing relevant document content. + """ + client = _create_search_client() + + request = vectorsearch.SearchDataObjectsRequest( + parent=collection_path, + semantic_search=vectorsearch.SemanticSearch( + search_text=query, + search_field="text_embedding", + task_type="QUESTION_ANSWERING", + top_k=top_k, + output_fields=vectorsearch.OutputFields( + data_fields=[ + "product_id", + "name", + "price", + "description", + "category", + "brand", + "rating", + "stock", + ] + ), + ), + ) + + results = client.search_data_objects(request) + + formatted_parts = [ + _format_result(i + 1, result) for i, result in enumerate(results) + ] + + if not formatted_parts: + return "No matching products found. Suggest broadening the search." + + return ( + "Found " + + str(len(formatted_parts)) + + " products. " + + ". ".join(formatted_parts) + ) + + +def search(query: str, top_k: int = 5) -> str: + """Run the SKILL.md smoke-test query against the configured collection. + + Reads the collection path from the ``VECTOR_SEARCH_COLLECTION`` env var, + which is what ``scripts/setup.py`` prints at the end of a successful run. + + Args: + query: The product search query in natural language. + top_k: Maximum number of matching products to return. + + Returns: + A formatted string with the matching products, or a "no matches" + message if the collection returned nothing. + + Raises: + RuntimeError: If ``VECTOR_SEARCH_COLLECTION`` is not set, or is set + to a malformed path (typically with an embedded newline from a + wrapped multi-line shell paste). + """ + raw = os.environ.get("VECTOR_SEARCH_COLLECTION") + if not raw: + raise RuntimeError( + "VECTOR_SEARCH_COLLECTION env var is not set. Set it to the collection " + "path printed by scripts/setup.py, e.g. " + "projects//locations/us-central1/collections/retail-skill-products-collection" + ) + # Strip whitespace -- a newline embedded mid-path (from a wrapped multi-line + # shell paste) silently causes the SDK to 501 from the wrong endpoint. + collection_path = raw.strip() + if not _COLLECTION_PATH_RE.match(collection_path): + raise RuntimeError( + "VECTOR_SEARCH_COLLECTION is malformed -- expected " + "'projects//locations//collections/' with no whitespace. " + f"Got: {raw!r}. " + "Tip: type the export on a single line, e.g. " + "export VECTOR_SEARCH_COLLECTION=" + '"projects/$GOOGLE_CLOUD_PROJECT/locations/us-central1/' + 'collections/retail-skill-products-collection"' + ) + return search_collection(query, collection_path, top_k=top_k) diff --git a/skills/retail/retail-product-search/scripts/setup.py b/skills/retail/retail-product-search/scripts/setup.py new file mode 100644 index 0000000000..8bb9515164 --- /dev/null +++ b/skills/retail/retail-product-search/scripts/setup.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Automated project setup driven by design-spec.md. + +Reads the design-spec.md configuration and runs the appropriate ingestion +steps based on the user's answers. This is the single entry point for both +the `vs` CLI and manual setup. + +Runs scripts in-process via direct Python imports -- does NOT shell out to +`python scripts/...`. That makes setup safe to invoke from any cwd, against +any Python interpreter, regardless of whether the skill files were copied +into the workspace. + +Usage: + python scripts/setup.py --config ./design-spec.md + python scripts/setup.py --config ./design-spec.md --dry-run +""" + +# pylint: disable=line-too-long +# (log lines and error messages with config paths are intentionally long.) + +import argparse +import logging +import pathlib +import sys +from typing import Any + +# Allow imports from the script's own directory before pip install -e is run. +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +# pylint: disable=wrong-import-position +from _paths import SAMPLE_PRODUCTS_CSV # noqa: E402 +from _setup_utils import load_config # noqa: E402 + +import validate_schema # noqa: E402 +import ingest_bigquery # noqa: E402 +import ingest_vertex_search # noqa: E402 +# pylint: enable=wrong-import-position + +logging.basicConfig(level=logging.INFO, format="%(message)s") +logger = logging.getLogger(__name__) + + +# Indirection layer so tests can monkeypatch these without touching the +# underlying scripts. Production: forwards to the real implementations. +def _validate_file(file_path, fields_level): + return validate_schema.validate_file(file_path, fields_level) + + +def _run_bigquery_ingest( # pylint: disable=too-many-arguments + *, project_id, dataset_id, table_id, source, source_format, if_exists +): + return ingest_bigquery.ingest( + project_id=project_id, + dataset_id=dataset_id, + table_id=table_id, + source=source, + source_format=source_format, + if_exists=if_exists, + ) + + +def _run_vertex_ingest(**kwargs): + return ingest_vertex_search.ingest_pipeline(**kwargs) + + +def _resolve_data_source(cfg: dict[str, Any]) -> tuple[str | None, str]: + """Resolve ``data_source`` from the spec into ``(source_path, format)``. + + Args: + cfg: Parsed design-spec config dict. + + Returns: + ``(source_path, format)`` where ``source_path`` is a local path or + ``gs://`` URI and ``format`` is ``"csv"`` or ``"json"``. Returns + ``(None, "")`` if the source is missing or broken so the caller + can error out cleanly. + """ + raw = (cfg.get("data_source") or "").strip() + + if raw.startswith("gs://"): + return raw, _format_from_path(raw) + + if raw and raw not in ("default", "assets/sample-products.csv"): + # User specified an explicit local path. + if pathlib.Path(raw).exists(): + return raw, _format_from_path(raw) + logger.error("\n data_source '%s' does not exist.", raw) + return None, "" + + # Default: the bundled sample inside the install dir. + if SAMPLE_PRODUCTS_CSV.exists(): + return str(SAMPLE_PRODUCTS_CSV), "csv" + + logger.error("\n Bundled sample not found at %s", SAMPLE_PRODUCTS_CSV) + return None, "" + + +def _format_from_path(path: str) -> str: + """Return ``"json"`` for ``.json``/``.jsonl`` paths, else ``"csv"``. + + Args: + path: Data file path (local or gs:// URI). + + Returns: + Either ``"json"`` or ``"csv"`` for the BigQuery loader. + """ + return "json" if path.endswith((".json", ".jsonl")) else "csv" + + +def setup(config_path: str, dry_run: bool = False) -> bool: # pylint: disable=too-many-locals + """Run the full setup pipeline based on design-spec.md. + + Args: + config_path: Path to ``design-spec.md`` with the user's config. + dry_run: If True, log the pipeline shape without running ingest. + + Returns: + True if every step succeeded; False if the data source couldn't + be resolved or validation produced errors. + """ + cfg = load_config(config_path) + + project_id = cfg.get("gcp_project_id", "") + if not project_id: + logger.error("gcp_project_id is required in design-spec.md") + sys.exit(1) + + product_fields = cfg.get("product_fields", "Extended") + dataset_id = cfg.get("dataset_id", "retail_skill_products") + table_id = cfg.get("table_id", "products") + catalog_size = cfg.get("catalog_size", "1K-50K") + gcp_region = cfg.get("gcp_region", "us-central1") + collection_id = cfg.get("collection_id", "retail-skill-products-collection") + embedding_model = cfg.get("embedding_model", "gemini-embedding-001") + raw_fields = cfg.get("embedding_fields", "name, description, category, brand") + embedding_fields = [f.strip() for f in raw_fields.split(",")] + + logger.info("=" * 60) + logger.info("SETUP PIPELINE (driven by design-spec.md)") + logger.info("=" * 60) + logger.info(" Project: %s", project_id) + logger.info(" Region: %s", gcp_region) + + source, source_format = _resolve_data_source(cfg) + if source is None: + return False + + logger.info(" Data source: %s", source) + + if dry_run: + logger.info( + "\n[dry-run] Would validate, ingest to BigQuery, then create Vector Search collection." + ) + return True + + # Step A: validate. + # validate_schema.load_records reads from the local filesystem only; + # skip the schema-shape check for gs:// sources and let + # ingest_bigquery.load_from_csv validate inline as it streams the file. + if source.startswith("gs://"): + logger.info( + "\n Skipping schema validation for gs:// source (validated inline during ingest)." + ) + else: + logger.info("\n Validating sample data...") + valid_count, errors = _validate_file(source, product_fields) + if errors: + logger.error(" Validation produced %d errors. First 5:", len(errors)) + for e in errors[:5]: + logger.error(" %s", e) + return False + logger.info(" Validated %d records.", valid_count) + + # Step B: ingest to BigQuery (idempotent re-run via if_exists='skip') + logger.info("\n Ingesting into BigQuery...") + _run_bigquery_ingest( + project_id=project_id, + dataset_id=dataset_id, + table_id=table_id, + source=source, + source_format=source_format, + if_exists="skip", + ) + + # Step C: create Vector Search collection + ingest + logger.info("\n Creating Vector Search collection...") + collection_path = _run_vertex_ingest( + project_id=project_id, + location=gcp_region, + dataset_id=dataset_id, + table_id=table_id, + collection_id=collection_id, + embedding_model=embedding_model, + embedding_fields=embedding_fields, + ) + + if catalog_size == "500K+": + logger.info("\n NOTE: Catalog size 500K+ detected.") + logger.info( + " Consider using Dataflow for batch ingestion instead of direct inserts." + ) + + logger.info("\n" + "=" * 60) + logger.info("SETUP COMPLETE") + logger.info("=" * 60) + if collection_path: + logger.info(" Collection path: %s", collection_path) + logger.info( + " Set VECTOR_SEARCH_COLLECTION env var to this path in your agent." + ) + return True + + +def main(): + """Parse CLI arguments and run the setup pipeline.""" + parser = argparse.ArgumentParser( + description="Run setup pipeline from design-spec.md" + ) + parser.add_argument("--config", required=True, help="Path to design-spec.md") + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would run without doing it", + ) + + args = parser.parse_args() + ok = setup(args.config, args.dry_run) + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/skills/retail/retail-product-search/scripts/validate_schema.py b/skills/retail/retail-product-search/scripts/validate_schema.py new file mode 100644 index 0000000000..f688350a2c --- /dev/null +++ b/skills/retail/retail-product-search/scripts/validate_schema.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Validate a product catalog CSV/JSON against the expected schema. + +Usage: + python validate_schema.py --file products.csv --fields-level Standard + python validate_schema.py --file products.json --fields-level Extended +""" + +# pylint: disable=line-too-long +# (validation error messages with field names are intentionally long.) + +import argparse +import csv +import json +import logging +import pathlib +import sys +from typing import Any + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +FIELD_LEVELS = { + "Basic": { + "required": ["product_id", "name", "price", "description"], + "optional": [], + }, + "Standard": { + "required": ["product_id", "name", "price", "description"], + "optional": ["category", "brand", "image_url"], + }, + "Extended": { + "required": ["product_id", "name", "price", "description"], + "optional": [ + "category", + "brand", + "image_url", + "rating", + "stock", + "manufacturer", + ], + }, + "Full": { + "required": ["product_id", "name", "price", "description"], + "optional": [ + "category", + "brand", + "image_url", + "rating", + "stock", + "manufacturer", + "variants", + "tags", + "specifications", + "reviews", + ], + }, +} + + +def load_records(file_path: pathlib.Path) -> list[dict[str, Any]]: + """Load records from a CSV, JSON, or JSONL file. + + Args: + file_path: Path to the product data file. The file type is + inferred from the suffix (``.csv``, ``.json``, or ``.jsonl``). + + Returns: + List of records as dicts. + + Raises: + ValueError: If the suffix isn't supported, or the JSON content is + neither a list nor a ``{"products": [...]}`` object. + """ + suffix = file_path.suffix.lower() + + if suffix == ".csv": + with open(file_path, encoding="utf-8") as f: + return list(csv.DictReader(f)) + elif suffix in (".json", ".jsonl"): + content = file_path.read_text(encoding="utf-8") + if suffix == ".jsonl": + return [ + json.loads(line) + for line in content.strip().splitlines() + if line.strip() + ] + parsed = json.loads(content) + if isinstance(parsed, list): + return parsed + if isinstance(parsed, dict) and "products" in parsed: + return parsed["products"] + raise ValueError('JSON must be an array or {"products": [...]}') + else: + raise ValueError(f"Unsupported file type: {suffix}. Use .csv, .json, or .jsonl") + + +def _validate_single_record( + record: dict[str, Any], + row_num: int, + required: list[str], + all_fields: set[str], + higher_level_fields: set[str], + seen_warned_fields: set[str], +) -> list[str]: + """Validate a single record against the schema. + + Args: + record: One product record as a dict from CSV/JSON. + row_num: 1-based row number for error messages. + required: Required field names at the selected fields-level. + all_fields: Set of fields recognized at the selected fields-level. + higher_level_fields: Fields recognized at any higher level; these + warn (and are ignored) instead of failing the row. + seen_warned_fields: Mutable set tracking which higher-level fields + already produced a warning, so each fires once per run. + + Returns: + List of error strings for this row (empty if the row is valid). + """ + row_errors = [] + + for field in required: + if field not in record or not record[field]: + row_errors.append(f"Row {row_num}: missing required field '{field}'") + + if "price" in record and record["price"]: + try: + float(record["price"]) + except (ValueError, TypeError): + row_errors.append( + f"Row {row_num}: 'price' must be numeric, got '{record['price']}'" + ) + + if "rating" in record and record["rating"]: + try: + val = float(record["rating"]) + if not 0 <= val <= 5: + row_errors.append(f"Row {row_num}: 'rating' should be 0-5, got {val}") + except (ValueError, TypeError): + row_errors.append(f"Row {row_num}: 'rating' must be numeric") + + if "stock" in record and record["stock"]: + try: + int(record["stock"]) + except (ValueError, TypeError): + row_errors.append(f"Row {row_num}: 'stock' must be an integer") + + extra_fields = set(record.keys()) - all_fields + known_extras = extra_fields & higher_level_fields + unknown_extras = extra_fields - higher_level_fields + + for field in known_extras - seen_warned_fields: + logger.warning( + "Field '%s' present in data but outside the '%s' schema. " + "It will be ignored. Pick a higher --fields-level if you want it indexed.", + field, + "current", + ) + seen_warned_fields.add(field) + + if unknown_extras: + row_errors.append(f"Row {row_num}: unrecognized fields: {sorted(unknown_extras)}") + + return row_errors + + +def validate( + records: list[dict[str, Any]], fields_level: str +) -> tuple[int, list[str]]: + """Validate records against the schema for ``fields_level``. + + Args: + records: All product records loaded from the source. + fields_level: One of ``"Basic"``, ``"Standard"``, ``"Extended"``, + ``"Full"``. + + Returns: + ``(valid_count, errors)`` where ``valid_count`` is the number of + rows that passed and ``errors`` is the concatenated list of error + strings across all rows. + """ + schema = FIELD_LEVELS[fields_level] + required = schema["required"] + all_fields = set(required + schema["optional"]) + # Fields that exist at any higher level -- treat as known-but-out-of-scope. + # Warn instead of erroring so users with richer CSVs don't have to strip columns. + higher_level_fields: set[str] = set() + for level_schema in FIELD_LEVELS.values(): + higher_level_fields.update(level_schema["required"]) + higher_level_fields.update(level_schema["optional"]) + + errors = [] + seen_warned_fields: set[str] = set() + valid = 0 + + for i, record in enumerate(records, start=1): + row_errors = _validate_single_record( + record, i, required, all_fields, higher_level_fields, seen_warned_fields + ) + if row_errors: + errors.extend(row_errors) + else: + valid += 1 + + return valid, errors + + +def validate_file(file_path, fields_level: str) -> tuple[int, list[str]]: + """Load a CSV/JSON file and validate it in one call. + + Convenience wrapper for orchestrators that don't need the records. + + Args: + file_path: Path to the product data file (CSV, JSON, or JSONL). + fields_level: One of ``"Basic"``, ``"Standard"``, ``"Extended"``, + ``"Full"``. + + Returns: + ``(valid_count, errors)`` -- same shape as ``validate()``. + """ + records = load_records(pathlib.Path(file_path)) + return validate(records, fields_level) + + +def main(): + """Parse CLI arguments and run schema validation.""" + parser = argparse.ArgumentParser(description="Validate product catalog schema") + parser.add_argument( + "--file", required=True, help="Path to product data file (CSV/JSON)" + ) + parser.add_argument( + "--fields-level", + choices=["Basic", "Standard", "Extended", "Full"], + default="Standard", + help="Product fields level (default: Standard)", + ) + + args = parser.parse_args() + file_path = pathlib.Path(args.file) + + if not file_path.exists(): + logger.error("File not found: %s", file_path) + sys.exit(1) + + records = load_records(file_path) + logger.info("Loaded %d records from %s", len(records), file_path) + + valid_count, errors = validate(records, args.fields_level) + + if errors: + logger.warning("%d validation errors found:", len(errors)) + for err in errors[:20]: + logger.warning(" %s", err) + if len(errors) > 20: + logger.warning(" ... and %d more", len(errors) - 20) + + logger.info( + "Validation complete: %d/%d records valid", valid_count, len(records) + ) + + if errors: + sys.exit(1) + + +if __name__ == "__main__": + main() From 516f2790b2e662d2e46aee4208bc2c614d20fd16 Mon Sep 17 00:00:00 2001 From: Shahin Saadati Date: Wed, 29 Jul 2026 15:36:45 -0700 Subject: [PATCH 2/5] feat: add stale-python-version and runnability-test checks to align-recipe-pyproject and add docstring regression tests to extract-python-environment-variables --- .../skills/align-recipe-pyproject/SKILL.md | 38 +- .../scripts/align_pyproject.py | 244 ++++++++++- .../align-recipe-pyproject/tests/conftest.py | 24 ++ .../tests/test_align_pyproject.py | 208 ++++++++++ .../SKILL.md | 60 ++- .../scripts/extract_env_vars.py | 387 +++++++++++++++--- .../tests/test_extract_env_vars.py | 311 ++++++++++++++ .../generate-python-runnability-test/SKILL.md | 22 +- .../scripts/generate_runnability_test.py | 195 ++++++++- .../tests/conftest.py | 24 ++ .../tests/test_generate_runnability_test.py | 187 +++++++++ .agents/skills/prepare-python-recipe/SKILL.md | 164 ++++++-- 12 files changed, 1757 insertions(+), 107 deletions(-) create mode 100644 .agents/skills/align-recipe-pyproject/tests/conftest.py create mode 100644 .agents/skills/align-recipe-pyproject/tests/test_align_pyproject.py create mode 100644 .agents/skills/generate-python-runnability-test/tests/conftest.py create mode 100644 .agents/skills/generate-python-runnability-test/tests/test_generate_runnability_test.py diff --git a/.agents/skills/align-recipe-pyproject/SKILL.md b/.agents/skills/align-recipe-pyproject/SKILL.md index 44d9561375..61d5397d67 100644 --- a/.agents/skills/align-recipe-pyproject/SKILL.md +++ b/.agents/skills/align-recipe-pyproject/SKILL.md @@ -15,7 +15,7 @@ description: > metadata: author: Google license: Apache-2.0 - version: 1.0.0 + version: 1.1.0 --- # Align Recipe pyproject.toml @@ -28,7 +28,7 @@ Scope: **`pyproject.toml` only**. Standalone `ruff.toml` / `.ruff.toml` files ar ## What This Skill Checks -Runs `scripts/align_pyproject.py` against a recipe directory. Six rules: +Runs `scripts/align_pyproject.py` against a recipe directory. Eight rules: | Rule ID | What it checks | Auto-fix | |---|---|---| @@ -38,6 +38,28 @@ Runs `scripts/align_pyproject.py` against a recipe directory. Six rules: | `description-matches-manifest` | If `[project].description` is set, it must equal `manifest.description`. Field is optional; skipped when absent. | Only with `--description-source={pyproject,manifest,delete}`. Refuses to touch description otherwise. | | `build-system-present` | `[build-system]` must have both `requires` and `build-backend`. Without it, `uv build` and `pip install .` fail. | **No** — backend choice is editorial. Reported for the human to fix. | | `default-pypi-index` | `[[tool.uv.index]]` must have an entry with `default = true` pointing at public PyPI (`https://pypi.org/simple[/]`). Required so `uv sync` works on Google corp workstations without corp Airlock auth — see the block comment in the root `pyproject.toml` for the full rationale. | Yes when the block is entirely missing — appends it. **No** when a default entry exists but points elsewhere (custom private index, TestPyPI, mirror) — reported for the human to reconcile, since the divergence may be intentional. | +| `stale-python-version-refs` | Scans **every text file in the recipe** for references to a Python version below the 3.11 floor. Raising `requires-python` is not a self-contained edit: the version is typically repeated in prose (`README.md`, `SKILL.md`) and in executable setup code. | **No** — report-only. Which references are stale and how to reword them is editorial. | +| `runnability-test-in-testpaths` | If `[tool.pytest.ini_options].testpaths` is set, at least one entry must collect `tests/test_runnability.py` (which `.github/policy.yml` requires every Python recipe to have). | **No** — report-only. Broadening `testpaths` changes what CI collects. | + +#### Why `stale-python-version-refs` exists + +A bootstrap script that picks an interpreter from an allowlist still +containing the old floor will happily build a venv the recipe then refuses to +install into: + +```bash +for py in python3.13 python3.12 python3.11 python3.10 python3; do +# ... +# ERROR: Package requires a different Python: 3.10.x not in '>=3.11' +``` + +The scan requires a Python-ish context (`python 3.10`, `Python :: 3.10`, +`>=3.10`) rather than matching bare digits, so a model name like +`gemini-3.5-flash` is not a false positive. It skips lockfiles, virtualenvs, +caches, and binary files, and ignores `requires-python` in `pyproject.toml` +itself (owned by `python-version-floor`, and still pre-rewrite in memory when +this check runs). Hits are capped at 40 in `details.hits`, but +`details.files` always lists every affected file. ### Edit safety @@ -66,9 +88,13 @@ Runs `scripts/align_pyproject.py` against a recipe directory. Six rules: 7. **If `default-pypi-index` returns `report_only`**, the recipe declares a default index that is NOT public PyPI (e.g. a private mirror, TestPyPI). The skill will not overwrite an intentional choice. Show the user the current `url` from `details.current_url` and ask whether it's deliberate. If yes, they can `# noqa`-comment it or update the repo standard; if no, they should change the URL to `https://pypi.org/simple/`. Do not auto-rewrite. -8. **After apply mode succeeds**, remind the user to run `uv sync` in the recipe directory if the pyproject changes touched dependencies (the script emits this in `notes` when relevant). +8. **If `stale-python-version-refs` returns `report_only`**, list the affected files from `details.files` and call out any **executable** ones first (`.sh`, `.py`, `Makefile`, CI YAML) — a stale interpreter allowlist in a bootstrap script is a live bug, not a docs nit, and it produces a confusing `requires a different Python` failure at install time. Prose files (`README.md`, `SKILL.md`) are lower priority but still inconsistent. Offer to update them; do not rewrite without asking, since some references are legitimately historical ("dropped 3.10 support in v2"). + +9. **If `runnability-test-in-testpaths` returns `report_only`**, the recipe's `testpaths` excludes `tests/test_runnability.py`, so a bare `uv run pytest` never runs it — the recipe looks tested while its import-smoke test silently never executes. Show `details.testpaths` and suggest adding `"tests"`. Do not auto-rewrite. + +10. **After apply mode succeeds**, remind the user to run `uv sync` in the recipe directory if the pyproject changes touched dependencies (the script emits this in `notes` when relevant). -9. **Do not commit any changes.** Show the diff or file contents; let the user commit. +11. **Do not commit any changes.** Show the diff or file contents; let the user commit. --- @@ -168,7 +194,7 @@ Status-specific guidance for what to put in the **Details** cell: - **`would_fix`** (dry-run) — describe the current-state problem, then say what apply would do. Include the `from` → `to` or the list of tables to be removed. - **`fixed`** (apply) — one-liner confirming the change (new value or list of removed tables). - **`needs_input`** (only `description-matches-manifest`) — the Details cell says something like `"descriptions differ — needs --description-source={pyproject,manifest,delete}"`. Do not put the two long descriptions inside the table. See "Follow-up content" below. -- **`report_only`** (two rules can hit this: `build-system-present` and `default-pypi-index` when a non-PyPI default is declared) — the Details cell names what's missing or non-conforming (e.g. `"[build-system] missing; recipe cannot be built as a package"` or `"default index is TestPyPI, not public PyPI"`). Follow-up content goes below the table (see next section). +- **`report_only`** (four rules can hit this: `build-system-present`, `default-pypi-index` when a non-PyPI default is declared, `stale-python-version-refs`, and `runnability-test-in-testpaths`) — the Details cell names what's missing or non-conforming (e.g. `"[build-system] missing; recipe cannot be built as a package"` or `"default index is TestPyPI, not public PyPI"`). Follow-up content goes below the table (see next section). - **`error`** — the Details cell shows the message verbatim; if it's very long, truncate with `…` and put the full text below. ### Follow-up content below the table @@ -210,6 +236,8 @@ Only for statuses that need extra context. Order: table first, then this content - **Dry-run with only `report_only`** (and no `would_fix` rows) — after the table, address the specific case: - `build-system-present`: show the two `[build-system]` template snippets and stop. This is a manual edit; the skill does not auto-fix it. - `default-pypi-index`: quote `details.current_url`, explain that this is not public PyPI, and ask whether it's intentional. If not, tell the user to change the URL to `https://pypi.org/simple/`. Do not auto-rewrite. + - `stale-python-version-refs`: list `details.files`, executable files first, and offer to update them. + - `runnability-test-in-testpaths`: quote `details.testpaths` and suggest adding `"tests"`. - **Dry-run with only `error` rows** — do not offer to apply. Errors mean the script bailed before it could compute a fix; the user has to resolve the underlying issue first. diff --git a/.agents/skills/align-recipe-pyproject/scripts/align_pyproject.py b/.agents/skills/align-recipe-pyproject/scripts/align_pyproject.py index 343bcf265c..a7a311af44 100644 --- a/.agents/skills/align-recipe-pyproject/scripts/align_pyproject.py +++ b/.agents/skills/align-recipe-pyproject/scripts/align_pyproject.py @@ -79,6 +79,8 @@ import argparse import json +import os +import re import sys from dataclasses import asdict, dataclass, field from pathlib import Path @@ -170,9 +172,7 @@ def to_json(self) -> str: # ---------- no-local-ruff-config: no [tool.ruff*] tables ------------------- -def check_no_local_ruff_config( - doc: tomlkit.TOMLDocument, apply: bool -) -> Check: +def check_no_local_ruff_config(doc: tomlkit.TOMLDocument, apply: bool) -> Check: """Remove any [tool.ruff*] table from pyproject.toml. Assumes doc["tool"] (if present) is a table — run() validates that the @@ -321,9 +321,7 @@ def _validate_and_apply_python_floor_rewrite( ) -def check_python_version_floor( - doc: tomlkit.TOMLDocument, apply: bool -) -> Check: +def check_python_version_floor(doc: tomlkit.TOMLDocument, apply: bool) -> Check: """Ensure [project].requires-python is compatible with MIN_PYTHON. Interpretation B (aligned with CI in .github/workflows/ @@ -905,6 +903,224 @@ def check_default_pypi_index(doc: tomlkit.TOMLDocument, apply: bool) -> Check: ) +# ---------- stale-python-version-refs (report-only) ------------------------ +# +# Raising [project].requires-python is not a self-contained edit. A recipe +# typically repeats its supported Python version in prose (README, SKILL.md) +# and — far more importantly — in EXECUTABLE setup code. A bootstrap script +# that picks an interpreter from an allowlist still containing the old floor +# will happily build a venv the recipe then refuses to install into: +# +# for py in python3.13 python3.12 python3.11 python3.10 python3; do +# ... +# ERROR: Package requires a different Python: 3.10.x not in '>=3.11' +# +# This check is report-only: which of the hits matter, and how to reword +# them, is the maintainer's call. Its job is simply that the bump never lands +# silently. + +# Directories that are never the recipe's own source. +_SCAN_SKIP_DIRS = frozenset( + { + ".git", + ".ruff_cache", + ".mypy_cache", + ".pytest_cache", + "__pycache__", + ".venv", + "venv", + "env", + "build", + "dist", + "node_modules", + } +) + +# Files whose Python-version mentions are generated, not authored. +_SCAN_SKIP_NAMES = frozenset({"uv.lock", "poetry.lock", "Pipfile.lock"}) + +# Two contexts that make "3.x" a Python-version reference rather than a +# coincidence. Without the context requirement, a model name like +# "gemini-3.5-flash" would match and drown the report in false positives. +_PY_REF_PATTERNS = ( + # python3.10 / python 3.10 / Python 3.10 / python@3.10 / python_3.10, and + # the trove classifier "Programming Language :: Python :: 3.10" (hence + # ':' in the separator class and a width of 4). + re.compile(r"python[\s@._:-]{0,4}3\.(\d+)", re.IGNORECASE), + # >=3.10 / ==3.9 / ~=3.10 — version-specifier syntax + re.compile(r"[<>=~!]=\s*3\.(\d+)"), +) + +# Cap the reported hits so a pathological recipe can't produce a wall of JSON. +_MAX_REPORTED_REFS = 40 + + +def _iter_scannable_files(recipe_dir: Path) -> list[Path]: + """Every text-ish file in the recipe that could carry a version claim.""" + found: list[Path] = [] + for root, dirs, files in os.walk(recipe_dir): + dirs[:] = [ + d + for d in dirs + if d not in _SCAN_SKIP_DIRS and not d.endswith(".egg-info") + ] + for name in sorted(files): + if name in _SCAN_SKIP_NAMES: + continue + found.append(Path(root) / name) + return sorted(found) + + +# `requires-python` is owned by the python-version-floor check, which rewrites +# it in-memory and persists only at the end of run(). Scanning it here would +# report the pre-rewrite value as a stale reference in apply mode even though +# the file on disk ends up correct — a pure false positive either way, since +# the floor check already reports that key authoritatively. +_REQUIRES_PYTHON_LINE_RE = re.compile(r"^\s*requires-python\s*=") + + +def _sub_floor_refs_in_text( + text: str, skip_requires_python: bool = False +) -> list[tuple[int, str]]: + """Return (line_number, line) for lines claiming a sub-floor Python.""" + hits: list[tuple[int, str]] = [] + for lineno, line in enumerate(text.splitlines(), start=1): + if skip_requires_python and _REQUIRES_PYTHON_LINE_RE.match(line): + continue + for pattern in _PY_REF_PATTERNS: + if any( + int(minor) < MIN_PYTHON[1] + for minor in pattern.findall(line) + if minor.isdigit() + ): + hits.append((lineno, line.strip())) + break + return hits + + +def check_stale_python_version_refs(recipe_dir: Path) -> Check: + """Report references to a Python version below the enforced floor.""" + offenders: dict[str, list[dict[str, Any]]] = {} + total = 0 + for path in _iter_scannable_files(recipe_dir): + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue # binary or unreadable — not a prose/version claim + rel = str(path.relative_to(recipe_dir)) + hits = _sub_floor_refs_in_text( + text, skip_requires_python=(rel == "pyproject.toml") + ) + if not hits: + continue + offenders[rel] = [ + {"line": lineno, "text": line} for lineno, line in hits + ] + total += len(hits) + + if not offenders: + return Check( + "stale-python-version-refs", + OK, + f"No references to a Python version below {MIN_PYTHON_STR}.", + ) + + # Trim the payload without losing the file list. + trimmed = dict(offenders) + if total > _MAX_REPORTED_REFS: + budget = _MAX_REPORTED_REFS + trimmed = {} + for rel, hits in offenders.items(): + if budget <= 0: + break + trimmed[rel] = hits[:budget] + budget -= len(trimmed[rel]) + + return Check( + "stale-python-version-refs", + REPORT_ONLY, + f"{total} reference(s) to a Python version below {MIN_PYTHON_STR} " + f"across {len(offenders)} file(s): {', '.join(sorted(offenders))}. " + f"[project].requires-python is enforced at >={MIN_PYTHON_STR}, so " + f"these are now inconsistent. Executable ones matter most — an " + f"interpreter-picking loop in a bootstrap script that still accepts " + f"the old floor will build a venv the recipe then refuses to install " + f"into. This skill does not auto-fix: which references are stale and " + f"how to reword them is editorial.", + { + "floor": MIN_PYTHON_STR, + "total": total, + "files": sorted(offenders), + "hits": trimmed, + "truncated": total > _MAX_REPORTED_REFS, + }, + ) + + +# ---------- runnability-test-in-testpaths (report-only) -------------------- + + +def _testpaths_cover_runnability(entries: list[str]) -> bool: + """Whether any testpaths entry collects tests/test_runnability.py.""" + for raw in entries: + entry = str(raw).strip().rstrip("/") + if entry in {"", ".", "tests", "tests/test_runnability.py"}: + return True + return False + + +def check_runnability_test_in_testpaths(doc: tomlkit.TOMLDocument) -> Check: + """Report a `testpaths` setting that excludes the runnability test. + + `tests/test_runnability.py` is a required file for Python recipes + (.github/policy.yml), but a narrower `testpaths` — e.g. + `["tests/unit", "tests/integration"]` — means a bare `uv run pytest` + never collects it. The recipe then looks tested while its one + import-smoke test silently never runs. + """ + ini = ( + doc.get("tool", {}).get("pytest", {}).get("ini_options", {}) + if doc.get("tool") is not None + else {} + ) + entries = ini.get("testpaths") if hasattr(ini, "get") else None + if entries is None: + return Check( + "runnability-test-in-testpaths", + OK, + "No [tool.pytest.ini_options].testpaths — pytest collects the " + "whole recipe, including tests/test_runnability.py.", + ) + if isinstance(entries, str): + entries = [entries] + if not isinstance(entries, (list, tuple)): + return Check( + "runnability-test-in-testpaths", + OK, + "testpaths is not a list; nothing to check.", + ) + + listed = [str(e) for e in entries] + if _testpaths_cover_runnability(listed): + return Check( + "runnability-test-in-testpaths", + OK, + f"testpaths {listed} collects tests/test_runnability.py.", + ) + + return Check( + "runnability-test-in-testpaths", + REPORT_ONLY, + f"[tool.pytest.ini_options].testpaths is {listed}, none of which " + f"collects tests/test_runnability.py — the required runnability " + f'test would never run under a bare `pytest`. Add "tests" to ' + f"testpaths, or narrow it deliberately and run the file by path. " + f"Not auto-fixed: broadening testpaths changes what CI collects, " + f"which is the maintainer's call.", + {"testpaths": listed, "expected": "tests/test_runnability.py"}, + ) + + # ---------- Orchestration ------------------------------------------------- @@ -1081,9 +1297,7 @@ def run( report.add( _run_check( "project-name-matches-folder", - lambda: check_project_name_matches_folder( - recipe_dir, doc, apply - ), + lambda: check_project_name_matches_folder(recipe_dir, doc, apply), ) ) report.add( @@ -1107,6 +1321,18 @@ def run( lambda: check_default_pypi_index(doc, apply), ) ) + report.add( + _run_check( + "stale-python-version-refs", + lambda: check_stale_python_version_refs(recipe_dir), + ) + ) + report.add( + _run_check( + "runnability-test-in-testpaths", + lambda: check_runnability_test_in_testpaths(doc), + ) + ) # Persist edits — only in apply mode, and only if at least one auto-fix # actually changed something. diff --git a/.agents/skills/align-recipe-pyproject/tests/conftest.py b/.agents/skills/align-recipe-pyproject/tests/conftest.py new file mode 100644 index 0000000000..2d55a0f3c1 --- /dev/null +++ b/.agents/skills/align-recipe-pyproject/tests/conftest.py @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Make this skill's scripts/ directory importable from its tests. + +Keeping the path shim inside the skill (rather than in the repo-root pytest +config) preserves the skill as a self-contained, portable bundle. +""" + +import sys +from pathlib import Path + +SCRIPTS_DIR = Path(__file__).parent.parent / "scripts" +sys.path.insert(0, str(SCRIPTS_DIR)) diff --git a/.agents/skills/align-recipe-pyproject/tests/test_align_pyproject.py b/.agents/skills/align-recipe-pyproject/tests/test_align_pyproject.py new file mode 100644 index 0000000000..7b0c47a732 --- /dev/null +++ b/.agents/skills/align-recipe-pyproject/tests/test_align_pyproject.py @@ -0,0 +1,208 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the align-recipe-pyproject skill script. + +Covers the two checks added after a vertical skill under skills/ showed that +raising `requires-python` silently desynced the recipe's own bootstrap script, +and that a narrow `testpaths` can exclude the required runnability test. +""" + +from pathlib import Path + +import align_pyproject as m +import tomlkit + + +def _write(path: Path, content: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def _check(recipe_dir: Path) -> m.Check: + return m.check_stale_python_version_refs(recipe_dir) + + +# --------------------------------------------------------------------------- +# stale-python-version-refs +# --------------------------------------------------------------------------- + + +def test_clean_recipe_reports_ok(tmp_path): + _write(tmp_path / "README.md", "- Python 3.11+\n") + _write(tmp_path / "pyproject.toml", 'requires-python = ">=3.11"\n') + assert _check(tmp_path).status == m.OK + + +def test_flags_prose_version_claim(tmp_path): + _write(tmp_path / "README.md", "# R\n\n- Python 3.10+\n") + check = _check(tmp_path) + assert check.status == m.REPORT_ONLY + assert check.details["files"] == ["README.md"] + + +def test_flags_interpreter_allowlist_in_shell_script(tmp_path): + # The case that actually broke a recipe: bootstrap.sh would still build a + # 3.10 venv that the recipe then refuses to install into. + _write( + tmp_path / "scripts" / "bootstrap.sh", + "for py in python3.12 python3.11 python3.10 python3; do\n", + ) + check = _check(tmp_path) + assert check.status == m.REPORT_ONLY + assert check.details["files"] == ["scripts/bootstrap.sh"] + + +def test_flags_version_specifier_syntax(tmp_path): + _write(tmp_path / "TROUBLE.md", "not in '>=3.10'\n") + assert _check(tmp_path).status == m.REPORT_ONLY + + +def test_does_not_flag_model_names(tmp_path): + # "gemini-3.5-flash" contains "3.5" but is not a Python version claim. + # A context-free digit scan would drown the report in these. + _write( + tmp_path / "agent.py", + 'MODEL = "gemini-3.5-flash"\nOTHER = "gemini-2.0-pro"\n', + ) + assert _check(tmp_path).status == m.OK + + +def test_does_not_flag_versions_at_or_above_floor(tmp_path): + _write( + tmp_path / "bootstrap.sh", + "for py in python3.13 python3.12 python3.11; do\n", + ) + assert _check(tmp_path).status == m.OK + + +def test_ignores_requires_python_line_in_pyproject(tmp_path): + # Owned by the python-version-floor check, which rewrites it in memory + # and persists at the end of run(); reporting it here would be a false + # positive in apply mode. + _write(tmp_path / "pyproject.toml", 'requires-python = ">=3.10"\n') + assert _check(tmp_path).status == m.OK + + +def test_still_flags_other_pyproject_lines(tmp_path): + _write( + tmp_path / "pyproject.toml", + 'requires-python = ">=3.10"\n' + 'classifiers = ["Programming Language :: Python :: 3.10"]\n', + ) + check = _check(tmp_path) + assert check.status == m.REPORT_ONLY + assert check.details["hits"]["pyproject.toml"][0]["line"] == 2 + + +def test_skips_lockfiles_and_venvs(tmp_path): + _write(tmp_path / "uv.lock", 'requires-python = ">=3.10"\n') + _write(tmp_path / ".venv" / "x.py", "# python3.9\n") + _write(tmp_path / "__pycache__" / "y.txt", "python3.9\n") + assert _check(tmp_path).status == m.OK + + +def test_binary_file_does_not_crash_the_scan(tmp_path): + (tmp_path / "logo.png").write_bytes(b"\x89PNG\r\n\x1a\n\xff\xfe") + _write(tmp_path / "README.md", "- Python 3.10+\n") + check = _check(tmp_path) + assert check.status == m.REPORT_ONLY + assert check.details["files"] == ["README.md"] + + +def test_hits_are_truncated_but_file_list_is_not(tmp_path): + body = "\n".join(f"python3.10 line {i}" for i in range(60)) + _write(tmp_path / "many.txt", body) + check = _check(tmp_path) + assert check.details["truncated"] is True + assert check.details["total"] == 60 + reported = sum(len(v) for v in check.details["hits"].values()) + assert reported == m._MAX_REPORTED_REFS + assert check.details["files"] == ["many.txt"] + + +# --------------------------------------------------------------------------- +# runnability-test-in-testpaths +# --------------------------------------------------------------------------- + + +def _testpaths_check(toml: str) -> m.Check: + return m.check_runnability_test_in_testpaths(tomlkit.parse(toml)) + + +def test_no_testpaths_is_ok(): + assert _testpaths_check('[project]\nname = "x"\n').status == m.OK + + +def test_narrow_testpaths_is_reported(): + check = _testpaths_check( + "[tool.pytest.ini_options]\n" + 'testpaths = ["tests/unit", "tests/integration"]\n' + ) + assert check.status == m.REPORT_ONLY + assert check.details["testpaths"] == ["tests/unit", "tests/integration"] + + +def test_tests_dir_in_testpaths_is_ok(): + check = _testpaths_check( + '[tool.pytest.ini_options]\ntestpaths = ["tests", "docs"]\n' + ) + assert check.status == m.OK + + +def test_trailing_slash_is_normalised(): + check = _testpaths_check( + '[tool.pytest.ini_options]\ntestpaths = ["tests/"]\n' + ) + assert check.status == m.OK + + +def test_dot_testpath_is_ok(): + check = _testpaths_check('[tool.pytest.ini_options]\ntestpaths = ["."]\n') + assert check.status == m.OK + + +def test_explicit_runnability_path_is_ok(): + check = _testpaths_check( + '[tool.pytest.ini_options]\ntestpaths = ["tests/test_runnability.py"]\n' + ) + assert check.status == m.OK + + +def test_string_testpaths_is_handled(): + check = _testpaths_check( + '[tool.pytest.ini_options]\ntestpaths = "tests/unit"\n' + ) + assert check.status == m.REPORT_ONLY + + +# --------------------------------------------------------------------------- +# Both checks are wired into run() +# --------------------------------------------------------------------------- + + +def test_new_checks_appear_in_report(tmp_path): + _write( + tmp_path / "pyproject.toml", + '[project]\nname = "alrec"\ndescription = "d"\n' + 'requires-python = ">=3.11"\n' + '[tool.pytest.ini_options]\ntestpaths = ["tests/unit"]\n', + ) + _write(tmp_path / "README.md", "- Python 3.10+\n") + + report = m.run(tmp_path, dry_run=True, description_source=None) + by_id = {c.id: c for c in report.checks} + + assert by_id["stale-python-version-refs"].status == m.REPORT_ONLY + assert by_id["runnability-test-in-testpaths"].status == m.REPORT_ONLY diff --git a/.agents/skills/extract-python-environment-variables/SKILL.md b/.agents/skills/extract-python-environment-variables/SKILL.md index 9a9705cfd1..7471f9aaf3 100644 --- a/.agents/skills/extract-python-environment-variables/SKILL.md +++ b/.agents/skills/extract-python-environment-variables/SKILL.md @@ -13,10 +13,15 @@ description: > (`"my-project-id"`, `"changeme"`, `"<...>"`) are downgraded to the TODO placeholder but the source string is preserved in the marker comment. Also detects hardcoded model-name string literals (e.g. - `"gemini-3.5-flash"` in `agent.py`) and rewrites them to bare - `os.getenv("MODEL_NAME")` (single model) or - `os.getenv("MODEL_NAME_GENERATED_1")` / `MODEL_NAME_GENERATED_2`, … - (multiple models) — no fallback default in the Python source. The + `"gemini-3.5-flash"` in `agent.py`) and rewrites them to an + `os.getenv(...)` call. The variable name is derived from the assignment + target when it names a model (`DEFAULT_EMBEDDING_MODEL` → + `EMBEDDING_MODEL`), else `MODEL_NAME` (single model) or + `MODEL_NAME_GENERATED_1` / `MODEL_NAME_GENERATED_2`, … (multiple models). + Normally no fallback default is written into the Python source; the one + exception is when no `load_dotenv()` bootstrap could be installed (no + package `__init__.py`), where the original literal is kept as the + fallback so the lookup cannot evaluate to `None` at runtime. The model string is written as the value in `.env.example` with a comment prompting a rename. When re-run against a recipe whose `.env.example` already has entries, the writer classifies each entry (skill-authored vs. user-authored, @@ -46,7 +51,7 @@ description: > metadata: author: Google license: Apache-2.0 - version: 2.1.0 + version: 2.2.0 --- # Extract Python Environment Variables @@ -115,6 +120,12 @@ Runs `scripts/extract_env_vars.py` against a recipe directory. The script: If `load_dotenv` is already present the injection is skipped. + **If no package `__init__.py` exists** (common in vertical skills under + `skills/`, where the code lives in a plain `scripts/` directory rather + than an importable package) the injection is skipped with a `[WARN]`, + and the step reports that no bootstrap is in place. Step 4 below depends + on that answer. + **Additionally — always, regardless of whether we injected** — appends `# noqa: E402 -- must come after load_dotenv()` to any top-level relative import (`from .x import y`) that sits AFTER a non-import module-level @@ -137,13 +148,38 @@ Runs `scripts/extract_env_vars.py` against a recipe directory. The script: Idempotent: a line that already carries `# noqa: E402` is skipped. 4. **Replaces hardcoded model names** in source (e.g. `model="gemini-3.5-flash"` - in `agent.py`) with **bare `os.getenv(...)`** — no default argument: - - Single model → `os.getenv("MODEL_NAME")` - - Multiple models → `os.getenv("MODEL_NAME_GENERATED_1")`, `os.getenv("MODEL_NAME_GENERATED_2")`, … (sorted alphabetically for determinism) + in `agent.py`) with an `os.getenv(...)` call. + + **Variable name.** The assignment target that holds the literal is used + when it names a model, since it carries far more meaning than a generic + fallback — `DEFAULT_EMBEDDING_MODEL = "gemini-embedding-001"` and + `embedding_model = cfg.get("embedding_model", "gemini-embedding-001")` + both yield `EMBEDDING_MODEL`. This matters in recipes that already read a + *different* model var (say `GEMINI_MODEL` for the LLM), where a second + bare `MODEL_NAME` would be actively misleading. When a string is assigned + to conflicting target names, the most frequent wins, ties broken + alphabetically. Otherwise: + - A single unnamed model → `MODEL_NAME` + - The rest → `MODEL_NAME_GENERATED_1`, `MODEL_NAME_GENERATED_2`, … (sorted + alphabetically for determinism) + + **Fallback argument.** Normally the emitted call is **bare** — + `os.getenv("EMBEDDING_MODEL")`, no default — because default values are + the maintainer's decision, not the skill's. That is only safe when the + `load_dotenv()` bootstrap from step 3 is in place to populate the + environment. + + **When step 3 could not install a bootstrap** (no package `__init__.py`), + nothing reads `.env`, so a bare lookup would evaluate to `None` at runtime + and silently break the recipe. In that case the original literal is + preserved as the fallback — `os.getenv("EMBEDDING_MODEL", + "gemini-embedding-001")` — which keeps behaviour identical to before the + rewrite while still lifting the value into the environment. The step logs + an `[INFO]` line explaining the choice. The actual model string is written as the value in `.env.example` (e.g. - `MODEL_NAME_GENERATED_1=gemini-3.5-flash`) with a comment prompting the - maintainer to rename the variable to something meaningful before shipping. + `EMBEDDING_MODEL=gemini-embedding-001`) with a comment prompting the + maintainer to rename the variable if the derived name isn't right. 5. **Updates `pyproject.toml`** — adds `python-dotenv>=1.0.0` to `[project]` dependencies if it is not already there. @@ -195,7 +231,9 @@ are LEFT UNTOUCHED. The skill's only writes to Python files are: (once, only if not already present). - Appending `# noqa: E402` to trailing relative imports that would otherwise trip Ruff after the env-bootstrap block. -- Replacing hardcoded model literals with bare `os.getenv(...)` calls. +- Replacing hardcoded model literals with `os.getenv(...)` calls — bare + when a `load_dotenv()` bootstrap is in place, otherwise retaining the + original literal as the fallback (see step 4 above). Note that scanning `os.environ.setdefault(...)` and lifting its value into `.env.example` (v2) does NOT violate Rule 2 — the skill READS from diff --git a/.agents/skills/extract-python-environment-variables/scripts/extract_env_vars.py b/.agents/skills/extract-python-environment-variables/scripts/extract_env_vars.py index ce9f0346da..eacdb653d5 100644 --- a/.agents/skills/extract-python-environment-variables/scripts/extract_env_vars.py +++ b/.agents/skills/extract-python-environment-variables/scripts/extract_env_vars.py @@ -1099,31 +1099,62 @@ def write_model_vars_to_env_example( # --------------------------------------------------------------------------- +# Opening quotes of a module docstring, allowing any legal string prefix. +# Python permits r/u/b (and Rb/bR/... pairs) before the quote; a docstring in +# practice is r"""...""" or plain, but the scan must not be fooled by any of +# them. Historical bug: this was a bare startswith('"""') test, so a file whose +# docstring was written as r\"\"\"...\"\"\" (common when the text contains +# backslashes) had `import os` injected ABOVE the docstring, demoting it to a +# dead string expression and losing __doc__. +_DOCSTRING_OPEN_RE = re.compile(r'^[rRuUbB]{0,2}("""|\'\'\'|"|\')') + + def _post_header_index(lines: list[str]) -> int: """ Return the line index after which new top-level code should be inserted. Skips (in order): 1. Leading license / comment block and blank lines. - 2. An optional module-level docstring (single- or triple-quoted). + 2. An optional module-level docstring, whatever its quote style or + string prefix. This prevents imports from being injected before the module docstring, which would cause documentation tools to miss it. + + The docstring boundary is resolved with the AST wherever the source + parses, since that is the only way to be right about every quote style, + prefix, and implicit string concatenation. The textual scan below is a + fallback for sources that do not parse (the caller may be mid-edit). """ - i = 0 n = len(lines) - # Skip license header (comment lines and blank lines) + # AST path — authoritative when the source is syntactically valid. + try: + tree = ast.parse("".join(lines)) + except (SyntaxError, ValueError): + tree = None + if tree is not None and tree.body: + first = tree.body[0] + if ( + isinstance(first, ast.Expr) + and isinstance(first.value, ast.Constant) + and isinstance(first.value.value, str) + ): + # end_lineno is the 1-based last line of the docstring, which is + # exactly the 0-based index of the line after it. + return min(first.value.end_lineno or first.value.lineno, n) + + # Fallback: skip the license header (comment and blank lines)... + i = 0 while i < n and (lines[i].strip().startswith("#") or not lines[i].strip()): i += 1 - # Skip module docstring if present + # ...then an optional module docstring. if i < n: - stripped = lines[i].strip() - for quote in ('"""', "'''"): - if not stripped.startswith(quote): - continue - rest = stripped[len(quote) :] + match = _DOCSTRING_OPEN_RE.match(lines[i].strip()) + if match: + quote = match.group(1) + rest = lines[i].strip()[match.end() :] if rest.endswith(quote) and len(rest) >= len(quote): i += 1 # single-line docstring else: @@ -1131,9 +1162,8 @@ def _post_header_index(lines: list[str]) -> int: while i < n and quote not in lines[i]: i += 1 i += 1 # include the line that contains the closing quotes - break - return i + return min(i, n) def _docstring_node_ids(tree: ast.AST) -> set[int]: @@ -1254,7 +1284,7 @@ def _imports_os(tree: ast.AST) -> bool: _NOQA_E402_SUFFIX = " # noqa: E402 -- must come after load_dotenv()" -def _suppress_e402_on_late_relative_imports( # noqa: C901 +def _suppress_e402_on_late_relative_imports( tree: ast.Module | None, lines: list[str] ) -> tuple[list[str], int]: """Append `# noqa: E402` to top-level `from .x import y` statements that @@ -1579,6 +1609,74 @@ def _find_dependencies_close_bracket(content: str) -> int | None: return _scan_matching_close_bracket(content, deps_start.end() - 1) +def _split_trailing_comment(line: str) -> tuple[str, str]: + """ + Split a single TOML line into ``(code, comment)``. + + A ``#`` only opens a comment when it sits outside a string, so the scan + tracks basic strings (``"..."``, backslash escapes honoured) and literal + strings (``'...'``, no escapes). ``comment`` starts at the ``#`` and is + ``""`` when the line has none; any whitespace between the code and the + ``#`` stays on the ``code`` side so callers can preserve the layout. + """ + quote: str | None = None + escaped = False + for idx, ch in enumerate(line): + if quote is not None: + if escaped: + escaped = False + elif quote == '"' and ch == "\\": + escaped = True + elif ch == quote: + quote = None + continue + if ch in "\"'": + quote = ch + elif ch == "#": + return line[:idx], line[idx:] + return line, "" + + +def _array_body_has_entries(array_body: str) -> bool: + """Whether an array body holds a real entry (not just blanks/comments).""" + code_only = "".join( + _split_trailing_comment(ln)[0] for ln in array_body.split("\n") + ) + return bool(code_only.strip()) + + +def _ensure_trailing_comma(trimmed: str) -> str: + """ + Append a ``,`` to the last array entry in ``trimmed`` if it lacks one. + + The comma must land on the ENTRY, never inside a trailing comment. + Historical bug: a plain ``trimmed.endswith(",")`` test inspected the end + of the *comment* text, so an array ending in:: + + "pyOpenSSL>=23.0", # mTLS during auth in some envs + + grew a stray comma inside the comment, and one ending in:: + + "pyOpenSSL>=23.0" # no comma + + produced invalid TOML — the separating comma was commented out, which + the round-trip guard in :func:`ensure_python_dotenv_dependency` then + rejected, silently skipping the dependency insertion altogether. + """ + lines = trimmed.split("\n") + for i in range(len(lines) - 1, -1, -1): + code, comment = _split_trailing_comment(lines[i]) + if not code.strip(): + continue # blank or comment-only line — keep looking backwards + stripped = code.rstrip() + if stripped.endswith(",") or stripped.endswith("["): + return trimmed # already terminated, or the array is empty + gap = code[len(stripped) :] + lines[i] = stripped + "," + gap + comment + return "\n".join(lines) + return trimmed + + def _insert_before_close(content: str, close_idx: int) -> str: """ Insert a `"python-dotenv>=1.0.0",` line into the dependencies array @@ -1596,12 +1694,12 @@ def _insert_before_close(content: str, close_idx: int) -> str: # the layout of the insertion. trimmed = prefix.rstrip() - # If the array has any content, make sure the last entry has a trailing - # comma before we append our own. + # If the array holds any real entry, make sure the last one has a + # trailing comma before we append our own. array_open = trimmed.rfind("[") array_body = trimmed[array_open + 1 :] if array_open >= 0 else "" - if array_body.strip() and not trimmed.endswith(","): - trimmed += "," + if _array_body_has_entries(array_body): + trimmed = _ensure_trailing_comma(trimmed) return trimmed + '\n "python-dotenv>=1.0.0",\n' + suffix @@ -1729,16 +1827,122 @@ def extract_hardcoded_models( return hits +def _assignment_target_name(node: ast.AST) -> str | None: + """Return the single ``Name`` target of an assignment, else None.""" + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + ): + return node.targets[0].id + if ( + isinstance(node, ast.AnnAssign) + and node.value is not None + and isinstance(node.target, ast.Name) + ): + return node.target.id + return None + + +def _var_name_from_target(target: str) -> str | None: + """ + Derive an env-var name from the assignment target holding a model string. + + ``DEFAULT_EMBEDDING_MODEL = "gemini-embedding-001"`` tells us far more + about the variable's role than the generic ``MODEL_NAME`` fallback does — + especially in a recipe that already reads a *different* model var (an LLM) + from the environment, where a second bare ``MODEL_NAME`` is actively + misleading. + + Only targets that name a model are used; anything else returns None so the + caller falls back to the generic scheme. The leading ``DEFAULT_`` is + dropped because it describes the constant, not the model. + """ + name = target.strip().lstrip("_").upper() + if name.startswith("DEFAULT_"): + name = name[len("DEFAULT_") :] + if not name.isidentifier(): + return None + if "MODEL" not in name: + return None + if name == "MODEL": + return "MODEL_NAME" + return name + + +def extract_model_var_hints(py_files: list[Path]) -> dict[str, str]: + """ + Suggest an env-var name for each hardcoded model string, from the + assignment target that holds it. + + Both ``DEFAULT_EMBEDDING_MODEL = "gemini-embedding-001"`` and + ``embedding_model = cfg.get("embedding_model", "gemini-embedding-001")`` + yield the hint ``EMBEDDING_MODEL``. The same exclusions as + :func:`extract_hardcoded_models` apply, so a model name mentioned in a + docstring or already serving as a getenv default never votes. + + When a model string is assigned to differently-named targets across the + recipe, the most frequent target wins, ties broken alphabetically, so the + result is deterministic. + + Returns: + {model_string: suggested_var_name} — only for strings with a usable hint. + """ + votes: dict[str, dict[str, int]] = {} + + for py_file in py_files: + try: + source = py_file.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(py_file)) + except (SyntaxError, UnicodeDecodeError): + continue + + excluded_ids = _docstring_node_ids(tree) | _getenv_default_node_ids( + tree + ) + + for node in ast.walk(tree): + target = _assignment_target_name(node) + if target is None: + continue + derived = _var_name_from_target(target) + if derived is None: + continue + for child in ast.walk(node.value): + if not isinstance(child, ast.Constant): + continue + if id(child) in excluded_ids: + continue + if not isinstance(child.value, str): + continue + if any( + child.value.startswith(prefix) for prefix in MODEL_PREFIXES + ): + tally = votes.setdefault(child.value, {}) + tally[derived] = tally.get(derived, 0) + 1 + + return { + model_str: sorted(tally.items(), key=lambda kv: (-kv[1], kv[0]))[0][0] + for model_str, tally in votes.items() + } + + def assign_model_var_names( model_strings: set[str], existing_vars: set[str] | None = None, + hints: dict[str, str] | None = None, ) -> dict[str, str]: """ - Assign a standardised MODEL_NAME_* env var name to each unique model string. + Assign a standardised env var name to each unique model string. Rules (applied to the sorted list for determinism): - - If there is only one model → MODEL_NAME (no suffix), unless MODEL_NAME - is already taken, in which case the counter scheme below is used. + - If ``hints`` supplies a name derived from the assignment target (see + :func:`extract_model_var_hints`) and that name is free, use it. A + recipe that writes ``DEFAULT_EMBEDDING_MODEL`` gets ``EMBEDDING_MODEL`` + rather than a generic ``MODEL_NAME`` that collides conceptually with + an LLM model var it may already read. + - Otherwise, if exactly one model is left unnamed → MODEL_NAME (no + suffix), unless MODEL_NAME is already taken. - Otherwise → MODEL_NAME_GENERATED_1, MODEL_NAME_GENERATED_2, … skipping any index whose name is already present in existing_vars (e.g. from a prior run or a manually added entry in .env.example). @@ -1753,21 +1957,39 @@ def assign_model_var_names( model_strings: the set of unique hardcoded model strings found in source. existing_vars: names already declared in .env.example (or anywhere else that should be treated as taken). Defaults to empty set. + hints: optional {model_string: suggested_name} from the assignment + targets in source. Returns: {model_string: env_var_name} """ taken = set(existing_vars) if existing_vars else set() + hints = hints or {} sorted_strings = sorted(model_strings) - if len(sorted_strings) == 1: - if "MODEL_NAME" not in taken: - return {sorted_strings[0]: "MODEL_NAME"} - # Fall through to the counter scheme if MODEL_NAME is already taken. - mapping: dict[str, str] = {} - counter = 1 + unresolved: list[str] = [] + + # Pass 1 — honour a hint derived from the assignment target. for model_str in sorted_strings: + hint = hints.get(model_str) + if hint and hint not in taken: + mapping[model_str] = hint + taken.add(hint) + else: + unresolved.append(model_str) + + if not unresolved: + return mapping + + # Pass 2 — a lone unnamed model gets the unsuffixed MODEL_NAME. + if len(unresolved) == 1 and "MODEL_NAME" not in taken: + mapping[unresolved[0]] = "MODEL_NAME" + return mapping + + # Pass 3 — counter scheme for everything still unnamed. + counter = 1 + for model_str in unresolved: while f"MODEL_NAME_GENERATED_{counter}" in taken: counter += 1 var_name = f"MODEL_NAME_GENERATED_{counter}" @@ -1778,11 +2000,23 @@ def assign_model_var_names( return mapping +def _py_string_literal(value: str) -> str: + """Render ``value`` as a one-line, double-quoted Python string literal.""" + body = ( + value.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + ) + return f'"{body}"' + + def _model_replacement( node: ast.AST, excluded_ids: set[int], name_map: dict[str, str], lines: list[str], + keep_defaults: bool = False, ) -> tuple[int, int, str, str, str] | None: """ Return (start, end, new_text, model_str, var_name) if node is a @@ -1792,6 +2026,22 @@ def _model_replacement( are already serving as ``os.getenv``/``environ.get``/``setdefault`` default arguments. The latter must NOT be replaced — see :func:`_getenv_default_node_ids` for the correctness rationale. + + ``keep_defaults`` controls the shape of the emitted call: + + * False (the default) → BARE ``os.getenv("VAR")``. This is the skill's + normal rule: we do NOT write inferred defaults into Python source. + Default values are the maintainer's decision; the skill's job is to + lift the constant out of the source, not to also decide what fallback + the maintainer wants. It is only SAFE because a ``load_dotenv()`` + bootstrap guarantees ``.env`` has been read by the time the call runs. + + * True → ``os.getenv("VAR", "")``. Used when no + bootstrap could be installed (see :func:`run_step_load_dotenv`), so + nothing loads ``.env`` and a bare lookup would evaluate to ``None`` + at runtime — silently breaking the recipe. Preserving the literal as + the fallback keeps behaviour identical to before the rewrite while + still lifting the value into the environment. """ if not isinstance(node, ast.Constant): return None @@ -1804,18 +2054,18 @@ def _model_replacement( return None start = _flat_offset(lines, node.lineno, node.col_offset) end = _flat_offset(lines, node.end_lineno, node.end_col_offset) - # Emit BARE `os.getenv("VAR")` — no default argument. This is a hard - # rule of the skill: we do NOT write inferred defaults into Python - # source, even though `os.getenv("MODEL_NAME", node.value)` would be - # trivially "safer" (the recipe would keep working after the rewrite - # even without .env set up). Default values are the maintainer's - # decision; the skill's job is to lift the constant out of the source, - # not to also decide what fallback the maintainer wants. - return start, end, f'os.getenv("{var_name}")', node.value, var_name + if keep_defaults: + new_text = f'os.getenv("{var_name}", {_py_string_literal(node.value)})' + else: + new_text = f'os.getenv("{var_name}")' + return start, end, new_text, node.value, var_name def _collect_model_replacements( - tree: ast.AST, source: str, name_map: dict[str, str] + tree: ast.AST, + source: str, + name_map: dict[str, str], + keep_defaults: bool = False, ) -> tuple[list[tuple[int, int, str]], dict[str, str]]: """Walk ``tree`` and collect model-string replacement plans. @@ -1838,7 +2088,9 @@ def _collect_model_replacements( replacements: list[tuple[int, int, str]] = [] file_substituted: dict[str, str] = {} for node in ast.walk(tree): - replacement = _model_replacement(node, excluded_ids, name_map, lines) + replacement = _model_replacement( + node, excluded_ids, name_map, lines, keep_defaults=keep_defaults + ) if replacement is None: continue start, end, new_text, model_str, var_name = replacement @@ -1883,12 +2135,16 @@ def replace_hardcoded_models( hits: dict[Path, list[tuple[int, str]]], name_map: dict[str, str], dry_run: bool = False, + keep_defaults: bool = False, ) -> dict[str, str]: """ Replace each hardcoded model string with the correct - os.getenv("MODEL_NAME_*") call in-place, using the mapping produced by + os.getenv(...) call in-place, using the mapping produced by assign_model_var_names(). + ``keep_defaults`` preserves the original literal as the ``os.getenv`` + fallback — see :func:`_model_replacement` for when and why. + Replacement is AST-position-based, which means: - All quote styles (single, double, triple, raw) are handled correctly because the AST abstracts away quoting entirely. @@ -1912,7 +2168,7 @@ def replace_hardcoded_models( continue replacements, file_substituted = _collect_model_replacements( - tree, source, name_map + tree, source, name_map, keep_defaults=keep_defaults ) if not replacements: continue @@ -2040,18 +2296,24 @@ def run_step_env_vars( return env_example -def run_step_load_dotenv(recipe_dir: Path, dry_run: bool = False) -> None: +def run_step_load_dotenv(recipe_dir: Path, dry_run: bool = False) -> bool: """Step 4: inject load_dotenv() bootstrap into the package __init__.py, and suppress Ruff E402 on any trailing relative imports that come after non-import statements (whether we just injected them or the author had - already written a bootstrap by hand).""" + already written a bootstrap by hand). + + Returns whether a ``load_dotenv()`` bootstrap is in place afterwards. + Step 6 needs this: without a bootstrap nothing reads ``.env``, so the + bare ``os.getenv("VAR")`` calls it would otherwise emit evaluate to + ``None`` at runtime. See :func:`_model_replacement`. + """ init_py = find_package_init(recipe_dir) if not init_py: print( "[WARN] No Python package (subdirectory with __init__.py) found. " "load_dotenv() injection skipped." ) - return + return False rel = init_py.relative_to(recipe_dir) injected, noqa_added = inject_load_dotenv(init_py, dry_run=dry_run) if injected: @@ -2068,6 +2330,7 @@ def run_step_load_dotenv(recipe_dir: Path, dry_run: bool = False) -> None: "E402 in Phase 4 (ordering is intentional: env must be " "populated before importing agent submodules)." ) + return True def run_step_pyproject(recipe_dir: Path, dry_run: bool = False) -> None: @@ -2105,8 +2368,15 @@ def run_step_model_names( py_files: list[Path], env_example: Path, dry_run: bool = False, + dotenv_active: bool = True, ) -> None: - """Step 6: detect hardcoded model strings, replace with os.getenv().""" + """Step 6: detect hardcoded model strings, replace with os.getenv(). + + ``dotenv_active`` comes from :func:`run_step_load_dotenv`. When it is + False nothing in the recipe reads ``.env``, so the original literal is + preserved as the ``os.getenv`` fallback rather than emitting a bare + lookup that would evaluate to ``None``. + """ model_hits = extract_hardcoded_models(py_files) if not model_hits: print("\n[PASS] No hardcoded model names detected.") @@ -2118,7 +2388,8 @@ def run_step_model_names( for _lineno, model_str in file_hits } existing_vars = read_defined_vars(env_example) - name_map = assign_model_var_names(all_model_strings, existing_vars) + hints = extract_model_var_hints(py_files) + name_map = assign_model_var_names(all_model_strings, existing_vars, hints) print("\n[INFO] Detected hardcoded model name(s):") for py_file, file_hits in model_hits.items(): @@ -2128,8 +2399,21 @@ def run_step_model_names( f' — "{model_str}" → {name_map[model_str]}' ) + keep_defaults = not dotenv_active + if keep_defaults: + print( + "[INFO] No load_dotenv() bootstrap is in place, so nothing would " + "read .env at runtime. Keeping the original literal as the " + "os.getenv() fallback instead of emitting a bare lookup that " + "would evaluate to None." + ) + substituted = replace_hardcoded_models( - py_files, model_hits, name_map, dry_run=dry_run + py_files, + model_hits, + name_map, + dry_run=dry_run, + keep_defaults=keep_defaults, ) if not substituted: return @@ -2145,9 +2429,14 @@ def run_step_model_names( replace_verb = "Would replace" if dry_run else "Replaced" for model_str, var_name in substituted.items(): + call = ( + f'os.getenv("{var_name}", "{model_str}")' + if keep_defaults + else f'os.getenv("{var_name}")' + ) print( f'[{_tag(dry_run)}] {replace_verb} hardcoded "{model_str}" with' - f' os.getenv("{var_name}") in source.' + f" {call} in source." ) if added_models: add_verb = "Would add" if dry_run else "Added" @@ -2206,9 +2495,15 @@ def main() -> None: print(f" {f.relative_to(recipe_dir)}") env_example = run_step_env_vars(recipe_dir, py_files, dry_run=dry_run) - run_step_load_dotenv(recipe_dir, dry_run=dry_run) + dotenv_active = run_step_load_dotenv(recipe_dir, dry_run=dry_run) run_step_pyproject(recipe_dir, dry_run=dry_run) - run_step_model_names(recipe_dir, py_files, env_example, dry_run=dry_run) + run_step_model_names( + recipe_dir, + py_files, + env_example, + dry_run=dry_run, + dotenv_active=dotenv_active, + ) print(f"\n{'=' * 50}") if dry_run: diff --git a/.agents/skills/extract-python-environment-variables/tests/test_extract_env_vars.py b/.agents/skills/extract-python-environment-variables/tests/test_extract_env_vars.py index 19c1b2908e..99bdb91977 100644 --- a/.agents/skills/extract-python-environment-variables/tests/test_extract_env_vars.py +++ b/.agents/skills/extract-python-environment-variables/tests/test_extract_env_vars.py @@ -2798,3 +2798,314 @@ def test_invariant_appended_entries_are_skill_owned(tmp_path): assert ( entries["DOWNGRADED"].classification == m.EntryClassification.SKILL_TODO ) + + +# --------------------------------------------------------------------------- +# _post_header_index — module-docstring detection +# +# Regression: the docstring scan was a bare startswith('"""') test, so any +# legal string prefix (r"""..., u'''...) slipped past it and `import os` was +# injected ABOVE the docstring. That demotes the docstring to a dead string +# expression and loses __doc__. +# --------------------------------------------------------------------------- + + +def _insert_point(source: str) -> str: + """Return the line `import os` would be inserted before.""" + lines = source.splitlines(keepends=True) + idx = m._post_header_index(lines) + return lines[idx] if idx < len(lines) else "" + + +def test_post_header_index_skips_plain_docstring(): + src = '# license\n\n"""Doc.\n\nMore.\n"""\n\nimport sys\n' + assert _insert_point(src) == "\n" + + +def test_post_header_index_skips_raw_prefixed_docstring(): + src = '# license\n\nr"""Doc.\n\nMore.\n"""\n\nimport sys\n' + assert _insert_point(src) == "\n" + + +def test_post_header_index_skips_unicode_prefixed_docstring(): + src = '# license\n\nu"""Doc.\n"""\n\nimport sys\n' + assert _insert_point(src) == "\n" + + +def test_post_header_index_skips_raw_single_quoted_docstring(): + src = "# license\n\nr'''Doc.\nMore.\n'''\n\nimport sys\n" + assert _insert_point(src) == "\n" + + +def test_post_header_index_skips_one_line_raw_docstring(): + src = '# license\n\nr"""Doc."""\n\nimport sys\n' + assert _insert_point(src) == "\n" + + +def test_post_header_index_handles_shebang_and_raw_docstring(): + src = '#!/usr/bin/env python3\n# license\n\nr"""Doc.\n"""\n\nimport sys\n' + assert _insert_point(src) == "\n" + + +def test_post_header_index_without_docstring_lands_after_comments(): + src = "# license\n\nimport sys\n" + assert _insert_point(src) == "import sys\n" + + +def test_post_header_index_unparseable_source_falls_back_to_scan(): + # The AST path is unavailable, but the textual fallback must still + # respect a prefixed docstring. + src = '# license\n\nr"""Doc.\n"""\n\nimport sys\ndef (\n' + assert _insert_point(src) == "\n" + + +def test_post_header_index_empty_file(): + assert m._post_header_index([]) == 0 + + +def test_os_import_injected_below_raw_docstring(tmp_path): + # End-to-end: the module docstring must remain the first statement, so + # ast.get_docstring() still finds it after the rewrite. + src = ( + "# Copyright\n" + "\n" + 'r"""Module docstring with a \\d regex mention."""\n' + "\n" + 'DEFAULT_MODEL = "gemini-embedding-001"\n' + ) + py = _write(tmp_path / "ingest.py", src) + + hits = m.extract_hardcoded_models([py]) + name_map = m.assign_model_var_names({"gemini-embedding-001"}) + m.replace_hardcoded_models([py], hits, name_map) + + result = py.read_text(encoding="utf-8") + tree = ast.parse(result) + assert ast.get_docstring(tree) is not None, ( + f"docstring lost — import injected above it:\n{result}" + ) + assert "import os" in result + + +# --------------------------------------------------------------------------- +# _insert_before_close — trailing comma must land on the entry, not a comment +# --------------------------------------------------------------------------- + + +def _deps_after_dotenv(body: str): + """Add python-dotenv to a dependencies array and parse the result.""" + src = '[project]\nname = "x"\n' + body + "\n" + out = m._compute_pyproject_with_dotenv(src) + assert out is not None + return out, m.tomllib.loads(out)["project"]["dependencies"] + + +def test_insert_dotenv_preserves_comment_when_entry_has_comma(): + out, deps = _deps_after_dotenv( + 'dependencies = [\n "requests>=2.28",\n' + ' "pyOpenSSL>=23.0", # mTLS during auth\n]' + ) + assert deps == ["requests>=2.28", "pyOpenSSL>=23.0", "python-dotenv>=1.0.0"] + # The comment must be untouched — no comma appended inside it. + assert "# mTLS during auth\n" in out + assert "# mTLS during auth," not in out + + +def test_insert_dotenv_adds_comma_before_trailing_comment(): + # Previously produced invalid TOML: the separating comma was written + # inside the comment, so the array never closed. + out, deps = _deps_after_dotenv( + 'dependencies = [\n "requests>=2.28",\n' + ' "pyOpenSSL>=23.0" # no comma here\n]' + ) + assert deps == ["requests>=2.28", "pyOpenSSL>=23.0", "python-dotenv>=1.0.0"] + assert '"pyOpenSSL>=23.0", # no comma here' in out + + +def test_insert_dotenv_with_comment_only_last_line(): + _out, deps = _deps_after_dotenv( + 'dependencies = [\n "requests>=2.28",\n # a note\n]' + ) + assert deps == ["requests>=2.28", "python-dotenv>=1.0.0"] + + +def test_insert_dotenv_into_empty_array(): + _out, deps = _deps_after_dotenv("dependencies = [\n]") + assert deps == ["python-dotenv>=1.0.0"] + + +def test_insert_dotenv_into_empty_array_with_comment(): + _out, deps = _deps_after_dotenv("dependencies = [\n # nothing yet\n]") + assert deps == ["python-dotenv>=1.0.0"] + + +def test_insert_dotenv_into_single_line_array(): + _out, deps = _deps_after_dotenv('dependencies = ["a", "b"]') + assert deps == ["a", "b", "python-dotenv>=1.0.0"] + + +def test_insert_dotenv_ignores_hash_inside_string(): + # A '#' inside a PEP 508 URL is not a comment marker. + _out, deps = _deps_after_dotenv( + 'dependencies = [\n "pkg @ https://x/y#egg=pkg"\n]' + ) + assert deps == ["pkg @ https://x/y#egg=pkg", "python-dotenv>=1.0.0"] + + +def test_split_trailing_comment_respects_quotes(): + assert m._split_trailing_comment('"a#b" # real') == ( + '"a#b" ', + "# real", + ) + assert m._split_trailing_comment("'a#b'") == ("'a#b'", "") + assert m._split_trailing_comment('"esc\\"#q" # c') == ( + '"esc\\"#q" ', + "# c", + ) + + +# --------------------------------------------------------------------------- +# Model rewrite: bare getenv only when a load_dotenv bootstrap exists +# --------------------------------------------------------------------------- + + +def test_model_rewrite_keeps_default_when_no_dotenv_bootstrap(tmp_path): + py = _write( + tmp_path / "ingest.py", + 'DEFAULT_MODEL = "gemini-embedding-001"\n', + ) + hits = m.extract_hardcoded_models([py]) + name_map = {"gemini-embedding-001": "EMBEDDING_MODEL"} + + m.replace_hardcoded_models([py], hits, name_map, keep_defaults=True) + + result = py.read_text(encoding="utf-8") + assert 'os.getenv("EMBEDDING_MODEL", "gemini-embedding-001")' in result + # Still valid Python, and the value can never be None. + ast.parse(result) + + +def test_model_rewrite_bare_getenv_by_default(tmp_path): + py = _write( + tmp_path / "agent.py", + 'MODEL = "gemini-3.5-flash"\n', + ) + hits = m.extract_hardcoded_models([py]) + name_map = {"gemini-3.5-flash": "MODEL_NAME"} + + m.replace_hardcoded_models([py], hits, name_map) + + result = py.read_text(encoding="utf-8") + assert 'os.getenv("MODEL_NAME")' in result + assert "gemini-3.5-flash" not in result + + +def test_run_step_load_dotenv_returns_false_without_package(tmp_path, capsys): + assert m.run_step_load_dotenv(tmp_path) is False + assert "No Python package" in capsys.readouterr().out + + +def test_run_step_load_dotenv_returns_true_with_package(tmp_path): + _write(tmp_path / "app" / "__init__.py", "") + _write(tmp_path / "app" / "agent.py", "") + assert m.run_step_load_dotenv(tmp_path) is True + + +def test_py_string_literal_escapes(): + assert m._py_string_literal("plain") == '"plain"' + assert m._py_string_literal('has"quote') == '"has\\"quote"' + assert m._py_string_literal("back\\slash") == '"back\\\\slash"' + assert m._py_string_literal("nl\n") == '"nl\\n"' + + +# --------------------------------------------------------------------------- +# Model var naming derived from the assignment target +# --------------------------------------------------------------------------- + + +def test_model_var_hint_from_default_prefixed_constant(tmp_path): + py = _write( + tmp_path / "ingest.py", + 'DEFAULT_EMBEDDING_MODEL = "gemini-embedding-001"\n', + ) + assert m.extract_model_var_hints([py]) == { + "gemini-embedding-001": "EMBEDDING_MODEL" + } + + +def test_model_var_hint_from_nested_call_value(tmp_path): + py = _write( + tmp_path / "setup.py", + 'embedding_model = cfg.get("embedding_model", "gemini-embedding-001")\n', + ) + assert m.extract_model_var_hints([py]) == { + "gemini-embedding-001": "EMBEDDING_MODEL" + } + + +def test_model_var_hint_ignores_target_without_model_in_name(tmp_path): + py = _write(tmp_path / "agent.py", 'LLM = "gemini-3.5-flash"\n') + assert m.extract_model_var_hints([py]) == {} + + +def test_model_var_hint_bare_model_target_maps_to_model_name(tmp_path): + py = _write(tmp_path / "agent.py", 'MODEL = "gemini-3.5-flash"\n') + assert m.extract_model_var_hints([py]) == {"gemini-3.5-flash": "MODEL_NAME"} + + +def test_model_var_hint_ignores_docstring_mention(tmp_path): + py = _write( + tmp_path / "agent.py", + '"""Uses gemini-3.5-flash by default."""\n', + ) + assert m.extract_model_var_hints([py]) == {} + + +def test_model_var_hint_deterministic_across_conflicting_targets(tmp_path): + a = _write(tmp_path / "a.py", 'EMBEDDING_MODEL = "gemini-embedding-001"\n') + b = _write(tmp_path / "b.py", 'VECTOR_MODEL = "gemini-embedding-001"\n') + c = _write(tmp_path / "c.py", 'VECTOR_MODEL = "gemini-embedding-001"\n') + # VECTOR_MODEL has two votes to EMBEDDING_MODEL's one. + assert m.extract_model_var_hints([a, b, c]) == { + "gemini-embedding-001": "VECTOR_MODEL" + } + + +def test_assign_model_var_names_uses_hint(): + mapping = m.assign_model_var_names( + {"gemini-embedding-001"}, + existing_vars={"GEMINI_MODEL"}, + hints={"gemini-embedding-001": "EMBEDDING_MODEL"}, + ) + assert mapping == {"gemini-embedding-001": "EMBEDDING_MODEL"} + + +def test_assign_model_var_names_falls_back_when_hint_taken(): + mapping = m.assign_model_var_names( + {"gemini-embedding-001"}, + existing_vars={"EMBEDDING_MODEL"}, + hints={"gemini-embedding-001": "EMBEDDING_MODEL"}, + ) + assert mapping == {"gemini-embedding-001": "MODEL_NAME"} + + +def test_assign_model_var_names_mixed_hinted_and_unhinted(): + mapping = m.assign_model_var_names( + {"gemini-embedding-001", "gemini-3.5-flash"}, + hints={"gemini-embedding-001": "EMBEDDING_MODEL"}, + ) + assert mapping["gemini-embedding-001"] == "EMBEDDING_MODEL" + # The lone remaining model still gets the unsuffixed generic name. + assert mapping["gemini-3.5-flash"] == "MODEL_NAME" + + +def test_assign_model_var_names_unchanged_without_hints(): + # Backwards compatibility with the pre-hint behaviour. + assert m.assign_model_var_names({"gemini-3.5-flash"}) == { + "gemini-3.5-flash": "MODEL_NAME" + } + two = m.assign_model_var_names({"gemini-a", "gemini-b"}) + assert set(two.values()) == { + "MODEL_NAME_GENERATED_1", + "MODEL_NAME_GENERATED_2", + } diff --git a/.agents/skills/generate-python-runnability-test/SKILL.md b/.agents/skills/generate-python-runnability-test/SKILL.md index c7cd88e1f0..094f26d09f 100644 --- a/.agents/skills/generate-python-runnability-test/SKILL.md +++ b/.agents/skills/generate-python-runnability-test/SKILL.md @@ -15,7 +15,7 @@ description: > metadata: author: Google license: Apache-2.0 - version: 1.0.0 + version: 1.1.0 --- # Generate Python Runnability Test @@ -45,12 +45,28 @@ Runs `scripts/generate_runnability_test.py` against a recipe directory. Steps: Emission is post-processed through `ruff format` when available, so multi-patch `with (...):` blocks come out already wrapped per the repo's ruff config. -5. **Write it** to `/tests/test_runnability.py` (creating `tests/` if needed). Refuses to clobber an existing file unless `--overwrite` is passed. +5. **Check that the import can actually resolve.** The generated test does `import `, which only works if the RECIPE ROOT is on `sys.path`. That is not automatic — under pytest's default `prepend` import mode only the test file's own directory (`/tests`) is inserted. The recipe root gets there in one of these ways, reported as `import_support`: + + | `import_support` | Meaning | + |---|---| + | `installable` | `pyproject.toml` declares a `[build-system]`, so `uv sync` installs the project and the package is importable. | + | `pythonpath-ini` | `[tool.pytest.ini_options].pythonpath` includes `"."`. | + | `existing-conftest` | A `conftest.py` sits at the recipe root (sufficient whatever it contains — pytest puts each conftest's own directory on `sys.path`, and for that one it IS the recipe root), or a `tests/conftest.py` that demonstrably extends `sys.path`. | + | `generated-conftest` | None of the above held, so the skill wrote `tests/conftest.py` with a path shim. | + + A `tests/conftest.py` is judged by AST, not text search: it counts only if it really touches `sys.path`, so a comment or docstring that merely *mentions* `sys.path` cannot wrongly certify it. + + **Historical bug closed by this:** the recipe root being importable was assumed rather than checked. A recipe with no `[build-system]` — common for vertical skills under `skills/`, where code lives in a plain `scripts/` directory rather than an installed package — got a test that always died with `ModuleNotFoundError`, while `prepare-python-recipe`'s `py_compile` verification still reported success. + + Adding a `[build-system]` is the better fix; the generated conftest says so and tells the maintainer to delete it once they do. + +6. **Write it** to `/tests/test_runnability.py` (creating `tests/` if needed), plus `tests/conftest.py` when step 5 called for it. Refuses to clobber an existing file unless `--overwrite` is passed. ### Edit safety - No files outside the target recipe directory are read (beyond the recipe's own `.py` files) or written. - Existing `tests/test_runnability.py` is never silently overwritten. The user must explicitly opt in with `--overwrite`. +- An existing `tests/conftest.py` is **never** overwritten, not even with `--overwrite` — the skill reports `conftest_action: skipped` and warns instead. A conftest it did not write may do something clobbering would break. - `tests/` directory is created if missing (`mkdir -p` equivalent). No other directory or file is added. - Ruff-clean by construction — the generated file passes `ruff check` and `ruff format --check` under the root config. @@ -102,7 +118,7 @@ uv run --no-project python3 .agents/skills/generate-python-runnability-test/scri --recipe-dir --dry-run ``` -Output on stdout: JSON with `agent_file`, `module_name`, `detections`, `test_content`, `action` (`would_write` / `refused_overwrite` / `error`), and `message`. Exit code `0`. +Output on stdout: JSON with `agent_file`, `module_name`, `detections`, `import_support`, `conftest_path`, `conftest_action` (`would_write` / `wrote` / `skipped` / `null`), `test_content`, `action` (`would_write` / `refused_overwrite` / `error`), `message`, and `warnings` (a list — surface every entry to the user). Exit code `0`. Note: no `--with` flags are needed — the script only uses Python's stdlib (`ast`, `argparse`, `json`, `pathlib`, `dataclasses`, `os`, `sys`, `subprocess`, `textwrap`). `uv run --no-project python3` is used (rather than a bare `python3`) to guarantee a modern managed interpreter, consistent with the other Python recipe skills; the system `python3` on macOS can still be an old version. Dry-runs remain cheap and side-effect-free. diff --git a/.agents/skills/generate-python-runnability-test/scripts/generate_runnability_test.py b/.agents/skills/generate-python-runnability-test/scripts/generate_runnability_test.py index c9e6dfaf3e..6f0c24c068 100644 --- a/.agents/skills/generate-python-runnability-test/scripts/generate_runnability_test.py +++ b/.agents/skills/generate-python-runnability-test/scripts/generate_runnability_test.py @@ -57,6 +57,7 @@ import ast import json import os +import re import subprocess import sys import textwrap @@ -64,6 +65,11 @@ from pathlib import Path from typing import Any +try: # tomllib is stdlib from 3.11; this script still supports 3.9. + import tomllib +except ModuleNotFoundError: # pragma: no cover - depends on interpreter + tomllib = None # type: ignore[assignment] + # ---------- Constants ------------------------------------------------------ # Directories the recipe walker must never descend into. @@ -111,6 +117,26 @@ """ +CONFTEST_BODY = '''"""Put the recipe root on ``sys.path`` for the runnability test. + +The test imports ``{module_name}`` from the recipe root, but this recipe +declares no ``[build-system]``, so nothing installs it as a package. Under +pytest's default ``prepend`` import mode only the test file's own directory +goes on ``sys.path``, so the import would fail with ``ModuleNotFoundError``. + +Delete this file once the recipe declares a ``[build-system]`` and is +installed with ``uv sync`` — the shim is then redundant. +""" + +import sys +from pathlib import Path + +RECIPE_ROOT = Path(__file__).resolve().parent.parent +if str(RECIPE_ROOT) not in sys.path: + sys.path.insert(0, str(RECIPE_ROOT)) +''' + + # ---------- Report dataclasses --------------------------------------------- @@ -134,6 +160,13 @@ class Report: test_content: str | None = None action: str | None = None # would_write / wrote / refused_overwrite / error message: str = "" + # How `import ` is expected to resolve when pytest runs. + # One of: installable / pythonpath-ini / existing-conftest / + # generated-conftest / unresolved. + import_support: str | None = None + conftest_path: str | None = None + conftest_action: str | None = None # would_write / wrote / skipped / none + warnings: list[str] = field(default_factory=list) def to_json(self) -> str: return json.dumps( @@ -144,9 +177,13 @@ def to_json(self) -> str: "module_name": self.module_name, "target_path": self.target_path, "detections": asdict(self.detections), + "import_support": self.import_support, + "conftest_path": self.conftest_path, + "conftest_action": self.conftest_action, "test_content": self.test_content, "action": self.action, "message": self.message, + "warnings": self.warnings, }, indent=2, ) @@ -202,6 +239,117 @@ def module_path_from_file(agent_file: Path, recipe_dir: Path) -> str: return ".".join(parts) +# ---------- Import-support detection --------------------------------------- +# +# The generated test does `import `, which only resolves if the +# RECIPE ROOT is on sys.path. That is not automatic: under pytest's default +# `prepend` import mode, only the test file's own basedir (`/tests`) +# is inserted. The recipe root gets there when the project is installed +# (`uv sync`, which needs a [build-system]) or when something explicitly puts +# it there. +# +# Historical gap: this was assumed rather than checked, so a recipe without a +# [build-system] — common for vertical skills under skills/, where code lives +# in a plain scripts/ directory — got a test that always died with +# ModuleNotFoundError, while the pipeline's py_compile check still reported +# success. + + +_BUILD_SYSTEM_RE = re.compile(r"^\s*\[build-system\]", re.MULTILINE) + + +def _load_pyproject(pyproject: Path) -> dict[str, Any] | None: + """Parse pyproject.toml, or None when unavailable/unparseable.""" + if tomllib is None or not pyproject.is_file(): + return None + try: + return tomllib.loads(pyproject.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, ValueError): + return None + + +def _has_build_system(pyproject: Path) -> bool: + """Whether pyproject.toml declares a [build-system] table.""" + data = _load_pyproject(pyproject) + if data is not None: + return isinstance(data.get("build-system"), dict) + # tomllib unavailable (< 3.11) or unparseable — fall back to a text scan + # rather than silently claiming the recipe is not installable. + try: + return bool(_BUILD_SYSTEM_RE.search(pyproject.read_text("utf-8"))) + except (OSError, UnicodeDecodeError): + return False + + +def _pytest_pythonpath_covers_root(pyproject: Path) -> bool: + """Whether [tool.pytest.ini_options].pythonpath puts the root on sys.path.""" + data = _load_pyproject(pyproject) + if not data: + return False + ini = data.get("tool", {}).get("pytest", {}).get("ini_options", {}) + entries = ini.get("pythonpath") + if isinstance(entries, str): + entries = [entries] + if not isinstance(entries, list): + return False + return any(str(e).strip() in {".", "./", ""} for e in entries) + + +def _extends_syspath(path: Path) -> bool: + """Whether a file actually touches ``sys.path`` in executable code. + + AST-based on purpose: a substring search counts a comment or docstring + that merely *mentions* sys.path (``# does not touch sys.path``), which + would wrongly certify a conftest as providing import support. Comments + never appear in the AST and a prose mention is an ``ast.Constant``, not + an attribute access, so both are excluded for free. + """ + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, SyntaxError, ValueError): + return False + for node in ast.walk(tree): + # `sys.path...` + if ( + isinstance(node, ast.Attribute) + and node.attr == "path" + and isinstance(node.value, ast.Name) + and node.value.id == "sys" + ): + return True + # `from sys import path` + if isinstance(node, ast.ImportFrom) and node.module == "sys": + if any(alias.name == "path" for alias in node.names): + return True + return False + + +def detect_import_support(recipe_dir: Path) -> str: + """Classify how the recipe root reaches sys.path under pytest. + + Returns one of ``installable`` / ``pythonpath-ini`` / + ``existing-conftest`` / ``unresolved``. + """ + pyproject = recipe_dir / "pyproject.toml" + if _has_build_system(pyproject): + return "installable" + if _pytest_pythonpath_covers_root(pyproject): + return "pythonpath-ini" + # A conftest.py AT THE RECIPE ROOT is sufficient whatever it contains: + # under prepend import mode pytest puts each conftest's own directory on + # sys.path, and for this one that directory IS the recipe root. Verified + # empirically — an entirely empty root conftest.py makes `import pkg.mod` + # resolve. + if (recipe_dir / "conftest.py").is_file(): + return "existing-conftest" + # A tests/conftest.py, by the same rule, only adds `/tests` — which + # does NOT help. It counts only if it explicitly extends sys.path itself. + tests_conftest = recipe_dir / "tests" / "conftest.py" + if tests_conftest.is_file() and _extends_syspath(tests_conftest): + return "existing-conftest" + return "unresolved" + + # ---------- AST detection helpers ------------------------------------------ @@ -734,6 +882,36 @@ def run( target = tests_dir / "test_runnability.py" report.target_path = str(target) + # Will `import ` actually resolve? If not, the test compiles + # fine but dies at run time — so provide the missing sys.path shim rather + # than emitting a test that cannot pass. + report.import_support = detect_import_support(recipe_dir) + conftest = tests_dir / "conftest.py" + if report.import_support == "unresolved": + if conftest.exists(): + # Never clobber a conftest we did not write; it may already do + # something we would break. + report.conftest_action = "skipped" + report.conftest_path = str(conftest) + report.warnings.append( + f"{recipe_dir / 'pyproject.toml'} declares no [build-system], " + f"so the recipe is not installed and the recipe root is not " + f"on sys.path. {conftest} already exists and was left alone — " + f"confirm it puts the recipe root on sys.path, or " + f"`import {report.module_name}` will fail at run time." + ) + else: + report.conftest_action = "would_write" if dry_run else "wrote" + report.conftest_path = str(conftest) + report.import_support = "generated-conftest" + report.warnings.append( + f"No [build-system] in pyproject.toml, so nothing installs " + f"the recipe and `import {report.module_name}` would fail " + f"under pytest. Generated {conftest} to put the recipe root " + f"on sys.path. Adding a [build-system] is the better fix; " + f"delete the conftest once you do." + ) + if dry_run: report.action = "would_write" return report @@ -754,8 +932,23 @@ def run( report.message = f"Failed to write {target}: {e}" return report + if report.conftest_action == "wrote": + content = LICENSE_HEADER + CONFTEST_BODY.format( + module_name=report.module_name + ) + try: + conftest.write_text(content, encoding="utf-8") + except OSError as e: + # The test itself landed; report the shim failure without + # pretending the whole run failed. + report.conftest_action = "error" + report.warnings.append(f"Failed to write {conftest}: {e}") + report.action = "wrote" - report.message = f"Wrote {target}." + written = [str(target)] + if report.conftest_action == "wrote": + written.append(str(conftest)) + report.message = f"Wrote {', '.join(written)}." return report diff --git a/.agents/skills/generate-python-runnability-test/tests/conftest.py b/.agents/skills/generate-python-runnability-test/tests/conftest.py new file mode 100644 index 0000000000..2d55a0f3c1 --- /dev/null +++ b/.agents/skills/generate-python-runnability-test/tests/conftest.py @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Make this skill's scripts/ directory importable from its tests. + +Keeping the path shim inside the skill (rather than in the repo-root pytest +config) preserves the skill as a self-contained, portable bundle. +""" + +import sys +from pathlib import Path + +SCRIPTS_DIR = Path(__file__).parent.parent / "scripts" +sys.path.insert(0, str(SCRIPTS_DIR)) diff --git a/.agents/skills/generate-python-runnability-test/tests/test_generate_runnability_test.py b/.agents/skills/generate-python-runnability-test/tests/test_generate_runnability_test.py new file mode 100644 index 0000000000..5735a9166c --- /dev/null +++ b/.agents/skills/generate-python-runnability-test/tests/test_generate_runnability_test.py @@ -0,0 +1,187 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the generate-python-runnability-test skill script. + +Focused on import-support detection: the generated test does +``import ``, which only resolves when the RECIPE ROOT is on +``sys.path``. That was previously assumed rather than checked, so a recipe +with no ``[build-system]`` got a test that always died with +``ModuleNotFoundError`` while ``py_compile`` still reported success. +""" + +import ast +from pathlib import Path + +import generate_runnability_test as m + + +def _write(path: Path, content: str = "") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def _recipe(tmp_path: Path, pyproject: str, pkg: str = "app") -> Path: + _write(tmp_path / pkg / "agent.py", "root_agent = object()\n") + _write(tmp_path / "pyproject.toml", pyproject) + return tmp_path + + +# --------------------------------------------------------------------------- +# detect_import_support +# --------------------------------------------------------------------------- + + +def test_build_system_means_installable(tmp_path): + recipe = _recipe( + tmp_path, + '[project]\nname = "x"\n' + '[build-system]\nrequires = ["hatchling"]\n' + 'build-backend = "hatchling.build"\n', + ) + assert m.detect_import_support(recipe) == "installable" + + +def test_pythonpath_ini_covering_root(tmp_path): + recipe = _recipe( + tmp_path, + '[project]\nname = "x"\n' + '[tool.pytest.ini_options]\npythonpath = ["."]\n', + ) + assert m.detect_import_support(recipe) == "pythonpath-ini" + + +def test_pythonpath_ini_not_covering_root(tmp_path): + recipe = _recipe( + tmp_path, + '[project]\nname = "x"\n' + '[tool.pytest.ini_options]\npythonpath = ["src"]\n', + ) + assert m.detect_import_support(recipe) == "unresolved" + + +def test_root_conftest_is_sufficient_even_when_empty(tmp_path): + # Verified against real pytest: under prepend import mode a conftest's own + # directory goes on sys.path, and for a ROOT conftest that is the recipe + # root — so its contents are irrelevant. + recipe = _recipe(tmp_path, '[project]\nname = "x"\n') + _write(recipe / "conftest.py", "") + assert m.detect_import_support(recipe) == "existing-conftest" + + +def test_tests_conftest_counts_only_when_it_touches_syspath(tmp_path): + recipe = _recipe(tmp_path, '[project]\nname = "x"\n') + _write(recipe / "tests" / "conftest.py", "# just fixtures\n") + # tests/conftest.py only adds /tests, which does not help. + assert m.detect_import_support(recipe) == "unresolved" + + # A mere MENTION of sys.path in prose must not count as import support. + _write( + recipe / "tests" / "conftest.py", + '"""Fixtures. Deliberately does not touch sys.path."""\n', + ) + assert m.detect_import_support(recipe) == "unresolved" + + _write( + recipe / "tests" / "conftest.py", + "import sys\nsys.path.insert(0, '..')\n", + ) + assert m.detect_import_support(recipe) == "existing-conftest" + + +def test_missing_pyproject_is_unresolved(tmp_path): + _write(tmp_path / "app" / "agent.py", "root_agent = object()\n") + assert m.detect_import_support(tmp_path) == "unresolved" + + +def test_unparseable_pyproject_falls_back_to_text_scan(tmp_path): + # Malformed TOML must not be read as "no build system" — the text scan + # is the safety net. + recipe = _recipe( + tmp_path, + '[project\nname = "x"\n[build-system]\nrequires = ["hatchling"]\n', + ) + assert m.detect_import_support(recipe) == "installable" + + +# --------------------------------------------------------------------------- +# conftest generation +# --------------------------------------------------------------------------- + + +def test_generates_conftest_when_import_would_fail(tmp_path): + recipe = _recipe(tmp_path, '[project]\nname = "x"\n', pkg="scripts") + + report = m.run(recipe, None, dry_run=False, overwrite=False) + + assert report.action == "wrote" + assert report.import_support == "generated-conftest" + assert report.conftest_action == "wrote" + conftest = recipe / "tests" / "conftest.py" + assert conftest.is_file() + content = conftest.read_text(encoding="utf-8") + assert "sys.path.insert" in content + ast.parse(content) # must be valid Python + assert any("build-system" in w for w in report.warnings) + + +def test_no_conftest_generated_when_installable(tmp_path): + recipe = _recipe( + tmp_path, + '[project]\nname = "x"\n' + '[build-system]\nrequires = ["hatchling"]\n' + 'build-backend = "hatchling.build"\n', + ) + + report = m.run(recipe, None, dry_run=False, overwrite=False) + + assert report.import_support == "installable" + assert report.conftest_action is None + assert not (recipe / "tests" / "conftest.py").exists() + assert report.warnings == [] + + +def test_existing_conftest_is_never_clobbered(tmp_path): + recipe = _recipe(tmp_path, '[project]\nname = "x"\n') + original = "# hand-written, does not touch sys.path\n" + _write(recipe / "tests" / "conftest.py", original) + + report = m.run(recipe, None, dry_run=False, overwrite=False) + + assert report.conftest_action == "skipped" + assert (recipe / "tests" / "conftest.py").read_text() == original + assert any("left alone" in w for w in report.warnings) + + +def test_dry_run_writes_nothing(tmp_path): + recipe = _recipe(tmp_path, '[project]\nname = "x"\n') + + report = m.run(recipe, None, dry_run=True, overwrite=False) + + assert report.action == "would_write" + assert report.conftest_action == "would_write" + assert not (recipe / "tests" / "conftest.py").exists() + assert not (recipe / "tests" / "test_runnability.py").exists() + + +def test_report_json_includes_new_fields(tmp_path): + import json + + recipe = _recipe(tmp_path, '[project]\nname = "x"\n') + report = m.run(recipe, None, dry_run=True, overwrite=False) + + payload = json.loads(report.to_json()) + assert payload["import_support"] == "generated-conftest" + assert payload["conftest_action"] == "would_write" + assert isinstance(payload["warnings"], list) diff --git a/.agents/skills/prepare-python-recipe/SKILL.md b/.agents/skills/prepare-python-recipe/SKILL.md index c60dbbf304..a2446d15e5 100644 --- a/.agents/skills/prepare-python-recipe/SKILL.md +++ b/.agents/skills/prepare-python-recipe/SKILL.md @@ -4,11 +4,13 @@ description: > End-to-end orchestration to prepare or update a Python recipe under core/python/, contrib/python/, or skills/// so it passes every check in .github/workflows/python-validate-recipe.yml. - Runs seven phases in + Runs eight phases in order on an already-in-place recipe: manifest.yaml generation, environment-variable extraction, pyproject.toml alignment, ruff - format+check, per-recipe `uv lock`, runnability-test generation, and a - final `py_compile` verification of the generated test file. Assumes + format+check, per-recipe `uv lock`, runnability-test generation, + compile-and-run verification of the generated test file, and a final + pass through the repo's own `validate manifest` / `validate structure` + validators. Assumes the user has already done the manual prep (deactivated any venv, `git pull` and `uv sync` from the repo root, placed the recipe at its target path, @@ -25,7 +27,7 @@ description: > metadata: author: Google license: Apache-2.0 - version: 1.0.0 + version: 1.1.0 --- # Prepare Python Recipe @@ -65,25 +67,28 @@ If the user has NOT done these and asks you to run the skill anyway, tell them t ## What This Skill Does -Runs seven ordered phases against a target recipe. Each phase either invokes an existing sub-skill (or its underlying script) or runs a repo-standard command: +Runs eight ordered phases against a target recipe. Each phase either invokes an existing sub-skill (or its underlying script) or runs a repo-standard command: 1. **Manifest** — generate `manifest.yaml` if missing. Ownership placeholders (`ownership.team`, `ownership.poc`) are LEFT AS-IS — never replaced mid-pipeline. See "Canonical placeholder strings" above. 2. **Environment variables** — extract env vars used by the recipe into `.env.example`; ensure `load_dotenv()` is bootstrapped and `python-dotenv` is a dep. -3. **Align pyproject.toml** — remove `[tool.ruff*]`, raise `requires-python` floor, ensure `[project].name` matches folder, reconcile description with manifest, and ensure `[[tool.uv.index]]` declares public PyPI as default (needed to bypass corp Airlock). +3. **Align pyproject.toml** — remove `[tool.ruff*]`, raise `requires-python` floor, ensure `[project].name` matches folder, reconcile description with manifest, ensure `[[tool.uv.index]]` declares public PyPI as default (needed to bypass corp Airlock), and report stale sub-3.11 version references plus a `testpaths` that would exclude the runnability test. 4. **Lint** — `ruff format` + `ruff check --fix` on the recipe (from the repo root, so the root ruff config wins). **Must run AFTER Phase 3** — align removes any recipe-local `[tool.ruff*]` block, and that removal is what makes the root config the effective one. Running lint before align would check against the recipe's (often more permissive) local config and miss violations that CI will later catch. 5. **Recipe `uv lock`** — regenerate `uv.lock` so it reflects the post-align `pyproject.toml`. Does NOT install into `.venv/` — that's a heavier step the user runs after they've reviewed the diff. `uv lock` just resolves and records; `uv sync` would download and install every wheel, which is scope-creep for a "prepare" pipeline. -6. **Runnability test** — generate `tests/test_runnability.py` if missing (or ask before overwriting). -7. **Verify (compile-check)** — `uv run --no-project python3 -m py_compile /tests/test_runnability.py` — a lightweight sanity check that the generated (or existing) test file is syntactically valid Python. Deterministic; does NOT execute the test, resolve imports, or require `.env` to exist. If it fails, the master reports the error verbatim and moves on (the summary marks Phase 7 as failed). The master does NOT attempt to diagnose or fix — that's a human review task. +6. **Runnability test** — generate `tests/test_runnability.py` if missing (or ask before overwriting), plus a `tests/conftest.py` path shim when the recipe isn't installable. +7. **Verify (compile + run)** — `py_compile` the runnability test, then run it with pytest. The compile step is a syntax check; running it is what proves the test's `import` can actually resolve (`--collect-only` would not — the guarded test shape puts the import inside the test function). The test is side-effect-free by construction. +8. **Validate (repo validators)** — run `uv run validate manifest` and `uv run validate structure` on the recipe. This is the phase that catches everything the seven build phases don't model: required files, required directories (`tests/unit/` for vertical skills), size limits, naming. At the end, print a summary table and remind the user to `git diff` and commit — the skill never commits. +**Why Phase 8 exists.** The pipeline used to end at Phase 7 and report a clean run while `uv run validate structure` failed — the recipe passed every phase the skill modelled and was still rejected by CI. Reimplementing policy checks inside this skill would guarantee drift, so the pipeline defers to the repo's own validators as the last word. + --- ## Rules for the Agent -1. **Ask for `--recipe-dir` up front** if the user hasn't given one. All seven phases operate on the same recipe. +1. **Ask for `--recipe-dir` up front** if the user hasn't given one. All eight phases operate on the same recipe. -2. **Confirm before starting**. The pipeline touches many files. Show the user the plan (the seven phases + the target recipe path) and ask for a single "yes, go ahead" before Phase 1. Do NOT prompt again for each phase unless a decision is required (see rules 5 and 6). +2. **Confirm before starting**. The pipeline touches many files. Show the user the plan (the eight phases + the target recipe path) and ask for a single "yes, go ahead" before Phase 1. Do NOT prompt again for each phase unless a decision is required (see rules 5 and 6). 3. **Invoke sub-skill SCRIPTS directly** (not the sub-skills' own agent-facing SKILL.md). Reason: sub-skills each have their own "want me to apply?" prompt. In master-orchestration mode the user has already opted into apply for the whole pipeline; individual prompts would be noise. Command lines for each sub-script are given in each phase below. @@ -143,7 +148,7 @@ If the user has not specified the recipe directory, ask for it before proceeding If it isn't a directory, stop immediately with that message — do NOT show the plan or prompt. -**Step 0b — Verify the recipe folder name matches CI's naming rules.** `python-validate-recipe.yml`'s Check 1 (folder-name regex + max length) rejects folders that don't match `^[a-z][a-z-]*$` or exceed `.github/policy.yml` `recipe_naming.max_folder_name_length`. Historically the pipeline was BLIND to this — it would run all 7 phases against a folder named `data_science` or `MyBadName`, report success, and let CI reject the PR later (or worse: Phase 3's `project-name-matches-folder` would propagate the bad name into `[project].name`). This check catches it up front. +**Step 0b — Verify the recipe folder name matches CI's naming rules.** `python-validate-recipe.yml`'s Check 1 (folder-name regex + max length) rejects folders that don't match `^[a-z][a-z-]*$` or exceed `.github/policy.yml` `recipe_naming.max_folder_name_length`. Historically the pipeline was BLIND to this — it would run every phase against a folder named `data_science` or `MyBadName`, report success, and let CI reject the PR later (or worse: Phase 3's `project-name-matches-folder` would propagate the bad name into `[project].name`). This check catches it up front. ```bash MAX_LEN=$(uv run --no-project --with pyyaml python3 .github/scripts/load_policy.py recipe_naming.max_folder_name_length) @@ -157,7 +162,30 @@ The check exits 0 silently on a compliant name; on violation it exits 1 with the **Only proceed past this step if the folder-name check passed.** -**Step 0c — Show the plan and get confirmation.** Before composing the plan, glance at the recipe for anything non-standard (package not called `app/`, `.env.example` outside root, missing `tests/`, extra Python source dirs, deprecated model literals per `AGENTS.md`). If any will affect what the pipeline does, flag them briefly in the plan message so the user isn't surprised mid-pipeline. Skip the flags entirely for a standard recipe. +**Step 0c — For a recipe under `skills/`, check the required directories.** `.github/policy.yml` `required_dirs.by_root.skills` mandates a fixed shape for every vertical skill — `scripts/`, `assets/`, `references/`, and `tests/unit/`. None of the eight phases creates these, so a missing one survives the whole pipeline and fails `validate structure` in Phase 8 (and CI). Surfacing it here means the user can create the directory before anything else runs, rather than reading about it in the final summary. + +Skip this step entirely for `core/` and `contrib/` recipes — `required_dirs.by_root` is empty for both. + +```bash +uv run --no-project --with pyyaml python3 -c " +import pathlib, sys, yaml +recipe = pathlib.Path('') +policy = yaml.safe_load(open('.github/policy.yml')) +needed = policy.get('required_dirs', {}).get('by_root', {}).get('skills', []) or [] +missing = [d for d in needed if not (recipe / d).is_dir()] +print('MISSING_DIRS: ' + (', '.join(missing) if missing else '(none)')) +" +``` + +This is INFORMATIONAL, not a halt. An empty directory satisfies the check, and git cannot commit an empty directory, so the fix is a `.gitkeep`: + +```bash +mkdir -p /tests/unit && touch /tests/unit/.gitkeep +``` + +Mention any missing directories in the Step 0d plan message and offer to create them with `.gitkeep` files as part of the run. If the user agrees, create them right after they confirm the plan and before Phase 1; record them in the summary's "Files created" list. If they decline, carry the item into the final TODO list. Do NOT create them unasked — an empty scaffold directory the user didn't want is still clutter. + +**Step 0d — Show the plan and get confirmation.** Before composing the plan, glance at the recipe for anything non-standard (package not called `app/`, `.env.example` outside root, missing `tests/`, extra Python source dirs, deprecated model literals per `AGENTS.md`). If any will affect what the pipeline does, flag them briefly in the plan message so the user isn't surprised mid-pipeline. Skip the flags entirely for a standard recipe. Then flag the assumptions the pipeline is making and show the user the plan. Do NOT frame these as "prerequisites" — they're a heads-up so the user can push back if any assumption is wrong, not a preflight checklist for the user to tick off: @@ -166,17 +194,23 @@ Then flag the assumptions the pipeline is making and show the user the plan. Do > - You've run `git pull` and `uv sync` at the repo root. > - `` is already at its target path (and renamed to its final basename). > -> I'll run the prepare-python-recipe pipeline on `` — 7 phases: +> I'll run the prepare-python-recipe pipeline on `` — 8 phases: > 1. Generate manifest.yaml (if missing) > 2. Extract env vars into .env.example > 3. Align pyproject.toml > 4. Ruff format + check --fix > 5. uv lock inside the recipe (regenerates uv.lock; does NOT install .venv/) > 6. Generate tests/test_runnability.py (if missing) -> 7. Compile-check the runnability test (`py_compile`; reports failure but does not debug) +> 7. Verify the runnability test compiles and runs +> 8. Run the repo validators (`validate manifest`, `validate structure`) > > Nothing gets committed — you'll `git diff` at the end. Proceed? +If Step 0c found missing required directories, add one line before "Nothing gets committed": + +> `` is a vertical skill and is missing `tests/unit/`, which +> `.github/policy.yml` requires. Want me to create it with a `.gitkeep`? + Get a yes-or-no. If no, stop. ### Phase 1 — manifest.yaml @@ -239,9 +273,11 @@ uv run --no-project --with tomlkit --with 'ruamel.yaml' --with packaging \ [--description-source=] ``` -**The align script exits `1` (non-zero) whenever any check is `report_only`** — that is expected, not a hard error, so do NOT apply rule 7's halt to it. Decide from the JSON, not the exit code: if the apply run's only non-clean checks are `report_only`, note them in the summary (the master does NOT auto-fix these) and continue; halt only if a check has status `error`. Two rules can produce `report_only`: - - `build-system-present` (missing `[build-system]` — backend choice is editorial) +**The align script exits `1` (non-zero) whenever any check is `report_only`** — that is expected, not a hard error, so do NOT apply rule 7's halt to it. Decide from the JSON, not the exit code: if the apply run's only non-clean checks are `report_only`, note them in the summary (the master does NOT auto-fix these) and continue; halt only if a check has status `error`. Four rules can produce `report_only`: + - `build-system-present` (missing `[build-system]` — backend choice is editorial). **Note the knock-on for Phase 6:** with no `[build-system]` the recipe is never installed, so the runnability test's `import` cannot resolve on its own and the generator will emit a `tests/conftest.py` path shim. Mention both together in the summary rather than as two unrelated items. - `default-pypi-index` (a default index is declared but points somewhere other than public PyPI — divergence may be intentional) + - `stale-python-version-refs` (files still referencing a Python below the 3.11 floor). Relay `details.files` in the summary and flag **executable** files first — a bootstrap script whose interpreter allowlist still accepts 3.10 will build a venv the recipe then refuses to install into. Prose files are lower priority. + - `runnability-test-in-testpaths` (a `testpaths` setting that excludes `tests/test_runnability.py`, so a bare `pytest` never collects it) Progress line: `Phase 3 (align): fix(es) applied; report-only issue(s) left.` @@ -306,32 +342,83 @@ uv run --no-project python3 .agents/skills/generate-python-runnability-test/scri If the script errors (no `agent.py` found), surface the message and offer to re-run with `--agent-file ` when the user tells you where the entry point is. -Progress line: `Phase 6 (runnability test): generated | kept existing | regenerated.` +**Read the report's `import_support`, `conftest_action`, and `warnings` fields.** The generated test does `import `, which only resolves if the recipe root is on `sys.path` — not automatic under pytest. When the recipe declares no `[build-system]` the generator also writes a `tests/conftest.py` path shim (`conftest_action: wrote`) and explains why in `warnings`. Relay every warning; do not silently drop them. If `conftest_action` is `skipped`, an existing `tests/conftest.py` was left untouched and may not provide the shim — carry that into the summary as a Manual TODO. -### Phase 7 — verify (compile-check the runnability test) +Progress line: `Phase 6 (runnability test): generated | kept existing | regenerated[; tests/conftest.py path shim written (recipe has no [build-system])].` -Runs LAST. Lightweight sanity check that the generated (or existing) `tests/test_runnability.py` is at least syntactically valid Python. Deliberately weaker than `uv run pytest`: it does NOT execute the test, resolve imports, or require `.env` to be populated. Its only purpose is to catch generator bugs (invalid Python emitted by Phase 6) and gross syntax errors in a hand-edited test file. +### Phase 7 — verify (compile + run the runnability test) -**7a. Check the test file exists.** If Phase 6 skipped generation (agent.py not found, so no test was written) or the user chose not to regenerate an existing broken test, there may be nothing to compile. Skip and record it. +Two escalating checks on the generated (or existing) `tests/test_runnability.py`. The test is designed to be side-effect-free (it patches `vertexai.init` and `google.auth.default`), so neither step needs `.env`, ADC, or network. + +**7a. Check the test file exists.** If Phase 6 skipped generation (agent.py not found, so no test was written) or the user chose not to regenerate an existing broken test, there may be nothing to check. Skip and record it. ```bash [ -f /tests/test_runnability.py ] && echo exists || echo missing ``` -**7b. Compile.** +**7b. Compile** — is it valid Python? ```bash uv run --no-project python3 -m py_compile /tests/test_runnability.py ``` -Use `uv run --no-project python3` here too — not a bare `python`/`python3`. Two reasons: `python` may not be on PATH at all on some systems, and (more importantly) a guarded test with multiple patches emits a parenthesized `with (...)` block, which is **Python 3.10+ syntax**. Compiling it under an older system interpreter would report a spurious `SyntaxError` on a file that is actually valid. uv's managed interpreter is 3.11+, so this is a true syntax check rather than a version artifact. +Use `uv run --no-project python3` here — not a bare `python`/`python3`. Two reasons: `python` may not be on PATH at all on some systems, and (more importantly) a guarded test with multiple patches emits a parenthesized `with (...)` block, which is **Python 3.10+ syntax**. Compiling it under an older system interpreter would report a spurious `SyntaxError` on a file that is actually valid. uv's managed interpreter is 3.11+, so this is a true syntax check rather than a version artifact. + +**7c. Run it** — does the import actually resolve? + +```bash +uv run --no-project --with pytest pytest tests/test_runnability.py -q +``` + +Run this WITH `workdir = ` so pytest's rootdir matches what a user would get running the test themselves. + +**Why run it rather than `--collect-only`.** `py_compile` only parses; it passes happily on a test whose `import app.agent` can never resolve. But `--collect-only` is no better here: the *guarded* test shape (the common one for ADK recipes) puts the import INSIDE the test function, under a `with patch(...)` block, so collection imports the test module without ever touching the recipe's module. Only actually running the test exercises the import. + +**A third-party `ModuleNotFoundError` is EXPECTED here, not a finding.** Phase 5 ran `uv lock`, not `uv sync`, so the recipe's dependencies are NOT installed. Classify by the module named in the error: + +- The name is a **dependency** (`vertexai`, `google.adk`, `pandas`) → expected. Report `deps not installed`. Note that this outcome is **inconclusive** about the import path: the dependency failure fires before the recipe's own import is reached, so the test proves nothing either way. Fall back to Phase 6's `import_support` field for that question (`installable` / `pythonpath-ini` / `existing-conftest` / `generated-conftest` all mean the path is provided; only `unresolved` is a problem). +- The name is the recipe's **own top-level module** (the first segment of Phase 6's `module_name` — e.g. `scripts` from `scripts.agent`) → REAL finding. The import path is broken and the test can never pass, whatever else is installed. Report per outcome: -- Exit 0 → **pass.** Progress line: `Phase 7 (verify): compile OK.` -- Exit non-zero → **fail.** Print the stderr verbatim in the summary as a Manual TODO. Do NOT attempt to diagnose, retry, or auto-fix. Do NOT halt the pipeline (Phase 7 is the last phase anyway; the summary still gets printed). Progress line: `Phase 7 (verify): compile FAILED — .` +- Compile 0, test passes → **pass.** Progress line: `Phase 7 (verify): compile OK, test passes.` +- Compile 0, fails on a dependency → **pass with note.** Progress line: `Phase 7 (verify): compile OK; test not run ( not installed — run uv sync). Import path: .` +- Compile 0, fails on the recipe's own module → **fail.** Progress line: `Phase 7 (verify): compile OK but is not importable.` Record as a Manual TODO; the usual cause is a missing `[build-system]` (Phase 3 report-only) with no conftest shim. +- Compile 0, fails an ASSERTION (`root_agent is None`) → **fail.** A genuine recipe defect, not an environment one. Report it verbatim. +- Compile non-zero → **fail.** Print the stderr verbatim in the summary as a Manual TODO and SKIP 7c (a file that doesn't parse cannot run). Do NOT attempt to diagnose, retry, or auto-fix. Progress line: `Phase 7 (verify): compile FAILED — .` - File missing → **skip.** Progress line: `Phase 7 (verify): skipped (no tests/test_runnability.py to check).` -Note: passing Phase 7 does NOT mean the recipe actually runs — it means the test file is valid Python. Actually running the test (which validates that `agent.py` imports and `root_agent` is non-None) is still a manual `uv run pytest` step listed under "Next steps" in the summary. +Never halt the pipeline on Phase 7 — Phase 8 still runs and the summary still prints. + +Note: a `deps not installed` result means Phase 7 has NOT proven the recipe runs. Say so plainly in the summary rather than implying a clean bill of health; the real confirmation is the manual `uv sync && uv run pytest` step under "Next steps". + +### Phase 8 — validate (the repo's own validators) + +Runs LAST. Phases 1–7 model what the pipeline knows how to *build*; this phase asks the repo whether the result is actually acceptable. It is deliberately a thin wrapper: the checks live in `tools/` and `.github/policy.yml`, and reimplementing any of them here would guarantee drift. + +**Historical gap this closes:** the pipeline ended at Phase 7 and printed a clean summary for a recipe that `validate structure` rejected for a missing `tests/unit/` directory. Every modelled phase passed; CI still failed the PR. + +**8a. Validate the manifest.** + +```bash +uv run validate manifest +``` + +**8b. Validate the structure.** + +```bash +uv run validate structure +``` + +Both run from the repo root with a repo-root-relative ``, never an absolute path. Both exit non-zero on failure — do NOT apply rule 7's halt (Phase 8 is last, and its failures are findings to report, not crashes). + +**Interpreting the output — two distinct kinds of failure:** + +- **Ownership placeholder failures are EXPECTED.** `ownership.team` / `ownership.poc` still hold the canonical placeholders, and the validator fails deliberately until a human replaces them. Report these as `expected` in the summary, not as a problem the pipeline caused. They are already item 1 of the TODO list. +- **Everything else is a REAL finding.** Missing required files or directories, size-limit violations, schema errors, naming violations. List each verbatim in Section 3 with the fix. + +Progress line: `Phase 8 (validate): manifest ; structure .` + +If the only failures across both validators are the two ownership placeholders, the recipe is in the expected end state — say so plainly rather than presenting it as a failure. --- @@ -340,9 +427,9 @@ Note: passing Phase 7 does NOT mean the recipe actually runs — it means the te While the pipeline runs, print a short progress line per phase (see above). Do NOT dump raw JSON. Do NOT re-render sub-skill tables. **Track three things as the pipeline runs** so you can report them at the end: -- Every file the pipeline created or modified (across all seven phases). Observe this from each sub-script's stdout plus your own knowledge of what each phase touches (Phase 1 → `manifest.yaml`; Phase 2 → `.env.example`, package `__init__.py`, `pyproject.toml`, any source files where a hardcoded model name was replaced; Phase 3 → `pyproject.toml`; Phase 4 → any `.py` under the recipe; Phase 5 → `uv.lock`; Phase 6 → `tests/test_runnability.py`; Phase 7 → nothing). -- Every action the pipeline **attempted but couldn't complete** (a phase halted by rule 7, Phase 4 unfixable ruff, Phase 7 compile fail). -- Every deferred item that needs human follow-up (Phase 3 `report_only`). +- Every file the pipeline created or modified (across all eight phases). Observe this from each sub-script's stdout plus your own knowledge of what each phase touches (Phase 0c → `tests/unit/.gitkeep` and any other required dir, if the user opted in; Phase 1 → `manifest.yaml`; Phase 2 → `.env.example`, package `__init__.py`, `pyproject.toml`, any source files where a hardcoded model name was replaced; Phase 3 → `pyproject.toml`; Phase 4 → any `.py` under the recipe; Phase 5 → `uv.lock`; Phase 6 → `tests/test_runnability.py` and possibly `tests/conftest.py`; Phases 7 and 8 → nothing). +- Every action the pipeline **attempted but couldn't complete** (a phase halted by rule 7, Phase 4 unfixable ruff, Phase 7 compile or run failure). +- Every deferred item that needs human follow-up (Phase 3 `report_only`, Phase 8 real validator findings). At the end, print the sections below in order. Section 3 is **conditional** — omit it entirely if nothing belongs in it. Sections 1, 2, and 4 always print. @@ -356,10 +443,13 @@ At the end, print the sections below in order. Section 3 is **conditional** — | 4. Lint | ok | 12 files formatted, 4 issues auto-fixed | | 5. Recipe lock | ok | done | | 6. Runnability test | ok | generated | -| 7. Verify (compile-check) | ok | tests/test_runnability.py compiles | +| 7. Verify (compile + run) | ok | compiles; test passes | +| 8. Validate (repo validators) | ok | manifest + structure clean except ownership placeholders | Use plain words in the Outcome column (`ok` / `skipped` / `failed`). No emoji unless the user asked for them. +Phase 8 is `ok` when the only failures are the two ownership placeholders — that is the expected end state, not a defect. Use `failed` only for real validator findings. + ### 2. Files created or modified Short bullet list, grouped **Created** and **Modified**, one line per file. Aggregate large groups (e.g. "12 `.py` files formatted (Phase 4)") rather than listing each individually. Omit files that were checked but untouched. If a group is empty, drop its heading. If nothing changed at all, print `Nothing changed — the recipe was already fully aligned.` @@ -369,13 +459,15 @@ Example: > **Created** > - `manifest.yaml` (Phase 1) > - `tests/test_runnability.py` (Phase 6) +> - `tests/conftest.py` (Phase 6 — sys.path shim; recipe has no `[build-system]`) +> - `tests/unit/.gitkeep` (Phase 0c — required for `skills/` recipes) > - `uv.lock` (Phase 5) > > **Modified** > - `pyproject.toml` (Phases 2, 3 — added `python-dotenv`; aligned rules) > - `/__init__.py` (Phase 2 — added `load_dotenv()`) > - `.env.example` (Phase 2 — 3 vars added) -> - `agent.py` (Phase 2 — replaced hardcoded model name with `os.getenv("MODEL_NAME")`) +> - `agent.py` (Phase 2 — replaced hardcoded model name with `os.getenv("EMBEDDING_MODEL")`) > - 12 `.py` files formatted, 4 auto-fixed (Phase 4) ### 3. What the skill tried but couldn't complete (conditional — omit section if empty) @@ -388,12 +480,16 @@ Cases that go here: - **Halted phase (rule 7 hard error)**: a phase's script exited non-zero and the pipeline stopped. Show the phase, the command that failed, and a one-line snippet of the stderr. Explicitly note which phases did NOT run as a result. - **Phase 4 unfixable ruff**: `ruff check --fix` ran but couldn't auto-fix some violations. Show `:` — `` for each. +- **Phase 6 conftest skipped**: an existing `tests/conftest.py` blocked the path shim the generator wanted to write. - **Phase 7 compile fail**: `py_compile` on `tests/test_runnability.py` returned an error. Show a one-line snippet of the stderr. +- **Phase 7 failure on the recipe's own module**: the test's import can't resolve. (A dependency-not-installed failure does NOT belong here — that's expected after `uv lock`.) +- **Phase 8 real validator findings**: anything `validate manifest` / `validate structure` reported other than the two ownership placeholders. One line each, verbatim. Example: > - **Phase 4 (ruff)** — 2 violations remain that `ruff check --fix` can't auto-fix: `app/deploy.py:276` (`C901`, `PLR0915`), `app/tools.py:52` (`C901`). Refactor or add `# noqa: ` at the def line. > - **Phase 7 (verify)** — `uv run --no-project python3 -m py_compile tests/test_runnability.py` failed: `SyntaxError: invalid syntax (line 14)`. Likely a generator bug or hand-edit; regenerate or fix before running pytest. +> - **Phase 8 (validate)** — `validate structure` reports `Required directory 'tests/unit/' is missing`. Create it with a `.gitkeep`. ### 4. What you still need to do @@ -407,8 +503,12 @@ A single short TODO list. Keep every entry to one line. Standard items come firs **Conditional — include ONLY if the phase raised it:** -- **Phase 3 report-only, `build-system`**: add a `[build-system]` block to `pyproject.toml` — see `.agents/skills/align-recipe-pyproject/SKILL.md` for hatchling / uv_build templates. +- **Phase 3 report-only, `build-system`**: add a `[build-system]` block to `pyproject.toml` — see `.agents/skills/align-recipe-pyproject/SKILL.md` for hatchling / uv_build templates. If Phase 6 also wrote a `tests/conftest.py` shim, note that adding the build-system makes the shim redundant and it can be deleted. - **Phase 3 report-only, `pypi-index`**: `[[tool.uv.index]]` default points somewhere other than public PyPI — verify this is intentional or fix per the align skill. +- **Phase 3 report-only, `stale-python-version-refs`**: list the files, executable ones first (a bootstrap script's interpreter allowlist is a live bug; prose is cosmetic). +- **Phase 3 report-only, `runnability-test-in-testpaths`**: add `"tests"` to `[tool.pytest.ini_options].testpaths`, or the runnability test never runs under a bare `pytest`. +- **Phase 0c / Phase 8, missing required directory**: create it with a `.gitkeep` (e.g. `mkdir -p /tests/unit && touch /tests/unit/.gitkeep`) if the user declined during Phase 0c. +- **Phase 8, other validator findings**: one line per finding with the fix. **Commands — always show:** @@ -419,6 +519,6 @@ uv run pytest tests/test_runnability.py -v # confirm the runnability test ac # commit when you're happy ``` -`uv sync` is what actually installs the recipe's dependencies into `.venv/`. The pipeline stopped at `uv lock` on purpose — installing is heavier and better done after you've reviewed the diff. +`uv sync` is what actually installs the recipe's dependencies into `.venv/`. The pipeline stopped at `uv lock` on purpose — installing is heavier and better done after you've reviewed the diff. It is also why Phase 7 usually cannot run the test to completion — it stops at the first missing dependency. Then stop. Do NOT commit. End your turn. From 5133852e3faa02e84e5bfbce41d10870caeaa8b6 Mon Sep 17 00:00:00 2001 From: Shahin Saadati Date: Wed, 29 Jul 2026 15:58:08 -0700 Subject: [PATCH 3/5] chore: remove retail-product-search skill and associated files --- skills/retail/retail-product-search/EVAL.yaml | 209 ------- skills/retail/retail-product-search/README.md | 80 --- skills/retail/retail-product-search/SKILL.md | 271 --------- .../assets/design-spec.md | 37 -- .../assets/sample-products.csv | 6 - .../retail-product-search/pyproject.toml | 39 -- .../references/agent-example.md | 71 --- .../references/architecture.md | 49 -- .../references/dependencies.md | 27 - .../references/ingestion-scripts.md | 165 ------ .../references/install-paths.md | 44 -- .../references/troubleshooting.md | 39 -- .../retail-product-search/scripts/_paths.py | 35 -- .../scripts/_setup_utils.py | 59 -- .../retail-product-search/scripts/agent.py | 144 ----- .../scripts/bootstrap.sh | 110 ---- .../retail-product-search/scripts/cleanup.py | 533 ------------------ .../scripts/ingest_bigquery.py | 467 --------------- .../scripts/ingest_vertex_search.py | 467 --------------- .../scripts/retrievers.py | 163 ------ .../retail-product-search/scripts/setup.py | 246 -------- .../scripts/validate_schema.py | 281 --------- 22 files changed, 3542 deletions(-) delete mode 100644 skills/retail/retail-product-search/EVAL.yaml delete mode 100644 skills/retail/retail-product-search/README.md delete mode 100644 skills/retail/retail-product-search/SKILL.md delete mode 100644 skills/retail/retail-product-search/assets/design-spec.md delete mode 100644 skills/retail/retail-product-search/assets/sample-products.csv delete mode 100644 skills/retail/retail-product-search/pyproject.toml delete mode 100644 skills/retail/retail-product-search/references/agent-example.md delete mode 100644 skills/retail/retail-product-search/references/architecture.md delete mode 100644 skills/retail/retail-product-search/references/dependencies.md delete mode 100644 skills/retail/retail-product-search/references/ingestion-scripts.md delete mode 100644 skills/retail/retail-product-search/references/install-paths.md delete mode 100644 skills/retail/retail-product-search/references/troubleshooting.md delete mode 100644 skills/retail/retail-product-search/scripts/_paths.py delete mode 100644 skills/retail/retail-product-search/scripts/_setup_utils.py delete mode 100644 skills/retail/retail-product-search/scripts/agent.py delete mode 100755 skills/retail/retail-product-search/scripts/bootstrap.sh delete mode 100644 skills/retail/retail-product-search/scripts/cleanup.py delete mode 100644 skills/retail/retail-product-search/scripts/ingest_bigquery.py delete mode 100644 skills/retail/retail-product-search/scripts/ingest_vertex_search.py delete mode 100644 skills/retail/retail-product-search/scripts/retrievers.py delete mode 100644 skills/retail/retail-product-search/scripts/setup.py delete mode 100644 skills/retail/retail-product-search/scripts/validate_schema.py diff --git a/skills/retail/retail-product-search/EVAL.yaml b/skills/retail/retail-product-search/EVAL.yaml deleted file mode 100644 index 552f49928a..0000000000 --- a/skills/retail/retail-product-search/EVAL.yaml +++ /dev/null @@ -1,209 +0,0 @@ -# uses_catalog: true -cases: -- id: headphones_under_200 - prompt: "I'm looking for wireless headphones under $200, what do you have?" - metadata: - skill_name: "retail-product-search" - products: - - "Vertex AI Vector Search" - - "BigQuery" - rubric: - - "The response MUST return product results that include wireless headphones from the catalog." - - "The response MUST only include products priced under $200." - - "The response MUST include key product details: name, price, and description for each result." - - "The response MUST NOT hallucinate product names, brands, or prices not present in the catalog." - - "The response SHOULD use semantic search to match the user intent, not just keyword matching." - assertions: - - type: contains_product - field: name - value: Wireless Headphones - - type: price_under - value: 200 - - type: tool_called - tool: retrieve_docs - - type: no_hallucination - forbidden: - - Bose - - AirPods - - Sennheiser - -- id: cheapest_item - prompt: "What is the cheapest product you have available right now?" - metadata: - skill_name: "retail-product-search" - products: - - "Vertex AI Vector Search" - - "BigQuery" - rubric: - - "The response MUST identify the lowest-priced product in the catalog." - - "The response MUST include the exact price as a dollar amount." - - "The response MUST include the product name and a brief description." - - "The response MUST NOT fabricate prices or products not present in the catalog." - assertions: - - type: tool_called - tool: retrieve_docs - - type: mentions_price - value: true - - type: response_not_empty - -- id: keyboard_search - prompt: "I need a mechanical keyboard for coding, what do you recommend?" - metadata: - skill_name: "retail-product-search" - products: - - "Vertex AI Vector Search" - - "BigQuery" - rubric: - - "The response MUST return at least one mechanical keyboard product from the catalog." - - "The response MUST include the brand and price for each returned product." - - "The response MUST provide details relevant to coding use (e.g., key switch type, ergonomics)." - - "The response MUST NOT recommend products outside the catalog inventory." - - "The response SHOULD explain why the recommended product suits coding workflows." - assertions: - - type: contains_product - field: name - value: Mechanical Keyboard - - type: mentions_field - field: brand - - type: mentions_field - field: price - - type: tool_called - tool: retrieve_docs - -- id: category_filter - prompt: "Show me all audio products you carry." - metadata: - skill_name: "retail-product-search" - products: - - "Vertex AI Vector Search" - - "BigQuery" - rubric: - - "The response MUST return products categorized under Audio from the catalog." - - "The response MUST include the name and price for each audio product listed." - - "The response MUST NOT include products from non-audio categories in the results." - - "The response MUST NOT hallucinate audio products not present in the catalog (e.g., speaker systems, turntables)." - assertions: - - type: tool_called - tool: retrieve_docs - - type: contains_product - field: category - value: Audio - - type: no_hallucination - forbidden: - - speaker system - - turntable - -- id: brand_search - prompt: "Do you have any Sony products in your store?" - metadata: - skill_name: "retail-product-search" - products: - - "Vertex AI Vector Search" - - "BigQuery" - rubric: - - "The response MUST return products from the Sony brand present in the catalog." - - "The response MUST include the product name, category, and price for each Sony item." - - "The response MUST NOT fabricate Sony products that are not in the catalog." - - "The response MUST NOT include products from other brands unless explicitly relevant for comparison." - assertions: - - type: tool_called - tool: retrieve_docs - - type: contains_product - field: brand - value: Sony - - type: response_not_empty - -- id: no_results_handling - prompt: "Do you have any cars for sale?" - metadata: - skill_name: "retail-product-search" - products: - - "Vertex AI Vector Search" - - "BigQuery" - rubric: - - "The response MUST clearly state that no matching products were found in the catalog." - - "The response MUST NOT fabricate or hallucinate car products." - - "The response SHOULD suggest alternative actions (e.g., browsing other categories or refining the query)." - - "The response MUST remain helpful and polite even when no results are available." - assertions: - - type: tool_called - tool: retrieve_docs - - type: graceful_no_results - -- id: comparison_query - prompt: "Can you compare the keyboard and the headphones you have?" - metadata: - skill_name: "retail-product-search" - products: - - "Vertex AI Vector Search" - - "BigQuery" - rubric: - - "The response MUST include a side-by-side comparison covering at least name, price, and category." - - "The response MUST reference actual products from the catalog for both the keyboard and headphones." - - "The response MUST NOT invent specifications or features not listed in the catalog." - - "The response SHOULD highlight key differences that help the user make a purchase decision." - assertions: - - type: tool_called - tool: retrieve_docs - - type: mentions_field - field: price - - type: response_min_length - value: 50 - -- id: price_range - prompt: "Show me products between $50 and $150." - metadata: - skill_name: "retail-product-search" - products: - - "Vertex AI Vector Search" - - "BigQuery" - rubric: - - "The response MUST only include products with prices between $50 and $150 inclusive." - - "The response MUST display the price for each returned product." - - "The response MUST NOT return products priced outside the specified range." - - "The response MUST NOT hallucinate products or prices not present in the catalog." - assertions: - - type: tool_called - tool: retrieve_docs - - type: mentions_price - value: true - - type: response_not_empty - -- id: vague_query_handling - prompt: "I need something for my desk, any suggestions?" - metadata: - skill_name: "retail-product-search" - products: - - "Vertex AI Vector Search" - - "BigQuery" - rubric: - - "The response MUST return relevant desk-related products from the catalog (e.g., keyboards, monitors)." - - "The response MUST include product name and price for each suggestion." - - "The response MUST NOT hallucinate products not in the catalog." - - "The response SHOULD interpret the vague query using semantic search rather than exact keyword matching." - assertions: - - type: tool_called - tool: retrieve_docs - - type: response_not_empty - - type: response_min_length - value: 30 - -- id: skill_covers_vector_search - prompt: "How does the product search system work under the hood?" - metadata: - skill_name: "retail-product-search" - products: - - "Vertex AI Vector Search" - - "BigQuery" - - "Cloud Storage" - rubric: - - "The response MUST mention vector search as part of the product retrieval mechanism." - - "The response MUST reference BigQuery as the data storage or query layer." - - "The response MUST NOT claim the system uses technologies or services it does not employ." - - "The response SHOULD explain how semantic embeddings power product matching." - assertions: - - type: skill_covers - topic: vector search - - type: skill_covers - topic: bigquery - - type: response_not_empty diff --git a/skills/retail/retail-product-search/README.md b/skills/retail/retail-product-search/README.md deleted file mode 100644 index ae730f31ab..0000000000 --- a/skills/retail/retail-product-search/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# Retail Product Search - -Semantic product search agent on Google Cloud (Vertex AI Vector Search, -BigQuery, embeddings). Use to build e-commerce search, catalog discovery, or -shopping assistant agents. - -## Install - -```bash -npx skills add tanvisinghal-0105/solution_skills --skill retail-product-search -``` - -Hosts that support the skills spec (Claude Code, Gemini CLI, Codex, ...) -auto-detect and drop `SKILL.md` into their skills directory -(e.g. `~/.claude/skills/`, `~/.agents/skills/`). - -## Prerequisites - -- Python 3.10+ -- [`gcloud` CLI](https://cloud.google.com/sdk/docs/install) with ADC - configured (`gcloud auth application-default login`) -- A GCP project with billing enabled and BigQuery + Vertex AI APIs on: - ```bash - gcloud services enable bigquery.googleapis.com aiplatform.googleapis.com - ``` - -## Run - -In a fresh workspace, launch your AI coding agent and trigger the skill: - -``` -Use the retail-product-search skill to set up a product search agent on Google Cloud. -``` - -The agent walks Q-MODE (2 questions Quick / 4 questions Full), runs -`scripts/bootstrap.sh` to create the venv, then `scripts/setup.py` to validate -the catalog, ingest to BigQuery, create the Vector Search collection, and -launch the ADK web UI. - -## Use your own catalog - -Choose Full setup and point Q-B at a local CSV or `gs://` URI. CSV needs -`product_id`, `name`, `price`, `description`. Optional: `category`, `brand`, -`image_url`, `rating`, `stock`. - -## Cleanup - -In the agent chat: - -``` -clean up the GCP resources -``` - -Runs `cleanup.py --confirm` to delete the BigQuery dataset and Vector Search -collection. - -## Troubleshooting - -| Error | Fix | -|---|---| -| `MethodNotImplemented: 501` from Vector Search | `VECTOR_SEARCH_COLLECTION` has a newline. Re-export on one line | -| `ModuleNotFoundError: google.adk` | `bash -c "pip install -e '${SKILL_DIR}[adk]'"` (the `bash -c` matters in zsh) | -| `Package requires Python: 3.9.X` | Recreate venv with `python3.12 -m venv .venv` | -| `BILLING_DISABLED` / `PERMISSION_DENIED` | GCP project setup — see [references/troubleshooting.md](references/troubleshooting.md) | - -Full table: [references/troubleshooting.md](references/troubleshooting.md). - -## What gets built - -- BigQuery dataset `retail_skill_products.products` -- Vertex AI Vector Search collection `retail-skill-products-collection` in - `us-central1`, with auto-embeddings via `gemini-embedding-001` -- Workspace venv with the skill installed editable + a `design-spec.md` - -The skill's source code stays in the install directory — nothing is copied -to your workspace. - -## License - -Apache 2.0 diff --git a/skills/retail/retail-product-search/SKILL.md b/skills/retail/retail-product-search/SKILL.md deleted file mode 100644 index 8c7718dc1f..0000000000 --- a/skills/retail/retail-product-search/SKILL.md +++ /dev/null @@ -1,271 +0,0 @@ ---- -name: retail-product-search -description: >- - Creates product search agents with semantic search and RAG on Google Cloud - (Vertex AI Vector Search, BigQuery, embeddings). Use when the user wants to - "build a product search agent", "create an e-commerce search", "make a - shopping assistant", "set up semantic catalog discovery", "ingest products - into Vector Search", or "deploy a retail RAG agent". Handles the full - pipeline: catalog data ingestion to BigQuery, Vertex AI Vector Search - collection setup, ADK agent scaffolding, evaluation, and Cloud Run deployment. -metadata: - author: Google - license: Apache-2.0 - version: 0.1.0 ---- - -# Product Search Agent - -Creates product search agents with semantic search and RAG on Google Cloud. - -## STOP — Q-MODE FIRST - -**If a catalog is already loaded** (system context says "DEPLOYED search agent" -or provides a `` block), skip Q-MODE and answer product queries -directly using the catalog. - -**Otherwise**, your first message MUST be exactly this: - -``` -[skill: retail-product-search] active. -Q-MODE: Pick a setup mode? [default: 1] - 1. Quick start -- 2 questions, smart defaults, ~60s. Best for demos and first-timers. - 2. Full setup -- 4 questions, ~2 min. Best for real builds. -``` - -Then stop and wait. Accept `1`, `quick`, empty/Enter (Quick), or `2`, `full` (Full). - -## Execution Rules - -1. Q-MODE first. No preamble, no plan-proposing. -2. One question at a time. Show `[default: ...]`. Empty input = default. -3. Save answers to `./design-spec.md` in the workspace as you collect them. -4. After interview, run `scripts/setup.py` (see Workspace Setup below). -5. User can say "configure more" mid-Quick-Start to switch to Full. - -## Workspace Setup - -The skill has two locations: -- **Install dir** — where SKILL.md and scripts live (varies by host) -- **Workspace** — the agent's cwd; design-spec.md, .venv, and per-run state live here - -By the end of this section the workspace must have `.venv/` (with the skill -installed editable + `[adk]` extras), `design-spec.md`, and `SKILL_DIR` -exported in the shell. - -Run this as ONE shell command — splitting it across tool calls loses state: - -```bash -SKILL_DIR=$(for d in ~/.claude/skills ~/.agents/skills ~/.gemini/skills ~/.cursor/skills; do - [ -f "$d/retail-product-search/SKILL.md" ] && echo "$d/retail-product-search" && break -done) -bash "$SKILL_DIR/scripts/bootstrap.sh" -``` - -`bootstrap.sh` finds a Python 3.10+ interpreter (with absolute-path fallback -for sandboxed shells), creates `.venv`, installs the skill editable, and -copies `design-spec.md` into the workspace. - -All scripts run from the install dir against the workspace config. **Use -`.venv/bin/python`, not bare `python`** — bare `python` may resolve to a -Python without the skill's editable install on sys.path. - -```bash -.venv/bin/python "$SKILL_DIR/scripts/setup.py" --config ./design-spec.md -.venv/bin/python "$SKILL_DIR/scripts/cleanup.py" --config ./design-spec.md --confirm -``` - -Details in [references/install-paths.md](references/install-paths.md). - -## Skill Dependencies - -This skill works in conjunction with the following core Google Cloud skills: -- `bigquery-basics` (for database configuration guidelines) -- `gemini-api` (for Vertex AI API / Google Gen AI SDK best practices) - -Verify if these skills are installed in your active skills directory. If they are missing, recommend the developer to install them by running: -`npx skills add google/skills --skill bigquery-basics gemini-api` - -## Mode 1: Quick Start (2 questions) - -| Q | Question | Default | -|---|---|---| -| Q-A | GCP project ID? | `$GOOGLE_CLOUD_PROJECT` or `gcloud config get-value project` | -| Q-B | Where's your product data? | `assets/sample-products.csv` (bundled) | - -Accepted for Q-B: empty / `default` (bundled), `/path/to/file.csv`, or `gs://...`. - -Silent defaults: Extended fields, `us-central1`. - -After Q-A and Q-B, do this automatically (don't ask the user to copy/paste). -**Run these steps SEQUENTIALLY — do not parallelize.** Steps 2-3 modify the -file bootstrap copies in step 1; running them concurrently is a race. - -1. **Run bootstrap first and wait for completion.** `bash "$SKILL_DIR/scripts/bootstrap.sh"` - copies the YAML-frontmatter design-spec template into the workspace at - `./design-spec.md`. Do NOT touch `./design-spec.md` until bootstrap exits. -2. **Mutate the existing `./design-spec.md`** — do NOT rewrite it from scratch. - `setup.py` parses YAML frontmatter via `_setup_utils.py`. A Markdown-only - file fails with `'NoneType' object has no attribute 'get'`. Use Edit / sed - to replace specific lines: - - `gcp_project_id: ""` → `gcp_project_id: ""` - - `data_source: assets/sample-products.csv` → `data_source: ` (only if user gave a non-default) -3. Say: "Taking defaults for the rest. Running setup — this takes 2-5 min to - create a BigQuery dataset and Vector Search collection. Say 'configure - more' to switch to Full setup." -4. Run `.venv/bin/python "$SKILL_DIR/scripts/setup.py" --config ./design-spec.md` -5. Stream output. On non-zero exit, surface the error and check - [references/troubleshooting.md](references/troubleshooting.md) -6. On success, set `VECTOR_SEARCH_COLLECTION` and proceed to Test - -## Mode 2: Full Setup - -Adds two more questions: product fields level and GCP region. - -| Q | Question | Default | Notes | -|---|---|---|---| -| Q-fields | Product fields level | `Extended` | `Basic` / `Standard` / `Extended` / `Full`. Match this to your CSV's columns. Don't offer "Custom" — `validate_schema.py` rejects it. | -| Q-region | GCP region | `us-central1` | **Only confirmed-working region for Vector Search 2.0.** Other regions return `501 MethodNotImplemented`. | - -Otherwise identical to Quick Start. - -## When to Use - -- E-commerce product search, shopping assistants, semantic catalog discovery - -Don't use for generic document search, simple keyword search, or non-retail. - -## Project Tree - -``` -retail-product-search/ - assets/ - design-spec.md # Source of truth -- filled by Q-MODE - sample-products.csv # Bundled 5-product demo catalog - references/ # Deep-dive docs (load on demand) - scripts/ - agent.py # Reference ADK agent - retrievers.py # Vector Search retrieval logic - setup.py # Pipeline driver (reads design-spec.md) - bootstrap.sh # Workspace bootstrap (called from Workspace Setup) - validate_schema.py - ingest_bigquery.py - ingest_vertex_search.py - cleanup.py -``` - -Customize: rewrite `scripts/agent.py` (see -[references/agent-example.md](references/agent-example.md)) and -`scripts/retrievers.py` with your product-specific fields. - -## Test - -After `setup.py` succeeds, set the collection env var (one line, no newlines): - -```bash -export VECTOR_SEARCH_COLLECTION="projects/$GOOGLE_CLOUD_PROJECT/locations/us-central1/collections/retail-skill-products-collection" -``` - -Then either: - -**With ADK** (interactive UI): -```bash -# Use the WORKSPACE VENV's adk (not bare `adk`) so the skill's editable -# install is on sys.path. Bare `adk` may resolve to a global Python (pyenv, -# brew, etc.) whose ADK can't find the skill and reports an empty app list. -.venv/bin/adk web "$SKILL_DIR/scripts" --port 8765 -``` -Open http://127.0.0.1:8765, click `scripts`, query. - -⚠️ Two things must be right: -- **Point `adk web` at `$SKILL_DIR/scripts`, not at `.`** — agent code lives - in the install dir, not the workspace. `adk web .` fails with "No agents - found in current folder". -- **Use `.venv/bin/adk`, not bare `adk`** — bare `adk` may launch the wrong - Python and silently fail to load the agent (UI loads, but `/list-apps` - returns `[]` and queries time out). - -**Without ADK** (direct smoke test): -```bash -.venv/bin/python -c "from scripts.retrievers import search; print(search('laptop for video editing', top_k=3))" -``` - -Semantic-only retrieval — no structured filters on price, stock, or rating. -For demo queries and how to add structured filtering, see -[references/architecture.md](references/architecture.md). - -## Evaluate - -```bash -cd -./vs eval retail-product-search --project-id $PROJECT -``` - -`EVAL.yaml` declares `rubric` (LLM-as-judge) + `assertions` (deterministic -checks). Target: 80%+ passing. - -## Deploy - -**Never deploy without explicit human approval.** - -Cloud Run service account needs `roles/bigquery.dataViewer` on the dataset and -`roles/aiplatform.user` on the project. Deploy via `gcloud run deploy` or your -org's existing tooling. - -## Gotchas - -- **No results**: collection empty or `VECTOR_SEARCH_COLLECTION` not set -- **Slow search**: check region and `top_k` -- **No structured filters**: `search()` is pure semantic similarity. Price / - stock / currency filters happen client-side in the LLM, so results may - include items outside the constraint. Don't promise hard filters -- **ADK session memory**: if the retriever errored in earlier turns, the - model "learns" the tool is broken. Click "New Session" in `adk web` after - fixing the underlying issue - -## Troubleshooting - -Most-common failures inline; full table in -[references/troubleshooting.md](references/troubleshooting.md). - -| Error | Fix | -|---|---| -| `setup.py` exits with `'NoneType' object has no attribute 'get'` | `design-spec.md` was written as plain Markdown instead of mutating the YAML-frontmatter template bootstrap copied | Wait for bootstrap to finish, then **edit** (not rewrite) `./design-spec.md` — only change the field values inside the existing `---...---` frontmatter | -| `adk web` starts but `/list-apps` returns `[]` / browser shows "No agents found" | Bare `adk` resolved to a global Python that lacks the editable install. Kill it and restart with `.venv/bin/adk web "$SKILL_DIR/scripts" --port 8765` | -| `MethodNotImplemented: 501` from Vector Search | `VECTOR_SEARCH_COLLECTION` has a newline. Re-export on one line | -| `ModuleNotFoundError: google.adk` | `bash -c "pip install -e '${SKILL_DIR}[adk]'"` (the `bash -c` matters in zsh) | -| `Package requires Python: 3.9.X` | venv used system Python 3.9. Recreate with `python3.12 -m venv .venv` | -| `BILLING_DISABLED` / `PERMISSION_DENIED` / `API has not been used` | GCP project setup — see troubleshooting.md | - -## MCP Migration - -This skill uses `gcloud` CLI + Python SDKs (`google-genai`, -`google-cloud-bigquery`, `google-cloud-aiplatform`). Per -[Phase 2 Skills guidelines](https://github.com/google/skills), 1p skills -should prefer remote MCP tools when available. Migration map: - -| Service | Where | Future MCP | -|---|---|---| -| BigQuery | `ingest_bigquery.py`, `validate_schema.py` | BigQuery MCP | -| Vertex AI Vector Search | `ingest_vertex_search.py`, `setup.py` | Vertex AI MCP | -| Vertex AI Embeddings | `retrievers.py` | Vertex AI MCP | -| Cloud Run | `gcloud run deploy` | Cloud Run MCP | - -## Completion Checklist - -- [ ] Product fields level and data source confirmed -- [ ] Data ingestion ran; Vector Search populated -- [ ] `retrieve_docs` returns results in ADK web UI -- [ ] Evaluation passes success criteria -- [ ] Deployed (if beyond prototype) - -## References - -Load on demand: - -- [references/install-paths.md](references/install-paths.md) — host install dirs, Python fallback, `bash -c` rationale -- [references/dependencies.md](references/dependencies.md) — pip deps and install quirks -- [references/architecture.md](references/architecture.md) — what retrieval does (semantic-only), demo queries, structured-filter strategies -- [references/troubleshooting.md](references/troubleshooting.md) — full error table -- [references/agent-example.md](references/agent-example.md) — how agent.py + retrievers.py fit together -- [references/ingestion-scripts.md](references/ingestion-scripts.md) — per-script CLI reference diff --git a/skills/retail/retail-product-search/assets/design-spec.md b/skills/retail/retail-product-search/assets/design-spec.md deleted file mode 100644 index 57542ed5f5..0000000000 --- a/skills/retail/retail-product-search/assets/design-spec.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -# Product Search Agent - Design Spec -# Fill in your values and use this as input for project generation. - -# --- Required --- -gcp_project_id: "" # REQUIRED: your GCP project ID - -# --- Data --- -data_source: assets/sample-products.csv # local CSV path or gs://bucket/path/products.csv -product_fields: Extended # Basic | Standard | Extended | Full - -# --- GCP --- -gcp_region: us-central1 # Vector Search 2.0 region (us-central1 is the only confirmed-working region today) -dataset_id: retail_skill_products -table_id: products - -# --- Optional warnings --- -catalog_size: "1K-50K" # only used to trigger a >500K Dataflow hint ---- - -# Product Search Agent - -## Overview - -This design spec captures the configuration decisions for a retail product search -agent built on Google Cloud. It is generated during the SKILL.md interview and -used by ingestion scripts and agent scaffolding. - -## How to Use - -1. Fill in `gcp_project_id` above (or let the coding agent do it conversationally). -2. Pass this file to any script: `python scripts/ingest_bigquery.py --config assets/design-spec.md`. -3. CLI args always override values from this file. - -## Design Decisions - -Document any non-obvious choices here so future contributors understand the "why". diff --git a/skills/retail/retail-product-search/assets/sample-products.csv b/skills/retail/retail-product-search/assets/sample-products.csv deleted file mode 100644 index f75d833758..0000000000 --- a/skills/retail/retail-product-search/assets/sample-products.csv +++ /dev/null @@ -1,6 +0,0 @@ -product_id,name,price,description,category,brand,image_url,rating,stock -prod-001,Wireless Headphones,199.99,Premium noise-cancelling over-ear headphones with 30-hour battery,Audio,Sony,,4.5,50 -prod-002,Laptop Stand,49.99,Aluminum adjustable laptop stand with cable management,Accessories,Generic,,4.2,100 -prod-003,USB-C Cable,19.99,6ft braided USB-C to USB-C cable with 100W charging,Cables,Anker,,4.7,200 -prod-004,Mechanical Keyboard,129.99,Compact 75% mechanical keyboard with hot-swappable switches,Peripherals,Keychron,,4.6,30 -prod-005,Monitor Light Bar,59.99,Screen-mounted LED light bar with adjustable color temperature,Lighting,BenQ,,4.4,75 diff --git a/skills/retail/retail-product-search/pyproject.toml b/skills/retail/retail-product-search/pyproject.toml deleted file mode 100644 index 8c129d94d0..0000000000 --- a/skills/retail/retail-product-search/pyproject.toml +++ /dev/null @@ -1,39 +0,0 @@ -[project] -name = "retail-product-search" -version = "0.1.0" -description = "Semantic product search agent using Vertex AI Vector Search and BigQuery" -requires-python = ">=3.10" -license = {text = "Apache-2.0"} - -dependencies = [ - "google-cloud-bigquery>=3.0", - "google-cloud-storage>=2.0", - "google-cloud-vectorsearch>=0.5,<1.0", # preview/beta -- pin to known-working range - "google-cloud-aiplatform>=1.30", - "google-genai>=1.0", - "pyyaml>=6.0", - "requests>=2.28", - "pyOpenSSL>=23.0", # mTLS during BigQuery / Vector Search auth in some envs -] - -[project.optional-dependencies] -adk = ["google-adk>=2.2.0"] # agent runtime for `adk web` and the ADK framework wrapper - -[tool.setuptools.packages.find] -include = ["scripts*"] - -[dependency-groups] -dev = [ - "pytest>=8.0", - "pytest-mock>=3.14", - "pytest-cov>=4.1", -] - -[tool.pytest.ini_options] -testpaths = ["tests/unit", "tests/integration"] -addopts = "-s -v" -log_cli = true -log_cli_level = "INFO" -markers = [ - "live: integration test that hits real GCP. Requires LIVE_EVAL=1 and a fresh GCP project to ingest into.", -] diff --git a/skills/retail/retail-product-search/references/agent-example.md b/skills/retail/retail-product-search/references/agent-example.md deleted file mode 100644 index 3fe74238f8..0000000000 --- a/skills/retail/retail-product-search/references/agent-example.md +++ /dev/null @@ -1,71 +0,0 @@ -# Example Agent Code - -The shipping implementation is in `scripts/agent.py` and `scripts/retrievers.py`. -Read those files directly -- this doc summarizes the shape so you know what -to look at when adapting it. - -## Files - -- **`scripts/agent.py`** -- defines `root_agent`, a single ADK `Agent` with one - tool (`retrieve_docs`) and a static instruction string focused on product - search. -- **`scripts/retrievers.py`** -- defines `search_collection`, the Vector Search - 2.0 query that backs `retrieve_docs`, plus a `search()` convenience - wrapper used by the Step 9 smoke test. - -## Shape - -``` -User query - | -root_agent (Gemini, ADK) - | instruction = "search the catalog for every query, present results..." - | -retrieve_docs(query) - | -search_collection(query, collection_path, top_k=10) - | -vectorsearch.DataObjectSearchServiceClient.search_data_objects( - SemanticSearch(search_text=query, search_field="text_embedding", - task_type="QUESTION_ANSWERING", top_k=10, - output_fields=[product_id, name, price, description, - category, brand, rating, stock]) -) - | -Formatted "Product 1: name, $price, by brand, rated X/5, " string -``` - -## Configuration - -The agent reads these env vars at runtime: - -| Variable | Default | Purpose | -|---|---|---| -| `GEMINI_MODEL` | `gemini-3.5-flash` | LLM used by `root_agent` | -| `GOOGLE_CLOUD_PROJECT` | from ADC | GCP project for Vertex AI | -| `GOOGLE_CLOUD_LOCATION` | `global` | LLM region | -| `VECTOR_SEARCH_LOCATION` | `us-central1` | Vector Search region (used to build the default collection path) | -| `VECTOR_SEARCH_COLLECTION` | `projects//locations//collections/retail-skill-products-collection` | Full collection resource path | - -`VECTOR_SEARCH_COLLECTION` must match the regex -`projects/

/locations//collections/` with no whitespace. A -newline embedded mid-path (common from multi-line shell pastes) causes the -SDK to return a confusing 501 -- `scripts/agent.py` validates this up front. - -## Extending - -To add a tool (e.g. cart actions, inventory checks, price-history lookups): - -1. Define a function in `scripts/agent.py` with a docstring describing when to - use it -- ADK feeds the docstring to the LLM as the tool description. -2. Append it to the `tools=[...]` list on `root_agent`. -3. Update the `instruction` string so the LLM knows when to pick the new - tool over `retrieve_docs`. - -To change the LLM, set `GEMINI_MODEL` -- no code changes needed. - -To change the retrieval contract (different fields, different `top_k`, -filtering), edit `search_collection` in `scripts/retrievers.py`. Note that VS -2.0 semantic search has no built-in structured filters; any -price/stock/category gating must happen in the agent's prompt or in a -client-side post-filter. diff --git a/skills/retail/retail-product-search/references/architecture.md b/skills/retail/retail-product-search/references/architecture.md deleted file mode 100644 index 7ff0de9a2f..0000000000 --- a/skills/retail/retail-product-search/references/architecture.md +++ /dev/null @@ -1,49 +0,0 @@ -# What the retrieval layer actually does - -Read this when demoing the skill or extending it — covers what's -semantic vs what isn't, and where structured filtering would have to -live. - -## Semantic search, not filtered search - -`search(query, top_k)` runs Vertex AI Vector Search semantic similarity -over the embedding fields configured in `design-spec.md` (default: -`name, description, category, brand`). **It does NOT apply structured -filters on price, currency, stock, or rating** -- those words in a -query are just text the embedder sees. - -Implications when you demo: - -- `"headphones under 100 EUR"` returns headphones ranked by semantic - match; any price filtering happens in the agent's LLM, not the - retriever, so results may include items above the threshold (the LLM - may then narrate "no matches" if it filters client-side). -- The bundled `sample-products.csv` has USD prices. If the user asks in - EUR, the agent silently treats them as equivalent. For a real - multi-currency demo, add a `price_eur` or `currency` column to the - catalog and prompt the agent to use it. - -## Test queries - -| Query | What you're testing | -|---|---| -| `"laptop for video editing"` | Pure semantic match (works as documented) | -| `"I need a gift"` | Vague-query clarification flow (agent prompt-driven) | -| `"Which one has the best battery life?"` | RAG-style follow-up | -| `"wireless headphones under $100"` | Semantic + LLM-side price filtering; agent will narrate the price filter even though the retriever doesn't apply it | - -## Where to add structured filtering - -If a real catalog needs `price < X` to actually constrain results -(not just narrate around them), there are three places to add it: - -1. **Inside the retriever** — pre-filter the BigQuery dataset - before embedding ingest, or post-filter the Vector Search results - in `scripts/retrievers.py:search()`. -2. **As a separate tool on the agent** — add a `filter_by_price` tool - alongside `retrieve_docs` and let the LLM compose them. -3. **In the agent's system prompt** — give the LLM filter rules and - trust it to apply them. Cheapest, least reliable. - -Option 1 is the right answer for production; options 2 and 3 are -useful for demos that need quick wins. diff --git a/skills/retail/retail-product-search/references/dependencies.md b/skills/retail/retail-product-search/references/dependencies.md deleted file mode 100644 index 22c38fddff..0000000000 --- a/skills/retail/retail-product-search/references/dependencies.md +++ /dev/null @@ -1,27 +0,0 @@ -# Dependencies - -`pip install -e .` from the install dir resolves everything from -`pyproject.toml`. You don't install these manually. - -## Required - -- `google-cloud-bigquery>=3.0` -- `google-cloud-storage>=2.0` -- `google-cloud-vectorsearch>=0.5,<1.0` (preview, pinned) -- `google-cloud-aiplatform>=1.30` -- `google-genai>=1.0` -- `pyyaml>=6.0`, `requests>=2.28`, `pyOpenSSL` (mTLS for BigQuery) - -## Optional `[adk]` extra - -- `google-adk>=2.2.0` — for the ADK web UI - -## Install quirks (handled by bootstrap.sh) - -1. **zsh expands `[adk]` as a glob** and silently drops extras. Workaround: - `bash -c "pip install -e '${SKILL_DIR}[adk]'"`. -2. **Agent shell tools reset state between calls.** Run the workspace setup - block as a single shell invocation. -3. **Stripped PATH** in sandboxed terminals hides brew/pyenv Python. The - bootstrap script falls back to absolute paths - (`/opt/homebrew/bin/python3.13`, `~/.pyenv/shims/python3.12`, etc.). diff --git a/skills/retail/retail-product-search/references/ingestion-scripts.md b/skills/retail/retail-product-search/references/ingestion-scripts.md deleted file mode 100644 index ad9b10e451..0000000000 --- a/skills/retail/retail-product-search/references/ingestion-scripts.md +++ /dev/null @@ -1,165 +0,0 @@ -# Product Search - Data Ingestion Scripts Reference - -Reference documentation for the scripts in `scripts/`. These are standalone -Python scripts that handle product data ingestion and vector search indexing. - -All scripts support `--config design-spec.md` to load defaults from your -project configuration. - ---- - -## Script 1: ingest_bigquery.py - -**Purpose**: Load product catalog from CSV/JSON into BigQuery -**Location**: `scripts/ingest_bigquery.py` - -### Usage - -```bash -# From GCS -python scripts/ingest_bigquery.py \ - --project-id my-project \ - --gcs-bucket my-project-products \ - --gcs-path products.csv - -# From local file -python scripts/ingest_bigquery.py \ - --project-id my-project \ - --local-file data/products.json \ - --format json - -# Using design-spec.md for defaults -python scripts/ingest_bigquery.py \ - --config design-spec.md \ - --local-file data/products.csv -``` - -### Schema - -The script uses a fixed schema matching the Extended product field level: - -| Field | Type | Mode | Notes | -|-------|------|------|-------| -| product_id | STRING | REQUIRED | Unique identifier | -| name | STRING | REQUIRED | Product name | -| price | FLOAT64 | REQUIRED | Price in configured currency | -| description | STRING | REQUIRED | Product description | -| category | STRING | NULLABLE | Product category | -| brand | STRING | NULLABLE | Brand name | -| image_url | STRING | NULLABLE | Product image URL | -| rating | FLOAT64 | NULLABLE | Rating (0-5) | -| stock | INT64 | NULLABLE | Stock quantity | - -To customize the schema (e.g. for Basic or Full field levels), edit -`REQUIRED_FIELDS`, `OPTIONAL_FIELDS`, and `SCHEMA` at the top of the script. - -### Validation - -The script validates each row before loading: -- Required fields must be present and non-empty -- `price` must be numeric -- `stock` must be an integer -- Invalid rows are skipped with warnings (not fatal) - ---- - -## Script 2: ingest_vertex_search.py - -**Purpose**: Create a Vector Search 2.0 Collection and ingest products -**Location**: `scripts/ingest_vertex_search.py` - -Uses Vector Search 2.0 Collections with auto-embeddings. No manual -embedding generation or GCS bucket needed -- the Collection's configured -embedding model generates embeddings automatically when data objects -are inserted. - -### Usage - -```bash -# Basic usage -python scripts/ingest_vertex_search.py \ - --project-id my-project \ - --collection-id retail-skill-products-collection - -# Using design-spec.md for defaults -python scripts/ingest_vertex_search.py --config design-spec.md -``` - -### Pipeline - -1. Fetch products from BigQuery (`retail_skill_products.products`) -2. Create Vector Search 2.0 Collection if it doesn't exist (with auto-embedding config) -3. Insert products as data objects one at a time, catching `AlreadyExists` for idempotent re-runs -4. VS 2.0 auto-generates embeddings from the configured text template - -### Configuration - -| Flag | Default | Config key | -|------|---------|------------| -| `--collection-id` | retail-skill-products-collection | `collection_id` | -| `--embedding-model` | gemini-embedding-001 | `embedding_model` | -| `--embedding-fields` | name,description,category,brand | `embedding_fields` | -| `--location` | us-central1 | `gcp_region` | - -### Output - -After ingestion, set the `VECTOR_SEARCH_COLLECTION` environment variable -in your agent to the collection path printed by the script: - -```bash -export VECTOR_SEARCH_COLLECTION="projects/PROJECT_ID/locations/us-central1/collections/retail-skill-products-collection" -``` - ---- - -## Script 3: validate_schema.py - -**Purpose**: Validate product data files before ingestion -**Location**: `scripts/validate_schema.py` - -### Usage - -```bash -# Validate CSV at Extended level (matches sample-products.csv) -python scripts/validate_schema.py \ - --file data/products.csv \ - --fields-level Extended - -# Validate JSON at Standard level (format auto-detected from .json/.jsonl suffix) -python scripts/validate_schema.py \ - --file data/products.json \ - --fields-level Standard -``` - -### Field Levels - -| Level | Required | Optional | -|-------|----------|----------| -| Basic | product_id, name, price, description | (none) | -| Standard | Basic | category, brand, image_url | -| Extended | Basic | Standard + rating, stock, manufacturer | -| Full | Basic | Extended + variants, tags, specifications, reviews | - ---- - -## Ingestion Order - -Run scripts in this order: - -1. `validate_schema.py` -- validate your data file -2. `ingest_bigquery.py` -- load to BigQuery -3. `ingest_vertex_search.py` -- create collection + ingest products - -**Always use these retail skill scripts** -- generic document ingestion -pipelines are incompatible with product catalogs. - ---- - -## Customizing the Schema - -To change the product schema: - -1. Edit `REQUIRED_FIELDS`, `OPTIONAL_FIELDS`, and `SCHEMA` in `ingest_bigquery.py` -2. Edit `DEFAULT_EMBEDDING_FIELDS` and `PRODUCT_DATA_FIELDS` in `ingest_vertex_search.py` -3. Update `validate_schema.py` field level definitions if needed -4. Update `design-spec.md` with the new field level and embedding fields diff --git a/skills/retail/retail-product-search/references/install-paths.md b/skills/retail/retail-product-search/references/install-paths.md deleted file mode 100644 index f2fca63ffd..0000000000 --- a/skills/retail/retail-product-search/references/install-paths.md +++ /dev/null @@ -1,44 +0,0 @@ -# Install Paths and Environment Quirks - -## Known install dirs per host - -| Host | Install dir | -|---|---| -| Claude Code | `~/.claude/skills//` | -| `npx skills add` (Codex, Gemini CLI, GitHub Copilot, others) | `~/.agents/skills//` | -| Gemini CLI standalone | `~/.gemini/skills//` | -| Cursor | `~/.cursor/skills//` | - -`bootstrap.sh` tries these paths in order. If your host installs elsewhere, -set `SKILL_DIR` manually: - -```bash -export SKILL_DIR=/path/to/your/install/dir -``` - -## Python interpreter fallback - -`bootstrap.sh` tries `command -v python3.{13,12,11,10}` first, then falls -back to absolute paths (`/opt/homebrew/bin/`, `/usr/local/bin/`, -`~/.pyenv/shims/`). The fallback handles sandboxed shells with stripped PATH. - -For conda/asdf/other layouts, set `PYTHON_BIN` and skip the loop. - -## Why `bash -c` wraps the pip command - -`pip install -e "$SKILL_DIR[adk]"` looks correct but breaks in zsh: zsh treats -`[adk]` as a glob character class, silently expands to empty, and the -`[adk]` extras are dropped. `bash -c "pip install -e '${SKILL_DIR}[adk]'"` -sidesteps this — pip runs under bash, which doesn't glob-expand `[adk]`. - -## Why the workspace setup must run as one shell command - -Agent shell tools reset cwd and clear variables between calls. If you set -`SKILL_DIR` in call 1 and run pip in call 2, `$SKILL_DIR` is empty in call 2 -and the install becomes `pip install -e [adk]` (invalid). - -## Description tuning for triggering - -If your skill is registered but the agent doesn't pick it for relevant -prompts, the `description` field is what hosts' routers match against. -Include concrete trigger phrases alongside the abstract description. diff --git a/skills/retail/retail-product-search/references/troubleshooting.md b/skills/retail/retail-product-search/references/troubleshooting.md deleted file mode 100644 index 3af752deb4..0000000000 --- a/skills/retail/retail-product-search/references/troubleshooting.md +++ /dev/null @@ -1,39 +0,0 @@ -# Troubleshooting - -Read this when `setup.py` or a per-step script exits non-zero. Match -the error message against the table below before guessing. - -## Error table - -| Error pattern | Likely cause | Fix | -|---|---|---| -| `BILLING_DISABLED` / `Billing must be enabled` | GCP project has no billing account | Link a billing account in Cloud Console, then re-run | -| `PERMISSION_DENIED` on BigQuery | Service account missing IAM | `gcloud projects add-iam-policy-binding $PROJECT_ID --member=user:$EMAIL --role=roles/bigquery.dataEditor` | -| `PERMISSION_DENIED` on Vertex AI | Missing `aiplatform.user` role | `gcloud projects add-iam-policy-binding $PROJECT_ID --member=user:$EMAIL --role=roles/aiplatform.user` | -| `API has not been used` / `is disabled` | Required API not enabled | `gcloud services enable bigquery.googleapis.com aiplatform.googleapis.com` | -| `Collection not found` at query time | Vector Search collection never created or wrong path | Re-run `ingest_vertex_search.py`; verify `VECTOR_SEARCH_COLLECTION` env var | -| `setup.py` exits with `AttributeError: 'NoneType' object has no attribute 'get'` from `_setup_utils.py` | `design-spec.md` is not in YAML-frontmatter form. The agent wrote it as plain Markdown instead of editing the YAML template bootstrap copied. `_setup_utils.load_config()` returns `None` for non-YAML files, then `setup.py` calls `.get(...)` on it | Wait for `bootstrap.sh` to finish before touching `./design-spec.md`. Then **edit** the file (don't rewrite) — modify only the values inside the existing `---...---` frontmatter block. The canonical template is at `/assets/design-spec.md` | -| `adk web` boots but `/list-apps` returns `[]`, browser shows "No agents found in current folder" | Bare `adk` resolved through `PATH` to a global Python (pyenv, brew, etc.) that doesn't have the skill's editable install on `sys.path`. The server runs but can't import the agent module | Kill it (`lsof -ti :8765 \| xargs kill -9`) and restart with the workspace venv's adk explicitly: `.venv/bin/adk web "$SKILL_DIR/scripts" --port 8765`. Same root cause if `python "$SKILL_DIR/scripts/setup.py"` succeeds but the smoke test fails — use `.venv/bin/python` instead of bare `python` | -| `VECTOR_SEARCH_COLLECTION is malformed` | env var has embedded whitespace (usually a newline from a wrapped paste) | Re-`export` the value on a single line; prefer the `$PROJECT_ID` short form | -| `MethodNotImplemented: 501` from Vector Search in `us-central1` | Almost always a malformed `VECTOR_SEARCH_COLLECTION` (see above) -- not a real region/service issue | Echo `$VECTOR_SEARCH_COLLECTION` and confirm it has no `\n` or whitespace | -| `ModuleNotFoundError: google.cloud.aiplatform` | Python deps not installed | `pip install -e .` from the skill dir, or `pip install google-cloud-aiplatform google-cloud-bigquery google-genai` | -| `Package 'retail-product-search' requires a different Python: 3.9.X not in '>=3.10'` | venv was created with system Python 3.9 (macOS default) | Recreate the venv with a 3.10+ interpreter: `rm -rf .venv && python3.12 -m venv .venv && source .venv/bin/activate && bash -c "pip install -e '${SKILL_DIR}[adk]'"` (substitute the Python version you have installed). Use the interpreter-detection loop from the Workspace Setup section to auto-find one. | -| `ModuleNotFoundError: google.adk` | Optional ADK dep missing | Use the standalone test path in Step 9, or `pip install -e '.[adk]'` from the skill dir (single-quote the `.[adk]` so zsh doesn't glob-expand the brackets) | -| `Schema mismatch` from `validate_schema.py` | CSV missing required fields (`product_id`, `name`, `price`) | Add the missing columns and re-run; or pass `--fields-level Standard` if your data is sparse | -| `Quota exceeded` on embedding requests | Free-tier embedding quota hit | Wait an hour, re-run with a smaller catalog slice, or request a quota increase. The script catches per-row failures and continues, so a partial run is fine to resume | -| Setup script hangs on Vector Search create | Collection creation is async and takes 2-5 min | Wait. If >10 min with no progress, check Cloud Console > Vertex AI > Vector Search for collection status | -| `projects//locations/...` in error path | `$GOOGLE_CLOUD_PROJECT` was empty when you ran the `export VECTOR_SEARCH_COLLECTION=...` -- substitution produced an empty project ID | `export GOOGLE_CLOUD_PROJECT=` first, then re-run the `export VECTOR_SEARCH_COLLECTION` line, then restart `adk web` | -| Agent in `adk web` keeps saying "tool encountered an error" / "I am still unable to search" even after env fixed | ADK session memory -- model learned the tool is broken from earlier turns | Click "New Session" in the ADK web UI; the fresh session will retry the tool | -| Agent ignores price/currency filters in user queries | Retriever is pure semantic similarity; no structured filters at retrieval time | Expected behavior. Document the limit in your demo; or add a `currency`/`price_eur` column to the catalog and prompt the agent to filter client-side | - -## ADK web UI: "tool encountered an error" loop - -If `adk web` returns "tool encountered an error" repeatedly even after -the env var is correct: ADK persists session history in -`scripts/.adk/session.db`. If the first few queries in a session got a -tool error (typically a malformed env var, since fixed), the model -"learns" the tool is broken and starts skipping it, apologizing without -retrying. - -Click "New Session" in the ADK web UI after fixing the underlying -issue -- a fresh session will retry the tool from scratch. diff --git a/skills/retail/retail-product-search/scripts/_paths.py b/skills/retail/retail-product-search/scripts/_paths.py deleted file mode 100644 index 1640839cdb..0000000000 --- a/skills/retail/retail-product-search/scripts/_paths.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Install-dir resolution for the retail-product-search skill. - -INSTALL_DIR is the directory containing SKILL.md -- i.e. the root of the -installed skill, regardless of where the user invokes a script from. -Computed from this file's own location at import time, so it stays stable -even when the user's cwd changes mid-session. - -Use this for any "where do I find my bundled assets?" lookup. -Do NOT use this for "where should I write user state?" -- user-mutable -state goes to the agent's current working directory. -""" - -from __future__ import annotations - -import pathlib - -INSTALL_DIR: pathlib.Path = pathlib.Path(__file__).resolve().parent.parent - -ASSETS_DIR: pathlib.Path = INSTALL_DIR / "assets" -SAMPLE_PRODUCTS_CSV: pathlib.Path = ASSETS_DIR / "sample-products.csv" -DEFAULT_DESIGN_SPEC: pathlib.Path = ASSETS_DIR / "design-spec.md" diff --git a/skills/retail/retail-product-search/scripts/_setup_utils.py b/skills/retail/retail-product-search/scripts/_setup_utils.py deleted file mode 100644 index c7847edecc..0000000000 --- a/skills/retail/retail-product-search/scripts/_setup_utils.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Shared utilities for skill setup scripts. - -Provides load_config() for reading YAML frontmatter from design-spec.md. - -Vendored locally so this skill ships as a self-contained package -- no -parent-directory _shared/ folder required to run setup. -""" - -import logging -import pathlib -from typing import Any - -import yaml # type: ignore[import-untyped] - -logger = logging.getLogger(__name__) - - -def load_config(config_path: str) -> dict[str, Any]: - """Load design-spec.md YAML frontmatter. - - Args: - config_path: Path to the design-spec markdown file. - - Returns: - Parsed YAML frontmatter as a dict. Empty dict if the file is - missing, has no frontmatter, or the frontmatter is empty. - """ - if not pathlib.Path(config_path).exists(): - return {} - text = pathlib.Path(config_path).read_text(encoding="utf-8") - if text.startswith("---"): - lines = text.split("\n") - yaml_lines = [] - in_frontmatter = False - for line in lines: - if line.strip() == "---": - if not in_frontmatter: - in_frontmatter = True - continue - break - if in_frontmatter: - yaml_lines.append(line) - yaml_text = "\n".join(yaml_lines) - return yaml.safe_load(yaml_text) or {} - return yaml.safe_load(text) or {} diff --git a/skills/retail/retail-product-search/scripts/agent.py b/skills/retail/retail-product-search/scripts/agent.py deleted file mode 100644 index b0585d94d8..0000000000 --- a/skills/retail/retail-product-search/scripts/agent.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""ADK agent definition for the retail product search skill. - -Exposes a :data:`root_agent` (and :data:`app`) that uses Vertex AI Vector -Search 2.0 for semantic product retrieval via the :func:`retrieve_docs` tool. -""" - -# pylint: disable=line-too-long -# (error messages and example URLs are intentionally long; breaking them hurts copy-paste UX.) - -import os -import re - -import google -from google.adk import agents -from google.adk import apps -from google.adk import models -import vertexai - -from scripts.retrievers import search_collection - -LLM_LOCATION = os.getenv("GOOGLE_CLOUD_LOCATION", "global") -VECTOR_SEARCH_LOCATION = os.getenv("VECTOR_SEARCH_LOCATION", "us-central1") -LLM = os.getenv("GEMINI_MODEL", "gemini-3.5-flash") - -_, _default_project = google.auth.default() -_project_id = os.getenv("GOOGLE_CLOUD_PROJECT", _default_project) -os.environ.setdefault("GOOGLE_CLOUD_PROJECT", _project_id) -os.environ.setdefault("GOOGLE_CLOUD_LOCATION", LLM_LOCATION) -os.environ.setdefault("GOOGLE_GENAI_USE_VERTEXAI", "True") -vertexai.init(project=_project_id, location=LLM_LOCATION) - - -_COLLECTION_PATH_RE = re.compile( - r"^projects/[^/\s]+/locations/[^/\s]+/collections/[^/\s]+$" -) - - -def _get_vector_search_collection() -> str: - """Return the Vector Search collection resource path. - - Reads from the ``VECTOR_SEARCH_COLLECTION`` env var when set; otherwise - builds the default path from ``GOOGLE_CLOUD_PROJECT`` and - ``VECTOR_SEARCH_LOCATION``. - - Returns: - The fully qualified Vector Search collection path. - - Raises: - ValueError: If ``VECTOR_SEARCH_COLLECTION`` is set to a malformed - path (e.g. contains a newline from a wrapped shell paste, which - silently causes the Vector Search SDK to return a 501). - """ - raw = os.getenv("VECTOR_SEARCH_COLLECTION") - if raw is None: - return ( - f"projects/{_project_id}/locations/{VECTOR_SEARCH_LOCATION}" - "/collections/retail-skill-products-collection" - ) - # Strip whitespace -- multi-line shell pastes can embed a newline mid-path, - # which the Vector Search SDK silently maps to a 501 from the wrong endpoint. - cleaned = raw.strip() - if not _COLLECTION_PATH_RE.match(cleaned): - raise ValueError( - "VECTOR_SEARCH_COLLECTION is malformed -- expected " - "'projects//locations//collections/' with no whitespace. " - f"Got: {raw!r}. " - "Tip: type the export on a single line, e.g. " - "export VECTOR_SEARCH_COLLECTION=" - '"projects/$GOOGLE_CLOUD_PROJECT/locations/us-central1/' - 'collections/retail-skill-products-collection"' - ) - return cleaned - - -def retrieve_docs(query: str) -> str: - """Search the product catalog using semantic similarity. - - Call this for every user query about products -- by name, by attributes - (price/brand/category), by intent ('something for my desk'), or by - comparison ('X vs Y'). Returns a formatted list of matching products - with name, price, brand, rating, and a description snippet. - - Args: - query: The product search query in natural language. - - Returns: - Formatted product list, or a "no matches" message. - """ - try: - return search_collection( - query=query, - collection_path=_get_vector_search_collection(), - ) - except Exception as e: # pylint: disable=broad-exception-caught - return ( - f"Calling retrieval tool with query:\n\n{query}\n\n" - f"raised the following error:\n\n{type(e)}: {e}" - ) - - -INSTRUCTION = """You are a retail product search assistant. -Use the retrieve_docs tool to search the product catalog for every user query. - -IMPORTANT: After receiving tool results, you MUST immediately present the results to the user. -Do not stop after saying "let me search" -- always continue to present the full results. - -When presenting search results: -- Say each product's name, price, brand, and a short description. -- Keep it conversational and concise. -- Never make up products -- only mention products returned by the tool. -- If no products match, say so and suggest broadening the search.""" - - -root_agent = agents.Agent( - name="root_agent", - model=models.Gemini(model=LLM), - instruction=INSTRUCTION, - tools=[retrieve_docs], -) - -app = apps.App( - root_agent=root_agent, - # ADK's `adk web` auto-names the app from the agent module's parent - # directory. The agent lives at scripts/agent.py, so ADK names it - # "scripts". The App name MUST match the auto-discovered name or - # session creation fails with "Session not found: . The runner - # is configured with app name '', but the root agent was loaded - # from '/.../scripts'." - name="scripts", -) diff --git a/skills/retail/retail-product-search/scripts/bootstrap.sh b/skills/retail/retail-product-search/scripts/bootstrap.sh deleted file mode 100755 index 29a14bfc90..0000000000 --- a/skills/retail/retail-product-search/scripts/bootstrap.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Workspace bootstrap for retail-product-search. -# -# Discovers the skill's install dir, finds a Python 3.10+ interpreter, -# creates .venv in the current directory, installs the skill editable -# with the [adk] extras, and copies design-spec.md into the workspace. -# -# Run from the workspace directory as a single shell invocation: -# bash /path/to/scripts/bootstrap.sh -# -# All defensive logic (interpreter fallback, bash -c around pip, -# stripped-PATH workarounds) is documented in -# references/install-paths.md. - -set -e - -# 1. Locate the install dir. Try common spec-compliant locations. -SKILL_DIR="" -for candidate in \ - ~/.claude/skills/retail-product-search \ - ~/.agents/skills/retail-product-search \ - ~/.gemini/skills/retail-product-search \ - ~/.cursor/skills/retail-product-search; do - if [ -f "$candidate/SKILL.md" ]; then - SKILL_DIR="$candidate" - break - fi -done - -# Bootstrap may itself be invoked via absolute path; if so, derive -# SKILL_DIR from $0 as a final fallback. -if [ -z "$SKILL_DIR" ]; then - script_dir="$(cd "$(dirname "$0")" && pwd)" - if [ -f "$script_dir/../SKILL.md" ]; then - SKILL_DIR="$(cd "$script_dir/.." && pwd)" - fi -fi - -if [ -z "$SKILL_DIR" ]; then - echo "ERROR: retail-product-search skill not installed. Install it first." >&2 - exit 1 -fi - -# 2. Pick a Python 3.10+ interpreter. -# Try PATH lookup first; fall back to absolute paths for sandboxed -# shells that launch with a stripped PATH. -PYTHON_BIN="" -for py in python3.13 python3.12 python3.11 python3.10 python3; do - if command -v "$py" >/dev/null 2>&1; then - ver=$("$py" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null) - case "$ver" in 3.10|3.11|3.12|3.13) PYTHON_BIN="$py"; break ;; esac - fi -done - -if [ -z "$PYTHON_BIN" ]; then - for path in \ - /opt/homebrew/bin/python3.13 \ - /opt/homebrew/bin/python3.12 \ - /opt/homebrew/bin/python3.11 \ - /opt/homebrew/bin/python3.10 \ - /usr/local/bin/python3.13 \ - /usr/local/bin/python3.12 \ - /usr/local/bin/python3.11 \ - /usr/local/bin/python3.10 \ - "$HOME/.pyenv/shims/python3.13" \ - "$HOME/.pyenv/shims/python3.12" \ - "$HOME/.pyenv/shims/python3.11" \ - "$HOME/.pyenv/shims/python3.10"; do - if [ -x "$path" ]; then - ver=$("$path" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null) - case "$ver" in 3.10|3.11|3.12|3.13) PYTHON_BIN="$path"; break ;; esac - fi - done -fi - -if [ -z "$PYTHON_BIN" ]; then - echo "ERROR: need Python 3.10+. Install one (brew install python@3.12) and retry." >&2 - exit 1 -fi - -echo "Using PYTHON_BIN=$PYTHON_BIN" -echo "Using SKILL_DIR=$SKILL_DIR" - -# 3. Create venv, activate, install the skill with extras. -# bash -c around pip ensures the [adk] extras aren't glob-expanded by zsh. -if [ ! -d .venv ]; then - "$PYTHON_BIN" -m venv .venv -fi -source .venv/bin/activate -bash -c "pip install -e '${SKILL_DIR}[adk]'" - -# 4. Copy the design-spec template into the workspace. -if [ ! -f ./design-spec.md ]; then - cp "$SKILL_DIR/assets/design-spec.md" ./design-spec.md -fi - -echo "" -echo "READY" -echo " SKILL_DIR=$SKILL_DIR" -echo " workspace=$(pwd)" -echo " venv=$(pwd)/.venv" -echo " design-spec=$(pwd)/design-spec.md" diff --git a/skills/retail/retail-product-search/scripts/cleanup.py b/skills/retail/retail-product-search/scripts/cleanup.py deleted file mode 100644 index e9c3466fee..0000000000 --- a/skills/retail/retail-product-search/scripts/cleanup.py +++ /dev/null @@ -1,533 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -r"""Clean up all GCP resources created by the product search agent. - -Deletes BigQuery datasets, Vector Search 2.0 collections (and legacy -v1 MatchingEngineIndex if present), and Cloud Run services. - -Usage: - # Dry run (show what would be deleted, don't delete) - python scripts/cleanup.py --config design-spec.md --dry-run - - # Delete everything - python scripts/cleanup.py --config design-spec.md --confirm - - # Delete only specific resources - python scripts/cleanup.py --config design-spec.md --confirm \\ - --only bigquery,vectorsearch -""" - -# pylint: disable=line-too-long -# (error messages with resource paths and gcloud commands are intentionally long.) - -import argparse -import logging -import pathlib -import subprocess -import sys -from typing import Any - -from _setup_utils import load_config # noqa: E402 # pylint: disable=wrong-import-position -from google.api_core import exceptions -from google.cloud import aiplatform -from google.cloud import bigquery -from google.cloud import vectorsearch - -# Allow imports from the script's own directory before pip install -e is run. -sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -ALL_RESOURCE_TYPES = ["bigquery", "vectorsearch", "cloudrun"] -# Quick Start only creates BigQuery + Vector Search. Cloud Run is only used by -# the deploy extension, so we skip it by default to avoid slow `gcloud describe` -# roundtrips. Users who deployed to Cloud Run can opt in with --only cloudrun. -DEFAULT_RESOURCE_TYPES = ["bigquery", "vectorsearch"] - -# Bound how long we'll wait for a single `gcloud describe` to answer. Without -# this, an unreachable network / wrong project can hang cleanup indefinitely. -GCLOUD_DESCRIBE_TIMEOUT_SEC = 15 - - -def delete_bigquery(project_id: str, dataset_id: str, dry_run: bool) -> bool: - """Delete a BigQuery dataset and all of its tables. - - Args: - project_id: GCP project ID. - dataset_id: Bare dataset name (no project prefix). - dry_run: If True, log what would be deleted but don't delete. - - Returns: - True on successful delete or when the dataset is already absent. - False if the delete call raised an unexpected exception. - """ - client = bigquery.Client(project=project_id) - dataset_ref = f"{project_id}.{dataset_id}" - - try: - client.get_dataset(dataset_ref) - except Exception: # pylint: disable=broad-exception-caught - logger.info("BigQuery dataset %s does not exist, skipping", dataset_ref) - return True - - if dry_run: - logger.info( - "[DRY RUN] Would delete BigQuery dataset: %s (and all tables)", dataset_ref - ) - return True - - try: - client.delete_dataset(dataset_ref, delete_contents=True, not_found_ok=True) - logger.info("Deleted BigQuery dataset: %s", dataset_ref) - return True - except Exception as e: # pylint: disable=broad-exception-caught - logger.error("Failed to delete BigQuery dataset %s: %s", dataset_ref, e) - return False - - -def _drain_data_objects(collection_path: str) -> int: - """Delete all data objects inside a Vector Search collection. - - The API rejects collection deletion while data objects exist, so we - list them via semantic search (broad query) and batch-delete in chunks. - - Args: - collection_path: Fully qualified Vector Search collection path - (``projects/.../locations/.../collections/...``). - - Returns: - The total number of data objects deleted. - """ - search_client = vectorsearch.DataObjectSearchServiceClient() - data_client = vectorsearch.DataObjectServiceClient() - - # Broad-stroke search to enumerate IDs server-side. top_k is capped by the - # API; we loop until a search returns no new IDs (or we hit a sane ceiling). - seen: set[str] = set() - deleted_total = 0 - batch_size = 100 - max_iterations = ( - 50 # 50 * 100 = 5000 objects; raise if you have a bigger collection - ) - - for _ in range(max_iterations): - request = vectorsearch.SearchDataObjectsRequest( - parent=collection_path, - semantic_search=vectorsearch.SemanticSearch( - search_text="product", # broad query -- collection is product catalog - search_field="text_embedding", - task_type="QUESTION_ANSWERING", # required by the API; matches scripts/retrievers.py - top_k=batch_size, - output_fields=vectorsearch.OutputFields(data_fields=["product_id"]), - ), - ) - - batch_ids: list[str] = [] - for result in search_client.search_data_objects(request): - object_name = result.data_object.name - if object_name and object_name not in seen: - seen.add(object_name) - batch_ids.append(object_name) - - if not batch_ids: - break - - delete_requests = [ - vectorsearch.DeleteDataObjectRequest(name=name) for name in batch_ids - ] - data_client.batch_delete_data_objects( - request=vectorsearch.BatchDeleteDataObjectsRequest( - parent=collection_path, - requests=delete_requests, - ) - ) - deleted_total += len(batch_ids) - logger.info(" Drained %d data objects (%d total)", len(batch_ids), deleted_total) - - return deleted_total - - -def delete_vectorsearch_collection( - project_id: str, location: str, collection_id: str, dry_run: bool -) -> bool: - """Delete a Vector Search 2.0 Collection. - - The API requires the collection to be empty before deletion, so we - first drain all contained data objects. - - Args: - project_id: GCP project ID. - location: GCP region of the collection. - collection_id: Bare collection ID (no project/location prefix). - dry_run: If True, log what would be deleted but don't delete. - - Returns: - True on successful delete, when the collection is already absent, - or when Vector Search isn't available in this region. False if the - drain or delete call raised an unexpected exception. - """ - client = vectorsearch.VectorSearchServiceClient() - collection_name = ( - f"projects/{project_id}/locations/{location}/collections/{collection_id}" - ) - - try: - client.get_collection( - request=vectorsearch.GetCollectionRequest(name=collection_name) - ) - except exceptions.NotFound: - logger.info( - "Vector Search collection %s does not exist, skipping", collection_id - ) - return True - except exceptions.MethodNotImplemented: - # Vector Search 2.0 isn't available in this region. Nothing to delete - # because nothing could have been created here in the first place. - logger.info( - "Vector Search not available in %s; no collection to clean up.", location - ) - return True - - if dry_run: - logger.info( - "[DRY RUN] Would drain data objects and delete Vector Search collection: %s", - collection_name, - ) - return True - - try: - logger.info("Draining data objects from %s before deletion...", collection_id) - drained = _drain_data_objects(collection_name) - logger.info("Drained %d data object(s) from %s", drained, collection_id) - except Exception as e: # pylint: disable=broad-exception-caught - logger.error( - "Failed to drain data objects from %s: %s\n" - " Collection deletion will fail until it is empty. Try the manual path:\n" - " gcloud ai vector-search-collections delete %s " - "--region=%s --project=%s --force", - collection_id, - e, - collection_id, - location, - project_id, - ) - return False - - try: - operation = client.delete_collection( - request=vectorsearch.DeleteCollectionRequest(name=collection_name) - ) - operation.result() - logger.info("Deleted Vector Search collection: %s", collection_name) - return True - except Exception as e: # pylint: disable=broad-exception-caught - logger.error( - "Failed to delete Vector Search collection %s: %s\n" - " Data objects were drained, but the delete still failed. Possible causes:\n" - " - An async embedding/index operation is still in flight (wait 2-3 min and retry)\n" - " - Stragglers remain that the drain pass missed; verify in Cloud Console.\n" - " Manual cleanup:\n" - " gcloud ai vector-search-collections delete %s " - "--region=%s --project=%s --force", - collection_id, - e, - collection_id, - location, - project_id, - ) - return False - - -def delete_vectorsearch_v1_index( - project_id: str, location: str, index_name: str, dry_run: bool -) -> bool: - """Delete legacy Vector Search v1 MatchingEngineIndex (if any remain). - - Args: - project_id: GCP project ID. - location: GCP region of the legacy index. - index_name: Display name of the legacy MatchingEngineIndex. - dry_run: If True, log what would be deleted but don't delete. - - Returns: - True on success, or when there's no matching index to delete. - False if a delete call failed. - """ - aiplatform.init(project=project_id, location=location) - - indexes = aiplatform.MatchingEngineIndex.list(filter=f'display_name="{index_name}"') - - if not indexes: - return True - - ok = True - for index in indexes: - if dry_run: - logger.info( - "[DRY RUN] Would delete legacy Vector Search index: %s", - index.resource_name, - ) - continue - - try: - index.delete() # type: ignore[attr-defined] - logger.info("Deleted legacy Vector Search index: %s", index.resource_name) - except Exception as e: # pylint: disable=broad-exception-caught - logger.error( - "Failed to delete legacy index %s: %s", index.resource_name, e - ) - ok = False - - return ok - - -def delete_cloudrun( - project_id: str, location: str, service_name: str, dry_run: bool -) -> bool: - """Delete a Cloud Run service. - - Args: - project_id: GCP project ID. - location: GCP region of the service. - service_name: Cloud Run service name. - dry_run: If True, log what would be deleted but don't delete. - - Returns: - True on successful delete, when the service is already absent, or - when ``gcloud describe`` times out (treated as expected-absent). - False if the delete call raised an unexpected error. - """ - try: - result = subprocess.run( - [ - "gcloud", - "run", - "services", - "describe", - service_name, - "--region", - location, - "--project", - project_id, - "--format", - "value(name)", - ], - capture_output=True, - text=True, - timeout=GCLOUD_DESCRIBE_TIMEOUT_SEC, - check=False, - ) - except subprocess.TimeoutExpired: - logger.warning( - "gcloud timed out checking Cloud Run service %s after %ds. Skipping. " - "If you did deploy to Cloud Run, delete the service manually: " - "gcloud run services delete %s --region=%s --project=%s", - service_name, - GCLOUD_DESCRIBE_TIMEOUT_SEC, - service_name, - location, - project_id, - ) - return True - - if result.returncode != 0: - logger.info("Cloud Run service %s does not exist, skipping", service_name) - return True - - if dry_run: - logger.info("[DRY RUN] Would delete Cloud Run service: %s", service_name) - return True - - try: - subprocess.run( - [ - "gcloud", - "run", - "services", - "delete", - service_name, - "--region", - location, - "--project", - project_id, - "--quiet", - ], - check=True, - ) - logger.info("Deleted Cloud Run service: %s", service_name) - return True - except subprocess.CalledProcessError as e: - logger.error("Failed to delete Cloud Run service %s: %s", service_name, e) - return False - - -def cleanup( - config: dict[str, Any], - dry_run: bool, - only: list[str], - dataset_id: str = "retail_skill_products", -) -> bool: - """Run cleanup for all or selected resource types. - - Args: - config: Parsed design-spec config dict. - dry_run: If True, log what would be deleted but don't delete. - only: Resource types to delete; subset of - ``["bigquery", "vectorsearch", "cloudrun"]``. - dataset_id: BigQuery dataset name to delete. - - Returns: - True if every step (across the resource types in ``only``) - succeeded; False if any step returned failure. - """ - project_id = config.get("gcp_project_id", "") - if not project_id: - logger.error("gcp_project_id not set in config") - sys.exit(1) - - location = config.get("gcp_region", "us-central1") - project_name = config.get("project_name", "product-search") - collection_id = config.get("collection_id", "retail-skill-products-collection") - - mode = "[DRY RUN] " if dry_run else "" - logger.info("%sCleaning up resources for project: %s", mode, project_id) - logger.info("%sResource types: %s", mode, ", ".join(only)) - logger.info("%sBigQuery dataset: %s", mode, dataset_id) - logger.info("") - - failures: list[str] = [] - - def _track(label: str, success: bool) -> None: - if not success: - failures.append(label) - - if "bigquery" in only: - _track("bigquery", delete_bigquery(project_id, dataset_id, dry_run)) - - if "vectorsearch" in only: - _track( - "vectorsearch (collection)", - delete_vectorsearch_collection( - project_id, location, collection_id, dry_run - ), - ) - _track( - "vectorsearch (v1 index)", - delete_vectorsearch_v1_index( - project_id, location, "retail_skill_products_index", dry_run - ), - ) - - if "cloudrun" in only: - _track( - "cloudrun", - delete_cloudrun(project_id, location, project_name, dry_run), - ) - - logger.info("") - if dry_run: - logger.info("Dry run complete. No resources were deleted.") - logger.info("Run with --confirm to actually delete.") - return True - - if failures: - logger.error( - "Cleanup finished with errors. Failed steps: %s", ", ".join(failures) - ) - logger.error("See messages above for the manual cleanup commands.") - return False - - logger.info("Cleanup complete. All selected resources deleted.") - return True - - -def main(): - """Parse CLI arguments and run cleanup.""" - parser = argparse.ArgumentParser( - description="Clean up all GCP resources created by the product search agent" - ) - parser.add_argument("--config", required=True, help="Path to design-spec.md") - parser.add_argument( - "--confirm", - action="store_true", - help="Actually delete resources (without this flag, runs in dry-run mode)", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Show what would be deleted without deleting", - ) - parser.add_argument( - "--only", - default="", - help=( - f"Comma-separated resource types to delete. Valid: {','.join(ALL_RESOURCE_TYPES)}. " - f"Default if omitted: {','.join(DEFAULT_RESOURCE_TYPES)} " - "(Cloud Run excluded by default; opt in with --only cloudrun if you deployed to Cloud Run)." # noqa: E501 - ), - ) - parser.add_argument( - "--dataset-id", - default="retail_skill_products", - help="BigQuery dataset name to delete (default: retail_skill_products)", - ) - - args = parser.parse_args() - - config = load_config(args.config) - if not config: - logger.error("Config file not found or empty: %s", args.config) - sys.exit(1) - - # Honor dataset_id from design-spec.md unless overridden on the CLI. - # Otherwise per-case suffixes get ignored and the wrong dataset is "deleted" - # (i.e. cleanup looks for the default and reports a false success). - dataset_id = args.dataset_id - if dataset_id == "retail_skill_products" and config.get("dataset_id"): - dataset_id = config["dataset_id"] - - dry_run = not args.confirm or args.dry_run - - if args.only: - only = [r.strip() for r in args.only.split(",")] - invalid = [r for r in only if r not in ALL_RESOURCE_TYPES] - if invalid: - parser.error( - f"Invalid resource types: {invalid}. Valid: {ALL_RESOURCE_TYPES}" - ) - else: - only = DEFAULT_RESOURCE_TYPES - - if not dry_run: - project_id = config.get("gcp_project_id", "unknown") - print( - f"\nYou are about to permanently delete GCP resources in project: {project_id}" - ) - print(f"Resource types: {', '.join(only)}") - try: - answer = input("\nAre you sure? Type 'yes' to confirm: ").strip().lower() - except (EOFError, KeyboardInterrupt): - answer = "" - if answer != "yes": - print("Aborted. No resources were deleted.") - sys.exit(0) - - ok = cleanup(config, dry_run, only, dataset_id=dataset_id) - sys.exit(0 if ok else 1) - - -if __name__ == "__main__": - main() diff --git a/skills/retail/retail-product-search/scripts/ingest_bigquery.py b/skills/retail/retail-product-search/scripts/ingest_bigquery.py deleted file mode 100644 index 3d35a56e5d..0000000000 --- a/skills/retail/retail-product-search/scripts/ingest_bigquery.py +++ /dev/null @@ -1,467 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -r"""Ingest product catalog data into BigQuery. - -Supports CSV and JSON source formats from GCS or local files. -Validates products against the configured schema before loading. - -Usage: - python ingest_bigquery.py \\ - --project-id my-project \\ - --gcs-bucket my-project-products \\ - --gcs-path products.csv - - python ingest_bigquery.py \\ - --project-id my-project \\ - --local-file data/products.json \\ - --format json - - # Or use design-spec.md for defaults: - python ingest_bigquery.py --config design-spec.md \\ - --local-file data/products.csv -""" - -# pylint: disable=line-too-long -# (validation error messages and argparse help strings are intentionally long.) - -import argparse -import csv -import io -import json -import logging -import pathlib -import sys -from typing import Any - -from google.cloud import bigquery, storage - -# Allow imports from the script's own directory before pip install -e is run. -sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) -from _setup_utils import load_config # noqa: E402 # pylint: disable=wrong-import-position - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -# Configure these for your product schema. -# Basic: product_id, name, price, description -# Standard: + category, brand, image_url -# Extended: + rating, stock, manufacturer -# Full: + variants, tags, specifications, reviews -REQUIRED_FIELDS = ["product_id", "name", "price", "description"] -OPTIONAL_FIELDS = ["category", "brand", "image_url", "rating", "stock"] - -SCHEMA = [ - bigquery.SchemaField("product_id", "STRING", mode="REQUIRED"), - bigquery.SchemaField("name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("price", "FLOAT64", mode="REQUIRED"), - bigquery.SchemaField("description", "STRING", mode="REQUIRED"), - bigquery.SchemaField("category", "STRING", mode="NULLABLE"), - bigquery.SchemaField("brand", "STRING", mode="NULLABLE"), - bigquery.SchemaField("image_url", "STRING", mode="NULLABLE"), - bigquery.SchemaField("rating", "FLOAT64", mode="NULLABLE"), - bigquery.SchemaField("stock", "INT64", mode="NULLABLE"), -] - - -def validate_product(product: dict[str, Any], row_num: int) -> list[str]: - """Validate a single product record. - - Args: - product: One product as a dict from the source file. - row_num: 1-based row number for error messages. - - Returns: - List of error strings for this row (empty if valid). - """ - errors = [] - for field in REQUIRED_FIELDS: - if field not in product or not product[field]: - errors.append(f"Row {row_num}: missing required field '{field}'") - - if "price" in product and product["price"]: - try: - float(product["price"]) - except (ValueError, TypeError): - errors.append( - f"Row {row_num}: 'price' must be numeric, got '{product['price']}'" - ) - - if "stock" in product and product["stock"]: - try: - int(product["stock"]) - except (ValueError, TypeError): - errors.append( - f"Row {row_num}: 'stock' must be an integer, got '{product['stock']}'" - ) - - return errors - - -def convert_types(product: dict[str, Any]) -> dict[str, Any]: - """Convert string values to proper types for BigQuery. - - Args: - product: Raw product dict (typically with string-typed values - from CSV parsing). - - Returns: - New dict with ``price``/``rating`` cast to float, ``stock`` cast - to int, and string fields passed through. - """ - converted = {} - converted["product_id"] = product.get("product_id", "") - converted["name"] = product.get("name", "") - converted["description"] = product.get("description", "") - - if "price" in product and product["price"]: - converted["price"] = float(product["price"]) - - if "rating" in product and product["rating"]: - converted["rating"] = float(product["rating"]) - - if "stock" in product and product["stock"]: - converted["stock"] = int(product["stock"]) - - for field in ["category", "brand", "image_url"]: - if field in product: - converted[field] = product[field] - - return converted - - -def load_from_csv(source: str) -> list[dict[str, Any]]: - """Load products from CSV (GCS URI or local path). - - Args: - source: Either a ``gs://bucket/path.csv`` URI or a local filesystem - path to a CSV file. - - Returns: - List of validated, type-converted product dicts. - """ - if source.startswith("gs://"): - parts = source.replace("gs://", "").split("/", 1) - client = storage.Client() - blob = client.bucket(parts[0]).blob(parts[1]) - content = blob.download_as_text() - reader = csv.DictReader(content.splitlines()) - else: - with open(source, encoding="utf-8") as f: - reader = csv.DictReader(f) - return _validate_and_convert(list(reader)) - - return _validate_and_convert(list(reader)) - - -def load_from_json(source: str) -> list[dict[str, Any]]: - """Load products from JSON or JSONL (GCS URI or local path). - - Args: - source: Either a ``gs://bucket/path.json[l]`` URI or a local - filesystem path. JSONL is detected from the suffix. - - Returns: - List of validated, type-converted product dicts. - """ - if source.startswith("gs://"): - parts = source.replace("gs://", "").split("/", 1) - client = storage.Client() - blob = client.bucket(parts[0]).blob(parts[1]) - content = blob.download_as_text() - else: - content = pathlib.Path(source).read_text(encoding="utf-8") - - if source.endswith(".jsonl"): - raw = [ - json.loads(line) for line in content.strip().splitlines() if line.strip() - ] - else: - parsed = json.loads(content) - raw = parsed if isinstance(parsed, list) else parsed.get("products", []) - - return _validate_and_convert(raw) - - -def _validate_and_convert( - raw_products: list[dict[str, Any]], -) -> list[dict[str, Any]]: - """Validate and type-convert a list of product dicts. - - Logs a summary of skipped rows when validation produces errors. - - Args: - raw_products: Raw dicts straight from the source loader. - - Returns: - List of cleaned, type-converted product dicts (skipped rows are - excluded). - """ - products = [] - all_errors = [] - - for i, product in enumerate(raw_products, start=1): - errors = validate_product(product, i) - if errors: - all_errors.extend(errors) - continue - products.append(convert_types(product)) - - if all_errors: - logger.warning("Skipped rows with %d validation errors:", len(all_errors)) - for err in all_errors[:10]: - logger.warning(" %s", err) - if len(all_errors) > 10: - logger.warning(" ... and %d more", len(all_errors) - 10) - - logger.info("Loaded %d valid products", len(products)) - return products - - -def _ensure_dataset_and_table( - client: bigquery.Client, - dataset_ref: str, - table_ref: str, - table_id: str, - if_exists: str, -) -> bool: - """Ensure the BigQuery dataset and table exist based on if_exists policy. - - Args: - client: An initialized BigQuery client. - dataset_ref: Fully qualified dataset reference (``project.dataset``). - table_ref: Fully qualified table reference (``project.dataset.table``). - table_id: Bare table name (used when creating the table). - if_exists: Behavior when the dataset already exists: ``"error"``, - ``"skip"``, or ``"rename"`` (interactive only). - - Returns: - ``True`` if ingestion should continue (dataset/table ready), - ``False`` if the existing data should be left alone (skip path). - - Raises: - ValueError: When ``if_exists="rename"`` and the user supplied a new - dataset name; the new name is the exception's argument. The - caller must re-derive refs from this name and re-invoke. - """ - try: - client.get_dataset(dataset_ref) - dataset_exists = True - except Exception: # pylint: disable=broad-exception-caught - dataset_exists = False - - if dataset_exists: - if if_exists == "skip": - try: - client.get_table(table_ref) - count_query = f"SELECT COUNT(*) AS n FROM `{table_ref}`" - row_count = next(iter(client.query(count_query).result())).n - if row_count > 0: - logger.info( - "Dataset %s and table %s already exist with %d rows " - "-- skipping ingestion.", - dataset_ref, - table_ref, - row_count, - ) - return False - logger.info("Table %s exists but is empty -- loading rows.", table_ref) - except Exception: # pylint: disable=broad-exception-caught - logger.info( - "Dataset %s exists but table %s missing -- creating table.", - dataset_ref, - table_ref, - ) - table = bigquery.Table(table_ref, schema=SCHEMA) - client.create_table(table) - elif if_exists == "rename" and sys.stdin.isatty(): - logger.warning("Dataset %s already exists.", dataset_ref) - try: - new_name = input( - " Enter a different dataset name (or Ctrl+C to cancel): " - ).strip() - except (EOFError, KeyboardInterrupt): - print() - logger.info("Cancelled.") - sys.exit(0) - if not new_name: - logger.error("No name provided. Exiting.") - sys.exit(1) - # Caller must re-derive refs from new_name; signal by returning None - # We return the new dataset ID via sys.exit to keep signature simple. - # Raise so the caller can re-try with new name. - raise ValueError(new_name) - else: - logger.error( - "Dataset %s already exists. " - "Re-run with --if-exists skip to reuse it, or --dataset-id .", - dataset_ref, - ) - sys.exit(1) - - if not dataset_exists: - logger.info("Creating dataset %s", dataset_ref) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - client.create_dataset(dataset) - - logger.info("Creating table %s", table_ref) - table = bigquery.Table(table_ref, schema=SCHEMA) - client.create_table(table) - - return True - - -def ingest( - project_id: str, - dataset_id: str, - table_id: str, - source: str, - source_format: str = "csv", - if_exists: str = "error", -): - """Ingest products into BigQuery. - - if_exists controls behavior when the target dataset already exists: - - "error" (default): fail with a clear message - - "skip": no-op if the table already has rows (idempotent re-runs) - - "rename": prompt for a new name (interactive only) - """ - client = bigquery.Client(project=project_id) - - dataset_ref = f"{project_id}.{dataset_id}" - table_ref = f"{project_id}.{dataset_id}.{table_id}" - - try: - should_continue = _ensure_dataset_and_table( - client, dataset_ref, table_ref, table_id, if_exists - ) - except ValueError as new_name: - # "rename" path: recurse with the new dataset name. - dataset_id = str(new_name) - dataset_ref = f"{project_id}.{dataset_id}" - table_ref = f"{project_id}.{dataset_id}.{table_id}" - try: - client.get_dataset(dataset_ref) - logger.error( - "Dataset %s also exists. Re-run with --dataset-id .", - dataset_ref, - ) - sys.exit(1) - except Exception: # pylint: disable=broad-exception-caught - pass - should_continue = _ensure_dataset_and_table( - client, dataset_ref, table_ref, table_id, "error" - ) - - if not should_continue: - return - - # Load products - if source_format == "json": - products = load_from_json(source) - else: - products = load_from_csv(source) - - if not products: - logger.error("No valid products to ingest") - sys.exit(1) - - # Use a load job (batch) instead of streaming insert. Streaming inserts go - # through a buffer that can keep stale dataset-uuid references for ~5-10 min - # after a delete, which causes "Dataset is deleted" errors when the same - # dataset name is recreated quickly (e.g. test re-runs). Load jobs route - # through a different path with no such race. - job_config = bigquery.LoadJobConfig( - schema=SCHEMA, - write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE, - source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON, - ) - ndjson = "\n".join(json.dumps(p) for p in products).encode("utf-8") - load_job = client.load_table_from_file( - io.BytesIO(ndjson), - table_ref, - job_config=job_config, - ) - load_job.result() # Wait for completion; raises on error. - if load_job.errors: - logger.error("Load job errors: %s", load_job.errors) - sys.exit(1) - - logger.info("Successfully ingested %d products to %s", len(products), table_ref) - - -def main(): - """Parse CLI arguments and ingest products to BigQuery.""" - parser = argparse.ArgumentParser(description="Ingest product catalog to BigQuery") - parser.add_argument( - "--config", - default="", - help="Path to design-spec.md (provides defaults for other args)", - ) - parser.add_argument("--project-id", help="GCP project ID") - parser.add_argument( - "--dataset-id", - default="retail_skill_products", - help="BigQuery dataset ID", - ) - parser.add_argument("--table-id", default="products", help="BigQuery table ID") - parser.add_argument("--gcs-bucket", help="GCS bucket name (used with --gcs-path)") - parser.add_argument("--gcs-path", help="Path to data file in GCS bucket") - parser.add_argument("--local-file", help="Path to local data file") - parser.add_argument( - "--format", choices=["csv", "json"], default="csv", help="Source format" - ) - parser.add_argument( - "--if-exists", - choices=["error", "skip", "rename"], - default="error", - help=( - "Behavior when dataset already exists: error (default), " - "skip (idempotent re-run), rename (interactive)" - ), - ) - - args = parser.parse_args() - - # Load design-spec.md defaults -- CLI args override config values - if args.config: - cfg = load_config(args.config) - if not args.project_id: - args.project_id = cfg.get("gcp_project_id", "") - - if not args.project_id: - parser.error( - "--project-id is required (or set gcp_project_id in design-spec.md)" - ) - if args.local_file: - source = args.local_file - elif args.gcs_bucket and args.gcs_path: - source = f"gs://{args.gcs_bucket}/{args.gcs_path}" - else: - parser.error("Provide either --local-file or both --gcs-bucket and --gcs-path") - - ingest( - project_id=args.project_id, - dataset_id=args.dataset_id, - table_id=args.table_id, - source=source, - source_format=args.format, - if_exists=args.if_exists, - ) - - -if __name__ == "__main__": - main() diff --git a/skills/retail/retail-product-search/scripts/ingest_vertex_search.py b/skills/retail/retail-product-search/scripts/ingest_vertex_search.py deleted file mode 100644 index e3e3345290..0000000000 --- a/skills/retail/retail-product-search/scripts/ingest_vertex_search.py +++ /dev/null @@ -1,467 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -r"""Ingest products into Vertex AI Vector Search 2.0 Collection. - -Reads products from BigQuery, creates a Vector Search 2.0 Collection -(if it doesn't exist), and inserts products one at a time. `AlreadyExists` -errors are caught so re-runs are idempotent. Embeddings are auto-generated -by the Collection's configured model. - -Usage: - python ingest_vertex_search.py \\ - --project-id my-project \\ - --collection-id my-products - - # Or use design-spec.md for defaults: - python ingest_vertex_search.py --config design-spec.md -""" - -# pylint: disable=line-too-long -# (error messages with collection_id hints and gcloud commands are intentionally long.) - -import argparse -import logging -import pathlib -import sys -from typing import Any - -from google.api_core import exceptions -from google.cloud import bigquery, vectorsearch -from google.protobuf import struct_pb2 - -# Allow imports from the script's own directory before pip install -e is run. -sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) -from _setup_utils import load_config # noqa: E402 # pylint: disable=wrong-import-position - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -DEFAULT_EMBEDDING_MODEL = "gemini-embedding-001" -DEFAULT_EMBEDDING_FIELDS = ["name", "description", "category", "brand"] - -# All product fields stored as data object fields -PRODUCT_DATA_FIELDS = { - "product_id": "string", - "name": "string", - "price": "number", - "description": "string", - "category": "string", - "brand": "string", - "image_url": "string", - "rating": "number", - "stock": "integer", -} - - -def fetch_products( - project_id: str, dataset_id: str, table_id: str -) -> list[dict[str, Any]]: - """Fetch all products from a BigQuery table. - - Args: - project_id: GCP project ID. - dataset_id: BigQuery dataset name. - table_id: BigQuery table name. - - Returns: - List of product dicts (one per BQ row). - """ - client = bigquery.Client(project=project_id) - query = f"SELECT * FROM `{project_id}.{dataset_id}.{table_id}`" - - logger.info("Fetching products from %s.%s.%s", project_id, dataset_id, table_id) - results = client.query(query).result() - products = [dict(row.items()) for row in results] - logger.info("Fetched %d products", len(products)) - return products - - -def build_text_template(embedding_fields: list[str]) -> str: - """Build the ``text_template`` for auto-embedding from field names. - - The template uses ``{field_name}`` placeholders that VS 2.0 resolves - from the data object's data fields. - - Args: - embedding_fields: Field names to include in the template. - - Returns: - Pipe-separated template string, e.g. - ``"name: {name} | description: {description}"``. - """ - parts = [f"{field}: {{{field}}}" for field in embedding_fields] - return " | ".join(parts) - - -def get_data_schema(products: list[dict[str, Any]]) -> dict[str, Any]: - """Build the data schema from known product fields. - - Only includes fields that exist in the actual data (sampled from the - first 10 products). - - Args: - products: Product dicts loaded from BigQuery. - - Returns: - JSON-Schema-style dict ``{"type": "object", "properties": {...}}`` - with one property per detected field. - """ - properties: dict[str, Any] = {} - sample_keys: set[str] = set() - for p in products[:10]: - sample_keys.update(p.keys()) - - for field, field_type in PRODUCT_DATA_FIELDS.items(): - if field in sample_keys: - properties[field] = {"type": field_type} - - return {"type": "object", "properties": properties} - - -def create_collection_if_needed( # pylint: disable=too-many-arguments - project_id: str, - location: str, - collection_id: str, - embedding_model: str, - embedding_fields: list[str], - data_schema: dict[str, Any], -) -> str: - """Create a Vector Search 2.0 Collection if it doesn't exist. - - Args: - project_id: GCP project ID. - location: GCP region for the collection. - collection_id: Collection ID (RFC1035-compliant, max 63 chars). - embedding_model: Auto-embedding model ID (e.g. - ``"gemini-embedding-001"``). - embedding_fields: Fields used in the embedding text template. - data_schema: JSON-Schema-style dict from :func:`get_data_schema`. - - Returns: - Fully qualified collection resource path. - """ - client = vectorsearch.VectorSearchServiceClient() - parent = f"projects/{project_id}/locations/{location}" - collection_name = f"{parent}/collections/{collection_id}" - - try: - client.get_collection( - request=vectorsearch.GetCollectionRequest(name=collection_name) - ) - logger.info("Collection '%s' already exists", collection_id) - return collection_name - except exceptions.NotFound: - pass - - text_template = build_text_template(embedding_fields) - logger.info( - "Creating collection '%s' with text_template: %s", collection_id, text_template - ) - - request = vectorsearch.CreateCollectionRequest( - parent=parent, - collection_id=collection_id, - collection={ - "data_schema": data_schema, - "vector_schema": { - "text_embedding": { - "dense_vector": { - "dimensions": 768, - "vertex_embedding_config": { - "model_id": embedding_model, - "text_template": text_template, - "task_type": "RETRIEVAL_DOCUMENT", - }, - }, - }, - }, - }, - ) - - try: - operation = client.create_collection(request=request) - logger.info("Waiting for collection creation to complete...") - operation.result() - logger.info("Collection created successfully") - except Exception as e: # pylint: disable=broad-exception-caught - # The SDK often mangles the underlying API error into a confusing gRPC - # StatusCode mismatch. Most common real causes: - # 1. collection_id > 63 chars (RFC1035 limit) - # 2. collection_id has uppercase or invalid characters - # 3. location/region doesn't support Vector Search - # 4. project lacks aiplatform.user IAM - hints = [] - if len(collection_id) > 63: - hints.append( - f"collection_id is {len(collection_id)} chars (max 63 per RFC1035). " - "Shorten it via the 'collection_id' key in design-spec.md." - ) - if not collection_id.islower() or any(c.isupper() for c in collection_id): - hints.append( - "collection_id must be all lowercase letters, digits, and dashes." - ) - hint_text = ( - ("\n Likely cause: " + " ".join(hints)) - if hints - else ( - "\n Possible causes: collection_id too long (>63), uppercase chars, " - "region doesn't support Vector Search, or missing aiplatform.user IAM." - ) - ) - logger.error( - "Error creating collection '%s': %s%s\n" - " Manual cleanup if the collection was partially created:\n" - " gcloud ai vector-search-collections delete %s " - "--region=%s --project=%s", - collection_id, - e, - hint_text, - collection_id, - location, - project_id, - ) - sys.exit(1) - - return collection_name - - -def ingest_products( - collection_path: str, - products: list[dict[str, Any]], -) -> None: - """Insert products into the Vector Search 2.0 Collection. - - Embeddings are auto-generated by the Collection's configured model. - Uses individual inserts for reliability. - """ - client = vectorsearch.DataObjectServiceClient() - - created = 0 - skipped = 0 - errors = 0 - - for i, product in enumerate(products): - product_id = str(product.get("product_id", "")) - # Ensure ID is RFC1035 compliant (lowercase, starts with letter) - safe_id = product_id.lower().replace("_", "-") - if safe_id and not safe_id[0].isalpha(): - safe_id = "p-" + safe_id - - data_struct = struct_pb2.Struct() # pylint: disable=no-member - for field, field_type in PRODUCT_DATA_FIELDS.items(): - value = product.get(field) - if value is None: - continue - # Preserve proper types for the schema - if field_type == "number": - data_struct.update({field: float(value)}) - elif field_type == "integer": - data_struct.update({field: int(value)}) - else: - data_struct.update({field: str(value)}) - - try: - client.create_data_object( - parent=collection_path, - data_object_id=safe_id, - data_object=vectorsearch.DataObject(data=data_struct), - ) - created += 1 - except exceptions.AlreadyExists: - skipped += 1 - except Exception as e: # pylint: disable=broad-exception-caught - errors += 1 - logger.warning("Failed to insert %s: %s", safe_id, e) - - if (i + 1) % 50 == 0 or (i + 1) == len(products): - logger.info("Processed %d/%d products", i + 1, len(products)) - - logger.info( - "Ingestion complete: %d created, %d skipped, %d errors", - created, - skipped, - errors, - ) - - -def ingest_pipeline( # pylint: disable=too-many-arguments - *, - project_id: str, - location: str, - dataset_id: str, - table_id: str, - collection_id: str, - embedding_model: str, - embedding_fields: list[str], -) -> str | None: - """Run the Vector Search ingestion pipeline programmatically. - - Args: - project_id: GCP project ID. - location: GCP region for the collection. - dataset_id: BigQuery dataset to read products from. - table_id: BigQuery table to read products from. - collection_id: Vector Search collection ID. - embedding_model: Auto-embedding model ID. - embedding_fields: Fields used in the embedding text template. - - Returns: - The collection path on success (whether freshly created or - already populated and skipped). ``None`` on error (e.g. no - products found in BigQuery). - """ - collection_path = ( - f"projects/{project_id}/locations/{location}/collections/{collection_id}" - ) - - # Re-run short-circuit: if the collection already exists AND is populated, - # skip the BQ fetch + per-row insert loop. - try: - vs_client = vectorsearch.VectorSearchServiceClient() - vs_client.get_collection( - request=vectorsearch.GetCollectionRequest(name=collection_path) - ) - search_client = vectorsearch.DataObjectSearchServiceClient() - probe = search_client.search_data_objects( - request=vectorsearch.SearchDataObjectsRequest( - parent=collection_path, - semantic_search=vectorsearch.SemanticSearch( - search_text="probe", - search_field="text_embedding", - task_type="QUESTION_ANSWERING", - top_k=1, - output_fields=vectorsearch.OutputFields(data_fields=["product_id"]), - ), - ) - ) - if any(True for _ in probe): - logger.info( - "Collection '%s' already populated -- " - "skipping ingestion (re-run idempotent path).", - collection_id, - ) - logger.info("Collection path: %s", collection_path) - return collection_path - except exceptions.NotFound: - pass - - products = fetch_products(project_id, dataset_id, table_id) - if not products: - logger.error("No products found in BigQuery") - return None - - data_schema = get_data_schema(products) - collection_path = create_collection_if_needed( - project_id, - location, - collection_id, - embedding_model, - embedding_fields, - data_schema, - ) - - ingest_products(collection_path, products) - - logger.info( - "Successfully ingested %d products into collection '%s'", - len(products), - collection_id, - ) - logger.info("Collection path: %s", collection_path) - return collection_path - - -def main(): - """Parse CLI arguments and run the Vector Search ingestion pipeline.""" - parser = argparse.ArgumentParser( - description="Ingest products to Vertex AI Vector Search 2.0" - ) - parser.add_argument( - "--config", - default="", - help="Path to design-spec.md (provides defaults for other args)", - ) - parser.add_argument("--project-id", help="GCP project ID") - parser.add_argument("--location", default="us-central1", help="GCP region") - parser.add_argument( - "--dataset-id", default="retail_skill_products", help="BigQuery dataset" - ) - parser.add_argument("--table-id", default="products", help="BigQuery table") - parser.add_argument( - "--collection-id", - default="retail-skill-products-collection", - help="Vector Search 2.0 Collection ID", - ) - parser.add_argument( - "--embedding-model", - default=DEFAULT_EMBEDDING_MODEL, - help="Auto-embedding model (default: gemini-embedding-001)", - ) - parser.add_argument( - "--embedding-fields", - default=",".join(DEFAULT_EMBEDDING_FIELDS), - help="Comma-separated fields for embedding text template", - ) - - args = parser.parse_args() - - # Load design-spec.md defaults -- CLI args override config values - if args.config: - cfg = load_config(args.config) - if not args.project_id: - args.project_id = cfg.get("gcp_project_id", "") - if args.location == "us-central1" and cfg.get("gcp_region"): - args.location = cfg["gcp_region"] - if cfg.get("dataset_id") and args.dataset_id == "retail_skill_products": - args.dataset_id = cfg["dataset_id"] - if cfg.get("table_id") and args.table_id == "products": - args.table_id = cfg["table_id"] - if ( - cfg.get("collection_id") - and args.collection_id == "retail-skill-products-collection" - ): - args.collection_id = cfg["collection_id"] - if cfg.get("embedding_model"): - model_name = cfg["embedding_model"].split(" ")[0] - if args.embedding_model == DEFAULT_EMBEDDING_MODEL: - args.embedding_model = model_name - if cfg.get("embedding_fields") and args.embedding_fields == ",".join( - DEFAULT_EMBEDDING_FIELDS - ): - args.embedding_fields = cfg["embedding_fields"] - - if not args.project_id: - parser.error( - "--project-id is required (or set gcp_project_id in design-spec.md)" - ) - - embedding_fields = [f.strip() for f in args.embedding_fields.split(",")] - - result_path = ingest_pipeline( - project_id=args.project_id, - location=args.location, - dataset_id=args.dataset_id, - table_id=args.table_id, - collection_id=args.collection_id, - embedding_model=args.embedding_model, - embedding_fields=embedding_fields, - ) - if result_path: - logger.info("Set VECTOR_SEARCH_COLLECTION env var to this path in your agent.") - - -if __name__ == "__main__": - main() diff --git a/skills/retail/retail-product-search/scripts/retrievers.py b/skills/retail/retail-product-search/scripts/retrievers.py deleted file mode 100644 index 89927c9dcb..0000000000 --- a/skills/retail/retail-product-search/scripts/retrievers.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Retrieval helpers for the retail product search agent. - -Provides :func:`search_collection` for semantic product lookup via -Vertex AI Vector Search 2.0, and :func:`search` as a convenience -wrapper that reads the collection path from the environment. -""" - -# pylint: disable=line-too-long -# (error messages and example URLs are intentionally long; breaking them hurts copy-paste UX.) - -import os -import re - -from google.cloud import vectorsearch - -_COLLECTION_PATH_RE = re.compile( - r"^projects/[^/\s]+/locations/[^/\s]+/collections/[^/\s]+$" -) - - -def _create_search_client(): - return vectorsearch.DataObjectSearchServiceClient() - - -def _format_result(index: int, result) -> str: - """Format a single search result as a compact one-line string. - - Args: - index: 1-based product index used in the formatted output. - result: A Vector Search ``SearchDataObjectsResponse`` entry. - - Returns: - A one-line string of the form - ``"Product N: name, $price, by brand, rated R/5, "``. - """ - data = result.data_object.data - name = data.get("name", "Unknown") - price = data.get("price", "N/A") - brand = data.get("brand", "") - rating = data.get("rating", "") - description = data.get("description", "") - - parts = [f"{name}, ${price}"] - if brand: - parts.append(f"by {brand}") - if rating: - parts.append(f"rated {rating} out of 5") - if description: - parts.append(description[:80]) - - return f"Product {index}: " + ", ".join(parts) - - -def search_collection( - query: str, - collection_path: str, - top_k: int = 10, -) -> str: - """Search a Vector Search 2.0 Collection using semantic search. - - Args: - query: The search query text. - collection_path: Full resource path of the collection. - top_k: Number of results to return. - - Returns: - Formatted string containing relevant document content. - """ - client = _create_search_client() - - request = vectorsearch.SearchDataObjectsRequest( - parent=collection_path, - semantic_search=vectorsearch.SemanticSearch( - search_text=query, - search_field="text_embedding", - task_type="QUESTION_ANSWERING", - top_k=top_k, - output_fields=vectorsearch.OutputFields( - data_fields=[ - "product_id", - "name", - "price", - "description", - "category", - "brand", - "rating", - "stock", - ] - ), - ), - ) - - results = client.search_data_objects(request) - - formatted_parts = [ - _format_result(i + 1, result) for i, result in enumerate(results) - ] - - if not formatted_parts: - return "No matching products found. Suggest broadening the search." - - return ( - "Found " - + str(len(formatted_parts)) - + " products. " - + ". ".join(formatted_parts) - ) - - -def search(query: str, top_k: int = 5) -> str: - """Run the SKILL.md smoke-test query against the configured collection. - - Reads the collection path from the ``VECTOR_SEARCH_COLLECTION`` env var, - which is what ``scripts/setup.py`` prints at the end of a successful run. - - Args: - query: The product search query in natural language. - top_k: Maximum number of matching products to return. - - Returns: - A formatted string with the matching products, or a "no matches" - message if the collection returned nothing. - - Raises: - RuntimeError: If ``VECTOR_SEARCH_COLLECTION`` is not set, or is set - to a malformed path (typically with an embedded newline from a - wrapped multi-line shell paste). - """ - raw = os.environ.get("VECTOR_SEARCH_COLLECTION") - if not raw: - raise RuntimeError( - "VECTOR_SEARCH_COLLECTION env var is not set. Set it to the collection " - "path printed by scripts/setup.py, e.g. " - "projects//locations/us-central1/collections/retail-skill-products-collection" - ) - # Strip whitespace -- a newline embedded mid-path (from a wrapped multi-line - # shell paste) silently causes the SDK to 501 from the wrong endpoint. - collection_path = raw.strip() - if not _COLLECTION_PATH_RE.match(collection_path): - raise RuntimeError( - "VECTOR_SEARCH_COLLECTION is malformed -- expected " - "'projects//locations//collections/' with no whitespace. " - f"Got: {raw!r}. " - "Tip: type the export on a single line, e.g. " - "export VECTOR_SEARCH_COLLECTION=" - '"projects/$GOOGLE_CLOUD_PROJECT/locations/us-central1/' - 'collections/retail-skill-products-collection"' - ) - return search_collection(query, collection_path, top_k=top_k) diff --git a/skills/retail/retail-product-search/scripts/setup.py b/skills/retail/retail-product-search/scripts/setup.py deleted file mode 100644 index 8bb9515164..0000000000 --- a/skills/retail/retail-product-search/scripts/setup.py +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Automated project setup driven by design-spec.md. - -Reads the design-spec.md configuration and runs the appropriate ingestion -steps based on the user's answers. This is the single entry point for both -the `vs` CLI and manual setup. - -Runs scripts in-process via direct Python imports -- does NOT shell out to -`python scripts/...`. That makes setup safe to invoke from any cwd, against -any Python interpreter, regardless of whether the skill files were copied -into the workspace. - -Usage: - python scripts/setup.py --config ./design-spec.md - python scripts/setup.py --config ./design-spec.md --dry-run -""" - -# pylint: disable=line-too-long -# (log lines and error messages with config paths are intentionally long.) - -import argparse -import logging -import pathlib -import sys -from typing import Any - -# Allow imports from the script's own directory before pip install -e is run. -sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) - -# pylint: disable=wrong-import-position -from _paths import SAMPLE_PRODUCTS_CSV # noqa: E402 -from _setup_utils import load_config # noqa: E402 - -import validate_schema # noqa: E402 -import ingest_bigquery # noqa: E402 -import ingest_vertex_search # noqa: E402 -# pylint: enable=wrong-import-position - -logging.basicConfig(level=logging.INFO, format="%(message)s") -logger = logging.getLogger(__name__) - - -# Indirection layer so tests can monkeypatch these without touching the -# underlying scripts. Production: forwards to the real implementations. -def _validate_file(file_path, fields_level): - return validate_schema.validate_file(file_path, fields_level) - - -def _run_bigquery_ingest( # pylint: disable=too-many-arguments - *, project_id, dataset_id, table_id, source, source_format, if_exists -): - return ingest_bigquery.ingest( - project_id=project_id, - dataset_id=dataset_id, - table_id=table_id, - source=source, - source_format=source_format, - if_exists=if_exists, - ) - - -def _run_vertex_ingest(**kwargs): - return ingest_vertex_search.ingest_pipeline(**kwargs) - - -def _resolve_data_source(cfg: dict[str, Any]) -> tuple[str | None, str]: - """Resolve ``data_source`` from the spec into ``(source_path, format)``. - - Args: - cfg: Parsed design-spec config dict. - - Returns: - ``(source_path, format)`` where ``source_path`` is a local path or - ``gs://`` URI and ``format`` is ``"csv"`` or ``"json"``. Returns - ``(None, "")`` if the source is missing or broken so the caller - can error out cleanly. - """ - raw = (cfg.get("data_source") or "").strip() - - if raw.startswith("gs://"): - return raw, _format_from_path(raw) - - if raw and raw not in ("default", "assets/sample-products.csv"): - # User specified an explicit local path. - if pathlib.Path(raw).exists(): - return raw, _format_from_path(raw) - logger.error("\n data_source '%s' does not exist.", raw) - return None, "" - - # Default: the bundled sample inside the install dir. - if SAMPLE_PRODUCTS_CSV.exists(): - return str(SAMPLE_PRODUCTS_CSV), "csv" - - logger.error("\n Bundled sample not found at %s", SAMPLE_PRODUCTS_CSV) - return None, "" - - -def _format_from_path(path: str) -> str: - """Return ``"json"`` for ``.json``/``.jsonl`` paths, else ``"csv"``. - - Args: - path: Data file path (local or gs:// URI). - - Returns: - Either ``"json"`` or ``"csv"`` for the BigQuery loader. - """ - return "json" if path.endswith((".json", ".jsonl")) else "csv" - - -def setup(config_path: str, dry_run: bool = False) -> bool: # pylint: disable=too-many-locals - """Run the full setup pipeline based on design-spec.md. - - Args: - config_path: Path to ``design-spec.md`` with the user's config. - dry_run: If True, log the pipeline shape without running ingest. - - Returns: - True if every step succeeded; False if the data source couldn't - be resolved or validation produced errors. - """ - cfg = load_config(config_path) - - project_id = cfg.get("gcp_project_id", "") - if not project_id: - logger.error("gcp_project_id is required in design-spec.md") - sys.exit(1) - - product_fields = cfg.get("product_fields", "Extended") - dataset_id = cfg.get("dataset_id", "retail_skill_products") - table_id = cfg.get("table_id", "products") - catalog_size = cfg.get("catalog_size", "1K-50K") - gcp_region = cfg.get("gcp_region", "us-central1") - collection_id = cfg.get("collection_id", "retail-skill-products-collection") - embedding_model = cfg.get("embedding_model", "gemini-embedding-001") - raw_fields = cfg.get("embedding_fields", "name, description, category, brand") - embedding_fields = [f.strip() for f in raw_fields.split(",")] - - logger.info("=" * 60) - logger.info("SETUP PIPELINE (driven by design-spec.md)") - logger.info("=" * 60) - logger.info(" Project: %s", project_id) - logger.info(" Region: %s", gcp_region) - - source, source_format = _resolve_data_source(cfg) - if source is None: - return False - - logger.info(" Data source: %s", source) - - if dry_run: - logger.info( - "\n[dry-run] Would validate, ingest to BigQuery, then create Vector Search collection." - ) - return True - - # Step A: validate. - # validate_schema.load_records reads from the local filesystem only; - # skip the schema-shape check for gs:// sources and let - # ingest_bigquery.load_from_csv validate inline as it streams the file. - if source.startswith("gs://"): - logger.info( - "\n Skipping schema validation for gs:// source (validated inline during ingest)." - ) - else: - logger.info("\n Validating sample data...") - valid_count, errors = _validate_file(source, product_fields) - if errors: - logger.error(" Validation produced %d errors. First 5:", len(errors)) - for e in errors[:5]: - logger.error(" %s", e) - return False - logger.info(" Validated %d records.", valid_count) - - # Step B: ingest to BigQuery (idempotent re-run via if_exists='skip') - logger.info("\n Ingesting into BigQuery...") - _run_bigquery_ingest( - project_id=project_id, - dataset_id=dataset_id, - table_id=table_id, - source=source, - source_format=source_format, - if_exists="skip", - ) - - # Step C: create Vector Search collection + ingest - logger.info("\n Creating Vector Search collection...") - collection_path = _run_vertex_ingest( - project_id=project_id, - location=gcp_region, - dataset_id=dataset_id, - table_id=table_id, - collection_id=collection_id, - embedding_model=embedding_model, - embedding_fields=embedding_fields, - ) - - if catalog_size == "500K+": - logger.info("\n NOTE: Catalog size 500K+ detected.") - logger.info( - " Consider using Dataflow for batch ingestion instead of direct inserts." - ) - - logger.info("\n" + "=" * 60) - logger.info("SETUP COMPLETE") - logger.info("=" * 60) - if collection_path: - logger.info(" Collection path: %s", collection_path) - logger.info( - " Set VECTOR_SEARCH_COLLECTION env var to this path in your agent." - ) - return True - - -def main(): - """Parse CLI arguments and run the setup pipeline.""" - parser = argparse.ArgumentParser( - description="Run setup pipeline from design-spec.md" - ) - parser.add_argument("--config", required=True, help="Path to design-spec.md") - parser.add_argument( - "--dry-run", - action="store_true", - help="Show what would run without doing it", - ) - - args = parser.parse_args() - ok = setup(args.config, args.dry_run) - sys.exit(0 if ok else 1) - - -if __name__ == "__main__": - main() diff --git a/skills/retail/retail-product-search/scripts/validate_schema.py b/skills/retail/retail-product-search/scripts/validate_schema.py deleted file mode 100644 index f688350a2c..0000000000 --- a/skills/retail/retail-product-search/scripts/validate_schema.py +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Validate a product catalog CSV/JSON against the expected schema. - -Usage: - python validate_schema.py --file products.csv --fields-level Standard - python validate_schema.py --file products.json --fields-level Extended -""" - -# pylint: disable=line-too-long -# (validation error messages with field names are intentionally long.) - -import argparse -import csv -import json -import logging -import pathlib -import sys -from typing import Any - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -FIELD_LEVELS = { - "Basic": { - "required": ["product_id", "name", "price", "description"], - "optional": [], - }, - "Standard": { - "required": ["product_id", "name", "price", "description"], - "optional": ["category", "brand", "image_url"], - }, - "Extended": { - "required": ["product_id", "name", "price", "description"], - "optional": [ - "category", - "brand", - "image_url", - "rating", - "stock", - "manufacturer", - ], - }, - "Full": { - "required": ["product_id", "name", "price", "description"], - "optional": [ - "category", - "brand", - "image_url", - "rating", - "stock", - "manufacturer", - "variants", - "tags", - "specifications", - "reviews", - ], - }, -} - - -def load_records(file_path: pathlib.Path) -> list[dict[str, Any]]: - """Load records from a CSV, JSON, or JSONL file. - - Args: - file_path: Path to the product data file. The file type is - inferred from the suffix (``.csv``, ``.json``, or ``.jsonl``). - - Returns: - List of records as dicts. - - Raises: - ValueError: If the suffix isn't supported, or the JSON content is - neither a list nor a ``{"products": [...]}`` object. - """ - suffix = file_path.suffix.lower() - - if suffix == ".csv": - with open(file_path, encoding="utf-8") as f: - return list(csv.DictReader(f)) - elif suffix in (".json", ".jsonl"): - content = file_path.read_text(encoding="utf-8") - if suffix == ".jsonl": - return [ - json.loads(line) - for line in content.strip().splitlines() - if line.strip() - ] - parsed = json.loads(content) - if isinstance(parsed, list): - return parsed - if isinstance(parsed, dict) and "products" in parsed: - return parsed["products"] - raise ValueError('JSON must be an array or {"products": [...]}') - else: - raise ValueError(f"Unsupported file type: {suffix}. Use .csv, .json, or .jsonl") - - -def _validate_single_record( - record: dict[str, Any], - row_num: int, - required: list[str], - all_fields: set[str], - higher_level_fields: set[str], - seen_warned_fields: set[str], -) -> list[str]: - """Validate a single record against the schema. - - Args: - record: One product record as a dict from CSV/JSON. - row_num: 1-based row number for error messages. - required: Required field names at the selected fields-level. - all_fields: Set of fields recognized at the selected fields-level. - higher_level_fields: Fields recognized at any higher level; these - warn (and are ignored) instead of failing the row. - seen_warned_fields: Mutable set tracking which higher-level fields - already produced a warning, so each fires once per run. - - Returns: - List of error strings for this row (empty if the row is valid). - """ - row_errors = [] - - for field in required: - if field not in record or not record[field]: - row_errors.append(f"Row {row_num}: missing required field '{field}'") - - if "price" in record and record["price"]: - try: - float(record["price"]) - except (ValueError, TypeError): - row_errors.append( - f"Row {row_num}: 'price' must be numeric, got '{record['price']}'" - ) - - if "rating" in record and record["rating"]: - try: - val = float(record["rating"]) - if not 0 <= val <= 5: - row_errors.append(f"Row {row_num}: 'rating' should be 0-5, got {val}") - except (ValueError, TypeError): - row_errors.append(f"Row {row_num}: 'rating' must be numeric") - - if "stock" in record and record["stock"]: - try: - int(record["stock"]) - except (ValueError, TypeError): - row_errors.append(f"Row {row_num}: 'stock' must be an integer") - - extra_fields = set(record.keys()) - all_fields - known_extras = extra_fields & higher_level_fields - unknown_extras = extra_fields - higher_level_fields - - for field in known_extras - seen_warned_fields: - logger.warning( - "Field '%s' present in data but outside the '%s' schema. " - "It will be ignored. Pick a higher --fields-level if you want it indexed.", - field, - "current", - ) - seen_warned_fields.add(field) - - if unknown_extras: - row_errors.append(f"Row {row_num}: unrecognized fields: {sorted(unknown_extras)}") - - return row_errors - - -def validate( - records: list[dict[str, Any]], fields_level: str -) -> tuple[int, list[str]]: - """Validate records against the schema for ``fields_level``. - - Args: - records: All product records loaded from the source. - fields_level: One of ``"Basic"``, ``"Standard"``, ``"Extended"``, - ``"Full"``. - - Returns: - ``(valid_count, errors)`` where ``valid_count`` is the number of - rows that passed and ``errors`` is the concatenated list of error - strings across all rows. - """ - schema = FIELD_LEVELS[fields_level] - required = schema["required"] - all_fields = set(required + schema["optional"]) - # Fields that exist at any higher level -- treat as known-but-out-of-scope. - # Warn instead of erroring so users with richer CSVs don't have to strip columns. - higher_level_fields: set[str] = set() - for level_schema in FIELD_LEVELS.values(): - higher_level_fields.update(level_schema["required"]) - higher_level_fields.update(level_schema["optional"]) - - errors = [] - seen_warned_fields: set[str] = set() - valid = 0 - - for i, record in enumerate(records, start=1): - row_errors = _validate_single_record( - record, i, required, all_fields, higher_level_fields, seen_warned_fields - ) - if row_errors: - errors.extend(row_errors) - else: - valid += 1 - - return valid, errors - - -def validate_file(file_path, fields_level: str) -> tuple[int, list[str]]: - """Load a CSV/JSON file and validate it in one call. - - Convenience wrapper for orchestrators that don't need the records. - - Args: - file_path: Path to the product data file (CSV, JSON, or JSONL). - fields_level: One of ``"Basic"``, ``"Standard"``, ``"Extended"``, - ``"Full"``. - - Returns: - ``(valid_count, errors)`` -- same shape as ``validate()``. - """ - records = load_records(pathlib.Path(file_path)) - return validate(records, fields_level) - - -def main(): - """Parse CLI arguments and run schema validation.""" - parser = argparse.ArgumentParser(description="Validate product catalog schema") - parser.add_argument( - "--file", required=True, help="Path to product data file (CSV/JSON)" - ) - parser.add_argument( - "--fields-level", - choices=["Basic", "Standard", "Extended", "Full"], - default="Standard", - help="Product fields level (default: Standard)", - ) - - args = parser.parse_args() - file_path = pathlib.Path(args.file) - - if not file_path.exists(): - logger.error("File not found: %s", file_path) - sys.exit(1) - - records = load_records(file_path) - logger.info("Loaded %d records from %s", len(records), file_path) - - valid_count, errors = validate(records, args.fields_level) - - if errors: - logger.warning("%d validation errors found:", len(errors)) - for err in errors[:20]: - logger.warning(" %s", err) - if len(errors) > 20: - logger.warning(" ... and %d more", len(errors) - 20) - - logger.info( - "Validation complete: %d/%d records valid", valid_count, len(records) - ) - - if errors: - sys.exit(1) - - -if __name__ == "__main__": - main() From bdcec2d725eaa11f006b01301f4b6c66cfc510fc Mon Sep 17 00:00:00 2001 From: Shahin Saadati Date: Wed, 29 Jul 2026 16:13:48 -0700 Subject: [PATCH 4/5] feat: exclude structural model literals from extraction to prevent breaking collection lookups and comparisons --- .../SKILL.md | 30 +++- .../scripts/extract_env_vars.py | 151 +++++++++++++++++- .../tests/test_extract_env_vars.py | 125 +++++++++++++++ 3 files changed, 298 insertions(+), 8 deletions(-) diff --git a/.agents/skills/extract-python-environment-variables/SKILL.md b/.agents/skills/extract-python-environment-variables/SKILL.md index 7471f9aaf3..9951162d73 100644 --- a/.agents/skills/extract-python-environment-variables/SKILL.md +++ b/.agents/skills/extract-python-environment-variables/SKILL.md @@ -51,7 +51,7 @@ description: > metadata: author: Google license: Apache-2.0 - version: 2.2.0 + version: 2.3.0 --- # Extract Python Environment Variables @@ -150,6 +150,34 @@ Runs `scripts/extract_env_vars.py` against a recipe directory. The script: 4. **Replaces hardcoded model names** in source (e.g. `model="gemini-3.5-flash"` in `agent.py`) with an `os.getenv(...)` call. + **Position decides eligibility.** A model string is only promoted when it + is a *configurable constant*. Three positions mean it is something else, + and are left untouched: + + - **Collection-literal entries** (dict keys and values, list/set/tuple + items) — lookup tables and enumerations of supported models: + + ```python + IMAGE_MODELS = { + "flash": "gemini-2.5-flash-image", + "pro": "gemini-2.5-pro-image", + } + ``` + + Rewriting the values collapses the table onto one env var; rewriting the + **keys** is worse, because `IMAGE_MODELS.get("flash")` then never + matches anything. A dict key must stay a static literal. + - **Subscript indices** — `IMAGE_MODELS["gemini-3.1"]` is a key *into* + such a table, so replacing it looks up a different entry. + - **Comparison operands** — `if "gemini-3.1" in model_id` tests a value + rather than configuring one. + + Skipping a legitimate extraction is cheap (lift it by hand); silently + breaking a lookup table is not, so this errs towards skipping. Every + skipped literal is listed in an `[INFO]` block, so nothing is hidden — and + because these strings never reach the naming step, a dict target like + `IMAGE_MODELS` can no longer leak into `.env.example` as a variable name. + **Variable name.** The assignment target that holds the literal is used when it names a model, since it carries far more meaning than a generic fallback — `DEFAULT_EMBEDDING_MODEL = "gemini-embedding-001"` and diff --git a/.agents/skills/extract-python-environment-variables/scripts/extract_env_vars.py b/.agents/skills/extract-python-environment-variables/scripts/extract_env_vars.py index eacdb653d5..3dc4451aec 100644 --- a/.agents/skills/extract-python-environment-variables/scripts/extract_env_vars.py +++ b/.agents/skills/extract-python-environment-variables/scripts/extract_env_vars.py @@ -1259,6 +1259,72 @@ def _mark_default(call: ast.Call) -> None: return ids +def _structural_exclusion_node_ids(tree: ast.AST) -> set[int]: + """ + Return the id() of every ast.Constant that must NOT be rewritten because + of WHERE it sits, regardless of what it looks like. + + A model string is only a candidate for promotion to an env var when it is + a *configurable constant*. Three positions mean it is something else, and + rewriting them corrupts working code: + + 1. **An element of a collection literal** (dict key or value, list, set, + or tuple entry). These are lookup tables and enumerations of supported + models, not a single configurable choice:: + + IMAGE_MODELS = { + "flash": "gemini-2.5-flash-image", + "pro": "gemini-2.5-pro-image", + } + + Rewriting the values makes the table resolve at import time from a + single env var; rewriting the KEYS is worse still, because + ``IMAGE_MODELS.get("flash")`` then never matches anything. A dict key + must stay a static literal. + + 2. **A subscript index** — ``IMAGE_MODELS["gemini-3.1"]`` is a key INTO + such a table. Replacing it looks up a different (probably absent) + entry. + + 3. **An operand of a comparison** — ``if "gemini-3.1" in model_id`` tests + a value rather than configuring one. Replacing it changes the + predicate's meaning. + + Skipping a legitimate extraction is cheap (the user can lift the value by + hand); silently breaking a lookup table is not. When in doubt this errs + towards skipping, and the skipped literals are reported so nothing is + hidden — see :func:`extract_skipped_model_literals`. + """ + ids: set[int] = set() + for parent in ast.walk(tree): + if isinstance(parent, ast.Dict): + # `keys` holds None for a `**expansion` entry — guard for it. + for node in list(parent.keys) + list(parent.values): + if isinstance(node, ast.Constant): + ids.add(id(node)) + elif isinstance(parent, (ast.List, ast.Set, ast.Tuple)): + for node in parent.elts: + if isinstance(node, ast.Constant): + ids.add(id(node)) + elif isinstance(parent, ast.Subscript): + if isinstance(parent.slice, ast.Constant): + ids.add(id(parent.slice)) + elif isinstance(parent, ast.Compare): + for node in [parent.left, *parent.comparators]: + if isinstance(node, ast.Constant): + ids.add(id(node)) + return ids + + +def _model_exclusion_node_ids(tree: ast.AST) -> set[int]: + """Every Constant the model-replacement path must leave alone.""" + return ( + _docstring_node_ids(tree) + | _getenv_default_node_ids(tree) + | _structural_exclusion_node_ids(tree) + ) + + def _flat_offset(lines: list[str], lineno: int, col: int) -> int: """Convert a 1-based lineno + 0-based col_offset to a flat char offset.""" return sum(len(ln) for ln in lines[: lineno - 1]) + col @@ -1797,6 +1863,9 @@ def extract_hardcoded_models( :func:`_getenv_default_node_ids`) — those are already serving as env-var defaults; replacing them would silently regress the type from ``str`` to ``str | None`` and could break at runtime. + * String literals whose POSITION means they aren't a configurable + constant — collection-literal entries, subscript indices, and + comparison operands (see :func:`_structural_exclusion_node_ids`). Returns: {file_path: [(line_number, model_string), ...]} @@ -1810,9 +1879,7 @@ def extract_hardcoded_models( except (SyntaxError, UnicodeDecodeError): continue - excluded_ids = _docstring_node_ids(tree) | _getenv_default_node_ids( - tree - ) + excluded_ids = _model_exclusion_node_ids(tree) for node in ast.walk(tree): if not isinstance(node, ast.Constant): @@ -1827,6 +1894,61 @@ def extract_hardcoded_models( return hits +def extract_skipped_model_literals( + py_files: list[Path], +) -> dict[Path, list[tuple[int, str]]]: + """ + Find model strings deliberately NOT rewritten because of their position. + + These are the literals :func:`_structural_exclusion_node_ids` protects — + lookup-table entries, subscript keys, comparison operands. They are + reported rather than silently dropped so the maintainer knows the recipe + still hardcodes a model somewhere, and can decide whether that table + ought to be configurable. + + Docstring mentions are excluded (prose, not code). Identical + (line, string) pairs are de-duplicated: ``{"gemini-x": "gemini-x"}`` + holds two distinct nodes on one line, and listing it twice only makes + the report harder to read. + + Returns: + {file_path: [(line_number, model_string), ...]} + """ + skipped: dict[Path, list[tuple[int, str]]] = {} + + for py_file in py_files: + try: + source = py_file.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(py_file)) + except (SyntaxError, UnicodeDecodeError): + continue + + structural = _structural_exclusion_node_ids(tree) + docstrings = _docstring_node_ids(tree) + seen: set[tuple[int, str]] = set() + + for node in ast.walk(tree): + if not isinstance(node, ast.Constant): + continue + if id(node) not in structural or id(node) in docstrings: + continue + if not isinstance(node.value, str): + continue + if not any(node.value.startswith(p) for p in MODEL_PREFIXES): + continue + key = (node.lineno, node.value) + if key in seen: + continue + seen.add(key) + skipped.setdefault(py_file, []).append(key) + + if skipped: + for hits in skipped.values(): + hits.sort() + + return skipped + + def _assignment_target_name(node: ast.AST) -> str | None: """Return the single ``Name`` target of an assignment, else None.""" if ( @@ -1897,9 +2019,7 @@ def extract_model_var_hints(py_files: list[Path]) -> dict[str, str]: except (SyntaxError, UnicodeDecodeError): continue - excluded_ids = _docstring_node_ids(tree) | _getenv_default_node_ids( - tree - ) + excluded_ids = _model_exclusion_node_ids(tree) for node in ast.walk(tree): target = _assignment_target_name(node) @@ -2083,7 +2203,7 @@ def _collect_model_replacements( type of a call like ``os.getenv("V", "gemini-3.5-flash")`` from ``str`` to ``str | None``. """ - excluded_ids = _docstring_node_ids(tree) | _getenv_default_node_ids(tree) + excluded_ids = _model_exclusion_node_ids(tree) lines = source.splitlines(keepends=True) replacements: list[tuple[int, int, str]] = [] file_substituted: dict[str, str] = {} @@ -2378,6 +2498,23 @@ def run_step_model_names( lookup that would evaluate to ``None``. """ model_hits = extract_hardcoded_models(py_files) + skipped = extract_skipped_model_literals(py_files) + + if skipped: + total = sum(len(v) for v in skipped.values()) + print( + f"\n[INFO] Left {total} model literal(s) in place — their position " + "means they are not a configurable constant (lookup-table entry, " + "subscript key, or comparison operand). Rewriting them would " + "break the code:" + ) + for py_file, file_hits in skipped.items(): + for lineno, model_str in file_hits: + print( + f" {py_file.relative_to(recipe_dir)}:{lineno}" + f' — "{model_str}"' + ) + if not model_hits: print("\n[PASS] No hardcoded model names detected.") return diff --git a/.agents/skills/extract-python-environment-variables/tests/test_extract_env_vars.py b/.agents/skills/extract-python-environment-variables/tests/test_extract_env_vars.py index 99bdb91977..55abc5cd26 100644 --- a/.agents/skills/extract-python-environment-variables/tests/test_extract_env_vars.py +++ b/.agents/skills/extract-python-environment-variables/tests/test_extract_env_vars.py @@ -3109,3 +3109,128 @@ def test_assign_model_var_names_unchanged_without_hints(): "MODEL_NAME_GENERATED_1", "MODEL_NAME_GENERATED_2", } + + +# --------------------------------------------------------------------------- +# Structural exclusions: position decides whether a model literal is a +# configurable constant. +# +# Regression: the extractor rewrote every gemini-* string it found, including +# the keys AND values of a model lookup table: +# +# IMAGE_MODELS = {"flash": "gemini-2.5-flash-image", ...} +# +# Turning a dict key into os.getenv(...) means IMAGE_MODELS.get("flash") +# never matches again — a silent, total break of the lookup. +# --------------------------------------------------------------------------- + + +LOOKUP_TABLE_SRC = """IMAGE_MODELS = { + "gemini-2.5-flash-image": "gemini-2.5-flash-image", + "flash": "gemini-2.5-flash-image", + "pro": "gemini-2.5-pro-image", +} + + +def resolve(label): + if "gemini-3.1" in label: + return IMAGE_MODELS["gemini-2.5-pro-image"] + return IMAGE_MODELS.get(label, IMAGE_MODELS["gemini-2.5-flash-image"]) +""" + + +def test_dict_literal_entries_are_not_detected(tmp_path): + py = _write(tmp_path / "models.py", LOOKUP_TABLE_SRC) + assert m.extract_hardcoded_models([py]) == {} + + +def test_dict_literal_is_left_byte_identical(tmp_path): + py = _write(tmp_path / "models.py", LOOKUP_TABLE_SRC) + hits = m.extract_hardcoded_models([py]) + name_map = m.assign_model_var_names( + {"gemini-2.5-flash-image", "gemini-2.5-pro-image", "gemini-3.1"} + ) + + m.replace_hardcoded_models([py], hits, name_map) + + assert py.read_text(encoding="utf-8") == LOOKUP_TABLE_SRC + + +def test_list_set_and_tuple_entries_are_excluded(tmp_path): + py = _write( + tmp_path / "c.py", + 'ALLOWED = ["gemini-3.5-flash"]\n' + 'SUPPORTED = {"gemini-3.5-pro"}\n' + 'PAIR = ("gemini-3.1-flash-image",)\n', + ) + assert m.extract_hardcoded_models([py]) == {} + + +def test_subscript_index_is_excluded(tmp_path): + py = _write(tmp_path / "c.py", 'X = TABLE["gemini-3.5-flash"]\n') + assert m.extract_hardcoded_models([py]) == {} + + +def test_comparison_operand_is_excluded(tmp_path): + py = _write( + tmp_path / "c.py", + 'if "gemini-3.1" in model_id:\n pass\n', + ) + assert m.extract_hardcoded_models([py]) == {} + + +def test_scalar_assignment_is_still_detected(tmp_path): + # The exclusions must not swallow the case the skill exists for. + py = _write(tmp_path / "agent.py", 'MODEL = "gemini-3.5-flash"\n') + hits = m.extract_hardcoded_models([py]) + assert [v for hits_ in hits.values() for _, v in hits_] == [ + "gemini-3.5-flash" + ] + + +def test_call_keyword_argument_is_still_detected(tmp_path): + py = _write(tmp_path / "agent.py", 'a = Agent(model="gemini-3.5-flash")\n') + hits = m.extract_hardcoded_models([py]) + assert [v for hits_ in hits.values() for _, v in hits_] == [ + "gemini-3.5-flash" + ] + + +def test_dict_target_does_not_produce_a_var_name_hint(tmp_path): + # `IMAGE_MODELS = {...}` names a TABLE, not a model. Deriving the env var + # name IMAGE_MODELS from it produced a nonsensical .env.example entry. + py = _write(tmp_path / "models.py", LOOKUP_TABLE_SRC) + assert m.extract_model_var_hints([py]) == {} + + +def test_skipped_literals_are_reported(tmp_path): + py = _write(tmp_path / "models.py", LOOKUP_TABLE_SRC) + skipped = m.extract_skipped_model_literals([py]) + + values = [v for _, v in skipped[py]] + assert "gemini-2.5-flash-image" in values # dict key + value + assert "gemini-2.5-pro-image" in values + assert "gemini-3.1" in values # comparison operand + + +def test_skipped_literals_deduplicate_per_line(tmp_path): + # `"x": "x"` is two nodes on one line; report it once. + py = _write( + tmp_path / "d.py", + 'T = {\n "gemini-3.5-flash": "gemini-3.5-flash",\n}\n', + ) + skipped = m.extract_skipped_model_literals([py]) + assert skipped[py] == [(2, "gemini-3.5-flash")] + + +def test_skipped_literals_ignore_docstring_mentions(tmp_path): + py = _write( + tmp_path / "d.py", + '"""Uses gemini-3.5-flash by default."""\n', + ) + assert m.extract_skipped_model_literals([py]) == {} + + +def test_skipped_report_empty_when_nothing_excluded(tmp_path): + py = _write(tmp_path / "agent.py", 'MODEL = "gemini-3.5-flash"\n') + assert m.extract_skipped_model_literals([py]) == {} From b436cddabdef8ef8b1a50e3100d6be848c215901 Mon Sep 17 00:00:00 2001 From: Shahin Saadati Date: Wed, 29 Jul 2026 16:17:48 -0700 Subject: [PATCH 5/5] feat: add packaging, ruamel-yaml, and tomlkit to dev dependencies and update workflow docs --- .github/workflows/tools-tests.yml | 7 ++++--- pyproject.toml | 13 +++++++++++++ uv.lock | 28 +++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tools-tests.yml b/.github/workflows/tools-tests.yml index fce125a81d..8ba9ded09e 100644 --- a/.github/workflows/tools-tests.yml +++ b/.github/workflows/tools-tests.yml @@ -58,9 +58,10 @@ jobs: pytest: name: pytest (tools + skills) runs-on: ubuntu-latest - # Tooling + skills tests are lightweight (stdlib + pytest); 15 min - # leaves headroom for cold uv-sync while still protecting against a - # stuck job riding the default 6-hour timeout. + # Tooling + skills tests are lightweight (pytest plus a few pure-Python + # parsing libs in the dev group); 15 min leaves headroom for cold + # uv-sync while still protecting against a stuck job riding the default + # 6-hour timeout. timeout-minutes: 15 steps: diff --git a/pyproject.toml b/pyproject.toml index 649ebf25b1..11c14eb871 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,8 +16,21 @@ validate = "validate:main" [dependency-groups] # Test-only dependencies for the repo tooling (tools/) and skill scripts # (.agents/skills/*/scripts/). Install with `uv sync --dev`. +# +# tomlkit, ruamel.yaml and packaging are imported directly by +# .agents/skills/align-recipe-pyproject/scripts/align_pyproject.py, which +# pytest imports via that skill's tests. Without them, collection of the +# whole suite fails. They live here rather than in [project.dependencies] +# because nothing under tools/ needs them at runtime. +# +# packaging is also imported by .github/scripts/check_recipe_pyproject.py, +# but that gate injects its own deps via `uv run --with` and does not read +# this group -- so it is declared here for the test run, not for that gate. dev = [ "pytest>=8", + "tomlkit>=0.13", + "ruamel.yaml>=0.18", + "packaging>=24", ] [build-system] diff --git a/uv.lock b/uv.lock index 3d45c36765..4ec7638a12 100644 --- a/uv.lock +++ b/uv.lock @@ -13,7 +13,10 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "packaging" }, { name = "pytest" }, + { name = "ruamel-yaml" }, + { name = "tomlkit" }, ] [package.metadata] @@ -23,7 +26,12 @@ requires-dist = [ ] [package.metadata.requires-dev] -dev = [{ name = "pytest", specifier = ">=8" }] +dev = [ + { name = "packaging", specifier = ">=24" }, + { name = "pytest", specifier = ">=8" }, + { name = "ruamel-yaml", specifier = ">=0.18" }, + { name = "tomlkit", specifier = ">=0.13" }, +] [[package]] name = "attrs" @@ -314,6 +322,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, ] +[[package]] +name = "ruamel-yaml" +version = "0.19.1" +source = { registry = "https://pypi.org/simple/" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/3b/ebda527b56beb90cb7652cb1c7e4f91f48649fbcd8d2eb2fb6e77cd3329b/ruamel_yaml-0.19.1.tar.gz", hash = "sha256:53eb66cd27849eff968ebf8f0bf61f46cdac2da1d1f3576dd4ccee9b25c31993", size = 142709, upload-time = "2026-01-02T16:50:31.84Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/0c/51f6841f1d84f404f92463fc2b1ba0da357ca1e3db6b7fbda26956c3b82a/ruamel_yaml-0.19.1-py3-none-any.whl", hash = "sha256:27592957fedf6e0b62f281e96effd28043345e0e66001f97683aa9a40c667c93", size = 118102, upload-time = "2026-01-02T16:50:29.201Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.15.1" +source = { registry = "https://pypi.org/simple/" } +sdist = { url = "https://files.pythonhosted.org/packages/94/96/e07752635b98536177fa1f37671c8f3cdde2e724c6bcf6034b2cfb571565/tomlkit-0.15.1.tar.gz", hash = "sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97", size = 180129, upload-time = "2026-07-17T01:48:04.562Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/bc/8c13eb66537dce1d2bd3a57132902f38d0e7f5bb46fa9f4daed9fe9d76ee/tomlkit-0.15.1-py3-none-any.whl", hash = "sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304", size = 49449, upload-time = "2026-07-17T01:48:05.728Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0"