From 84f38beeeadeb87eae91f6c7ecc0d5e73cda4005 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 27 Aug 2026 07:31:26 +0000 Subject: [PATCH] fix(seo): keep query views and markdown aliases off googlebot The Pages coverage export showed Google fetching ~100k noindex query-string URLs and saturating the origin. Disallow those views and .md aliases for the generic crawler, tag markdown as noindex, cache social cards for a day, and return 503 on uncaught origin failures. Co-authored-by: Steven Lynn --- docs/seo/README.md | 2 +- docs/seo/crawling.md | 25 +++-- docs/seo/indexation.md | 10 +- docs/seo/recommendations.md | 14 +-- docs/seo/search-console.md | 97 ++++++++++++++++++++ frontend/src/pages/artifact-og/og-image.tsx | 27 ++++-- frontend/src/pages/markdown/markdown.test.ts | 1 + frontend/src/pages/markdown/response.ts | 5 + frontend/src/pages/seo/robots.test.ts | 8 ++ frontend/src/pages/seo/robots.ts | 15 +++ frontend/workers/app.ts | 19 +++- frontend/workers/edge-cache.test.ts | 12 +++ frontend/workers/edge-cache.ts | 8 +- 13 files changed, 213 insertions(+), 30 deletions(-) diff --git a/docs/seo/README.md b/docs/seo/README.md index 154c257..2a0a401 100644 --- a/docs/seo/README.md +++ b/docs/seo/README.md @@ -9,7 +9,7 @@ every page the site owns, including every plugin it indexes. - [`indexation.md`](indexation.md) — what is offered to the index and what is withheld. - [`structured-data.md`](structured-data.md) — the schema.org graph and where each node comes from. - [`crawling.md`](crawling.md) — robots.txt, ads.txt, the sitemap set, internal link graph. -- [`search-console.md`](search-console.md) — first Google Search Console export, and what it changed. +- [`search-console.md`](search-console.md) — Search Console exports, and what they changed. - [`recommendations.md`](recommendations.md) — prioritised follow-up work (P0/P1/P2). Language conventions for writing code — adding a locale, adding a key — are in diff --git a/docs/seo/crawling.md b/docs/seo/crawling.md index 4e2c0fe..0761a41 100644 --- a/docs/seo/crawling.md +++ b/docs/seo/crawling.md @@ -2,23 +2,33 @@ ## robots.txt -Served from `frontend/src/pages/seo/robots.ts` at `/robots.txt`. It disallows -`/api/` and names the sitemap index. +Served from `frontend/src/pages/seo/robots.ts` at `/robots.txt`. The generic +crawler (`User-agent: *`, Googlebot included) is allowed the HTML catalog and +disallowed three classes of URL: + +| Rule | Why | +|---|---| +| `Disallow: /api/` | Machine-only JSON. There is no HTML `noindex` to read, so blocking it saves crawl budget without hiding an indexation instruction. | +| `Disallow: /*?` | Every query string is a view of a path that already has a canonical home (`/browse?q=`, `?offset=`, `?profile=`). The HTML of those views is `noindex, follow`; fetching them still costs the budget. Plugin rows on listing page two onwards stay in the artifact sitemap. | +| `Disallow: /*.md$` | Agent markdown aliases of the same HTML documents. Retrieval user-agents keep `Allow: /` in their own group, so they still reach `.md`. `/llms.txt` does not end in `.md` and stays crawlable. | Account pages remain crawlable even though they are `noindex, follow`. A search engine must fetch a page to read that directive; blocking `/dashboard`, `/device`, or `/sign-in` in robots.txt can leave the URL indexed without a snippet because the crawler knows the URL exists but cannot see its `noindex`. -The API is different: it is machine-only JSON with no HTML directive to read, -so blocking it saves crawl budget without hiding an indexation instruction. +Query-string views are the opposite case: they were never supposed to enter +the index, and the coverage export in [`search-console.md`](search-console.md) +showed Google spending the budget confirming `noindex` on ~100k of them. Nothing there is a security boundary. robots.txt is a request, and the paths it names are exactly the paths anyone can read in it. Search/retrieval agents (`OAI-SearchBot`, `ChatGPT-User`, `Claude-SearchBot`, -`Claude-User`) may crawl public pages and the catalog snapshot; training -crawlers (`GPTBot`, `ClaudeBot`) are denied. Public responses also send +`Claude-User`) may crawl public pages, `.md` aliases, and the catalog snapshot; +training crawlers (`GPTBot`, `ClaudeBot`) are denied. Public responses also send `Content-Signal: ai-train=no, search=yes, ai-input=yes, use=reference`. +Markdown responses add `X-Robots-Tag: noindex, follow` so a crawler that +fetches them anyway does not treat the alias as a second document. ## ads.txt @@ -339,7 +349,8 @@ serves every language variant, same as the site card. It has to be a PNG: Slack, X and the other link-preview fetchers do not rasterise SVG `og:image`s, so an SVG route would preview as nothing. The Wasm cost (~1 MB of bundle, ~150–300 ms cold) lands only on link-preview fetches, never on the HTML path; -responses carry the same `public, max-age=3600` contract as the sitemap XML. +responses carry `public, max-age=86400` and are stored in the Worker Cache API +so a crawler wave does not re-render every card. A sibling route, `/a/:artifactId/badge.svg`, serves the shields-style README badge (`dsh.fish | A · 78`, or the star count with `?metric=stars`). The diff --git a/docs/seo/indexation.md b/docs/seo/indexation.md index 8d77351..15576a1 100644 --- a/docs/seo/indexation.md +++ b/docs/seo/indexation.md @@ -33,7 +33,7 @@ The catalog can mint effectively unlimited URLs: any combination of `kind`, `category`, `sort`, `verified`, `q` and `offset` is a distinct query string over the same rows. This is the classic way a directory burns its crawl budget. -Three mechanisms, together: +Three mechanisms in HTML, plus one in robots.txt: 1. **Any query at all makes `/browse` `noindex`.** The loader reports `filtered: [...url.searchParams.keys()].length > 0`. @@ -42,9 +42,15 @@ Three mechanisms, together: and the sitemap carries. A combination filter is a view of it. 3. **The single-facet listings get real paths**, so the terms worth ranking for have a stable, linkable, canonical document instead of a query string. +4. **`robots.txt` `Disallow: /*?` for `User-agent: *`.** `nofollow` is a hint; + Google still fetched ~100k `noindex` query URLs in the first coverage + export. Blocking the fetch keeps that budget for sitemap URLs. Retrieval + agents keep `Allow: /` in their own group. Keyword links on a plugin page point at `/browse?q=` and are -`nofollow` for the same reason. +`nofollow` for the same reason. Pagination stays `noindex, follow` in HTML so +a client that ignores robots.txt can still walk it; the artifact sitemap is +how Google is supposed to find those rows. ## Locale quality gate diff --git a/docs/seo/recommendations.md b/docs/seo/recommendations.md index c32832c..34a2b17 100644 --- a/docs/seo/recommendations.md +++ b/docs/seo/recommendations.md @@ -25,12 +25,14 @@ Then, in Search Console, check the **International Targeting** report. It is the only place a broken `hreflang` cluster shows up as an error rather than as silence. -The domain property is verified and the sitemap is submitted. The first export -is in [`search-console.md`](search-console.md). Remaining operator work from -that export: URL Inspection on `/`, `/browse`, `/docs`, and -`/ko/a/dsh-better-edit`; the International Targeting report for `hreflang` -errors. HTTP and retired-locale URLs already 301 — do not add a second -redirect, wait for them to leave the page report. +The domain property is verified and the sitemap is submitted. Exports are in +[`search-console.md`](search-console.md). Remaining operator work: URL +Inspection on `/`, `/browse`, `/docs`, and `/ko/a/dsh-better-edit`; the +International Targeting report for `hreflang` errors; after the coverage-fix +deploy, validate the "Excluded by `noindex`" set so those URLs move to +"Blocked by robots.txt" instead of being refetched. HTTP and retired-locale +URLs already 301 — do not add a second redirect, wait for them to leave the +page report. ### Confirm `PUBLIC_BASE_URL` is the production origin diff --git a/docs/seo/search-console.md b/docs/seo/search-console.md index 75d97c7..a6b6c36 100644 --- a/docs/seo/search-console.md +++ b/docs/seo/search-console.md @@ -95,3 +95,100 @@ only thing Korea can be shown. The backlog that follows from this export is in [`recommendations.md`](recommendations.md). + +--- + +# Coverage export, 27 August 2026 + +Pages report, property `dsh.fish`, sitemap "All known pages". Chart through +21 August (typical reporting delay). This is the inventory counterpart of the +traffic export above. + +## Totals + +| Slice | Pages | +|---|---| +| Indexed | 2,445 | +| Not indexed | 118,540 | +| Impressions (21 Aug) | 735 | + +The jump on 18 August — 11 indexed / 3 not indexed the day before, then 2,445 / +118,540 — is the sitemap being ingested, not a traffic trend. Impressions then +grew 17 → 735 over three days **without** more URLs entering the index: Google +spent the rest of the budget elsewhere. + +Live catalog at the time of the reading: 5,985 artifacts. Sitemap invitation: + +| File | `` count | +|---|---| +| `/sitemaps/pages.xml` | 264 | +| `/sitemaps/artifacts/0.xml` … `4.xml` | 6,000 each (1,000 artifacts × 6 locales) | +| `/sitemaps/artifacts/5.xml` | 5,910 | +| **Offered** | **~36,174** | + +GSC knew ~120,985 URLs. The extra ~85k were not in the sitemap. + +## Why the extra URLs were `noindex` + +| Reason | Source | Pages | +|---|---|---| +| Excluded by `noindex` tag | Website | 103,428 | +| Discovered – currently not indexed | Google systems | 12,919 | +| Server error (5xx) | Website | 1,306 | +| Crawled – currently not indexed | Google systems | 853 | +| Not found (404) | Website | 20 | +| Page with redirect | Website | 12 | +| Soft 404 | Website | 1 | +| Duplicate, Google chose different canonical | Google systems | 1 | + +Production HTML for `/ja/a/dsh-better-edit` is `index, follow` with +`SEO_LOCALE_GATING=false`. The 103k `noindex` rows are not untranslated plugin +pages. They match the URL classes that already emit `noindex, follow`: + +- `/browse?q=` from the twelve keyword chips on each plugin page + (`rel="nofollow"` is a hint; Google still fetched them) +- `/browse?offset=`, `/kind/…?offset=`, `/category/…?offset=` +- other query views (`?sort=`, `?profile=`, combined filters) + +~5,985 artifacts × up to 12 keywords, plus locale-prefixed copies and +pagination, is the right order of magnitude. Each fetch is a D1 search. A +wave of 100k of those is also the 1,306 5xx: the origin was saturating, not +returning a broken document. + +The 12,919 "discovered, not indexed" and the 853 "crawled, not indexed" are +the leftover **wanted** URLs — sitemap plugin pages waiting because the budget +went to the `noindex` views. Locale shells with English README body and a +translated frame are a quality reason some crawled pages stay out; do not +turn on `SEO_LOCALE_GATING` until the Korean cluster from the first export +is associated. + +12 redirects (HTTP, `/en/`, retired `/de` `/fr` `/es` `/pt-BR`) and 20 404s +are residual. Do not add a second redirect. + +## What changed in code + +Generic crawler (`User-agent: *`): + +- `Disallow: /*?` — stop fetching query-string views; the artifact sitemap + remains the inventory for rows past listing page one. +- `Disallow: /*.md$` — stop fetching agent markdown aliases. Retrieval + user-agents keep `Allow: /` in their own group. + +Markdown responses send `X-Robots-Tag: noindex, follow`. Social cards cache +for a day and sit in the Worker Cache API. An uncaught origin failure is 503 +with `Retry-After`, not a hard 500. + +## Operator follow-up + +1. After deploy, Search Console → Pages → "Excluded by `noindex`" → Validate + fix. The count should move toward "Blocked by robots.txt" as Google recrawls + `robots.txt`, then drop. +2. URL Inspection on `https://dsh.fish/`, `/browse`, `/docs`, a head plugin + page, and `/ko/a/dsh-better-edit`. Confirm `robots.txt` is fetched and the + HTML URL is allowed. +3. Leave `SEO_LOCALE_GATING` off until the Korean CTR test from the first + export has a second crawl to compare. +4. Do not request indexing of `/browse?q=` or `*.md` URLs. + +The backlog that follows from this export is in +[`recommendations.md`](recommendations.md). diff --git a/frontend/src/pages/artifact-og/og-image.tsx b/frontend/src/pages/artifact-og/og-image.tsx index 5fe4e53..0f98d0c 100644 --- a/frontend/src/pages/artifact-og/og-image.tsx +++ b/frontend/src/pages/artifact-og/og-image.tsx @@ -10,7 +10,8 @@ import { artifactOgCard, OG_CARD_SIZE } from './og-card' * (name, grade, counts) changes as the catalog re-crawls, and there are too * many artifacts to pre-render. It is only ever fetched by link-preview * crawlers, so the ~1 MB of Wasm this pulls into the Worker never touches the - * HTML path, and the hour-long cache lifetime matches the sitemap XML. + * HTML path. Cards are cached for a day: a crawler wave that hits every + * plugin page must not re-render Wasm on each fetch. * * A resource route, not a page: the loader is the whole response. */ @@ -24,11 +25,21 @@ export async function loader({ context, params }: Route.LoaderArgs) { return new Response(null, { status: 404 }) } - const png = await renderOgPng(artifactOgCard(artifact), OG_CARD_SIZE) - return new Response(png as BodyInit, { - headers: { - 'content-type': 'image/png', - 'cache-control': 'public, max-age=3600', - }, - }) + try { + const png = await renderOgPng(artifactOgCard(artifact), OG_CARD_SIZE) + return new Response(png as BodyInit, { + headers: { + 'content-type': 'image/png', + // Social cards can lag the catalog by a day. A shorter lifetime + // stampedes Wasm renders when a crawler wave hits every plugin page. + 'cache-control': 'public, max-age=86400', + }, + }) + } catch (error) { + console.error('og_render_failed', params.artifactId, String(error)) + throw new Response(null, { + status: 503, + headers: { 'retry-after': '60' }, + }) + } } diff --git a/frontend/src/pages/markdown/markdown.test.ts b/frontend/src/pages/markdown/markdown.test.ts index ee43545..3504d1e 100644 --- a/frontend/src/pages/markdown/markdown.test.ts +++ b/frontend/src/pages/markdown/markdown.test.ts @@ -120,6 +120,7 @@ describe('maybeMarkdownResponse', () => { expect(response).not.toBeNull() expect(response!.headers.get('content-type')).toBe('text/markdown; charset=utf-8') expect(response!.headers.get('vary')).toBe('accept') + expect(response!.headers.get('x-robots-tag')).toBe('noindex, follow') expect(Number(response!.headers.get('x-markdown-tokens'))).toBeGreaterThan(0) expect(await response!.text()).toContain('# @acme/dsh-hello') }) diff --git a/frontend/src/pages/markdown/response.ts b/frontend/src/pages/markdown/response.ts index ff920bd..027ada5 100644 --- a/frontend/src/pages/markdown/response.ts +++ b/frontend/src/pages/markdown/response.ts @@ -18,6 +18,11 @@ export function markdownResponse(markdown: string, init?: { status?: number }): 'cache-control': status >= 400 ? 'no-store' : 'public, max-age=300', 'x-markdown-tokens': String(estimateTokens(markdown)), 'content-signal': 'ai-train=no, search=yes, ai-input=yes, use=reference', + // Markdown is the agent mirror, not a second document for the HTML index. + // robots.txt already Disallows `*.md` for Googlebot; this header covers + // crawlers that fetch anyway, including `Accept: text/markdown` on the + // HTML URL. + 'x-robots-tag': 'noindex, follow', }, }) } diff --git a/frontend/src/pages/seo/robots.test.ts b/frontend/src/pages/seo/robots.test.ts index f2e8bcd..1de54a3 100644 --- a/frontend/src/pages/seo/robots.test.ts +++ b/frontend/src/pages/seo/robots.test.ts @@ -16,6 +16,14 @@ describe('robotsText', () => { expect(body).not.toContain('Disallow: /*/') }) + it('keeps query-string views and markdown aliases off the generic crawler', () => { + expect(body).toContain('User-agent: *\nAllow: /\nDisallow: /api/\nDisallow: /*?\nDisallow: /*.md$') + // Retrieval agents keep their own Allow: / group, so they still reach + // `.md` aliases and filtered listings. The rules above must not leak into it. + expect(body).toContain('User-agent: OAI-SearchBot\nAllow: /\nAllow: /api/v1/catalog/snapshot\nDisallow: /api/\n') + expect(body).not.toContain('User-agent: OAI-SearchBot\nAllow: /\nAllow: /api/v1/catalog/snapshot\nDisallow: /api/\nDisallow: /*?') + }) + it('allows retrieval agents but denies training crawlers', () => { expect(body).toContain('User-agent: OAI-SearchBot\nAllow: /') expect(body).toContain('Allow: /api/v1/catalog/snapshot') diff --git a/frontend/src/pages/seo/robots.ts b/frontend/src/pages/seo/robots.ts index 7863e22..011880e 100644 --- a/frontend/src/pages/seo/robots.ts +++ b/frontend/src/pages/seo/robots.ts @@ -11,6 +11,16 @@ import { hubContext } from '@/shared/api/hub-context' * `/api/` is disallowed for the same reason: it answers JSON, and a crawler * enumerating it learns nothing the HTML pages do not already say. * + * Query strings and `.md` aliases are disallowed for the generic crawler + * (`User-agent: *`, which is Googlebot). Every `?` URL is a view of a path + * that already has a canonical home — `/browse?q=`, `?offset=`, `?profile=` — + * and the Pages coverage export showed Google spending the + * crawl budget fetching ~100k of those `noindex` views instead of the ~36k + * sitemap URLs. `.md` aliases are the agent mirror of those same pages; + * retrieval bots keep `Allow: /` in their own group so they still reach them. + * Plugin pages on listing page two onwards stay discoverable through the + * artifact sitemap, so blocking `?offset=` does not hide the catalog. + * * Nothing here is a security boundary. robots.txt is a request, and the paths * it names are exactly the paths anyone can read in it. */ @@ -45,6 +55,11 @@ export function robotsText(baseUrl: string): string { 'User-agent: *', 'Allow: /', 'Disallow: /api/', + // Faceted views and pagination. The sitemap already lists every artifact; + // these URLs are `noindex, follow` in HTML and must not spend crawl budget. + 'Disallow: /*?', + // Agent markdown aliases. Retrieval user-agents above keep Allow: /. + 'Disallow: /*.md$', '', '# Atom feeds live at /feed.xml and //feed.xml; the IndexNow key', '# file at /indexnow-.txt. Both are crawlable by design.', diff --git a/frontend/workers/app.ts b/frontend/workers/app.ts index c97694c..406f3a3 100644 --- a/frontend/workers/app.ts +++ b/frontend/workers/app.ts @@ -135,10 +135,21 @@ async function handleRequest( export default { async fetch(request, env, ctx) { - return withPublicSignals( - await withEdgeCache(request, ctx, () => handleRequest(request, env, ctx)), - request, - ) + try { + return withPublicSignals( + await withEdgeCache(request, ctx, () => handleRequest(request, env, ctx)), + request, + ) + } catch (error) { + // An uncaught loader failure is an origin 500. Google records that as a + // hard server error; 503 + Retry-After is the honest signal when D1 or + // CPU is saturated, which is what a coverage burst does. + console.error('worker_fetch_failed', String(error)) + return new Response(null, { + status: 503, + headers: { 'retry-after': '60' }, + }) + } }, /** diff --git a/frontend/workers/edge-cache.test.ts b/frontend/workers/edge-cache.test.ts index 9420ab6..be7b9a6 100644 --- a/frontend/workers/edge-cache.test.ts +++ b/frontend/workers/edge-cache.test.ts @@ -135,4 +135,16 @@ describe('isStorableResponse', () => { false, ) }) + + it('stores artifact social cards so a crawler wave does not re-render Wasm', () => { + expect( + isStorableResponse( + '/a/dsh-hello/og.png', + response({ contentType: 'image/png' }), + ), + ).toBe(true) + expect( + isStorableResponse('/a/dsh-hello/og.png', response({ status: 503, contentType: 'image/png' })), + ).toBe(false) + }) }) diff --git a/frontend/workers/edge-cache.ts b/frontend/workers/edge-cache.ts index 3c76709..d1867ae 100644 --- a/frontend/workers/edge-cache.ts +++ b/frontend/workers/edge-cache.ts @@ -70,8 +70,9 @@ export function isMachineReadablePath(pathname: string): boolean { /** * True when a response may be stored: a 200 without `Set-Cookie`, on either a - * machine-readable route or an anonymous page (`text/html` / `text/markdown` - * outside `/api/`). Redirects, errors, and the rest of the API are never stored. + * machine-readable route, an artifact social card (`/a/:id/og.png`), or an + * anonymous page (`text/html` / `text/markdown` outside `/api/`). Redirects, + * errors, and the rest of the API are never stored. */ export function isStorableResponse(pathname: string, response: Response): boolean { if (response.status !== 200 || response.headers.has('set-cookie')) { @@ -84,6 +85,9 @@ export function isStorableResponse(pathname: string, response: Response): boolea return false } const contentType = response.headers.get('content-type') ?? '' + if (pathname.endsWith('/og.png') && contentType.includes('image/png')) { + return true + } return contentType.includes('text/html') || contentType.includes('text/markdown') }