diff --git a/.env.example b/.env.example index d787eb9..376e6d8 100644 --- a/.env.example +++ b/.env.example @@ -37,3 +37,9 @@ VAPID_SUBJECT=mailto:admin@threatcrush.com # ─── Self-Hosting ─── # PORT=3000 # NODE_ENV=production + +# Crawl gateway (@profullstack/x402-gateway): AI training crawlers pay $1/day over +# x402. A SCOPED CoinPay key (payments:create) and the EVM address that receives +# the USDC. Unset = crawlers still get 402, nothing sold. +COINPAY_X402_KEY= +CRAWL_PAY_TO= diff --git a/apps/web/package.json b/apps/web/package.json index 7afb89c..e1a7950 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -13,6 +13,7 @@ "@profullstack/autoblog": "github:profullstack/autoblog#v0.4.0", "@profullstack/pluginstore": "^0.1.1", "@profullstack/stack": "^0.1.3", + "@profullstack/x402-gateway": "^0.1.0", "@supabase/supabase-js": "^2.101.1", "@threatcrush/scan": "workspace:*", "isomorphic-dompurify": "^3.12.0", diff --git a/apps/web/src/app/robots.ts b/apps/web/src/app/robots.ts deleted file mode 100644 index 8ede898..0000000 --- a/apps/web/src/app/robots.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { MetadataRoute } from "next"; -import { SITE_URL } from "@/lib/blog"; - -const DISALLOW = ["/admin", "/admin/", "/account", "/dashboard", "/api/"]; - -const AI_CRAWLERS = [ - "GPTBot", - "OAI-SearchBot", - "ChatGPT-User", - "ClaudeBot", - "anthropic-ai", - "Claude-Web", - "PerplexityBot", - "Perplexity-User", - "Google-Extended", - "Applebot-Extended", - "CCBot", - "Amazonbot", - "Bytespider", - "Meta-ExternalAgent", - "Meta-ExternalFetcher", - "DuckAssistBot", - "YouBot", - "cohere-ai", - "Diffbot", - "FacebookBot", - "Timpibot", -]; - -export default function robots(): MetadataRoute.Robots { - return { - rules: [ - { userAgent: "*", allow: "/", disallow: DISALLOW }, - ...AI_CRAWLERS.map((userAgent) => ({ - userAgent, - allow: "/", - disallow: DISALLOW, - })), - ], - sitemap: `${SITE_URL}/sitemap.xml`, - host: SITE_URL, - }; -} diff --git a/apps/web/src/app/robots.txt/route.ts b/apps/web/src/app/robots.txt/route.ts new file mode 100644 index 0000000..319d762 --- /dev/null +++ b/apps/web/src/app/robots.txt/route.ts @@ -0,0 +1,9 @@ +import { robotsRoute } from "@profullstack/x402-gateway/next"; +import { gateway } from "@/lib/crawl-gateway"; + +// Generated from the same crawler lists the gateway enforces: training +// crawlers are refused everywhere but /crawl (where they can buy a pass), +// retrieval crawlers are named as welcome, everyone else gets the rules below. +export const GET = robotsRoute(gateway, { + disallow: ["/api/"], +}); diff --git a/apps/web/src/lib/crawl-gateway.ts b/apps/web/src/lib/crawl-gateway.ts new file mode 100644 index 0000000..f38c549 --- /dev/null +++ b/apps/web/src/lib/crawl-gateway.ts @@ -0,0 +1,27 @@ +import { createGateway } from "@profullstack/x402-gateway"; +import { x402Proxy } from "@profullstack/x402-gateway/next"; + +/** + * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, + * meta-externalagent, Bytespider, Applebot-Extended, ...) by the day over + * x402, settled by CoinPay in USDC. People, Googlebot and the retrieval + * crawlers behind AI search pass through untouched. + * + * Runs inside the middleware, so nothing here may import Node-only modules. + * The env is read through a non-literal key on purpose: Next inlines + * `process.env.NAME` at build time, and these are runtime secrets. Without + * COINPAY_X402_KEY and CRAWL_PAY_TO the gateway still answers training + * crawlers with 402, just with an empty offer. + */ +const env = (name: string) => process.env[name]; + +export const gateway = createGateway({ + siteUrl: env("SITE_URL") || env("NEXT_PUBLIC_SITE_URL") || "https://threatcrush.com", + siteName: "ThreatCrush", + coinpay: { apiKey: env("COINPAY_X402_KEY") }, + payTo: env("CRAWL_PAY_TO"), + contact: "mailto:support@threatcrush.com", +}); + +/** Resolves to a Response for a refused crawler, or undefined to carry on. */ +export const gate = x402Proxy(gateway); diff --git a/apps/web/src/middleware.ts b/apps/web/src/middleware.ts index 98de259..1eb517f 100644 --- a/apps/web/src/middleware.ts +++ b/apps/web/src/middleware.ts @@ -1,3 +1,4 @@ +import { gate } from "@/lib/crawl-gateway"; import { NextResponse } from "next/server"; import type { NextRequest } from "next/server"; @@ -83,7 +84,13 @@ function rateLimit(req: NextRequest): NextResponse | undefined { return undefined; } -export function middleware(req: NextRequest): NextResponse | undefined { +export async function middleware(req: NextRequest): Promise { + // Crawl gateway first: AI training crawlers get 402 Payment Required (or the + // sales page at /crawl) unless they present a paid pass. People, Googlebot + // and retrieval crawlers fall through to everything below. + const answer = await gate(req); + if (answer) return answer; + if (req.headers.get("next-action")) { return new NextResponse(null, { status: 410 }); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d74e9bf..c25af17 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -334,6 +334,9 @@ importers: '@profullstack/stack': specifier: ^0.1.3 version: 0.1.3(next@16.2.2(@opentelemetry/api@1.9.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4) + '@profullstack/x402-gateway': + specifier: ^0.1.0 + version: 0.1.0 '@supabase/supabase-js': specifier: ^2.101.1 version: 2.101.1 @@ -2874,6 +2877,10 @@ packages: react: optional: true + '@profullstack/x402-gateway@0.1.0': + resolution: {integrity: sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ==} + engines: {node: '>=20.11'} + '@radix-ui/react-compose-refs@1.0.0': resolution: {integrity: sha512-0KaSv6sx787/hK3eF53iOkiSLwAGlFMx5lotrqD2pTjB18KbybKoEIgkNZTKC60YECDQTKGTRcDBILwZVqVKvA==} peerDependencies: @@ -11297,6 +11304,8 @@ snapshots: next: 16.2.2(@opentelemetry/api@1.9.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) react: 19.2.4 + '@profullstack/x402-gateway@0.1.0': {} + '@radix-ui/react-compose-refs@1.0.0(react@18.3.1)': dependencies: '@babel/runtime': 7.29.2