feat(crawl): add exclude URL regex to crawl and sitemap endpoints (#1982)

Stainless-Generated-From: 1d0af2f2628afa39c837cb2a1d8a5837e5f0d929

Author: stlc-bot

pkg/cmd/web.go

@@ -483,6 +483,11 @@ var webWebCrawlMd = requestflag.WithInnerFlags(cli.Command{
 			Usage:    "Remove matching elements after inclusions. Exclusions take precedence.",
 			BodyPath: "excludeSelectors",
 		},
+		&requestflag.Flag[string]{
+			Name:     "exclude-url-regex",
+			Usage:    "RE2 regex pattern. URLs matching this pattern are never followed or scraped, even when they match urlRegex. The starting URL is always scraped.",
+			BodyPath: "excludeUrlRegex",
+		},
 		&requestflag.Flag[bool]{
 			Name:     "follow-subdomains",
 			Usage:    "When true, follow links on subdomains of the starting URL's domain (e.g. docs.example.com when starting from example.com). www and apex are always treated as equivalent.",

pkg/cmd/web_test.go

@@ -325,6 +325,7 @@ func TestWebWebCrawlMd(t *testing.T) {
 			"--url", "https://example.com",
 			"--country", "de",
 			"--exclude-selector", "[x]",
+			"--exclude-url-regex", "/tag/|/author/",
 			"--follow-subdomains=true",
 			"--include-frames=true",
 			"--include-images=true",
@@ -359,6 +360,7 @@ func TestWebWebCrawlMd(t *testing.T) {
 			"--url", "https://example.com",
 			"--country", "de",
 			"--exclude-selector", "[x]",
+			"--exclude-url-regex", "/tag/|/author/",
 			"--follow-subdomains=true",
 			"--include-frames=true",
 			"--include-images=true",
@@ -392,6 +394,7 @@ func TestWebWebCrawlMd(t *testing.T) {
 			"country: de\n" +
 			"excludeSelectors:\n" +
 			"  - x\n" +
+			"excludeUrlRegex: /tag/|/author/\n" +
 			"followSubdomains: true\n" +
 			"includeFrames: true\n" +
 			"includeImages: true\n" +