diff --git a/README.md b/README.md index 9345aca..cb7798f 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,16 @@ For Debian / Ubuntu: apt install npm For production use, generate a random `secret` to place in the Berghain configuration file using `openssl rand -base64 32`. +### Optional User-Agent policy + +[`examples/haproxy/haproxy-ua-policy.cfg`](examples/haproxy/haproxy-ua-policy.cfg) is an alternative +HAProxy example that challenges known AI crawlers and browser-like clients, allows selected tools and public API/feed +paths, and tarpits selected scraper libraries. The default example remains User-Agent neutral. + +Customize the maps under [`examples/haproxy/maps/`](examples/haproxy/maps/) and the `public_endpoint` ACLs before use. +Map keys are lowercase literal substrings without whitespace. User-Agent headers are trivial to spoof, so this policy is +traffic shaping only; it must not protect authenticated or otherwise sensitive endpoints. + ## Running with Docker To run the project using Docker, follow these steps: diff --git a/examples/haproxy/haproxy-ua-policy.cfg b/examples/haproxy/haproxy-ua-policy.cfg new file mode 100644 index 0000000..bf7bb93 --- /dev/null +++ b/examples/haproxy/haproxy-ua-policy.cfg @@ -0,0 +1,109 @@ +# Optional User-Agent and endpoint policy for Berghain. +# +# This is a complete alternative to examples/haproxy/haproxy.cfg. It leaves the +# default example generic while demonstrating the policy requested in issue #19: +# +# * challenge known AI crawlers and clients claiming to be browsers; +# * let explicitly identified command-line clients and feed readers through; +# * tarpit selected scraper libraries; and +# * leave explicitly public feed/API endpoints unchallenged. +# +# User-Agent strings are self-reported. Treat these rules as traffic shaping, +# never as authentication or authorization. Review all lists and paths before +# using this policy in production. +# +# Validate from the repository root with: +# haproxy -c -f examples/haproxy/haproxy-ua-policy.cfg + +global + log stdout format raw local0 + +defaults + mode http + log global + timeout client 5s + timeout server 5s + timeout connect 5s + timeout tarpit 5s + option httplog + +listen stats + bind 127.0.0.1:8000 + stats enable + stats uri / + stats refresh 15s + +frontend test + bind *:8080 + log-format "%ci:%cp\ [%t]\ %ft\ %b/%s\ %Th/%Ti/%TR/%Tq/%Tw/%Tc/%Tr/%Tt\ %ST\ %B\ %CC\ %CS\ %tsc\ %ac/%fc/%bc/%sc/%rc\ %sq/%bq\ %hr\ %hs\ %{+Q}r\ %ID spoa-error:\ %[var(txn.berghain.error)]" + + acl berghain_path path /cdn-cgi/challenge-platform/challenge + + # These paths deliberately bypass the UA policy. Use exact paths or a + # slash-terminated prefix so an unrelated path cannot match accidentally. + acl public_endpoint path -i /feed /feed.xml /rss /rss.xml /atom.xml + acl public_endpoint path_beg -i /api/ + + # map_sub performs a literal, case-sensitive substring lookup. Lowercase + # the request first because all map keys are lowercase. Separate maps make + # precedence deterministic when a UA contains more than one marker. + http-request set-var(txn.ua_ai) req.hdr(User-Agent),lower,map_sub(examples/haproxy/maps/ua_ai_crawlers.map,-) + http-request set-var(txn.ua_allowed) req.hdr(User-Agent),lower,map_sub(examples/haproxy/maps/ua_allowed_tools.map,-) + http-request set-var(txn.ua_tarpit) req.hdr(User-Agent),lower,map_sub(examples/haproxy/maps/ua_tarpit_tools.map,-) + + acl is_ai_crawler var(txn.ua_ai) -m str challenge + acl is_allowed_tool var(txn.ua_allowed) -m str allow + acl is_tarpit_tool var(txn.ua_tarpit) -m str tarpit + acl claims_browser req.hdr(User-Agent) -m sub -i Mozilla/ + + # Policy precedence is: public endpoint, tarpit, explicit AI challenge, + # browser challenge, pass-through. An allowed-tool marker cannot cancel an + # AI-crawler marker. Keep the Berghain endpoint reachable for challenge POSTs. + http-request tarpit deny_status 429 if is_tarpit_tool !public_endpoint !berghain_path + + filter spoe engine berghain config examples/haproxy/berghain.cfg + + # The challenge endpoint must receive the same level on its follow-up GET + # and POST; only validation and interstitial rendering are path-gated below. + http-request set-var(req.berghain.level) int(2) if is_ai_crawler !public_endpoint + http-request set-var(req.berghain.level) int(2) if claims_browser !is_allowed_tool !public_endpoint + + acl berghain_active var(req.berghain.level) -m found + + http-request send-spoe-group berghain validate if !berghain_path berghain_active + http-request return status 501 if { var(txn.berghain.error) -m found } + + acl berghain_valid var(txn.berghain.valid) -m bool + acl is_ssl ssl_fc + + http-request return status 403 content-type "text/html" file "web/dist/default/index.html" if !berghain_valid !berghain_path berghain_active !is_ssl + http-request return status 403 content-type "text/html" file "web/dist/native-crypto/index.html" if !berghain_valid !berghain_path berghain_active is_ssl + http-request wait-for-body time 5s if berghain_path METH_POST + use_backend berghain_http if berghain_path + + default_backend app_backend + +backend app_backend + mode http + http-request return status 200 content-type "text/plain" string "Hello World!" + +backend berghain_http + mode http + filter spoe engine berghain_challenge config examples/haproxy/berghain.cfg + + acl is_challenge_path path /cdn-cgi/challenge-platform/challenge + + http-request send-spoe-group berghain_challenge challenge if is_challenge_path + http-request return status 501 if { var(txn.berghain.error) -m found } + + acl has_token var(txn.berghain.token) -m found + + http-after-response add-header set-cookie "berghain=%[var(txn.berghain.token)]; %[var(txn.berghain.domain)] path=/;" if has_token + http-request return status 200 content-type "application/json" lf-string "%[var(txn.berghain.response)]" if is_challenge_path + + http-request return status 404 + +backend berghain_spop + mode tcp + option spop-check + server localhost unix@./spop.sock check diff --git a/examples/haproxy/maps/ua_ai_crawlers.map b/examples/haproxy/maps/ua_ai_crawlers.map new file mode 100644 index 0000000..9817cff --- /dev/null +++ b/examples/haproxy/maps/ua_ai_crawlers.map @@ -0,0 +1,22 @@ +# Literal lowercase substrings matched with map_sub(). Map keys cannot contain +# whitespace. Format: challenge. Illustrative; keep this current. +gptbot challenge +oai-searchbot challenge +chatgpt-user challenge +claudebot challenge +claude-web challenge +anthropic-ai challenge +ccbot challenge +google-extended challenge +googleother challenge +bytespider challenge +perplexitybot challenge +amazonbot challenge +applebot-extended challenge +meta-externalagent challenge +facebookbot challenge +imagesiftbot challenge +diffbot challenge +cohere-ai challenge +timpibot challenge +omgili challenge diff --git a/examples/haproxy/maps/ua_allowed_tools.map b/examples/haproxy/maps/ua_allowed_tools.map new file mode 100644 index 0000000..35728e2 --- /dev/null +++ b/examples/haproxy/maps/ua_allowed_tools.map @@ -0,0 +1,14 @@ +# Literal lowercase substrings matched with map_sub(). Map keys cannot contain +# whitespace. Format: allow. These markers only exempt the generic +# browser-UA rule; entries in ua_ai_crawlers.map still require a challenge. +curl/ allow +wget/ allow +feedly allow +feedbin allow +inoreader allow +newsblur allow +theoldreader allow +feedparser/ allow +ttrss allow +miniflux/ allow +nextcloud-news/ allow diff --git a/examples/haproxy/maps/ua_tarpit_tools.map b/examples/haproxy/maps/ua_tarpit_tools.map new file mode 100644 index 0000000..da4f639 --- /dev/null +++ b/examples/haproxy/maps/ua_tarpit_tools.map @@ -0,0 +1,14 @@ +# Literal lowercase substrings matched with map_sub(). Map keys cannot contain +# whitespace. Format: tarpit. Tarpits consume a connection for the +# configured timeout; keep this list narrow and capacity-plan before enabling it. +python-requests/ tarpit +python-urllib/ tarpit +aiohttp/ tarpit +python-httpx/ tarpit +go-http-client/ tarpit +libwww-perl/ tarpit +okhttp/ tarpit +node-fetch/ tarpit +axios/ tarpit +scrapy/ tarpit +guzzlehttp/ tarpit