feat(haproxy): add optional User-Agent policy

README.md

@@ -64,6 +64,16 @@ For Debian / Ubuntu: apt install npm
 
 For production use, generate a random `secret` to place in the Berghain configuration file using `openssl rand -base64 32`.
 
+### Optional User-Agent policy
+
+[`examples/haproxy/haproxy-ua-policy.cfg`](examples/haproxy/haproxy-ua-policy.cfg) is an alternative
+HAProxy example that challenges known AI crawlers and browser-like clients, allows selected tools and public API/feed
+paths, and tarpits selected scraper libraries. The default example remains User-Agent neutral.
+
+Customize the maps under [`examples/haproxy/maps/`](examples/haproxy/maps/) and the `public_endpoint` ACLs before use.
+Map keys are lowercase literal substrings without whitespace. User-Agent headers are trivial to spoof, so this policy is
+traffic shaping only; it must not protect authenticated or otherwise sensitive endpoints.
+
 ## Running with Docker
 
 To run the project using Docker, follow these steps:

examples/haproxy/haproxy-ua-policy.cfg

@@ -0,0 +1,109 @@
+# Optional User-Agent and endpoint policy for Berghain.
+#
+# This is a complete alternative to examples/haproxy/haproxy.cfg. It leaves the
+# default example generic while demonstrating the policy requested in issue #19:
+#
+#   * challenge known AI crawlers and clients claiming to be browsers;
+#   * let explicitly identified command-line clients and feed readers through;
+#   * tarpit selected scraper libraries; and
+#   * leave explicitly public feed/API endpoints unchallenged.
+#
+# User-Agent strings are self-reported. Treat these rules as traffic shaping,
+# never as authentication or authorization. Review all lists and paths before
+# using this policy in production.
+#
+# Validate from the repository root with:
+#   haproxy -c -f examples/haproxy/haproxy-ua-policy.cfg
+
+global
+    log stdout format raw local0
+
+defaults
+    mode http
+    log global
+    timeout client 5s
+    timeout server 5s
+    timeout connect 5s
+    timeout tarpit 5s
+    option httplog
+
+listen stats
+    bind 127.0.0.1:8000
+    stats enable
+    stats uri /
+    stats refresh 15s
+
+frontend test
+    bind *:8080
+    log-format "%ci:%cp\ [%t]\ %ft\ %b/%s\ %Th/%Ti/%TR/%Tq/%Tw/%Tc/%Tr/%Tt\ %ST\ %B\ %CC\ %CS\ %tsc\ %ac/%fc/%bc/%sc/%rc\ %sq/%bq\ %hr\ %hs\ %{+Q}r\ %ID spoa-error:\ %[var(txn.berghain.error)]"
+
+    acl berghain_path path /cdn-cgi/challenge-platform/challenge
+
+    # These paths deliberately bypass the UA policy. Use exact paths or a
+    # slash-terminated prefix so an unrelated path cannot match accidentally.
+    acl public_endpoint path -i /feed /feed.xml /rss /rss.xml /atom.xml
+    acl public_endpoint path_beg -i /api/
+
+    # map_sub performs a literal, case-sensitive substring lookup. Lowercase
+    # the request first because all map keys are lowercase. Separate maps make
+    # precedence deterministic when a UA contains more than one marker.
+    http-request set-var(txn.ua_ai) req.hdr(User-Agent),lower,map_sub(examples/haproxy/maps/ua_ai_crawlers.map,-)
+    http-request set-var(txn.ua_allowed) req.hdr(User-Agent),lower,map_sub(examples/haproxy/maps/ua_allowed_tools.map,-)
+    http-request set-var(txn.ua_tarpit) req.hdr(User-Agent),lower,map_sub(examples/haproxy/maps/ua_tarpit_tools.map,-)
+
+    acl is_ai_crawler var(txn.ua_ai) -m str challenge
+    acl is_allowed_tool var(txn.ua_allowed) -m str allow
+    acl is_tarpit_tool var(txn.ua_tarpit) -m str tarpit
+    acl claims_browser req.hdr(User-Agent) -m sub -i Mozilla/
+
+    # Policy precedence is: public endpoint, tarpit, explicit AI challenge,
+    # browser challenge, pass-through. An allowed-tool marker cannot cancel an
+    # AI-crawler marker. Keep the Berghain endpoint reachable for challenge POSTs.
+    http-request tarpit deny_status 429 if is_tarpit_tool !public_endpoint !berghain_path
+
+    filter spoe engine berghain config examples/haproxy/berghain.cfg
+
+    # The challenge endpoint must receive the same level on its follow-up GET
+    # and POST; only validation and interstitial rendering are path-gated below.
+    http-request set-var(req.berghain.level) int(2) if is_ai_crawler !public_endpoint
+    http-request set-var(req.berghain.level) int(2) if claims_browser !is_allowed_tool !public_endpoint
+
+    acl berghain_active var(req.berghain.level) -m found
+
+    http-request send-spoe-group berghain validate if !berghain_path berghain_active
+    http-request return status 501 if { var(txn.berghain.error) -m found }
+
+    acl berghain_valid var(txn.berghain.valid) -m bool
+    acl is_ssl ssl_fc
+
+    http-request return status 403 content-type "text/html" file "web/dist/default/index.html" if !berghain_valid !berghain_path berghain_active !is_ssl
+    http-request return status 403 content-type "text/html" file "web/dist/native-crypto/index.html" if !berghain_valid !berghain_path berghain_active is_ssl
+    http-request wait-for-body time 5s if berghain_path METH_POST
+    use_backend berghain_http if berghain_path
+
+    default_backend app_backend
+
+backend app_backend
+    mode http
+    http-request return status 200 content-type "text/plain" string "Hello World!"
+
+backend berghain_http
+    mode http
+    filter spoe engine berghain_challenge config examples/haproxy/berghain.cfg
+
+    acl is_challenge_path path /cdn-cgi/challenge-platform/challenge
+
+    http-request send-spoe-group berghain_challenge challenge if is_challenge_path
+    http-request return status 501 if { var(txn.berghain.error) -m found }
+
+    acl has_token var(txn.berghain.token) -m found
+
+    http-after-response add-header set-cookie "berghain=%[var(txn.berghain.token)]; %[var(txn.berghain.domain)] path=/;" if has_token
+    http-request return status 200 content-type "application/json" lf-string "%[var(txn.berghain.response)]" if is_challenge_path
+
+    http-request return status 404
+
+backend berghain_spop
+    mode tcp
+    option spop-check
+    server localhost unix@./spop.sock check

examples/haproxy/maps/ua_ai_crawlers.map

@@ -0,0 +1,22 @@
+# Literal lowercase substrings matched with map_sub(). Map keys cannot contain
+# whitespace. Format: <substring> challenge. Illustrative; keep this current.
+gptbot challenge
+oai-searchbot challenge
+chatgpt-user challenge
+claudebot challenge
+claude-web challenge
+anthropic-ai challenge
+ccbot challenge
+google-extended challenge
+googleother challenge
+bytespider challenge
+perplexitybot challenge
+amazonbot challenge
+applebot-extended challenge
+meta-externalagent challenge
+facebookbot challenge
+imagesiftbot challenge
+diffbot challenge
+cohere-ai challenge
+timpibot challenge
+omgili challenge

examples/haproxy/maps/ua_allowed_tools.map

@@ -0,0 +1,14 @@
+# Literal lowercase substrings matched with map_sub(). Map keys cannot contain
+# whitespace. Format: <substring> allow. These markers only exempt the generic
+# browser-UA rule; entries in ua_ai_crawlers.map still require a challenge.
+curl/ allow
+wget/ allow
+feedly allow
+feedbin allow
+inoreader allow
+newsblur allow
+theoldreader allow
+feedparser/ allow
+ttrss allow
+miniflux/ allow
+nextcloud-news/ allow

examples/haproxy/maps/ua_tarpit_tools.map

@@ -0,0 +1,14 @@
+# Literal lowercase substrings matched with map_sub(). Map keys cannot contain
+# whitespace. Format: <substring> tarpit. Tarpits consume a connection for the
+# configured timeout; keep this list narrow and capacity-plan before enabling it.
+python-requests/ tarpit
+python-urllib/ tarpit
+aiohttp/ tarpit
+python-httpx/ tarpit
+go-http-client/ tarpit
+libwww-perl/ tarpit
+okhttp/ tarpit
+node-fetch/ tarpit
+axios/ tarpit
+scrapy/ tarpit
+guzzlehttp/ tarpit