feat: add exposure map collectors

.env.example

@@ -29,6 +29,10 @@ DEFAULT_ADMIN_DISPLAY_NAME=Administrator
 LOG_LEVEL=INFO
 SCHEDULER_API_BASE=http://127.0.0.1:8088
 SCHEDULER_METRICS_PORT=9100
+BRAIN_HOST_PORT=8088
+FRONTEND_HOST_PORT=8080
+HOMELABSEC_UAT_FRONTEND_BIND=0.0.0.0
+HOMELABSEC_UAT_FRONTEND_PORT=18080
 TARGET_SUBNET=10.0.0.0/24
 DISCOVERY_INTERVAL_MINUTES=30
 REPORT_HOUR_UTC=8
@@ -55,8 +59,9 @@ EDGE_OIDC_CLIENT_SECRET=
 EDGE_OIDC_COOKIE_SECRET=
 EDGE_OIDC_REDIRECT_URL=
 EDGE_OIDC_EMAIL_DOMAINS=*
-EDGE_OIDC_SCOPE=openid email profile
+EDGE_OIDC_SCOPE="openid email profile"
 EDGE_OIDC_WHITELIST_DOMAINS=
+MONITORING_HOST_BIND=127.0.0.1
 PROMETHEUS_HOST_PORT=9090
 ALERTMANAGER_HOST_PORT=9093
 ALERTMANAGER_DEFAULT_RECEIVER=null

.env.uat.example

@@ -0,0 +1,99 @@
+# HomelabSec UAT / single-host deployment example.
+#
+# Copy to .env on the deployment host and change secrets before use:
+#   cp .env.uat.example .env
+#
+# This profile mirrors the pi4-style deployment: private Postgres, host-local
+# API, LAN/reverse-proxy frontend, and monitoring services on non-default ports.
+
+POSTGRES_DB=homelabsec
+POSTGRES_USER=homelabsec
+POSTGRES_PASSWORD=change-me
+
+OLLAMA_URL=http://host.containers.internal:11434
+OLLAMA_HOST_URL=http://localhost:11434
+OLLAMA_MODEL=homelabsec-classifier
+OLLAMA_TIMEOUT_SECONDS=120
+
+FINGERBANK_ENABLED=true
+FINGERBANK_API_KEY=
+FINGERBANK_BASE_URL=https://api.fingerbank.org
+FINGERBANK_TIMEOUT_SECONDS=10
+FINGERBANK_MIN_SCORE_ACCEPT=51
+FINGERBANK_MIN_SCORE_AUTO_ACCEPT=76
+
+COLLECTORS_ENABLED=true
+COLLECTOR_INTERFACE=any
+COLLECTOR_DHCP_ENABLED=true
+COLLECTOR_MDNS_ENABLED=true
+COLLECTOR_SSDP_ENABLED=true
+
+CLASSIFICATION_FALLBACK_ROLE=unknown
+CLASSIFICATION_FALLBACK_CONFIDENCE=0.10
+OBSERVATIONS_LIST_LIMIT=200
+FINGERPRINTS_LIST_LIMIT=200
+NOTABLE_ASSET_LIMIT=20
+ADMIN_STALE_SCAN_MINUTES=90
+AUTH_SESSION_DAYS=7
+DEFAULT_ADMIN_USERNAME=admin
+DEFAULT_ADMIN_PASSWORD=change-me-now
+DEFAULT_ADMIN_DISPLAY_NAME=Administrator
+LOG_LEVEL=INFO
+
+# UAT port layout. These avoid common defaults so the stack can coexist with
+# Nginx Proxy Manager, other monitoring systems, and local development stacks.
+BRAIN_HOST_PORT=18088
+HOMELABSEC_UAT_FRONTEND_BIND=0.0.0.0
+HOMELABSEC_UAT_FRONTEND_PORT=18080
+MONITORING_HOST_BIND=0.0.0.0
+PROMETHEUS_HOST_PORT=19090
+GRAFANA_HOST_PORT=13001
+ALERTMANAGER_HOST_PORT=19093
+
+SCHEDULER_METRICS_PORT=9100
+TARGET_SUBNET=10.0.0.0/24
+DISCOVERY_INTERVAL_MINUTES=30
+REPORT_HOUR_UTC=8
+TOP_PORTS=100
+API_RETRY_ATTEMPTS=5
+API_RETRY_DELAY_SECONDS=5
+STARTUP_API_TIMEOUT_SECONDS=120
+STARTUP_DISCOVERY=false
+
+LYNIS_POLL_INTERVAL_SECONDS=10
+LYNIS_SSH_TIMEOUT_SECONDS=30
+LYNIS_AUDIT_TIMEOUT_SECONDS=1800
+LYNIS_REPO_URL=https://github.com/CISOfy/lynis.git
+
+# Optional secure edge overlay values. Leave unused unless starting
+# compose.exposed.yaml or compose.oidc.yaml.
+EDGE_AUTH_USERNAME=admin
+EDGE_AUTH_PASSWORD=change-me-now
+EDGE_AUTH_MODE=basic
+EDGE_SERVER_NAME=localhost
+EDGE_TLS_MODE=self_signed
+EDGE_HTTP_PORT=18081
+EDGE_HTTPS_PORT=18443
+EDGE_OIDC_PROVIDER=oidc
+EDGE_OIDC_ISSUER_URL=
+EDGE_OIDC_CLIENT_ID=
+EDGE_OIDC_CLIENT_SECRET=
+EDGE_OIDC_COOKIE_SECRET=
+EDGE_OIDC_REDIRECT_URL=
+EDGE_OIDC_EMAIL_DOMAINS=*
+EDGE_OIDC_SCOPE="openid email profile"
+EDGE_OIDC_WHITELIST_DOMAINS=
+
+# Alertmanager defaults to a null receiver so the stack is safe before real
+# notification targets are configured.
+ALERTMANAGER_DEFAULT_RECEIVER=null
+ALERTMANAGER_WEBHOOK_URL=
+ALERTMANAGER_VALIDATION_WEBHOOK_URL=
+ALERTMANAGER_EMAIL_TO=
+ALERTMANAGER_EMAIL_FROM=
+ALERTMANAGER_SMARTHOST=
+ALERTMANAGER_SMTP_AUTH_USERNAME=
+ALERTMANAGER_SMTP_AUTH_PASSWORD=
+ALERTMANAGER_SMTP_REQUIRE_TLS=true
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=change-me-now

.gitignore

@@ -22,6 +22,16 @@ discovery/parsed/
 # Secrets
 .env
 *.env
+secrets/runtime/
+secrets/plaintext/
+secrets/tmp/
+*.key
+*.pem
+*.p12
+*.pfx
+*.kdbx
+*.vault
+*bitwarden*export*
 
 # OS
 .DS_Store

.pytest-alertmanager.yml

@@ -0,0 +1,24 @@
+
+route:
+  receiver: "null"
+  group_by: ["alertname", "job"]
+  group_wait: 10s
+  group_interval: 30s
+  repeat_interval: 4h
+
+  routes:
+    - receiver: "delivery-validation"
+      matchers:
+        - alertname: "HomelabSecDeliveryValidation"
+
+
+receivers:
+  - name: "null"
+
+  - name: "delivery-validation"
+    webhook_configs:
+      - url: "http://host.containers.internal:19094/homelabsec-alert-validation"
+        send_resolved: true
+
+
+templates: []

.sops.yaml

@@ -0,0 +1,8 @@
+# SOPS configuration for HomelabSec automation secrets.
+#
+# Only commit files that match the encrypted path rules below. Plaintext
+# runtime files are generated locally by scripts/secrets/render_sops_env.py and
+# must remain ignored.
+creation_rules:
+  - path_regex: ^secrets/sops/.*\.enc\.(json|yaml|yml)$
+    age: age16zlfacpnp7yszl9jxp8ua3t3dtvd9zed625qkt7cpmqvd7uht55qxfnmuf

BACKLOG.md

@@ -23,6 +23,90 @@ Use it as the working queue. `TODO.md` remains the broader status and historical
 
 ## Current Prioritized Queue
 
+### Operational P0: Secret Management Programme
+Priority: `P0`
+Status: `in progress`
+
+Goal:
+- Bring API keys, SSH keys, tokens, certificates, recovery material, and service credentials under deliberate management before broad backup implementation.
+
+Reference:
+- `docs/operations/homelab-secret-management-strategy.md`
+- `docs/operations/secret-management-runbook.md`
+- `docs/operations/secret-inventory-template.md`
+- `secrets/README.md`
+
+Delivered so far:
+- installed Faye-side `age`, `sops`, and Bitwarden CLI tooling with Bitwarden CLI pointed at Bitwarden EU
+- generated Faye's initial SOPS age recipient and wired it into `.sops.yaml`
+- added an encrypted sample secret bundle, metadata-only inventory example, guardrail validator, and local env renderer
+
+Deliver:
+- use Robert's existing Bitwarden EU account as the primary human/recovery vault and SOPS + age as the automation secret format
+- inventory high-value secrets by metadata only: owner, consumer, storage, rotation, recovery test, blast radius, and revocation path
+- migrate known high-value secrets out of ad-hoc `.env` files and loose key locations into Bitwarden records, SOPS-encrypted files, or documented local materialization steps
+- create and test encrypted backups for the Bitwarden/export recovery path, SOPS/age recovery material, backup repository passwords, and break-glass runbook
+- rotate old, broad, or unclear-provenance credentials in staged batches
+- add HomelabSec posture checks later without collecting raw secret values
+
+Safety rules:
+- never put plaintext private keys, tokens, passwords, seeds, macaroons, backup repository passwords, rendered `.env` files, or vault exports in GitHub, backlog files, dashboards, alerts, logs, or Telegram
+- report inventory coverage, freshness, rotation due dates, and recovery-test status only
+
+Next action:
+- with Robert present, login/unlock Bitwarden EU, create recovery records for Faye's SOPS age identity, and start migrating the first real HomelabSec runtime secrets into encrypted SOPS bundles.
+
+### Operational P0: Self-Hosted Git Migration Programme
+Priority: `P0`
+Status: `planned`
+
+Goal:
+- Move selected repositories from GitHub-only hosting to a self-hosted Git service without losing mirrors, backups, recovery options, or public-discovery benefits where they matter.
+
+Reference:
+- `docs/operations/self-hosted-git-migration-strategy.md`
+
+Deliver:
+- choose the self-hosted Git platform, with Forgejo as the current recommended pilot
+- deploy the platform behind the homelab HTTPS/DNS/certificate chain with SSH Git access
+- implement backup and restore drills before moving operational repositories
+- migrate low-risk pilot repositories first, then active personal repositories, then operational/private repositories
+- keep GitHub mirrors or fallback remotes until restore and rollback are proven
+- defer high-impact autonomous/financial repositories until explicit go/no-go after lower-risk migration success
+
+Safety rules:
+- do not delete or archive GitHub repositories during the pilot phase
+- do not print or commit GitHub/Forgejo tokens, deploy keys, webhook secrets, Actions secrets, or private SSH keys
+- do not make self-hosted Git the only copy of an important repository until backup and restore are verified
+
+Next action:
+- confirm Forgejo as the pilot platform or choose an alternative, then write/deploy the first LAN-only Forgejo runbook and test it with a Tier 0 repository.
+
+### Operational P0: Thunderbluff 3-2-1 Backup Programme
+Priority: `P0`
+Status: `planned`
+
+Goal:
+- Make Thunderbluff the primary encrypted backup landing zone, then complete a 3-2-1 posture with an independent/offsite or offline third copy.
+
+Reference:
+- `docs/operations/homelab-backup-strategy.md`
+
+Deliver:
+- confirm Thunderbluff backup share/path, access, capacity, snapshot/immutability support, and restricted backup users/keys
+- inventory all important homelab apps and classify by RPO/RTO/data criticality
+- implement the first three monitored jobs: Faye/Hermes runtime, HomelabSec Postgres/manifests, and proxy/DNS/certificate control-plane state
+- extend to Home Assistant, Trading Team, WordPress, media/document apps, monitoring, Proxmox guests, and Lightning/Umbrel critical state
+- replicate encrypted backups to a separate/offsite/offline third copy and run quarterly restore drills
+
+Safety rules:
+- do not start implementation until Robert explicitly asks; this is currently a planning/backlog item
+- application-aware database dumps before raw volume copies
+- report presence, age, size, snapshot IDs, and restore-test status only; never expose backup contents or secrets
+
+Next action:
+- secret-management choice is made; next backup step, only after Robert explicitly asks to start backup implementation, is to obtain/verify Thunderbluff access and implement the first three jobs.
+
 ### Slice 1: Alert Routing
 Priority: `P0`
 Status: `done`
@@ -176,6 +260,59 @@ Next priority:
 - richer alert delivery validation
 - backup retention policy and off-host storage
 
+### Slice 11: Exposure Map Dashboard
+Priority: `P0`
+Status: `in progress`
+
+Goal:
+- Turn HomelabSec into an operator-facing exposure map by correlating Nmap observations with HCM targets/certificates, NPM routes, Heimdall launcher links, and DNS answers.
+
+Reference:
+- `docs/threat-exposure-map-and-dashboard-spec.md`
+
+Deliver:
+- add route, DNS, launcher-link, and exposure-finding data models
+- add read-only exposure summary/routes/DNS/launcher/findings APIs
+- add secret-safe HCM and NPM collectors
+- add Heimdall hygiene checks for raw-IP or legacy links where a preferred HTTPS route exists
+- add DNS alignment checks that distinguish candidate records from production failures
+- add dashboard cards/tables for control-plane risk, raw service exposure, TLS status, unknown services, and accepted findings
+
+Delivered so far:
+- Added migration-backed exposure tables for routes, DNS records, launcher links, and exposure findings.
+- Added authenticated read-only API skeletons for `/exposure/summary`, `/exposure/routes`, `/exposure/dns`, `/exposure/launcher-links`, and `/exposure/findings`.
+- Added a dashboard exposure-map panel and frontend/API contract tests.
+
+Remaining:
+- Add unit tests for route classifiers and finding severity rules once collectors/classifiers are introduced.
+- Add fixture-based integration tests joining Nmap, HCM, NPM, Heimdall, and DNS payloads.
+- Add secret-safe HCM, NPM, Heimdall, and DNS collectors.
+- Add dashboard tables/cards for correlated control-plane risk, raw service exposure, TLS status, unknown services, and accepted findings.
+
+### Slice 12: Sandfly Security Finding Integration
+Priority: `P2`
+Status: `parked`
+
+Goal:
+- Add Sandfly support as a future enrichment/source-of-findings integration for HomelabSec, after the core exposure map work is stable.
+
+Context:
+- Public Sandfly tooling and docs appear sufficient to build the integration if a licensed Sandfly server is available.
+- Expected integration points include API authentication, Sandfly-managed host inventory, check listing, scan launch, ad-hoc IP range or SSH credential scans, result retrieval, and mapping Sandfly alerts/findings into HomelabSec findings/remediation surfaces.
+- This is deliberately not a current implementation item.
+
+Deliver later:
+- document required Sandfly server/API configuration and secret handling
+- add a read-only Sandfly collector for hosts, checks, scans, and findings
+- map Sandfly severity/status/remediation into HomelabSec exposure findings without breaking existing finding shapes
+- add opt-in scan launch controls with safe defaults and clear operator confirmation
+- add fixture-backed tests using sanitized Sandfly API payloads
+
+Verification later:
+- unit tests for Sandfly payload parsing and severity/status mapping
+- integration tests with mocked Sandfly API responses
+- dashboard contract tests for Sandfly-origin findings
+
 ## Suggested Execution Order
 
 1. Slice 1: Alert Routing
@@ -188,6 +325,10 @@ Next priority:
 8. Slice 8: DB Migration Discipline
 9. Slice 9: Backup And Restore Drill
 10. Slice 10: Admin UX Improvements
+11. Operational P0: Secret Management Programme
+12. Operational P0: Thunderbluff 3-2-1 Backup Programme
+13. Slice 11: Exposure Map Dashboard
+14. Slice 12: Sandfly Security Finding Integration, only after a Sandfly server/API path is selected
 
 ## Parking Lot
 
@@ -199,3 +340,4 @@ These are valid ideas, but not current execution priorities:
 - multi-node deployment support
 - replacing host-network scheduler design
 - major frontend redesign
+- Sandfly support until the core exposure map is stable and Robert decides to connect a licensed Sandfly server