diff --git a/CLAUDE.md b/CLAUDE.md index 18c16cc..7b7b132 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -240,9 +240,10 @@ aws --profile smelt s3 cp README.md s3://my-bucket/ `scripts/s3-key.sh` creates the hilt tenant if needed, mints an access key with every S3 permission, and writes the profile with the stack's region, ingot's endpoint and path-style -addressing. `TENANT=` / `PROFILE=` override the defaults. Rerun it after `make down && make up`: -hilt-vault is in-memory, so the tenant's signing key is gone and the script moves on to the -next free tenant id (`dev-2`, ...) with a fresh key. +addressing. `TENANT=` / `PROFILE=` override the defaults. hilt-vault keeps the tenant's +signing key on a volume, so the profile keeps working across `make down && make up`. A stack +whose vault has lost the key (one restored from a snapshot saved before the vault persisted) +makes the script move on to the next free tenant id (`dev-2`, ...) with a fresh key. ### Regenerating Keys and Proofs @@ -329,7 +330,7 @@ All host-side ports live in a dedicated `15XXX` range to avoid collision with co | piri-{N} | 15100 + N | HTTP/UCAN | Storage node(s); N defined by `smelt.yml` (default 1, max 9) | | hilt | 15110 | HTTP/UCAN | Tenant management (Tenant API + UCAN RPC) | | hilt-postgres | 15111 | PostgreSQL | Hilt tenant/provider store | -| hilt-vault | 15112 | HTTP | Hilt key vault (OpenBao dev mode) | +| hilt-vault | 15112 | HTTP | Hilt key vault (OpenBao, raft storage) | | plc | 15120 | HTTP | did:plc directory (reference implementation) | | plc-postgres | 15121 | PostgreSQL | did:plc directory store | | ingot | 15130 | S3/HTTP | S3 gateway over Forge | diff --git a/README.md b/README.md index e583451..fb64e8a 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ All host ports live in a dedicated `15XXX` range to avoid collision with common | piri-{N} | 15100+N | Storage node(s) with PDP proofs; N declared in `smelt.yml` (default 1, max 9) | | hilt | 15110 | Tenant management (Tenant API + UCAN RPC) | | hilt-postgres | 15111 | Postgres for hilt's tenant/provider store | -| hilt-vault | 15112 | Hilt key vault (OpenBao in dev mode) | +| hilt-vault | 15112 | Hilt key vault (OpenBao, raft storage) | | plc | 15120 | did:plc directory (reference implementation) | | plc-postgres | 15121 | Postgres for the did:plc directory | | ingot | 15130 | S3 gateway over Forge | diff --git a/docs/PERF_TESTING.md b/docs/PERF_TESTING.md index 8985be3..3ea0829 100644 --- a/docs/PERF_TESTING.md +++ b/docs/PERF_TESTING.md @@ -131,8 +131,9 @@ LABEL=after ./scripts/perf-s3-speedtest.sh run `setup` mints a key for tenant `perf` (AWS CLI profile `smelt-perf`, the same way `make s3-key` does), creates the bucket and generates the test files in -`generated/perf/testfiles`. Re-run it after `make down && make up`: hilt's dev -vault is in memory, so the old key stops working. `setup` then moves to the +`generated/perf/testfiles`. The key survives `make down && make up`; re-run +`setup` after `make clean` or a snapshot restore. If hilt has the tenant but +not its key (a snapshot saved before the vault persisted), `setup` moves to the next free tenant (`perf-2`, ...) and creates that tenant's bucket (`perf-s3-speedtest-perf-2`), since ingot bucket names are global and the old one belongs to `perf`. @@ -153,7 +154,8 @@ The `large` set writes 75 GiB per run, so plan for about 200 GB of free disk per run and 400 GB for a before/after pair. `SNAPSHOT` runs `make down && make up SNAPSHOT=...` and then `setup` again, -because the restore loses hilt's access key the same way a restart does. An +because the restore returns hilt to the snapshot's state, which holds only the +keys that existed when it was saved. An exported `SMELT_MANIFEST` takes precedence over the snapshot's own manifest, so the script refuses to restore when the two differ. @@ -196,8 +198,9 @@ LABEL=after ./scripts/perf-drill.sh run `setup` mints a key for tenant `drill` (AWS CLI profile `smelt-drill`, the same way `make s3-key` does) and writes it with ingot's endpoint and region to `generated/perf-runs/drill/provider/.env`, readable only by you. Neither key -is printed. Re-run `setup` after `make down && make up`: hilt's dev vault is -in memory, so the old key stops working and `setup` moves to tenant `drill-2`. +is printed. The key survives `make down && make up`. If hilt has the tenant but +not its key (a snapshot saved before the vault persisted), `setup` moves to +tenant `drill-2`. Re-run it after `make clean` too, which deletes the tenant along with its key; `setup` then creates tenant `drill` again. Without it, `run` stops at its smoke check with `InvalidAccessKeyId`. diff --git a/docs/SNAPSHOTS.md b/docs/SNAPSHOTS.md index f4befcc..65758f6 100644 --- a/docs/SNAPSHOTS.md +++ b/docs/SNAPSHOTS.md @@ -122,6 +122,8 @@ generated/snapshots// ├── guppy-data.tar.gz # client login + spaces ├── swarf-postgres-data.tar.gz # revocation records ├── hilt-postgres-data.tar.gz # tenant state + ├── hilt-vault-data.tar.gz # tenant/access-key private keys (OpenBao raft storage) + ├── hilt-vault-init.tar.gz # its unseal share (dev-only custody) ├── plc-postgres-data.tar.gz # DID PLC registry ├── ingot-data.tar.gz # ingot node state ├── ingot-postgres-data.tar.gz # ingot registry diff --git a/embed.go b/embed.go index 5ac17a6..710b489 100644 --- a/embed.go +++ b/embed.go @@ -9,6 +9,7 @@ import "embed" //go:embed compose.yml .env //go:embed systems/common/compose.yml +//go:embed systems/common/openbao/* //go:embed systems/blockchain/compose.yml //go:embed systems/blockchain/state/deployed-addresses.json @@ -22,6 +23,7 @@ import "embed" //go:embed systems/hilt/compose.yml //go:embed systems/hilt/register-provider.sh +//go:embed systems/hilt/openbao/* //go:embed systems/ingot/compose.yml //go:embed systems/ingot/config/* diff --git a/images_test.go b/images_test.go index e5d8a2a..0dc4177 100644 --- a/images_test.go +++ b/images_test.go @@ -22,8 +22,9 @@ var thirdPartyImages = map[string]map[string]string{ "postgres": "${POSTGRES_IMAGE:-postgres:16-alpine}", }, "systems/hilt/compose.yml": { - "hilt-vault": "${OPENBAO_IMAGE:-openbao/openbao:2.6}", - "hilt-postgres": "${POSTGRES_IMAGE:-postgres:16-alpine}", + "hilt-vault": "${OPENBAO_IMAGE:-openbao/openbao:2.6}", + "hilt-vault-init": "${OPENBAO_IMAGE:-openbao/openbao:2.6}", + "hilt-postgres": "${POSTGRES_IMAGE:-postgres:16-alpine}", }, "systems/swarf/compose.yml": { "swarf-postgres": "${POSTGRES_IMAGE:-postgres:16-alpine}", diff --git a/pkg/snapshot/volumes.go b/pkg/snapshot/volumes.go index c76a318..0b728fc 100644 --- a/pkg/snapshot/volumes.go +++ b/pkg/snapshot/volumes.go @@ -37,6 +37,8 @@ func resolveVolumes(m *manifest.Manifest) ([]string, error) { "guppy-data", // systems/guppy — client's login/space state "swarf-postgres-data", // systems/swarf — revocation records "hilt-postgres-data", // systems/hilt — tenant state + "hilt-vault-data", // systems/hilt — tenant/access-key private keys (raft storage) + "hilt-vault-init", // systems/hilt — unseal share for the above; restored together "plc-postgres-data", // systems/plc — DID PLC registry "ingot-data", // systems/ingot — node state "ingot-postgres-data", // systems/ingot — registry diff --git a/pkg/stack/stack_test.go b/pkg/stack/stack_test.go index 8046af6..d5b2285 100644 --- a/pkg/stack/stack_test.go +++ b/pkg/stack/stack_test.go @@ -100,6 +100,9 @@ func TestEmbeddedFilesExist(t *testing.T) { "systems/upload/register-providers.sh", "systems/ingot/openbao/config.hcl", "systems/ingot/openbao/init.sh", + "systems/hilt/openbao/config.hcl", + "systems/hilt/openbao/init.sh", + "systems/common/openbao/bootstrap.sh", } for _, f := range files { diff --git a/scripts/perf-drill.sh b/scripts/perf-drill.sh index 99a2c7e..5ca797d 100755 --- a/scripts/perf-drill.sh +++ b/scripts/perf-drill.sh @@ -380,7 +380,7 @@ smoke_fail() { $err The drill would fail the same way. Check the stack's logs: docker compose logs ingot piri-0 InvalidAccessKeyId means hilt no longer has the key, as after 'make clean' or a -restart; '$0 setup' mints a new one. +snapshot restore; '$0 setup' mints a new one. If piri reports a signature mismatch, the keys and proofs in generated/ are likely out of sync; 'make regen', then 'make clean && make up' and '$0 setup' regenerates both." } diff --git a/scripts/perf-s3-speedtest.sh b/scripts/perf-s3-speedtest.sh index 4e82dff..5408a3d 100755 --- a/scripts/perf-s3-speedtest.sh +++ b/scripts/perf-s3-speedtest.sh @@ -16,7 +16,7 @@ # LABEL run label, required for `run` # RUNS repeats per file (default 1) # SNAPSHOT run: `make down && make up SNAPSHOT=...` first, then re-run setup -# (hilt's dev vault is in memory, so a restore loses the access key). +# (the restore returns hilt to the snapshot's keys, so the access key goes). # With SMELT_MANIFEST also set, the two manifests must match. # PERF_EXTRA_METADATA a JSON object recorded verbatim as `extra` in metadata.json # and runs.jsonl; anything else stops the run before it starts @@ -39,8 +39,9 @@ RUNS="${RUNS:-1}" TENANT=perf PROFILE=smelt-perf # The bucket is named after the tenant the profile ended up on (see -# perf_bucket): ingot bucket names are global, and after a restart s3-key.sh -# may move to perf-2, which cannot reuse the bucket perf created. +# perf_bucket): ingot bucket names are global, and when hilt has lost the +# tenant's key s3-key.sh moves to perf-2, which cannot reuse the bucket perf +# created. BUCKET_PREFIX=perf-s3-speedtest TARGETS_TEMPLATE="$PROJECT/generated/perf/s3_targets.ini" # Containers whose CPU/memory and logs are captured around each run. diff --git a/scripts/s3-key.sh b/scripts/s3-key.sh index 8c623de..ff764ae 100755 --- a/scripts/s3-key.sh +++ b/scripts/s3-key.sh @@ -12,10 +12,11 @@ # returns it once); this script never prints it. The profile also carries a # custom `tenant_id` key naming the tenant it belongs to. # -# After `make down && make up` hilt has forgotten the tenant's signing key -# (hilt-vault is in-memory), so the script moves on to the next free tenant id -# (dev-2, dev-3, ...) and the previously saved key stops working; rerun -# `make s3-key` to get a working profile again. +# hilt-vault keeps the tenant's signing key on a volume, so a saved profile +# keeps working across `make down && make up`. If the vault has lost the key +# (a stack restored from a snapshot saved before the vault persisted), the +# script moves on to the next free tenant id (dev-2, dev-3, ...) and the +# previously saved key stops working; rerun `make s3-key` for a working profile. # # Needs: a running stack, docker, curl, jq, and AWS CLI v2 (2.13+, for the per-profile # endpoint_url setting). @@ -95,10 +96,10 @@ mint_key() { } # Hilt keeps tenant records in postgres but the tenant's signing key in -# hilt-vault, which runs OpenBao in dev mode: in memory, gone after -# `make down && make up`. The tenant then still exists (PUT says 200) but -# every access-key request fails with 500, and so does DELETE, which needs -# the same key to sign the did:plc tombstone. Nothing can revive that +# hilt-vault. If the vault has lost the key (a stack restored from a snapshot +# saved before the vault persisted), the tenant still exists (PUT says 200) +# but every access-key request fails with 500, and so does DELETE, which +# needs the same key to sign the did:plc tombstone. Nothing can revive that # tenant, so fall back to the next free id (dev-2, dev-3, ...). tenant="$TENANT" status= @@ -107,7 +108,7 @@ for attempt in $(seq 2 9); do status="$(mint_key "$tenant")" if [ "$status" = "200" ] || [ "$status" = "201" ]; then break; fi if [ "$status" = "500" ] && [ "$state" = "exists" ]; then - echo "tenant $tenant exists but hilt cannot sign for it (hilt-vault lost its key on restart); trying $TENANT-$attempt" >&2 + echo "tenant $tenant exists but hilt cannot sign for it (hilt-vault has lost its key); trying $TENANT-$attempt" >&2 tenant="$TENANT-$attempt" status= continue diff --git a/systems/common/openbao/bootstrap.sh b/systems/common/openbao/bootstrap.sh new file mode 100644 index 0000000..545f59f --- /dev/null +++ b/systems/common/openbao/bootstrap.sh @@ -0,0 +1,116 @@ +#!/bin/sh +# Shared first half of an OpenBao init job, sourced by each system's +# openbao/init.sh. It brings a non-dev `bao server` with raft storage to a +# usable state, idempotently, so the init job can run on every `make up`: +# +# 1. first boot: `bao operator init` (one unseal share); the share and the +# root token are kept on the init volume (/init). Dev-only custody: +# production replaces the stored share with a transit seal against a +# central OpenBao. +# 2. every boot: unseal if sealed, then wait for the node to go active +# (raft serves nothing until it has a leader). +# +# On return BAO_TOKEN holds the root token, for the caller's provisioning. +# The caller sets BAO_ADDR and OPENBAO_NAME (the server's compose service +# name, used in log lines and error hints). +# +# Nothing secret is ever echoed: no `set -x`, and bao output that carries +# key material goes to files or /dev/null. + +: "${BAO_ADDR:?BAO_ADDR must point at the OpenBao server}" +: "${OPENBAO_NAME:?OPENBAO_NAME (the compose service name of the server) must be set}" + +INIT_DIR=/init +UNSEAL_KEY_FILE="$INIT_DIR/unseal-key" +ROOT_TOKEN_FILE="$INIT_DIR/root-token" + +log() { echo "$OPENBAO_NAME-init: $*"; } +die() { echo "$OPENBAO_NAME-init: $*" >&2; exit 1; } + +# Server state comes from exit codes, which are stable across output formats. +# Documented codes: +# bao status 0 unsealed, 1 error, 2 sealed +# https://openbao.org/docs/commands/status/ +# bao operator init -status 0 initialized, 1 error, 2 not initialized +# https://openbao.org/docs/commands/operator/init/ +# Two cases the docs leave implicit, confirmed against openbao/openbao:2.6: +# an uninitialized server is also sealed, so `bao status` exits 2 for it, and +# `operator init -status` exits 2 (not 1) when the server is unreachable. So +# reachability is settled first via `bao status`, then initialization, then +# the seal. +is_initialized() { bao operator init -status >/dev/null 2>&1; } +is_sealed() { + rc=0 + bao status >/dev/null 2>&1 || rc=$? + [ "$rc" -eq 2 ] +} + +log "waiting for OpenBao at $BAO_ADDR..." +waited=0 +while :; do + rc=0 + bao status >/dev/null 2>&1 || rc=$? + if [ "$rc" -ne 1 ]; then break; fi + if [ "$waited" -ge 120 ]; then + die "OpenBao never answered after ${waited}s; aborting" + fi + sleep 1 + waited=$((waited + 1)) +done +log "OpenBao is answering (took ${waited}s)" + +# json_string JSON KEY prints the string value of KEY from JSON, for a value +# that is either a plain string or the first element of an array. Whitespace +# is stripped first so pretty-printed and compact output parse the same; +# base64 and token values contain no whitespace or quotes. +json_string() { + printf '%s' "$1" | tr -d ' \n\r\t' | sed -n "s/.*\"$2\":\[\{0,1\}\"\([^\"]*\)\".*/\1/p" +} + +mkdir -p "$INIT_DIR" + +if ! is_initialized; then + log "server is uninitialized; running operator init (1 share)" + # Stale material from a previous data volume is useless now; overwrite. + init_json=$(bao operator init -key-shares=1 -key-threshold=1 -format=json) + unseal_key=$(json_string "$init_json" unseal_keys_b64) + root_token=$(json_string "$init_json" root_token) + [ -n "$unseal_key" ] || die "could not parse the unseal key from operator init output" + [ -n "$root_token" ] || die "could not parse the root token from operator init output" + umask 077 + printf '%s' "$unseal_key" > "$UNSEAL_KEY_FILE" + printf '%s' "$root_token" > "$ROOT_TOKEN_FILE" + unset init_json unseal_key root_token + log "initialized; unseal share and root token stored on the init volume" +elif [ ! -s "$UNSEAL_KEY_FILE" ] || [ ! -s "$ROOT_TOKEN_FILE" ]; then + die "server is initialized but $INIT_DIR holds no unseal material (init volume lost); run 'make clean' to reset both $OPENBAO_NAME volumes" +fi + +if is_sealed; then + log "unsealing" + bao operator unseal "$(cat "$UNSEAL_KEY_FILE")" >/dev/null +fi +is_sealed && die "server is still sealed after unseal" +log "unsealed" + +# An unsealed raft node is not yet serving: it has to win its leadership +# election first, and until it does every request is refused with 500 +# "local node not active but active cluster node not found". sys/health's +# default codes settle it, since 200 means initialized, unsealed and active +# (a standby answers 429, a sealed node 503) — unlike the compose +# healthcheck, which passes sealedcode and uninitcode so it can gate the init +# job on a server that is merely listening. +# -T bounds each probe, so one hung connect cannot outlast the deadline, which +# is wall-clock rather than a count of attempts. +log "waiting for the node to become active..." +started=$(date +%s) +until wget -q -T 2 --spider "$BAO_ADDR/v1/sys/health" 2>/dev/null; do + if [ "$(($(date +%s) - started))" -ge 60 ]; then + die "node never became active after 60s; aborting" + fi + sleep 1 +done +log "node is active (took $(($(date +%s) - started))s)" + +BAO_TOKEN=$(cat "$ROOT_TOKEN_FILE") +export BAO_TOKEN diff --git a/systems/hilt/README.md b/systems/hilt/README.md index 5e2ca76..8f67995 100644 --- a/systems/hilt/README.md +++ b/systems/hilt/README.md @@ -19,8 +19,12 @@ the tenant as a customer with the upload service via `/customer/add`. - **hilt** - Tenant management service (`ghcr.io/fil-forge/hilt:main`) - **hilt-postgres** - PostgreSQL for hilt's tenant/provider/access-key stores (goose migrations run at hilt startup) -- **hilt-vault** - OpenBao in dev mode for tenant/access-key private keys - (KV v2 at the `secret` mount) +- **hilt-vault** - OpenBao for tenant/access-key private keys (server mode, + raft storage on `hilt-vault-data`, KV v2 at the `secret` mount) +- **hilt-vault-init** - one-shot: initializes and unseals `hilt-vault`, then + provisions the KV mount, hilt's policy and hilt's scoped token + (`openbao/init.sh`, with the shared bootstrap in + `../common/openbao/bootstrap.sh`) ## Ports @@ -41,11 +45,11 @@ All configuration is via `HILT_*` environment variables in `compose.yml`: only; never use a production key here. - Storage: postgres via the `hilt-postgres` sidecar (dev-only `hilt:hilt` credentials; data persists in the `hilt-postgres-data` volume). -- Vault: OpenBao dev mode via `hilt-vault` (`HILT_VAULT_TYPE=openbao`), - token auth with the dev root token (`HILT_VAULT_TOKEN`, default - `dev-root-token` — local dev only). Keys survive hilt restarts, but - dev-mode OpenBao stores in memory, so a `hilt-vault` container restart - clears them. +- Vault: OpenBao via `hilt-vault` (`HILT_VAULT_TYPE=openbao`), token auth + with a scoped token that `hilt-vault-init` provisions on every boot + (`HILT_VAULT_TOKEN`, default `dev-hilt-vault-token` — local dev only). + Keys persist in the `hilt-vault-data` volume, alongside the records in + `hilt-postgres-data`; `make clean` resets both. - PLC directory: the local reference server at `http://plc:3000`. - Upload service: `did:web:upload` at `http://upload:80`, presenting the `upload → hilt` `/customer/add` delegation from @@ -78,6 +82,9 @@ Registration is idempotent — when the record already exists in postgres the ## Volumes - `hilt-postgres-data` - Hilt's tenant/provider/access-key records +- `hilt-vault-data` - OpenBao raft storage: tenant and access-key private keys +- `hilt-vault-init` - hilt-vault's unseal share and root token (dev-only + custody) ## Dependencies @@ -85,7 +92,7 @@ Registration is idempotent — when the record already exists in postgres the - upload (service_healthy) - swarf (service_healthy) - hilt-postgres (service_healthy) -- hilt-vault (service_healthy) +- hilt-vault-init (service_completed_successfully) ## Requirements and Notes diff --git a/systems/hilt/compose.yml b/systems/hilt/compose.yml index 6d259c3..7fdfb26 100644 --- a/systems/hilt/compose.yml +++ b/systems/hilt/compose.yml @@ -35,8 +35,9 @@ services: - HILT_VAULT_TYPE=openbao - HILT_VAULT_OPENBAO_ADDRESS=http://hilt-vault:8200 - HILT_VAULT_OPENBAO_AUTH_METHOD=token - # Dev-mode root token (local dev only, like the partner key above) - - HILT_VAULT_OPENBAO_TOKEN=${HILT_VAULT_TOKEN:-dev-root-token} + # Scoped token hilt-vault-init provisions (local dev only, like the + # partner key above) + - HILT_VAULT_OPENBAO_TOKEN=${HILT_VAULT_TOKEN:-dev-hilt-vault-token} - HILT_PLC_DIRECTORY=http://plc:3000 - HILT_UPLOAD_SERVICE_ID=did:web:upload - HILT_UPLOAD_SERVICE_URL=http://upload:80 @@ -62,8 +63,9 @@ services: condition: service_healthy hilt-postgres: condition: service_healthy - hilt-vault: - condition: service_healthy + # The vault is unsealed and hilt's token exists once the init job has run. + hilt-vault-init: + condition: service_completed_successfully healthcheck: test: ["CMD", "curl", "-sf", "http://localhost:80/health"] start_interval: 1s @@ -108,20 +110,26 @@ services: networks: - forge-network - # OpenBao in dev mode for hilt's tenant/access-key private keys (the same - # server production runs). Dev mode auto-unseals and mounts KV v2 at - # `secret` (hilt's default mount). Storage is in-memory: keys survive hilt - # restarts but not a hilt-vault container restart. The image's default - # command is `server -dev`; the entrypoint reads the BAO_DEV_* vars below. + # OpenBao for hilt's tenant/access-key private keys (the same server + # production runs). A non-dev server with raft storage on hilt-vault-data, + # so the keys survive down/up and snapshot restores alongside the tenant + # and access-key rows in hilt-postgres. The healthcheck only means + # "listening": a fresh server is uninitialized and a restarted one is sealed + # until hilt-vault-init runs again (which `make up` does, since one-shot + # services re-run on every up). hilt-vault: image: ${OPENBAO_IMAGE:-openbao/openbao:2.6} + command: ["server"] ports: - "${SMELT_HILT_VAULT_PORT:-15112:8200}" environment: - - BAO_DEV_ROOT_TOKEN_ID=${HILT_VAULT_TOKEN:-dev-root-token} - - BAO_DEV_LISTEN_ADDRESS=0.0.0.0:8200 + # For `docker compose exec hilt-vault bao ...` (the CLI defaults to https) + - BAO_ADDR=http://127.0.0.1:8200 + volumes: + - ./openbao/config.hcl:/openbao/config/config.hcl:ro + - hilt-vault-data:/openbao/file healthcheck: - test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8200/v1/sys/health"] + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8200/v1/sys/health?sealedcode=200&uninitcode=200"] start_interval: 1s interval: 10s timeout: 5s @@ -131,6 +139,28 @@ services: networks: - forge-network + # One-shot initializer for hilt-vault: `operator init` on first boot (one + # unseal share + root token kept on the hilt-vault-init volume, dev-only + # custody), unseal on every boot, then make sure KV v2 at `secret`, the hilt + # policy, and hilt's scoped token exist. See openbao/init.sh. + hilt-vault-init: + image: ${OPENBAO_IMAGE:-openbao/openbao:2.6} + user: "0:0" # the image runs as `openbao`; the init volume is root-owned + entrypoint: ["sh", "/init.sh"] + environment: + - BAO_ADDR=http://hilt-vault:8200 + - HILT_VAULT_TOKEN=${HILT_VAULT_TOKEN:-dev-hilt-vault-token} + volumes: + - ./openbao/init.sh:/init.sh:ro + - ../common/openbao/bootstrap.sh:/openbao-bootstrap.sh:ro + - hilt-vault-init:/init + depends_on: + hilt-vault: + condition: service_healthy + restart: "no" + networks: + - forge-network + # Postgres for hilt's tenant/provider/access-key stores. Hilt runs goose # migrations on startup, so the `hilt` database created by POSTGRES_DB is # enough — no init sidecar needed. Named hilt-postgres because `postgres` @@ -158,3 +188,5 @@ services: volumes: hilt-postgres-data: + hilt-vault-data: # raft storage: tenant and access-key private keys + hilt-vault-init: # unseal share + root token (dev-only custody) diff --git a/systems/hilt/openbao/config.hcl b/systems/hilt/openbao/config.hcl new file mode 100644 index 0000000..ef7730c --- /dev/null +++ b/systems/hilt/openbao/config.hcl @@ -0,0 +1,22 @@ +# hilt-vault server config (local dev). +# +# The OpenBao that holds hilt's tenant and access-key private keys: a non-dev +# `bao server` with integrated raft storage on the hilt-vault-data volume, so +# the keys survive `make down` / `make up` and snapshot restores alongside +# the tenant and access-key rows in hilt-postgres. hilt-vault-init +# initializes, unseals, and provisions it on every boot. + +ui = false + +api_addr = "http://hilt-vault:8200" +cluster_addr = "http://hilt-vault:8201" + +listener "tcp" { + address = "0.0.0.0:8200" + tls_disable = true +} + +storage "raft" { + path = "/openbao/file" + node_id = "hilt-vault" +} diff --git a/systems/hilt/openbao/init.sh b/systems/hilt/openbao/init.sh new file mode 100644 index 0000000..c7171be --- /dev/null +++ b/systems/hilt/openbao/init.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# Initialize, unseal, and provision hilt-vault. +# +# Runs as the one-shot `hilt-vault-init` compose service, gated on +# hilt-vault's healthcheck (which only means "listening": a fresh server is +# uninitialized and a restarted one is sealed). The shared bootstrap +# (systems/common/openbao/bootstrap.sh) initializes on first boot, unseals, +# and waits for the node to go active; this script then makes sure the KV v2 +# engine at `secret` (hilt's default mount), the hilt policy, and hilt's +# scoped token exist. Every step is idempotent so the job runs on each +# `make up`. + +set -eu + +MOUNT=secret +POLICY_NAME=hilt-keys +: "${HILT_VAULT_TOKEN:?HILT_VAULT_TOKEN (the token id hilt uses) must be set}" + +OPENBAO_NAME=hilt-vault +. /openbao-bootstrap.sh + +if bao secrets list -format=json | grep -q "\"$MOUNT/\""; then + : +else + log "enabling KV v2 at $MOUNT" + bao secrets enable -path="$MOUNT" kv-v2 >/dev/null +fi + +# Hilt's policy: read and write key material, and delete a key outright +# (hilt removes a key's metadata, which drops every version). +bao policy write "$POLICY_NAME" - >/dev/null </dev/null 2>&1 || true +if ! out=$(bao token create -id="$HILT_VAULT_TOKEN" -policy="$POLICY_NAME" -no-default-policy \ + -orphan -display-name=hilt 2>&1); then + echo "$out" >&2 + die "creating hilt's token failed" +fi +unset out BAO_TOKEN + +log "KV v2 at $MOUNT ready; hilt token provisioned" diff --git a/systems/ingot/README.md b/systems/ingot/README.md index 256434d..0c037dc 100644 --- a/systems/ingot/README.md +++ b/systems/ingot/README.md @@ -15,7 +15,8 @@ guppy-style edge client. (`openbao/openbao:2.6`, server mode, raft storage on `ingot-openbao-data`). - **ingot-openbao-init** — one-shot: initializes and unseals `ingot-openbao`, then provisions the transit engine, the region KEK, and ingot's token - (`openbao/init.sh`). + (`openbao/init.sh`, with the shared bootstrap in + `../common/openbao/bootstrap.sh`). ## Ports @@ -154,10 +155,12 @@ The profile stores `endpoint_url`, which needs AWS CLI v2.13 or newer. To drive the tenant API by hand instead, see the smoke test in [systems/hilt/README.md](../hilt/README.md). -Rerun `make s3-key` after `make down && make up`: hilt-vault runs in memory, -so a restart loses the tenant's signing key and the saved access key stops -working. The script then provisions the next free tenant id (`dev-2`, ...) -and writes a fresh key to the profile. +The saved key keeps working across `make down && make up`: hilt-vault keeps +the tenant's signing key on a volume. Rerun `make s3-key` after `make clean`, +or after restoring a snapshot saved before the key was minted. If hilt has +the tenant but not its key (a snapshot saved before the vault persisted), the +script provisions the next free tenant id (`dev-2`, ...) and writes a fresh +key to the profile. ### Region KEK smoke test diff --git a/systems/ingot/compose.yml b/systems/ingot/compose.yml index 5e92d3a..c820832 100644 --- a/systems/ingot/compose.yml +++ b/systems/ingot/compose.yml @@ -123,6 +123,7 @@ services: - INGOT_REGION_KEK=region-kek volumes: - ./openbao/init.sh:/init.sh:ro + - ../common/openbao/bootstrap.sh:/openbao-bootstrap.sh:ro - ingot-openbao-init:/init depends_on: ingot-openbao: diff --git a/systems/ingot/openbao/init.sh b/systems/ingot/openbao/init.sh index a17342b..6c0d62d 100644 --- a/systems/ingot/openbao/init.sh +++ b/systems/ingot/openbao/init.sh @@ -3,121 +3,21 @@ # # Runs as the one-shot `ingot-openbao-init` compose service, gated on # ingot-openbao's healthcheck (which only means "listening": a fresh server -# is uninitialized and a restarted one is sealed). Every step is idempotent -# so the script runs on each `make up`: -# -# 1. first boot: `bao operator init` (one unseal share); the share and the -# root token are kept on the ingot-openbao-init volume (/init). Dev-only -# custody: production replaces the stored share with a transit seal -# against a central OpenBao. -# 2. every boot: unseal if sealed, wait for the node to go active (raft -# serves nothing until it has a leader), then make sure the transit -# engine, the region KEK (aes256-gcm96, derived=true, exportable=false), -# the ingot policy, and ingot's scoped token exist. -# -# Nothing secret is ever echoed: no `set -x`, and bao output that carries -# key material goes to files or /dev/null. +# is uninitialized and a restarted one is sealed). The shared bootstrap +# (systems/common/openbao/bootstrap.sh) initializes on first boot, unseals, +# and waits for the node to go active; this script then makes sure the +# transit engine, the region KEK (aes256-gcm96, derived=true, +# exportable=false), the ingot policy, and ingot's scoped token exist. Every +# step is idempotent so the job runs on each `make up`. set -eu -INIT_DIR=/init -UNSEAL_KEY_FILE="$INIT_DIR/unseal-key" -ROOT_TOKEN_FILE="$INIT_DIR/root-token" KEK="${INGOT_REGION_KEK:-region-kek}" POLICY_NAME=ingot-region-kek -: "${BAO_ADDR:?BAO_ADDR must point at ingot-openbao}" : "${INGOT_OPENBAO_TOKEN:?INGOT_OPENBAO_TOKEN (the token id ingot uses) must be set}" -log() { echo "ingot-openbao-init: $*"; } -die() { echo "ingot-openbao-init: $*" >&2; exit 1; } - -# Server state comes from exit codes, which are stable across output formats. -# Documented codes: -# bao status 0 unsealed, 1 error, 2 sealed -# https://openbao.org/docs/commands/status/ -# bao operator init -status 0 initialized, 1 error, 2 not initialized -# https://openbao.org/docs/commands/operator/init/ -# Two cases the docs leave implicit, confirmed against openbao/openbao:2.6: -# an uninitialized server is also sealed, so `bao status` exits 2 for it, and -# `operator init -status` exits 2 (not 1) when the server is unreachable. So -# reachability is settled first via `bao status`, then initialization, then -# the seal. -is_initialized() { bao operator init -status >/dev/null 2>&1; } -is_sealed() { - rc=0 - bao status >/dev/null 2>&1 || rc=$? - [ "$rc" -eq 2 ] -} - -log "waiting for OpenBao at $BAO_ADDR..." -waited=0 -while :; do - rc=0 - bao status >/dev/null 2>&1 || rc=$? - if [ "$rc" -ne 1 ]; then break; fi - if [ "$waited" -ge 120 ]; then - die "OpenBao never answered after ${waited}s; aborting" - fi - sleep 1 - waited=$((waited + 1)) -done -log "OpenBao is answering (took ${waited}s)" - -# json_string JSON KEY prints the string value of KEY from JSON, for a value -# that is either a plain string or the first element of an array. Whitespace -# is stripped first so pretty-printed and compact output parse the same; -# base64 and token values contain no whitespace or quotes. -json_string() { - printf '%s' "$1" | tr -d ' \n\r\t' | sed -n "s/.*\"$2\":\[\{0,1\}\"\([^\"]*\)\".*/\1/p" -} - -mkdir -p "$INIT_DIR" - -if ! is_initialized; then - log "server is uninitialized; running operator init (1 share)" - # Stale material from a previous data volume is useless now; overwrite. - init_json=$(bao operator init -key-shares=1 -key-threshold=1 -format=json) - unseal_key=$(json_string "$init_json" unseal_keys_b64) - root_token=$(json_string "$init_json" root_token) - [ -n "$unseal_key" ] || die "could not parse the unseal key from operator init output" - [ -n "$root_token" ] || die "could not parse the root token from operator init output" - umask 077 - printf '%s' "$unseal_key" > "$UNSEAL_KEY_FILE" - printf '%s' "$root_token" > "$ROOT_TOKEN_FILE" - unset init_json unseal_key root_token - log "initialized; unseal share and root token stored on the init volume" -elif [ ! -s "$UNSEAL_KEY_FILE" ] || [ ! -s "$ROOT_TOKEN_FILE" ]; then - die "server is initialized but $INIT_DIR holds no unseal material (init volume lost); run 'make clean' to reset both ingot-openbao volumes" -fi - -if is_sealed; then - log "unsealing" - bao operator unseal "$(cat "$UNSEAL_KEY_FILE")" >/dev/null -fi -is_sealed && die "server is still sealed after unseal" -log "unsealed" - -# An unsealed raft node is not yet serving: it has to win its leadership -# election first, and until it does every request is refused with 500 -# "local node not active but active cluster node not found". sys/health's -# default codes settle it, since 200 means initialized, unsealed and active -# (a standby answers 429, a sealed node 503) — unlike the compose -# healthcheck, which passes sealedcode and uninitcode so it can gate this -# container on a server that is merely listening. -# -T bounds each probe, so one hung connect cannot outlast the deadline, which -# is wall-clock rather than a count of attempts. -log "waiting for the node to become active..." -started=$(date +%s) -until wget -q -T 2 --spider "$BAO_ADDR/v1/sys/health" 2>/dev/null; do - if [ "$(($(date +%s) - started))" -ge 60 ]; then - die "node never became active after 60s; aborting" - fi - sleep 1 -done -log "node is active (took $(($(date +%s) - started))s)" - -BAO_TOKEN=$(cat "$ROOT_TOKEN_FILE") -export BAO_TOKEN +OPENBAO_NAME=ingot-openbao +. /openbao-bootstrap.sh if bao secrets list -format=json | grep -q '"transit/"'; then :