diff --git a/.air.toml b/.air.toml index e374e3bfa..61270ed58 100644 --- a/.air.toml +++ b/.air.toml @@ -9,4 +9,4 @@ tmp_dir = "tmp" delay = 100 exclude_dir = ["examples", "dist", "docs", "website", "loadtest", "sdks", "spec-sdk-tests", "internal/portal/node_modules", "internal/portal/src"] include_ext = ["go"] - include_file = ["go.mod", "go.sum", ".env"] + include_file = ["go.mod", "go.sum", ".env", ".outpost.yaml"] diff --git a/.env.dev b/.env.dev index 232e30e23..b945d7c2e 100644 --- a/.env.dev +++ b/.env.dev @@ -4,23 +4,37 @@ # Load configuration from YAML file CONFIG=".outpost.yaml" -# ============================== Local Dev Dependencies ============================== -# These control which services `make up/deps` starts -# Comment out to disable, uncomment to enable +# ============================== Local Dev Stack ============================== +# These flags control which services `make up` brings up. Edit and re-run +# `make up` — compose reconciles the running stack to match. + +# --- Dependencies --- # KV store (Redis is default, uncomment to use Dragonfly instead) # LOCAL_DEV_DRAGONFLY=1 # Log store -LOCAL_DEV_POSTGRES=1 -# LOCAL_DEV_CLICKHOUSE=1 +LOCAL_DEV_CLICKHOUSE=1 +# LOCAL_DEV_POSTGRES=1 # Message queue LOCAL_DEV_RABBITMQ=1 # LOCAL_DEV_LOCALSTACK=1 # LOCAL_DEV_GCP=1 -# Optional: GUI tools (set port to enable) +# --- Add-on stacks --- + +# Forward proxy for testing DESTINATIONS_WEBHOOK_PROXY_URL flows +# LOCAL_DEV_ENVOY=1 + +# Observability (pick one — they conflict on otel-collector + OTLP ports) +# LOCAL_DEV_GRAFANA=1 +# LOCAL_DEV_UPTRACE=1 + +# Azure Service Bus + SQL Edge emulators (for destazure provider tests) +# LOCAL_DEV_AZURE=1 + +# --- GUI tools (set to 1 for default port, or a number to override) --- # LOCAL_DEV_REDIS_COMMANDER=28081 # LOCAL_DEV_PGADMIN=28082 # LOCAL_DEV_TABIX=28083 diff --git a/.outpost.yaml.dev b/.outpost.yaml.dev index ae55780a2..1b50ed751 100644 --- a/.outpost.yaml.dev +++ b/.outpost.yaml.dev @@ -49,12 +49,12 @@ mqs: # subscription: "outpost-publish-sub" # Log Store (uncomment one) -postgres: "postgres://outpost:outpost@postgres:5432/outpost?sslmode=disable" -# clickhouse: -# addr: "clickhouse:9000" -# username: "outpost" -# password: "outpost" -# database: "outpost" +# postgres: "postgres://outpost:outpost@postgres:5432/outpost?sslmode=disable" +clickhouse: + addr: "clickhouse:9000" + username: "outpost" + password: "outpost" + database: "outpost" # Application api_key: "apikey" @@ -85,6 +85,9 @@ publish_max_concurrency: 1 delivery_max_concurrency: 100 log_max_concurrency: 1 +# Tight threshold for local dev +log_batch_threshold_seconds: 1 + # Destinations destinations: webhook: diff --git a/Makefile b/Makefile index 5f32306a8..cb06f5ca3 100644 --- a/Makefile +++ b/Makefile @@ -35,55 +35,26 @@ clean: rm -f bin/outpost bin/outpost-server up: - make up/deps - make up/outpost + ./build/dev/dev.sh up down: - make down/outpost - make down/deps + ./build/dev/dev.sh down -up/outpost: - docker-compose -f build/dev/compose.yml --env-file .env up -d +nuke: + ./build/dev/dev.sh down --volumes -down/outpost: - docker-compose -f build/dev/compose.yml --env-file .env down +health: + @WAIT="$(WAIT)" ./build/dev/health.sh -up/deps: - COMPOSE_PROFILES=$$(./build/dev/deps/profiles.sh) docker-compose --env-file .env -f build/dev/deps/compose.yml -f build/dev/deps/compose-gui.yml up -d - -down/deps: - COMPOSE_PROFILES=$$(./build/dev/deps/profiles.sh --all) docker-compose -f build/dev/deps/compose.yml -f build/dev/deps/compose-gui.yml down - -nuke/deps: - COMPOSE_PROFILES=$$(docker compose -f build/dev/deps/compose.yml -f build/dev/deps/compose-gui.yml config --profiles | paste -sd, -) docker compose -f build/dev/deps/compose.yml -f build/dev/deps/compose-gui.yml down --volumes --remove-orphans - -up/mqs: - docker-compose -f build/dev/mqs/compose.yml up -d - -down/mqs: - docker-compose -f build/dev/mqs/compose.yml down - -up/grafana: - docker-compose -f build/dev/grafana/compose.yml up -d - -down/grafana: - docker-compose -f build/dev/grafana/compose.yml down - -up/uptrace: - docker-compose -f build/dev/uptrace/compose.yml up -d - -down/uptrace: - docker-compose -f build/dev/uptrace/compose.yml down +smoke: + ./build/dev/smoke.sh +# Run portal natively (vite hot reload). Portal is also available as a +# containerized service via `make up` — use this target only when you want +# the faster native dev loop. up/portal: cd internal/portal && npm install && npm run dev -up/azure: - docker-compose -f build/dev/azure/compose.yml up -d - -down/azure: - docker-compose -f build/dev/azure/compose.yml down --volumes - up/test: docker-compose -f build/test/compose.yml up -d @@ -167,9 +138,6 @@ migrate: redis/debug: go run cmd/redis-debug/main.go $(ARGS) -network: - docker network create outpost - logs: docker logs $$(docker ps -f name=outpost-${SERVICE} --format "{{.ID}}") -f $(ARGS) diff --git a/build/dev/README.md b/build/dev/README.md new file mode 100644 index 000000000..6f176bf33 --- /dev/null +++ b/build/dev/README.md @@ -0,0 +1,53 @@ +# Outpost Dev Stack + +One Docker Compose project (`outpost`). One command: `make up`. What runs is +declared in `.env` via `LOCAL_DEV_*` flags. + +## How it works + +`build/dev/dev.sh` reads `.env`, builds the docker compose invocation — +a list of `-f` files plus `COMPOSE_PROFILES` — and runs it. All files declare +`name: outpost` and join one auto-created network (`outpost_default`), so +service-to-service DNS (`api`, `redis`, `postgres`, …) just works. + +``` +.env → dev.sh → docker compose -f ... -f ... --profile ... up -d +LOCAL_DEV_X=1 +``` + +`make up` is declarative: edit `.env`, re-run `make up`, only the diff is +applied. There are no `up/` targets — flipping the flag is the +mechanism. + +## Layout + +| dir | purpose | gating | +|---|---|---| +| `compose.yml` | core (api, delivery, log, portal) — always on | — | +| `deps/` | redis/dragonfly, postgres, clickhouse, rabbitmq, localstack, gcp + GUIs | compose profiles, per service | +| `envoy/` | forward proxy for `DESTINATIONS_WEBHOOK_PROXY_URL` flows | `LOCAL_DEV_ENVOY=1` | +| `grafana/` | otel-collector + Prometheus + Grafana | `LOCAL_DEV_GRAFANA=1` | +| `uptrace/` | otel-collector + Uptrace (alternative to grafana) | `LOCAL_DEV_UPTRACE=1` | +| `azure/` | Azure Service Bus + SQL Edge emulators | `LOCAL_DEV_AZURE=1` | + +Add-ons are file-level: the file is only loaded when its flag is set. This +avoids cross-file service-name collisions (e.g. `otel-collector` exists in +both `grafana/` and `uptrace/`). + +## Adding a new add-on + +1. Create `build/dev//compose.yml` with `name: "outpost"` at the top + and no `networks:` block (compose creates the default network). +2. Add a `LOCAL_DEV_=1` branch in `dev.sh` appending the file. +3. Add the flag to `.env.dev` (commented) and document it in + `contributing/getting-started.md`. + +## Commands + +``` +make up # bring up everything enabled in .env +make down # stop and remove the stack +make nuke # stop + remove volumes (wipe state) +make up/portal # run portal natively for vite hot reload (escape hatch) +make up/test # separate test project (isolated lifecycle) +``` diff --git a/build/dev/azure/compose.yml b/build/dev/azure/compose.yml index 676617990..a7fa9d615 100644 --- a/build/dev/azure/compose.yml +++ b/build/dev/azure/compose.yml @@ -1,4 +1,4 @@ -name: "outpost-azure" +name: "outpost" services: azuresb: @@ -21,8 +21,3 @@ services: environment: ACCEPT_EULA: "Y" MSSQL_SA_PASSWORD: "Password!" - -networks: - default: - name: outpost - external: true diff --git a/build/dev/compose.yml b/build/dev/compose.yml index fbdc46b71..dbc5e0de0 100644 --- a/build/dev/compose.yml +++ b/build/dev/compose.yml @@ -1,6 +1,32 @@ name: "outpost" +# Wait for whichever deps are active in the current profile set; required:false +# means a dep that isn't in any active profile is skipped rather than erroring. +x-deps-depends-on: &deps_depends_on + redis: { condition: service_healthy, required: false } + dragonfly: { condition: service_healthy, required: false } + postgres: { condition: service_healthy, required: false } + clickhouse: { condition: service_healthy, required: false } + rabbitmq: { condition: service_healthy, required: false } + localstack: { condition: service_healthy, required: false } + gcp: { condition: service_healthy, required: false } + services: + migrate: + image: outpost + build: + context: ../../ + dockerfile: ./build/dev/Dockerfile + entrypoint: "" + command: ["go", "run", "./cmd/outpost", "migrate", "apply", "--yes"] + volumes: + - ../../:/app + - go_mod_cache:/go/pkg + - go_build_cache:/root/.cache/go-build + env_file: ../../.env + depends_on: *deps_depends_on + restart: "no" + api: image: outpost build: @@ -15,6 +41,14 @@ services: env_file: ../../.env environment: SERVICE: api + depends_on: + <<: *deps_depends_on + migrate: { condition: service_completed_successfully } + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:3333/api/v1/healthz"] + interval: 2s + timeout: 2s + retries: 60 delivery: image: outpost @@ -22,12 +56,12 @@ services: - ../..:/app - go_mod_cache:/go/pkg - go_build_cache:/root/.cache/go-build - depends_on: - api: - condition: service_started env_file: ../../.env environment: SERVICE: delivery + depends_on: + <<: *deps_depends_on + api: { condition: service_healthy } log: image: outpost @@ -35,18 +69,12 @@ services: - ../..:/app - go_mod_cache:/go/pkg - go_build_cache:/root/.cache/go-build - depends_on: - api: - condition: service_started env_file: ../../.env environment: SERVICE: log - - envoy: - image: envoyproxy/envoy:v1.31-latest - volumes: - - ./envoy/envoy.yaml:/etc/envoy/envoy.yaml:ro - command: ["envoy", "-c", "/etc/envoy/envoy.yaml", "--log-level", "info"] + depends_on: + <<: *deps_depends_on + api: { condition: service_healthy } portal: build: @@ -67,8 +95,3 @@ volumes: driver: local go_build_cache: driver: local - -networks: - default: - name: outpost - external: true diff --git a/build/dev/deps/compose-gui.yml b/build/dev/deps/compose-gui.yml index 8ee9170b4..3026fcd75 100644 --- a/build/dev/deps/compose-gui.yml +++ b/build/dev/deps/compose-gui.yml @@ -1,4 +1,4 @@ -name: "outpost-deps" +name: "outpost" services: # ============================== GUI Tools ============================== @@ -40,8 +40,3 @@ services: volumes: pgadmin: driver: local - -networks: - default: - name: outpost - external: true diff --git a/build/dev/deps/compose.yml b/build/dev/deps/compose.yml index d82689181..b832ae511 100644 --- a/build/dev/deps/compose.yml +++ b/build/dev/deps/compose.yml @@ -1,4 +1,4 @@ -name: "outpost-deps" +name: "outpost" services: # ============================== Redis ============================== @@ -10,9 +10,14 @@ services: ports: - "26379:6379" environment: - - REDIS_ARGS=--requirepass password + - REDIS_ARGS=--requirepass password --appendonly yes volumes: - redis:/data + healthcheck: + test: ["CMD", "redis-cli", "-a", "password", "--no-auth-warning", "PING"] + interval: 2s + timeout: 2s + retries: 15 dragonfly: profiles: ["dragonfly"] @@ -29,6 +34,11 @@ services: default: aliases: - redis + healthcheck: + test: ["CMD", "redis-cli", "-a", "password", "--no-auth-warning", "PING"] + interval: 2s + timeout: 2s + retries: 15 # ============================== Log Store ============================== # LOCAL_DEV_CLICKHOUSE=1, LOCAL_DEV_POSTGRES=1 (can enable both) @@ -67,6 +77,11 @@ services: - 25432:5432 volumes: - postgres:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U outpost -d outpost"] + interval: 2s + timeout: 2s + retries: 15 # ============================== MQs ============================== # LOCAL_DEV_RABBITMQ=1, LOCAL_DEV_LOCALSTACK=1, LOCAL_DEV_GCP=1 @@ -78,6 +93,11 @@ services: - 25673:15672 # Management UI volumes: - rabbitmq:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"] + interval: 5s + timeout: 10s + retries: 20 localstack: profiles: ["localstack"] @@ -89,6 +109,11 @@ services: - 24571:4571 volumes: - localstack:/var/lib/localstack + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:4566/_localstack/health"] + interval: 5s + timeout: 5s + retries: 20 gcp: profiles: ["gcp"] @@ -108,6 +133,11 @@ services: volumes: - gcp-config:/root/.config - gcp-kube:/root/.kube + healthcheck: + test: ["CMD-SHELL", "curl -sS http://localhost:8085 >/dev/null"] + interval: 5s + timeout: 5s + retries: 20 volumes: redis: @@ -126,8 +156,3 @@ volumes: driver: local gcp-kube: driver: local - -networks: - default: - name: outpost - external: true diff --git a/build/dev/deps/profiles.sh b/build/dev/deps/profiles.sh deleted file mode 100755 index 766d5cf41..000000000 --- a/build/dev/deps/profiles.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# Reads LOCAL_DEV_* vars from .env and outputs COMPOSE_PROFILES -# Usage: ./profiles.sh - outputs profiles based on .env -# ./profiles.sh --all - outputs all profiles (for down command) - -if [ "$1" = "--all" ]; then - echo "redis,dragonfly,postgres,clickhouse,rabbitmq,localstack,gcp,redis-commander,pgadmin,tabix" - exit 0 -fi - -PROFILES="" - -# Cache (Redis default, Dragonfly optional) -if grep -q "^LOCAL_DEV_DRAGONFLY=1" .env 2>/dev/null; then - PROFILES="dragonfly" -else - PROFILES="redis" -fi - -# Log stores -grep -q "^LOCAL_DEV_POSTGRES=1" .env 2>/dev/null && PROFILES="$PROFILES,postgres" -grep -q "^LOCAL_DEV_CLICKHOUSE=1" .env 2>/dev/null && PROFILES="$PROFILES,clickhouse" - -# Message queues -grep -q "^LOCAL_DEV_RABBITMQ=1" .env 2>/dev/null && PROFILES="$PROFILES,rabbitmq" -grep -q "^LOCAL_DEV_LOCALSTACK=1" .env 2>/dev/null && PROFILES="$PROFILES,localstack" -grep -q "^LOCAL_DEV_GCP=1" .env 2>/dev/null && PROFILES="$PROFILES,gcp" - -# GUI tools -grep -q "^LOCAL_DEV_REDIS_COMMANDER=" .env 2>/dev/null && PROFILES="$PROFILES,redis-commander" -grep -q "^LOCAL_DEV_PGADMIN=" .env 2>/dev/null && PROFILES="$PROFILES,pgadmin" -grep -q "^LOCAL_DEV_TABIX=" .env 2>/dev/null && PROFILES="$PROFILES,tabix" - -echo "$PROFILES" diff --git a/build/dev/dev.sh b/build/dev/dev.sh new file mode 100755 index 000000000..88b794e43 --- /dev/null +++ b/build/dev/dev.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Drives `make up` / `make down`. Reads LOCAL_DEV_* from .env and assembles +# the docker compose invocation: list of -f files + COMPOSE_PROFILES. +# +# Usage: dev.sh up|down [extra docker compose args...] +set -euo pipefail + +cmd="${1:-}" +shift || true + +if [ -z "$cmd" ]; then + echo "usage: dev.sh up|down [extra args]" >&2 + exit 2 +fi + +# Source .env so LOCAL_DEV_* vars are visible. Missing .env is fine — nothing +# add-on is enabled by default. +if [ -f .env ]; then + set -a + # shellcheck disable=SC1091 + . ./.env + set +a +fi + +files=( + -f build/dev/compose.yml + -f build/dev/deps/compose.yml + -f build/dev/deps/compose-gui.yml +) +profiles=() + +# Cache: redis is default; dragonfly opts in and aliases as `redis` on the network. +if [ "${LOCAL_DEV_DRAGONFLY:-}" = "1" ]; then + profiles+=(dragonfly) +else + profiles+=(redis) +fi + +# Log stores +[ "${LOCAL_DEV_POSTGRES:-}" = "1" ] && profiles+=(postgres) +[ "${LOCAL_DEV_CLICKHOUSE:-}" = "1" ] && profiles+=(clickhouse) + +# Message queues +[ "${LOCAL_DEV_RABBITMQ:-}" = "1" ] && profiles+=(rabbitmq) +[ "${LOCAL_DEV_LOCALSTACK:-}" = "1" ] && profiles+=(localstack) +[ "${LOCAL_DEV_GCP:-}" = "1" ] && profiles+=(gcp) + +# GUI tools (any non-empty value enables; value doubles as port via ${VAR:-default}) +[ -n "${LOCAL_DEV_REDIS_COMMANDER:-}" ] && profiles+=(redis-commander) +[ -n "${LOCAL_DEV_PGADMIN:-}" ] && profiles+=(pgadmin) +[ -n "${LOCAL_DEV_TABIX:-}" ] && profiles+=(tabix) + +# Add-on stacks: each is a separate compose file merged into the `outpost` +# project only when its flag is on. File-level inclusion (rather than service +# profiles) avoids cross-file service-name collisions, e.g. otel-collector +# defined by both grafana and uptrace. +[ "${LOCAL_DEV_ENVOY:-}" = "1" ] && files+=(-f build/dev/envoy/compose.yml) +[ "${LOCAL_DEV_GRAFANA:-}" = "1" ] && files+=(-f build/dev/grafana/compose.yml) +[ "${LOCAL_DEV_UPTRACE:-}" = "1" ] && files+=(-f build/dev/uptrace/compose.yml) +[ "${LOCAL_DEV_AZURE:-}" = "1" ] && files+=(-f build/dev/azure/compose.yml) + +# grafana and uptrace both define otel-collector and overlap on OTLP ports. +# Enabling both at once is almost always a misconfiguration. +if [ "${LOCAL_DEV_GRAFANA:-}" = "1" ] && [ "${LOCAL_DEV_UPTRACE:-}" = "1" ]; then + echo "error: LOCAL_DEV_GRAFANA and LOCAL_DEV_UPTRACE cannot both be enabled (they conflict on otel-collector + ports)" >&2 + exit 1 +fi + +COMPOSE_PROFILES="$(IFS=,; echo "${profiles[*]}")" +export COMPOSE_PROFILES + +case "$cmd" in + up) + exec docker compose --env-file .env "${files[@]}" up -d "$@" + ;; + down) + # --remove-orphans cleans up containers from add-on files that may have + # been included previously but aren't in the current invocation. + exec docker compose --env-file .env "${files[@]}" down --remove-orphans "$@" + ;; + *) + echo "usage: dev.sh up|down [extra args]" >&2 + exit 2 + ;; +esac diff --git a/build/dev/envoy/compose.yml b/build/dev/envoy/compose.yml new file mode 100644 index 000000000..2ef52a1ab --- /dev/null +++ b/build/dev/envoy/compose.yml @@ -0,0 +1,8 @@ +name: "outpost" + +services: + envoy: + image: envoyproxy/envoy:v1.31-latest + volumes: + - ./envoy.yaml:/etc/envoy/envoy.yaml:ro + command: ["envoy", "-c", "/etc/envoy/envoy.yaml", "--log-level", "info"] diff --git a/build/dev/grafana/compose.yml b/build/dev/grafana/compose.yml index 455aac06c..3048a4fbf 100644 --- a/build/dev/grafana/compose.yml +++ b/build/dev/grafana/compose.yml @@ -1,4 +1,4 @@ -name: "outpost-grafana" +name: "outpost" services: otel-collector: @@ -36,8 +36,3 @@ services: volumes: grafana-data: - -networks: - default: - name: outpost - external: true diff --git a/build/dev/health.sh b/build/dev/health.sh new file mode 100755 index 000000000..b35c69777 --- /dev/null +++ b/build/dev/health.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Reachability check for the running dev stack. Walks LOCAL_DEV_* flags +# the same way dev.sh does and probes only the services that should be up. +# Exits non-zero if any probe fails. +# +# Polls for up to WAIT seconds (default 5) before declaring failure. +set -uo pipefail + +if [ -f .env ]; then + set -a; . ./.env; set +a +fi + +WAIT="${WAIT:-5}" +URL="${OUTPOST_URL:-http://localhost:3333/api/v1}" +KEY="${OUTPOST_API_KEY:-apikey}" + +probe_http() { + local name=$1 url=$2 extra=${3:-} + # shellcheck disable=SC2086 + if curl -fsS --max-time 3 $extra "$url" >/dev/null 2>&1; then + printf "\033[32m✓\033[0m %s (%s)\n" "$name" "$url" + else + printf "\033[31m✗\033[0m %s — no response from %s\n" "$name" "$url" + return 1 + fi +} + +probe_exec() { + local name=$1 container=$2; shift 2 + if docker exec "$container" "$@" >/dev/null 2>&1; then + printf "\033[32m✓\033[0m %s\n" "$name" + else + printf "\033[31m✗\033[0m %s — %s probe failed: %s\n" "$name" "$container" "$*" + return 1 + fi +} + +run_probes() { + local fail=0 + + # api healthz transitively verifies that deps are reachable — api would + # not have started otherwise (depends_on with service_healthy). + probe_http "api" "$URL/healthz" || fail=1 + + [ "${LOCAL_DEV_ENVOY:-}" = "1" ] && { probe_http "envoy" "http://localhost:9901/ready" || fail=1; } + [ "${LOCAL_DEV_GRAFANA:-}" = "1" ] && { probe_http "grafana" "http://localhost:3000/api/health" || fail=1; } + [ "${LOCAL_DEV_UPTRACE:-}" = "1" ] && { probe_http "uptrace" "http://localhost:14318" || fail=1; } + [ "${LOCAL_DEV_AZURE:-}" = "1" ] && { probe_exec "azuresb" outpost-azuresb-1 sh -c 'true' || fail=1; } + + return $fail +} + +deadline=$(( $(date +%s) + WAIT )) +attempt=0 +while :; do + attempt=$(( attempt + 1 )) + output=$(run_probes) + status=$? + if [ $status -eq 0 ] || [ "$(date +%s)" -ge "$deadline" ]; then + break + fi + sleep 1 +done + +echo "$output" +if [ $status -ne 0 ]; then + echo "Not healthy after ${WAIT}s." >&2 + exit 1 +fi diff --git a/build/dev/smoke.sh b/build/dev/smoke.sh new file mode 100755 index 000000000..306ce6a49 --- /dev/null +++ b/build/dev/smoke.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# End-to-end smoke test: provisions a tenant + webhook destination, publishes +# one event to mock.hookdeck.com, and polls until the delivery attempt is +# persisted. Cleans up on exit. +# +# Validates the full publish → delivery → log pipeline transitively: +# api, redis, mq, delivery worker, egress, log worker, log store. +set -uo pipefail + +API="${OUTPOST_URL:-http://localhost:3333/api/v1}" +KEY="${OUTPOST_API_KEY:-apikey}" +WEBHOOK="${OUTPOST_SMOKE_WEBHOOK:-https://mock.hookdeck.com}" +TOPIC="${OUTPOST_SMOKE_TOPIC:-user.created}" +TIMEOUT_S="${OUTPOST_SMOKE_TIMEOUT:-10}" + +# Unique-per-run ids so re-runs don't share state with each other. +SUFFIX="$(date +%s)$$" +TENANT="smoke_${SUFFIX}" +DEST="smoke_${SUFFIX}" +TRACE="smoke-${SUFFIX}" + +if ! command -v jq >/dev/null 2>&1; then + echo "smoke.sh requires jq" >&2 + exit 2 +fi + +step() { printf " → %s\n" "$1"; } +ok() { printf " \033[32m✓\033[0m %s\n" "$1"; } +fail() { printf " \033[31m✗\033[0m %s\n" "$1" >&2; exit 1; } + +cleaned=0 +cleanup() { + [ "$cleaned" = "1" ] && return + cleaned=1 + curl -fsS -X DELETE -H "Authorization: Bearer $KEY" \ + "$API/tenants/$TENANT/destinations/$DEST" >/dev/null 2>&1 || true + curl -fsS -X DELETE -H "Authorization: Bearer $KEY" \ + "$API/tenants/$TENANT" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +req() { + local method=$1 path=$2 body=${3:-} + if [ -n "$body" ]; then + curl -fsS -X "$method" \ + -H "Authorization: Bearer $KEY" \ + -H "Content-Type: application/json" \ + -d "$body" \ + "$API$path" + else + curl -fsS -X "$method" \ + -H "Authorization: Bearer $KEY" \ + "$API$path" + fi +} + +start=$(date +%s) + +# 1. Tenant +step "PUT /tenants/$TENANT" +req PUT "/tenants/$TENANT" >/dev/null || fail "tenant upsert failed (api unreachable or rejecting auth)" +ok "tenant ready" + +# 2. Destination (webhook) +step "POST /tenants/$TENANT/destinations" +dest_body=$(jq -nc \ + --arg id "$DEST" \ + --arg url "$WEBHOOK" \ + --arg topic "$TOPIC" \ + '{id:$id, type:"webhook", topics:[$topic], config:{url:$url}}') +dest_resp=$(req POST "/tenants/$TENANT/destinations" "$dest_body") \ + || fail "destination create failed" +dest_id=$(echo "$dest_resp" | jq -r '.id') +[ -n "$dest_id" ] && [ "$dest_id" != "null" ] || fail "destination create returned no id: $dest_resp" +ok "destination ready ($dest_id → $WEBHOOK)" + +# 3. Publish +step "POST /publish (topic=$TOPIC, trace=$TRACE)" +event_body=$(jq -nc \ + --arg tenant "$TENANT" \ + --arg topic "$TOPIC" \ + --arg trace "$TRACE" \ + '{tenant_id:$tenant, topic:$topic, data:{trace:$trace, smoke:true}}') +pub_resp=$(req POST "/publish" "$event_body") \ + || fail "publish failed (rabbitmq or publishmq config likely wrong)" +event_id=$(echo "$pub_resp" | jq -r '.id // empty') +ok "event ingested${event_id:+ (id=$event_id)}" + +# 4. Poll attempts until at least one successful delivery is logged. +step "polling attempts every 1s (up to ${TIMEOUT_S}s) — exercises mq → delivery → log → store" +poll_start=$(date +%s) +deadline=$(( poll_start + TIMEOUT_S )) +attempt_status="" +while [ "$(date +%s)" -lt "$deadline" ]; do + elapsed=$(( $(date +%s) - poll_start )) + attempts=$(req GET "/attempts?limit=20" 2>/dev/null || echo '{}') + attempt_status=$(echo "$attempts" | jq -r --arg eid "$event_id" \ + '.models[] | select(.event_id == $eid) | .status' 2>/dev/null | head -1) + printf " +%-2ss %s\n" "$elapsed" "${attempt_status:-pending}" + if [ -n "$attempt_status" ]; then + case "$attempt_status" in + success|delivered) break ;; + esac + fi + sleep 1 +done + +elapsed=$(( $(date +%s) - start )) + +if [ -z "$attempt_status" ]; then + fail "no delivery attempt logged after ${TIMEOUT_S}s — delivery worker or log worker likely stuck" +fi +case "$attempt_status" in + success|delivered) + ok "delivery attempt logged as $attempt_status" + echo + printf "\033[32mPipeline OK in %ss.\033[0m\n" "$elapsed" + ;; + *) + fail "delivery attempt logged but status=$attempt_status (expected success)" + ;; +esac diff --git a/build/dev/uptrace/compose.yml b/build/dev/uptrace/compose.yml index a4c923c25..79b529339 100644 --- a/build/dev/uptrace/compose.yml +++ b/build/dev/uptrace/compose.yml @@ -1,4 +1,4 @@ -name: "outpost-uptrace" +name: "outpost" services: otel-collector: image: otel/opentelemetry-collector-contrib:latest @@ -64,8 +64,3 @@ services: volumes: ch_data2: pg_data3: - -networks: - default: - name: outpost - external: true diff --git a/contributing/getting-started.md b/contributing/getting-started.md index 6c494d627..aa59f2a0b 100644 --- a/contributing/getting-started.md +++ b/contributing/getting-started.md @@ -9,39 +9,45 @@ cd outpost cp .env.dev .env cp .outpost.yaml.dev .outpost.yaml -# Create Docker network (one-time setup) -make network - # Start everything -make up/deps -make up/outpost +make up ``` The API is now available at `http://localhost:3333`. **Default setup includes:** - Redis (cache and state) -- PostgreSQL (log store) +- ClickHouse (log store) - RabbitMQ (message queue) -See [Configuration](#configuration) to customize with ClickHouse, AWS SQS, GCP PubSub, Azure ServiceBus, and more. +See [Configuration](#configuration) to customize with PostgreSQL, AWS SQS, GCP PubSub, Azure ServiceBus, and more. ## Day-to-Day Development ```sh -# Start -make up/deps -make up/outpost +make up # bring up everything enabled in .env +make down # stop and remove the stack +make nuke # stop + remove volumes (wipe state) +``` + +`make up` is declarative — it reads `.env` and reconciles the running stack +to match. Edit `.env`, re-run `make up`, and only the diff is applied. -# Stop -make down/outpost -make down/deps +### Verifying the stack -# Or use shortcuts -make up # starts deps + outpost -make down # stops outpost + deps +```sh +make health # reachability check, polls up to 5s by default +make health WAIT=30 # override poll window (handy right after make up) +make smoke # end-to-end pipeline test (tenant → publish → log) ``` +`make health` walks the same `LOCAL_DEV_*` flags as `make up` and probes +only services that should be running. `make smoke` provisions a tenant + +webhook destination, publishes one event to `https://mock.hookdeck.com`, +and polls until the delivery attempt lands in the log store, then cleans +up — exercising api, redis, mq, delivery worker, egress, log worker, and +log store in one shot. + ## Running Tests ```sh @@ -70,11 +76,11 @@ See the [Test](test.md) documentation for more details. Outpost can be configured via environment variables (`.env`) or a YAML file (`.outpost.yaml`). See [Configuration](config.md) for all options. -> **Tip:** For smoother local DX, consider using `.outpost.yaml` for Outpost settings. Air detects changes to `.outpost.yaml` and automatically restarts the services, whereas `.env` changes require `make down/outpost && make up/outpost`. +> **Tip:** For smoother local DX, consider using `.outpost.yaml` for Outpost settings. Air detects changes to `.outpost.yaml` and automatically restarts the services, whereas `.env` changes require `make down && make up`. -### Local Dependencies (.env) +### Local Dev Stack (.env) -The `.env` file controls which services `make up/deps` starts. Add or remove `LOCAL_DEV_*` variables to customize your stack: +The `.env` file controls which services `make up` starts. Add or remove `LOCAL_DEV_*` variables to customize your stack, then re-run `make up`: ```sh # .env @@ -83,13 +89,13 @@ The `.env` file controls which services `make up/deps` starts. Add or remove `LO # LOCAL_DEV_DRAGONFLY=1 # Log store -LOCAL_DEV_POSTGRES=1 +LOCAL_DEV_CLICKHOUSE=1 # Message queue LOCAL_DEV_RABBITMQ=1 # Optional: Additional services -# LOCAL_DEV_CLICKHOUSE=1 +# LOCAL_DEV_POSTGRES=1 # LOCAL_DEV_LOCALSTACK=1 # LOCAL_DEV_GCP=1 @@ -110,16 +116,17 @@ LOCAL_DEV_RABBITMQ=1 | `LOCAL_DEV_REDIS_COMMANDER=` | Enable Redis Commander | | `LOCAL_DEV_PGADMIN=` | Enable pgAdmin | | `LOCAL_DEV_TABIX=` | Enable Tabix (ClickHouse UI) | +| `LOCAL_DEV_ENVOY=1` | Forward proxy for `DESTINATIONS_WEBHOOK_PROXY_URL` testing | +| `LOCAL_DEV_GRAFANA=1` | OTel collector + Prometheus + Grafana | +| `LOCAL_DEV_UPTRACE=1` | Uptrace observability (alternative to Grafana — cannot be combined) | +| `LOCAL_DEV_AZURE=1` | Azure Service Bus + SQL Edge emulators | -After changing `.env`, restart dependencies: +After changing `.env`, reconcile the stack: ```sh -make down/deps -make up/deps +make up ``` -> **Note:** Azure ServiceBus has its own stack (`make up/azure`) because it's shared between dev and test environments. - ### Host Ports Reference | Service | Host Port | Credentials |