From e9ab53cfd1604e1c8a09ea7f179766daaa822ad2 Mon Sep 17 00:00:00 2001 From: Pooria Mehregan Date: Thu, 6 Aug 2026 12:51:29 +0200 Subject: [PATCH] feat: add local db copy --- .gitignore | 9 +- README.md | 3 +- docker-compose.yml | 18 ++- scripts/db/README.md | 147 +++++++++++++++++ scripts/db/environments.conf | 10 ++ scripts/db/export-bacpac.sh | 139 ++++++++++++++++ scripts/db/import-bacpac.sh | 132 ++++++++++++++++ scripts/db/lib.sh | 281 +++++++++++++++++++++++++++++++++ scripts/db/refresh-local-db.sh | 151 ++++++++++++++++++ scripts/db/start-sqlserver.sh | 181 +++++++++++++++++++++ scripts/db/tests/test-lib.sh | 73 +++++++++ umbraco-infoportal/Program.cs | 9 ++ umbraco-infoportal/README.md | 5 + 13 files changed, 1155 insertions(+), 3 deletions(-) create mode 100644 scripts/db/README.md create mode 100644 scripts/db/environments.conf create mode 100755 scripts/db/export-bacpac.sh create mode 100755 scripts/db/import-bacpac.sh create mode 100644 scripts/db/lib.sh create mode 100755 scripts/db/refresh-local-db.sh create mode 100755 scripts/db/start-sqlserver.sh create mode 100644 scripts/db/tests/test-lib.sh diff --git a/.gitignore b/.gitignore index 51cce17c..ea868c9c 100644 --- a/.gitignore +++ b/.gitignore @@ -443,4 +443,11 @@ override.tf.json *_override.tf.json # MacOS -.DS_Store \ No newline at end of file +.DS_Store + +# Local database copies — may contain production personal data. Never commit. +.bacpacs/ +*.bacpac +# Local developer config overrides (may contain local credentials). Never commit. +appsettings.Local.json +appsettings.*.Local.json diff --git a/README.md b/README.md index f4634d91..0dcc908f 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,5 @@ Repo for Altinn infoportal This repository is organized into distinct applications for the frontend and backend: - **Frontend**: Running instructions and documentation for the frontend app can be found in the [Astro Infoportal README](./astro-infoportal/README.md). -- **Backend**: Running instructions and documentation for the CMS app can be found in the [Umbraco Infoportal README](./umbraco-infoportal/README.md). \ No newline at end of file +- **Backend**: Running instructions and documentation for the CMS app can be found in the [Umbraco Infoportal README](./umbraco-infoportal/README.md). +- **Local database**: To run Umbraco against a copy of a real environment's database, see the [local database copy README](./scripts/db/README.md). \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index a002080e..626520f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,8 +12,24 @@ services: - umbraco_logs:/app/umbraco/Logs - umbraco_data:/app/umbraco/Data - umbraco_media:/app/wwwroot/media - + + mssql: + image: ${MSSQL_IMAGE:-mcr.microsoft.com/mssql/server:2022-latest} + platform: ${MSSQL_PLATFORM:-linux/amd64} + container_name: infoportal-mssql + profiles: ["localdb"] + restart: unless-stopped + environment: + ACCEPT_EULA: "Y" + MSSQL_PID: "Developer" + MSSQL_SA_PASSWORD: "${MSSQL_SA_PASSWORD:-}" + ports: + - "${MSSQL_PORT:-1433}:1433" + volumes: + - mssql_data:/var/opt/mssql + volumes: umbraco_logs: umbraco_data: umbraco_media: + mssql_data: diff --git a/scripts/db/README.md b/scripts/db/README.md new file mode 100644 index 00000000..f6494cb1 --- /dev/null +++ b/scripts/db/README.md @@ -0,0 +1,147 @@ +# Local copy of the Umbraco database + +Copy an environment's database to your machine and run Umbraco against it. + +```bash +scripts/db/refresh-local-db.sh at22 --auth token +``` + +Then start Umbraco. The connection string is written for you. + +## 1. Get access + +You need to be a member of the Entra group **`DIS AzSQL Admin Dev IP`**, which is the Entra +admin on the dev SQL servers. Ask the platform team. Prod is a separate grant. + +You also need to be **connected to the VPN** — the databases are Private Link addresses and +do not resolve otherwise. + +## 2. Install the tools + +**All platforms** + +```bash +dotnet tool install -g microsoft.sqlpackage # ensure ~/.dotnet/tools is on PATH +az login # used for database authentication +``` + +**macOS** — Microsoft ships no arm64 SQL Server image, and it only runs under Rosetta-backed +emulation, so use Colima rather than Docker Desktop or podman: + +```bash +brew install colima docker docker-compose sqlcmd +colima start --vm-type=vz --vz-rosetta --cpu 4 --memory 8 --disk 60 +``` + +**Linux** + +```bash +sudo apt-get install -y docker.io sqlcmd # or your distro's equivalent +``` + +**Windows** — work inside WSL2 (Ubuntu); the scripts are not run from PowerShell. Either +enable Docker Desktop's WSL2 integration, or install Docker inside the distro, then follow the +Linux steps. Keep the repo on the Linux filesystem, not `/mnt/c`, or everything will be slow. + +## 3. Run it + +```bash +scripts/db/refresh-local-db.sh at22 --auth token +``` + +Exports the database, starts SQL Server locally, imports, and writes the connection string to +`umbraco-infoportal/appsettings.Local.json`, which Umbraco loads automatically in Development. + +**That file is git-ignored and contains a password. Never commit it.** + +## 4. Start Umbraco + +```bash +cd umbraco-infoportal +dotnet run +``` + +Then, to log in: + +1. **Turn off the Entra auto-redirect.** In + `umbraco-infoportal/App_Plugins/ExternalLoginProviders/umbraco-package.json`, set + `"autoRedirect"` to `false`. + + Without this, `/umbraco` immediately redirects to Microsoft Entra ID, which is not + configured locally, and you get + `No authentication handler is registered for the scheme 'Umbraco.MicrosoftEntraId'`. + +2. **Open https://localhost:44391/umbraco** and log in as `alv.gullbrand.lia@digdir.no` / + `Test1234`. Use the local username and password form, not the Microsoft button. + + The backoffice requires HTTPS — plain `http://` returns + `error:invalid_request ... This server only accepts HTTPS requests`. If the browser + distrusts the certificate, run `dotnet dev-certs https --trust` once. + +3. **Revert `umbraco-package.json` before committing.** It is a tracked file, so the change + shows up in `git status` and must not be pushed. + +Passwords come from the copied environment. If that account does not work in the environment +you copied, ask the team for a test account or reset one against your local container. + +Use plain `dotnet run` so it starts in Development: that is what loads `appsettings.Local.json` +and keeps Key Vault disabled. Running with `--no-launch-profile` or a Production environment +fails with `Configuration value 'KeyVault:AkvUri' must be configured`. + +The generated `appsettings.Local.json` also turns uSync's automatic export off +(`ExportOnSave: "None"`). Without it, working against a copied environment writes that +environment's entire schema and content to `umbraco-infoportal/uSync/` — thousands of +generated files sitting next to tracked ones. Exporting by hand from the backoffice still +works, which is how uSync files are meant to be produced. + +Useful flags: + +| Flag | Effect | +|---|---| +| `--skip-export` | reuse the newest `.bacpac` instead of downloading again | +| `--keep-container` | import into the running container instead of recreating it | +| `--auth password` | prompt for an Entra password instead of using an `az` token | + +The four scripts also run standalone: `export-bacpac.sh`, `start-sqlserver.sh`, +`import-bacpac.sh`, `refresh-local-db.sh`. Pass `--help` to any of them. + +## What you get, and what you don't + +- **Images will be broken.** Media lives in Azure Blob Storage, not in the database. +- **Log in with the copied environment's backoffice users**, not local ones. +- For doctype and schema work, use uSync instead — it is lighter and involves no production + data. + +## Handling the data + +A copy of a real environment contains real accounts and personal data. + +- Keep it on an encrypted disk. +- Delete the `.bacpac` when you are done. +- Do not share it. `.bacpacs/` and `*.bacpac` are git-ignored; leave it that way. + +Remote environments are strictly read-only: these scripts only ever run a connectivity check +and `sqlpackage /Action:Export` against them, and the import refuses any target that is not +localhost. + +## If something fails + +| Symptom | Cause and fix | +|---|---| +| `cannot resolve ` | Not on the VPN. | +| `Login failed for user ''` | You are not in the `DIS AzSQL Admin Dev IP` group. | +| `AADSTS50126` | Password auth is blocked for your account — use `--auth token`. | +| `SQL Server segfaulted on startup (exit 139)` | Your runtime emulates with qemu, not Rosetta. Use Colima as above, or set the fallback below. | +| `x509: negative serial number` | Only affects hand-run `sqlcmd`; prefix it with `GODEBUG=x509negativeserial=1`. | +| Import fails on a foreign key | The export caught the source mid-write. Re-export. | + +**Without Rosetta** (arm64 machines that cannot use Colima) put this in `scripts/db/.env` to use +the arm64-native Azure SQL Edge instead. It works, but it is a reduced engine and not what the +environments run: + +``` +MSSQL_IMAGE=mcr.microsoft.com/azure-sql-edge +MSSQL_PLATFORM=linux/arm64 +``` + +To point the scripts at a specific container runtime, set `CONTAINER_CLI` in `scripts/db/.env`. diff --git a/scripts/db/environments.conf b/scripts/db/environments.conf new file mode 100644 index 00000000..dbb17346 --- /dev/null +++ b/scripts/db/environments.conf @@ -0,0 +1,10 @@ +# Azure SQL servers per environment. Private Link addresses — reachable only over VPN. +# Source: Confluence, "Systemdokumentasjon Altinn Infoportal" -> AzureSQL +# +# Only at22 is active. Enable another environment by uncommenting its line. +# +# env host catalog +at22 infop-at22-vllfov-server.privatelink.database.windows.net umbraco +#at23 infop-at23-vtjzsk-server.privatelink.database.windows.net umbraco +#tt02 infop-tt02-ksalae-server.privatelink.database.windows.net umbraco +#prod infop-prod-hxhapc-server.privatelink.database.windows.net umbraco diff --git a/scripts/db/export-bacpac.sh b/scripts/db/export-bacpac.sh new file mode 100755 index 00000000..c9e0b12f --- /dev/null +++ b/scripts/db/export-bacpac.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# +# Export an environment's Umbraco database to a local .bacpac file. +# +# READ-ONLY against the remote server. This script performs exactly two remote +# operations: a TCP reachability check and sqlpackage /Action:Export. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +. "$SCRIPT_DIR/lib.sh" + +usage() { + cat >&2 < [--out DIR] [--user ENTRA_EMAIL] [--auth METHOD] + + environment name; one of: $(list_environments) + --out DIR output directory (default: /.bacpacs) + --user EMAIL Entra ID account (default: \$INFOPORTAL_DB_USER, or + git config user.email when it is a @digdir.no address) + --auth METHOD password (default) or token. + + 'password' uses Authentication=Active Directory Password and + prompts. If Entra answers AADSTS50126 even with the correct + password, ROPC is blocked for your account -- use token. + + 'token' takes an Entra access token from the Azure CLI + (az login required). Nothing is prompted. +EOF + exit 1 +} + +ENV_NAME="" +OUT_DIR="$INFOPORTAL_REPO_ROOT/.bacpacs" +DB_USER="${INFOPORTAL_DB_USER:-}" +AUTH_METHOD="password" + +while [ $# -gt 0 ]; do + case "$1" in + --out) [ $# -ge 2 ] || usage; OUT_DIR="$2"; shift 2 ;; + --user) [ $# -ge 2 ] || usage; DB_USER="$2"; shift 2 ;; + --auth) [ $# -ge 2 ] || usage; AUTH_METHOD="$2"; shift 2 ;; + -h|--help) usage ;; + -*) log "unknown option: $1"; usage ;; + *) [ -z "$ENV_NAME" ] || usage; ENV_NAME="$1"; shift ;; + esac +done + +case "$AUTH_METHOD" in + password|token) ;; + *) log "unknown --auth value '$AUTH_METHOD'"; usage ;; +esac + +[ -n "$ENV_NAME" ] || usage + +require_sqlpackage +SQLPACKAGE="$(find_sqlpackage)" + +ENV_OUTPUT="$(resolve_environment "$ENV_NAME")" +read -r HOST CATALOG <&2 + read -r CONFIRM + [ "$CONFIRM" = "prod" ] || die "aborted" +fi + +log "checking connectivity to $HOST:1433 ..." +require_remote_reachable "$HOST" 1433 + +CONN="Server=tcp:$HOST,1433; Initial Catalog=$CATALOG; Encrypt=True; TrustServerCertificate=True" + +if [ "$AUTH_METHOD" = "token" ]; then + command -v az >/dev/null 2>&1 || die "the token auth method needs the Azure CLI. +Install it, run 'az login', then retry." + + log "requesting an Entra access token for database.windows.net ..." + # Never echo, log or persist the token. It goes straight into sqlpackage's + # argv and nowhere else. + DB_TOKEN="$(az account get-access-token --resource https://database.windows.net/ \ + --query accessToken -o tsv 2>/dev/null || true)" + [ -n "$DB_TOKEN" ] || die "could not get an access token. Run 'az login' and check +that 'az account show' names the tenant that owns this database." +else + if [ -z "$DB_USER" ]; then + DB_USER="$(git -C "$INFOPORTAL_REPO_ROOT" config user.email 2>/dev/null || true)" + case "$DB_USER" in *@digdir.no) ;; *) DB_USER="" ;; esac + fi + printf 'Entra ID email%s: ' "${DB_USER:+ [$DB_USER]}" >&2 + read -r USER_INPUT + [ -n "$USER_INPUT" ] && DB_USER="$USER_INPUT" + [ -n "$DB_USER" ] || die "no Entra ID email given" + + printf 'Password for %s: ' "$DB_USER" >&2 + read -rs DB_PASSWORD + printf '\n' >&2 + [ -n "$DB_PASSWORD" ] || die "no password given" + + CONN="$CONN; Authentication=Active Directory Password; User ID=$DB_USER; Password=$DB_PASSWORD" +fi + +mkdir -p "$OUT_DIR" +TARGET="$OUT_DIR/$ENV_NAME-umbraco-$(date +%Y%m%d-%H%M%S).bacpac" + +log "exporting $ENV_NAME ($CATALOG) -> $TARGET" +log "this reads the whole database and can take several minutes ..." + +if [ "$AUTH_METHOD" = "token" ]; then + # An access token lasts about an hour. A very large export can outlive it; + # if that happens, re-run — sqlpackage cannot refresh it mid-flight. + "$SQLPACKAGE" /Action:Export \ + /SourceConnectionString:"$CONN" \ + /AccessToken:"$DB_TOKEN" \ + /TargetFile:"$TARGET" \ + || die "export failed. + +If it says \"Login failed for user ''\", your Entra +account authenticated but has no login on this server. Ask for access to be +granted on $HOST." +else + "$SQLPACKAGE" /Action:Export \ + /SourceConnectionString:"$CONN" \ + /TargetFile:"$TARGET" \ + || die "export failed. + +If it says AADSTS50126, Entra rejected the password. That usually means password +(ROPC) authentication is blocked for your account rather than the password being +wrong. Retry with: $(basename "$0") $ENV_NAME --auth token" +fi + +log "exported $(du -h "$TARGET" | cut -f1) to:" +[ -s "$TARGET" ] || die "export reported success but $TARGET is missing or empty" +printf '%s\n' "$TARGET" diff --git a/scripts/db/import-bacpac.sh b/scripts/db/import-bacpac.sh new file mode 100755 index 00000000..81aa1eb6 --- /dev/null +++ b/scripts/db/import-bacpac.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# +# Import a .bacpac into the local container. +# +# Writes ONLY to localhost. assert_local_target makes that structural: a +# mistyped host exits rather than reaching Azure. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +. "$SCRIPT_DIR/lib.sh" + +ENV_FILE="$SCRIPT_DIR/.env" + +usage() { + cat >&2 < [--db NAME] [--port N] + + --db NAME target database name (default: umbraco) + --port N host port of the container (default: MSSQL_PORT from .env, else 1433) +EOF + exit 1 +} + +BACPAC="" +DB_NAME="umbraco" +PORT_OVERRIDE="" + +while [ $# -gt 0 ]; do + case "$1" in + --db) [ $# -ge 2 ] || usage; DB_NAME="$2"; shift 2 ;; + --port) [ $# -ge 2 ] || usage; PORT_OVERRIDE="$2"; shift 2 ;; + -h|--help) usage ;; + -*) log "unknown option: $1"; usage ;; + *) [ -z "$BACPAC" ] || usage; BACPAC="$1"; shift ;; + esac +done + +case "$DB_NAME" in + ''|*[!A-Za-z0-9_]*) die "invalid --db value '$DB_NAME': letters, digits and underscore only" ;; +esac + +[ -n "$BACPAC" ] || usage +[ -f "$BACPAC" ] || die "no such file: $BACPAC" + +[ -f "$ENV_FILE" ] || die "$ENV_FILE not found. Run start-sqlserver.sh first." +# shellcheck source=/dev/null +. "$ENV_FILE" + +PORT="${PORT_OVERRIDE:-${MSSQL_PORT:-1433}}" +HOST="localhost" + +# Validate the resolved port before it is ever interpolated into a connection +# string. .NET's connection-string parser uses the LAST occurrence of a +# duplicate key, so a non-numeric port (e.g. containing a `;`) can inject +# additional keys - including a `Server=` that redirects this write to a +# remote host. This must run before assert_local_target, which only inspects +# the literal host and never sees the assembled connection string. +case "$PORT" in + ''|*[!0-9]*) die "invalid port '$PORT': must be a number. A non-numeric port can inject additional connection-string keys and redirect this write to a remote server." ;; +esac + +# The guard. Do not remove, and do not turn into a prompt. +assert_local_target "$HOST" + +require_sqlpackage +require_sqlcmd +SQLPACKAGE="$(find_sqlpackage)" + +SA_PASSWORD="${MSSQL_SA_PASSWORD:?MSSQL_SA_PASSWORD missing from $ENV_FILE}" +CONN="Server=$HOST,$PORT;Initial Catalog=$DB_NAME;User ID=sa;Password=$SA_PASSWORD;Encrypt=True;TrustServerCertificate=True" + +log "dropping existing database '$DB_NAME' if present ..." +run_sqlcmd -S "$HOST,$PORT" -U sa -P "$SA_PASSWORD" -C \ + -Q "DROP DATABASE IF EXISTS [$DB_NAME];" >/dev/null + +log "importing $(basename "$BACPAC") into '$DB_NAME' ..." +START="$(date +%s)" +"$SQLPACKAGE" /Action:Import \ + /SourceFile:"$BACPAC" \ + /TargetConnectionString:"$CONN" \ + || die "import failed. + +If it failed on a foreign-key constraint, the export caught the source database +mid-write - bacpac export is not transactionally consistent. Re-export and try +again; see scripts/db/README.md if it keeps happening." +log "imported in $(( $(date +%s) - START ))s" + +log "verifying content ..." +ROWS="$(run_sqlcmd -S "$HOST,$PORT" -U sa -P "$SA_PASSWORD" -C -d "$DB_NAME" \ + -h -1 -W -Q "SET NOCOUNT ON; SELECT COUNT(*) FROM umbracoNode;" | tr -d '[:space:]')" +[ "$ROWS" -gt 0 ] 2>/dev/null || die "umbracoNode is empty - the import did not land data" +log "umbracoNode rows: $ROWS" + +log "checking constraint consistency ..." +CHECK_RESULT="$(run_sqlcmd -S "$HOST,$PORT" -U sa -P "$SA_PASSWORD" -C -d "$DB_NAME" \ + -h -1 -W -Q "SET NOCOUNT ON; +BEGIN TRY + CREATE TABLE #cc (TableName sysname, ConstraintName sysname, WhereClause nvarchar(max)); + INSERT INTO #cc EXEC('DBCC CHECKCONSTRAINTS WITH ALL_CONSTRAINTS'); + SELECT 'CHECKOK ' + CAST(COUNT(*) AS varchar(20)) FROM #cc; +END TRY +BEGIN CATCH + SELECT 'CHECKFAILED ' + ERROR_MESSAGE(); +END CATCH" 2>&1 | grep -E '^CHECK(OK|FAILED)' | tail -1 || true)" + +# The `|| true` above is load-bearing. grep exits 1 when it matches nothing, +# and under `set -o pipefail` that failure propagates to the assignment and +# errexit aborts the script -- which would make the catch-all branch below +# unreachable and turn a failed check into a bare `exit 1` with no diagnostic. +# Absorbing it lets an empty result reach the case and explain itself. +case "$CHECK_RESULT" in + 'CHECKOK '*) VIOLATIONS="${CHECK_RESULT#CHECKOK }" ;; + 'CHECKFAILED '*) die "constraint check failed to run: ${CHECK_RESULT#CHECKFAILED }" ;; + *) die "the constraint check produced no usable result. + +The check itself failed rather than finding violations - typically the +database became unreachable, or sqlcmd errored. The import itself already +succeeded; re-run the check by hand against '$DB_NAME' on localhost,$PORT." ;; +esac + +case "$VIOLATIONS" in + ''|*[!0-9]*) die "constraint check did not return a count (got: '$VIOLATIONS')" ;; +esac + +if [ "$VIOLATIONS" -eq 0 ]; then + log "no constraint violations" +else + log "WARNING: $VIOLATIONS constraint violation(s) found. The export caught the" + log "source mid-write. The copy is usable but may be internally inconsistent;" + log "re-export for a clean one." +fi diff --git a/scripts/db/lib.sh b/scripts/db/lib.sh new file mode 100644 index 00000000..918222e0 --- /dev/null +++ b/scripts/db/lib.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash +# Shared helpers for the local database copy scripts. +# Sourced, not executed. Must have no side effects on source. + +INFOPORTAL_DB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INFOPORTAL_REPO_ROOT="$(cd "$INFOPORTAL_DB_DIR/../.." && pwd)" +INFOPORTAL_DEFAULT_CONF="$INFOPORTAL_DB_DIR/environments.conf" + +log() { printf '%s\n' "$*" >&2; } +die() { printf 'error: %s\n' "$*" >&2; exit 1; } + +# list_environments [conf] -> "at22 tt02 " +list_environments() { + local conf="${1:-$INFOPORTAL_DEFAULT_CONF}" + awk 'substr($1,1,1) != "#" && NF >= 3 { printf "%s ", $1 }' "$conf" +} + +# resolve_environment [conf] -> " " +resolve_environment() { + local env_name="$1" + local conf="${2:-$INFOPORTAL_DEFAULT_CONF}" + [ -f "$conf" ] || die "environment config not found: $conf" + + local result + result="$(awk -v want="$env_name" ' + substr($1,1,1) != "#" && NF >= 3 && $1 == want { print $2, $3; found = 1; exit } + END { exit !found } + ' "$conf")" || die "unknown environment '$env_name'. Available: $(list_environments "$conf")" + + printf '%s\n' "$result" +} + +# assert_local_target +# +# Hard guard: import and any other write action must never reach a remote server. +# This exits rather than prompting — a mistyped argument must be incapable of +# writing to Azure. Do not soften this into a confirmation. +assert_local_target() { + local raw="${1:-}" + local host="${raw%%,*}" + + [ -n "$host" ] || die "no target host given; refusing to continue" + + case "$host" in + *database.windows.net*) + die "refusing to target Azure SQL host '$host'. Remote environments are read-only; this action only writes to a local container." + ;; + esac + + case "$host" in + localhost|127.0.0.1|::1|'[::1]') return 0 ;; + *) + die "refusing to target non-local host '$host'. Allowed targets: localhost, 127.0.0.1, ::1." + ;; + esac +} + +# find_sqlpackage -> path on stdout +# `dotnet tool install -g` does not put ~/.dotnet/tools on PATH, so check there too. +find_sqlpackage() { + if command -v sqlpackage >/dev/null 2>&1; then + command -v sqlpackage + return 0 + fi + if [ -x "$HOME/.dotnet/tools/sqlpackage" ]; then + printf '%s\n' "$HOME/.dotnet/tools/sqlpackage" + return 0 + fi + return 1 +} + +require_sqlpackage() { + find_sqlpackage >/dev/null 2>&1 || die "sqlpackage not found. Install it with: + dotnet tool install -g microsoft.sqlpackage +and ensure \$HOME/.dotnet/tools is on your PATH." +} + +require_sqlcmd() { + command -v sqlcmd >/dev/null 2>&1 || die "sqlcmd not found. Install it with: + brew install sqlcmd # macOS + sudo apt-get install mssql-tools18 # Debian/Ubuntu" +} + +# run_sqlcmd +# GODEBUG is mandatory: Azure SQL Edge presents a certificate with a negative +# serial number, which Go's x509 parser rejects outright. -C does not help +# because parsing fails before trust is evaluated. +run_sqlcmd() { + GODEBUG=x509negativeserial=1 sqlcmd "$@" +} + +# container_cli -> "docker" | "podman" +container_cli() { + if [ -n "${CONTAINER_CLI:-}" ]; then printf '%s\n' "$CONTAINER_CLI"; return 0; fi + + # Prefer a real docker over the podman-docker shim. On macOS `docker` is + # commonly a symlink to podman, and podman invoked under that name reports + # itself as docker -- so resolve every `docker` on PATH and pick the first + # that does NOT accept podman's --connection flag. Never trust the name. + local candidate + candidate="$(IFS=: + for dir in $PATH; do + [ -x "$dir/docker" ] || continue + if ! "$dir/docker" --help 2>&1 | grep -q -- '--connection'; then + printf '%s\n' "$dir/docker" + break + fi + done)" + if [ -n "$candidate" ]; then printf '%s\n' "$candidate"; return 0; fi + + if command -v docker >/dev/null 2>&1; then printf 'docker\n'; return 0; fi + if command -v podman >/dev/null 2>&1; then printf 'podman\n'; return 0; fi + return 1 +} + +# container_is_podman -> exit 0 if the detected CLI is podman. +# Do not grep --version: podman adopts a docker-compat personality when +# invoked through a `docker` symlink (podman-mac-helper) and reports +# "docker version X". Test for the flag we actually need instead — docker +# has no --connection global flag. +container_is_podman() { + local cli + cli="$(container_cli)" || return 1 + "$cli" --help 2>&1 | grep -q -- '--connection' +} + +# cre - container runtime exec, with the podman connection pinned. +# +# The active podman connection is global user state that any other project can +# change, and only one podman machine runs at a time. Pinning prevents the +# database landing in an unrelated project's VM. +cre() { + local cli + cli="$(container_cli)" || die "no container runtime found. Install docker or podman." + + if container_is_podman; then + if command -v podman >/dev/null 2>&1; then cli=podman; fi + "$cli" --connection "${INFOPORTAL_PODMAN_CONNECTION:-infoportal-db}" "$@" + else + "$cli" "$@" + fi +} + +# compose - run compose against the detected runtime. +# +# Not every CLI ships the compose plugin: Homebrew's `docker` formula is the +# bare client, so `docker compose` is an unknown command there. Fall back to a +# standalone docker-compose binary, pointed at the same daemon via the CLI's +# current context, so a developer does not have to install a plugin by hand. +compose() { + local cli + cli="$(container_cli)" || die "no container runtime found. Install docker or podman." + + if "$cli" compose version >/dev/null 2>&1; then + cre compose "$@" + return + fi + + if command -v docker-compose >/dev/null 2>&1; then + local host + host="$("$cli" context inspect --format '{{.Endpoints.docker.Host}}' 2>/dev/null || true)" + if [ -n "$host" ]; then + DOCKER_HOST="$host" docker-compose "$@" + else + docker-compose "$@" + fi + return + fi + + die "no compose implementation found for '$cli'. + +Install one of: + brew install docker-compose # standalone binary, picked up automatically + (podman ships 'podman compose' already)" +} + +# require_container_connection - verify the pinned podman connection exists. +# +# Deliberately NOT inside cre(). die() calls exit, and cre()'s call sites wrap it +# in command substitutions with `2>/dev/null` and `|| true` fallbacks — which +# discard the message and skip the fallbacks, so the script would abort with no +# output at all for exactly the misconfigured user this check exists to help. +# Call this explicitly, early, and unredirected. +require_container_connection() { + container_is_podman || return 0 + + local cli conn + cli="$(container_cli)" || die "no container runtime found. Install docker or podman." + if command -v podman >/dev/null 2>&1; then cli=podman; fi + conn="${INFOPORTAL_PODMAN_CONNECTION:-infoportal-db}" + + "$cli" system connection list --format '{{.Name}}' 2>/dev/null | grep -qx "$conn" && return 0 + + die "podman connection '$conn' not found. + +This tooling pins its connection so the database cannot land in another +project's VM. Create the machine, or point at an existing one with +INFOPORTAL_PODMAN_CONNECTION=. + +Only one podman machine runs at a time. To switch: + podman machine stop + podman machine start $conn" +} + +# tcp_check [timeout_seconds] +# +# Deliberately avoids timeout(1), which macOS does not ship. Runs the connect +# in a subshell and reaps it with a watchdog loop. +tcp_check() { + local host="$1" port="$2" timeout="${3:-5}" waited=0 pid + + ( exec 3<>"/dev/tcp/$host/$port" ) 2>/dev/null & + pid=$! + + while kill -0 "$pid" 2>/dev/null; do + if [ "$waited" -ge "$timeout" ]; then + kill "$pid" 2>/dev/null + wait "$pid" 2>/dev/null + return 1 + fi + sleep 1 + waited=$((waited + 1)) + done + + wait "$pid" +} + +# resolve_host_ipv4 -> prints an IPv4 address +# +# bash 3.2 (Apple's build) resolves /dev/tcp hostnames with gethostbyname, which +# ignores the split-horizon DNS a VPN installs on macOS. Private Link names then +# fail to resolve inside /dev/tcp even though dscacheutil, sqlcmd and sqlpackage +# all resolve them fine via getaddrinfo. So resolve with a system resolver first +# and connect to the address, never to the name. +resolve_host_ipv4() { + local host="$1" ip="" + + if command -v getent >/dev/null 2>&1; then + ip="$(getent ahostsv4 "$host" 2>/dev/null | awk '/STREAM/ { print $1; exit }')" + [ -n "$ip" ] || ip="$(getent hosts "$host" 2>/dev/null | awk '{ print $1; exit }')" + fi + if [ -z "$ip" ] && command -v dscacheutil >/dev/null 2>&1; then + ip="$(dscacheutil -q host -a name "$host" 2>/dev/null | awk '/^ip_address:/ { print $2; exit }')" + fi + if [ -z "$ip" ] && command -v dig >/dev/null 2>&1; then + ip="$(dig +short A "$host" 2>/dev/null | awk '/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ { print; exit }')" + fi + if [ -z "$ip" ] && command -v host >/dev/null 2>&1; then + ip="$(host -t A "$host" 2>/dev/null | awk '/has address/ { print $NF; exit }')" + fi + + [ -n "$ip" ] || return 1 + printf '%s\n' "$ip" +} + +# require_remote_reachable +# +# Splits the two failure modes the old single check could not tell apart: +# a DNS failure and a routing failure now produce different messages. +require_remote_reachable() { + local host="$1" port="$2" ip + + if ! ip="$(resolve_host_ipv4 "$host")"; then + die "cannot resolve $host. + +This is a Private Link address, so it only resolves over the VPN. + 1. Connect to the VPN. + 2. If you are connected, your VPN's DNS is not serving the privatelink zone." + fi + + tcp_check "$ip" "$port" 8 && return 0 + + die "$host resolves to $ip, but $ip:$port is not reachable. + +DNS is working, so this is routing or access rather than the VPN being down: + 1. The VPN may not route this subnet. + 2. Your account may not have been granted access to this database. + +Check with: + sqlcmd -S $host -d umbraco -C -U -P " +} diff --git a/scripts/db/refresh-local-db.sh b/scripts/db/refresh-local-db.sh new file mode 100755 index 00000000..d0bce806 --- /dev/null +++ b/scripts/db/refresh-local-db.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# +# Copy an environment's database into a local container, end to end. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +. "$SCRIPT_DIR/lib.sh" + +usage() { + cat >&2 < [--skip-export] [--keep-container] [--port N] + [--auth METHOD] [--user ENTRA_EMAIL] + + one of: $(list_environments) + --skip-export reuse the newest existing .bacpac for this environment + --keep-container import into the running container instead of recreating it + --port N host port for the container (default 1433) + --auth METHOD password (default) or token; passed to export-bacpac.sh. + Use token where Entra blocks password (ROPC) auth. + --user EMAIL Entra ID account; passed to export-bacpac.sh +EOF + exit 1 +} + +ENV_NAME="" +SKIP_EXPORT=0 +KEEP_CONTAINER=0 +PORT_ARGS="" +EXPORT_ARGS="" + +while [ $# -gt 0 ]; do + case "$1" in + --skip-export) SKIP_EXPORT=1; shift ;; + --keep-container) KEEP_CONTAINER=1; shift ;; + --port) [ $# -ge 2 ] || usage; PORT_ARGS="--port $2"; shift 2 ;; + --auth) [ $# -ge 2 ] || usage; EXPORT_ARGS="$EXPORT_ARGS --auth $2"; shift 2 ;; + --user) [ $# -ge 2 ] || usage; EXPORT_ARGS="$EXPORT_ARGS --user $2"; shift 2 ;; + -h|--help) usage ;; + -*) log "unknown option: $1"; usage ;; + *) [ -z "$ENV_NAME" ] || usage; ENV_NAME="$1"; shift ;; + esac +done + +[ -n "$ENV_NAME" ] || usage +resolve_environment "$ENV_NAME" >/dev/null + +BACPAC_DIR="$INFOPORTAL_REPO_ROOT/.bacpacs" + +if [ "$SKIP_EXPORT" -eq 1 ]; then + # Filenames are ENV-umbraco-TIMESTAMP.bacpac, a fixed alnum/dash charset from + # export-bacpac.sh; ls -t is sufficient and simpler than find here. + # shellcheck disable=SC2012 + BACPAC="$(ls -t "$BACPAC_DIR/$ENV_NAME-umbraco-"*.bacpac 2>/dev/null | head -1 || true)" + [ -n "$BACPAC" ] || die "no existing .bacpac for '$ENV_NAME' in $BACPAC_DIR" + log "==> reusing $(basename "$BACPAC")" +else + log "==> step 1/3: exporting from $ENV_NAME" + # shellcheck disable=SC2086 + BACPAC="$("$SCRIPT_DIR/export-bacpac.sh" "$ENV_NAME" $EXPORT_ARGS | tail -1)" +fi + +log "==> step 2/3: starting the local container" +KEEP_ARG="" +[ "$KEEP_CONTAINER" -eq 1 ] && KEEP_ARG="--keep" +# shellcheck disable=SC2086 +SA_PASSWORD="$("$SCRIPT_DIR/start-sqlserver.sh" $KEEP_ARG $PORT_ARGS | tail -1)" + +log "==> step 3/3: importing" +# shellcheck disable=SC2086 +"$SCRIPT_DIR/import-bacpac.sh" "$BACPAC" $PORT_ARGS + +# shellcheck source=/dev/null +. "$SCRIPT_DIR/.env" + +DSN="Server=localhost,${MSSQL_PORT};Database=umbraco;User Id=sa;Password=${SA_PASSWORD};TrustServerCertificate=True;Encrypt=False;MultipleActiveResultSets=True" +LOCAL_SETTINGS="$INFOPORTAL_REPO_ROOT/umbraco-infoportal/appsettings.Local.json" +MARKER='"_generatedBy": "scripts/db/refresh-local-db.sh"' + +# Write the connection string where Umbraco will pick it up on its own. +# appsettings.Local.json is git-ignored and Development-only, so the tracked +# appsettings.Development.json stays identical to main. +# +# Refuse to clobber a file we did not write: a developer may keep their own +# local overrides here, and silently overwriting them would be the kind of +# quiet destruction this tooling is meant to avoid. +if [ -f "$LOCAL_SETTINGS" ] && ! grep -qF "$MARKER" "$LOCAL_SETTINGS"; then + log "" + log "NOTE: $LOCAL_SETTINGS exists and was not written by this script, so it" + log "was left untouched. Add the connection string below to it by hand." + WROTE_LOCAL=0 +else + umask 077 + # uSync's ExportOnSave defaults to "All", so working against a copied + # environment writes that environment's whole schema and content to disk -- + # thousands of generated files sitting next to tracked ones, easy to commit by + # accident. Turn the automatic export off locally; exporting by hand from the + # backoffice still works, which is how uSync files are meant to be produced. + cat >"$LOCAL_SETTINGS" <&2 <"$ENV_FILE" </dev/null 2>&1 || true + # shellcheck disable=SC2086 + compose $COMPOSE_ARGS rm -f mssql >/dev/null 2>&1 || true + cre volume rm infoportal_mssql_data >/dev/null 2>&1 || true + # shellcheck disable=SC2086 + compose $COMPOSE_ARGS up -d mssql +} + +# --- start ------------------------------------------------------------------ +if [ "$RECREATE" -eq 1 ]; then + log "recreating container ..." + start_container_fresh +else + if cre ps --format '{{.Names}}' | grep -qx "$CONTAINER_NAME"; then + log "reusing running container $CONTAINER_NAME" + else + start_container_fresh + fi +fi + +# --- wait for readiness ----------------------------------------------------- +# Polled from the host: the Azure SQL Edge image ships no mssql-tools, so an +# in-container healthcheck cannot work for both images. +log "waiting for the database to accept connections ..." +WAITED=0 +until run_sqlcmd -S "localhost,$MSSQL_PORT" -U sa -P "$MSSQL_SA_PASSWORD" -C -Q "SELECT 1" >/dev/null 2>&1; do + # Don't just wait out the clock: if the engine died, say why now. + RUNNING="$(cre inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null || echo unknown)" + if [ "$RUNNING" = "false" ]; then + EXIT_CODE="$(cre inspect -f '{{.State.ExitCode}}' "$CONTAINER_NAME" 2>/dev/null || echo '?')" + log "container logs:" + cre logs "$CONTAINER_NAME" 2>&1 | tail -15 >&2 + if [ "$EXIT_CODE" = "139" ]; then + die "SQL Server segfaulted on startup (exit 139). + +That means your container runtime is emulating x86 with qemu rather than +Rosetta. SQL Server does not survive qemu user-mode emulation. + +Either use a Rosetta-backed runtime -- Colima: + colima start --vm-type=vz --vz-rosetta --cpu 4 --memory 8 + CONTAINER_CLI=\$(command -v docker) # must be a real docker, not the podman shim +or fall back to the arm64-native engine by putting this in scripts/db/.env: + MSSQL_IMAGE=mcr.microsoft.com/azure-sql-edge + MSSQL_PLATFORM=linux/arm64 +(Azure SQL Edge is a reduced engine and not what the environments run.)" + fi + die "the database container exited (code $EXIT_CODE) before becoming ready" + fi + + if [ "$WAITED" -ge 180 ]; then + log "container logs:" + cre logs "$CONTAINER_NAME" 2>&1 | tail -20 >&2 + die "database did not become ready within 180s" + fi + sleep 5 + WAITED=$((WAITED + 5)) +done + +log "ready after ${WAITED}s" +printf '%s\n' "$MSSQL_SA_PASSWORD" diff --git a/scripts/db/tests/test-lib.sh b/scripts/db/tests/test-lib.sh new file mode 100644 index 00000000..8531e425 --- /dev/null +++ b/scripts/db/tests/test-lib.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -uo pipefail + +TESTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../lib.sh +. "$TESTS_DIR/../lib.sh" + +PASS=0 +FAIL=0 + +assert_eq() { + local expected="$1" actual="$2" label="$3" + if [ "$expected" = "$actual" ]; then + PASS=$((PASS + 1)); printf 'ok %s\n' "$label" + else + FAIL=$((FAIL + 1)); printf 'FAIL %s\n expected: %s\n actual: %s\n' "$label" "$expected" "$actual" + fi +} + +assert_fails() { + local label="$2" + if ( eval "$1" ) >/dev/null 2>&1; then + FAIL=$((FAIL + 1)); printf 'FAIL %s (expected non-zero exit)\n' "$label" + else + PASS=$((PASS + 1)); printf 'ok %s\n' "$label" + fi +} + +assert_succeeds() { + local label="$2" + if ( eval "$1" ) >/dev/null 2>&1; then + PASS=$((PASS + 1)); printf 'ok %s\n' "$label" + else + FAIL=$((FAIL + 1)); printf 'FAIL %s (expected zero exit)\n' "$label" + fi +} + +FIXTURE="$(mktemp)" +cat >"$FIXTURE" <<'EOF' +# env host catalog +at22 at22.example.invalid umbraco +#at23 at23.example.invalid umbraco +EOF + +assert_eq "at22.example.invalid umbraco" "$(resolve_environment at22 "$FIXTURE")" "resolve_environment returns host and catalog" +assert_eq "at22" "$(list_environments "$FIXTURE" | tr -d ' ')" "list_environments omits commented entries" +assert_fails "resolve_environment at23 '$FIXTURE'" "commented environment is not resolvable" +assert_fails "resolve_environment nope '$FIXTURE'" "unknown environment exits non-zero" + +rm -f "$FIXTURE" + +assert_succeeds "assert_local_target localhost" "localhost allowed" +assert_succeeds "assert_local_target 127.0.0.1" "127.0.0.1 allowed" +assert_succeeds "assert_local_target 'localhost,1433'" "host with port allowed" + +assert_fails "assert_local_target infop-at22-vllfov-server.privatelink.database.windows.net" "azure sql host rejected" +assert_fails "assert_local_target 'infop-prod-hxhapc-server.privatelink.database.windows.net,1433'" "azure sql host with port rejected" +assert_fails "assert_local_target 10.0.0.5" "arbitrary remote host rejected" +assert_fails "assert_local_target ''" "empty host rejected" + +# tcp_check must fail fast against a port nothing listens on, and must not +# depend on timeout(1), which macOS does not ship. +START="$(date +%s)" +assert_fails "tcp_check 127.0.0.1 9 2" "tcp_check fails on a closed port" +ELAPSED=$(( $(date +%s) - START )) +if [ "$ELAPSED" -le 6 ]; then + PASS=$((PASS+1)); printf 'ok tcp_check respects its timeout (%ss)\n' "$ELAPSED" +else + FAIL=$((FAIL+1)); printf 'FAIL tcp_check took %ss\n' "$ELAPSED" +fi + +printf '\n%d passed, %d failed\n' "$PASS" "$FAIL" +[ "$FAIL" -eq 0 ] diff --git a/umbraco-infoportal/Program.cs b/umbraco-infoportal/Program.cs index 1bcc80ea..481d9f1c 100644 --- a/umbraco-infoportal/Program.cs +++ b/umbraco-infoportal/Program.cs @@ -24,6 +24,15 @@ options.KnownProxies.Clear(); }); +// Local developer overrides, e.g. the connection string for a database copy +// produced by scripts/db/refresh-local-db.sh. Git-ignored and Development-only, +// so it can hold local credentials without any risk of being committed and +// without diverging appsettings.Development.json from main. +if (builder.Environment.IsDevelopment()) +{ + builder.Configuration.AddJsonFile("appsettings.Local.json", optional: true, reloadOnChange: true); +} + builder.Services.Configure( builder.Configuration.GetSection(KeyVaultOptions.SectionName)); diff --git a/umbraco-infoportal/README.md b/umbraco-infoportal/README.md index bcaf566f..e70231c0 100644 --- a/umbraco-infoportal/README.md +++ b/umbraco-infoportal/README.md @@ -24,6 +24,11 @@ Once running, navigate to the default port indicated in your console output to v ## Database The project utilizes a local SQLite database by default, housed within the `umbraco/Data/` directory. +To work against real content instead, copy an environment's database to your machine and run +Umbraco against it — see [scripts/db/README.md](../scripts/db/README.md). That writes +`appsettings.Local.json`, which is loaded automatically in Development and takes precedence +over the SQLite default. + ## Elasticsearch (Search) Search is powered by Elasticsearch. For local development, run Elasticsearch in Docker: