diff --git a/config/complete/packit.yml b/config/complete/packit.yml index 2b29670..dd1f074 100644 --- a/config/complete/packit.yml +++ b/config/complete/packit.yml @@ -87,17 +87,6 @@ orderly-runner: ## network proxy: enabled: true - ssl: - ## This section describes how to get the certificate in. We - ## support two sources: - ## - ## 1. self signed certificates - just leave this section blank - ## - ## 2. certificates from strings - include the strings directly in - ## the keys here, or more likely use a VAULT:: - ## string to extract them from the vault. - certificate: "VAULT:secret/cert:value" - key: "VAULT:secret/key:value" hostname: localhost port_http: 80 port_https: 443 @@ -105,6 +94,21 @@ proxy: name: packit-proxy tag: main +## Standard configuration for using LetsEncrypt certs with acme-buddy. +## If this section is not included, the proxy will create +## a self-signed certificate. +acme_buddy: + image: + repo: ghcr.io/reside-ic + name: acme-buddy + tag: main + dns_provider: hdb + email: test@reside.com + env: + HDB_ACME_USERNAME: VAULT:secret/certbot-hdb/credentials:username + HDB_ACME_PASSWORD: VAULT:secret/certbot-hdb/credentials:password + port: 2112 + vault: ## Address of the vault server. This should be a string if it is ## present. diff --git a/config/self-signed/packit.yml b/config/self-signed/packit.yml new file mode 100644 index 0000000..7cb3889 --- /dev/null +++ b/config/self-signed/packit.yml @@ -0,0 +1,126 @@ +## The name of the docker network that containers will be attached to. +## If you want to proxy Packit to the host, you will need to +## arrange a proxy on this network, or use dev_mode in the web section +## below. +## Prefix for container names; we'll use {container_prefix}-(container_name) +container_prefix: packit + +## Set this flag to true to prevent use of --volumes in the cli to remove +## volumes on stop +protect_data: false + +## Docker org for images +repo: ghcr.io/mrc-ide + +## The name of the docker network that containers will be attached to. +## If you want to proxy Packit to the host, you will need to +## arrange a proxy on this network +network: packit-network + +## Names of the docker volumes to use: +## +## outpack: stores the outpack metadata +## proxy_logs: stores logs from the reverse proxy (only used if proxy is given) +## (More volumes are anticipated as the tool develops) +volumes: + outpack: outpack_volume + proxy_logs: packit_proxy_logs + packit_db: packit_db + packit_db_backup: packit_db_backup + orderly_library: orderly_library + orderly_logs: orderly_logs + +outpack: + server: + name: outpack_server + tag: main + migrate: + name: outpack.orderly + tag: main + +packit: + base_url: https://localhost + api: + name: packit-api + tag: main + app: + name: packit + tag: main + db: + name: packit-db + tag: main + user: VAULT:secret/db/user:value + password: VAULT:secret/db/password:value + auth: + enabled: true + auth_method: github + expiry_days: 1 + github_api_org: mrc-ide + github_api_team: packit + # Details of your Github OAuth app, which should be kept in the vault. The app's Authorization callback url must + # have the same root as the packit_api_root specified below, and should be of the form + # {PACKIT_API_ROOT}/login/oauth2/code/github + github_client: + id: VAULT:secret/auth/githubclient/id:value + secret: VAULT:secret/auth/githubclient/secret:value + jwt: + # Secret used to generate JWT tokens - this can be any string, the secret at this key in the vault is a random + # 32 char string, and is probably fine to re-use + secret: VAULT:secret/auth/jwt/secret:value + oauth2: + redirect: + # Root url which OAuth2 app will use to redirect back to packit api - must match OAuth2 app's registered url + packit_api_root: "https://packit/api" + url: "https://packit/redirect" # Url for redirecting back to the front end after successful authentication + cors_allowed_origins: "https://packit.example.com" + +orderly-runner: + image: + name: orderly.runner + tag: main + git: + url: https://github.com/reside-ic/orderly2-example.git + workers: 1 + env: + FOO: bar + +## If running a proxy directly, fill this section in. Otherwise you +## are responsible for proxying the application out of the docker +## network +proxy: + enabled: true + hostname: localhost + port_http: 80 + port_https: 443 + image: + name: packit-proxy + tag: main + +## Standard configuration for using LetsEncrypt certs with acme-buddy. +## If this section is not included, the proxy will create +## a self-signed certificate. +acme_buddy: + image: + repo: ghcr.io/reside-ic + name: acme-buddy + tag: main + email: test@reside.com + env: + ACME_BUDDY_SELF_SIGNED: 1 + port: 2112 + +vault: + ## Address of the vault server. This should be a string if it is + ## present. + addr: ~ + auth: + ## Authentication type - must be either "token" or the name of a + ## supported authentication method. These seem to be poorly + ## documented in the hvac, but include "github" for github + ## authentication. + ## + ## On a vault client object, see auth.implemented_class_names for + ## a list, which is currently + ## + ## azure, github, gcp, kubernetes, ldap, mfa, okta + method: token diff --git a/pyproject.toml b/pyproject.toml index 68bb23c..0e8f3ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,8 @@ path = "src/packit_deploy/__about__.py" [tool.hatch.envs.default] dependencies = [ "coverage[toml]>=6.5", + "cryptography", + "docker", "pytest", "pytest-mock", "tenacity", diff --git a/src/packit_deploy/__about__.py b/src/packit_deploy/__about__.py index 6c65da4..013cd6b 100644 --- a/src/packit_deploy/__about__.py +++ b/src/packit_deploy/__about__.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2023-present Alex Hill # # SPDX-License-Identifier: MIT -__version__ = "0.1.3" +__version__ = "0.1.4" diff --git a/src/packit_deploy/cli.py b/src/packit_deploy/cli.py index 5ee1724..7579723 100644 --- a/src/packit_deploy/cli.py +++ b/src/packit_deploy/cli.py @@ -25,7 +25,7 @@ def cli_configure(name): f"This packit instance is already configured as '{prev}', " f"but you are trying to reconfigure it as '{name}'. " "If you really want to do do this, then delete the file " - "'{IDENTITY_FILE}' from this directory and try again" + f"'{IDENTITY_FILE}' from this directory and try again" ) raise Exception(msg) else: diff --git a/src/packit_deploy/config.py b/src/packit_deploy/config.py index 6eda32b..e84599d 100644 --- a/src/packit_deploy/config.py +++ b/src/packit_deploy/config.py @@ -195,11 +195,14 @@ def __init__(self, path, extra=None, options=None) -> None: self.proxy_hostname = config.config_string(dat, ["proxy", "hostname"]) self.proxy_port_http = config.config_integer(dat, ["proxy", "port_http"]) self.proxy_port_https = config.config_integer(dat, ["proxy", "port_https"]) - ssl = config.config_dict(dat, ["proxy", "ssl"], True) - self.proxy_ssl_self_signed = ssl is None - if not self.proxy_ssl_self_signed: - self.proxy_ssl_certificate = config.config_string(dat, ["proxy", "ssl", "certificate"], True) - self.proxy_ssl_key = config.config_string(dat, ["proxy", "ssl", "key"], True) + + acme_key = "acme_buddy" + self.use_acme = acme_key in dat + if self.use_acme: + self.acme_config = config.config_acme(dat, acme_key) + self.containers["acme-buddy"] = "acme-buddy" + self.images["acme-buddy"] = self.acme_config.ref + self.volumes["packit-tls"] = "packit-tls" self.proxy_name = config.config_string(dat, ["proxy", "image", "name"]) self.proxy_tag = config.config_string(dat, ["proxy", "image", "tag"]) diff --git a/src/packit_deploy/packit_constellation.py b/src/packit_deploy/packit_constellation.py index 08549af..6c95f41 100644 --- a/src/packit_deploy/packit_constellation.py +++ b/src/packit_deploy/packit_constellation.py @@ -2,7 +2,7 @@ import constellation import docker -from constellation import docker_util, vault +from constellation import acme, docker_util, vault from packit_deploy.config import PackitConfig from packit_deploy.docker_helpers import DockerClient @@ -13,7 +13,8 @@ def __init__(self, cfg: PackitConfig): # resolve secrets early so we can set these env vars from vault values if cfg.vault and cfg.vault.url: vault.resolve_secrets(cfg, cfg.vault.client()) - + if cfg.use_acme: # pragma: no cover + vault.resolve_secrets(cfg.acme_config, cfg.vault.client()) outpack = outpack_server_container(cfg) packit_db = packit_db_container(cfg) packit_api = packit_api_container(cfg) @@ -24,6 +25,15 @@ def __init__(self, cfg: PackitConfig): if cfg.proxy_enabled: proxy = proxy_container(cfg, packit_api, packit) containers.append(proxy) + if cfg.use_acme: + acme_container = acme.acme_buddy_container( + cfg.acme_config, + "acme-buddy", + proxy.name_external(cfg.container_prefix), + "packit-tls", + cfg.proxy_hostname, + ) + containers.append(acme_container) if cfg.orderly_runner_enabled: containers.append(redis_container(cfg)) @@ -222,6 +232,8 @@ def proxy_container(cfg: PackitConfig, packit_api=None, packit=None): packit_addr = packit.name_external(cfg.container_prefix) proxy_args = [cfg.proxy_hostname, str(cfg.proxy_port_http), str(cfg.proxy_port_https), packit_api_addr, packit_addr] proxy_mounts = [constellation.ConstellationVolumeMount("proxy_logs", "/var/log/nginx")] + if cfg.use_acme: + proxy_mounts += [constellation.ConstellationVolumeMount("packit-tls", "/run/proxy")] proxy_ports = [cfg.proxy_port_http, cfg.proxy_port_https] proxy = constellation.ConstellationContainer( proxy_name, cfg.proxy_ref, ports=proxy_ports, args=proxy_args, mounts=proxy_mounts, configure=proxy_configure @@ -231,13 +243,9 @@ def proxy_container(cfg: PackitConfig, packit_api=None, packit=None): def proxy_configure(container, cfg: PackitConfig): print("[proxy] Configuring proxy container") - if cfg.proxy_ssl_self_signed: + if not cfg.use_acme: print("[proxy] Generating self-signed certificates for proxy") docker_util.exec_safely(container, ["self-signed-certificate", "/run/proxy"]) - else: - print("[proxy] Copying ssl certificate and key into proxy") - docker_util.string_into_container(cfg.proxy_ssl_certificate, container, "/run/proxy/certificate.pem") - docker_util.string_into_container(cfg.proxy_ssl_key, container, "/run/proxy/key.pem") def redis_container(cfg: PackitConfig): diff --git a/tests/test_config.py b/tests/test_config.py index 1eb1220..940c90c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -40,18 +40,19 @@ def test_config_proxy_disabled(): def test_config_proxy(): cfg = PackitConfig("config/novault") assert cfg.proxy_enabled - assert cfg.proxy_ssl_self_signed + assert not cfg.use_acme assert "proxy" in cfg.containers assert str(cfg.images["proxy"]) == "ghcr.io/mrc-ide/packit-proxy:main" assert cfg.proxy_hostname == "localhost" assert cfg.proxy_port_http == 80 assert cfg.proxy_port_https == 443 - cfg = PackitConfig("config/complete") assert cfg.proxy_enabled - assert not cfg.proxy_ssl_self_signed - assert cfg.proxy_ssl_certificate == "VAULT:secret/cert:value" - assert cfg.proxy_ssl_key == "VAULT:secret/key:value" + assert cfg.use_acme + assert str(cfg.images["acme-buddy"]) == "ghcr.io/reside-ic/acme-buddy:main" + assert cfg.acme_config.port == 2112 + assert "acme-buddy" in cfg.containers + assert "packit-tls" in cfg.volumes def test_github_auth(): @@ -152,7 +153,7 @@ def test_workers_can_be_enabled(): assert cfg.orderly_runner_ref.tag == "main" assert cfg.orderly_runner_workers == 1 - assert len(cfg.images) == 7 + assert len(cfg.images) == 8 assert str(cfg.images["orderly-runner"]) == "ghcr.io/mrc-ide/orderly.runner:main" assert str(cfg.images["redis"]) == "library/redis:8.0" diff --git a/tests/test_integration.py b/tests/test_integration.py index c4553fb..e8b334f 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -10,6 +10,8 @@ import vault_dev from click.testing import CliRunner from constellation import docker_util +from cryptography import x509 +from cryptography.hazmat.backends import default_backend from packit_deploy import cli from packit_deploy.config import PackitConfig @@ -84,7 +86,7 @@ def test_start_and_stop_proxy(): while len(json.loads(res)) < 1 and retries < 5: res = http_get("http://localhost/api/packets") time.sleep(5) - retries = retries + 1 + retries += 1 assert len(json.loads(res)) > 1 finally: stop_packit(path) @@ -106,15 +108,33 @@ def test_proxy_ssl_configured(): url = f"http://localhost:{s.port}" options = {"vault": {"addr": url, "auth": {"args": {"token": s.token}}}} write_secrets_to_vault(s.client()) - cli.cli_start.callback(pull=False, name=path, options=options) + client = docker.from_env() + container = client.containers.get("packit-acme-buddy") + env = container.attrs["Config"]["Env"] + env_dict = dict(e.split("=", 1) for e in env) + assert "hdb-us3r" in env_dict["HDB_ACME_USERNAME"] + assert "hdb-p@assword" in env_dict["HDB_ACME_PASSWORD"] - cfg = PackitConfig(path) - proxy = cfg.get_container("proxy") - cert = docker_util.string_from_container(proxy, "run/proxy/certificate.pem") - key = docker_util.string_from_container(proxy, "run/proxy/key.pem") - assert "c3rt" in cert - assert "s3cret" in key + finally: + stop_packit(path) + + +def test_acme_buddy_writes_cert(): + path = "config/self-signed" + try: + with vault_dev.Server() as s: + url = f"http://localhost:{s.port}" + options = {"vault": {"addr": url, "auth": {"args": {"token": s.token}}}} + write_secrets_to_vault(s.client()) + cli.cli_start.callback(pull=False, name=path, options=options) + client = docker.from_env() + proxy = client.containers.get("packit-proxy") + cert_str = docker_util.string_from_container(proxy, "/run/proxy/certificate.pem") + cert = x509.load_pem_x509_certificate(cert_str.encode(), default_backend()) + assert cert.subject == cert.issuer + pubkey = cert.public_key() + pubkey.verify(cert.signature, cert.tbs_certificate_bytes) finally: stop_packit(path) @@ -310,8 +330,7 @@ def stop_packit(path): def write_secrets_to_vault(cl): - cl.write("secret/cert", value="c3rt") - cl.write("secret/key", value="s3cret") + cl.write("secret/certbot-hdb/credentials", username="hdb-us3r", password="hdb-p@assword") cl.write("secret/db/user", value="us3r") cl.write("secret/db/password", value="p@ssword") cl.write("secret/ssh", public="publ1c", private="private") diff --git a/tests/test_packit.py b/tests/test_packit.py index 0cbfadb..25bff92 100644 --- a/tests/test_packit.py +++ b/tests/test_packit.py @@ -25,7 +25,7 @@ def test_environment_with_private_runner_contains_url_and_key(): env = packit_api_get_env(cfg) assert env["PACKIT_ORDERLY_RUNNER_URL"] == "http://packit-orderly-runner-api:8001" assert env["PACKIT_ORDERLY_RUNNER_REPOSITORY_URL"] == "git@github.com:reside-ic/orderly2-example-private.git" - assert type(env["PACKIT_ORDERLY_RUNNER_REPOSITORY_SSH_KEY"]) is str + assert isinstance(env["PACKIT_ORDERLY_RUNNER_REPOSITORY_SSH_KEY"], str) assert env["PACKIT_ORDERLY_RUNNER_LOCATION_URL"] == "http://packit-outpack-server:8000"