Use lets-encrypt for certs

config/complete/packit.yml

@@ -87,24 +87,28 @@ orderly-runner:
 ## network
 proxy:
   enabled: true
-  ssl:
-    ## This section describes how to get the certificate in.  We
-    ## support two sources:
-    ##
-    ## 1. self signed certificates - just leave this section blank
-    ##
-    ## 2. certificates from strings - include the strings directly in
-    ##    the keys here, or more likely use a VAULT:<path>:<key>
-    ##    string to extract them from the vault.
-    certificate: "VAULT:secret/cert:value"
-    key: "VAULT:secret/key:value"
   hostname: localhost
   port_http: 80
   port_https: 443
   image:
     name: packit-proxy
     tag: main
 
+## Standard configuration for using LetsEncrypt certs with acme-buddy.
+## If this section is not included, the proxy will create
+## a self-signed certificate.
+acme_buddy:
+  image:
+    repo: ghcr.io/reside-ic
+    name: acme-buddy
+    tag: main
+  dns_provider: hdb
+  email: [email protected]
+  env:
+    HDB_ACME_USERNAME: VAULT:secret/certbot-hdb/credentials:username
+    HDB_ACME_PASSWORD: VAULT:secret/certbot-hdb/credentials:password
+  port: 2112
+
 vault:
   ## Address of the vault server.  This should be a string if it is
   ## present.

config/self-signed/packit.yml

@@ -0,0 +1,126 @@
+## The name of the docker network that containers will be attached to.
+## If you want to proxy Packit to the host, you will need to
+## arrange a proxy on this network, or use dev_mode in the web section
+## below.
+## Prefix for container names; we'll use {container_prefix}-(container_name)
+container_prefix: packit
+
+## Set this flag to true to prevent use of --volumes in the cli to remove
+## volumes on stop
+protect_data: false
+
+## Docker org for images
+repo: ghcr.io/mrc-ide
+
+## The name of the docker network that containers will be attached to.
+## If you want to proxy Packit to the host, you will need to
+## arrange a proxy on this network
+network: packit-network
+
+## Names of the docker volumes to use:
+##
+## outpack: stores the outpack metadata
+## proxy_logs: stores logs from the reverse proxy (only used if proxy is given)
+## (More volumes are anticipated as the tool develops)
+volumes:
+  outpack: outpack_volume
+  proxy_logs: packit_proxy_logs
+  packit_db: packit_db
+  packit_db_backup: packit_db_backup
+  orderly_library: orderly_library
+  orderly_logs: orderly_logs
+
+outpack:
+  server:
+    name: outpack_server
+    tag: main
+  migrate:
+    name: outpack.orderly
+    tag: main
+
+packit:
+  base_url: https://localhost
+  api:
+    name: packit-api
+    tag: main
+  app:
+    name: packit
+    tag: main
+  db:
+    name: packit-db
+    tag: main
+    user: VAULT:secret/db/user:value
+    password: VAULT:secret/db/password:value
+  auth:
+    enabled: true
+    auth_method: github
+    expiry_days: 1
+    github_api_org: mrc-ide
+    github_api_team: packit
+    # Details of your Github OAuth app, which should be kept in the vault. The app's Authorization callback url must
+    # have the same root as the packit_api_root specified below, and should be of the form
+    # {PACKIT_API_ROOT}/login/oauth2/code/github
+    github_client:
+      id: VAULT:secret/auth/githubclient/id:value
+      secret: VAULT:secret/auth/githubclient/secret:value
+    jwt:
+      # Secret used to generate JWT tokens - this can be any string, the secret at this key in the vault is a random
+      # 32 char string, and is probably fine to re-use
+      secret: VAULT:secret/auth/jwt/secret:value
+    oauth2:
+      redirect:
+        # Root url which OAuth2 app will use to redirect back to packit api - must match OAuth2 app's registered url
+        packit_api_root: "https://packit/api"
+        url: "https://packit/redirect"  # Url for redirecting back to the front end after successful authentication
+  cors_allowed_origins: "https://packit.example.com"
+
+orderly-runner:
+  image:
+    name: orderly.runner
+    tag: main
+  git:
+    url: https://github.com/reside-ic/orderly2-example.git
+  workers: 1
+  env:
+    FOO: bar
+
+## If running a proxy directly, fill this section in.  Otherwise you
+## are responsible for proxying the application out of the docker
+## network
+proxy:
+  enabled: true
+  hostname: localhost
+  port_http: 80
+  port_https: 443
+  image:
+    name: packit-proxy
+    tag: main
+
+## Standard configuration for using LetsEncrypt certs with acme-buddy.
+## If this section is not included, the proxy will create
+## a self-signed certificate.
+acme_buddy:
+  image:
+    repo: ghcr.io/reside-ic
+    name: acme-buddy
+    tag: main
+  email: [email protected]
+  env:
+    ACME_BUDDY_SELF_SIGNED: 1
+  port: 2112
+
+vault:
+  ## Address of the vault server.  This should be a string if it is
+  ## present.
+  addr: ~
+  auth:
+    ## Authentication type - must be either "token" or the name of a
+    ## supported authentication method.  These seem to be poorly
+    ## documented in the hvac, but include "github" for github
+    ## authentication.
+    ##
+    ## On a vault client object, see auth.implemented_class_names for
+    ## a list, which is currently
+    ##
+    ##     azure, github, gcp, kubernetes, ldap, mfa, okta
+    method: token

pyproject.toml

@@ -44,6 +44,8 @@ path = "src/packit_deploy/__about__.py"
 [tool.hatch.envs.default]
 dependencies = [
     "coverage[toml]>=6.5",
+    "cryptography",
+    "docker",
     "pytest",
     "pytest-mock",
     "tenacity",

src/packit_deploy/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2023-present Alex Hill <[email protected]>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.1.3"
+__version__ = "0.1.4"

src/packit_deploy/cli.py

@@ -25,7 +25,7 @@ def cli_configure(name):
                 f"This packit instance is already configured as '{prev}', "
                 f"but you are trying to reconfigure it as '{name}'. "
                 "If you really want to do do this, then delete the file "
-                "'{IDENTITY_FILE}' from this directory and try again"
+                f"'{IDENTITY_FILE}' from this directory and try again"
             )
             raise Exception(msg)
         else:

src/packit_deploy/config.py

@@ -195,11 +195,14 @@ def __init__(self, path, extra=None, options=None) -> None:
             self.proxy_hostname = config.config_string(dat, ["proxy", "hostname"])
             self.proxy_port_http = config.config_integer(dat, ["proxy", "port_http"])
             self.proxy_port_https = config.config_integer(dat, ["proxy", "port_https"])
-            ssl = config.config_dict(dat, ["proxy", "ssl"], True)
-            self.proxy_ssl_self_signed = ssl is None
-            if not self.proxy_ssl_self_signed:
-                self.proxy_ssl_certificate = config.config_string(dat, ["proxy", "ssl", "certificate"], True)
-                self.proxy_ssl_key = config.config_string(dat, ["proxy", "ssl", "key"], True)
+
+            acme_key = "acme_buddy"
+            self.use_acme = acme_key in dat
+            if self.use_acme:
+                self.acme_config = config.config_acme(dat, acme_key)
+                self.containers["acme-buddy"] = "acme-buddy"
+                self.images["acme-buddy"] = self.acme_config.ref
+                self.volumes["packit-tls"] = "packit-tls"
 
             self.proxy_name = config.config_string(dat, ["proxy", "image", "name"])
             self.proxy_tag = config.config_string(dat, ["proxy", "image", "tag"])

src/packit_deploy/packit_constellation.py

@@ -2,7 +2,7 @@
 
 import constellation
 import docker
-from constellation import docker_util, vault
+from constellation import acme, docker_util, vault
 
 from packit_deploy.config import PackitConfig
 from packit_deploy.docker_helpers import DockerClient
@@ -13,7 +13,8 @@ def __init__(self, cfg: PackitConfig):
         # resolve secrets early so we can set these env vars from vault values
         if cfg.vault and cfg.vault.url:
             vault.resolve_secrets(cfg, cfg.vault.client())
-
+            if cfg.use_acme:  # pragma: no cover
+                vault.resolve_secrets(cfg.acme_config, cfg.vault.client())
         outpack = outpack_server_container(cfg)
         packit_db = packit_db_container(cfg)
         packit_api = packit_api_container(cfg)
@@ -24,6 +25,15 @@ def __init__(self, cfg: PackitConfig):
         if cfg.proxy_enabled:
             proxy = proxy_container(cfg, packit_api, packit)
             containers.append(proxy)
+            if cfg.use_acme:
+                acme_container = acme.acme_buddy_container(
+                    cfg.acme_config,
+                    "acme-buddy",
+                    proxy.name_external(cfg.container_prefix),
+                    "packit-tls",
+                    cfg.proxy_hostname,
+                )
+                containers.append(acme_container)
 
         if cfg.orderly_runner_enabled:
             containers.append(redis_container(cfg))
@@ -222,6 +232,8 @@ def proxy_container(cfg: PackitConfig, packit_api=None, packit=None):
     packit_addr = packit.name_external(cfg.container_prefix)
     proxy_args = [cfg.proxy_hostname, str(cfg.proxy_port_http), str(cfg.proxy_port_https), packit_api_addr, packit_addr]
     proxy_mounts = [constellation.ConstellationVolumeMount("proxy_logs", "/var/log/nginx")]
+    if cfg.use_acme:
+        proxy_mounts += [constellation.ConstellationVolumeMount("packit-tls", "/run/proxy")]
     proxy_ports = [cfg.proxy_port_http, cfg.proxy_port_https]
     proxy = constellation.ConstellationContainer(
         proxy_name, cfg.proxy_ref, ports=proxy_ports, args=proxy_args, mounts=proxy_mounts, configure=proxy_configure
@@ -231,13 +243,9 @@ def proxy_container(cfg: PackitConfig, packit_api=None, packit=None):
 
 def proxy_configure(container, cfg: PackitConfig):
     print("[proxy] Configuring proxy container")
-    if cfg.proxy_ssl_self_signed:
+    if not cfg.use_acme:
         print("[proxy] Generating self-signed certificates for proxy")
         docker_util.exec_safely(container, ["self-signed-certificate", "/run/proxy"])
-    else:
-        print("[proxy] Copying ssl certificate and key into proxy")
-        docker_util.string_into_container(cfg.proxy_ssl_certificate, container, "/run/proxy/certificate.pem")
-        docker_util.string_into_container(cfg.proxy_ssl_key, container, "/run/proxy/key.pem")
 
 
 def redis_container(cfg: PackitConfig):

tests/test_config.py

@@ -40,18 +40,19 @@ def test_config_proxy_disabled():
 def test_config_proxy():
     cfg = PackitConfig("config/novault")
     assert cfg.proxy_enabled
-    assert cfg.proxy_ssl_self_signed
+    assert not cfg.use_acme
     assert "proxy" in cfg.containers
     assert str(cfg.images["proxy"]) == "ghcr.io/mrc-ide/packit-proxy:main"
     assert cfg.proxy_hostname == "localhost"
     assert cfg.proxy_port_http == 80
     assert cfg.proxy_port_https == 443
-
     cfg = PackitConfig("config/complete")
     assert cfg.proxy_enabled
-    assert not cfg.proxy_ssl_self_signed
-    assert cfg.proxy_ssl_certificate == "VAULT:secret/cert:value"
-    assert cfg.proxy_ssl_key == "VAULT:secret/key:value"
+    assert cfg.use_acme
+    assert str(cfg.images["acme-buddy"]) == "ghcr.io/reside-ic/acme-buddy:main"
+    assert cfg.acme_config.port == 2112
+    assert "acme-buddy" in cfg.containers
+    assert "packit-tls" in cfg.volumes
 
 
 def test_github_auth():
@@ -152,7 +153,7 @@ def test_workers_can_be_enabled():
     assert cfg.orderly_runner_ref.tag == "main"
     assert cfg.orderly_runner_workers == 1
 
-    assert len(cfg.images) == 7
+    assert len(cfg.images) == 8
     assert str(cfg.images["orderly-runner"]) == "ghcr.io/mrc-ide/orderly.runner:main"
     assert str(cfg.images["redis"]) == "library/redis:8.0"
 

tests/test_integration.py

@@ -10,6 +10,8 @@
 import vault_dev
 from click.testing import CliRunner
 from constellation import docker_util
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
 
 from packit_deploy import cli
 from packit_deploy.config import PackitConfig
@@ -84,7 +86,7 @@ def test_start_and_stop_proxy():
         while len(json.loads(res)) < 1 and retries < 5:
             res = http_get("http://localhost/api/packets")
             time.sleep(5)
-            retries = retries + 1
+            retries += 1
         assert len(json.loads(res)) > 1
     finally:
         stop_packit(path)
@@ -106,15 +108,33 @@ def test_proxy_ssl_configured():
             url = f"http://localhost:{s.port}"
             options = {"vault": {"addr": url, "auth": {"args": {"token": s.token}}}}
             write_secrets_to_vault(s.client())
-
             cli.cli_start.callback(pull=False, name=path, options=options)
+            client = docker.from_env()
+            container = client.containers.get("packit-acme-buddy")
+            env = container.attrs["Config"]["Env"]
+            env_dict = dict(e.split("=", 1) for e in env)
+            assert "hdb-us3r" in env_dict["HDB_ACME_USERNAME"]
+            assert "hdb-p@assword" in env_dict["HDB_ACME_PASSWORD"]
 
-            cfg = PackitConfig(path)
-            proxy = cfg.get_container("proxy")
-            cert = docker_util.string_from_container(proxy, "run/proxy/certificate.pem")
-            key = docker_util.string_from_container(proxy, "run/proxy/key.pem")
-            assert "c3rt" in cert
-            assert "s3cret" in key
+    finally:
+        stop_packit(path)
+
+
+def test_acme_buddy_writes_cert():
+    path = "config/self-signed"
+    try:
+        with vault_dev.Server() as s:
+            url = f"http://localhost:{s.port}"
+            options = {"vault": {"addr": url, "auth": {"args": {"token": s.token}}}}
+            write_secrets_to_vault(s.client())
+            cli.cli_start.callback(pull=False, name=path, options=options)
+            client = docker.from_env()
+            proxy = client.containers.get("packit-proxy")
+            cert_str = docker_util.string_from_container(proxy, "/run/proxy/certificate.pem")
+            cert = x509.load_pem_x509_certificate(cert_str.encode(), default_backend())
+            assert cert.subject == cert.issuer
+            pubkey = cert.public_key()
+            pubkey.verify(cert.signature, cert.tbs_certificate_bytes)
 
     finally:
         stop_packit(path)
@@ -310,8 +330,7 @@ def stop_packit(path):
 
 
 def write_secrets_to_vault(cl):
-    cl.write("secret/cert", value="c3rt")
-    cl.write("secret/key", value="s3cret")
+    cl.write("secret/certbot-hdb/credentials", username="hdb-us3r", password="hdb-p@assword")
     cl.write("secret/db/user", value="us3r")
     cl.write("secret/db/password", value="p@ssword")
     cl.write("secret/ssh", public="publ1c", private="private")

tests/test_packit.py

@@ -25,7 +25,7 @@ def test_environment_with_private_runner_contains_url_and_key():
     env = packit_api_get_env(cfg)
     assert env["PACKIT_ORDERLY_RUNNER_URL"] == "http://packit-orderly-runner-api:8001"
     assert env["PACKIT_ORDERLY_RUNNER_REPOSITORY_URL"] == "[email protected]:reside-ic/orderly2-example-private.git"
-    assert type(env["PACKIT_ORDERLY_RUNNER_REPOSITORY_SSH_KEY"]) is str
+    assert isinstance(env["PACKIT_ORDERLY_RUNNER_REPOSITORY_SSH_KEY"], str)
     assert env["PACKIT_ORDERLY_RUNNER_LOCATION_URL"] == "http://packit-outpack-server:8000"