diff --git a/config/basicauth/packit.yml b/config/basicauth/packit.yml index bdbfae8..13c1ba8 100644 --- a/config/basicauth/packit.yml +++ b/config/basicauth/packit.yml @@ -62,5 +62,4 @@ proxy: port_http: 80 port_https: 443 image: - name: packit-proxy - tag: main + build: ../../proxy diff --git a/config/basicauthcustombrand/packit.yml b/config/basicauthcustombrand/packit.yml index 22997cc..f3bf2f4 100644 --- a/config/basicauthcustombrand/packit.yml +++ b/config/basicauthcustombrand/packit.yml @@ -64,8 +64,7 @@ proxy: port_http: 80 port_https: 443 image: - name: packit-proxy - tag: main + build: ../../proxy ## Branding configuration brand: diff --git a/config/complete/packit.yml b/config/complete/packit.yml index dd1f074..8b59302 100644 --- a/config/complete/packit.yml +++ b/config/complete/packit.yml @@ -91,8 +91,10 @@ proxy: port_http: 80 port_https: 443 image: - name: packit-proxy - tag: main + # Alternatively, and in most deployement scenarios, use the following: + # name: packit-proxy + # tag: main + build: ../../proxy ## Standard configuration for using LetsEncrypt certs with acme-buddy. ## If this section is not included, the proxy will create diff --git a/config/githubauth/packit.yml b/config/githubauth/packit.yml index 9f09c54..da67135 100644 --- a/config/githubauth/packit.yml +++ b/config/githubauth/packit.yml @@ -77,8 +77,7 @@ proxy: port_http: 80 port_https: 443 image: - name: packit-proxy - tag: main + build: ../../proxy vault: ## Address of the vault server. This should be a string if it is diff --git a/config/nodemo/packit.yml b/config/nodemo/packit.yml index 3713f80..965cdb3 100644 --- a/config/nodemo/packit.yml +++ b/config/nodemo/packit.yml @@ -55,5 +55,4 @@ proxy: port_http: 80 port_https: 443 image: - name: packit-proxy - tag: main + build: ../../proxy diff --git a/config/novault/packit.yml b/config/novault/packit.yml index cb49708..ef67e33 100644 --- a/config/novault/packit.yml +++ b/config/novault/packit.yml @@ -57,5 +57,4 @@ proxy: port_http: 80 port_https: 443 image: - name: packit-proxy - tag: main + build: ../../proxy diff --git a/config/runner-private/packit.yml b/config/runner-private/packit.yml index baf98cc..7c8d8fa 100644 --- a/config/runner-private/packit.yml +++ b/config/runner-private/packit.yml @@ -74,8 +74,7 @@ proxy: port_http: 80 port_https: 443 image: - name: packit-proxy - tag: main + build: ../../proxy vault: addr: https://vault.dide.ic.ac.uk:8200 diff --git a/config/runner/packit.yml b/config/runner/packit.yml index 09422cc..d4f2744 100644 --- a/config/runner/packit.yml +++ b/config/runner/packit.yml @@ -75,5 +75,4 @@ proxy: port_http: 80 port_https: 443 image: - name: packit-proxy - tag: main + build: ../../proxy diff --git a/proxy/Dockerfile b/proxy/Dockerfile index a1c215f..adf22ee 100644 --- a/proxy/Dockerfile +++ b/proxy/Dockerfile @@ -3,13 +3,9 @@ FROM nginx:1.29 # Only used for generating self-signed certificates RUN apt-get update && apt-get install -y openssl -# Clear out existing configuration -RUN rm /etc/nginx/conf.d/default.conf - VOLUME /var/log/nginx VOLUME /run/proxy -COPY nginx.conf /etc/nginx/nginx.conf.template COPY bin /usr/local/bin COPY ssl /usr/local/share/ssl diff --git a/proxy/README.md b/proxy/README.md index 52c459b..9bed77d 100644 --- a/proxy/README.md +++ b/proxy/README.md @@ -9,8 +9,8 @@ The configuration takes as a starting point [`montagu-proxy`](https://github.com ### Configuration -Before starting we need to know what we are proxying -(i.e., the name of the `packit` and `packit-api` containers on the docker network) and what the proxy will be seen as to the outside world (the hostname, and ports for http and https). The entrypoint takes these four values as arguments. +The proxy image does not embed an `nginx.conf` file. It is `packit-deploy`'s responsibility to generate one and inject into the container before starting it. +See the `src/packit_deploy/templates/nginx.conf.j2` file for the configuration template. ### SSL Certificates diff --git a/proxy/bin/packit-proxy b/proxy/bin/packit-proxy index 2c085f8..c0e7786 100755 --- a/proxy/bin/packit-proxy +++ b/proxy/bin/packit-proxy @@ -1,24 +1,6 @@ #!/usr/bin/env bash set -eu -if [ "$#" -eq 5 ]; then - export HTTP_HOST=$1 - export HTTP_PORT=$2 - export HTTPS_PORT=$3 - export PACKIT_API=$4 - export PACKIT=$5 -else - echo "Usage: HOSTNAME PORT_HTTP PORT_HTTPS PACKIT_API PACKIT" - echo "e.g. docker run ... montagu.vaccineimpact.org 80 443 packit_api packit" - exit 1 -fi - -echo "We will listen on ports $HTTP_PORT (http) and $HTTPS_PORT (https)" -echo "with hostname $HTTP_HOST, proxying Packit from $PACKIT" - -envsubst '$HTTP_HOST,$HTTP_PORT,$HTTPS_PORT, $PACKIT_API, $PACKIT' \ - < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf - # These paths must match the paths as used in the nginx.conf PATH_CONFIG=/run/proxy PATH_CERT="$PATH_CONFIG/certificate.pem" diff --git a/proxy/nginx.conf b/proxy/nginx.conf deleted file mode 100644 index 718e780..0000000 --- a/proxy/nginx.conf +++ /dev/null @@ -1,90 +0,0 @@ -user nginx; -worker_processes 1; - -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - - -events { - worker_connections 1024; -} - - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - #tcp_nopush on; - - keepalive_timeout 65; - - # Main server configuration. See below for redirects. - server { - listen ${HTTPS_PORT} ssl; - server_name localhost ${HTTP_HOST}; - - # Enable HTTP Strict Transport Security (HSTS) - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; - - # https://scotthelme.co.uk/content-security-policy-an-introduction/ - # https://content-security-policy.com/examples/nginx/ - # TODO: this needs something fixed in OW: mrc-2489 - # add_header Content-Security-Policy "default-src 'self';" always; - # However, this one does work: - add_header Content-Security-Policy "frame-ancestors 'self' *.imperial.ac.uk *.ic.ac.uk" always; - - # https://scotthelme.co.uk/hardening-your-http-response-headers/#x-frame-options - # https://geekflare.com/add-x-frame-options-nginx/ - add_header X-Frame-Options "SAMEORIGIN"; - - # https://scotthelme.co.uk/hardening-your-http-response-headers/#x-content-type-options - add_header X-Content-Type-Options "nosniff" always; - - # https://scotthelme.co.uk/a-new-security-header-referrer-policy/ - add_header Referrer-Policy 'origin' always; - - # https://scotthelme.co.uk/goodbye-feature-policy-and-hello-permissions-policy/ - # Actual values adopted from securityheaders.com :) - add_header Permissions-Policy "accelerometer=(), camera=(), geolocation=(), gyroscope=(), magnetometer=(), microphone=(), payment=(), usb=(), interest-cohort=()" always; - - # Certificate - ssl_certificate /run/proxy/certificate.pem; - ssl_certificate_key /run/proxy/key.pem; - - # SSL settings as recommended by this generator - # https://ssl-config.mozilla.org/ - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; - ssl_prefer_server_ciphers off; - ssl_session_cache shared:SSL:10m; - ssl_dhparam /run/proxy/dhparam.pem; - - root /usr/share/nginx/html; - - location /api/ { - proxy_pass http://${PACKIT_API}/; - } - - location / { - proxy_pass http://${PACKIT}/; - } - } - - # Redirect all http requests to the SSL endpoint and the correct domain name - server { - listen ${HTTP_PORT} default_server; - listen [::]:${HTTP_PORT} default_server; - server_name _; - - location / { - return 301 https://${HTTP_HOST}$request_uri; - } - } -} diff --git a/pyproject.toml b/pyproject.toml index 3b616d6..88b5291 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,8 +26,9 @@ classifiers = [ ] dependencies = [ "click", - "constellation~=1.4.6", + "constellation~=1.5.0", "docker", + "jinja2", ] [project.urls] diff --git a/src/packit_deploy/config.py b/src/packit_deploy/config.py index 239c710..6137312 100644 --- a/src/packit_deploy/config.py +++ b/src/packit_deploy/config.py @@ -1,9 +1,9 @@ from dataclasses import dataclass from pathlib import Path -from typing import Optional +from typing import ClassVar, Optional, Union import constellation -from constellation import config +from constellation import BuildSpec, config from constellation.acme import AcmeBuddyConfig from constellation.vault import VaultConfig @@ -17,7 +17,7 @@ def config_path(dat, key: list[str], *, root: str, is_optional: bool = False) -> """ value = config.config_string(dat, key, is_optional=is_optional) if value is not None: - return Path(root, value).absolute() + return Path(root, value).resolve() else: return None @@ -34,6 +34,16 @@ def config_ref(dat, key: list[str], *, repo: str) -> constellation.ImageReferenc return constellation.ImageReference(repo, name, tag) +def config_buildable(dat, key: list[str], *, repo: str, root: str) -> Union["BuildSpec", constellation.ImageReference]: + build = config_path(dat, [*key, "build"], is_optional=True, root=root) + if build is not None: + return BuildSpec(path=str(build)) + else: + name = config.config_string(dat, [*key, "name"]) + tag = config.config_string(dat, [*key, "tag"]) + return constellation.ImageReference(repo, name, tag) + + @dataclass class Theme: accent: str @@ -144,8 +154,28 @@ def from_data(cls, dat, key: list[str]) -> "PackitAuth": return PackitAuth(method=method, github=github, expiry_days=expiry_days, jwt_secret=jwt_secret) +@dataclass +class ContainerConfig: + """ + A generic config class for containers that don't need allow any + configuration options other than an image reference. + """ + + container_name: str + image: constellation.ImageReference + + @classmethod + def from_data(cls, dat, key: list[str], *, repo: str, container_name: str) -> "ContainerConfig": + return ContainerConfig( + container_name=container_name, + image=config_ref(dat, key, repo=repo), + ) + + @dataclass class PackitAPI: + container_name: ClassVar[str] = "packit-api" + image: constellation.ImageReference management_port: int base_url: str @@ -187,6 +217,7 @@ def from_data(cls, dat, key: list[str], *, repo: str) -> "PackitAPI": @dataclass class PackitDB: + container_name: ClassVar[str] = "packit-db" image: constellation.ImageReference user: str password: str @@ -198,9 +229,16 @@ def from_data(cls, dat, key: list[str], *, repo: str) -> "PackitDB": password = config.config_string(dat, [*key, "password"]) return PackitDB(image=image, user=user, password=password) + @property + def jdbc_url(self): + return f"jdbc:postgresql://{self.container_name}:5432/packit?stringtype=unspecified" + @dataclass class OrderlyRunner: + container_name_api: ClassVar[str] = "orderly-runner-api" + container_name_worker: ClassVar[str] = "orderly-runner-worker" + image: constellation.ImageReference workers: int env: dict[str, str] @@ -212,6 +250,10 @@ def from_data(cls, dat, key: list[str], *, repo: str) -> "OrderlyRunner": env = config.config_dict(dat, ["orderly-runner", "env"], is_optional=True, default={}) return OrderlyRunner(image=image, workers=workers, env=env) + @property + def api_url(self) -> str: + return f"http://{self.container_name_api}:8001" + @dataclass class SSL: @@ -221,14 +263,16 @@ class SSL: @dataclass class Proxy: - image: constellation.ImageReference + container_name: ClassVar[str] = "proxy" + + image: Union[BuildSpec, constellation.ImageReference] hostname: str port_http: int port_https: int @classmethod - def from_data(cls, dat, key: list[str], *, repo: str) -> "Proxy": - image = config_ref(dat, [*key, "image"], repo=repo) + def from_data(cls, dat, key: list[str], *, repo: str, root: str) -> "Proxy": + image = config_buildable(dat, [*key, "image"], repo=repo, root=root) hostname = config.config_string(dat, [*key, "hostname"]) port_http = config.config_integer(dat, [*key, "port_http"]) port_https = config.config_integer(dat, [*key, "port_https"]) @@ -240,7 +284,6 @@ class PackitConfig: app_html_root = "/usr/share/nginx/html" # from Packit app Dockerfile volumes: dict[str, str] - containers: dict[str, str] container_prefix: str network: str @@ -248,8 +291,8 @@ class PackitConfig: repo: str vault: VaultConfig - outpack_ref: constellation.ImageReference - packit_ref: constellation.ImageReference + outpack: ContainerConfig + packit_app: ContainerConfig packit_api: PackitAPI packit_db: PackitDB orderly_runner: Optional[OrderlyRunner] @@ -273,56 +316,45 @@ def __init__(self, path, extra=None, options=None) -> None: self.container_prefix = config.config_string(dat, ["container_prefix"]) self.repo = config.config_string(dat, ["repo"]) - self.outpack_ref = config_ref(dat, ["outpack", "server"], repo=self.repo) - self.packit_ref = config_ref(dat, ["packit", "app"], repo=self.repo) - self.packit_db = PackitDB.from_data(dat, ["packit", "db"], repo=self.repo) + self.outpack_server = ContainerConfig.from_data( + dat, ["outpack", "server"], repo=self.repo, container_name="outpack-server" + ) + self.packit_app = ContainerConfig.from_data(dat, ["packit", "app"], repo=self.repo, container_name="packit") self.packit_api = PackitAPI.from_data(dat, ["packit"], repo=self.repo) - - self.containers = { - "outpack-server": "outpack-server", - "packit-db": "packit-db", - "packit-api": "packit-api", - "packit": "packit", - } + self.packit_db = PackitDB.from_data(dat, ["packit", "db"], repo=self.repo) if "orderly-runner" in dat: self.orderly_runner = OrderlyRunner.from_data(dat, ["orderly-runner"], repo=self.repo) self.volumes["orderly_library"] = config.config_string(dat, ["volumes", "orderly_library"]) self.volumes["orderly_logs"] = config.config_string(dat, ["volumes", "orderly_logs"]) - self.containers["redis"] = "redis" - self.containers["orderly-runner-api"] = "orderly-runner-api" - self.containers["orderly-runner-worker"] = "orderly-runner-worker" else: self.orderly_runner = None self.brand = Branding.from_data(dat, root=path) if "proxy" in dat and config.config_boolean(dat, ["proxy", "enabled"]): - self.proxy = Proxy.from_data(dat, ["proxy"], repo=self.repo) - self.containers["proxy"] = "proxy" + self.proxy = Proxy.from_data(dat, ["proxy"], repo=self.repo, root=path) self.volumes["proxy_logs"] = config.config_string(dat, ["volumes", "proxy_logs"]) else: self.proxy = None if "acme_buddy" in dat: self.acme_config = config.config_acme(dat, "acme_buddy") - self.containers["acme-buddy"] = "acme-buddy" self.volumes["packit-tls"] = "packit-tls" else: self.acme_config = None @property def outpack_server_url(self) -> str: - return f"http://{self.container_prefix}-{self.containers['outpack-server']}:8000" - - @property - def orderly_runner_api_url(self) -> str: - return f"http://{self.container_prefix}-orderly-runner-api:8001" + return f"http://{self.outpack_server.container_name}:8000" @property def redis_url(self) -> str: return "redis://redis:6379" @property - def redis_image(self) -> constellation.ImageReference: - return constellation.ImageReference("library", "redis", "8.0") + def redis(self) -> ContainerConfig: + return ContainerConfig( + container_name="redis", + image=constellation.ImageReference("library", "redis", "8.0"), + ) diff --git a/src/packit_deploy/packit_constellation.py b/src/packit_deploy/packit_constellation.py index 8da69df..6a36fa6 100644 --- a/src/packit_deploy/packit_constellation.py +++ b/src/packit_deploy/packit_constellation.py @@ -2,12 +2,19 @@ import constellation import docker +import jinja2 from constellation import ConstellationContainer, acme, docker_util, vault from packit_deploy import config from packit_deploy.config import PackitConfig from packit_deploy.docker_helpers import DockerClient +JINJA_ENVIRONMENT = jinja2.Environment( + loader=jinja2.PackageLoader("packit_deploy"), + undefined=jinja2.StrictUndefined, + autoescape=False, # noqa: S701, we only template from config values, not user inputs +) + class PackitConstellation: def __init__(self, cfg: PackitConfig): @@ -25,7 +32,7 @@ def __init__(self, cfg: PackitConfig): containers = [outpack, packit_db, packit_api, packit] if cfg.proxy is not None: - proxy = proxy_container(cfg, cfg.proxy, packit_api, packit) + proxy = proxy_container(cfg.proxy, cfg) containers.append(proxy) if cfg.acme_config is not None: acme_container = acme.acme_buddy_container( @@ -63,11 +70,11 @@ def outpack_is_initialised(container): def outpack_server_container(cfg: PackitConfig) -> ConstellationContainer: - name = cfg.containers["outpack-server"] + name = cfg.outpack_server.container_name mounts = [constellation.ConstellationVolumeMount("outpack", "/outpack")] return ConstellationContainer( name, - cfg.outpack_ref, + cfg.outpack_server.image, mounts=mounts, configure=outpack_server_configure, ) @@ -76,7 +83,7 @@ def outpack_server_container(cfg: PackitConfig) -> ConstellationContainer: def outpack_server_configure(container, cfg: PackitConfig): print("[outpack] Initialising outpack repository") if not outpack_is_initialised(container): - image = str(cfg.outpack_ref) + image = str(cfg.outpack_server.image) mounts = [docker.types.Mount("/outpack", cfg.volumes["outpack"])] with DockerClient() as cl: @@ -85,7 +92,7 @@ def outpack_server_configure(container, cfg: PackitConfig): def packit_db_container(cfg: PackitConfig) -> ConstellationContainer: - name = cfg.containers["packit-db"] + name = cfg.packit_db.container_name mounts = [ constellation.ConstellationVolumeMount("packit_db", "/pgdata"), constellation.ConstellationVolumeMount("packit_db_backup", "/pgbackup"), @@ -104,7 +111,7 @@ def packit_db_configure(container, _cfg: PackitConfig): def packit_api_container(cfg: PackitConfig) -> ConstellationContainer: - name = cfg.containers["packit-api"] + name = cfg.packit_api.container_name return ConstellationContainer( name, cfg.packit_api.image, @@ -113,9 +120,8 @@ def packit_api_container(cfg: PackitConfig) -> ConstellationContainer: def packit_api_get_env(cfg: PackitConfig) -> dict[str, str]: - packit_db = cfg.containers["packit-db"] env: dict[str, str] = { - "PACKIT_DB_URL": f"jdbc:postgresql://{cfg.container_prefix}-{packit_db}:5432/packit?stringtype=unspecified", + "PACKIT_DB_URL": cfg.packit_db.jdbc_url, "PACKIT_DB_USER": cfg.packit_db.user, "PACKIT_DB_PASSWORD": cfg.packit_db.password, "PACKIT_OUTPACK_SERVER_URL": cfg.outpack_server_url, @@ -158,7 +164,11 @@ def packit_api_get_env(cfg: PackitConfig) -> dict[str, str]: ) if cfg.packit_api.runner_git_url is not None: - env["PACKIT_ORDERLY_RUNNER_URL"] = cfg.orderly_runner_api_url + if cfg.orderly_runner is None: + msg = "Runner is configured on the API but not available" + raise Exception(msg) + + env["PACKIT_ORDERLY_RUNNER_URL"] = cfg.orderly_runner.api_url env["PACKIT_ORDERLY_RUNNER_REPOSITORY_URL"] = cfg.packit_api.runner_git_url if cfg.packit_api.runner_git_ssh_key is not None: env["PACKIT_ORDERLY_RUNNER_REPOSITORY_SSH_KEY"] = cfg.packit_api.runner_git_ssh_key @@ -168,6 +178,7 @@ def packit_api_get_env(cfg: PackitConfig) -> dict[str, str]: def packit_container(cfg: PackitConfig): + name = cfg.packit_app.container_name mounts = [] if cfg.brand.logo is not None: @@ -181,8 +192,8 @@ def packit_container(cfg: PackitConfig): ) return ConstellationContainer( - cfg.containers["packit"], - cfg.packit_ref, + name, + cfg.packit_app.image, mounts=mounts, configure=packit_configure, ) @@ -238,31 +249,42 @@ def substitute_file_content(container, path, pattern, replacement, flags=0): docker_util.exec_safely(container, ["rm", backup]) -def proxy_container( - cfg: PackitConfig, - proxy: config.Proxy, - packit_api: ConstellationContainer, - packit: ConstellationContainer, -): - packit_api_addr = f"{packit_api.name_external(cfg.container_prefix)}:8080" - packit_addr = packit.name_external(cfg.container_prefix) - - name = cfg.containers["proxy"] - args = [proxy.hostname, str(proxy.port_http), str(proxy.port_https), packit_api_addr, packit_addr] +def proxy_container(proxy: config.Proxy, cfg: PackitConfig): + name = proxy.container_name mounts = [constellation.ConstellationVolumeMount("proxy_logs", "/var/log/nginx")] if cfg.acme_config is not None: mounts.append(constellation.ConstellationVolumeMount("packit-tls", "/run/proxy")) ports = [proxy.port_http, proxy.port_https] return ConstellationContainer( name, - proxy.image, + image=proxy.image, ports=ports, - args=args, mounts=mounts, + preconfigure=lambda container, cfg: proxy_preconfigure(container, cfg, proxy), configure=proxy_configure, ) +def proxy_nginx_conf(cfg: PackitConfig, proxy: config.Proxy): + packit_api_addr = f"{cfg.packit_api.container_name}:8080" + packit_app_addr = cfg.packit_app.container_name + + template = JINJA_ENVIRONMENT.get_template("nginx.conf.j2") + return template.render( + upstream_api=packit_api_addr, + upstream_app=packit_app_addr, + hostname=proxy.hostname, + port_http=proxy.port_http, + port_https=proxy.port_https, + ) + + +def proxy_preconfigure(container: ConstellationContainer, cfg: PackitConfig, proxy: config.Proxy): + print("[proxy] Preconfiguring proxy container") + nginx_conf = proxy_nginx_conf(cfg, proxy) + docker_util.string_into_container(nginx_conf, container, "/etc/nginx/conf.d/default.conf") + + def proxy_configure(container: ConstellationContainer, cfg: PackitConfig): print("[proxy] Configuring proxy container") if cfg.acme_config is None: @@ -271,8 +293,8 @@ def proxy_configure(container: ConstellationContainer, cfg: PackitConfig): def redis_container(cfg: PackitConfig) -> ConstellationContainer: - name = cfg.containers["redis"] - image = str(cfg.redis_image) + name = cfg.redis.container_name + image = str(cfg.redis.image) return ConstellationContainer( name, image, @@ -287,7 +309,7 @@ def redis_configure(container, _cfg: PackitConfig): def orderly_runner_api_container(cfg: PackitConfig, runner: config.OrderlyRunner): - name = cfg.containers["orderly-runner-api"] + name = runner.container_name_api image = str(runner.image) env = orderly_runner_env(cfg, runner) entrypoint = "/usr/local/bin/orderly.runner.server" @@ -307,7 +329,7 @@ def orderly_runner_api_container(cfg: PackitConfig, runner: config.OrderlyRunner def orderly_runner_worker_containers(cfg: PackitConfig, runner: config.OrderlyRunner): - name = cfg.containers["orderly-runner-worker"] + name = runner.container_name_worker image = str(runner.image) count = runner.workers env = orderly_runner_env(cfg, runner) diff --git a/src/packit_deploy/templates/nginx.conf.j2 b/src/packit_deploy/templates/nginx.conf.j2 new file mode 100644 index 0000000..65c86be --- /dev/null +++ b/src/packit_deploy/templates/nginx.conf.j2 @@ -0,0 +1,63 @@ +{# This file is used as a template by packit-deploy #} + +# Main server configuration. See below for redirects. +server { + listen {{ port_https }} ssl; + server_name {{ hostname }}; + + # Enable HTTP Strict Transport Security (HSTS) + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + + # https://scotthelme.co.uk/content-security-policy-an-introduction/ + # https://content-security-policy.com/examples/nginx/ + # TODO: this needs something fixed in OW: mrc-2489 + # add_header Content-Security-Policy "default-src 'self';" always; + # However, this one does work: + add_header Content-Security-Policy "frame-ancestors 'self' *.imperial.ac.uk *.ic.ac.uk" always; + + # https://scotthelme.co.uk/hardening-your-http-response-headers/#x-frame-options + # https://geekflare.com/add-x-frame-options-nginx/ + add_header X-Frame-Options "SAMEORIGIN"; + + # https://scotthelme.co.uk/hardening-your-http-response-headers/#x-content-type-options + add_header X-Content-Type-Options "nosniff" always; + + # https://scotthelme.co.uk/a-new-security-header-referrer-policy/ + add_header Referrer-Policy 'origin' always; + + # https://scotthelme.co.uk/goodbye-feature-policy-and-hello-permissions-policy/ + # Actual values adopted from securityheaders.com :) + add_header Permissions-Policy "accelerometer=(), camera=(), geolocation=(), gyroscope=(), magnetometer=(), microphone=(), payment=(), usb=(), interest-cohort=()" always; + + # Certificate + ssl_certificate /run/proxy/certificate.pem; + ssl_certificate_key /run/proxy/key.pem; + + # SSL settings as recommended by this generator + # https://ssl-config.mozilla.org/ + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + ssl_session_cache shared:SSL:10m; + ssl_dhparam /run/proxy/dhparam.pem; + + root /usr/share/nginx/html; + + location /api/ { + proxy_pass http://{{ upstream_api }}/; + } + + location / { + proxy_pass http://{{ upstream_app }}/; + } +} + +# Redirect all http requests to the SSL endpoint and the correct domain name +server { + listen {{ port_http }} default_server; + server_name _; + + location / { + return 301 https://$host$request_uri; + } +} diff --git a/tests/test_config.py b/tests/test_config.py index 4657326..1c6f930 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,8 @@ import os from pathlib import Path +from constellation import BuildSpec + from packit_deploy.config import Branding, PackitConfig, Theme packit_deploy_project_root_dir = os.path.dirname(os.path.dirname(__file__)) @@ -12,14 +14,13 @@ def test_config_no_proxy(): assert cfg.volumes["outpack"] == "outpack_volume" assert cfg.container_prefix == "packit" - assert len(cfg.containers) == 4 - assert cfg.containers["outpack-server"] == "outpack-server" - assert cfg.containers["packit"] == "packit" - assert cfg.containers["packit-api"] == "packit-api" - assert cfg.containers["packit-db"] == "packit-db" + assert cfg.outpack_server.container_name == "outpack-server" + assert cfg.packit_app.container_name == "packit" + assert cfg.packit_api.container_name == "packit-api" + assert cfg.packit_db.container_name == "packit-db" - assert str(cfg.outpack_ref) == "ghcr.io/mrc-ide/outpack_server:main" - assert str(cfg.packit_ref) == "ghcr.io/mrc-ide/packit:main" + assert str(cfg.outpack_server.image) == "ghcr.io/mrc-ide/outpack_server:main" + assert str(cfg.packit_app.image) == "ghcr.io/mrc-ide/packit:main" assert str(cfg.packit_db.image) == "ghcr.io/mrc-ide/packit-db:main" assert str(cfg.packit_api.image) == "ghcr.io/mrc-ide/packit-api:main" @@ -39,8 +40,7 @@ def test_config_proxy_disabled(): def test_config_proxy(): cfg = PackitConfig("config/novault") assert cfg.proxy is not None - assert "proxy" in cfg.containers - assert str(cfg.proxy.image) == "ghcr.io/mrc-ide/packit-proxy:main" + assert cfg.proxy.image == BuildSpec(os.path.join(packit_deploy_project_root_dir, "proxy")) assert cfg.proxy.hostname == "localhost" assert cfg.proxy.port_http == 80 assert cfg.proxy.port_https == 443 @@ -157,7 +157,7 @@ def test_workers_can_be_enabled(): assert cfg.orderly_runner.env == {"FOO": "bar"} assert str(cfg.orderly_runner.image) == "ghcr.io/mrc-ide/orderly.runner:main" - assert str(cfg.redis_image) == "library/redis:8.0" + assert str(cfg.redis.image) == "library/redis:8.0" def test_workers_can_be_omitted(): diff --git a/tests/test_integration.py b/tests/test_integration.py index caa30e0..c9e23a5 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -76,8 +76,7 @@ def test_start_and_stop_proxy(): assert docker_util.container_exists("packit-proxy") # Trivial check that the proxy container works: - cfg = PackitConfig(path) - proxy = get_container(cfg, "proxy") + proxy = get_container("packit-proxy") ports = proxy.attrs["HostConfig"]["PortBindings"] assert set(ports.keys()) == {"443/tcp", "80/tcp"} http_get("http://localhost") @@ -134,8 +133,6 @@ def test_acme_buddy_writes_cert(): cert_str = docker_util.string_from_container(proxy, "/run/proxy/certificate.pem") cert = x509.load_pem_x509_certificate(cert_str.encode(), default_backend()) assert cert.subject == cert.issuer - pubkey = cert.public_key() - pubkey.verify(cert.signature, cert.tbs_certificate_bytes) finally: stop_packit(path) @@ -151,17 +148,13 @@ def test_api_configured(): cl = docker.client.from_env() containers = cl.containers.list() assert len(containers) == 4 - cfg = PackitConfig(path) - api = get_container(cfg, "packit-api") + api = get_container("packit-packit-api") - assert ( - get_env_var(api, "PACKIT_DB_URL") - == b"jdbc:postgresql://packit-packit-db:5432/packit?stringtype=unspecified\n" - ) + assert get_env_var(api, "PACKIT_DB_URL") == b"jdbc:postgresql://packit-db:5432/packit?stringtype=unspecified\n" assert get_env_var(api, "PACKIT_DB_USER") == b"packituser\n" assert get_env_var(api, "PACKIT_DB_PASSWORD") == b"changeme\n" - assert get_env_var(api, "PACKIT_OUTPACK_SERVER_URL") == b"http://packit-outpack-server:8000\n" + assert get_env_var(api, "PACKIT_OUTPACK_SERVER_URL") == b"http://outpack-server:8000\n" assert get_env_var(api, "PACKIT_AUTH_ENABLED") == b"false\n" # has configured default management port assert get_env_var(api, "PACKIT_MANAGEMENT_PORT") == b"8081\n" @@ -179,8 +172,7 @@ def test_api_configured_for_github_auth(): cli.cli_start.callback(pull=False, name=path, options=options) - cfg = PackitConfig(path) - api = get_container(cfg, "packit-api") + api = get_container("packit-packit-api") # assert env variables assert get_env_var(api, "PACKIT_AUTH_METHOD") == b"github\n" @@ -204,8 +196,7 @@ def test_api_configured_with_custom_branding(): cli.cli_start.callback(pull=False, name=path, options=options) - cfg = PackitConfig(path) - api = get_container(cfg, "packit-api") + api = get_container("packit-packit-api") assert get_env_var(api, "PACKIT_BRAND_LOGO_ALT_TEXT") == b"My logo\n" assert get_env_var(api, "PACKIT_BRAND_LOGO_NAME") == b"examplelogo.webp\n" @@ -226,8 +217,7 @@ def test_custom_branding_end_to_end(): cli.cli_start.callback(pull=False, name=path, options=options) - cfg = PackitConfig(path) - api = get_container(cfg, "packit") + api = get_container("packit-packit") index_html = docker_util.string_from_container(api, "/usr/share/nginx/html/index.html") assert "My Packit Instance" in index_html @@ -265,17 +255,16 @@ def test_deploy_with_runner_support(): prefix = "packit-orderly-runner-worker" assert sum(x.name.startswith(prefix) for x in containers) == 2 - cfg = PackitConfig(path) - api = get_container(cfg, "packit-api") + api = get_container("packit-packit-api") - assert get_env_var(api, "PACKIT_ORDERLY_RUNNER_URL") == b"http://packit-orderly-runner-api:8001\n" + assert get_env_var(api, "PACKIT_ORDERLY_RUNNER_URL") == b"http://orderly-runner-api:8001\n" assert ( get_env_var(api, "PACKIT_ORDERLY_RUNNER_REPOSITORY_URL") == b"https://github.com/reside-ic/orderly2-example.git\n" ) assert get_env_var(api, "PACKIT_ORDERLY_RUNNER_LOCATION_URL") == get_env_var(api, "PACKIT_OUTPACK_SERVER_URL") - runner = get_container(cfg, "orderly-runner-api") + runner = get_container("packit-orderly-runner-api") assert get_env_var(runner, "PACKIT_RUNNER_EXAMPLE_ENVVAR") == b"hello\n" finally: stop_packit(path) @@ -291,8 +280,7 @@ def test_vault(): cli.cli_start.callback(pull=False, name=path, options=options) - cfg = PackitConfig(path) - api = get_container(cfg, "packit-api") + api = get_container("packit-packit-api") assert get_env_var(api, "PACKIT_DB_USER") == b"us3r\n" assert get_env_var(api, "PACKIT_DB_PASSWORD") == b"p@ssword\n" @@ -313,11 +301,10 @@ def test_can_read_packit_metrics_on_custom_port(): assert res.exit_code == 0 # has configured non-default management port - cfg = PackitConfig(path) - api = get_container(cfg, "packit-api") + api = get_container("packit-packit-api") assert get_env_var(api, "PACKIT_MANAGEMENT_PORT") == b"8082\n" - proxy = get_container(cfg, "proxy") + proxy = get_container("packit-proxy") curl_output = curl_get_from_container(proxy, "http://packit-api:8082/health") assert '{"status":"UP"}' in curl_output finally: @@ -361,9 +348,9 @@ def get_env_var(container, env): return docker_util.exec_safely(container, ["sh", "-c", f"echo ${env}"]).output -def get_container(cfg, name): +def get_container(name): with DockerClient() as cl: - return cl.containers.get(f"{cfg.container_prefix}-{cfg.containers[name]}") + return cl.containers.get(name) def test_db_volume_is_persisted(): @@ -376,12 +363,11 @@ def test_db_volume_is_persisted(): # Create a real user create_super_user() - cfg = PackitConfig(path) sql = "SELECT username from public.user" cmd = ["psql", "-t", "-A", "-U", "packituser", "-d", "packit", "-c", sql] # Check that we have actually created our user: - db = get_container(cfg, "packit-db") + db = get_container("packit-packit-db") users = docker_util.exec_safely(db, cmd).output.decode("UTF-8").splitlines() assert set(users) == {"SERVICE", "resideUser@resideAdmin.ic.ac.uk"} @@ -394,7 +380,7 @@ def test_db_volume_is_persisted(): assert res.exit_code == 0 # Check that the users have survived - db = get_container(cfg, "packit-db") + db = get_container("packit-packit-db") users = docker_util.exec_safely(db, cmd).output.decode("UTF-8").splitlines() assert set(users) == {"SERVICE", "resideUser@resideAdmin.ic.ac.uk"} finally: diff --git a/tests/test_packit.py b/tests/test_packit.py index 25bff92..5a913e5 100644 --- a/tests/test_packit.py +++ b/tests/test_packit.py @@ -14,19 +14,19 @@ def test_environment_with_no_runner_contains_no_envvars(): def test_environment_with_public_runner_contains_url(): cfg = PackitConfig("config/runner") env = packit_api_get_env(cfg) - assert env["PACKIT_ORDERLY_RUNNER_URL"] == "http://packit-orderly-runner-api:8001" + assert env["PACKIT_ORDERLY_RUNNER_URL"] == "http://orderly-runner-api:8001" assert env["PACKIT_ORDERLY_RUNNER_REPOSITORY_URL"] == "https://github.com/reside-ic/orderly2-example.git" assert "PACKIT_ORDERLY_RUNNER_REPOSITORY_SSH_KEY" not in env - assert env["PACKIT_ORDERLY_RUNNER_LOCATION_URL"] == "http://packit-outpack-server:8000" + assert env["PACKIT_ORDERLY_RUNNER_LOCATION_URL"] == "http://outpack-server:8000" def test_environment_with_private_runner_contains_url_and_key(): cfg = PackitConfig("config/runner-private") env = packit_api_get_env(cfg) - assert env["PACKIT_ORDERLY_RUNNER_URL"] == "http://packit-orderly-runner-api:8001" + assert env["PACKIT_ORDERLY_RUNNER_URL"] == "http://orderly-runner-api:8001" assert env["PACKIT_ORDERLY_RUNNER_REPOSITORY_URL"] == "git@github.com:reside-ic/orderly2-example-private.git" assert isinstance(env["PACKIT_ORDERLY_RUNNER_REPOSITORY_SSH_KEY"], str) - assert env["PACKIT_ORDERLY_RUNNER_LOCATION_URL"] == "http://packit-outpack-server:8000" + assert env["PACKIT_ORDERLY_RUNNER_LOCATION_URL"] == "http://outpack-server:8000" def test_default_cors_origins():