Merge pull request #42 from mrc-ide/proxy-metrics

Export packit and outpack metrics.

Author: EmmaLRussell

config/multipackit/packit.yml

@@ -22,6 +22,7 @@ instances:
     volumes:
       outpack: foo_outpack_volume
       packit_db: foo_packit_db
+      packit_db_backup: foo_packit_db_backup
 
     outpack:
       server:
@@ -48,6 +49,7 @@ instances:
     volumes:
       outpack: bar_outpack_volume
       packit_db: bar_packit_db
+      packit_db_backup: bar_packit_db_backup
 
     outpack:
       server:
@@ -78,6 +80,7 @@ proxy:
   hostname: localhost
   port_http: 80
   port_https: 443
+  port_metrics: 8080
   image:
     build: ../../proxy
 

config/runner/packit.yml

@@ -74,5 +74,6 @@ proxy:
   hostname: localhost
   port_http: 80
   port_https: 443
+  port_metrics: 8080
   image:
     build: ../../proxy

config/self-signed/packit.yml

@@ -92,6 +92,7 @@ proxy:
   hostname: localhost
   port_http: 80
   port_https: 443
+  port_metrics: 8080
   image:
     name: packit-proxy
     tag: main

src/packit_deploy/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2023-present Alex Hill <[email protected]>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.1.4"
+__version__ = "0.1.5"

src/packit_deploy/config.py

@@ -199,6 +199,7 @@ def from_data(cls, dat, key: list[str], *, container_name: str, ctx: Context) ->
 class PackitAPI:
     container_name: str
     image: constellation.ImageReference
+    # port at which api provides health metrics, separately proxied by montagu API - different from Proxy port_metrics!
     management_port: int
     base_url: str
     cors_allowed_origins: str
@@ -312,15 +313,24 @@ class Proxy:
     hostname: str
     port_http: int
     port_https: int
+    # port at which proxy will provide api and outpack server metrics. Different from PackitAPI management_port!
+    port_metrics: Optional[int]
 
     @classmethod
     def from_data(cls, dat, key: list[str], *, ctx: Context) -> "Proxy":
         image = config_buildable(dat, [*key, "image"], repo=ctx.repo, root=ctx.root)
         hostname = config.config_string(dat, [*key, "hostname"])
         port_http = config.config_integer(dat, [*key, "port_http"])
         port_https = config.config_integer(dat, [*key, "port_https"])
+        port_metrics = config.config_integer(dat, [*key, "port_metrics"], is_optional=True)
 
-        return Proxy(image=image, hostname=hostname, port_http=port_http, port_https=port_https)
+        return Proxy(
+            image=image,
+            hostname=hostname,
+            port_http=port_http,
+            port_https=port_https,
+            port_metrics=port_metrics,
+        )
 
 
 @dataclass
@@ -397,12 +407,16 @@ def outpack_server_url(self) -> str:
         return f"http://{self.outpack_server.container_name}:8000"
 
     @property
-    def packit_app_endpoint(self) -> str:
-        return f"{self.packit_app.container_name}:80"
+    def packit_app_url(self) -> str:
+        return f"http://{self.packit_app.container_name}:80"
+
+    @property
+    def packit_api_url(self) -> str:
+        return f"http://{self.packit_api.container_name}:8080"
 
     @property
-    def packit_api_endpoint(self) -> str:
-        return f"{self.packit_api.container_name}:8080"
+    def packit_api_management_url(self) -> str:
+        return f"http://{self.packit_api.container_name}:{self.packit_api.management_port}"
 
 
 class PackitConfig:

src/packit_deploy/packit_constellation.py

@@ -262,6 +262,8 @@ def proxy_container(proxy: config.Proxy, cfg: PackitConfig):
     if cfg.acme_config is not None:
         mounts.append(constellation.ConstellationVolumeMount("packit-tls", "/run/proxy"))
     ports = [proxy.port_http, proxy.port_https]
+    if proxy.port_metrics is not None:
+        ports.append(proxy.port_metrics)
     return ConstellationContainer(
         name,
         image=proxy.image,
@@ -278,8 +280,10 @@ def proxy_preconfigure(container: ConstellationContainer, cfg: PackitConfig, pro
     instances = [
         {
             "hostname": instance_hostname(name, proxy.hostname),
-            "upstream_api": instance.packit_api_endpoint,
-            "upstream_app": instance.packit_app_endpoint,
+            "outpack_server_url": instance.outpack_server_url,
+            "packit_app_url": instance.packit_app_url,
+            "packit_api_url": instance.packit_api_url,
+            "packit_api_management_url": instance.packit_api_management_url,
             "name": instance.brand.name or name,
         }
         for name, instance in cfg.instances.items()
@@ -296,6 +300,7 @@ def proxy_preconfigure(container: ConstellationContainer, cfg: PackitConfig, pro
         instances=instances,
         port_http=proxy.port_http,
         port_https=proxy.port_https,
+        port_metrics=proxy.port_metrics,
         index_hostname=index_hostname,
     )
     write_to_container(nginx_conf.encode("utf-8"), container, "/etc/nginx/conf.d/default.conf")

src/packit_deploy/templates/nginx.conf.j2

@@ -45,11 +45,11 @@ server {
     root /usr/share/nginx/html;
 
     location /api/ {
-        proxy_pass http://{{ instance.upstream_api }}/;
+        proxy_pass {{ instance.packit_api_url }}/;
     }
 
     location / {
-        proxy_pass http://{{ instance.upstream_app }}/;
+        proxy_pass {{ instance.packit_app_url }}/;
     }
 }
 {%- endfor -%}
@@ -108,3 +108,20 @@ server {
         return 301 https://$host$request_uri;
     }
 }
+
+{%- if port_metrics is not none -%}
+{%- for instance in instances -%}
+server {
+    listen       {{ port_metrics }};
+    server_name  {{ instance.hostname }};
+
+    location /metrics/outpack_server {
+        proxy_pass {{ instance.outpack_server_url }}/metrics;
+    }
+    location /metrics/packit-api {
+        proxy_pass {{ instance.packit_api_management_url }}/prometheus;
+    }
+
+}
+{%- endfor -%}
+{%- endif -%}

tests/test_integration.py

@@ -289,11 +289,11 @@ def test_vault():
 
 
 # Test that the custom management port defined in the novault config
-# has been correctly configured in the api so we can get metrics from
+# has been correctly configured in the api so we can get health metrics from
 # within the network - we currently do not expose packit metrics through
 # the proxy as this will be done through montagu proxy, and handled
 # separately in the nix deployment
-def test_can_read_packit_metrics_on_custom_port():
+def test_can_read_packit_health_metrics_on_custom_port():
     path = "config/novault"
     try:
         runner = CliRunner()
@@ -311,6 +311,46 @@ def test_can_read_packit_metrics_on_custom_port():
         stop_packit(path)
 
 
+def test_can_read_metrics_from_proxy_single_instance():
+    path = "config/runner"
+    try:
+        runner = CliRunner()
+        res = runner.invoke(cli.cli, ["start", "--pull", "--name", path])
+        assert res.exit_code == 0
+
+        retries = 50
+        api_res = http_get("http://localhost:8080/metrics/packit-api", retries=retries)
+        assert "application_ready_time_seconds" in api_res
+
+        outpack_res = http_get("http://localhost:8080/metrics/outpack_server", retries=retries)
+        assert "outpack_server_build_info" in outpack_res
+    finally:
+        stop_packit(path)
+
+
+def test_can_read_metrics_from_proxy_multi_instance():
+    path = "config/multipackit"
+    try:
+        runner = CliRunner()
+        res = runner.invoke(cli.cli, ["start", "--pull", "--name", path])
+        assert res.exit_code == 0
+
+        retries = 50
+        expected_api_metrics_content = "application_ready_time_seconds"
+        assert expected_api_metrics_content in http_get("http://foo.localhost:8080/metrics/packit-api", retries=retries)
+        assert expected_api_metrics_content in http_get("http://bar.localhost:8080/metrics/packit-api", retries=retries)
+
+        expected_outpack_metrics_content = "outpack_server_build_info"
+        assert expected_outpack_metrics_content in http_get(
+            "http://foo.localhost:8080/metrics/outpack_server", retries=retries
+        )
+        assert expected_outpack_metrics_content in http_get(
+            "http://bar.localhost:8080/metrics/outpack_server", retries=retries
+        )
+    finally:
+        stop_packit(path)
+
+
 def stop_packit(path):
     with mock.patch("packit_deploy.cli._prompt_yes_no") as prompt:
         prompt.return_value = True