Add end-to-end tests (#94)

This pull request adds a small pod to the Cortex helm chart, which can
be used to perform commands, such as end-to-end tests. This pod will
have access to the database and also the Cortex configuration, so that
it can make OpenStack calls, making the implementation of e2e tests
significantly easier. In our deployment pipeline, we can use the new pod
to run `kubectl exec -it cortex-cli -- /usr/bin/cortex checks` to
execute the tests. This tooling was also added to the Tilt setup, with a
new command "Run E2E Tests".

Author: PhilippMatthes

.golangci.yaml

@@ -91,7 +91,6 @@ linters:
         - emptyStringTest
         - evalOrder
         - httpNoBody
-        - importShadow
         - initClause
         - methodExprCall
         - paramTypeCombine

Tiltfile

@@ -56,6 +56,17 @@ k8s_resource('cortex-mqtt', port_forwards=[
     port_forward(8004, 8080), # Websocket connection
 ], labels=['Core-Services'])
 
+########### Cortex Commands
+k8s_resource('cortex-cli', labels=['Commands'])
+local_resource(
+    'Run E2E Tests',
+    'kubectl exec -it cortex-cli -- /usr/bin/cortex checks',
+    deps=['./internal/checks'],
+    labels=['Commands'],
+    trigger_mode=TRIGGER_MODE_MANUAL,
+    auto_init=False,
+)
+
 ########### Postgres DB for Cortex Core Service
 k8s_yaml(helm('./helm/postgres', name='cortex-postgres'))
 k8s_resource('cortex-postgresql', port_forwards=[

helm/cortex/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.9.0
+version: 0.10.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/cortex/templates/cli.yaml

@@ -0,0 +1,54 @@
+# Copyright 2025 SAP SE
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ $.Chart.Name }}-cli
+  labels:
+    app: {{ $.Chart.Name }}-cli
+    {{- include "cortex.labels" $ | nindent 4 }}
+spec:
+  {{- with $.Values.imagePullSecrets }}
+  imagePullSecrets:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  securityContext:
+    {{- toYaml $.Values.podSecurityContext | nindent 4 }}
+  containers:
+    - name: {{ $.Chart.Name }}-cli
+      command:
+        - "/bin/sh"
+        - "-c"
+        - "echo 'Waiting for commands...' && sleep infinity"
+      securityContext:
+        {{- toYaml $.Values.securityContext | nindent 8 }}
+      image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag | default $.Chart.AppVersion }}"
+      imagePullPolicy: {{ $.Values.image.pullPolicy }}
+      resources:
+        {{- toYaml $.Values.resources | nindent 8 }}
+      volumeMounts:
+        - name: {{ include "cortex.fullname" $ }}-config-volume
+          mountPath: /etc/config
+        {{- with $.Values.volumeMounts }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+  volumes:
+    - name: {{ include "cortex.fullname" $ }}-config-volume
+      configMap:
+        name: {{ include "cortex.fullname" $ }}-config
+    {{- with $.Values.volumes }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with $.Values.nodeSelector }}
+  nodeSelector:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with $.Values.affinity }}
+  affinity:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with $.Values.tolerations }}
+  tolerations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}

internal/commands/checks/checks.go

@@ -0,0 +1,95 @@
+// Copyright 2025 SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package checks
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"strconv"
+
+	"github.com/cobaltcore-dev/cortex/internal/conf"
+	httpapi "github.com/cobaltcore-dev/cortex/internal/scheduler/api/http"
+	cortexopenstack "github.com/cobaltcore-dev/cortex/internal/sync/openstack"
+	"github.com/gophercloud/gophercloud/v2"
+	"github.com/gophercloud/gophercloud/v2/openstack"
+	"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/hypervisors"
+	"github.com/sapcc/go-bits/must"
+)
+
+// Run all checks.
+func RunChecks(ctx context.Context, config conf.Config) {
+	checkSchedulerReturnsValidHosts(ctx, config)
+}
+
+// Check that the scheduler returns a valid set of hosts.
+func checkSchedulerReturnsValidHosts(ctx context.Context, config conf.Config) {
+	osConf := config.GetSyncConfig().OpenStack
+	slog.Info("authenticating against openstack", "url", osConf.Keystone.URL)
+	authOptions := gophercloud.AuthOptions{
+		IdentityEndpoint: osConf.Keystone.URL,
+		Username:         osConf.Keystone.OSUsername,
+		DomainName:       osConf.Keystone.OSUserDomainName,
+		Password:         osConf.Keystone.OSPassword,
+		AllowReauth:      true,
+		Scope: &gophercloud.AuthScope{
+			ProjectName: osConf.Keystone.OSProjectName,
+			DomainName:  osConf.Keystone.OSProjectDomainName,
+		},
+	}
+	pc := must.Return(openstack.NewClient(authOptions.IdentityEndpoint))
+	must.Succeed(openstack.Authenticate(ctx, pc, authOptions))
+	url := must.Return(pc.EndpointLocator(gophercloud.EndpointOpts{
+		Type:         "compute",
+		Availability: gophercloud.Availability(osConf.Nova.Availability),
+	}))
+	sc := &gophercloud.ServiceClient{ProviderClient: pc, Endpoint: url, Type: "compute"}
+	slog.Info("authenticated against openstack", "url", url)
+	slog.Info("listing hypervisors")
+	pages := must.Return(hypervisors.List(sc, hypervisors.ListOpts{}).AllPages(ctx))
+	var data = &struct {
+		Hypervisors []cortexopenstack.Hypervisor `json:"hypervisors"`
+	}{}
+	must.Succeed(pages.(hypervisors.HypervisorPage).ExtractInto(data))
+	if len(data.Hypervisors) == 0 {
+		panic("no hypervisors found")
+	}
+	slog.Info("found hypervisors", "count", len(data.Hypervisors))
+
+	var hosts []httpapi.ExternalSchedulerHost
+	weights := make(map[string]float64)
+	for _, h := range data.Hypervisors {
+		weights[h.ServiceHost] = 1.0
+		hosts = append(hosts, httpapi.ExternalSchedulerHost{
+			ComputeHost:        h.ServiceHost,
+			HypervisorHostname: h.Hostname,
+		})
+	}
+	request := httpapi.ExternalSchedulerRequest{
+		Hosts:   hosts,
+		Weights: weights,
+	}
+	port := strconv.Itoa(config.GetSchedulerConfig().API.Port)
+	apiURL := "http://cortex-scheduler:" + port + "/scheduler/nova/external"
+	slog.Info("sending request to external scheduler", "apiURL", apiURL)
+
+	requestBody := must.Return(json.Marshal(request))
+	buf := bytes.NewBuffer(requestBody)
+	req := must.Return(http.NewRequestWithContext(ctx, http.MethodPost, apiURL, buf))
+	req.Header.Set("Content-Type", "application/json")
+	//nolint:bodyclose // We don't care about the body here.
+	respRaw := must.Return(http.DefaultClient.Do(req))
+	defer respRaw.Body.Close()
+	if respRaw.StatusCode != http.StatusOK {
+		panic("external scheduler API returned non-200 status code")
+	}
+	var resp httpapi.ExternalSchedulerResponse
+	must.Succeed(json.NewDecoder(respRaw.Body).Decode(&resp))
+	if len(resp.Hosts) == 0 {
+		panic("no hosts found in response")
+	}
+	slog.Info("check successful, got hosts", "count", len(resp.Hosts))
+}

main.go

@@ -12,6 +12,7 @@ import (
 	gosync "sync"
 	"time"
 
+	"github.com/cobaltcore-dev/cortex/internal/commands/checks"
 	"github.com/cobaltcore-dev/cortex/internal/conf"
 	"github.com/cobaltcore-dev/cortex/internal/db"
 	"github.com/cobaltcore-dev/cortex/internal/features"
@@ -131,7 +132,7 @@ func main() {
 		taskName = os.Args[1]
 		bininfo.SetTaskName(taskName)
 	} else {
-		panic(fmt.Sprintf("usage: %s [syncer | extractor | scheduler]", os.Args[0]))
+		panic(fmt.Sprintf("usage: %s [checks | syncer | extractor | scheduler]", os.Args[0]))
 	}
 
 	dbInstance := db.NewPostgresDB(config.GetDBConfig())
@@ -140,6 +141,14 @@ func main() {
 	migrater := db.NewMigrater(dbInstance)
 	migrater.Migrate(true)
 
+	// If we're running one-off tasks (commands), don't setup the monitoring server.
+	//nolint:gocritic // We may add more tasks in the future.
+	switch taskName {
+	case "checks":
+		checks.RunChecks(ctx, config)
+		return
+	}
+
 	monitoringConfig := config.GetMonitoringConfig()
 	registry := monitoring.NewRegistry(monitoringConfig)
 	go runMonitoringServer(ctx, registry, monitoringConfig)