[WIP] fix(sre): resolve OpenShift deployment failures across SRE tools and applications

scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml

@@ -34,7 +34,7 @@
       kubeconfig: "{{ applications_cluster.kubeconfig }}"
       platform: "{{ applications_cluster.platform }}"
 
-- name: Create Service Account with `anyuid` access
+- name: Create Service Account with SCC access
   when:
     - applications_cluster.platform == "openshift"
   block:
@@ -49,23 +49,52 @@
             namespace: "{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}"
         state: present
 
-    - name: Create RoleBinding for access to anyuid SecurityContextConstraint
+    # The demo services pin fixed UIDs (needs RunAsAny, like anyuid), but when
+    # Dynatrace OneAgent injection is enabled it also adds a seccomp annotation,
+    # which the built-in anyuid SCC forbids (bumping pods to restricted-v2 which
+    # then rejects the fixed UIDs). This custom SCC is anyuid-equivalent PLUS
+    # permits seccomp profiles, so the demo runs whether or not OneAgent injects.
+    - name: Create custom SecurityContextConstraint for OpenTelemetry Demo
       kubernetes.core.k8s:
         kubeconfig: "{{ applications_cluster.kubeconfig }}"
         resource_definition:
-          kind: RoleBinding
-          apiVersion: rbac.authorization.k8s.io/v1
+          apiVersion: security.openshift.io/v1
+          kind: SecurityContextConstraints
           metadata:
-            name: "{{ applications_managers.opentelemetry_demo.helm.release.name }}"
-            namespace: "{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}"
-          subjects:
-            - kind: ServiceAccount
-              name: "{{ applications_managers.opentelemetry_demo.helm.release.name }}"
-              namespace: "{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}"
-          roleRef:
-            apiGroup: rbac.authorization.k8s.io
-            kind: ClusterRole
-            name: system:openshift:scc:anyuid
+            name: it-bench-otel-demo-anyuid-seccomp
+            labels:
+              "app.kubernetes.io/managed-by": ITBench
+          allowHostDirVolumePlugin: false
+          allowHostIPC: false
+          allowHostNetwork: false
+          allowHostPID: false
+          allowHostPorts: false
+          allowPrivilegeEscalation: true
+          allowPrivilegedContainer: false
+          fsGroup:
+            type: RunAsAny
+          readOnlyRootFilesystem: false
+          requiredDropCapabilities:
+            - MKNOD
+          runAsUser:
+            type: RunAsAny
+          seLinuxContext:
+            type: MustRunAs
+          seccompProfiles:
+            - "*"
+          supplementalGroups:
+            type: RunAsAny
+          users:
+            - "system:serviceaccount:{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}:{{ applications_managers.opentelemetry_demo.helm.release.name }}"
+          volumes:
+            - configMap
+            - csi
+            - downwardAPI
+            - emptyDir
+            - ephemeral
+            - persistentVolumeClaim
+            - projected
+            - secret
         state: present
 
 - name: Install OpenTelemetry Demo (Astronomy Shop)

scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml

@@ -17,3 +17,15 @@
         name: "{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}"
     state: absent
     wait: true
+
+# The custom SCC is cluster-scoped, so namespace deletion does not remove it.
+- name: Delete custom SecurityContextConstraint for OpenTelemetry Demo
+  kubernetes.core.k8s:
+    kubeconfig: "{{ applications_cluster.kubeconfig }}"
+    api_version: security.openshift.io/v1
+    kind: SecurityContextConstraints
+    name: it-bench-otel-demo-anyuid-seccomp
+    state: absent
+    wait: true
+  when:
+    - applications_cluster.platform == "openshift"

scenarios/sre/project/roles/tools/tasks/install.yaml

@@ -45,9 +45,14 @@
       ansible.builtin.import_tasks:
         file: install_opentelemetry.yaml
 
+    # OpenShift ships metrics-server (the v1beta1.metrics.k8s.io APIService is
+    # owned by the cluster-monitoring-operator), so installing it here conflicts.
+    # Only install metrics-server on plain Kubernetes.
     - name: Import Kubernetes Metrics Server installation tasks
       ansible.builtin.import_tasks:
         file: install_kubernetes_metrics_server.yaml
+      when:
+        - tools_cluster.platform != "openshift"
 
     - name: Import cert-manager installation tasks
       ansible.builtin.import_tasks:

scenarios/sre/project/roles/tools/tasks/install_cert_manager.yaml

@@ -120,7 +120,11 @@
         namespace: cert-manager-operator
       register: tools_openshift_cert_manager_csv
       until:
-        - tools_openshift_cert_manager_csv.resources | ansible.builtin.selectattr("status.phase", "==", "Succeeded") | ansible.builtin.length == 0
+        - >-
+          tools_openshift_cert_manager_csv.resources
+          | ansible.builtin.selectattr("metadata.name", "match", "^cert-manager-operator")
+          | ansible.builtin.selectattr("status.phase", "==", "Succeeded")
+          | ansible.builtin.length > 0
       retries: 10
       delay: 30
 

scenarios/sre/project/roles/tools/tasks/install_chaos_mesh.yaml

@@ -20,7 +20,11 @@
           allowHostNetwork: false
           allowHostPID: true
           allowHostPorts: false
-          allowPrivilegeEscalation: false
+          # A privileged container implies privilege escalation; setting this to
+          # false while allowPrivilegedContainer is true produces an invalid pod
+          # spec that the API server rejects ("cannot set allowPrivilegeEscalation
+          # to false and privileged to true").
+          allowPrivilegeEscalation: true
           allowPrivilegedContainer: true
           allowedCapabilities:
             - IPC_LOCK

scenarios/sre/project/roles/tools/tasks/install_clickhouse_instances.yaml

@@ -125,13 +125,16 @@
                         - matchExpressions:
                             - key: node-role.itbench.io/sandbox
                               operator: DoesNotExist
-                securityContext:
-                  fsGroup: 101
-                  runAsGroup: 101
-                  runAsNonRoot: true
-                  runAsUser: 101
-                  seccompProfile:
-                    type: RuntimeDefault
+                # On OpenShift the restricted-v2 SCC assigns a UID/fsGroup from
+                # the namespace's allowed range, and rejects the fixed value 101.
+                # Omit the hardcoded IDs there so admission injects valid ones;
+                # keep the explicit ClickHouse-image UID (101) on plain Kubernetes.
+                securityContext: >-
+                  {{
+                    {'runAsNonRoot': true, 'seccompProfile': {'type': 'RuntimeDefault'}}
+                    if tools_cluster.platform == 'openshift'
+                    else {'fsGroup': 101, 'runAsGroup': 101, 'runAsNonRoot': true, 'runAsUser': 101, 'seccompProfile': {'type': 'RuntimeDefault'}}
+                  }}
                 containers:
                   - name: clickhouse
                     securityContext:

scenarios/sre/project/roles/tools/tasks/install_istio.yaml

@@ -165,8 +165,8 @@
 
 - name: Enable Host Routing
   kubernetes.core.k8s_json_patch:
-    api_version: network.operator.openshift.io/v1
-    kind: EgressRouter
+    api_version: operator.openshift.io/v1
+    kind: Network
     kubeconfig: "{{ tools_cluster.kubeconfig }}"
     name: cluster
     patch:

scenarios/sre/project/roles/tools/tasks/install_kubernetes_topology_monitor.yaml

@@ -1,16 +1,10 @@
 ---
-- name: Create temporary directory for Git checkout
-  ansible.builtin.tempfile:
-    prefix: topology_monitor
-    state: directory
-  register: tools_temporary_directory
-
-- name: Checkout Kubernetes Topology Monitor repository using Git
-  ansible.builtin.git:
-    depth: 1
-    dest: "{{ tools_temporary_directory.path }}"
-    repo: "{{ tools_managers.kubernetes_topology_monitor.git.repository }}"
-    version: "{{ tools_managers.kubernetes_topology_monitor.git.branch }}"
+# Resolve the in-tree Kubernetes Topology Monitor chart. Using the local copy
+# (rather than cloning upstream) keeps OpenShift/PodSecurity fixes to the chart
+# authoritative and reviewable in this repository.
+- name: Resolve the local Kubernetes Topology Monitor chart path
+  ansible.builtin.set_fact:
+    tools_topology_monitor_chart: "{{ [playbook_dir, '..', 'tools', 'kubernetes-topology-monitor', 'charts', 'kubernetes-topology-monitor'] | ansible.builtin.path_join | ansible.builtin.realpath }}"
 
 - name: Create Namespace for Kubernetes Topology Monitor
   kubernetes.core.k8s:
@@ -27,15 +21,10 @@
 
 - name: Install Kubernetes Topology Monitor
   kubernetes.core.helm:
-    chart_ref: "{{ tools_temporary_directory.path }}/scenarios/sre/tools/kubernetes-topology-monitor/charts/kubernetes-topology-monitor"
+    chart_ref: "{{ tools_topology_monitor_chart }}"
     kubeconfig: "{{ tools_cluster.kubeconfig }}"
     release_name: "{{ tools_managers.kubernetes_topology_monitor.helm.release.name }}"
     release_namespace: "{{ tools_managers.kubernetes_topology_monitor.kubernetes.namespace }}"
     release_state: present
     timeout: 10m0s
     wait: true
-
-- name: Delete the temporary directory
-  ansible.builtin.file:
-    path: "{{ tools_temporary_directory.path }}"
-    state: absent

scenarios/sre/project/roles/tools/tasks/install_openshift_routes.yaml

@@ -69,7 +69,7 @@
       apiVersion: route.openshift.io/v1
       kind: Route
       metadata:
-        name: "{{ tools_managers.opencost.chart.release.name }}"
+        name: "{{ tools_managers.opencost.helm.release.name }}"
         namespace: "{{ tools_managers.opencost.kubernetes.namespace }}"
       spec:
         httpHeaders:
@@ -79,7 +79,7 @@
           targetPort: 9090
         to:
           kind: Service
-          name: "{{ tools_managers.opencost.chart.release.name }}"
+          name: "{{ tools_managers.opencost.helm.release.name }}"
     state: present
   when:
     - tools_configuration.finops.enabled | ansible.builtin.default(false)

scenarios/sre/project/roles/tools/tasks/install_opentelemetry_operator.yaml

@@ -97,7 +97,7 @@
         namespace: openshift-opentelemetry-operator
       register: tools_openshift_otel_operator_csv
       until:
-        - tools_openshift_otel_operator_csv.resources | ansible.builtin.selectattr("status.phase", "==", "Succeeded") | ansible.builtin.length == 0
+        - tools_openshift_otel_operator_csv.resources | ansible.builtin.selectattr("status.phase", "==", "Succeeded") | ansible.builtin.length > 0
       retries: 10
       delay: 30
 
@@ -112,3 +112,45 @@
         - tools_openshift_otel_collector_crd.resources | ansible.builtin.length == 1
       retries: 10
       delay: 30
+
+    # The CSV reporting "Succeeded" does not guarantee the controller-manager
+    # (which serves the mutating/validating webhooks) is Ready. Applying an
+    # OpenTelemetryCollector before the webhook has endpoints fails with
+    # "no endpoints available for service ...-controller-manager-service".
+    # Wait for the controller-manager Deployment to be fully available.
+    - name: Wait for the OpenTelemetry Operator controller-manager to be available
+      kubernetes.core.k8s_info:
+        api_version: apps/v1
+        kind: Deployment
+        kubeconfig: "{{ tools_cluster.kubeconfig }}"
+        namespace: openshift-opentelemetry-operator
+        label_selectors:
+          - app.kubernetes.io/name=opentelemetry-operator
+      register: tools_openshift_otel_operator_deploy
+      until:
+        - tools_openshift_otel_operator_deploy.resources | ansible.builtin.length > 0
+        - >-
+          tools_openshift_otel_operator_deploy.resources
+          | map(attribute='status.unavailableReplicas', default=0)
+          | select('gt', 0) | list | ansible.builtin.length == 0
+        - >-
+          tools_openshift_otel_operator_deploy.resources
+          | map(attribute='status.readyReplicas', default=0)
+          | select('gt', 0) | list | ansible.builtin.length
+          == tools_openshift_otel_operator_deploy.resources | ansible.builtin.length
+      retries: 20
+      delay: 15
+
+    - name: Wait for the OpenTelemetry Operator webhook service to have endpoints
+      kubernetes.core.k8s_info:
+        api_version: v1
+        kind: Endpoints
+        kubeconfig: "{{ tools_cluster.kubeconfig }}"
+        namespace: openshift-opentelemetry-operator
+        name: opentelemetry-operator-controller-manager-service
+      register: tools_openshift_otel_webhook_endpoints
+      until:
+        - tools_openshift_otel_webhook_endpoints.resources | ansible.builtin.length == 1
+        - tools_openshift_otel_webhook_endpoints.resources[0].subsets | ansible.builtin.default([]) | ansible.builtin.length > 0
+      retries: 20
+      delay: 15

scenarios/sre/project/roles/tools/tasks/set_internal_endpoints_prometheus.yaml

@@ -43,29 +43,48 @@
   when:
     - tools_cluster.platform == "openshift"
   block:
-    - name: Retrieve Service for Prometheus
+    # On OpenShift, in-cluster metrics are served by the thanos-querier (which
+    # aggregates platform + user-workload Prometheus). It listens on HTTPS :9091
+    # and requires a bearer token; there is no plain-HTTP :9090 service.
+    - name: Retrieve Service for the OpenShift Thanos Querier
       kubernetes.core.k8s_info:
         api_version: v1
         kind: Service
         kubeconfig: "{{ tools_cluster.kubeconfig }}"
-        name: prometheus-k8s
+        name: thanos-querier
         namespace: openshift-monitoring
       register: tools_prometheus_service
 
-    - name: Validate that Prometheus service exists
+    - name: Validate that the Thanos Querier service exists
       ansible.builtin.assert:
         that:
           - tools_prometheus_service.api_found
           - tools_prometheus_service.resources | ansible.builtin.length == 1
-        fail_msg: Prometheus service not found. Please verify that Prometheus was successfully installed.
-        success_msg: Prometheus service found.
+        fail_msg: Thanos Querier service not found. Please verify that OpenShift monitoring is enabled.
+        success_msg: Thanos Querier service found.
+
+    - name: Create a bearer token for querying OpenShift monitoring
+      ansible.builtin.command:
+        argv:
+          - "{{ executables_oc_path | ansible.builtin.default('oc') }}"
+          - create
+          - token
+          - prometheus-k8s
+          - --namespace
+          - openshift-monitoring
+          - --duration
+          - 8760h
+      register: tools_prometheus_token
+      changed_when: false
+      no_log: true
 
     - name: Create Prometheus in-cluster endpoints
       ansible.builtin.set_fact:
+        tools_prometheus_bearer_token: "{{ tools_prometheus_token.stdout | ansible.builtin.trim }}"
         tools_prometheus_endpoint:
           http: |-
             {{
-              "http://" +
+              "https://" +
               (
                 [
                   tools_prometheus_service.resources[0].metadata.name,
@@ -76,5 +95,6 @@
                 ] |
                 ansible.builtin.join(".")
               ) +
-              ":9090"
+              ":9091"
             }}
+      no_log: true

scenarios/sre/project/roles/tools/tasks/uninstall_istio.yaml

@@ -1,9 +1,9 @@
 ---
 - name: Disable Host Routing
   kubernetes.core.k8s_json_patch:
-    api_version: network.operator.openshift.io/v1
-    kind: EgressRouter
-    kubeconfig: "{{ faults_cluster.kubeconfig }}"
+    api_version: operator.openshift.io/v1
+    kind: Network
+    kubeconfig: "{{ tools_cluster.kubeconfig }}"
     name: cluster
     patch:
       - op: replace

scenarios/sre/tools/kubernetes-topology-monitor/charts/kubernetes-topology-monitor/templates/statefulset.yaml

@@ -16,8 +16,14 @@ spec:
         app: topology-monitor
     spec:
       serviceAccountName: topology-monitor
+      # Pod-level security context compliant with the "restricted" Pod Security
+      # Standard / OpenShift restricted-v2 SCC. UID/GID/fsGroup are intentionally
+      # omitted so OpenShift assigns them from the namespace's allowed range; on
+      # plain Kubernetes the container runs as a non-root user from the image.
       securityContext:
-        fsGroup: 1000
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
       containers:
       - name: topology-monitor
         image: quay.io/it-bench/topology-monitor:0.0.4
@@ -29,8 +35,13 @@ spec:
         - "--interval=300"
         - "--max-snapshots=10"
         securityContext:
-          runAsUser: 1000
-          runAsGroup: 1000
+          allowPrivilegeEscalation: false
+          runAsNonRoot: true
+          capabilities:
+            drop:
+              - ALL
+          seccompProfile:
+            type: RuntimeDefault
         ports:
         - containerPort: 8080
         volumeMounts: