diff --git a/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml b/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml index ff5353cd6..29b477256 100644 --- a/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml +++ b/scenarios/sre/project/roles/applications/tasks/install_opentelemetry_demo.yaml @@ -34,7 +34,7 @@ kubeconfig: "{{ applications_cluster.kubeconfig }}" platform: "{{ applications_cluster.platform }}" -- name: Create Service Account with `anyuid` access +- name: Create Service Account with SCC access when: - applications_cluster.platform == "openshift" block: @@ -49,23 +49,52 @@ namespace: "{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}" state: present - - name: Create RoleBinding for access to anyuid SecurityContextConstraint + # The demo services pin fixed UIDs (needs RunAsAny, like anyuid), but when + # Dynatrace OneAgent injection is enabled it also adds a seccomp annotation, + # which the built-in anyuid SCC forbids (bumping pods to restricted-v2 which + # then rejects the fixed UIDs). This custom SCC is anyuid-equivalent PLUS + # permits seccomp profiles, so the demo runs whether or not OneAgent injects. + - name: Create custom SecurityContextConstraint for OpenTelemetry Demo kubernetes.core.k8s: kubeconfig: "{{ applications_cluster.kubeconfig }}" resource_definition: - kind: RoleBinding - apiVersion: rbac.authorization.k8s.io/v1 + apiVersion: security.openshift.io/v1 + kind: SecurityContextConstraints metadata: - name: "{{ applications_managers.opentelemetry_demo.helm.release.name }}" - namespace: "{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}" - subjects: - - kind: ServiceAccount - name: "{{ applications_managers.opentelemetry_demo.helm.release.name }}" - namespace: "{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}" - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:openshift:scc:anyuid + name: it-bench-otel-demo-anyuid-seccomp + labels: + "app.kubernetes.io/managed-by": ITBench + allowHostDirVolumePlugin: false + allowHostIPC: false + allowHostNetwork: false + allowHostPID: false + allowHostPorts: false + allowPrivilegeEscalation: true + allowPrivilegedContainer: false + fsGroup: + type: RunAsAny + readOnlyRootFilesystem: false + requiredDropCapabilities: + - MKNOD + runAsUser: + type: RunAsAny + seLinuxContext: + type: MustRunAs + seccompProfiles: + - "*" + supplementalGroups: + type: RunAsAny + users: + - "system:serviceaccount:{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}:{{ applications_managers.opentelemetry_demo.helm.release.name }}" + volumes: + - configMap + - csi + - downwardAPI + - emptyDir + - ephemeral + - persistentVolumeClaim + - projected + - secret state: present - name: Install OpenTelemetry Demo (Astronomy Shop) diff --git a/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml b/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml index d29409f73..0ccdeed52 100644 --- a/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml +++ b/scenarios/sre/project/roles/applications/tasks/uninstall_opentelemetry_demo.yaml @@ -17,3 +17,15 @@ name: "{{ applications_managers.opentelemetry_demo.kubernetes.namespace }}" state: absent wait: true + +# The custom SCC is cluster-scoped, so namespace deletion does not remove it. +- name: Delete custom SecurityContextConstraint for OpenTelemetry Demo + kubernetes.core.k8s: + kubeconfig: "{{ applications_cluster.kubeconfig }}" + api_version: security.openshift.io/v1 + kind: SecurityContextConstraints + name: it-bench-otel-demo-anyuid-seccomp + state: absent + wait: true + when: + - applications_cluster.platform == "openshift" diff --git a/scenarios/sre/project/roles/tools/tasks/install.yaml b/scenarios/sre/project/roles/tools/tasks/install.yaml index f0a5a23a3..4829fda71 100644 --- a/scenarios/sre/project/roles/tools/tasks/install.yaml +++ b/scenarios/sre/project/roles/tools/tasks/install.yaml @@ -45,9 +45,14 @@ ansible.builtin.import_tasks: file: install_opentelemetry.yaml + # OpenShift ships metrics-server (the v1beta1.metrics.k8s.io APIService is + # owned by the cluster-monitoring-operator), so installing it here conflicts. + # Only install metrics-server on plain Kubernetes. - name: Import Kubernetes Metrics Server installation tasks ansible.builtin.import_tasks: file: install_kubernetes_metrics_server.yaml + when: + - tools_cluster.platform != "openshift" - name: Import cert-manager installation tasks ansible.builtin.import_tasks: diff --git a/scenarios/sre/project/roles/tools/tasks/install_cert_manager.yaml b/scenarios/sre/project/roles/tools/tasks/install_cert_manager.yaml index f753f42cc..511a52574 100644 --- a/scenarios/sre/project/roles/tools/tasks/install_cert_manager.yaml +++ b/scenarios/sre/project/roles/tools/tasks/install_cert_manager.yaml @@ -120,7 +120,11 @@ namespace: cert-manager-operator register: tools_openshift_cert_manager_csv until: - - tools_openshift_cert_manager_csv.resources | ansible.builtin.selectattr("status.phase", "==", "Succeeded") | ansible.builtin.length == 0 + - >- + tools_openshift_cert_manager_csv.resources + | ansible.builtin.selectattr("metadata.name", "match", "^cert-manager-operator") + | ansible.builtin.selectattr("status.phase", "==", "Succeeded") + | ansible.builtin.length > 0 retries: 10 delay: 30 diff --git a/scenarios/sre/project/roles/tools/tasks/install_chaos_mesh.yaml b/scenarios/sre/project/roles/tools/tasks/install_chaos_mesh.yaml index 1ffe3f491..2cae1a09e 100644 --- a/scenarios/sre/project/roles/tools/tasks/install_chaos_mesh.yaml +++ b/scenarios/sre/project/roles/tools/tasks/install_chaos_mesh.yaml @@ -20,7 +20,11 @@ allowHostNetwork: false allowHostPID: true allowHostPorts: false - allowPrivilegeEscalation: false + # A privileged container implies privilege escalation; setting this to + # false while allowPrivilegedContainer is true produces an invalid pod + # spec that the API server rejects ("cannot set allowPrivilegeEscalation + # to false and privileged to true"). + allowPrivilegeEscalation: true allowPrivilegedContainer: true allowedCapabilities: - IPC_LOCK diff --git a/scenarios/sre/project/roles/tools/tasks/install_clickhouse_instances.yaml b/scenarios/sre/project/roles/tools/tasks/install_clickhouse_instances.yaml index 9c39c9b2e..32792e56a 100644 --- a/scenarios/sre/project/roles/tools/tasks/install_clickhouse_instances.yaml +++ b/scenarios/sre/project/roles/tools/tasks/install_clickhouse_instances.yaml @@ -125,13 +125,16 @@ - matchExpressions: - key: node-role.itbench.io/sandbox operator: DoesNotExist - securityContext: - fsGroup: 101 - runAsGroup: 101 - runAsNonRoot: true - runAsUser: 101 - seccompProfile: - type: RuntimeDefault + # On OpenShift the restricted-v2 SCC assigns a UID/fsGroup from + # the namespace's allowed range, and rejects the fixed value 101. + # Omit the hardcoded IDs there so admission injects valid ones; + # keep the explicit ClickHouse-image UID (101) on plain Kubernetes. + securityContext: >- + {{ + {'runAsNonRoot': true, 'seccompProfile': {'type': 'RuntimeDefault'}} + if tools_cluster.platform == 'openshift' + else {'fsGroup': 101, 'runAsGroup': 101, 'runAsNonRoot': true, 'runAsUser': 101, 'seccompProfile': {'type': 'RuntimeDefault'}} + }} containers: - name: clickhouse securityContext: diff --git a/scenarios/sre/project/roles/tools/tasks/install_istio.yaml b/scenarios/sre/project/roles/tools/tasks/install_istio.yaml index a1c7a00dd..28633b02b 100644 --- a/scenarios/sre/project/roles/tools/tasks/install_istio.yaml +++ b/scenarios/sre/project/roles/tools/tasks/install_istio.yaml @@ -165,8 +165,8 @@ - name: Enable Host Routing kubernetes.core.k8s_json_patch: - api_version: network.operator.openshift.io/v1 - kind: EgressRouter + api_version: operator.openshift.io/v1 + kind: Network kubeconfig: "{{ tools_cluster.kubeconfig }}" name: cluster patch: diff --git a/scenarios/sre/project/roles/tools/tasks/install_kubernetes_topology_monitor.yaml b/scenarios/sre/project/roles/tools/tasks/install_kubernetes_topology_monitor.yaml index 4649c6c1d..37f658fde 100644 --- a/scenarios/sre/project/roles/tools/tasks/install_kubernetes_topology_monitor.yaml +++ b/scenarios/sre/project/roles/tools/tasks/install_kubernetes_topology_monitor.yaml @@ -1,16 +1,10 @@ --- -- name: Create temporary directory for Git checkout - ansible.builtin.tempfile: - prefix: topology_monitor - state: directory - register: tools_temporary_directory - -- name: Checkout Kubernetes Topology Monitor repository using Git - ansible.builtin.git: - depth: 1 - dest: "{{ tools_temporary_directory.path }}" - repo: "{{ tools_managers.kubernetes_topology_monitor.git.repository }}" - version: "{{ tools_managers.kubernetes_topology_monitor.git.branch }}" +# Resolve the in-tree Kubernetes Topology Monitor chart. Using the local copy +# (rather than cloning upstream) keeps OpenShift/PodSecurity fixes to the chart +# authoritative and reviewable in this repository. +- name: Resolve the local Kubernetes Topology Monitor chart path + ansible.builtin.set_fact: + tools_topology_monitor_chart: "{{ [playbook_dir, '..', 'tools', 'kubernetes-topology-monitor', 'charts', 'kubernetes-topology-monitor'] | ansible.builtin.path_join | ansible.builtin.realpath }}" - name: Create Namespace for Kubernetes Topology Monitor kubernetes.core.k8s: @@ -27,15 +21,10 @@ - name: Install Kubernetes Topology Monitor kubernetes.core.helm: - chart_ref: "{{ tools_temporary_directory.path }}/scenarios/sre/tools/kubernetes-topology-monitor/charts/kubernetes-topology-monitor" + chart_ref: "{{ tools_topology_monitor_chart }}" kubeconfig: "{{ tools_cluster.kubeconfig }}" release_name: "{{ tools_managers.kubernetes_topology_monitor.helm.release.name }}" release_namespace: "{{ tools_managers.kubernetes_topology_monitor.kubernetes.namespace }}" release_state: present timeout: 10m0s wait: true - -- name: Delete the temporary directory - ansible.builtin.file: - path: "{{ tools_temporary_directory.path }}" - state: absent diff --git a/scenarios/sre/project/roles/tools/tasks/install_openshift_routes.yaml b/scenarios/sre/project/roles/tools/tasks/install_openshift_routes.yaml index f544f0479..0954a79cf 100644 --- a/scenarios/sre/project/roles/tools/tasks/install_openshift_routes.yaml +++ b/scenarios/sre/project/roles/tools/tasks/install_openshift_routes.yaml @@ -69,7 +69,7 @@ apiVersion: route.openshift.io/v1 kind: Route metadata: - name: "{{ tools_managers.opencost.chart.release.name }}" + name: "{{ tools_managers.opencost.helm.release.name }}" namespace: "{{ tools_managers.opencost.kubernetes.namespace }}" spec: httpHeaders: @@ -79,7 +79,7 @@ targetPort: 9090 to: kind: Service - name: "{{ tools_managers.opencost.chart.release.name }}" + name: "{{ tools_managers.opencost.helm.release.name }}" state: present when: - tools_configuration.finops.enabled | ansible.builtin.default(false) diff --git a/scenarios/sre/project/roles/tools/tasks/install_opentelemetry_operator.yaml b/scenarios/sre/project/roles/tools/tasks/install_opentelemetry_operator.yaml index 864c04804..4a26ee94b 100644 --- a/scenarios/sre/project/roles/tools/tasks/install_opentelemetry_operator.yaml +++ b/scenarios/sre/project/roles/tools/tasks/install_opentelemetry_operator.yaml @@ -97,7 +97,7 @@ namespace: openshift-opentelemetry-operator register: tools_openshift_otel_operator_csv until: - - tools_openshift_otel_operator_csv.resources | ansible.builtin.selectattr("status.phase", "==", "Succeeded") | ansible.builtin.length == 0 + - tools_openshift_otel_operator_csv.resources | ansible.builtin.selectattr("status.phase", "==", "Succeeded") | ansible.builtin.length > 0 retries: 10 delay: 30 @@ -112,3 +112,45 @@ - tools_openshift_otel_collector_crd.resources | ansible.builtin.length == 1 retries: 10 delay: 30 + + # The CSV reporting "Succeeded" does not guarantee the controller-manager + # (which serves the mutating/validating webhooks) is Ready. Applying an + # OpenTelemetryCollector before the webhook has endpoints fails with + # "no endpoints available for service ...-controller-manager-service". + # Wait for the controller-manager Deployment to be fully available. + - name: Wait for the OpenTelemetry Operator controller-manager to be available + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: Deployment + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: openshift-opentelemetry-operator + label_selectors: + - app.kubernetes.io/name=opentelemetry-operator + register: tools_openshift_otel_operator_deploy + until: + - tools_openshift_otel_operator_deploy.resources | ansible.builtin.length > 0 + - >- + tools_openshift_otel_operator_deploy.resources + | map(attribute='status.unavailableReplicas', default=0) + | select('gt', 0) | list | ansible.builtin.length == 0 + - >- + tools_openshift_otel_operator_deploy.resources + | map(attribute='status.readyReplicas', default=0) + | select('gt', 0) | list | ansible.builtin.length + == tools_openshift_otel_operator_deploy.resources | ansible.builtin.length + retries: 20 + delay: 15 + + - name: Wait for the OpenTelemetry Operator webhook service to have endpoints + kubernetes.core.k8s_info: + api_version: v1 + kind: Endpoints + kubeconfig: "{{ tools_cluster.kubeconfig }}" + namespace: openshift-opentelemetry-operator + name: opentelemetry-operator-controller-manager-service + register: tools_openshift_otel_webhook_endpoints + until: + - tools_openshift_otel_webhook_endpoints.resources | ansible.builtin.length == 1 + - tools_openshift_otel_webhook_endpoints.resources[0].subsets | ansible.builtin.default([]) | ansible.builtin.length > 0 + retries: 20 + delay: 15 diff --git a/scenarios/sre/project/roles/tools/tasks/set_internal_endpoints_prometheus.yaml b/scenarios/sre/project/roles/tools/tasks/set_internal_endpoints_prometheus.yaml index 0303688d5..60992261f 100644 --- a/scenarios/sre/project/roles/tools/tasks/set_internal_endpoints_prometheus.yaml +++ b/scenarios/sre/project/roles/tools/tasks/set_internal_endpoints_prometheus.yaml @@ -43,29 +43,48 @@ when: - tools_cluster.platform == "openshift" block: - - name: Retrieve Service for Prometheus + # On OpenShift, in-cluster metrics are served by the thanos-querier (which + # aggregates platform + user-workload Prometheus). It listens on HTTPS :9091 + # and requires a bearer token; there is no plain-HTTP :9090 service. + - name: Retrieve Service for the OpenShift Thanos Querier kubernetes.core.k8s_info: api_version: v1 kind: Service kubeconfig: "{{ tools_cluster.kubeconfig }}" - name: prometheus-k8s + name: thanos-querier namespace: openshift-monitoring register: tools_prometheus_service - - name: Validate that Prometheus service exists + - name: Validate that the Thanos Querier service exists ansible.builtin.assert: that: - tools_prometheus_service.api_found - tools_prometheus_service.resources | ansible.builtin.length == 1 - fail_msg: Prometheus service not found. Please verify that Prometheus was successfully installed. - success_msg: Prometheus service found. + fail_msg: Thanos Querier service not found. Please verify that OpenShift monitoring is enabled. + success_msg: Thanos Querier service found. + + - name: Create a bearer token for querying OpenShift monitoring + ansible.builtin.command: + argv: + - "{{ executables_oc_path | ansible.builtin.default('oc') }}" + - create + - token + - prometheus-k8s + - --namespace + - openshift-monitoring + - --duration + - 8760h + register: tools_prometheus_token + changed_when: false + no_log: true - name: Create Prometheus in-cluster endpoints ansible.builtin.set_fact: + tools_prometheus_bearer_token: "{{ tools_prometheus_token.stdout | ansible.builtin.trim }}" tools_prometheus_endpoint: http: |- {{ - "http://" + + "https://" + ( [ tools_prometheus_service.resources[0].metadata.name, @@ -76,5 +95,6 @@ ] | ansible.builtin.join(".") ) + - ":9090" + ":9091" }} + no_log: true diff --git a/scenarios/sre/project/roles/tools/tasks/uninstall_istio.yaml b/scenarios/sre/project/roles/tools/tasks/uninstall_istio.yaml index 52dd4b986..66d13c9ac 100644 --- a/scenarios/sre/project/roles/tools/tasks/uninstall_istio.yaml +++ b/scenarios/sre/project/roles/tools/tasks/uninstall_istio.yaml @@ -1,9 +1,9 @@ --- - name: Disable Host Routing kubernetes.core.k8s_json_patch: - api_version: network.operator.openshift.io/v1 - kind: EgressRouter - kubeconfig: "{{ faults_cluster.kubeconfig }}" + api_version: operator.openshift.io/v1 + kind: Network + kubeconfig: "{{ tools_cluster.kubeconfig }}" name: cluster patch: - op: replace diff --git a/scenarios/sre/tools/kubernetes-topology-monitor/charts/kubernetes-topology-monitor/templates/statefulset.yaml b/scenarios/sre/tools/kubernetes-topology-monitor/charts/kubernetes-topology-monitor/templates/statefulset.yaml index 2ddb83b03..51e481c46 100644 --- a/scenarios/sre/tools/kubernetes-topology-monitor/charts/kubernetes-topology-monitor/templates/statefulset.yaml +++ b/scenarios/sre/tools/kubernetes-topology-monitor/charts/kubernetes-topology-monitor/templates/statefulset.yaml @@ -16,8 +16,14 @@ spec: app: topology-monitor spec: serviceAccountName: topology-monitor + # Pod-level security context compliant with the "restricted" Pod Security + # Standard / OpenShift restricted-v2 SCC. UID/GID/fsGroup are intentionally + # omitted so OpenShift assigns them from the namespace's allowed range; on + # plain Kubernetes the container runs as a non-root user from the image. securityContext: - fsGroup: 1000 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault containers: - name: topology-monitor image: quay.io/it-bench/topology-monitor:0.0.4 @@ -29,8 +35,13 @@ spec: - "--interval=300" - "--max-snapshots=10" securityContext: - runAsUser: 1000 - runAsGroup: 1000 + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault ports: - containerPort: 8080 volumeMounts: