From 419856ed90db031d438e9d33f4608311528485ed Mon Sep 17 00:00:00 2001 From: Pawel Palucki Date: Wed, 12 Feb 2020 15:19:57 +0100 Subject: [PATCH 1/5] Static numa allocator --- .../kubernetes/monitoring/wca/daemonset.yaml | 2 +- .../monitoring/wca/kustomization.yaml | 1 + .../wca/wca-config-static-allocator.yaml | 73 +++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml diff --git a/examples/kubernetes/monitoring/wca/daemonset.yaml b/examples/kubernetes/monitoring/wca/daemonset.yaml index 9dcb46cbf..98019ceda 100644 --- a/examples/kubernetes/monitoring/wca/daemonset.yaml +++ b/examples/kubernetes/monitoring/wca/daemonset.yaml @@ -56,7 +56,7 @@ spec: # command: # - /usr/bin/wca.pex args: - - "--config=/etc/wca/wca-config.yaml" + - "--config=/etc/wca/wca-config-static-allocator.yaml" - "--log=info" - "--root" resources: diff --git a/examples/kubernetes/monitoring/wca/kustomization.yaml b/examples/kubernetes/monitoring/wca/kustomization.yaml index 6de669338..f01cab250 100644 --- a/examples/kubernetes/monitoring/wca/kustomization.yaml +++ b/examples/kubernetes/monitoring/wca/kustomization.yaml @@ -18,3 +18,4 @@ configMapGenerator: - name: wca-config files: - wca-config.yaml + - wca-config-static-allocator.yaml diff --git a/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml b/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml new file mode 100644 index 000000000..f0109994d --- /dev/null +++ b/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml @@ -0,0 +1,73 @@ +# Full example for Kubernetes running as daemonset (in-cluster k8s config) +# Requires environment variables: +# - KUBERNETES_SERVICE_HOST - provided by default by Kuberenetes +# - KUBERNETES_SERVICE_PORT - provided by default by Kubernetes +# - HOST_IP - need to explicitly enabled in Pod spec using downward-api +# - HOSTNAME - default provided by OS +runner: !AllocationRunner + + allocations_storage: !LogStorage + overwrite: True + output_filename: /var/lib/wca/allocations.prom + + anomalies_storage: !LogStorage + output_filename: /dev/null + + allocator: !StaticAllocator + + + rules: + - name: Bind to numa node1 + labels: + numanode: node1 + allocations: + migrate_pages: 0 + + - name: Bind to numa node2 + labels: + numanode: node2 + allocations: + migrate_pages: 1 + + - name: Bind to numa node2 + labels: + numanode: node2 + allocations: + migrate_pages: 2 + + - name: Bind to numa node3 + labels: + numanode: node3 + allocations: + migrate_pages: 3 + + measurement_runner: !MeasurementRunner + interval: 5.0 + node: !KubernetesNode + cgroup_driver: cgroupfs + monitored_namespaces: ["default"] + kubeapi_host: !Env KUBERNETES_SERVICE_HOST + kubeapi_port: !Env KUBERNETES_SERVICE_PORT + node_ip: !Env HOST_IP + + + metrics_storage: !LogStorage + overwrite: True + output_filename: /var/lib/wca/metrics.prom + + extra_labels: + node: !Env HOSTNAME + event_names: + - task_cycles # do not remove: USED BY E2E tests + - task_instructions + - task_offcore_requests_demand_data_rd # required fro R/W characterization + - task_offcore_requests_demand_rfo + enable_derived_metrics: True + uncore_event_names: + - platform_cas_count_reads + - platform_cas_count_writes + - platform_pmm_bandwidth_reads + - platform_pmm_bandwidth_writes + + wss_reset_interval: 1 + gather_hw_mm_topology: True From 91c28f99aaab488c07452bb1ef3d90e512a36430 Mon Sep 17 00:00:00 2001 From: Pawel Palucki Date: Wed, 12 Feb 2020 17:15:59 +0100 Subject: [PATCH 2/5] fix rules --- .../monitoring/wca/wca-config-static-allocator.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml b/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml index f0109994d..82bd34cc9 100644 --- a/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml +++ b/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml @@ -19,13 +19,13 @@ runner: !AllocationRunner rules: - name: Bind to numa node1 labels: - numanode: node1 + numanode: node0 allocations: migrate_pages: 0 - name: Bind to numa node2 labels: - numanode: node2 + numanode: node1 allocations: migrate_pages: 1 From 06c0ecc4fbbaec0388a874e254323d48aa9c1429 Mon Sep 17 00:00:00 2001 From: Pawel Palucki Date: Thu, 13 Feb 2020 12:34:10 +0100 Subject: [PATCH 3/5] fix rules --- .../monitoring/wca/wca-config-static-allocator.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml b/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml index 82bd34cc9..e22ac9ce5 100644 --- a/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml +++ b/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml @@ -4,8 +4,15 @@ # - KUBERNETES_SERVICE_PORT - provided by default by Kubernetes # - HOST_IP - need to explicitly enabled in Pod spec using downward-api # - HOSTNAME - default provided by OS +loggers: + wca.containers: TRACE + wca.cgroups_allocations: TRACE + wca.platforms: TRACE + wca.extra: TRACE + runner: !AllocationRunner + allocations_storage: !LogStorage overwrite: True output_filename: /var/lib/wca/allocations.prom @@ -21,24 +28,28 @@ runner: !AllocationRunner labels: numanode: node0 allocations: + cpuset_mems: '0' migrate_pages: 0 - name: Bind to numa node2 labels: numanode: node1 allocations: + cpuset_mems: '1' migrate_pages: 1 - name: Bind to numa node2 labels: numanode: node2 allocations: + cpuset_mems: '2' migrate_pages: 2 - name: Bind to numa node3 labels: numanode: node3 allocations: + cpuset_mems: '3' migrate_pages: 3 measurement_runner: !MeasurementRunner From 17783b79529e50092407e9350d2364b09ad8a74b Mon Sep 17 00:00:00 2001 From: Pawel Palucki Date: Thu, 13 Feb 2020 13:01:55 +0100 Subject: [PATCH 4/5] fix names of rules --- .../monitoring/wca/wca-config-static-allocator.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml b/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml index e22ac9ce5..526f4301f 100644 --- a/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml +++ b/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml @@ -24,14 +24,14 @@ runner: !AllocationRunner rules: - - name: Bind to numa node1 + - name: Bind to numa node0 labels: numanode: node0 allocations: cpuset_mems: '0' migrate_pages: 0 - - name: Bind to numa node2 + - name: Bind to numa node1 labels: numanode: node1 allocations: From b39bb4980a497ace1a682ffdb9dc6512bc7ca0fd Mon Sep 17 00:00:00 2001 From: Pawel Palucki Date: Thu, 13 Feb 2020 15:57:28 +0100 Subject: [PATCH 5/5] add comment --- .../monitoring/wca/wca-config-static-allocator.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml b/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml index 526f4301f..7b3d29164 100644 --- a/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml +++ b/examples/kubernetes/monitoring/wca/wca-config-static-allocator.yaml @@ -22,6 +22,13 @@ runner: !AllocationRunner allocator: !StaticAllocator + # cAdvisor/kubectl limitation in Topology/Discovery + # After new numanodes are provisioed and kubectl restarted: + #sudo sh -c 'echo 0-3 >/sys/fs/cgroup/cpuset/kubepods/cpuset.mems' + #sudo sh -c 'echo 0-3 >/sys/fs/cgroup/cpuset/kubepods/burstable/cpuset.mems' + #sudo sh -c 'echo 0-3 >/sys/fs/cgroup/cpuset/kubepods/besteffort/cpuset.mems' + #cat /sys/fs/cgroup/cpuset/kubepods/{,burstable/,besteffort/}cpuset.mems + #(migrate_pages both binary and syscall requires that /proc/self/status allows to be run on given numanode) rules: - name: Bind to numa node0