diff --git a/ci-metrics/job-completed.sh b/ci-metrics/job-completed.sh index dd036cf4..d4857175 100755 --- a/ci-metrics/job-completed.sh +++ b/ci-metrics/job-completed.sh @@ -144,8 +144,18 @@ else 'BEGIN{ if (p+0>0) printf "%.3f", q/p; }') fi -# Online CPU count — the denominator for utilisation when there is no cgroup quota -# (the common case for our runners). Overridable for deterministic tests. +# CPU-avg denominator candidates, in preference order: +# 1. cpu_limit_cores — hard cgroup quota (above), when set. +# 2. cpu_request_cores — the runner's CPU request from the Downward API env var +# CI_METRICS_CPU_REQUEST_MILLI (millicores). Exact; the right denominator for our +# burstable ARC runners (no quota, request set). Absent on WarpBuild. +# 3. cpu_online_count — nproc. Only correct when the runner owns the host (WarpBuild +# dedicated VM); on ARC this is the whole node, so it's the last resort. +if [[ "${CI_METRICS_CPU_REQUEST_MILLI:-}" =~ ^[0-9]+$ ]] && (( CI_METRICS_CPU_REQUEST_MILLI > 0 )); then + cpu_request_cores=$(awk -v m="$CI_METRICS_CPU_REQUEST_MILLI" 'BEGIN{printf "%.3f", m/1000}') +else + cpu_request_cores="" +fi cpu_online_count="${CI_METRICS_NPROC:-$(nproc 2>/dev/null || true)}" [[ "$cpu_online_count" =~ ^[0-9]+$ ]] || cpu_online_count="" @@ -298,7 +308,7 @@ for iface in "${!net_rx[@]}"; do done job_metrics_json=$(cat < "/ N cores (P%)" (hard limit) +# 2. CPU request -> "/ N cores requested (P%)" (ARC burstable; P can exceed 100% when bursting) +# 3. nproc -> "/ N cores available (P%)" (WarpBuild dedicated VM) +# Falls back to bare cores, then n/a. +# This rendering mirrors _rci_cpu_cell in report-ci-metrics/lib.sh; the expected output per tier +# is pinned by the "_rci_cpu_cell() denominator preference" spec — keep both in sync if changed. cpu_avg_cores="" if [[ -n "$cpu_usage_seconds" && -n "$duration_seconds" ]]; then cpu_avg_cores=$(awk -v u="$cpu_usage_seconds" -v d="$duration_seconds" \ @@ -361,11 +376,16 @@ if [[ -n "$cpu_usage_seconds" && -n "$duration_seconds" ]]; then fi if [[ -n "$cpu_avg_cores" && -n "$cpu_limit_cores" && -n "$cpu_avg_utilization" ]]; then v_cpu_avg="${cpu_avg_cores} / $(round2 "$cpu_limit_cores") cores ($(pct0 "$cpu_avg_utilization")%)" +elif [[ -n "$cpu_avg_cores" && -n "$cpu_request_cores" ]] \ + && awk -v r="$cpu_request_cores" 'BEGIN{exit !(r+0>0)}'; then + cpu_avg_pct=$(awk -v c="$cpu_avg_cores" -v r="$cpu_request_cores" \ + 'BEGIN{ printf "%.0f", (c/r)*100 }') + v_cpu_avg="${cpu_avg_cores} / $(round2 "$cpu_request_cores") cores requested (${cpu_avg_pct}%)" elif [[ -n "$cpu_avg_cores" && -n "$cpu_online_count" ]] \ && (( cpu_online_count > 0 )); then cpu_avg_pct=$(awk -v c="$cpu_avg_cores" -v n="$cpu_online_count" \ 'BEGIN{ printf "%.0f", (c/n)*100 }') - v_cpu_avg="${cpu_avg_cores} / ${cpu_online_count} cores (${cpu_avg_pct}%)" + v_cpu_avg="${cpu_avg_cores} / ${cpu_online_count} cores available (${cpu_avg_pct}%)" elif [[ -n "$cpu_avg_cores" ]]; then v_cpu_avg="${cpu_avg_cores} cores" else diff --git a/report-ci-metrics/lib.sh b/report-ci-metrics/lib.sh index 5d638e07..4e64b698 100644 --- a/report-ci-metrics/lib.sh +++ b/report-ci-metrics/lib.sh @@ -56,13 +56,12 @@ _rci_fmt_bytes() { fi } -# CPU-avg display cell for one job's JSON, mirroring the hook's step-summary logic: -# cores = usage_seconds / duration_seconds (2dp); -# denominator = limit_cores when avg_utilization is known, else online_count; -# " / cores (%)", falling back to bare cores, then "n/a". +# CPU-avg display cell for one job's JSON, mirroring the hook's step-summary logic. +# Denominator preference: limit_cores -> request_cores ("requested", ARC burstable) -> +# online_count ("available", WarpBuild VM) -> bare cores -> "n/a". _rci_cpu_cell() { local json=$1 - jq -r '.cgroup.cpu as $c | .duration_seconds as $d | if ($c.usage_seconds != null and $d != null and $d > 0) then (($c.usage_seconds / $d) * 100 | round / 100) as $cores | if ($c.limit_cores != null and $c.avg_utilization != null) then "\($cores) / \(($c.limit_cores*100|round)/100) cores (\(($c.avg_utilization*100)|round)%)" elif ($c.online_count != null and $c.online_count > 0) then "\($cores) / \($c.online_count) cores (\((($cores/$c.online_count)*100)|round)%)" else "\($cores) cores" end else "n/a" end' <<< "$json" + jq -r '.cgroup.cpu as $c | .duration_seconds as $d | if ($c.usage_seconds != null and $d != null and $d > 0) then (($c.usage_seconds / $d) * 100 | round / 100) as $cores | if ($c.limit_cores != null and $c.avg_utilization != null) then "\($cores) / \(($c.limit_cores*100|round)/100) cores (\(($c.avg_utilization*100)|round)%)" elif ($c.request_cores != null and $c.request_cores > 0) then "\($cores) / \(($c.request_cores*100|round)/100) cores requested (\((($cores/$c.request_cores)*100)|round)%)" elif ($c.online_count != null and $c.online_count > 0) then "\($cores) / \($c.online_count) cores available (\((($cores/$c.online_count)*100)|round)%)" else "\($cores) cores" end else "n/a" end' <<< "$json" } # Sum a numeric jq path across all record JSONs; nulls count as 0. Echoes an integer-ish sum. diff --git a/spec/report-ci-metrics_spec.sh b/spec/report-ci-metrics_spec.sh index 46aea662..2d15cc2f 100644 --- a/spec/report-ci-metrics_spec.sh +++ b/spec/report-ci-metrics_spec.sh @@ -181,11 +181,11 @@ Describe 'report-ci-metrics/lib.sh' End End - # schema_version 2 fixtures (one "\t" record each): + # schema_version 3 fixtures (one "\t" record each): # build = cache restored+saved, no flags; test = OOM-killed; lint = no disk, throttled. - J_BUILD='{"schema_version":2,"captured_at":"t","duration_seconds":62.0,"cgroup":{"cpu":{"usage_seconds":40.0,"throttled_seconds":0.0,"nr_throttled":0,"limit_cores":2.0,"online_count":4,"avg_utilization":0.32,"throttle_rate":null,"pressure_some_avg10":null,"pressure_some_avg60":null,"pressure_some_avg300":null},"memory":{"peak_bytes":3435973836,"limit_bytes":4294967296,"peak_utilization":0.80,"oom":0,"oom_kill":0}},"net":{"rx_bytes":1073741824,"tx_bytes":209715200,"by_interface":{}},"disk":{"path":"/","total_bytes":32212254720,"used_bytes":6442450944,"available_bytes":null,"utilization":0.20},"cache":[{"key":"maven-abc","cache_hit":true,"restore_key_hit":null,"backend":"s3","size_bytes_restored":471859200,"saved":true,"size_bytes_at_end":492832000}]}' - J_TEST='{"schema_version":2,"captured_at":"t","duration_seconds":62.0,"cgroup":{"cpu":{"usage_seconds":10.0,"throttled_seconds":0.0,"nr_throttled":0,"limit_cores":null,"online_count":4,"avg_utilization":null,"throttle_rate":null,"pressure_some_avg10":null,"pressure_some_avg60":null,"pressure_some_avg300":null},"memory":{"peak_bytes":104857600,"limit_bytes":4294967296,"peak_utilization":0.02,"oom":0,"oom_kill":1}},"net":{"rx_bytes":524288000,"tx_bytes":104857600,"by_interface":{}},"disk":{"path":"/","total_bytes":32212254720,"used_bytes":3221225472,"available_bytes":null,"utilization":0.10},"cache":[{"key":"npm-xyz","cache_hit":true,"restore_key_hit":null,"backend":"gha","size_bytes_restored":104857600,"saved":false,"size_bytes_at_end":null}]}' - J_LINT='{"schema_version":2,"captured_at":"t","duration_seconds":62.0,"cgroup":{"cpu":{"usage_seconds":2.0,"throttled_seconds":3.5,"nr_throttled":7,"limit_cores":2.0,"online_count":4,"avg_utilization":0.01,"throttle_rate":0.05,"pressure_some_avg10":null,"pressure_some_avg60":null,"pressure_some_avg300":null},"memory":{"peak_bytes":52428800,"limit_bytes":4294967296,"peak_utilization":0.01,"oom":0,"oom_kill":0}},"net":{"rx_bytes":0,"tx_bytes":0,"by_interface":{}},"disk":{"path":"/","total_bytes":null,"used_bytes":null,"available_bytes":null,"utilization":null}}' + J_BUILD='{"schema_version":3,"captured_at":"t","duration_seconds":62.0,"cgroup":{"cpu":{"usage_seconds":40.0,"throttled_seconds":0.0,"nr_throttled":0,"limit_cores":2.0,"request_cores":1.0,"online_count":4,"avg_utilization":0.32,"throttle_rate":null,"pressure_some_avg10":null,"pressure_some_avg60":null,"pressure_some_avg300":null},"memory":{"peak_bytes":3435973836,"limit_bytes":4294967296,"peak_utilization":0.80,"oom":0,"oom_kill":0}},"net":{"rx_bytes":1073741824,"tx_bytes":209715200,"by_interface":{}},"disk":{"path":"/","total_bytes":32212254720,"used_bytes":6442450944,"available_bytes":null,"utilization":0.20},"cache":[{"key":"maven-abc","cache_hit":true,"restore_key_hit":null,"backend":"s3","size_bytes_restored":471859200,"saved":true,"size_bytes_at_end":492832000}]}' + J_TEST='{"schema_version":3,"captured_at":"t","duration_seconds":62.0,"cgroup":{"cpu":{"usage_seconds":10.0,"throttled_seconds":0.0,"nr_throttled":0,"limit_cores":null,"request_cores":1.0,"online_count":4,"avg_utilization":null,"throttle_rate":null,"pressure_some_avg10":null,"pressure_some_avg60":null,"pressure_some_avg300":null},"memory":{"peak_bytes":104857600,"limit_bytes":4294967296,"peak_utilization":0.02,"oom":0,"oom_kill":1}},"net":{"rx_bytes":524288000,"tx_bytes":104857600,"by_interface":{}},"disk":{"path":"/","total_bytes":32212254720,"used_bytes":3221225472,"available_bytes":null,"utilization":0.10},"cache":[{"key":"npm-xyz","cache_hit":true,"restore_key_hit":null,"backend":"gha","size_bytes_restored":104857600,"saved":false,"size_bytes_at_end":null}]}' + J_LINT='{"schema_version":3,"captured_at":"t","duration_seconds":62.0,"cgroup":{"cpu":{"usage_seconds":2.0,"throttled_seconds":3.5,"nr_throttled":7,"limit_cores":2.0,"request_cores":1.0,"online_count":4,"avg_utilization":0.01,"throttle_rate":0.05,"pressure_some_avg10":null,"pressure_some_avg60":null,"pressure_some_avg300":null},"memory":{"peak_bytes":52428800,"limit_bytes":4294967296,"peak_utilization":0.01,"oom":0,"oom_kill":0}},"net":{"rx_bytes":0,"tx_bytes":0,"by_interface":{}},"disk":{"path":"/","total_bytes":null,"used_bytes":null,"available_bytes":null,"utilization":null}}' Describe 'render_headline()' It 'renders correct totals: CPU-seconds, worst peak mem with job, net, cache' @@ -289,6 +289,33 @@ Describe 'report-ci-metrics/lib.sh' End End + Describe '_rci_cpu_cell() denominator preference' + # cores = usage/duration. Denominator: limit -> request -> online_count -> bare. + It 'uses the cgroup limit when present' + j='{"duration_seconds":62.0,"cgroup":{"cpu":{"usage_seconds":31.0,"limit_cores":2.0,"avg_utilization":0.25,"request_cores":1.0,"online_count":32}}}' + When call _rci_cpu_cell "$j" + The output should equal '0.5 / 2 cores (25%)' + End + + It 'uses the CPU request (not nproc) when there is no limit — the ARC bug fix' + j='{"duration_seconds":62.0,"cgroup":{"cpu":{"usage_seconds":4.34,"limit_cores":null,"avg_utilization":null,"request_cores":1.0,"online_count":32}}}' + When call _rci_cpu_cell "$j" + The output should equal '0.07 / 1 cores requested (7%)' + End + + It 'falls back to online_count (available) on WarpBuild when no limit/request' + j='{"duration_seconds":62.0,"cgroup":{"cpu":{"usage_seconds":31.0,"limit_cores":null,"avg_utilization":null,"request_cores":null,"online_count":8}}}' + When call _rci_cpu_cell "$j" + The output should equal '0.5 / 8 cores available (6%)' + End + + It 'shows bare cores when no denominator is available' + j='{"duration_seconds":62.0,"cgroup":{"cpu":{"usage_seconds":31.0,"limit_cores":null,"avg_utilization":null,"request_cores":null,"online_count":null}}}' + When call _rci_cpu_cell "$j" + The output should equal '0.5 cores' + End + End + Describe 'render_cache_fold()' It 'renders a cache block when at least one job has a cache entry' records=$(printf '%s\t%s\n' 'build' "$J_BUILD" 'test' "$J_TEST" 'lint' "$J_LINT")