diff --git a/Runner/suites/Performance/Sysbench_Performance/run.sh b/Runner/suites/Performance/Sysbench_Performance/run.sh index 64844724..4afda1e8 100755 --- a/Runner/suites/Performance/Sysbench_Performance/run.sh +++ b/Runner/suites/Performance/Sysbench_Performance/run.sh @@ -255,16 +255,18 @@ log_info "FILEIO dir=$FILEIO_DIR total=$FILEIO_TOTAL_SIZE blk=$FILEIO_BLOCK_SIZE perf_clock_sanity_warn 2>/dev/null || true # ---------------- deps check ---------------- -deps_list="sysbench awk sed grep date mkfifo tee" if [ -n "${TASKSET_CPU_LIST:-}" ]; then - deps_list="$deps_list taskset" -fi - -# Use single call (no loop) -if ! check_dependencies "$deps_list"; then - log_skip "$TESTNAME SKIP - missing one or more dependencies: $deps_list" - echo "$TESTNAME SKIP" >"$RES_FILE" - exit 0 + if ! check_dependencies sysbench awk sed grep date mkfifo tee taskset; then + log_skip "$TESTNAME SKIP - missing one or more dependencies" + echo "$TESTNAME SKIP" >"$RES_FILE" + exit 0 + fi +else + if ! check_dependencies sysbench awk sed grep date mkfifo tee; then + log_skip "$TESTNAME SKIP - missing one or more dependencies" + echo "$TESTNAME SKIP" >"$RES_FILE" + exit 0 + fi fi set_performance_governor 2>/dev/null || true diff --git a/Runner/suites/Performance/Tiotest/README.md b/Runner/suites/Performance/Tiotest/README.md new file mode 100644 index 00000000..165c70f2 --- /dev/null +++ b/Runner/suites/Performance/Tiotest/README.md @@ -0,0 +1,272 @@ +# Tiotest + +Tiotest storage KPI runner for Yocto/LE images, aligned to qcom-linux-testkit performance suite conventions. + +It runs the requested tiotest modes (sequential + random) for one or more thread counts, for N iterations: +- `seqwr` Sequential Write +- `seqrd` Sequential Read +- `rndwr` Random Write +- `rndrd` Random Read + +Outputs: +- Per-run logs: `OUT_DIR/tiotest__t_iter.log` +- Metrics TSV: `OUT_DIR/tiotest_metrics.tsv` +- Summary: `OUT_DIR/tiotest_summary.txt` +- Result file: `./Tiotest.res` (OVERWRITTEN, not appended; LAVA-friendly) + +> Note: The `tiotest -h` shown on RB3GEN2 (v0.3.3) does NOT include `--direct-io`. +> Some internal forks may have extra flags; this runner follows your target help output and keeps everything non-hardcoded. + +--- + +## Prerequisites + +1. `tiotest` binary available on target (in PATH or pass `--tiotest-bin /path/to/tiotest`) +2. A writable directory on the storage you want to benchmark (filesystem mode), OR a block device (raw mode). +3. Recommended: keep system idle and (optionally) use performance governor for more stable KPIs. + +Dependencies (checked by script): +- `awk sed grep date tee` +- `tiotest` (or the path you pass) + +--- + +## Quick Start (filesystem mode) + +Run on a disk-backed path (recommended, avoids tmpfs variance): + +```sh +cd Runner/suites/Performance/Tiotest + +./run.sh \ + --out-dir ./tiotest_out \ + --iterations 3 \ + --threads-list "1 4" \ + --tiotest-bin tiotest \ + --use-raw 0 \ + --tiotest-dir /var/tmp/tiotest_fileio \ + --mode-list "seqwr seqrd rndwr rndrd" \ + --seq-block 524288 \ + --seq-file-mb 1024 \ + --rnd-block 4096 \ + --rnd-file-mb 1 \ + --rnd-ops 12500 \ + --hide-latency 1 \ + --drop-caches 0 \ + --set-perf-gov 1 \ + --require-non-tmpfs 1 +``` + +Result: +- `Tiotest.res` contains **one line**: `Tiotest PASS|FAIL|SKIP` +- Summary at `./tiotest_out/tiotest_summary.txt` + +--- + +## Raw Device Mode (USE_RAW=1) + +Raw mode uses tiotest `-R` and expects `--tiotest-dir` to be a **block device**: + +```sh +./run.sh \ + --out-dir ./tiotest_out_raw \ + --iterations 2 \ + --threads-list "1" \ + --tiotest-bin tiotest \ + --use-raw 1 \ + --tiotest-dir /dev/sda \ + --offset-mb 0 \ + --offset-first 0 \ + --mode-list "seqwr seqrd rndwr rndrd" +``` + +Options: +- `--offset-mb N` corresponds to `-o N` (offset between threads) when using `-R` +- `--offset-first 1` corresponds to `-O` (apply offset to first thread as well) + +> Caution: Raw mode can stress the device. Ensure you are using the correct target block device. + +--- + +## Running Only a Subset of Modes + +Example: sequential only + +```sh +./run.sh --mode-list "seqwr seqrd" +``` + +Example: random only + +```sh +./run.sh --mode-list "rndwr rndrd" +``` + +--- + +## Changing Block Size / File Size + +Sequential 1MB block size (per thread 1GB): + +```sh +./run.sh --seq-block 1048576 --seq-file-mb 1024 +``` + +Random 4KB blocks, larger file per thread, more ops: + +```sh +./run.sh --rnd-block 4096 --rnd-file-mb 64 --rnd-ops 12500 +``` + +--- + +## Latency Output + +Your tiotest supports: +- `-L` hide latency output + +Runner control: +- `--hide-latency 1` => adds `-L` +- `--hide-latency 0` => do not add `-L` (latency may be printed if tiotest emits it) + +The runner also supports an optional strict check: +- If latency is enabled (hide-latency != 1) and `perf_tiotest_latency_strict_check()` exists in `lib_performance.sh`, + the run can FAIL if `% >2 sec` or `% >10 sec` becomes non-zero. + +--- + +## Optional Baseline Gating + +If you provide a baseline file, the runner can evaluate average KPIs vs baseline with allowed deviation. + +- Baseline auto-detect: `./tiotest_baseline.conf` (same folder as `run.sh`) +- Or pass: `--baseline /path/to/tiotest_baseline.conf` +- Control deviation: `--delta 0.10` (10%) + +Example: + +```sh +./run.sh \ + --baseline ./tiotest_baseline.conf \ + --delta 0.10 +``` + +If gating fails: +- `.res` will be `Tiotest FAIL` +- exit code `1` (LAVA will still collect logs, and your YAML uses `|| true` if desired) + +--- + +## Output artifacts + +All artifacts are written under `--out-dir`: + +- `tiotest_summary.txt`: final human-readable summary (also printed to stdout) +- `tiotest_metrics.tsv`: per-iteration machine-readable metrics +- per-metric `.values` files (one value per iteration) used for averaging/gating +- `tiotest_seq_t_iter.log` and `tiotest_rnd_t_iter.log`: raw tiotest logs per iteration + +### tiotest_metrics.tsv format + +`tiotest_metrics.tsv` always has **8 tab-separated columns**: + +``` +mode threads mbps iops latavg_ms latmax_ms pct_gt2s pct_gt10s +``` + +Example: + +``` +rndrd 4 3127.443 800625 0.003 0.133 0.00000 0.00000 +``` + +## Baseline and gating + +The runner can gate measured averages against a baseline file (default: `tiotest_baseline.conf`). + +### Baseline file format + +Key-value format compatible with `perf_baseline_get_value()`: + +``` +# tiotest...baseline=... +# tiotest...goal=... +# tiotest...op=>=|<=|>|<|== + +tiotest.1.seqwr_mbps.baseline=180 +tiotest.1.seqwr_mbps.goal=180 +tiotest.1.seqwr_mbps.op=>= + +tiotest.1.seqrd_mbps.baseline=800 +tiotest.1.seqrd_mbps.goal=800 +tiotest.1.seqrd_mbps.op=>= + +tiotest.1.rndwr_mbps.baseline=45 +tiotest.1.rndwr_mbps.goal=45 +tiotest.1.rndwr_mbps.op=>= + +tiotest.1.rndwr_iops.baseline=11000 +tiotest.1.rndwr_iops.goal=11000 +tiotest.1.rndwr_iops.op=>= + +tiotest.1.rndrd_mbps.baseline=50 +tiotest.1.rndrd_mbps.goal=50 +tiotest.1.rndrd_mbps.op=>= + +tiotest.1.rndrd_iops.baseline=12000 +tiotest.1.rndrd_iops.goal=12000 +tiotest.1.rndrd_iops.op=>= + +tiotest.4.seqwr_mbps.baseline=500 +tiotest.4.seqwr_mbps.goal=500 +tiotest.4.seqwr_mbps.op=>= + +tiotest.4.seqrd_mbps.baseline=1200 +tiotest.4.seqrd_mbps.goal=1200 +tiotest.4.seqrd_mbps.op=>= + +tiotest.4.rndwr_mbps.baseline=80 +tiotest.4.rndwr_mbps.goal=80 +tiotest.4.rndwr_mbps.op=>= + +tiotest.4.rndwr_iops.baseline=30000 +tiotest.4.rndwr_iops.goal=30000 +tiotest.4.rndwr_iops.op=>= + +tiotest.4.rndrd_mbps.baseline=90 +tiotest.4.rndrd_mbps.goal=90 +tiotest.4.rndrd_mbps.op=>= + +tiotest.4.rndrd_iops.baseline=32000 +tiotest.4.rndrd_iops.goal=32000 +tiotest.4.rndrd_iops.op=>= +``` + +### Goal and delta behavior + +If `.goal` is missing, it can be derived from `.baseline` and `ALLOWED_DEVIATION` (delta) depending on the operator: + +- `>=` / `>`: `goal = baseline * (1 - delta)` +- `<=` / `<`: `goal = baseline * (1 + delta)` +- `==`: `goal = baseline` + +Gating output is logged and also appended to the summary. The summary prints `goal${op}${goal}` so you will see +strings like `goal=>=90` or `goal>90` exactly. + +## LAVA Usage + +Use the test definition YAML: `Tiotest.yaml` + +Typical steps: +- `cd Runner/suites/Performance/Tiotest/` +- Run `./run.sh ...` +- Send `.res` with: + `Runner/utils/send-to-lava.sh Tiotest.res` + +--- + +## Tips for Stable KPI Numbers + +- Use a disk-backed directory (e.g., `/var/tmp/...` or your storage mount), not tmpfs. +- Keep device idle; run 2–3 iterations and compare variance. +- Consider performance governor (`--set-perf-gov 1`) if supported on your platform. diff --git a/Runner/suites/Performance/Tiotest/run.sh b/Runner/suites/Performance/Tiotest/run.sh new file mode 100755 index 00000000..518b7062 --- /dev/null +++ b/Runner/suites/Performance/Tiotest/run.sh @@ -0,0 +1,471 @@ +#!/bin/sh +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear + +SCRIPT_DIR="$( + cd "$(dirname "$0")" || exit 1 + pwd +)" + +TESTNAME="Tiotest" +RES_FILE="./${TESTNAME}.res" + +# ---------------- Defaults (env overrides; CLI can override further) ---------------- +OUT_DIR="${OUT_DIR:-./tiotest_out}" + +ITERATIONS="${ITERATIONS:-1}" +THREADS_LIST="${THREADS_LIST:-1 4}" + +TIOTEST_DIR="${TIOTEST_DIR:-}" +USE_RAW="${USE_RAW:-0}" +OFFSET_MB="${OFFSET_MB:-}" +OFFSET_FIRST="${OFFSET_FIRST:-0}" + +SEQ_BLOCK_SIZE="${SEQ_BLOCK_SIZE:-524288}" +SEQ_FILE_SIZE_MB="${SEQ_FILE_SIZE_MB:-1024}" + +RND_BLOCK_SIZE="${RND_BLOCK_SIZE:-4096}" +RND_FILE_SIZE_MB="${RND_FILE_SIZE_MB:-1}" +RND_OPS="${RND_OPS:-12500}" + +MODE_LIST="${MODE_LIST:-seqwr seqrd rndwr rndrd}" + +HIDE_LATENCY="${HIDE_LATENCY:-1}" +TERSE="${TERSE:-0}" +SEQ_WRITE_PHASE="${SEQ_WRITE_PHASE:-0}" +SYNC_WRITES="${SYNC_WRITES:-0}" +CONSISTENCY="${CONSISTENCY:-0}" +DEBUG_LEVEL="${DEBUG_LEVEL:-}" + +DROP_CACHES="${DROP_CACHES:-0}" +SET_PERF_GOV="${SET_PERF_GOV:-1}" +REQUIRE_NON_TMPFS="${REQUIRE_NON_TMPFS:-1}" + +BASELINE_FILE="${BASELINE_FILE:-}" +ALLOWED_DEVIATION="${ALLOWED_DEVIATION:-0.10}" + +STANDALONE="${STANDALONE:-0}" + +TIOTEST_BIN="${TIOTEST_BIN:-tiotest}" + +usage() { + cat <&2 + usage + exit 2 + ;; + *) + echo "[ERROR] Unexpected argument: $1" >&2 + usage + exit 2 + ;; + esac +done + +# ---------------- locate and source init_env → functestlib.sh + lib_performance.sh ---------------- +if [ "$STANDALONE" = "1" ]; then + : "${TOOLS:=}" +else + INIT_ENV="" + SEARCH="$SCRIPT_DIR" + while [ "$SEARCH" != "/" ]; do + if [ -f "$SEARCH/init_env" ]; then + INIT_ENV="$SEARCH/init_env" + break + fi + SEARCH=$(dirname "$SEARCH") + done + + if [ -z "$INIT_ENV" ]; then + echo "[ERROR] Could not find init_env (starting at $SCRIPT_DIR). Use --standalone 1 to bypass." >&2 + exit 1 + fi + + if [ -z "${__INIT_ENV_LOADED:-}" ]; then + # shellcheck disable=SC1090 + . "$INIT_ENV" + __INIT_ENV_LOADED=1 + fi +fi + +if [ -z "${TOOLS:-}" ] || [ ! -f "$TOOLS/functestlib.sh" ] || [ ! -f "$TOOLS/lib_performance.sh" ]; then + echo "[ERROR] Missing TOOLS/functestlib.sh or TOOLS/lib_performance.sh. (TOOLS=$TOOLS)" >&2 + exit 1 +fi + +# shellcheck disable=SC1091 +. "$TOOLS/functestlib.sh" +# shellcheck disable=SC1091 +. "$TOOLS/lib_performance.sh" + +: >"$RES_FILE" +mkdir -p "$OUT_DIR" 2>/dev/null || true + +# Baseline auto-detect +if [ -z "$BASELINE_FILE" ] && [ -f "$SCRIPT_DIR/tiotest_baseline.conf" ]; then + BASELINE_FILE="$SCRIPT_DIR/tiotest_baseline.conf" +fi + +# Auto-pick TIOTEST_DIR if not set +if [ -z "$TIOTEST_DIR" ]; then + if [ "$USE_RAW" = "1" ]; then + TIOTEST_DIR="/dev/sda" + else + if [ -d /var/tmp ] && [ -w /var/tmp ]; then + TIOTEST_DIR="/var/tmp/tiotest_fileio" + else + TIOTEST_DIR="/tmp/tiotest_fileio" + fi + fi +fi + +if [ -n "$BASELINE_FILE" ] && [ ! -f "$BASELINE_FILE" ]; then + log_warn "Baseline file set but not found: $BASELINE_FILE (will run report-only)" + BASELINE_FILE="" +fi + +cleanup() { + restore_governor 2>/dev/null || true +} +trap cleanup EXIT + +# Clock sanity +if command -v ensure_reasonable_clock >/dev/null 2>&1; then + log_info "Ensuring system clock is reasonable before tiotest..." + if ! ensure_reasonable_clock; then + log_error "Clock is not reasonable; benchmark timestamps/gating may be invalid." + fi +else + log_info "ensure_reasonable_clock() not available, continuing." +fi + +log_info "Tiotest runner starting" +log_info "OUTDIR=$OUT_DIR BASELINE=${BASELINE_FILE:-none} DELTA=$ALLOWED_DEVIATION ITERATIONS=$ITERATIONS" +log_info "THREADS_LIST=$THREADS_LIST MODE_LIST=$MODE_LIST" +log_info "TIOTEST_BIN=$TIOTEST_BIN USE_RAW=$USE_RAW TIOTEST_DIR=$TIOTEST_DIR offset_mb=${OFFSET_MB:-none} offset_first=$OFFSET_FIRST" +log_info "SEQ blk=$SEQ_BLOCK_SIZE file_mb=$SEQ_FILE_SIZE_MB RND blk=$RND_BLOCK_SIZE file_mb=$RND_FILE_SIZE_MB ops=$RND_OPS" +log_info "FLAGS hide_lat=$HIDE_LATENCY terse=$TERSE W=$SEQ_WRITE_PHASE S=$SYNC_WRITES c=$CONSISTENCY D=${DEBUG_LEVEL:-none}" +log_info "STABILITY drop_caches=$DROP_CACHES set_perf_gov=$SET_PERF_GOV require_non_tmpfs=$REQUIRE_NON_TMPFS" + +perf_clock_sanity_warn 2>/dev/null || true + +# ---------------- deps check ---------------- +if ! check_dependencies awk sed grep date tee "$TIOTEST_BIN"; then + log_skip "$TESTNAME SKIP - missing one or more dependencies: awk sed grep date tee $TIOTEST_BIN" + echo "$TESTNAME SKIP" >"$RES_FILE" + exit 0 +fi + +if [ "$SET_PERF_GOV" = "1" ]; then + set_performance_governor 2>/dev/null || true +fi + +# Path checks +if [ "$USE_RAW" = "1" ]; then + if [ ! -b "$TIOTEST_DIR" ]; then + log_skip "$TESTNAME SKIP - USE_RAW=1 but TIOTEST_DIR not a block device: $TIOTEST_DIR" + echo "$TESTNAME SKIP" >"$RES_FILE" + exit 0 + fi +else + if ! mkdir -p "$TIOTEST_DIR" 2>/dev/null; then + log_skip "$TESTNAME SKIP - TIOTEST_DIR not creatable: $TIOTEST_DIR" + echo "$TESTNAME SKIP" >"$RES_FILE" + exit 0 + fi + if [ ! -w "$TIOTEST_DIR" ]; then + log_skip "$TESTNAME SKIP - TIOTEST_DIR not writable: $TIOTEST_DIR" + echo "$TESTNAME SKIP" >"$RES_FILE" + exit 0 + fi + if [ "$REQUIRE_NON_TMPFS" = "1" ] && tiotest_is_tmpfs_path "$TIOTEST_DIR"; then + log_skip "$TESTNAME SKIP - TIOTEST_DIR appears tmpfs: $TIOTEST_DIR" + echo "$TESTNAME SKIP" >"$RES_FILE" + exit 0 + fi +fi + +suite_rc=0 +gate_fail=0 + +summary="$OUT_DIR/tiotest_summary.txt" +metrics="$OUT_DIR/tiotest_metrics.tsv" +: >"$summary" +: >"$metrics" + +ITER_LIST=$(awk -v n="$ITERATIONS" 'BEGIN{for (k=1;k<=n;k++) printf "%d ", k}') + +# Determine if we need seq and/or rnd runs +NEED_SEQ=0 +NEED_RND=0 +for m in $MODE_LIST; do + case "$m" in + seqwr|seqrd) NEED_SEQ=1 ;; + rndwr|rndrd) NEED_RND=1 ;; + esac +done + +# If latency is hidden, don't track/print latency fields (avoids "unknown") +WANT_LAT=1 +if [ "$HIDE_LATENCY" = "1" ]; then + WANT_LAT=0 +fi + +# Pre-create per-mode value files only for requested modes +for tt in $THREADS_LIST; do + for mode in $MODE_LIST; do + : >"$OUT_DIR/${mode}_mbps_t${tt}.values" 2>/dev/null || true + : >"$OUT_DIR/${mode}_iops_t${tt}.values" 2>/dev/null || true + if [ "$WANT_LAT" = "1" ]; then + : >"$OUT_DIR/${mode}_latavg_t${tt}.values" 2>/dev/null || true + : >"$OUT_DIR/${mode}_latmax_t${tt}.values" 2>/dev/null || true + : >"$OUT_DIR/${mode}_pct2s_t${tt}.values" 2>/dev/null || true + : >"$OUT_DIR/${mode}_pct10s_t${tt}.values" 2>/dev/null || true + fi + done +done + +for tt in $THREADS_LIST; do + for it in $ITER_LIST; do + if [ "$DROP_CACHES" = "1" ]; then + tiotest_drop_caches_best_effort 2>/dev/null || true + fi + + if [ "$NEED_SEQ" = "1" ]; then + logf_seq="$OUT_DIR/tiotest_seq_t${tt}_iter${it}.log" + if ! perf_tiotest_run_seq_pair \ + "$TIOTEST_BIN" "$tt" "$TIOTEST_DIR" "$USE_RAW" \ + "$SEQ_BLOCK_SIZE" "$SEQ_FILE_SIZE_MB" \ + "$HIDE_LATENCY" "$TERSE" "$SEQ_WRITE_PHASE" "$SYNC_WRITES" "$CONSISTENCY" "$DEBUG_LEVEL" \ + "$OFFSET_MB" "$OFFSET_FIRST" \ + "$logf_seq" "$metrics" + then + log_warn "tiotest: seq pair metrics missing threads=$tt iter=$it (see $logf_seq)" + suite_rc=1 + fi + fi + + if [ "$NEED_RND" = "1" ]; then + logf_rnd="$OUT_DIR/tiotest_rnd_t${tt}_iter${it}.log" + if ! perf_tiotest_run_rnd_pair \ + "$TIOTEST_BIN" "$tt" "$TIOTEST_DIR" "$USE_RAW" \ + "$RND_BLOCK_SIZE" "$RND_FILE_SIZE_MB" "$RND_OPS" \ + "$HIDE_LATENCY" "$TERSE" "$SEQ_WRITE_PHASE" "$SYNC_WRITES" "$CONSISTENCY" "$DEBUG_LEVEL" \ + "$OFFSET_MB" "$OFFSET_FIRST" \ + "$logf_rnd" "$metrics" + then + log_warn "tiotest: rnd metrics missing threads=$tt iter=$it (see $logf_rnd)" + suite_rc=1 + fi + fi + + # Emit per-mode ITER summary + append values + for mode in $MODE_LIST; do + mbps=$(awk -v m="$mode" -v t="$tt" '($1==m && $2==t){v=$3} END{print v}' "$metrics" 2>/dev/null) + iops=$(awk -v m="$mode" -v t="$tt" '($1==m && $2==t){v=$4} END{print v}' "$metrics" 2>/dev/null) + + perf_append_if_number "$OUT_DIR/${mode}_mbps_t${tt}.values" "$mbps" + perf_append_if_number "$OUT_DIR/${mode}_iops_t${tt}.values" "$iops" + + if [ "$WANT_LAT" = "1" ]; then + lat_avg=$(awk -v m="$mode" -v t="$tt" '($1==m && $2==t){v=$5} END{print v}' "$metrics" 2>/dev/null) + lat_max=$(awk -v m="$mode" -v t="$tt" '($1==m && $2==t){v=$6} END{print v}' "$metrics" 2>/dev/null) + pct2=$(awk -v m="$mode" -v t="$tt" '($1==m && $2==t){v=$7} END{print v}' "$metrics" 2>/dev/null) + pct10=$(awk -v m="$mode" -v t="$tt" '($1==m && $2==t){v=$8} END{print v}' "$metrics" 2>/dev/null) + + perf_append_if_number "$OUT_DIR/${mode}_latavg_t${tt}.values" "$lat_avg" + perf_append_if_number "$OUT_DIR/${mode}_latmax_t${tt}.values" "$lat_max" + perf_append_if_number "$OUT_DIR/${mode}_pct2s_t${tt}.values" "$pct2" + perf_append_if_number "$OUT_DIR/${mode}_pct10s_t${tt}.values" "$pct10" + + log_info "ITER_SUMMARY threads=$tt iter=$it/$ITERATIONS mode=$mode mbps=$(perf_norm_metric "$mbps") iops=$(perf_norm_metric "$iops") lat_avg_ms=$(perf_norm_metric "$lat_avg") lat_max_ms=$(perf_norm_metric "$lat_max") pct_gt2s=$(perf_norm_metric "$pct2") pct_gt10s=$(perf_norm_metric "$pct10")" + else + log_info "ITER_SUMMARY threads=$tt iter=$it/$ITERATIONS mode=$mode mbps=$(perf_norm_metric "$mbps") iops=$(perf_norm_metric "$iops") lat=hidden" + fi + done + done + + # -------- Averages per thread -------- + { + echo "Threads=$tt" + for mode in $MODE_LIST; do + a_mbps=$(perf_values_avg "$OUT_DIR/${mode}_mbps_t${tt}.values") + a_iops=$(perf_values_avg "$OUT_DIR/${mode}_iops_t${tt}.values") + + echo " ${mode}_avg_mbps : $(perf_norm_metric "$a_mbps")" + echo " ${mode}_avg_iops : $(perf_norm_metric "$a_iops")" + + if [ "$WANT_LAT" = "1" ]; then + a_latavg=$(perf_values_avg "$OUT_DIR/${mode}_latavg_t${tt}.values") + a_latmax=$(perf_values_avg "$OUT_DIR/${mode}_latmax_t${tt}.values") + a_pct2=$(perf_values_avg "$OUT_DIR/${mode}_pct2s_t${tt}.values") + a_pct10=$(perf_values_avg "$OUT_DIR/${mode}_pct10s_t${tt}.values") + + echo " ${mode}_avg_lat_avg_ms : $(perf_norm_metric "$a_latavg")" + echo " ${mode}_avg_lat_max_ms : $(perf_norm_metric "$a_latmax")" + echo " ${mode}_avg_pct_gt2s : $(perf_norm_metric "$a_pct2")" + echo " ${mode}_avg_pct_gt10s : $(perf_norm_metric "$a_pct10")" + fi + done + } >>"$summary" + + # -------- Baseline gating (if helper exists) -------- + if [ -n "$BASELINE_FILE" ] && command -v perf_tiotest_gate_eval_line_safe >/dev/null 2>&1; then + for mode in $MODE_LIST; do + a_mbps=$(perf_values_avg "$OUT_DIR/${mode}_mbps_t${tt}.values") + a_iops=$(perf_values_avg "$OUT_DIR/${mode}_iops_t${tt}.values") + + if [ -n "$a_mbps" ]; then + line=$(perf_tiotest_gate_eval_line_safe "$BASELINE_FILE" "tiotest" "$tt" "${mode}_mbps" "$a_mbps" "$ALLOWED_DEVIATION") + rc=$? + base=$(kv "$line" "baseline"); goal=$(kv "$line" "goal"); op=$(kv "$line" "op"); score=$(kv "$line" "score_pct"); status=$(kv "$line" "status") + log_info "GATE tiotest ${mode}_mbps threads=$tt avg=$a_mbps baseline=${base:-NA} goal=${op}${goal:-NA} delta=$ALLOWED_DEVIATION score_pct=${score:-NA} status=${status:-NA}" + echo " gate_${mode}_mbps : status=${status:-NA} baseline=${base:-NA} goal=${op}${goal:-NA} score_pct=${score:-NA} delta=$ALLOWED_DEVIATION" >>"$summary" + [ "$rc" -eq 1 ] && gate_fail=1 + fi + + case "$mode" in + rndwr|rndrd) + if [ -n "$a_iops" ]; then + line=$(perf_tiotest_gate_eval_line_safe "$BASELINE_FILE" "tiotest" "$tt" "${mode}_iops" "$a_iops" "$ALLOWED_DEVIATION") + rc=$? + base=$(kv "$line" "baseline"); goal=$(kv "$line" "goal"); op=$(kv "$line" "op"); score=$(kv "$line" "score_pct"); status=$(kv "$line" "status") + log_info "GATE tiotest ${mode}_iops threads=$tt avg=$a_iops baseline=${base:-NA} goal=${op}${goal:-NA} delta=$ALLOWED_DEVIATION score_pct=${score:-NA} status=${status:-NA}" + echo " gate_${mode}_iops : status=${status:-NA} baseline=${base:-NA} goal=${op}${goal:-NA} score_pct=${score:-NA} delta=$ALLOWED_DEVIATION" >>"$summary" + [ "$rc" -eq 1 ] && gate_fail=1 + fi + ;; + esac + done + fi + + echo >>"$summary" +done + +log_info "Final summary written → $summary" +log_info "----- TIOTEST SUMMARY (stdout) -----" +cat "$summary" || true +log_info "----- END SUMMARY -----" + +# latency strict check applies only when we actually collected latency +if [ "$HIDE_LATENCY" != "1" ]; then + if command -v perf_tiotest_latency_strict_check >/dev/null 2>&1; then + if ! perf_tiotest_latency_strict_check "$summary"; then + log_fail "$TESTNAME FAIL - latency threshold breach" + echo "$TESTNAME FAIL" >"$RES_FILE" + exit 1 + fi + fi +fi + +if [ -n "$BASELINE_FILE" ] && [ "$gate_fail" -ne 0 ]; then + log_fail "$TESTNAME FAIL - one or more KPIs did not meet baseline thresholds" + echo "$TESTNAME FAIL" >"$RES_FILE" + exit 1 +fi + +log_pass "$TESTNAME PASS" +echo "$TESTNAME PASS" >"$RES_FILE" +exit "$suite_rc" diff --git a/Runner/suites/Performance/Tiotest/tiotest-performance.yaml b/Runner/suites/Performance/Tiotest/tiotest-performance.yaml new file mode 100755 index 00000000..ea1c4974 --- /dev/null +++ b/Runner/suites/Performance/Tiotest/tiotest-performance.yaml @@ -0,0 +1,102 @@ +metadata: + name: tiotest-performance + format: "Lava-Test Test Definition 1.0" + description: > + Tiotest storage performance suite (sequential + random read/write) + with per-iteration logs, summary averages, baseline gating (optional), + and latency collection (optional). Designed for Yocto / LE platforms. + os: + - linux + scope: + - performance + - functional + +params: + # Output (LAVA will collect everything under this dir if you add it to artifacts) + OUT_DIR: "./logs_Tiotest" + + # Iteration controls + ITERATIONS: "3" + THREADS_LIST: "1 4" + + # tiotest binary + TIOTEST_BIN: "tiotest" + + # Storage target + TIOTEST_DIR: "/var/tmp/tiotest_fileio" + USE_RAW: "0" # 0 => filesystem, 1 => raw block device + OFFSET_MB: "" # optional (--offset-mb) + OFFSET_FIRST: "0" # (--offset-first) + + # Modes to run + MODE_LIST: "seqwr seqrd rndwr rndrd" + + # Sequential parameters + SEQ_BLOCK_SIZE: "524288" # 512 KB + SEQ_FILE_SIZE_MB: "1024" # 1 GB per thread + + # Random parameters + RND_BLOCK_SIZE: "4096" # 4 KB + RND_FILE_SIZE_MB: "1" # MB + RND_OPS: "12500" + + # tiotest flags + # NOTE: set to "0" to ENABLE latency output and populate latency columns in metrics. + HIDE_LATENCY: "0" # --hide-latency (0 => show latency, 1 => hide latency) + TERSE: "0" # --terse + SEQ_WRITE_PHASE: "0" # --seq-write-phase + SYNC_WRITES: "0" # --sync-writes + CONSISTENCY: "0" # --consistency + DEBUG_LEVEL: "" # --debug-level + + # Stability / environment + DROP_CACHES: "1" + SET_PERF_GOV: "1" + REQUIRE_NON_TMPFS: "1" + + # Baseline / gating (optional) + # Point this to tiotest_baseline.conf if you want PASS/FAIL gating against thresholds. + BASELINE_FILE: "" # e.g. "/tmp/Runner/suites/Performance/Tiotest/tiotest_baseline.conf" + ALLOWED_DEVIATION: "0.10" # used when goal missing: derived from baseline +/- delta + + # Standalone mode (no init_env) + STANDALONE: "0" + +run: + steps: + - REPO_PATH=$PWD + - cd Runner/suites/Performance/Tiotest/ || exit 0 + + # Run runner; it is LAVA-friendly and should not hard-fail the job. + - > + ./run.sh + --out-dir "$OUT_DIR" + --iterations "$ITERATIONS" + --threads-list "$THREADS_LIST" + --tiotest-bin "$TIOTEST_BIN" + --tiotest-dir "$TIOTEST_DIR" + --use-raw "$USE_RAW" + --offset-mb "$OFFSET_MB" + --offset-first "$OFFSET_FIRST" + --mode-list "$MODE_LIST" + --seq-block "$SEQ_BLOCK_SIZE" + --seq-file-mb "$SEQ_FILE_SIZE_MB" + --rnd-block "$RND_BLOCK_SIZE" + --rnd-file-mb "$RND_FILE_SIZE_MB" + --rnd-ops "$RND_OPS" + --hide-latency "$HIDE_LATENCY" + --terse "$TERSE" + --seq-write-phase "$SEQ_WRITE_PHASE" + --sync-writes "$SYNC_WRITES" + --consistency "$CONSISTENCY" + --debug-level "$DEBUG_LEVEL" + --drop-caches "$DROP_CACHES" + --set-perf-gov "$SET_PERF_GOV" + --require-non-tmpfs "$REQUIRE_NON_TMPFS" + --baseline "$BASELINE_FILE" + --delta "$ALLOWED_DEVIATION" + --standalone "$STANDALONE" + || true + + # Emit result to LAVA (expects Tiotest.res in current directory) + - $REPO_PATH/Runner/utils/send-to-lava.sh Tiotest.res diff --git a/Runner/suites/Performance/Tiotest/tiotest_baseline.conf b/Runner/suites/Performance/Tiotest/tiotest_baseline.conf new file mode 100755 index 00000000..94d9522b --- /dev/null +++ b/Runner/suites/Performance/Tiotest/tiotest_baseline.conf @@ -0,0 +1,52 @@ +# Tiotest baseline (key=value), compatible with perf_baseline_get_value() +# Format: +# tiotest...baseline=... +# tiotest...goal=... +# tiotest...op=>=|<=|... +tiotest.1.seqwr_mbps.baseline=180 +tiotest.1.seqwr_mbps.goal=180 +tiotest.1.seqwr_mbps.op=>= + +tiotest.1.seqrd_mbps.baseline=800 +tiotest.1.seqrd_mbps.goal=800 +tiotest.1.seqrd_mbps.op=>= + +tiotest.1.rndwr_mbps.baseline=45 +tiotest.1.rndwr_mbps.goal=45 +tiotest.1.rndwr_mbps.op=>= + +tiotest.1.rndwr_iops.baseline=11000 +tiotest.1.rndwr_iops.goal=11000 +tiotest.1.rndwr_iops.op=>= + +tiotest.1.rndrd_mbps.baseline=50 +tiotest.1.rndrd_mbps.goal=50 +tiotest.1.rndrd_mbps.op=>= + +tiotest.1.rndrd_iops.baseline=12000 +tiotest.1.rndrd_iops.goal=12000 +tiotest.1.rndrd_iops.op=>= + +tiotest.4.seqwr_mbps.baseline=500 +tiotest.4.seqwr_mbps.goal=500 +tiotest.4.seqwr_mbps.op=>= + +tiotest.4.seqrd_mbps.baseline=1200 +tiotest.4.seqrd_mbps.goal=1200 +tiotest.4.seqrd_mbps.op=>= + +tiotest.4.rndwr_mbps.baseline=80 +tiotest.4.rndwr_mbps.goal=80 +tiotest.4.rndwr_mbps.op=>= + +tiotest.4.rndwr_iops.baseline=30000 +tiotest.4.rndwr_iops.goal=30000 +tiotest.4.rndwr_iops.op=>= + +tiotest.4.rndrd_mbps.baseline=90 +tiotest.4.rndrd_mbps.goal=90 +tiotest.4.rndrd_mbps.op=>= + +tiotest.4.rndrd_iops.baseline=32000 +tiotest.4.rndrd_iops.goal=32000 +tiotest.4.rndrd_iops.op=>= diff --git a/Runner/utils/lib_performance.sh b/Runner/utils/lib_performance.sh index 17158393..b4ba000f 100755 --- a/Runner/utils/lib_performance.sh +++ b/Runner/utils/lib_performance.sh @@ -2145,3 +2145,613 @@ perf_sysbench_gate_eval_line_safe() { perf_sysbench_gate_eval_line "$f" "$sb_case" "$thr" "$metric" "$avg" "$delta" return $? } + +############################################################################### +# Tiotest helpers (Storage_Tiotest) +############################################################################### +tiotest_is_tmpfs_path() { + # Best-effort: detect if a path is on tmpfs + # Usage: tiotest_is_tmpfs_path /path ; returns 0 if tmpfs, 1 otherwise + p=$1 + [ -z "$p" ] && return 1 + if command -v df >/dev/null 2>&1; then + # BusyBox df prints "Filesystem" and type may not be available. + # Use /proc/mounts as primary. + : + fi + if [ -r /proc/mounts ]; then + # Find the mountpoint for p and check fstype + # Simple heuristic: if any mount entry matches prefix and fstype=tmpfs + mp="" + while read -r _dev mnt fstype rest; do + case "$p" in + "$mnt"|"$mnt"/*) + # pick longest matching mountpoint + if [ -z "$mp" ] || [ "${#mnt}" -gt "${#mp}" ]; then + mp="$mnt" + mpfstype="$fstype" + fi + ;; + esac + done /dev/null 2>&1; then + perf_drop_caches 2>/dev/null || true + return 0 + fi + # fallback + if [ -w /proc/sys/vm/drop_caches ]; then + sync 2>/dev/null || true + echo 3 >/proc/sys/vm/drop_caches 2>/dev/null || true + return 0 + fi + return 1 +} + +tiotest_extract_mbps() { + # Extract MB/s rate from tiotest tables; returns the LAST matching row. + # Args: (ex: "Write" or "Read") + logf=$1 + lbl=$2 + + awk -v lbl="$lbl" ' + BEGIN{v=""} + /^[[:space:]]*\|/ && $0 ~ lbl { + if (match($0, /[0-9]+(\.[0-9]+)?[[:space:]]*MB\/s/)) { + s=substr($0, RSTART, RLENGTH) + gsub(/[[:space:]]*MB\/s/, "", s) + v=s + } + } + END{ + if (v=="") exit 1 + print v + } + ' "$logf" +} + +tiotest_extract_iops() { + # Many tiotest builds print IOPS only for random rows; if absent returns empty. + # Args: + logf=$1 + lbl=$2 + + # Try to capture number in an "IOPS" column if present. + # We do not assume fixed column count; instead grep a number near "IOPS". + awk -v lbl="$lbl" ' + BEGIN{v=""} + /^\|/ && $0 ~ lbl { + # If line contains IOPS number, try to pick it. + # Common format: "... | 11624 | ..." + # We take the largest integer on the line as a heuristic. + nmax="" + for (i=1;i<=NF;i++){ + if ($i ~ /^[0-9]+$/) { + if (nmax=="" || $i+0 > nmax+0) nmax=$i + } + } + v=nmax + } + END{ print v } + ' "$logf" 2>/dev/null +} + +tiotest_extract_latency_block() { + logf=$1 + item=$2 # "Write" / "Read" / "Random Write" / "Random Read" + + awk -v item="$item" ' + BEGIN { inblk=0; la=""; lm=""; p2=""; p10="" } + + /^Tiotest latency results/ { inblk=1; next } + inblk==1 && /^`/ { inblk=0 } # end of ascii table (best-effort) + + # Match the latency row for the requested item + inblk==1 && $0 ~ /^\|/ && $0 ~ ("|[[:space:]]*" item "[[:space:]]*\\|") { + # Extract numbers that appear *after* the item label. + # Latency rows look like: + # | Write | 0.002 ms | 0.022 ms | 0.00000 | 0.00000 | + # We collect numeric tokens only, ignoring "ms" and pipes. + n=0 + for (i=1; i<=NF; i++) { + if ($i ~ /^[0-9]+(\.[0-9]+)?$/) { + n++ + if (n==1) la=$i + else if (n==2) lm=$i + else if (n==3) p2=$i + else if (n==4) p10=$i + } + } + } + + END { + # Print only numeric fields; empty means "not found" + printf "%s\t%s\t%s\t%s\n", la, lm, p2, p10 + } + ' "$logf" 2>/dev/null +} + +tiotest_metrics_append() { + mf=$1; mode=$2; thr=$3; mbps=$4; iops=$5; latavg=$6; latmax=$7; pct2=$8; pct10=$9 + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$mode" "$thr" "${mbps:-}" "${iops:-}" "${latavg:-}" "${latmax:-}" "${pct2:-}" "${pct10:-}" >>"$mf" +} + +# ---------------- perf helpers: normalize/validate numeric metrics ---------------- + +perf_norm_metric() { + case "${1:-}" in + ""|"unknown"|"UNKNOWN"|"NA"|"N/A"|"n/a") printf "%s" "NA" ;; + *) printf "%s" "$1" ;; + esac +} + +perf_is_number() { + # integer or decimal (e.g., 10, 10.5, 0.000) + echo "${1:-}" | awk ' + $0 ~ /^[0-9]+([.][0-9]+)?$/ { exit 0 } + { exit 1 } + ' +} + +perf_append_if_number() { + # $1=file $2=value + if perf_is_number "${2:-}"; then + perf_values_append "$1" "$2" + fi +} + +tiotest_build_common_args() { + # Helper: build common flags list for tiotest + # Echoes args; caller can use: set -- $(tiotest_build_common_args ...) + # Args: + # threads dir use_raw block_bytes file_mb rnd_ops hide_lat terse W S c debug offset_mb offset_first + tt=$1; dir=$2; use_raw=$3 + bs=$4; fmb=$5; rops=$6 + hide_lat=$7; terse=$8; wphase=$9; syncw=${10}; cons=${11}; dbg=${12} + offmb=${13}; ofirst=${14} + + args="-t $tt -d $dir -b $bs -f $fmb" + if [ "$use_raw" = "1" ]; then + args="$args -R" + [ -n "$offmb" ] && args="$args -o $offmb" + [ "$ofirst" = "1" ] && args="$args -O" + fi + [ -n "$rops" ] && args="$args -r $rops" + [ "$hide_lat" = "1" ] && args="$args -L" + [ "$terse" = "1" ] && args="$args -T" + [ "$wphase" = "1" ] && args="$args -W" + [ "$syncw" = "1" ] && args="$args -S" + [ "$cons" = "1" ] && args="$args -c" + [ -n "$dbg" ] && args="$args -D $dbg" + + echo "$args" +} + +perf_tiotest_run_seq_pair() { + # Run sequential Write+Read in ONE invocation and collect: + # - MB/s for Write and Read + # - IOPS (estimated from MB/s and block size) + # - Latency stats if present (avg/max/%>2s/%>10s), otherwise leave empty + # + # Args: + # bin threads dir use_raw seq_block seq_file_mb hide_lat terse W S c debug offset_mb offset_first logf metrics + bin=$1; tt=$2; dir=$3; use_raw=$4 + bs=$5; fmb=$6 + hide_lat=$7; terse=$8; wphase=$9; syncw=${10}; cons=${11}; dbg=${12} + offmb=${13}; ofirst=${14} + logf=${15}; metrics=${16} + + : >"$logf" 2>/dev/null || true + + common=$(tiotest_build_common_args "$tt" "$dir" "$use_raw" "$bs" "$fmb" "" \ + "$hide_lat" "$terse" "$wphase" "$syncw" "$cons" "$dbg" "$offmb" "$ofirst") + + log_info "RUN: $bin $common -k 1 -k 3" + # shellcheck disable=SC2086 + "$bin" $common -k 1 -k 3 >>"$logf" 2>&1 || true + + # ---------------- MB/s ---------------- + seqwr_mbps=$(tiotest_extract_mbps "$logf" "Write" 2>/dev/null || true) + seqrd_mbps=$(tiotest_extract_mbps "$logf" "Read" 2>/dev/null || true) + + # ---------------- IOPS estimate ---------------- + seqwr_iops="" + seqrd_iops="" + if [ -n "$bs" ] && [ "$bs" -gt 0 ] 2>/dev/null; then + if [ -n "$seqwr_mbps" ]; then + seqwr_iops=$(awk -v mbps="$seqwr_mbps" -v b="$bs" 'BEGIN{ if (b>0) printf "%.0f", (mbps*1024*1024)/b }' 2>/dev/null) + fi + if [ -n "$seqrd_mbps" ]; then + seqrd_iops=$(awk -v mbps="$seqrd_mbps" -v b="$bs" 'BEGIN{ if (b>0) printf "%.0f", (mbps*1024*1024)/b }' 2>/dev/null) + fi + fi + + # ---------------- Latency (best-effort) ---------------- + # Strictly parse numeric columns from: + # | Write | ms | ms | | | + # | Read | ms | ms | | | + # Ensures we never output "Write"/"|" into metrics.tsv. + seqwr_latavg=""; seqwr_latmax=""; seqwr_pct2=""; seqwr_pct10="" + seqrd_latavg=""; seqrd_latmax=""; seqrd_pct2=""; seqrd_pct10="" + + row=$(awk ' + BEGIN { inlat=0; la=""; lm=""; p2=""; p10="" } + /^Tiotest latency results/ { inlat=1; next } + inlat==1 && $0 ~ /^\|/ && $0 ~ /|[[:space:]]*Write[[:space:]]*\|/ { + n=0 + for (i=1;i<=NF;i++){ + if ($i ~ /^[0-9]+(\.[0-9]+)?$/) { + n++ + if (n==1) la=$i + else if (n==2) lm=$i + else if (n==3) p2=$i + else if (n==4) p10=$i + } + } + } + END { + if (la=="") exit 1 + printf "%s\t%s\t%s\t%s\n", la, lm, p2, p10 + } + ' "$logf" 2>/dev/null || true) + + if [ -n "$row" ]; then + # split by tabs + seqwr_latavg=$(printf '%s' "$row" | awk -F'\t' '{print $1}') + seqwr_latmax=$(printf '%s' "$row" | awk -F'\t' '{print $2}') + seqwr_pct2=$(printf '%s' "$row" | awk -F'\t' '{print $3}') + seqwr_pct10=$(printf '%s' "$row" | awk -F'\t' '{print $4}') + fi + + row=$(awk ' + BEGIN { inlat=0; la=""; lm=""; p2=""; p10="" } + /^Tiotest latency results/ { inlat=1; next } + inlat==1 && $0 ~ /^\|/ && $0 ~ /|[[:space:]]*Read[[:space:]]*\|/ { + n=0 + for (i=1;i<=NF;i++){ + if ($i ~ /^[0-9]+(\.[0-9]+)?$/) { + n++ + if (n==1) la=$i + else if (n==2) lm=$i + else if (n==3) p2=$i + else if (n==4) p10=$i + } + } + } + END { + if (la=="") exit 1 + printf "%s\t%s\t%s\t%s\n", la, lm, p2, p10 + } + ' "$logf" 2>/dev/null || true) + + if [ -n "$row" ]; then + seqrd_latavg=$(printf '%s' "$row" | awk -F'\t' '{print $1}') + seqrd_latmax=$(printf '%s' "$row" | awk -F'\t' '{print $2}') + seqrd_pct2=$(printf '%s' "$row" | awk -F'\t' '{print $3}') + seqrd_pct10=$(printf '%s' "$row" | awk -F'\t' '{print $4}') + fi + + # ---------------- Emit metrics (8 columns) ---------------- + tiotest_metrics_append "$metrics" "seqwr" "$tt" "$seqwr_mbps" "$seqwr_iops" \ + "$seqwr_latavg" "$seqwr_latmax" "$seqwr_pct2" "$seqwr_pct10" + + tiotest_metrics_append "$metrics" "seqrd" "$tt" "$seqrd_mbps" "$seqrd_iops" \ + "$seqrd_latavg" "$seqrd_latmax" "$seqrd_pct2" "$seqrd_pct10" + + if [ -z "$seqwr_mbps" ] && [ -z "$seqrd_mbps" ]; then + return 1 + fi + return 0 +} + +perf_tiotest_run_rnd_pair() { + # Random pair runner with robust rndwr + rndrd capture and clean 8-column TSV emission. + # Primary: run BOTH (Write+Read) using -k 1 -k 3 (matches your proven manual command). + # Fallback: legacy probing if Read doesn't appear (some builds behave oddly / pid-file issues). + # + # Args: + # bin threads dir use_raw rnd_block rnd_file_mb rnd_ops hide_lat terse W S c debug offset_mb offset_first logf metrics + bin=$1; tt=$2; dir=$3; use_raw=$4 + bs=$5; fmb=$6; rops=$7 + hide_lat=$8; terse=$9; wphase=${10}; syncw=${11}; cons=${12}; dbg=${13} + offmb=${14}; ofirst=${15} + logf=${16}; metrics=${17} + + : >"$logf" 2>/dev/null || true + + common=$(tiotest_build_common_args "$tt" "$dir" "$use_raw" "$bs" "$fmb" "$rops" \ + "$hide_lat" "$terse" "$wphase" "$syncw" "$cons" "$dbg" "$offmb" "$ofirst") + + tmp_both="${logf}.rndboth.tmp" + tmp_wr="${logf}.rndwr.tmp" + tmp_rd="${logf}.rndrd.tmp" + : >"$tmp_both" 2>/dev/null || true + : >"$tmp_wr" 2>/dev/null || true + : >"$tmp_rd" 2>/dev/null || true + + rndwr_mbps=""; rndwr_iops=""; rndwr_latavg=""; rndwr_latmax=""; rndwr_pct2=""; rndwr_pct10="" + rndrd_mbps=""; rndrd_iops=""; rndrd_latavg=""; rndrd_latmax=""; rndrd_pct2=""; rndrd_pct10="" + + # ---------------- Primary: BOTH random write + random read ---------------- + log_info "RUN (rnd both): $bin $common -k 1 -k 3" + # shellcheck disable=SC2086 + "$bin" $common -k 1 -k 3 >"$tmp_both" 2>&1 || true + cat "$tmp_both" >>"$logf" 2>/dev/null || true + + rndwr_mbps=$(tiotest_extract_mbps "$tmp_both" "Write" 2>/dev/null || true) + rndrd_mbps=$(tiotest_extract_mbps "$tmp_both" "Read" 2>/dev/null || true) + + # Latency parsing (best-effort): strict match "| Write |" and "| Read |" + row=$(awk ' + BEGIN { inlat=0; la=""; lm=""; p2=""; p10="" } + /^Tiotest latency results/ { inlat=1; next } + inlat==1 && $0 ~ /^\|/ && $0 ~ /|[[:space:]]*Write[[:space:]]*\|/ { + n=0 + for (i=1;i<=NF;i++){ + if ($i ~ /^[0-9]+(\.[0-9]+)?$/) { + n++ + if (n==1) la=$i + else if (n==2) lm=$i + else if (n==3) p2=$i + else if (n==4) p10=$i + } + } + } + END { + if (la=="") exit 1 + printf "%s\t%s\t%s\t%s\n", la, lm, p2, p10 + } + ' "$tmp_both" 2>/dev/null || true) + if [ -n "$row" ]; then + rndwr_latavg=$(printf '%s' "$row" | awk -F'\t' '{print $1}') + rndwr_latmax=$(printf '%s' "$row" | awk -F'\t' '{print $2}') + rndwr_pct2=$(printf '%s' "$row" | awk -F'\t' '{print $3}') + rndwr_pct10=$(printf '%s' "$row" | awk -F'\t' '{print $4}') + fi + + row=$(awk ' + BEGIN { inlat=0; la=""; lm=""; p2=""; p10="" } + /^Tiotest latency results/ { inlat=1; next } + inlat==1 && $0 ~ /^\|/ && $0 ~ /|[[:space:]]*Read[[:space:]]*\|/ { + n=0 + for (i=1;i<=NF;i++){ + if ($i ~ /^[0-9]+(\.[0-9]+)?$/) { + n++ + if (n==1) la=$i + else if (n==2) lm=$i + else if (n==3) p2=$i + else if (n==4) p10=$i + } + } + } + END { + if (la=="") exit 1 + printf "%s\t%s\t%s\t%s\n", la, lm, p2, p10 + } + ' "$tmp_both" 2>/dev/null || true) + if [ -n "$row" ]; then + rndrd_latavg=$(printf '%s' "$row" | awk -F'\t' '{print $1}') + rndrd_latmax=$(printf '%s' "$row" | awk -F'\t' '{print $2}') + rndrd_pct2=$(printf '%s' "$row" | awk -F'\t' '{print $3}') + rndrd_pct10=$(printf '%s' "$row" | awk -F'\t' '{print $4}') + fi + + # If Read missing, fallback to legacy probing + if [ -z "$rndrd_mbps" ]; then + log_info "rndrd missing in primary run; falling back to legacy probing" + + # Phase 1: try to collect write + log_info "RUN (rndwr fallback): $bin $common -k 0 -k 2 -k 3" + # shellcheck disable=SC2086 + "$bin" $common -k 0 -k 2 -k 3 >"$tmp_wr" 2>&1 || true + cat "$tmp_wr" >>"$logf" 2>/dev/null || true + [ -z "$rndwr_mbps" ] && rndwr_mbps=$(tiotest_extract_mbps "$tmp_wr" "Write" 2>/dev/null || true) + + # Phase 2: prep + attempt read (best-effort) + log_info "RUN (rndrd+prep fallback): $bin -t $tt -d $dir -b $bs -f $fmb -k 1 -k 2 -k 3 ; then $bin $common -k 0 -k 1 -k 2" + "$bin" -t "$tt" -d "$dir" -b "$bs" -f "$fmb" -k 1 -k 2 -k 3 >>"$tmp_rd" 2>&1 || true + # shellcheck disable=SC2086 + "$bin" $common -k 0 -k 1 -k 2 >>"$tmp_rd" 2>&1 || true + cat "$tmp_rd" >>"$logf" 2>/dev/null || true + + [ -z "$rndwr_mbps" ] && rndwr_mbps=$(tiotest_extract_mbps "$tmp_rd" "Write" 2>/dev/null || true) + rndrd_mbps=$(tiotest_extract_mbps "$tmp_rd" "Read" 2>/dev/null || true) + + # Latency from fallback read log (if any) + if [ -z "$rndwr_latavg" ]; then + row=$(awk ' + BEGIN { inlat=0; la=""; lm=""; p2=""; p10="" } + /^Tiotest latency results/ { inlat=1; next } + inlat==1 && $0 ~ /^\|/ && $0 ~ /|[[:space:]]*Write[[:space:]]*\|/ { + n=0 + for (i=1;i<=NF;i++){ + if ($i ~ /^[0-9]+(\.[0-9]+)?$/) { + n++ + if (n==1) la=$i + else if (n==2) lm=$i + else if (n==3) p2=$i + else if (n==4) p10=$i + } + } + } + END { if (la=="") exit 1; printf "%s\t%s\t%s\t%s\n", la, lm, p2, p10 } + ' "$tmp_rd" 2>/dev/null || true) + if [ -n "$row" ]; then + rndwr_latavg=$(printf '%s' "$row" | awk -F'\t' '{print $1}') + rndwr_latmax=$(printf '%s' "$row" | awk -F'\t' '{print $2}') + rndwr_pct2=$(printf '%s' "$row" | awk -F'\t' '{print $3}') + rndwr_pct10=$(printf '%s' "$row" | awk -F'\t' '{print $4}') + fi + fi + + if [ -z "$rndrd_latavg" ]; then + row=$(awk ' + BEGIN { inlat=0; la=""; lm=""; p2=""; p10="" } + /^Tiotest latency results/ { inlat=1; next } + inlat==1 && $0 ~ /^\|/ && $0 ~ /|[[:space:]]*Read[[:space:]]*\|/ { + n=0 + for (i=1;i<=NF;i++){ + if ($i ~ /^[0-9]+(\.[0-9]+)?$/) { + n++ + if (n==1) la=$i + else if (n==2) lm=$i + else if (n==3) p2=$i + else if (n==4) p10=$i + } + } + } + END { if (la=="") exit 1; printf "%s\t%s\t%s\t%s\n", la, lm, p2, p10 } + ' "$tmp_rd" 2>/dev/null || true) + if [ -n "$row" ]; then + rndrd_latavg=$(printf '%s' "$row" | awk -F'\t' '{print $1}') + rndrd_latmax=$(printf '%s' "$row" | awk -F'\t' '{print $2}') + rndrd_pct2=$(printf '%s' "$row" | awk -F'\t' '{print $3}') + rndrd_pct10=$(printf '%s' "$row" | awk -F'\t' '{print $4}') + fi + fi + fi + + # ---------------- IOPS estimation ---------------- + if [ -n "$bs" ] && [ "$bs" -gt 0 ] 2>/dev/null; then + if [ -n "$rndwr_mbps" ]; then + rndwr_iops=$(awk -v mbps="$rndwr_mbps" -v b="$bs" 'BEGIN{ if (b>0) printf "%.0f", (mbps*1024*1024)/b }' 2>/dev/null) + fi + if [ -n "$rndrd_mbps" ]; then + rndrd_iops=$(awk -v mbps="$rndrd_mbps" -v b="$bs" 'BEGIN{ if (b>0) printf "%.0f", (mbps*1024*1024)/b }' 2>/dev/null) + fi + fi + + # ---------------- Emit metrics (8 columns) ---------------- + tiotest_metrics_append "$metrics" "rndwr" "$tt" "$rndwr_mbps" "$rndwr_iops" \ + "$rndwr_latavg" "$rndwr_latmax" "$rndwr_pct2" "$rndwr_pct10" + + tiotest_metrics_append "$metrics" "rndrd" "$tt" "$rndrd_mbps" "$rndrd_iops" \ + "$rndrd_latavg" "$rndrd_latmax" "$rndrd_pct2" "$rndrd_pct10" + + rm -f "$tmp_both" "$tmp_wr" "$tmp_rd" 2>/dev/null || true + + # Success if we got rndwr or rndrd + [ -n "$rndwr_mbps" ] && return 0 + [ -n "$rndrd_mbps" ] && return 0 + return 1 +} + +# --- Tiotest baseline format support (suite=... threads=... metric=... baseline=... goal=... op=...) --- +perf_tiotest_baseline_lookup_kv() { + f=$1; suite=$2; thr=$3; metric=$4 + # prints: "baseline op goal" or empty + awk -v s="$suite" -v t="$thr" -v m="$metric" ' + { + sv=""; tv=""; mv=""; b=""; g=""; o=""; + for (i=1; i<=NF; i++) { + split($i, a, "="); + if (a[1]=="suite") sv=a[2]; + else if (a[1]=="threads") tv=a[2]; + else if (a[1]=="metric") mv=a[2]; + else if (a[1]=="baseline") b=a[2]; + else if (a[1]=="goal") g=a[2]; + else if (a[1]=="op") o=a[2]; + } + if (sv==s && tv==t && mv==m) { + print b, o, g; + exit; + } + } + ' "$f" +} + +perf_tiotest_baseline_prefix() { + suite=$1 + thr=$2 + metric=$3 + echo "${suite}.${thr}.${metric}" +} + +# Tiotest-specific gating wrapper. +# Purpose: isolate tiotest gating logic from sysbench helpers to avoid regressions. +# Contract: returns 0=PASS, 1=FAIL, 2=NO_BASELINE/NO_AVG style (same shape as sysbench wrapper). +perf_tiotest_gate_eval_line_safe() { + f=$1 + suite=$2 + thr=$3 + metric=$4 + avg=$5 + delta=$6 + + if [ -z "$avg" ]; then + echo "status=NO_AVG baseline=NA goal=NA op=NA score_pct=NA key=NA" + return 2 + fi + + prefix=$(perf_tiotest_baseline_prefix "$suite" "$thr" "$metric") + + base=$(perf_baseline_get_value "$f" "${prefix}.baseline") + goal=$(perf_baseline_get_value "$f" "${prefix}.goal") + op=$(perf_baseline_get_value "$f" "${prefix}.op") + + if [ -z "$base" ] || [ -z "$op" ] || [ "$base" = "__FILL_ME__" ]; then + echo "status=NO_BASELINE baseline=NA goal=NA op=NA score_pct=NA key=${prefix}" + return 2 + fi + + # If goal missing, derive from baseline and delta (best-effort). + if [ -z "$goal" ] || [ "$goal" = "__FILL_ME__" ]; then + # delta default to 0 if empty/non-numeric + case "$delta" in + ""|*[!0-9.]*) + delta=0 + ;; + esac + + case "$op" in + ">="|">") + goal=$(awk -v b="$base" -v d="$delta" 'BEGIN{ printf "%.6f", (b+0)*(1.0-(d+0)) }' 2>/dev/null) + ;; + "<="|"<") + goal=$(awk -v b="$base" -v d="$delta" 'BEGIN{ printf "%.6f", (b+0)*(1.0+(d+0)) }' 2>/dev/null) + ;; + "="|"==") + goal=$base + ;; + *) + echo "status=BAD_OP baseline=$base goal=NA op=$op score_pct=NA key=${prefix}" + return 2 + ;; + esac + fi + + if [ -z "$goal" ]; then + echo "status=NO_BASELINE baseline=$base goal=NA op=$op score_pct=NA key=${prefix}" + return 2 + fi + + score=$(awk -v a="$avg" -v b="$base" 'BEGIN{ if ((b+0)>0) printf "%.1f", ((a+0)/(b+0))*100; else print "NA" }' 2>/dev/null) + + fail=0 + case "$op" in + ">=") awk -v a="$avg" -v g="$goal" 'BEGIN{exit ((a+0) >= (g+0))?0:1}' || fail=1 ;; + "<=") awk -v a="$avg" -v g="$goal" 'BEGIN{exit ((a+0) <= (g+0))?0:1}' || fail=1 ;; + ">") awk -v a="$avg" -v g="$goal" 'BEGIN{exit ((a+0) > (g+0))?0:1}' || fail=1 ;; + "<") awk -v a="$avg" -v g="$goal" 'BEGIN{exit ((a+0) < (g+0))?0:1}' || fail=1 ;; + "="|"==") awk -v a="$avg" -v g="$goal" 'BEGIN{exit ((a+0) == (g+0))?0:1}' || fail=1 ;; + *) + echo "status=BAD_OP baseline=$base goal=$goal op=$op score_pct=$score key=${prefix}" + return 2 + ;; + esac + + if [ "$fail" -eq 0 ]; then + echo "status=PASS baseline=$base goal=$goal op=$op score_pct=$score key=${prefix}" + return 0 + fi + + echo "status=FAIL baseline=$base goal=$goal op=$op score_pct=$score key=${prefix}" + return 1 +}