tonic/benchmarks/native-compiler-suite.toml at main · mikeyobrien/tonic · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
# Native compiler performance contract for LLVM roadmap baseline.
#
# This suite keeps the existing benchsuite command contract, but adds
# machine-readable native compiler SLOs + weighted Rust/Go competitiveness scoring.

[performance_contract]
baseline_path = "benchmarks/native-compiler-baselines.json"
candidate_target = "interpreter"
reference_targets = ["tonic_interpreter_baseline"]
relative_budget_pct = 50
pass_threshold = 0.82

  [performance_contract.metric_weights]
  latency_p50 = 0.35
  latency_p95 = 0.25
  rss = 0.20
  artifact_size = 0.10
  compile_latency = 0.10

  [performance_contract.slo]
  startup_p50_ms = 50
  runtime_p50_ms = 15
  runtime_p95_ms = 30
  rss_kb = 30720
  artifact_size_bytes = 12000000
  compile_latency_ms = 3000

  [performance_contract.regression_policy]
  quarantine_margin = 0.10
  rollback_margin = 0.20
  quarantine_max_soft_regressions = 2
  quarantine_max_score_gap = 0.03
  rollback_min_score_gap = 0.08

[[workload]]
name = "check_parse_control_flow"
command = ["check", "examples/parity/06-control-flow/for_multi_generator.tn"]
mode = "cold"
threshold_p50_ms = 10
threshold_p95_ms = 20
threshold_rss_kb = 30720
weight = 1.2
category = "startup"

[[workload]]
name = "check_module_load_project"
command = ["check", "examples/parity/07-modules/project_multifile_pipeline"]
mode = "cold"
threshold_p50_ms = 12
threshold_p95_ms = 24
threshold_rss_kb = 30720
weight = 1.2
category = "startup"

[[workload]]
name = "run_module_load_project"
command = ["run", "examples/parity/07-modules/project_multifile_pipeline"]
mode = "cold"
threshold_p50_ms = 14
threshold_p95_ms = 28
threshold_rss_kb = 30720
weight = 1.0
category = "module_load"

[[workload]]
name = "run_control_flow_multi_generator"
command = ["run", "examples/parity/06-control-flow/for_multi_generator.tn"]
mode = "warm"
threshold_p50_ms = 8
threshold_p95_ms = 16
threshold_rss_kb = 30720
weight = 1.3
category = "control_flow"

[[workload]]
name = "run_control_flow_for_into"
command = ["run", "examples/parity/06-control-flow/for_into.tn"]
mode = "warm"
threshold_p50_ms = 8
threshold_p95_ms = 16
threshold_rss_kb = 30720
weight = 1.0
category = "control_flow"

[[workload]]
name = "run_error_result_pipeline"
command = ["run", "examples/parity/08-errors/question_operator_success.tn"]
mode = "warm"
threshold_p50_ms = 8
threshold_p95_ms = 16
threshold_rss_kb = 30720
weight = 0.8
category = "runtime"

[[workload]]
name = "check_json_transform_shape"
command = ["check", "examples/parity/03-collections/map_literal_single_entry.tn"]
mode = "warm"
threshold_p50_ms = 8
threshold_p95_ms = 16
threshold_rss_kb = 30720
weight = 0.9
category = "json_transform"

[[workload]]
name = "check_file_pipeline_project"
command = ["check", "examples/parity/07-modules/project_multifile_pipeline"]
mode = "cold"
threshold_p50_ms = 12
threshold_p95_ms = 24
threshold_rss_kb = 30720
weight = 1.0
category = "file_io"

[[workload]]
name = "check_subprocess_host_interop"
command = ["check", "tests/fixtures/host_call/sum.tn"]
mode = "warm"
threshold_p50_ms = 8
threshold_p95_ms = 16
threshold_rss_kb = 30720
weight = 0.8
category = "subprocess"