Skip to content

Option to re-display a benchmark file #185

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 23 commits into from
Jul 10, 2025
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
58bca2c
Allow result file to be re-displayed
jaredoconnell Jun 12, 2025
90328e6
Added test for JSON
jaredoconnell Jun 12, 2025
07d8150
Added yaml test
jaredoconnell Jun 13, 2025
1ed37cd
Fix warning
jaredoconnell Jun 13, 2025
379c64f
Add uncommitted file
jaredoconnell Jun 13, 2025
2a8db0d
Fix linter errors
jaredoconnell Jun 13, 2025
c43573a
Use fixed width for CLI tests
jaredoconnell Jun 13, 2025
18d5897
Fix linter error and exclude test assets from linting
jaredoconnell Jun 13, 2025
2063d43
Added option to regenerate test artifact
jaredoconnell Jun 13, 2025
27e8391
Fix linter errors
jaredoconnell Jun 13, 2025
554a182
Merge branch 'main' into redisplay-results
sjmonson Jun 20, 2025
7f2dd40
Address review comments
jaredoconnell Jun 23, 2025
fff5c87
Allow reexporting reimported benchmarks
jaredoconnell Jun 26, 2025
9f9ddc9
Add test for reexporting and fix other tests
jaredoconnell Jun 26, 2025
bf3d175
Merge branch 'main' into redisplay-results
jaredoconnell Jun 26, 2025
83b0c77
Switch to internal dependency, and fix linter errors
jaredoconnell Jun 26, 2025
c0baf33
Update documentation to reflect command change
jaredoconnell Jun 26, 2025
9d41aa6
Merge branch 'main' into redisplay-results
jaredoconnell Jun 27, 2025
686fcee
Merge branch 'main' into redisplay-results
jaredoconnell Jun 27, 2025
534fbe4
Fix linter errors
jaredoconnell Jun 27, 2025
bece157
Revert docs changes
jaredoconnell Jul 8, 2025
be1730d
Update command to use hyphen
jaredoconnell Jul 8, 2025
2bd7f5e
Merge branch 'main' into redisplay-results
markurtz Jul 10, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ cython_debug/
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
.idea/


# MacOS files
Expand Down
2 changes: 2 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ repos:
rev: v4.6.0
hooks:
- id: trailing-whitespace
exclude: ^tests/?.*/assets/.+
- id: end-of-file-fixer
exclude: ^tests/?.*/assets/.+
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.11.7
hooks:
Expand Down
16 changes: 15 additions & 1 deletion src/guidellm/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@
import click

from guidellm.backend import BackendType
from guidellm.benchmark import ProfileType, benchmark_generative_text
from guidellm.benchmark import (
ProfileType,
benchmark_generative_text,
display_benchmarks_report,
)
from guidellm.config import print_config
from guidellm.preprocess.dataset import ShortPromptStrategy, process_dataset
from guidellm.scheduler import StrategyType
Expand Down Expand Up @@ -282,6 +286,16 @@ def benchmark(
)


@cli.command(help="Redisplay a saved benchmark report.")
@click.argument(
"path",
type=click.Path(),
default=Path.cwd() / "benchmarks.json",
)
def display(path):
display_benchmarks_report(path)


def decode_escaped_str(_ctx, _param, value):
"""
Click auto adds characters. For example, when using --pad-char "\n",
Expand Down
3 changes: 2 additions & 1 deletion src/guidellm/benchmark/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
StatusBreakdown,
)
from .benchmarker import Benchmarker, BenchmarkerResult, GenerativeBenchmarker
from .entrypoints import benchmark_generative_text
from .entrypoints import benchmark_generative_text, display_benchmarks_report
from .output import GenerativeBenchmarksConsole, GenerativeBenchmarksReport
from .profile import (
AsyncProfile,
Expand Down Expand Up @@ -63,4 +63,5 @@
"ThroughputProfile",
"benchmark_generative_text",
"create_profile",
"display_benchmarks_report",
]
16 changes: 10 additions & 6 deletions src/guidellm/benchmark/entrypoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,13 +121,8 @@ async def benchmark_generative_text(
)

if output_console:
orig_enabled = console.enabled
console.enabled = True
console.benchmarks = report.benchmarks
console.print_benchmarks_metadata()
console.print_benchmarks_info()
console.print_benchmarks_stats()
console.enabled = orig_enabled
console.print_full_report()

if output_path:
console.print_line("\nSaving benchmarks report...")
Expand All @@ -139,3 +134,12 @@ async def benchmark_generative_text(
console.print_line("\nBenchmarking complete.")

return report, saved_path

def display_benchmarks_report(file: Path):
console = GenerativeBenchmarksConsole(enabled=True)
if not file.exists():
console.print_line(f"File {file} not found.")
return
report = GenerativeBenchmarksReport.load_file(file)
console.benchmarks = report.benchmarks
console.print_full_report()
17 changes: 17 additions & 0 deletions src/guidellm/benchmark/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -944,3 +944,20 @@ def print_benchmarks_stats(self):
title="Benchmarks Stats",
sections=sections,
)

def print_full_report(self):
"""
Print out the benchmark statistics to the console.
Temporarily enables the console if it's disabled.

Format:
- Metadata
- Info
- Stats
"""
orig_enabled = self.enabled
self.enabled = True
self.print_benchmarks_metadata()
self.print_benchmarks_info()
self.print_benchmarks_stats()
self.enabled = orig_enabled
Empty file.
Loading