Option to re-display a benchmark file

.gitignore

@@ -168,7 +168,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
 
 
 # MacOS files

.pre-commit-config.yaml

@@ -3,7 +3,9 @@ repos:
   rev: v4.6.0
   hooks:
   - id: trailing-whitespace
+    exclude: ^tests/?.*/assets/.+
   - id: end-of-file-fixer
+    exclude: ^tests/?.*/assets/.+
 - repo: https://github.com/astral-sh/ruff-pre-commit
   rev: v0.11.7
   hooks:

src/guidellm/__main__.py

@@ -7,12 +7,16 @@
 from pydantic import ValidationError
 
 from guidellm.backend import BackendType
-from guidellm.benchmark import ProfileType
+from guidellm.benchmark import (
+    ProfileType,
+    reimport_benchmarks_report,
+)
 from guidellm.benchmark.entrypoints import benchmark_with_scenario
 from guidellm.benchmark.scenario import GenerativeTextScenario, get_builtin_scenarios
 from guidellm.config import print_config
 from guidellm.preprocess.dataset import ShortPromptStrategy, process_dataset
 from guidellm.scheduler import StrategyType
+from guidellm.utils import DefaultGroupHandler
 from guidellm.utils import cli as cli_tools
 
 STRATEGY_PROFILE_CHOICES = set(
@@ -25,7 +29,17 @@ def cli():
     pass
 
 
-@cli.command(
+@cli.group(
+    help="Commands to run a new benchmark or load a prior one.",
+    cls=DefaultGroupHandler,
+    default="run",
+)
+def benchmark():
+    pass
+
+
+@benchmark.command(
+    "run",
     help="Run a benchmark against a generative model using the specified arguments.",
     context_settings={"auto_envvar_prefix": "GUIDELLM"},
 )
@@ -230,7 +244,7 @@ def cli():
     type=int,
     help="The random seed to use for benchmarking to ensure reproducibility.",
 )
-def benchmark(
+def run(
     scenario,
     target,
     backend_type,
@@ -306,6 +320,37 @@ def benchmark(
     )
 
 
+@benchmark.command(
+    "from-file",
+    help="Load a saved benchmark report."
+)
+@click.argument(
+    "path",
+    type=click.Path(file_okay=True, dir_okay=False, exists=True),
+    default=Path.cwd() / "benchmarks.json",
+)
+@click.option(
+    "--output-path",
+    type=click.Path(file_okay=True, dir_okay=True, exists=False),
+    default=None,
+    is_flag=False,
+    flag_value=Path.cwd() / "benchmarks_reexported.json",
+    help=(
+        "Allows re-exporting the benchmarks to another format. "
+        "The path to save the output to. If it is a directory, "
+        "it will save benchmarks.json under it. "
+        "Otherwise, json, yaml, or csv files are supported for output types "
+        "which will be read from the extension for the file path. "
+        "This input is optional. If the output path flag is not provided, "
+        "the benchmarks will not be reexported. If the flag is present but "
+        "no value is specified, it will default to the current directory "
+        "with the file name `benchmarks_reexported.json`."
+    ),
+)
+def from_file(path, output_path):
+    reimport_benchmarks_report(path, output_path)
+
+
 def decode_escaped_str(_ctx, _param, value):
     """
     Click auto adds characters. For example, when using --pad-char "\n",
@@ -321,10 +366,11 @@ def decode_escaped_str(_ctx, _param, value):
 
 
 @cli.command(
+    short_help="Prints environment variable settings.",
     help=(
         "Print out the available configuration settings that can be set "
         "through environment variables."
-    )
+    ),
 )
 def config():
     print_config()

src/guidellm/benchmark/__init__.py

@@ -12,7 +12,7 @@
     StatusBreakdown,
 )
 from .benchmarker import Benchmarker, BenchmarkerResult, GenerativeBenchmarker
-from .entrypoints import benchmark_generative_text
+from .entrypoints import benchmark_generative_text, reimport_benchmarks_report
 from .output import GenerativeBenchmarksConsole, GenerativeBenchmarksReport
 from .profile import (
     AsyncProfile,
@@ -63,4 +63,5 @@
     "ThroughputProfile",
     "benchmark_generative_text",
     "create_profile",
+    "reimport_benchmarks_report",
 ]

src/guidellm/benchmark/entrypoints.py

@@ -133,13 +133,8 @@ async def benchmark_generative_text(
             )
 
     if output_console:
-        orig_enabled = console.enabled
-        console.enabled = True
         console.benchmarks = report.benchmarks
-        console.print_benchmarks_metadata()
-        console.print_benchmarks_info()
-        console.print_benchmarks_stats()
-        console.enabled = orig_enabled
+        console.print_full_report()
 
     if output_path:
         console.print_line("\nSaving benchmarks report...")
@@ -151,3 +146,20 @@ async def benchmark_generative_text(
     console.print_line("\nBenchmarking complete.")
 
     return report, saved_path
+
+
+def reimport_benchmarks_report(file: Path, output_path: Optional[Path]) -> None:
+    """
+    The command-line entry point for re-importing and displaying an
+    existing benchmarks report. Can also specify
+    Assumes the file provided exists.
+    """
+    console = GenerativeBenchmarksConsole(enabled=True)
+    report = GenerativeBenchmarksReport.load_file(file)
+    console.benchmarks = report.benchmarks
+    console.print_full_report()
+
+    if output_path:
+        console.print_line("\nSaving benchmarks report...")
+        saved_path = report.save_file(output_path)
+        console.print_line(f"Benchmarks report saved to {saved_path}")

src/guidellm/benchmark/output.py

@@ -242,7 +242,10 @@ def _file_setup(
         if path_suffix in [".csv"]:
             return path, "csv"
 
-        raise ValueError(f"Unsupported file extension: {path_suffix} for {path}.")
+        raise ValueError(
+            f"Unsupported file extension: {path_suffix} for {path}; "
+            "expected json, yaml, or csv."
+        )
 
     @staticmethod
     def _benchmark_desc_headers_and_values(
@@ -944,3 +947,20 @@ def print_benchmarks_stats(self):
             title="Benchmarks Stats",
             sections=sections,
         )
+
+    def print_full_report(self):
+        """
+        Print out the benchmark statistics to the console.
+        Temporarily enables the console if it's disabled.
+
+        Format:
+        - Metadata
+        - Info
+        - Stats
+        """
+        orig_enabled = self.enabled
+        self.enabled = True
+        self.print_benchmarks_metadata()
+        self.print_benchmarks_info()
+        self.print_benchmarks_stats()
+        self.enabled = orig_enabled

src/guidellm/utils/__init__.py

@@ -1,4 +1,5 @@
 from .colors import Colors
+from .default_group import DefaultGroupHandler
 from .hf_datasets import (
     SUPPORTED_TYPES,
     save_dataset_to_file,
@@ -20,6 +21,7 @@
 __all__ = [
     "SUPPORTED_TYPES",
     "Colors",
+    "DefaultGroupHandler",
     "EndlessTextCreator",
     "IntegerRangeSampler",
     "check_load_processor",

src/guidellm/utils/default_group.py

@@ -0,0 +1,104 @@
+"""
+File uses code adapted from code with the following license:
+
+Copyright (c) 2015-2023, Heungsub Lee
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+  Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+  Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+  Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+
+__all__ = ["DefaultGroupHandler"]
+
+import collections.abc as cabc
+
+import click
+
+
+class DefaultGroupHandler(click.Group):
+    """
+    Allows the migration to a new sub-command by allowing the group to run
+    one of its sub-commands as the no-args default command.
+    """
+
+    def __init__(self, *args, **kwargs):
+        # To resolve as the default command.
+        if not kwargs.get("ignore_unknown_options", True):
+            raise ValueError("Default group accepts unknown options")
+        self.ignore_unknown_options = True
+        self.default_cmd_name = kwargs.pop("default", None)
+        self.default_if_no_args = kwargs.pop("default_if_no_args", False)
+        super().__init__(*args, **kwargs)
+
+    def parse_args(self, ctx, args):
+        if not args and self.default_if_no_args:
+            args.insert(0, self.default_cmd_name)
+        return super().parse_args(ctx, args)
+
+    def get_command(self, ctx, cmd_name):
+        if cmd_name not in self.commands:
+            # If it doesn't match an existing command, use the default command name.
+            ctx.arg0 = cmd_name
+            cmd_name = self.default_cmd_name
+        return super().get_command(ctx, cmd_name)
+
+    def resolve_command(self, ctx, args):
+        cmd_name, cmd, args = super().resolve_command(ctx, args)
+        if hasattr(ctx, "arg0"):
+            args.insert(0, ctx.arg0)
+            cmd_name = cmd.name
+        return cmd_name, cmd, args
+
+    def format_commands(self, ctx, formatter):
+        """
+        Used to wrap the default formatter to clarify which command is the default.
+        """
+        formatter = DefaultCommandFormatter(self, formatter, mark=" (default)")
+        return super().format_commands(ctx, formatter)
+
+
+class DefaultCommandFormatter:
+    """
+    Wraps a formatter to edit the line for the default command to mark it
+    with the specified mark string.
+    """
+
+    def __init__(self, group, formatter, mark="*"):
+        self.group = group
+        self.formatter = formatter
+        self.mark = mark
+        super().__init__()
+
+    def __getattr__(self, attr):
+        return getattr(self.formatter, attr)
+
+    def write_dl(self, rows: cabc.Sequence[tuple[str, str]], *args, **kwargs):
+        rows_: list[tuple[str, str]] = []
+        for cmd_name, help_msg in rows:
+            if cmd_name == self.group.default_cmd_name:
+                rows_.insert(0, (cmd_name + self.mark, help_msg))
+            else:
+                rows_.append((cmd_name, help_msg))
+        return self.formatter.write_dl(rows_, *args, **kwargs)