pyproject.toml

# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Build
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.hatch.metadata]
allow-direct-references = true

[project]
name = "aiperf"
version = "0.12.0"
description = "AIPerf is a package for performance testing of AI models"
readme = "README.md"
license = {text = "Apache-2.0"}
license-files = ["LICENSE", "ATTRIBUTIONS.md"]
authors = [
    { name = "NVIDIA Inc.", email = "sw-dl-dynamo@nvidia.com" },
]
requires-python = ">=3.11,<3.14"
dependencies = [
  "aiofiles~=24.1.0",
  "aiohttp~=3.13.3",
  "cyclopts>=4,<5",
  "fastapi>=0.115,<1",
  "ffmpeg-python~=0.2.0",
  "filelock>=3.13", # Cross-process populate lock for the mmap dataset cache
  # datasets pulls pyarrow (its Arrow core), which has no Windows-on-ARM wheel
  # (apache/arrow#47195). Excluded on win_arm64 so the base install works there;
  # HF-backed --public-dataset is gated on that platform, but synthetic /
  # --input-file datasets (incl. SPEED-Bench) don't need it.
  "datasets>=3.0; platform_system != 'Windows' or platform_machine != 'ARM64'",
  "huggingface-hub>=0.34.0,<2.0",
  "jinja2~=3.1.5", # NOTE: Versions prior to 3.1.5 have vuln exploits
  "jmespath~=1.0.1",
  "kaleido~=1.2.0",
  "matplotlib>=3.10.0",
  "msgspec>=0.19.0,<1.0.0",
  "numpy>=1.26.4,<3",
  "nvidia-ml-py", # Note: No version specified to be most compatible with CUDA version
  "optuna>=3.6",
  "orjson~=3.11.7",
  "pandas~=2.3.3",
  "pillow~=12.2.0",
  "plotly~=6.4.0",
  "dash~=3.1.0",
  "dash-bootstrap-components~=2.0.0",
  "prometheus_client~=0.23.1",
  "protobuf>=4", # Loose on purpose: required by some HF tokenizers (e.g. T5/LLaMA SentencePiece converters); let it co-resolve with transformers/grpc/etc.
  "psutil~=7.0.0",
  # pyarrow ships no Windows-on-ARM wheel (apache/arrow#47195) and source-builds
  # fail there, so it's excluded on win_arm64; the parquet exporter self-disables
  # when it's absent. Required on every platform that has wheels.
  "pyarrow>=18.0.0; platform_system != 'Windows' or platform_machine != 'ARM64'",
  "pydantic>=2.10.0,<3.0.0",
  "pydantic-settings>=2.10.0,<3.0.0",
  "pyzmq~=26.4.0",
  "rich~=14.1.0",
  "ruamel.yaml~=0.18.12",
  "seaborn~=0.13.2",
  "scipy>=1.13.0",
  "sentencepiece>=0.2.0", # Loose on purpose: required by SentencePiece-based tokenizers (T5, LLaMA, XLNet, etc.) so the default install just works.
  "setproctitle~=1.3.6",
  "soundfile~=0.13.1",
  "starlette-compress>=1.0.1",
  "textual~=5.3.0",
  "tiktoken>=0.7.0,<1",
  "tqdm>=4.67.1",
  "transformers>=4.56.0", # Lowest compatible version for dynamo backends
  "uvicorn[standard]>=0.34,<1",
  "uvloop>=0.22.1; platform_system != 'Windows'",
  "zstandard>=0.25.0",
  "crick~=0.0.8",
]

# CLI Entrypoints
[project.scripts]
aiperf = "aiperf.cli:app"

# Plugin entry point definition to discover plugins at runtime
[project.entry-points."aiperf.plugins"]
aiperf = "aiperf.plugin:plugins.yaml"

[project.optional-dependencies]
mlflow = [
  "mlflow>=3.10.0,<4.0.0",
]
wandb = [
  "wandb>=0.19.0,<1.0.0",
]
otel = [
  "opentelemetry-exporter-otlp-proto-http>=1.24.0,<2.0.0",
  "opentelemetry-sdk>=1.24.0,<2.0.0",
]
test = [
  "httpx>=0.27.0",
  # hypothesis 6.156+ ships native (non-universal) wheels with no win_arm64
  # build and no sdist, so it can't install on Windows-on-ARM. Cap it there to
  # the last pure-Python release (6.155.x); leave it unconstrained elsewhere.
  "hypothesis>=6.0.0; platform_system != 'Windows' or platform_machine != 'ARM64'",
  "hypothesis>=6.0.0,<6.156; platform_system == 'Windows' and platform_machine == 'ARM64'",
  "jsonschema>=4.0.0",
  "looptime>=0.5",
  "pytest>=7.0.0",
  "pytest-asyncio",
  "pytest-cov",
  "pytest-rerunfailures>=15.0",
  "pytest-timeout>=2.0.0",
  "pytest-xdist>=3.8.0",
  # trustme pulls cryptography, which has no Windows-on-ARM wheel and
  # source-builds against openssl-sys (fails on win-arm). Only the TLS
  # transport tests use it; they are collect-ignored there.
  "trustme>=1.0.0; platform_system != 'Windows' or platform_machine != 'ARM64'",
]
dev = [
  "aiperf[mlflow,otel,test,wandb]",
  "black>=25.1.0",
  "pre-commit>=4.2.0",
  "ruff>=0.14.0,<0.15.0",
]
botorch = [
  "optuna-integration>=3.6",
  "botorch>=0.10",
  "gpytorch>=1.11",
  "torch>=2.0",
]
optuna = [
  "aiperf[botorch]",
]
accuracy = [
    # latex2sympy2-extended is co-resolved with lighteval (which pins
    # ==1.0.6 for its math-verify integration). 1.0.6 ships the same
    # ``latex2sympy`` API our math grader uses, so we accept the older
    # version as long as it satisfies our import.
    "latex2sympy2-extended>=1.0.6",
    "lighteval>=0.13.0",
    "sympy>=1.14.0",
    # deepeval is the upstream reference for the HellaSwag and
    # BigBench-Hard benchmark prompt templates. Aiperf calls
    # ``HellaSwagTemplate.generate_output`` (and
    # ``BigBenchHardTemplate.generate_output`` on the BigBench branch)
    # directly so prompts are byte-equal to what the trt-llm benchmark
    # recipe ships.
    "deepeval>=2.9.0,<5.0.0",
]

[dependency-groups]
dev = [
    "hypothesis>=6.152.4",
    "pre-commit>=4.6.0",
]

# uv resolver constraints. ``required-environments`` makes uv refuse
# to resolve to any package version that lacks a wheel or sdist for
# every platform listed, which auto-backtracks past partial-upload
# PyPI releases (e.g. yarl 1.24.0 + greenlet 3.5.1 on 2026-05-{19,20}
# both shipped without cp313/linux wheels and broke cold builds).
# Sdist counts as installable — crick ships only sdist on Linux
# aarch64 and is unaffected (env-builder compiles it from source).
[tool.uv]
required-environments = [
  "sys_platform == 'linux' and platform_machine == 'x86_64'",
  "sys_platform == 'linux' and platform_machine == 'aarch64'",
  "sys_platform == 'darwin' and platform_machine == 'arm64'",
]

[tool.ruff]
# Base settings for ruff
line-length = 88
indent-width = 4
exclude = ["__pycache__", "build", "dist", ".venv", "venv", "notebooks/TCO_calculator.ipynb"]

[tool.ruff.lint]
select = [
    # pycodestyle (except E501)
    "E",
    # Pyflakes
    "F",
    # pyupgrade
    "UP",
    # flake8-bugbear
    "B",
    # flake8-simplify
    "SIM",
    # isort
    "I",
]
# Ignore line length errors, ruff format will handle this but
# can have some lines that are slightly over due to the way formatting works
ignore = ["E501"]

# LLM-ergonomics rules (C901, TID251, S110, S112, ANN201, D103) are
# deliberately NOT in `select` above. They are enforced out-of-band via
# ``tools/ruff_baselined.py``, which filters ruff's output through
# ``tools/ruff_baseline.json`` (keyed on (rule, file, function-name)). This
# avoids per-line ``# noqa`` noise and per-file exemptions while still
# catching NEW violations in files that already have grandfathered ones.
# The threshold/config blocks below are kept so that ``ruff check --select
# C901,...`` (invoked by the wrapper and available for ad-hoc runs)
# honors the project's tunings.

[tool.ruff.lint.mccabe]
max-complexity = 10

[tool.ruff.lint.flake8-tidy-imports.banned-api]
# Enforced ONLY via the wrapper (``tools/ruff_baselined.py`` passes
# ``--select TID251`` explicitly) — plain ``ruff check`` / ``make lint``
# does NOT fire these because TID251 is intentionally absent from
# ``select`` above. ``tools/check_ergonomics.py`` is the second
# enforcement path: its ``stdlib-json`` axis flags the same imports via
# AST walk, independent of ruff. Both run as pre-commit hooks.
# Enforce orjson over stdlib json (CLAUDE.md standard).
"json.dumps".msg = "Use orjson.dumps(obj).decode() instead. Add option=orjson.OPT_INDENT_2 only when pretty-printed output is required (configs, manifests, debug exports) — the default must stay compact to preserve byte-for-byte equivalence for tests, JSONL, and cache keys."
"json.loads".msg = "Use orjson.loads instead."
"json.dump".msg = "Use orjson.dumps instead (no stdlib-style file writing — serialize then write bytes)."
"json.load".msg = "Use orjson.loads on file bytes instead."

[tool.pytest.ini_options]
# Basic pytest configuration
testpaths = ["tests"]
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
markers = [
    "integration: marks tests as integration tests",
    "performance: marks tests as performance tests (deselected by default)",
    "ffmpeg: marks tests that require ffmpeg to be installed (deselected by default)",
    "stress: marks tests as stress tests that generate high load (deselected by default)",
    "slow: marks tests as slow (>= 3s, deselected by default; opt in with -m slow)",
    "component_integration: marks tests as component integration tests",
    "statistical: marks tests requiring statistical validation with large samples, and may not be stable by default",
    "server_unit: marks tests as unit tests for the mock server",
    "fern: marks tests that validate Fern documentation (requires fern CLI)",
    "network: marks tests that require network access",
    "requires_deepeval: tests that need the real deepeval install (i.e. the [accuracy] extras) — skipped when only the fake-deepeval harness is registered",
    "looptime: marks tests that use virtual event-loop time",
]
# Better console output
console_output_style = "progress"
verbosity_assertions = 2
# Show extra test summary info
# Deselect performance, ffmpeg, stress, statistical, component_integration, integration, server_unit, fern, and slow tests by default
# To run them: pytest -m performance, pytest -m ffmpeg, pytest -m stress, pytest -m statistical, pytest -m slow
# IDE note: When running a specific test file/function from IDE, markers are often bypassed
addopts = "--strict-markers -m 'not performance and not ffmpeg and not stress and not statistical and not component_integration and not integration and not server_unit and not fern and not slow'"
# Filter out known warnings from third-party libraries and test infrastructure
filterwarnings = [
    "ignore:.*torch.jit.script.*:DeprecationWarning",
    "ignore::RuntimeWarning:looptime",
    "ignore:There is no current event loop:DeprecationWarning",
    "ignore:coroutine.*was never awaited:RuntimeWarning",
    "ignore:unclosed file:ResourceWarning",
    "ignore:builtin type SwigPyPacked has no __module__ attribute:DeprecationWarning",
    "ignore:builtin type SwigPyObject has no __module__ attribute:DeprecationWarning",
    "ignore:BoTorchSampler is experimental.*:optuna.exceptions.ExperimentalWarning",
    "ignore:GPSampler is experimental.*:optuna.exceptions.ExperimentalWarning",
    "ignore:Argument ``constraints_func`` is an experimental feature.*:optuna.exceptions.ExperimentalWarning",
    "ignore:ORJSONResponse is deprecated.*:fastapi.exceptions.FastAPIDeprecationWarning",
    "ignore:CUDA initialization.*The NVIDIA driver on your system is too old.*:UserWarning",
    "ignore:The model inputs are of type torch.float32.*",
    "ignore:Optimization failed in `gen_candidates_scipy`.*:RuntimeWarning",
    "ignore:Optimization failed on the second try.*:RuntimeWarning",
    "ignore:builtin type swigvarlink has no __module__ attribute:DeprecationWarning",
]

[tool.codespell]
skip = "*.pyc,*build*,tests/unit/transports/test_aiohttp_sse.py,tests/integration/assets/canary_reference_inputs.json,src/aiperf/server_metrics/units.py,src/aiperf/api/static/dashboard.html,src/aiperf/config/schema/aiperf-config.schema.json"
ignore-words-list = "timeslice,timeslices,optiona,disjointness,concurency,strat,clen"