From 6fdc38abe3c260d634e16c168f887a37a4e2d771 Mon Sep 17 00:00:00 2001 From: Qiong Wu Date: Wed, 10 Jun 2026 16:11:11 +0800 Subject: [PATCH 01/27] Add --memory flag to winml perf for process/device memory measurement Implement per-phase memory tracking that captures Working Set, Private Bytes, and device (NPU/GPU) memory at each benchmark phase boundary: baseline, post-load, post-compile, and post-inference. Key design decisions: - Default enabled (--no-memory to disable) since snapshots are taken between phases and add zero overhead to latency measurements - Pure ctypes implementation (K32GetProcessMemoryInfo) with no new dependencies - Device memory via single-shot PDH query reusing existing adapter resolution logic - Console output shows a table with per-phase deltas and peak summary - JSON output includes full memory profile under 'memory' key New files: - session/monitor/memory_tracker.py: MemoryTracker, MemorySnapshot, MemoryProfile dataclasses and Windows/Linux memory query functions - tests/unit/session/monitor/test_memory_tracker.py: unit tests --- src/winml/modelkit/commands/perf.py | 110 ++++++ .../session/monitor/memory_tracker.py | 357 ++++++++++++++++++ .../session/monitor/test_memory_tracker.py | 167 ++++++++ 3 files changed, 634 insertions(+) create mode 100644 src/winml/modelkit/session/monitor/memory_tracker.py create mode 100644 tests/unit/session/monitor/test_memory_tracker.py diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index b53578d3d..efbc16fc1 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -26,6 +26,7 @@ from rich.console import Console from rich.table import Table +from ..session.monitor.memory_tracker import MemoryProfile from ..utils import cli as cli_utils from ..utils.constants import EPName, EPNameOrAlias from ..utils.logging import configure_logging @@ -82,6 +83,7 @@ class BenchmarkConfig: skip_build: bool = True allow_unsupported_nodes: bool = False monitor: bool = False + memory: bool = True ep: EPNameOrAlias | None = None shape_config: dict | None = None @@ -129,6 +131,9 @@ class BenchmarkResult: # Hardware monitor metrics (from HWMonitor.to_dict()) hw_monitor: dict[str, Any] | None = None + # Memory profile (from MemoryTracker) + memory_profile: MemoryProfile | None = None + def to_dict(self) -> dict[str, Any]: """Convert to dictionary for JSON serialization.""" result = { @@ -169,6 +174,8 @@ def to_dict(self) -> dict[str, Any]: } if self.hw_monitor: result["hw_monitor"] = self.hw_monitor + if self.memory_profile: + result["memory"] = self.memory_profile.to_dict() return result @@ -281,6 +288,7 @@ def __init__(self, config: BenchmarkConfig) -> None: self.config = config self._model: WinMLPreTrainedModel | None = None self._inputs: dict[str, np.ndarray] | None = None + self._memory_tracker: Any = None def run(self) -> BenchmarkResult: """Execute full benchmark pipeline. @@ -288,11 +296,21 @@ def run(self) -> BenchmarkResult: Returns: BenchmarkResult with timing statistics """ + # Initialize memory tracker if enabled + if self.config.memory: + from ..session.monitor.memory_tracker import MemoryTracker + + self._memory_tracker = MemoryTracker() + self._memory_tracker.snapshot_baseline() + # [1] Load model logger.info("Loading model: %s", self.config.model_id) self._load_model() assert self._model is not None + if self._memory_tracker: + self._memory_tracker.snapshot_post_load() + # [2] Generate inputs logger.info("Generating benchmark inputs") self._generate_inputs() @@ -300,6 +318,10 @@ def run(self) -> BenchmarkResult: # Compile session early so model.device is resolved for display self._model._session.compile() + if self._memory_tracker: + adapter_luid = self._resolve_adapter_luid() + self._memory_tracker.snapshot_post_compile(adapter_luid=adapter_luid) + # Print model info before benchmark starts _print_model_info( self._model.io_config, @@ -317,6 +339,10 @@ def run(self) -> BenchmarkResult: ) stats = self._run_benchmark() + if self._memory_tracker: + adapter_luid = self._resolve_adapter_luid() + self._memory_tracker.snapshot_post_inference(adapter_luid=adapter_luid) + # [4] Collect results logger.info("Collecting results") return self._collect_results(stats) @@ -384,6 +410,40 @@ def _generate_inputs(self) -> None: batch_size=self.config.batch_size, ) + def _resolve_adapter_luid(self) -> str | None: + """Resolve the adapter LUID for device memory queries. + + Uses the same resolution logic as HWMonitor: device kind + EP name. + Returns None on non-Windows or when no adapter is available. + """ + import sys + + if sys.platform != "win32": + return None + + assert self._model is not None + device = self._model.device or self.config.device + ep_name = self._model.ep_name + + if device == "cpu": + return None + + try: + from ..sysinfo.pdh_adapters import resolve_adapter_luid + + if device == "npu": + return resolve_adapter_luid("npu", ep_name=ep_name) + if device == "gpu": + return resolve_adapter_luid("gpu", ep_name=ep_name) + # "auto" — try NPU first, then GPU + luid = resolve_adapter_luid("npu", ep_name=ep_name) + if luid: + return luid + return resolve_adapter_luid("gpu", ep_name=ep_name) + except Exception: + logger.debug("Could not resolve adapter LUID for memory query", exc_info=True) + return None + def _run_benchmark(self) -> PerfStats: """Execute benchmark iterations with timing.""" if self.config.monitor: @@ -517,6 +577,8 @@ def _collect_results(self, stats: PerfStats) -> BenchmarkResult: actual_ep=self._model.ep_name, # Hardware monitor metrics (only present when --monitor is used) hw_monitor=getattr(self, "_hw_metrics", None), + # Memory profile (only present when --memory is used) + memory_profile=(self._memory_tracker.profile() if self._memory_tracker else None), ) @@ -874,6 +936,46 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: f" CPU: {cpu.get('mean_pct', 0):.1f}% avg | Mem: {ram.get('used_mb', 0):.0f} MB" ) + # Memory section (only when --memory is enabled) + if result.memory_profile: + console.print() + console.print("[bold]Memory[/bold]") + mem = result.memory_profile + + mem_table = Table(show_header=True, header_style="bold cyan") + mem_table.add_column("Phase", style="dim") + mem_table.add_column("Working Set", justify="right") + mem_table.add_column("Private Bytes", justify="right") + mem_table.add_column("Device Mem", justify="right") + mem_table.add_column("Δ Working Set", justify="right") + + phases = [ + ("Baseline", mem.baseline, None), + ("Post-Load", mem.post_load, mem.load_delta_mb), + ("Post-Compile", mem.post_compile, mem.compile_delta_mb), + ("Post-Inference", mem.post_inference, mem.inference_delta_mb), + ] + for name, snap, delta in phases: + dev_str = f"{snap.device_local_mb:.1f} MB" if snap.device_local_mb > 0 else "—" + delta_str = f"+{delta:.1f} MB" if delta is not None else "—" + mem_table.add_row( + name, + f"{snap.working_set_mb:.1f} MB", + f"{snap.private_bytes_mb:.1f} MB", + dev_str, + delta_str, + ) + + console.print(mem_table) + + # Summary line + peak_dev = mem.peak_device_local_mb + dev_summary = f" + {peak_dev:.1f} MB (device)" if peak_dev > 0 else "" + console.print( + f" [dim]Peak Working Set: {mem.peak_working_set_mb:.1f} MB | " + f"Total Δ: +{mem.total_delta_mb:.1f} MB (process){dev_summary}[/dim]" + ) + console.print() @@ -1103,6 +1205,12 @@ def _run_simple_loop( show_default=True, help="Show live hardware utilization chart for the benchmarked device (NPU, GPU, or CPU)", ) +@click.option( + "--memory/--no-memory", + default=True, + show_default=True, + help="Measure process and device memory at each benchmark phase", +) @click.option( "--op-tracing", "op_tracing", @@ -1133,6 +1241,7 @@ def perf( allow_unsupported_nodes: bool, module_class: str | None, monitor: bool, + memory: bool, op_tracing: str | None, verbose: int, quiet: bool, @@ -1269,6 +1378,7 @@ def perf( skip_build=skip_build, allow_unsupported_nodes=allow_unsupported_nodes, monitor=monitor, + memory=memory, ep=ep, shape_config=shape_config, ) diff --git a/src/winml/modelkit/session/monitor/memory_tracker.py b/src/winml/modelkit/session/monitor/memory_tracker.py new file mode 100644 index 000000000..24e33198e --- /dev/null +++ b/src/winml/modelkit/session/monitor/memory_tracker.py @@ -0,0 +1,357 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +r"""Process memory tracking for perf benchmarking. + +Provides lightweight, zero-dependency process memory snapshots via Windows +``GetProcessMemoryInfo`` (ctypes). Used by ``winml perf --memory`` to measure +memory consumption at each benchmark phase. + +For device (NPU/GPU) memory, a single-shot PDH query is used to read +``\GPU Process Memory\Local Usage`` and ``\GPU Process Memory\Shared Usage``. +""" + +from __future__ import annotations + +import ctypes +import logging +import os +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, ClassVar + + +if sys.platform == "win32": + import ctypes.wintypes as wintypes + + +logger = logging.getLogger(__name__) + + +# ============================================================================= +# Process Memory via GetProcessMemoryInfo (Windows) +# ============================================================================= + +_MB = 1024 * 1024 + + +if sys.platform == "win32": + + class _ProcessMemoryCountersEx(ctypes.Structure): + """PROCESS_MEMORY_COUNTERS_EX structure from psapi.h.""" + + _fields_: ClassVar = [ + ("cb", wintypes.DWORD), + ("PageFaultCount", wintypes.DWORD), + ("PeakWorkingSetSize", ctypes.c_size_t), + ("WorkingSetSize", ctypes.c_size_t), + ("QuotaPeakPagedPoolUsage", ctypes.c_size_t), + ("QuotaPagedPoolUsage", ctypes.c_size_t), + ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t), + ("QuotaNonPagedPoolUsage", ctypes.c_size_t), + ("PagefileUsage", ctypes.c_size_t), + ("PeakPagefileUsage", ctypes.c_size_t), + ("PrivateUsage", ctypes.c_size_t), + ] + + +def _get_process_memory() -> tuple[float, float, float, float]: + """Get current process memory via K32GetProcessMemoryInfo. + + Uses kernel32.K32GetProcessMemoryInfo (Windows 7+) which supports + PROCESS_MEMORY_COUNTERS_EX natively. + + Returns: + (working_set_mb, peak_working_set_mb, private_bytes_mb, peak_private_bytes_mb) + """ + if sys.platform != "win32": + return _get_process_memory_linux() + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.GetCurrentProcess.restype = wintypes.HANDLE + kernel32.K32GetProcessMemoryInfo.restype = wintypes.BOOL + kernel32.K32GetProcessMemoryInfo.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(_ProcessMemoryCountersEx), + wintypes.DWORD, + ] + + handle = kernel32.GetCurrentProcess() + counters = _ProcessMemoryCountersEx() + counters.cb = ctypes.sizeof(counters) + + success = kernel32.K32GetProcessMemoryInfo(handle, ctypes.byref(counters), counters.cb) + if not success: + err = ctypes.get_last_error() + logger.warning("K32GetProcessMemoryInfo failed (error=%d), returning zeros", err) + return (0.0, 0.0, 0.0, 0.0) + + return ( + counters.WorkingSetSize / _MB, + counters.PeakWorkingSetSize / _MB, + counters.PrivateUsage / _MB, + counters.PeakPagefileUsage / _MB, + ) + + +def _get_process_memory_linux() -> tuple[float, float, float, float]: + """Fallback for Linux: read /proc/self/status.""" + try: + with Path("/proc/self/status").open() as f: + content = f.read() + + values: dict[str, float] = {} + for line in content.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0].rstrip(":") in ( + "VmRSS", + "VmPeak", + "VmSize", + ): + values[parts[0].rstrip(":")] = float(parts[1]) / 1024 # kB -> MB + + rss = values.get("VmRSS", 0.0) + peak = values.get("VmPeak", 0.0) + return (rss, peak, rss, peak) + except OSError: + return (0.0, 0.0, 0.0, 0.0) + + +# ============================================================================= +# Device Memory via single-shot PDH query +# ============================================================================= + + +def _get_device_memory_mb(luid: str | None) -> tuple[float, float]: + """Single-shot PDH query for device memory (local, shared) in MB. + + Args: + luid: Adapter LUID string. If None, returns (0, 0). + + Returns: + (local_mb, shared_mb) + """ + if luid is None or sys.platform != "win32": + return (0.0, 0.0) + + try: + from ._pdh import PdhQuery + + pid = os.getpid() + query = PdhQuery() + query.open() + + local_ok = query.add_counter( + "local", + rf"\GPU Process Memory(pid_{pid}_luid_{luid}_phys_0)\Local Usage", + fmt="large", + ) + shared_ok = query.add_counter( + "shared", + rf"\GPU Process Memory(pid_{pid}_luid_{luid}_phys_0)\Shared Usage", + fmt="large", + ) + + if not local_ok and not shared_ok: + query.close() + return (0.0, 0.0) + + # PDH large counters don't need priming (not rate-based) + query.prime() + values = query.collect() + query.close() + + local_bytes = values.get("local") or 0 + shared_bytes = values.get("shared") or 0 + return (local_bytes / _MB, shared_bytes / _MB) + except Exception: + logger.debug("Device memory query failed", exc_info=True) + return (0.0, 0.0) + + +# ============================================================================= +# Data Classes +# ============================================================================= + + +@dataclass +class MemorySnapshot: + """A point-in-time memory measurement.""" + + working_set_mb: float = 0.0 + peak_working_set_mb: float = 0.0 + private_bytes_mb: float = 0.0 + peak_private_bytes_mb: float = 0.0 + device_local_mb: float = 0.0 + device_shared_mb: float = 0.0 + + def to_dict(self) -> dict[str, float]: + """JSON-serializable dictionary.""" + return { + "working_set_mb": round(self.working_set_mb, 2), + "peak_working_set_mb": round(self.peak_working_set_mb, 2), + "private_bytes_mb": round(self.private_bytes_mb, 2), + "peak_private_bytes_mb": round(self.peak_private_bytes_mb, 2), + "device_local_mb": round(self.device_local_mb, 2), + "device_shared_mb": round(self.device_shared_mb, 2), + } + + +@dataclass +class MemoryProfile: + """Memory measurements across benchmark phases.""" + + baseline: MemorySnapshot + post_load: MemorySnapshot + post_compile: MemorySnapshot + post_inference: MemorySnapshot + + @property + def load_delta_mb(self) -> float: + """Working set increase from model loading.""" + return self.post_load.working_set_mb - self.baseline.working_set_mb + + @property + def compile_delta_mb(self) -> float: + """Working set increase from session compilation.""" + return self.post_compile.working_set_mb - self.post_load.working_set_mb + + @property + def inference_delta_mb(self) -> float: + """Working set increase during inference.""" + return self.post_inference.working_set_mb - self.post_compile.working_set_mb + + @property + def total_delta_mb(self) -> float: + """Total working set increase from baseline.""" + return self.post_inference.working_set_mb - self.baseline.working_set_mb + + @property + def peak_working_set_mb(self) -> float: + """Peak working set across all phases (from OS counter).""" + return self.post_inference.peak_working_set_mb + + @property + def peak_device_local_mb(self) -> float: + """Peak device local memory across all phases.""" + return max( + self.baseline.device_local_mb, + self.post_load.device_local_mb, + self.post_compile.device_local_mb, + self.post_inference.device_local_mb, + ) + + @property + def peak_device_shared_mb(self) -> float: + """Peak device shared memory across all phases.""" + return max( + self.baseline.device_shared_mb, + self.post_load.device_shared_mb, + self.post_compile.device_shared_mb, + self.post_inference.device_shared_mb, + ) + + def to_dict(self) -> dict[str, Any]: + """JSON-serializable dictionary.""" + return { + "baseline": self.baseline.to_dict(), + "post_load": self.post_load.to_dict(), + "post_compile": self.post_compile.to_dict(), + "post_inference": self.post_inference.to_dict(), + "peak_working_set_mb": round(self.peak_working_set_mb, 2), + "peak_device_local_mb": round(self.peak_device_local_mb, 2), + "peak_device_shared_mb": round(self.peak_device_shared_mb, 2), + "total_delta_working_set_mb": round(self.total_delta_mb, 2), + } + + +# ============================================================================= +# MemoryTracker +# ============================================================================= + + +class MemoryTracker: + """Lightweight memory tracker that takes snapshots at phase boundaries. + + Usage:: + + tracker = MemoryTracker() + tracker.snapshot_baseline() + # ... load model ... + tracker.snapshot_post_load() + # ... compile ... + tracker.snapshot_post_compile(adapter_luid="0x...") + # ... run benchmark ... + tracker.snapshot_post_inference(adapter_luid="0x...") + profile = tracker.profile() + """ + + def __init__(self) -> None: + self._baseline: MemorySnapshot | None = None + self._post_load: MemorySnapshot | None = None + self._post_compile: MemorySnapshot | None = None + self._post_inference: MemorySnapshot | None = None + + def _take_snapshot(self, adapter_luid: str | None = None) -> MemorySnapshot: + """Take a point-in-time memory snapshot.""" + ws, peak_ws, priv, peak_priv = _get_process_memory() + dev_local, dev_shared = _get_device_memory_mb(adapter_luid) + return MemorySnapshot( + working_set_mb=ws, + peak_working_set_mb=peak_ws, + private_bytes_mb=priv, + peak_private_bytes_mb=peak_priv, + device_local_mb=dev_local, + device_shared_mb=dev_shared, + ) + + def snapshot_baseline(self) -> None: + """Capture baseline memory before model loading.""" + self._baseline = self._take_snapshot() + + def snapshot_post_load(self) -> None: + """Capture memory after model loading.""" + self._post_load = self._take_snapshot() + + def snapshot_post_compile(self, adapter_luid: str | None = None) -> None: + """Capture memory after session compilation. + + Args: + adapter_luid: Adapter LUID for device memory query. + Available after compile resolves the EP. + """ + self._post_compile = self._take_snapshot(adapter_luid) + + def snapshot_post_inference(self, adapter_luid: str | None = None) -> None: + """Capture memory after benchmark completion. + + Args: + adapter_luid: Adapter LUID for device memory query. + """ + self._post_inference = self._take_snapshot(adapter_luid) + + def profile(self) -> MemoryProfile | None: + """Build a complete MemoryProfile from collected snapshots. + + Returns None if any phase snapshot is missing. + """ + if any( + s is None + for s in (self._baseline, self._post_load, self._post_compile, self._post_inference) + ): + logger.warning("Incomplete memory snapshots, cannot build profile") + return None + + assert self._baseline is not None + assert self._post_load is not None + assert self._post_compile is not None + assert self._post_inference is not None + + return MemoryProfile( + baseline=self._baseline, + post_load=self._post_load, + post_compile=self._post_compile, + post_inference=self._post_inference, + ) diff --git a/tests/unit/session/monitor/test_memory_tracker.py b/tests/unit/session/monitor/test_memory_tracker.py new file mode 100644 index 000000000..42bc310a2 --- /dev/null +++ b/tests/unit/session/monitor/test_memory_tracker.py @@ -0,0 +1,167 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +"""Tests for the memory_tracker module.""" + +from __future__ import annotations + +import pytest + +from winml.modelkit.session.monitor.memory_tracker import ( + MemoryProfile, + MemorySnapshot, + MemoryTracker, + _get_process_memory, +) + + +class TestGetProcessMemory: + """Test the process memory retrieval function.""" + + def test_returns_four_floats(self) -> None: + result = _get_process_memory() + assert len(result) == 4 + for val in result: + assert isinstance(val, float) + + def test_working_set_positive(self) -> None: + ws, peak_ws, priv, peak_priv = _get_process_memory() + # Our process should be using *some* memory + assert ws > 0 + assert peak_ws >= ws + assert priv > 0 + assert peak_priv >= priv + + +class TestMemorySnapshot: + """Test MemorySnapshot dataclass.""" + + def test_to_dict(self) -> None: + snap = MemorySnapshot( + working_set_mb=100.123, + peak_working_set_mb=120.456, + private_bytes_mb=80.789, + peak_private_bytes_mb=90.012, + device_local_mb=50.347, + device_shared_mb=10.678, + ) + d = snap.to_dict() + assert d["working_set_mb"] == 100.12 + assert d["peak_working_set_mb"] == 120.46 + assert d["private_bytes_mb"] == 80.79 + assert d["device_local_mb"] == 50.35 + assert d["device_shared_mb"] == 10.68 + + def test_defaults_are_zero(self) -> None: + snap = MemorySnapshot() + assert snap.working_set_mb == 0.0 + assert snap.device_local_mb == 0.0 + + +class TestMemoryProfile: + """Test MemoryProfile computed properties.""" + + @pytest.fixture + def profile(self) -> MemoryProfile: + return MemoryProfile( + baseline=MemorySnapshot( + working_set_mb=100.0, + peak_working_set_mb=100.0, + private_bytes_mb=120.0, + peak_private_bytes_mb=120.0, + ), + post_load=MemorySnapshot( + working_set_mb=300.0, + peak_working_set_mb=310.0, + private_bytes_mb=350.0, + peak_private_bytes_mb=350.0, + ), + post_compile=MemorySnapshot( + working_set_mb=320.0, + peak_working_set_mb=325.0, + private_bytes_mb=370.0, + peak_private_bytes_mb=375.0, + device_local_mb=50.0, + ), + post_inference=MemorySnapshot( + working_set_mb=330.0, + peak_working_set_mb=340.0, + private_bytes_mb=380.0, + peak_private_bytes_mb=385.0, + device_local_mb=52.0, + device_shared_mb=8.0, + ), + ) + + def test_load_delta(self, profile: MemoryProfile) -> None: + assert profile.load_delta_mb == pytest.approx(200.0) + + def test_compile_delta(self, profile: MemoryProfile) -> None: + assert profile.compile_delta_mb == pytest.approx(20.0) + + def test_inference_delta(self, profile: MemoryProfile) -> None: + assert profile.inference_delta_mb == pytest.approx(10.0) + + def test_total_delta(self, profile: MemoryProfile) -> None: + assert profile.total_delta_mb == pytest.approx(230.0) + + def test_peak_working_set(self, profile: MemoryProfile) -> None: + assert profile.peak_working_set_mb == pytest.approx(340.0) + + def test_peak_device_local(self, profile: MemoryProfile) -> None: + assert profile.peak_device_local_mb == pytest.approx(52.0) + + def test_peak_device_shared(self, profile: MemoryProfile) -> None: + assert profile.peak_device_shared_mb == pytest.approx(8.0) + + def test_to_dict(self, profile: MemoryProfile) -> None: + d = profile.to_dict() + assert "baseline" in d + assert "post_load" in d + assert "post_compile" in d + assert "post_inference" in d + assert d["peak_working_set_mb"] == 340.0 + assert d["total_delta_working_set_mb"] == 230.0 + + +class TestMemoryTracker: + """Test MemoryTracker snapshot collection.""" + + def test_full_workflow(self) -> None: + tracker = MemoryTracker() + tracker.snapshot_baseline() + tracker.snapshot_post_load() + tracker.snapshot_post_compile() + tracker.snapshot_post_inference() + profile = tracker.profile() + + assert profile is not None + assert profile.baseline.working_set_mb > 0 + assert profile.post_inference.working_set_mb > 0 + + def test_incomplete_returns_none(self) -> None: + tracker = MemoryTracker() + tracker.snapshot_baseline() + # Missing other phases + profile = tracker.profile() + assert profile is None + + def test_snapshots_are_nondecreasing(self) -> None: + """Working set should generally not decrease between adjacent snapshots.""" + tracker = MemoryTracker() + tracker.snapshot_baseline() + + # Allocate something to ensure memory grows + _data = [bytearray(1024 * 1024) for _ in range(5)] # ~5 MB + + tracker.snapshot_post_load() + tracker.snapshot_post_compile() + tracker.snapshot_post_inference() + profile = tracker.profile() + + assert profile is not None + # post_load should be >= baseline (we allocated memory) + assert profile.post_load.working_set_mb >= profile.baseline.working_set_mb + + del _data From e50a39c8742a7c0735b73e8cf5f397a56ac02596 Mon Sep 17 00:00:00 2001 From: Qiong Wu Date: Wed, 10 Jun 2026 16:23:20 +0800 Subject: [PATCH 02/27] Simplify memory output to single line with peak process + device memory --- src/winml/modelkit/commands/perf.py | 39 +++-------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index efbc16fc1..8937f71e0 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -938,43 +938,12 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: # Memory section (only when --memory is enabled) if result.memory_profile: - console.print() - console.print("[bold]Memory[/bold]") mem = result.memory_profile - - mem_table = Table(show_header=True, header_style="bold cyan") - mem_table.add_column("Phase", style="dim") - mem_table.add_column("Working Set", justify="right") - mem_table.add_column("Private Bytes", justify="right") - mem_table.add_column("Device Mem", justify="right") - mem_table.add_column("Δ Working Set", justify="right") - - phases = [ - ("Baseline", mem.baseline, None), - ("Post-Load", mem.post_load, mem.load_delta_mb), - ("Post-Compile", mem.post_compile, mem.compile_delta_mb), - ("Post-Inference", mem.post_inference, mem.inference_delta_mb), - ] - for name, snap, delta in phases: - dev_str = f"{snap.device_local_mb:.1f} MB" if snap.device_local_mb > 0 else "—" - delta_str = f"+{delta:.1f} MB" if delta is not None else "—" - mem_table.add_row( - name, - f"{snap.working_set_mb:.1f} MB", - f"{snap.private_bytes_mb:.1f} MB", - dev_str, - delta_str, - ) - - console.print(mem_table) - - # Summary line + peak_ws = mem.peak_working_set_mb peak_dev = mem.peak_device_local_mb - dev_summary = f" + {peak_dev:.1f} MB (device)" if peak_dev > 0 else "" - console.print( - f" [dim]Peak Working Set: {mem.peak_working_set_mb:.1f} MB | " - f"Total Δ: +{mem.total_delta_mb:.1f} MB (process){dev_summary}[/dim]" - ) + dev_str = f" | {peak_dev:.1f} MB (device)" if peak_dev > 0 else "" + console.print() + console.print(f"[bold]Memory:[/bold] {peak_ws:.1f} MB (process peak){dev_str}") console.print() From a8bef090e1cc26467608e0a3dd362840f9ff9e5a Mon Sep 17 00:00:00 2001 From: Qiong Wu Date: Wed, 10 Jun 2026 16:28:49 +0800 Subject: [PATCH 03/27] Remove unnecessary del statement (CodeQL) --- tests/unit/session/monitor/test_memory_tracker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/session/monitor/test_memory_tracker.py b/tests/unit/session/monitor/test_memory_tracker.py index 42bc310a2..a1a7c4a2c 100644 --- a/tests/unit/session/monitor/test_memory_tracker.py +++ b/tests/unit/session/monitor/test_memory_tracker.py @@ -163,5 +163,5 @@ def test_snapshots_are_nondecreasing(self) -> None: assert profile is not None # post_load should be >= baseline (we allocated memory) assert profile.post_load.working_set_mb >= profile.baseline.working_set_mb - - del _data + # Keep _data alive until assertions complete so memory isn't reclaimed early + assert _data is not None From 1b7d016429bea708cb5e503aa6e4e3aaaf1a6c42 Mon Sep 17 00:00:00 2001 From: Qiong Wu Date: Wed, 10 Jun 2026 16:32:22 +0800 Subject: [PATCH 04/27] Use post-inference working set instead of process peak for memory display Shows the actual memory footprint during inference (steady state) rather than the process lifetime peak which may include transient allocations from model loading or compilation. --- src/winml/modelkit/commands/perf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 8937f71e0..9f9b97cea 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -939,11 +939,11 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: # Memory section (only when --memory is enabled) if result.memory_profile: mem = result.memory_profile - peak_ws = mem.peak_working_set_mb - peak_dev = mem.peak_device_local_mb - dev_str = f" | {peak_dev:.1f} MB (device)" if peak_dev > 0 else "" + inference_ws = mem.post_inference.working_set_mb + inference_dev = mem.post_inference.device_local_mb + dev_str = f" | {inference_dev:.1f} MB (device)" if inference_dev > 0 else "" console.print() - console.print(f"[bold]Memory:[/bold] {peak_ws:.1f} MB (process peak){dev_str}") + console.print(f"[bold]Memory:[/bold] {inference_ws:.1f} MB (process){dev_str}") console.print() From 125c4556c85f923d12705c9a27e67936c98de374 Mon Sep 17 00:00:00 2001 From: Qiong Wu Date: Wed, 10 Jun 2026 18:33:34 +0800 Subject: [PATCH 05/27] Fix CI: add missing memory_profile mock in TestPerfFormatJson The test from PR #855 mocks BenchmarkResult but did not set memory_profile=None, causing MagicMock comparison failure in display_console_report. --- tests/unit/commands/test_perf_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/commands/test_perf_cli.py b/tests/unit/commands/test_perf_cli.py index fa4541493..a83bdc843 100644 --- a/tests/unit/commands/test_perf_cli.py +++ b/tests/unit/commands/test_perf_cli.py @@ -489,6 +489,7 @@ def test_format_text_shows_console_report( mock_result.samples_per_sec = 100.0 mock_result.batches_per_sec = 100.0 mock_result.hw_monitor = None + mock_result.memory_profile = None mock_instance = MagicMock() mock_instance.run.return_value = mock_result mock_benchmark_class.return_value = mock_instance From 56704ba06e912991058b62bc0c71b6489dff5894 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 12 Jun 2026 16:28:52 +0800 Subject: [PATCH 06/27] refactor: align memory tracker with reference 3-phase approach - Remove snapshot_post_load (load+compile combined as model_load) - Use psutil for RSS with ctypes/proc fallback - Simplify MemorySnapshot to rss_mb + peak_wset_mb + device fields - Add delta breakdown in console: model load / inference / total - Update tests for new 3-phase API (14 tests passing) --- src/winml/modelkit/commands/perf.py | 21 +- .../session/monitor/memory_tracker.py | 197 +++++++++--------- .../session/monitor/test_memory_tracker.py | 112 ++++------ 3 files changed, 151 insertions(+), 179 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 670ed7742..b07224a18 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -296,7 +296,9 @@ def run(self) -> BenchmarkResult: Returns: BenchmarkResult with timing statistics """ - # Initialize memory tracker if enabled + # Initialize memory tracker if enabled. + # Baseline is taken here — after Python/ORT/EP DLLs are loaded but + # before model-specific work, so EP initialization cost is excluded. if self.config.memory: from ..session.monitor.memory_tracker import MemoryTracker @@ -308,9 +310,6 @@ def run(self) -> BenchmarkResult: self._load_model() assert self._model is not None - if self._memory_tracker: - self._memory_tracker.snapshot_post_load() - # [2] Generate inputs logger.info("Generating benchmark inputs") self._generate_inputs() @@ -939,11 +938,17 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: # Memory section (only when --memory is enabled) if result.memory_profile: mem = result.memory_profile - inference_ws = mem.post_inference.working_set_mb - inference_dev = mem.post_inference.device_local_mb - dev_str = f" | {inference_dev:.1f} MB (device)" if inference_dev > 0 else "" + dev_str = ( + f" | {mem.peak_device_local_mb:.1f} MB (device)" if mem.peak_device_local_mb > 0 else "" + ) + rss = mem.post_inference.rss_mb console.print() - console.print(f"[bold]Memory:[/bold] {inference_ws:.1f} MB (process){dev_str}") + console.print(f"[bold]Memory:[/bold] {rss:.1f} MB (process){dev_str}") + console.print( + f" [dim]model load: +{mem.model_load_delta_mb:.1f} MB | " + f"inference: +{mem.inference_alloc_delta_mb:.1f} MB | " + f"total: +{mem.total_delta_mb:.1f} MB[/dim]" + ) console.print() diff --git a/src/winml/modelkit/session/monitor/memory_tracker.py b/src/winml/modelkit/session/monitor/memory_tracker.py index 24e33198e..880e674fc 100644 --- a/src/winml/modelkit/session/monitor/memory_tracker.py +++ b/src/winml/modelkit/session/monitor/memory_tracker.py @@ -2,14 +2,15 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. # -------------------------------------------------------------------------- -r"""Process memory tracking for perf benchmarking. +"""Process memory tracking for perf benchmarking. -Provides lightweight, zero-dependency process memory snapshots via Windows -``GetProcessMemoryInfo`` (ctypes). Used by ``winml perf --memory`` to measure -memory consumption at each benchmark phase. +Measures RSS (Resident Set Size) at benchmark phase boundaries to compute +memory deltas for model loading, compilation, and inference. Uses the same +approach as standalone memory measurement scripts: psutil for process memory +with a ctypes fallback on Windows. -For device (NPU/GPU) memory, a single-shot PDH query is used to read -``\GPU Process Memory\Local Usage`` and ``\GPU Process Memory\Shared Usage``. +The tracker excludes one-time EP initialization costs (DLL loading) by +taking the baseline *after* the EP registry is warmed up. """ from __future__ import annotations @@ -31,12 +32,36 @@ # ============================================================================= -# Process Memory via GetProcessMemoryInfo (Windows) +# Memory Measurement # ============================================================================= _MB = 1024 * 1024 +def _get_memory_mb() -> dict[str, float]: + """Return current RSS and peak working set in MB for this process. + + Tries psutil first (cross-platform), falls back to ctypes on Windows + or /proc/self/status on Linux. + """ + try: + import psutil + + proc = psutil.Process(os.getpid()) + info = proc.memory_info() + return { + "rss_mb": info.rss / _MB, + "peak_wset_mb": getattr(info, "peak_wset", info.rss) / _MB, + } + except ImportError: + pass + + # Fallback: platform-specific + if sys.platform == "win32": + return _get_memory_mb_win32() + return _get_memory_mb_linux() + + if sys.platform == "win32": class _ProcessMemoryCountersEx(ctypes.Structure): @@ -57,18 +82,8 @@ class _ProcessMemoryCountersEx(ctypes.Structure): ] -def _get_process_memory() -> tuple[float, float, float, float]: - """Get current process memory via K32GetProcessMemoryInfo. - - Uses kernel32.K32GetProcessMemoryInfo (Windows 7+) which supports - PROCESS_MEMORY_COUNTERS_EX natively. - - Returns: - (working_set_mb, peak_working_set_mb, private_bytes_mb, peak_private_bytes_mb) - """ - if sys.platform != "win32": - return _get_process_memory_linux() - +def _get_memory_mb_win32() -> dict[str, float]: + """Fallback for Windows: ctypes K32GetProcessMemoryInfo.""" kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) kernel32.GetCurrentProcess.restype = wintypes.HANDLE kernel32.K32GetProcessMemoryInfo.restype = wintypes.BOOL @@ -84,19 +99,16 @@ def _get_process_memory() -> tuple[float, float, float, float]: success = kernel32.K32GetProcessMemoryInfo(handle, ctypes.byref(counters), counters.cb) if not success: - err = ctypes.get_last_error() - logger.warning("K32GetProcessMemoryInfo failed (error=%d), returning zeros", err) - return (0.0, 0.0, 0.0, 0.0) + logger.warning("K32GetProcessMemoryInfo failed, returning zeros") + return {"rss_mb": 0.0, "peak_wset_mb": 0.0} - return ( - counters.WorkingSetSize / _MB, - counters.PeakWorkingSetSize / _MB, - counters.PrivateUsage / _MB, - counters.PeakPagefileUsage / _MB, - ) + return { + "rss_mb": counters.WorkingSetSize / _MB, + "peak_wset_mb": counters.PeakWorkingSetSize / _MB, + } -def _get_process_memory_linux() -> tuple[float, float, float, float]: +def _get_memory_mb_linux() -> dict[str, float]: """Fallback for Linux: read /proc/self/status.""" try: with Path("/proc/self/status").open() as f: @@ -105,18 +117,14 @@ def _get_process_memory_linux() -> tuple[float, float, float, float]: values: dict[str, float] = {} for line in content.splitlines(): parts = line.split() - if len(parts) >= 2 and parts[0].rstrip(":") in ( - "VmRSS", - "VmPeak", - "VmSize", - ): + if len(parts) >= 2 and parts[0].rstrip(":") in ("VmRSS", "VmPeak"): values[parts[0].rstrip(":")] = float(parts[1]) / 1024 # kB -> MB rss = values.get("VmRSS", 0.0) peak = values.get("VmPeak", 0.0) - return (rss, peak, rss, peak) + return {"rss_mb": rss, "peak_wset_mb": peak} except OSError: - return (0.0, 0.0, 0.0, 0.0) + return {"rss_mb": 0.0, "peak_wset_mb": 0.0} # ============================================================================= @@ -158,7 +166,6 @@ def _get_device_memory_mb(luid: str | None) -> tuple[float, float]: query.close() return (0.0, 0.0) - # PDH large counters don't need priming (not rate-based) query.prime() values = query.collect() query.close() @@ -180,20 +187,16 @@ def _get_device_memory_mb(luid: str | None) -> tuple[float, float]: class MemorySnapshot: """A point-in-time memory measurement.""" - working_set_mb: float = 0.0 - peak_working_set_mb: float = 0.0 - private_bytes_mb: float = 0.0 - peak_private_bytes_mb: float = 0.0 + rss_mb: float = 0.0 + peak_wset_mb: float = 0.0 device_local_mb: float = 0.0 device_shared_mb: float = 0.0 def to_dict(self) -> dict[str, float]: """JSON-serializable dictionary.""" return { - "working_set_mb": round(self.working_set_mb, 2), - "peak_working_set_mb": round(self.peak_working_set_mb, 2), - "private_bytes_mb": round(self.private_bytes_mb, 2), - "peak_private_bytes_mb": round(self.peak_private_bytes_mb, 2), + "rss_mb": round(self.rss_mb, 2), + "peak_wset_mb": round(self.peak_wset_mb, 2), "device_local_mb": round(self.device_local_mb, 2), "device_shared_mb": round(self.device_shared_mb, 2), } @@ -201,69 +204,62 @@ def to_dict(self) -> dict[str, float]: @dataclass class MemoryProfile: - """Memory measurements across benchmark phases.""" + """Memory measurements across benchmark phases. + + Mirrors the structure used in standalone memory measurement scripts: + baseline → after_compile → after_warmup, with computed deltas. + """ baseline: MemorySnapshot - post_load: MemorySnapshot post_compile: MemorySnapshot post_inference: MemorySnapshot @property - def load_delta_mb(self) -> float: - """Working set increase from model loading.""" - return self.post_load.working_set_mb - self.baseline.working_set_mb + def model_load_delta_mb(self) -> float: + """RSS increase from model loading + compilation.""" + return self.post_compile.rss_mb - self.baseline.rss_mb @property - def compile_delta_mb(self) -> float: - """Working set increase from session compilation.""" - return self.post_compile.working_set_mb - self.post_load.working_set_mb + def inference_alloc_delta_mb(self) -> float: + """RSS increase from inference (warmup + benchmark).""" + return self.post_inference.rss_mb - self.post_compile.rss_mb @property - def inference_delta_mb(self) -> float: - """Working set increase during inference.""" - return self.post_inference.working_set_mb - self.post_compile.working_set_mb + def total_delta_mb(self) -> float: + """Total RSS increase from baseline.""" + return self.post_inference.rss_mb - self.baseline.rss_mb @property - def total_delta_mb(self) -> float: - """Total working set increase from baseline.""" - return self.post_inference.working_set_mb - self.baseline.working_set_mb + def peak_wset_mb(self) -> float: + """Peak working set (from OS counter at end of benchmark).""" + return self.post_inference.peak_wset_mb @property - def peak_working_set_mb(self) -> float: - """Peak working set across all phases (from OS counter).""" - return self.post_inference.peak_working_set_mb + def peak_delta_mb(self) -> float: + """Peak working set increase from baseline.""" + return self.post_inference.peak_wset_mb - self.baseline.peak_wset_mb @property def peak_device_local_mb(self) -> float: """Peak device local memory across all phases.""" return max( self.baseline.device_local_mb, - self.post_load.device_local_mb, self.post_compile.device_local_mb, self.post_inference.device_local_mb, ) - @property - def peak_device_shared_mb(self) -> float: - """Peak device shared memory across all phases.""" - return max( - self.baseline.device_shared_mb, - self.post_load.device_shared_mb, - self.post_compile.device_shared_mb, - self.post_inference.device_shared_mb, - ) - def to_dict(self) -> dict[str, Any]: """JSON-serializable dictionary.""" return { - "baseline": self.baseline.to_dict(), - "post_load": self.post_load.to_dict(), - "post_compile": self.post_compile.to_dict(), - "post_inference": self.post_inference.to_dict(), - "peak_working_set_mb": round(self.peak_working_set_mb, 2), - "peak_device_local_mb": round(self.peak_device_local_mb, 2), - "peak_device_shared_mb": round(self.peak_device_shared_mb, 2), - "total_delta_working_set_mb": round(self.total_delta_mb, 2), + "rss_baseline_mb": round(self.baseline.rss_mb, 2), + "rss_after_compile_mb": round(self.post_compile.rss_mb, 2), + "rss_after_inference_mb": round(self.post_inference.rss_mb, 2), + "model_load_delta_mb": round(self.model_load_delta_mb, 2), + "inference_alloc_delta_mb": round(self.inference_alloc_delta_mb, 2), + "total_delta_mb": round(self.total_delta_mb, 2), + "peak_working_set_mb": round(self.peak_wset_mb, 2), + "peak_delta_mb": round(self.peak_delta_mb, 2), + "device_local_mb": round(self.peak_device_local_mb, 2), } @@ -275,13 +271,16 @@ def to_dict(self) -> dict[str, Any]: class MemoryTracker: """Lightweight memory tracker that takes snapshots at phase boundaries. + Follows the same measurement approach as standalone memory scripts: + - Baseline is taken *after* EP initialization (excludes DLL loading) + - Snapshots after compile and after inference warmup + - Deltas show model load cost and inference allocation cost + Usage:: tracker = MemoryTracker() tracker.snapshot_baseline() - # ... load model ... - tracker.snapshot_post_load() - # ... compile ... + # ... load model + compile ... tracker.snapshot_post_compile(adapter_luid="0x...") # ... run benchmark ... tracker.snapshot_post_inference(adapter_luid="0x...") @@ -290,33 +289,30 @@ class MemoryTracker: def __init__(self) -> None: self._baseline: MemorySnapshot | None = None - self._post_load: MemorySnapshot | None = None self._post_compile: MemorySnapshot | None = None self._post_inference: MemorySnapshot | None = None def _take_snapshot(self, adapter_luid: str | None = None) -> MemorySnapshot: """Take a point-in-time memory snapshot.""" - ws, peak_ws, priv, peak_priv = _get_process_memory() + mem = _get_memory_mb() dev_local, dev_shared = _get_device_memory_mb(adapter_luid) return MemorySnapshot( - working_set_mb=ws, - peak_working_set_mb=peak_ws, - private_bytes_mb=priv, - peak_private_bytes_mb=peak_priv, + rss_mb=mem["rss_mb"], + peak_wset_mb=mem["peak_wset_mb"], device_local_mb=dev_local, device_shared_mb=dev_shared, ) def snapshot_baseline(self) -> None: - """Capture baseline memory before model loading.""" - self._baseline = self._take_snapshot() + """Capture baseline memory. - def snapshot_post_load(self) -> None: - """Capture memory after model loading.""" - self._post_load = self._take_snapshot() + Should be called *after* EP registry initialization so that one-time + DLL loading costs are excluded from model measurements. + """ + self._baseline = self._take_snapshot() def snapshot_post_compile(self, adapter_luid: str | None = None) -> None: - """Capture memory after session compilation. + """Capture memory after model load + session compilation. Args: adapter_luid: Adapter LUID for device memory query. @@ -325,7 +321,7 @@ def snapshot_post_compile(self, adapter_luid: str | None = None) -> None: self._post_compile = self._take_snapshot(adapter_luid) def snapshot_post_inference(self, adapter_luid: str | None = None) -> None: - """Capture memory after benchmark completion. + """Capture memory after inference (warmup + benchmark). Args: adapter_luid: Adapter LUID for device memory query. @@ -337,21 +333,16 @@ def profile(self) -> MemoryProfile | None: Returns None if any phase snapshot is missing. """ - if any( - s is None - for s in (self._baseline, self._post_load, self._post_compile, self._post_inference) - ): + if any(s is None for s in (self._baseline, self._post_compile, self._post_inference)): logger.warning("Incomplete memory snapshots, cannot build profile") return None assert self._baseline is not None - assert self._post_load is not None assert self._post_compile is not None assert self._post_inference is not None return MemoryProfile( baseline=self._baseline, - post_load=self._post_load, post_compile=self._post_compile, post_inference=self._post_inference, ) diff --git a/tests/unit/session/monitor/test_memory_tracker.py b/tests/unit/session/monitor/test_memory_tracker.py index a1a7c4a2c..b00cb8470 100644 --- a/tests/unit/session/monitor/test_memory_tracker.py +++ b/tests/unit/session/monitor/test_memory_tracker.py @@ -12,26 +12,22 @@ MemoryProfile, MemorySnapshot, MemoryTracker, - _get_process_memory, + _get_memory_mb, ) -class TestGetProcessMemory: +class TestGetMemoryMb: """Test the process memory retrieval function.""" - def test_returns_four_floats(self) -> None: - result = _get_process_memory() - assert len(result) == 4 - for val in result: - assert isinstance(val, float) + def test_returns_dict_with_expected_keys(self) -> None: + result = _get_memory_mb() + assert "rss_mb" in result + assert "peak_wset_mb" in result - def test_working_set_positive(self) -> None: - ws, peak_ws, priv, peak_priv = _get_process_memory() - # Our process should be using *some* memory - assert ws > 0 - assert peak_ws >= ws - assert priv > 0 - assert peak_priv >= priv + def test_rss_positive(self) -> None: + result = _get_memory_mb() + assert result["rss_mb"] > 0 + assert result["peak_wset_mb"] >= result["rss_mb"] class TestMemorySnapshot: @@ -39,23 +35,20 @@ class TestMemorySnapshot: def test_to_dict(self) -> None: snap = MemorySnapshot( - working_set_mb=100.123, - peak_working_set_mb=120.456, - private_bytes_mb=80.789, - peak_private_bytes_mb=90.012, + rss_mb=100.127, + peak_wset_mb=120.456, device_local_mb=50.347, device_shared_mb=10.678, ) d = snap.to_dict() - assert d["working_set_mb"] == 100.12 - assert d["peak_working_set_mb"] == 120.46 - assert d["private_bytes_mb"] == 80.79 + assert d["rss_mb"] == 100.13 + assert d["peak_wset_mb"] == 120.46 assert d["device_local_mb"] == 50.35 assert d["device_shared_mb"] == 10.68 def test_defaults_are_zero(self) -> None: snap = MemorySnapshot() - assert snap.working_set_mb == 0.0 + assert snap.rss_mb == 0.0 assert snap.device_local_mb == 0.0 @@ -65,64 +58,49 @@ class TestMemoryProfile: @pytest.fixture def profile(self) -> MemoryProfile: return MemoryProfile( - baseline=MemorySnapshot( - working_set_mb=100.0, - peak_working_set_mb=100.0, - private_bytes_mb=120.0, - peak_private_bytes_mb=120.0, - ), - post_load=MemorySnapshot( - working_set_mb=300.0, - peak_working_set_mb=310.0, - private_bytes_mb=350.0, - peak_private_bytes_mb=350.0, - ), + baseline=MemorySnapshot(rss_mb=100.0, peak_wset_mb=100.0), post_compile=MemorySnapshot( - working_set_mb=320.0, - peak_working_set_mb=325.0, - private_bytes_mb=370.0, - peak_private_bytes_mb=375.0, + rss_mb=320.0, + peak_wset_mb=350.0, device_local_mb=50.0, ), post_inference=MemorySnapshot( - working_set_mb=330.0, - peak_working_set_mb=340.0, - private_bytes_mb=380.0, - peak_private_bytes_mb=385.0, + rss_mb=330.0, + peak_wset_mb=360.0, device_local_mb=52.0, device_shared_mb=8.0, ), ) - def test_load_delta(self, profile: MemoryProfile) -> None: - assert profile.load_delta_mb == pytest.approx(200.0) - - def test_compile_delta(self, profile: MemoryProfile) -> None: - assert profile.compile_delta_mb == pytest.approx(20.0) + def test_model_load_delta(self, profile: MemoryProfile) -> None: + assert profile.model_load_delta_mb == pytest.approx(220.0) - def test_inference_delta(self, profile: MemoryProfile) -> None: - assert profile.inference_delta_mb == pytest.approx(10.0) + def test_inference_alloc_delta(self, profile: MemoryProfile) -> None: + assert profile.inference_alloc_delta_mb == pytest.approx(10.0) def test_total_delta(self, profile: MemoryProfile) -> None: assert profile.total_delta_mb == pytest.approx(230.0) - def test_peak_working_set(self, profile: MemoryProfile) -> None: - assert profile.peak_working_set_mb == pytest.approx(340.0) + def test_peak_wset(self, profile: MemoryProfile) -> None: + assert profile.peak_wset_mb == pytest.approx(360.0) + + def test_peak_delta(self, profile: MemoryProfile) -> None: + assert profile.peak_delta_mb == pytest.approx(260.0) def test_peak_device_local(self, profile: MemoryProfile) -> None: assert profile.peak_device_local_mb == pytest.approx(52.0) - def test_peak_device_shared(self, profile: MemoryProfile) -> None: - assert profile.peak_device_shared_mb == pytest.approx(8.0) - def test_to_dict(self, profile: MemoryProfile) -> None: d = profile.to_dict() - assert "baseline" in d - assert "post_load" in d - assert "post_compile" in d - assert "post_inference" in d - assert d["peak_working_set_mb"] == 340.0 - assert d["total_delta_working_set_mb"] == 230.0 + assert d["rss_baseline_mb"] == 100.0 + assert d["rss_after_compile_mb"] == 320.0 + assert d["rss_after_inference_mb"] == 330.0 + assert d["model_load_delta_mb"] == 220.0 + assert d["inference_alloc_delta_mb"] == 10.0 + assert d["total_delta_mb"] == 230.0 + assert d["peak_working_set_mb"] == 360.0 + assert d["peak_delta_mb"] == 260.0 + assert d["device_local_mb"] == 52.0 class TestMemoryTracker: @@ -131,14 +109,13 @@ class TestMemoryTracker: def test_full_workflow(self) -> None: tracker = MemoryTracker() tracker.snapshot_baseline() - tracker.snapshot_post_load() tracker.snapshot_post_compile() tracker.snapshot_post_inference() profile = tracker.profile() assert profile is not None - assert profile.baseline.working_set_mb > 0 - assert profile.post_inference.working_set_mb > 0 + assert profile.baseline.rss_mb > 0 + assert profile.post_inference.rss_mb > 0 def test_incomplete_returns_none(self) -> None: tracker = MemoryTracker() @@ -148,20 +125,19 @@ def test_incomplete_returns_none(self) -> None: assert profile is None def test_snapshots_are_nondecreasing(self) -> None: - """Working set should generally not decrease between adjacent snapshots.""" + """RSS should generally not decrease between adjacent snapshots.""" tracker = MemoryTracker() tracker.snapshot_baseline() # Allocate something to ensure memory grows _data = [bytearray(1024 * 1024) for _ in range(5)] # ~5 MB - tracker.snapshot_post_load() tracker.snapshot_post_compile() tracker.snapshot_post_inference() profile = tracker.profile() assert profile is not None - # post_load should be >= baseline (we allocated memory) - assert profile.post_load.working_set_mb >= profile.baseline.working_set_mb - # Keep _data alive until assertions complete so memory isn't reclaimed early + # post_compile should be >= baseline (we allocated memory) + assert profile.post_compile.rss_mb >= profile.baseline.rss_mb + # Keep _data alive until assertions complete assert _data is not None From ed8fac10b4aa0919ff79b33207bf8afd842e4b71 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 12 Jun 2026 17:46:18 +0800 Subject: [PATCH 07/27] feat: warmup EP before memory baseline Call WinMLSession._init_winml_eps_once() before taking the baseline snapshot so one-time EP DLL loading costs are excluded from the model_load_delta metric. --- src/winml/modelkit/commands/perf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index b07224a18..1cefa7c50 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -297,11 +297,13 @@ def run(self) -> BenchmarkResult: BenchmarkResult with timing statistics """ # Initialize memory tracker if enabled. - # Baseline is taken here — after Python/ORT/EP DLLs are loaded but - # before model-specific work, so EP initialization cost is excluded. + # Warm up EP registry first so one-time DLL loading costs (~190 MB + # for OV plugin) are excluded from model memory measurements. if self.config.memory: from ..session.monitor.memory_tracker import MemoryTracker + from ..session.session import WinMLSession + WinMLSession._init_winml_eps_once() self._memory_tracker = MemoryTracker() self._memory_tracker.snapshot_baseline() From e57e8dc406c0e38f8b7487f7fd480a65915ae7cd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 12 Jun 2026 19:26:50 +0800 Subject: [PATCH 08/27] fix: correct memory measurement phase ordering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reorder: load → compile → snapshot → generate_inputs → benchmark (matches reference: input alloc now in inference delta, not model load) - Add gc.collect() before baseline and post_compile snapshots - Fix format bug: use :+.1f for deltas (shows -24.8 not +-24.8) --- src/winml/modelkit/commands/perf.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 1cefa7c50..92927afa4 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -296,14 +296,17 @@ def run(self) -> BenchmarkResult: Returns: BenchmarkResult with timing statistics """ + import gc + # Initialize memory tracker if enabled. # Warm up EP registry first so one-time DLL loading costs (~190 MB - # for OV plugin) are excluded from model memory measurements. + # for OV plugin) are excluded from model measurements. if self.config.memory: from ..session.monitor.memory_tracker import MemoryTracker from ..session.session import WinMLSession WinMLSession._init_winml_eps_once() + gc.collect() self._memory_tracker = MemoryTracker() self._memory_tracker.snapshot_baseline() @@ -312,14 +315,13 @@ def run(self) -> BenchmarkResult: self._load_model() assert self._model is not None - # [2] Generate inputs - logger.info("Generating benchmark inputs") - self._generate_inputs() - - # Compile session early so model.device is resolved for display + # [2] Compile session so model.device is resolved for display. + # Snapshot is taken RIGHT after compile (before input generation) + # to match reference: model_load_delta = load + compile only. self._model._session.compile() if self._memory_tracker: + gc.collect() adapter_luid = self._resolve_adapter_luid() self._memory_tracker.snapshot_post_compile(adapter_luid=adapter_luid) @@ -332,7 +334,11 @@ def run(self) -> BenchmarkResult: ep_name=self._model.ep_name, ) - # [3] Run benchmark + # [3] Generate inputs (after compile so its memory is in inference delta) + logger.info("Generating benchmark inputs") + self._generate_inputs() + + # [4] Run benchmark logger.info( "Running benchmark: %d iterations + %d warmup", self.config.iterations, @@ -344,7 +350,7 @@ def run(self) -> BenchmarkResult: adapter_luid = self._resolve_adapter_luid() self._memory_tracker.snapshot_post_inference(adapter_luid=adapter_luid) - # [4] Collect results + # [5] Collect results logger.info("Collecting results") return self._collect_results(stats) @@ -947,9 +953,9 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: console.print() console.print(f"[bold]Memory:[/bold] {rss:.1f} MB (process){dev_str}") console.print( - f" [dim]model load: +{mem.model_load_delta_mb:.1f} MB | " - f"inference: +{mem.inference_alloc_delta_mb:.1f} MB | " - f"total: +{mem.total_delta_mb:.1f} MB[/dim]" + f" [dim]model load: {mem.model_load_delta_mb:+.1f} MB | " + f"inference: {mem.inference_alloc_delta_mb:+.1f} MB | " + f"total: {mem.total_delta_mb:+.1f} MB[/dim]" ) console.print() From 9665a5498850dab09b5f9cced87df490e3e300eb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 15 Jun 2026 21:18:30 +0800 Subject: [PATCH 09/27] simplify: reduce memory_tracker to ~100 lines - Remove ctypes/proc fallback (psutil is already a dependency) - Remove peak_wset_mb, peak_delta_mb (not meaningful for inference) - Remove device_shared_mb (unused in display) - Remove MemorySnapshot class (just store raw floats) - Flatten MemoryProfile fields for direct access --- src/winml/modelkit/commands/perf.py | 9 +- .../session/monitor/memory_tracker.py | 291 +++--------------- .../session/monitor/test_memory_tracker.py | 88 ++---- 3 files changed, 67 insertions(+), 321 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 92927afa4..0be663ce0 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -946,12 +946,11 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: # Memory section (only when --memory is enabled) if result.memory_profile: mem = result.memory_profile - dev_str = ( - f" | {mem.peak_device_local_mb:.1f} MB (device)" if mem.peak_device_local_mb > 0 else "" - ) - rss = mem.post_inference.rss_mb + dev_str = f" | {mem.device_local_mb:.1f} MB (device)" if mem.device_local_mb > 0 else "" console.print() - console.print(f"[bold]Memory:[/bold] {rss:.1f} MB (process){dev_str}") + console.print( + f"[bold]Memory:[/bold] {mem.rss_after_inference_mb:.1f} MB (process){dev_str}" + ) console.print( f" [dim]model load: {mem.model_load_delta_mb:+.1f} MB | " f"inference: {mem.inference_alloc_delta_mb:+.1f} MB | " diff --git a/src/winml/modelkit/session/monitor/memory_tracker.py b/src/winml/modelkit/session/monitor/memory_tracker.py index 880e674fc..53c625eac 100644 --- a/src/winml/modelkit/session/monitor/memory_tracker.py +++ b/src/winml/modelkit/session/monitor/memory_tracker.py @@ -4,145 +4,35 @@ # -------------------------------------------------------------------------- """Process memory tracking for perf benchmarking. -Measures RSS (Resident Set Size) at benchmark phase boundaries to compute -memory deltas for model loading, compilation, and inference. Uses the same -approach as standalone memory measurement scripts: psutil for process memory -with a ctypes fallback on Windows. - -The tracker excludes one-time EP initialization costs (DLL loading) by -taking the baseline *after* the EP registry is warmed up. +Measures RSS at benchmark phase boundaries to compute memory deltas for +model loading and inference. """ from __future__ import annotations -import ctypes import logging import os import sys from dataclasses import dataclass -from pathlib import Path -from typing import Any, ClassVar - +from typing import Any -if sys.platform == "win32": - import ctypes.wintypes as wintypes +import psutil logger = logging.getLogger(__name__) - -# ============================================================================= -# Memory Measurement -# ============================================================================= - _MB = 1024 * 1024 -def _get_memory_mb() -> dict[str, float]: - """Return current RSS and peak working set in MB for this process. - - Tries psutil first (cross-platform), falls back to ctypes on Windows - or /proc/self/status on Linux. - """ - try: - import psutil - - proc = psutil.Process(os.getpid()) - info = proc.memory_info() - return { - "rss_mb": info.rss / _MB, - "peak_wset_mb": getattr(info, "peak_wset", info.rss) / _MB, - } - except ImportError: - pass - - # Fallback: platform-specific - if sys.platform == "win32": - return _get_memory_mb_win32() - return _get_memory_mb_linux() - - -if sys.platform == "win32": - - class _ProcessMemoryCountersEx(ctypes.Structure): - """PROCESS_MEMORY_COUNTERS_EX structure from psapi.h.""" - - _fields_: ClassVar = [ - ("cb", wintypes.DWORD), - ("PageFaultCount", wintypes.DWORD), - ("PeakWorkingSetSize", ctypes.c_size_t), - ("WorkingSetSize", ctypes.c_size_t), - ("QuotaPeakPagedPoolUsage", ctypes.c_size_t), - ("QuotaPagedPoolUsage", ctypes.c_size_t), - ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t), - ("QuotaNonPagedPoolUsage", ctypes.c_size_t), - ("PagefileUsage", ctypes.c_size_t), - ("PeakPagefileUsage", ctypes.c_size_t), - ("PrivateUsage", ctypes.c_size_t), - ] - - -def _get_memory_mb_win32() -> dict[str, float]: - """Fallback for Windows: ctypes K32GetProcessMemoryInfo.""" - kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) - kernel32.GetCurrentProcess.restype = wintypes.HANDLE - kernel32.K32GetProcessMemoryInfo.restype = wintypes.BOOL - kernel32.K32GetProcessMemoryInfo.argtypes = [ - wintypes.HANDLE, - ctypes.POINTER(_ProcessMemoryCountersEx), - wintypes.DWORD, - ] - - handle = kernel32.GetCurrentProcess() - counters = _ProcessMemoryCountersEx() - counters.cb = ctypes.sizeof(counters) - - success = kernel32.K32GetProcessMemoryInfo(handle, ctypes.byref(counters), counters.cb) - if not success: - logger.warning("K32GetProcessMemoryInfo failed, returning zeros") - return {"rss_mb": 0.0, "peak_wset_mb": 0.0} - - return { - "rss_mb": counters.WorkingSetSize / _MB, - "peak_wset_mb": counters.PeakWorkingSetSize / _MB, - } - - -def _get_memory_mb_linux() -> dict[str, float]: - """Fallback for Linux: read /proc/self/status.""" - try: - with Path("/proc/self/status").open() as f: - content = f.read() - - values: dict[str, float] = {} - for line in content.splitlines(): - parts = line.split() - if len(parts) >= 2 and parts[0].rstrip(":") in ("VmRSS", "VmPeak"): - values[parts[0].rstrip(":")] = float(parts[1]) / 1024 # kB -> MB - - rss = values.get("VmRSS", 0.0) - peak = values.get("VmPeak", 0.0) - return {"rss_mb": rss, "peak_wset_mb": peak} - except OSError: - return {"rss_mb": 0.0, "peak_wset_mb": 0.0} - +def _get_rss_mb() -> float: + """Return current RSS in MB for this process.""" + return psutil.Process(os.getpid()).memory_info().rss / _MB -# ============================================================================= -# Device Memory via single-shot PDH query -# ============================================================================= - -def _get_device_memory_mb(luid: str | None) -> tuple[float, float]: - """Single-shot PDH query for device memory (local, shared) in MB. - - Args: - luid: Adapter LUID string. If None, returns (0, 0). - - Returns: - (local_mb, shared_mb) - """ +def _get_device_memory_mb(luid: str | None) -> float: + """Single-shot PDH query for device local memory in MB.""" if luid is None or sys.platform != "win32": - return (0.0, 0.0) + return 0.0 try: from ._pdh import PdhQuery @@ -151,130 +41,63 @@ def _get_device_memory_mb(luid: str | None) -> tuple[float, float]: query = PdhQuery() query.open() - local_ok = query.add_counter( + ok = query.add_counter( "local", rf"\GPU Process Memory(pid_{pid}_luid_{luid}_phys_0)\Local Usage", fmt="large", ) - shared_ok = query.add_counter( - "shared", - rf"\GPU Process Memory(pid_{pid}_luid_{luid}_phys_0)\Shared Usage", - fmt="large", - ) - - if not local_ok and not shared_ok: + if not ok: query.close() - return (0.0, 0.0) + return 0.0 query.prime() values = query.collect() query.close() - - local_bytes = values.get("local") or 0 - shared_bytes = values.get("shared") or 0 - return (local_bytes / _MB, shared_bytes / _MB) + return (values.get("local") or 0) / _MB except Exception: logger.debug("Device memory query failed", exc_info=True) - return (0.0, 0.0) - - -# ============================================================================= -# Data Classes -# ============================================================================= - - -@dataclass -class MemorySnapshot: - """A point-in-time memory measurement.""" - - rss_mb: float = 0.0 - peak_wset_mb: float = 0.0 - device_local_mb: float = 0.0 - device_shared_mb: float = 0.0 - - def to_dict(self) -> dict[str, float]: - """JSON-serializable dictionary.""" - return { - "rss_mb": round(self.rss_mb, 2), - "peak_wset_mb": round(self.peak_wset_mb, 2), - "device_local_mb": round(self.device_local_mb, 2), - "device_shared_mb": round(self.device_shared_mb, 2), - } + return 0.0 @dataclass class MemoryProfile: - """Memory measurements across benchmark phases. + """Memory measurements across benchmark phases.""" - Mirrors the structure used in standalone memory measurement scripts: - baseline → after_compile → after_warmup, with computed deltas. - """ - - baseline: MemorySnapshot - post_compile: MemorySnapshot - post_inference: MemorySnapshot + rss_baseline_mb: float + rss_after_compile_mb: float + rss_after_inference_mb: float + device_local_mb: float = 0.0 @property def model_load_delta_mb(self) -> float: """RSS increase from model loading + compilation.""" - return self.post_compile.rss_mb - self.baseline.rss_mb + return self.rss_after_compile_mb - self.rss_baseline_mb @property def inference_alloc_delta_mb(self) -> float: """RSS increase from inference (warmup + benchmark).""" - return self.post_inference.rss_mb - self.post_compile.rss_mb + return self.rss_after_inference_mb - self.rss_after_compile_mb @property def total_delta_mb(self) -> float: """Total RSS increase from baseline.""" - return self.post_inference.rss_mb - self.baseline.rss_mb - - @property - def peak_wset_mb(self) -> float: - """Peak working set (from OS counter at end of benchmark).""" - return self.post_inference.peak_wset_mb - - @property - def peak_delta_mb(self) -> float: - """Peak working set increase from baseline.""" - return self.post_inference.peak_wset_mb - self.baseline.peak_wset_mb - - @property - def peak_device_local_mb(self) -> float: - """Peak device local memory across all phases.""" - return max( - self.baseline.device_local_mb, - self.post_compile.device_local_mb, - self.post_inference.device_local_mb, - ) + return self.rss_after_inference_mb - self.rss_baseline_mb def to_dict(self) -> dict[str, Any]: """JSON-serializable dictionary.""" return { - "rss_baseline_mb": round(self.baseline.rss_mb, 2), - "rss_after_compile_mb": round(self.post_compile.rss_mb, 2), - "rss_after_inference_mb": round(self.post_inference.rss_mb, 2), + "rss_baseline_mb": round(self.rss_baseline_mb, 2), + "rss_after_compile_mb": round(self.rss_after_compile_mb, 2), + "rss_after_inference_mb": round(self.rss_after_inference_mb, 2), "model_load_delta_mb": round(self.model_load_delta_mb, 2), "inference_alloc_delta_mb": round(self.inference_alloc_delta_mb, 2), "total_delta_mb": round(self.total_delta_mb, 2), - "peak_working_set_mb": round(self.peak_wset_mb, 2), - "peak_delta_mb": round(self.peak_delta_mb, 2), - "device_local_mb": round(self.peak_device_local_mb, 2), + "device_local_mb": round(self.device_local_mb, 2), } -# ============================================================================= -# MemoryTracker -# ============================================================================= - - class MemoryTracker: - """Lightweight memory tracker that takes snapshots at phase boundaries. - - Follows the same measurement approach as standalone memory scripts: - - Baseline is taken *after* EP initialization (excludes DLL loading) - - Snapshots after compile and after inference warmup - - Deltas show model load cost and inference allocation cost + """Lightweight memory tracker that takes RSS snapshots at phase boundaries. Usage:: @@ -288,53 +111,28 @@ class MemoryTracker: """ def __init__(self) -> None: - self._baseline: MemorySnapshot | None = None - self._post_compile: MemorySnapshot | None = None - self._post_inference: MemorySnapshot | None = None - - def _take_snapshot(self, adapter_luid: str | None = None) -> MemorySnapshot: - """Take a point-in-time memory snapshot.""" - mem = _get_memory_mb() - dev_local, dev_shared = _get_device_memory_mb(adapter_luid) - return MemorySnapshot( - rss_mb=mem["rss_mb"], - peak_wset_mb=mem["peak_wset_mb"], - device_local_mb=dev_local, - device_shared_mb=dev_shared, - ) + self._baseline: float | None = None + self._post_compile: float | None = None + self._post_inference: float | None = None + self._device_local_mb: float = 0.0 def snapshot_baseline(self) -> None: - """Capture baseline memory. - - Should be called *after* EP registry initialization so that one-time - DLL loading costs are excluded from model measurements. - """ - self._baseline = self._take_snapshot() + """Capture baseline RSS (call after EP warmup).""" + self._baseline = _get_rss_mb() def snapshot_post_compile(self, adapter_luid: str | None = None) -> None: - """Capture memory after model load + session compilation. - - Args: - adapter_luid: Adapter LUID for device memory query. - Available after compile resolves the EP. - """ - self._post_compile = self._take_snapshot(adapter_luid) + """Capture RSS after model load + compile.""" + self._post_compile = _get_rss_mb() + self._device_local_mb = max(self._device_local_mb, _get_device_memory_mb(adapter_luid)) def snapshot_post_inference(self, adapter_luid: str | None = None) -> None: - """Capture memory after inference (warmup + benchmark). - - Args: - adapter_luid: Adapter LUID for device memory query. - """ - self._post_inference = self._take_snapshot(adapter_luid) + """Capture RSS after inference.""" + self._post_inference = _get_rss_mb() + self._device_local_mb = max(self._device_local_mb, _get_device_memory_mb(adapter_luid)) def profile(self) -> MemoryProfile | None: - """Build a complete MemoryProfile from collected snapshots. - - Returns None if any phase snapshot is missing. - """ + """Build MemoryProfile. Returns None if any snapshot is missing.""" if any(s is None for s in (self._baseline, self._post_compile, self._post_inference)): - logger.warning("Incomplete memory snapshots, cannot build profile") return None assert self._baseline is not None @@ -342,7 +140,8 @@ def profile(self) -> MemoryProfile | None: assert self._post_inference is not None return MemoryProfile( - baseline=self._baseline, - post_compile=self._post_compile, - post_inference=self._post_inference, + rss_baseline_mb=self._baseline, + rss_after_compile_mb=self._post_compile, + rss_after_inference_mb=self._post_inference, + device_local_mb=self._device_local_mb, ) diff --git a/tests/unit/session/monitor/test_memory_tracker.py b/tests/unit/session/monitor/test_memory_tracker.py index b00cb8470..ab0364e17 100644 --- a/tests/unit/session/monitor/test_memory_tracker.py +++ b/tests/unit/session/monitor/test_memory_tracker.py @@ -10,46 +10,18 @@ from winml.modelkit.session.monitor.memory_tracker import ( MemoryProfile, - MemorySnapshot, MemoryTracker, - _get_memory_mb, + _get_rss_mb, ) -class TestGetMemoryMb: - """Test the process memory retrieval function.""" +class TestGetRssMb: + """Test the process RSS retrieval function.""" - def test_returns_dict_with_expected_keys(self) -> None: - result = _get_memory_mb() - assert "rss_mb" in result - assert "peak_wset_mb" in result - - def test_rss_positive(self) -> None: - result = _get_memory_mb() - assert result["rss_mb"] > 0 - assert result["peak_wset_mb"] >= result["rss_mb"] - - -class TestMemorySnapshot: - """Test MemorySnapshot dataclass.""" - - def test_to_dict(self) -> None: - snap = MemorySnapshot( - rss_mb=100.127, - peak_wset_mb=120.456, - device_local_mb=50.347, - device_shared_mb=10.678, - ) - d = snap.to_dict() - assert d["rss_mb"] == 100.13 - assert d["peak_wset_mb"] == 120.46 - assert d["device_local_mb"] == 50.35 - assert d["device_shared_mb"] == 10.68 - - def test_defaults_are_zero(self) -> None: - snap = MemorySnapshot() - assert snap.rss_mb == 0.0 - assert snap.device_local_mb == 0.0 + def test_returns_positive_float(self) -> None: + rss = _get_rss_mb() + assert isinstance(rss, float) + assert rss > 0 class TestMemoryProfile: @@ -58,18 +30,10 @@ class TestMemoryProfile: @pytest.fixture def profile(self) -> MemoryProfile: return MemoryProfile( - baseline=MemorySnapshot(rss_mb=100.0, peak_wset_mb=100.0), - post_compile=MemorySnapshot( - rss_mb=320.0, - peak_wset_mb=350.0, - device_local_mb=50.0, - ), - post_inference=MemorySnapshot( - rss_mb=330.0, - peak_wset_mb=360.0, - device_local_mb=52.0, - device_shared_mb=8.0, - ), + rss_baseline_mb=100.0, + rss_after_compile_mb=320.0, + rss_after_inference_mb=330.0, + device_local_mb=52.0, ) def test_model_load_delta(self, profile: MemoryProfile) -> None: @@ -81,15 +45,6 @@ def test_inference_alloc_delta(self, profile: MemoryProfile) -> None: def test_total_delta(self, profile: MemoryProfile) -> None: assert profile.total_delta_mb == pytest.approx(230.0) - def test_peak_wset(self, profile: MemoryProfile) -> None: - assert profile.peak_wset_mb == pytest.approx(360.0) - - def test_peak_delta(self, profile: MemoryProfile) -> None: - assert profile.peak_delta_mb == pytest.approx(260.0) - - def test_peak_device_local(self, profile: MemoryProfile) -> None: - assert profile.peak_device_local_mb == pytest.approx(52.0) - def test_to_dict(self, profile: MemoryProfile) -> None: d = profile.to_dict() assert d["rss_baseline_mb"] == 100.0 @@ -98,8 +53,6 @@ def test_to_dict(self, profile: MemoryProfile) -> None: assert d["model_load_delta_mb"] == 220.0 assert d["inference_alloc_delta_mb"] == 10.0 assert d["total_delta_mb"] == 230.0 - assert d["peak_working_set_mb"] == 360.0 - assert d["peak_delta_mb"] == 260.0 assert d["device_local_mb"] == 52.0 @@ -114,30 +67,25 @@ def test_full_workflow(self) -> None: profile = tracker.profile() assert profile is not None - assert profile.baseline.rss_mb > 0 - assert profile.post_inference.rss_mb > 0 + assert profile.rss_baseline_mb > 0 + assert profile.rss_after_inference_mb > 0 def test_incomplete_returns_none(self) -> None: tracker = MemoryTracker() tracker.snapshot_baseline() - # Missing other phases - profile = tracker.profile() - assert profile is None + assert tracker.profile() is None - def test_snapshots_are_nondecreasing(self) -> None: - """RSS should generally not decrease between adjacent snapshots.""" + def test_deltas_nonnegative_with_allocation(self) -> None: tracker = MemoryTracker() tracker.snapshot_baseline() - # Allocate something to ensure memory grows - _data = [bytearray(1024 * 1024) for _ in range(5)] # ~5 MB + # Allocate ~5 MB to ensure RSS grows + _data = [bytearray(1024 * 1024) for _ in range(5)] tracker.snapshot_post_compile() tracker.snapshot_post_inference() profile = tracker.profile() assert profile is not None - # post_compile should be >= baseline (we allocated memory) - assert profile.post_compile.rss_mb >= profile.baseline.rss_mb - # Keep _data alive until assertions complete + assert profile.model_load_delta_mb >= 0 assert _data is not None From b4e8bd3504d2a243ee5e46bd3fdb6ae2a75183c8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 15 Jun 2026 21:25:02 +0800 Subject: [PATCH 10/27] simplify: remove MemoryTracker/MemoryProfile classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inline RSS measurement directly in PerfBenchmark.run() — just 3 calls to get_rss_mb() and arithmetic. memory_tracker.py is now only two functions (~50 lines). No classes, no state management. --- src/winml/modelkit/commands/perf.py | 61 ++++++----- .../session/monitor/memory_tracker.py | 103 +----------------- .../session/monitor/test_memory_tracker.py | 82 ++------------ 3 files changed, 43 insertions(+), 203 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 0be663ce0..64ad27e5c 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -26,7 +26,6 @@ from rich.console import Console from rich.table import Table -from ..session.monitor.memory_tracker import MemoryProfile from ..utils import cli as cli_utils from ..utils.constants import EPName, EPNameOrAlias from ..utils.logging import configure_logging @@ -131,8 +130,8 @@ class BenchmarkResult: # Hardware monitor metrics (from HWMonitor.to_dict()) hw_monitor: dict[str, Any] | None = None - # Memory profile (from MemoryTracker) - memory_profile: MemoryProfile | None = None + # Memory profile dict (rss deltas from memory_tracker) + memory_profile: dict[str, float] | None = None def to_dict(self) -> dict[str, Any]: """Convert to dictionary for JSON serialization.""" @@ -175,7 +174,7 @@ def to_dict(self) -> dict[str, Any]: if self.hw_monitor: result["hw_monitor"] = self.hw_monitor if self.memory_profile: - result["memory"] = self.memory_profile.to_dict() + result["memory"] = self.memory_profile return result @@ -288,7 +287,7 @@ def __init__(self, config: BenchmarkConfig) -> None: self.config = config self._model: WinMLPreTrainedModel | None = None self._inputs: dict[str, np.ndarray] | None = None - self._memory_tracker: Any = None + self._memory: dict[str, float] | None = None def run(self) -> BenchmarkResult: """Execute full benchmark pipeline. @@ -298,32 +297,26 @@ def run(self) -> BenchmarkResult: """ import gc - # Initialize memory tracker if enabled. - # Warm up EP registry first so one-time DLL loading costs (~190 MB - # for OV plugin) are excluded from model measurements. + # Memory measurement: warmup EP, then take RSS at phase boundaries. if self.config.memory: - from ..session.monitor.memory_tracker import MemoryTracker + from ..session.monitor.memory_tracker import get_device_memory_mb, get_rss_mb from ..session.session import WinMLSession WinMLSession._init_winml_eps_once() gc.collect() - self._memory_tracker = MemoryTracker() - self._memory_tracker.snapshot_baseline() + rss_baseline = get_rss_mb() # [1] Load model logger.info("Loading model: %s", self.config.model_id) self._load_model() assert self._model is not None - # [2] Compile session so model.device is resolved for display. - # Snapshot is taken RIGHT after compile (before input generation) - # to match reference: model_load_delta = load + compile only. + # [2] Compile session self._model._session.compile() - if self._memory_tracker: + if self.config.memory: gc.collect() - adapter_luid = self._resolve_adapter_luid() - self._memory_tracker.snapshot_post_compile(adapter_luid=adapter_luid) + rss_after_compile = get_rss_mb() # Print model info before benchmark starts _print_model_info( @@ -334,11 +327,10 @@ def run(self) -> BenchmarkResult: ep_name=self._model.ep_name, ) - # [3] Generate inputs (after compile so its memory is in inference delta) + # [3] Generate inputs + run benchmark logger.info("Generating benchmark inputs") self._generate_inputs() - # [4] Run benchmark logger.info( "Running benchmark: %d iterations + %d warmup", self.config.iterations, @@ -346,11 +338,21 @@ def run(self) -> BenchmarkResult: ) stats = self._run_benchmark() - if self._memory_tracker: + if self.config.memory: + rss_after_inference = get_rss_mb() adapter_luid = self._resolve_adapter_luid() - self._memory_tracker.snapshot_post_inference(adapter_luid=adapter_luid) - - # [5] Collect results + device_mb = get_device_memory_mb(adapter_luid) + self._memory = { + "rss_baseline_mb": round(rss_baseline, 2), + "rss_after_compile_mb": round(rss_after_compile, 2), + "rss_after_inference_mb": round(rss_after_inference, 2), + "model_load_delta_mb": round(rss_after_compile - rss_baseline, 2), + "inference_alloc_delta_mb": round(rss_after_inference - rss_after_compile, 2), + "total_delta_mb": round(rss_after_inference - rss_baseline, 2), + "device_local_mb": round(device_mb, 2), + } + + # [4] Collect results logger.info("Collecting results") return self._collect_results(stats) @@ -585,7 +587,7 @@ def _collect_results(self, stats: PerfStats) -> BenchmarkResult: # Hardware monitor metrics (only present when --monitor is used) hw_monitor=getattr(self, "_hw_metrics", None), # Memory profile (only present when --memory is used) - memory_profile=(self._memory_tracker.profile() if self._memory_tracker else None), + memory_profile=self._memory, ) @@ -946,15 +948,16 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: # Memory section (only when --memory is enabled) if result.memory_profile: mem = result.memory_profile - dev_str = f" | {mem.device_local_mb:.1f} MB (device)" if mem.device_local_mb > 0 else "" + dev_mb = mem.get("device_local_mb", 0) + dev_str = f" | {dev_mb:.1f} MB (device)" if dev_mb > 0 else "" console.print() console.print( - f"[bold]Memory:[/bold] {mem.rss_after_inference_mb:.1f} MB (process){dev_str}" + f"[bold]Memory:[/bold] {mem['rss_after_inference_mb']:.1f} MB (process){dev_str}" ) console.print( - f" [dim]model load: {mem.model_load_delta_mb:+.1f} MB | " - f"inference: {mem.inference_alloc_delta_mb:+.1f} MB | " - f"total: {mem.total_delta_mb:+.1f} MB[/dim]" + f" [dim]model load: {mem['model_load_delta_mb']:+.1f} MB | " + f"inference: {mem['inference_alloc_delta_mb']:+.1f} MB | " + f"total: {mem['total_delta_mb']:+.1f} MB[/dim]" ) console.print() diff --git a/src/winml/modelkit/session/monitor/memory_tracker.py b/src/winml/modelkit/session/monitor/memory_tracker.py index 53c625eac..80cc67bd5 100644 --- a/src/winml/modelkit/session/monitor/memory_tracker.py +++ b/src/winml/modelkit/session/monitor/memory_tracker.py @@ -2,19 +2,13 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. # -------------------------------------------------------------------------- -"""Process memory tracking for perf benchmarking. - -Measures RSS at benchmark phase boundaries to compute memory deltas for -model loading and inference. -""" +"""Process memory helpers for perf benchmarking.""" from __future__ import annotations import logging import os import sys -from dataclasses import dataclass -from typing import Any import psutil @@ -24,23 +18,21 @@ _MB = 1024 * 1024 -def _get_rss_mb() -> float: +def get_rss_mb() -> float: """Return current RSS in MB for this process.""" return psutil.Process(os.getpid()).memory_info().rss / _MB -def _get_device_memory_mb(luid: str | None) -> float: +def get_device_memory_mb(luid: str | None) -> float: """Single-shot PDH query for device local memory in MB.""" if luid is None or sys.platform != "win32": return 0.0 - try: from ._pdh import PdhQuery pid = os.getpid() query = PdhQuery() query.open() - ok = query.add_counter( "local", rf"\GPU Process Memory(pid_{pid}_luid_{luid}_phys_0)\Local Usage", @@ -49,7 +41,6 @@ def _get_device_memory_mb(luid: str | None) -> float: if not ok: query.close() return 0.0 - query.prime() values = query.collect() query.close() @@ -57,91 +48,3 @@ def _get_device_memory_mb(luid: str | None) -> float: except Exception: logger.debug("Device memory query failed", exc_info=True) return 0.0 - - -@dataclass -class MemoryProfile: - """Memory measurements across benchmark phases.""" - - rss_baseline_mb: float - rss_after_compile_mb: float - rss_after_inference_mb: float - device_local_mb: float = 0.0 - - @property - def model_load_delta_mb(self) -> float: - """RSS increase from model loading + compilation.""" - return self.rss_after_compile_mb - self.rss_baseline_mb - - @property - def inference_alloc_delta_mb(self) -> float: - """RSS increase from inference (warmup + benchmark).""" - return self.rss_after_inference_mb - self.rss_after_compile_mb - - @property - def total_delta_mb(self) -> float: - """Total RSS increase from baseline.""" - return self.rss_after_inference_mb - self.rss_baseline_mb - - def to_dict(self) -> dict[str, Any]: - """JSON-serializable dictionary.""" - return { - "rss_baseline_mb": round(self.rss_baseline_mb, 2), - "rss_after_compile_mb": round(self.rss_after_compile_mb, 2), - "rss_after_inference_mb": round(self.rss_after_inference_mb, 2), - "model_load_delta_mb": round(self.model_load_delta_mb, 2), - "inference_alloc_delta_mb": round(self.inference_alloc_delta_mb, 2), - "total_delta_mb": round(self.total_delta_mb, 2), - "device_local_mb": round(self.device_local_mb, 2), - } - - -class MemoryTracker: - """Lightweight memory tracker that takes RSS snapshots at phase boundaries. - - Usage:: - - tracker = MemoryTracker() - tracker.snapshot_baseline() - # ... load model + compile ... - tracker.snapshot_post_compile(adapter_luid="0x...") - # ... run benchmark ... - tracker.snapshot_post_inference(adapter_luid="0x...") - profile = tracker.profile() - """ - - def __init__(self) -> None: - self._baseline: float | None = None - self._post_compile: float | None = None - self._post_inference: float | None = None - self._device_local_mb: float = 0.0 - - def snapshot_baseline(self) -> None: - """Capture baseline RSS (call after EP warmup).""" - self._baseline = _get_rss_mb() - - def snapshot_post_compile(self, adapter_luid: str | None = None) -> None: - """Capture RSS after model load + compile.""" - self._post_compile = _get_rss_mb() - self._device_local_mb = max(self._device_local_mb, _get_device_memory_mb(adapter_luid)) - - def snapshot_post_inference(self, adapter_luid: str | None = None) -> None: - """Capture RSS after inference.""" - self._post_inference = _get_rss_mb() - self._device_local_mb = max(self._device_local_mb, _get_device_memory_mb(adapter_luid)) - - def profile(self) -> MemoryProfile | None: - """Build MemoryProfile. Returns None if any snapshot is missing.""" - if any(s is None for s in (self._baseline, self._post_compile, self._post_inference)): - return None - - assert self._baseline is not None - assert self._post_compile is not None - assert self._post_inference is not None - - return MemoryProfile( - rss_baseline_mb=self._baseline, - rss_after_compile_mb=self._post_compile, - rss_after_inference_mb=self._post_inference, - device_local_mb=self._device_local_mb, - ) diff --git a/tests/unit/session/monitor/test_memory_tracker.py b/tests/unit/session/monitor/test_memory_tracker.py index ab0364e17..7a7fa73da 100644 --- a/tests/unit/session/monitor/test_memory_tracker.py +++ b/tests/unit/session/monitor/test_memory_tracker.py @@ -6,86 +6,20 @@ from __future__ import annotations -import pytest - -from winml.modelkit.session.monitor.memory_tracker import ( - MemoryProfile, - MemoryTracker, - _get_rss_mb, -) +from winml.modelkit.session.monitor.memory_tracker import get_rss_mb class TestGetRssMb: - """Test the process RSS retrieval function.""" + """Test process RSS retrieval.""" def test_returns_positive_float(self) -> None: - rss = _get_rss_mb() + rss = get_rss_mb() assert isinstance(rss, float) assert rss > 0 - -class TestMemoryProfile: - """Test MemoryProfile computed properties.""" - - @pytest.fixture - def profile(self) -> MemoryProfile: - return MemoryProfile( - rss_baseline_mb=100.0, - rss_after_compile_mb=320.0, - rss_after_inference_mb=330.0, - device_local_mb=52.0, - ) - - def test_model_load_delta(self, profile: MemoryProfile) -> None: - assert profile.model_load_delta_mb == pytest.approx(220.0) - - def test_inference_alloc_delta(self, profile: MemoryProfile) -> None: - assert profile.inference_alloc_delta_mb == pytest.approx(10.0) - - def test_total_delta(self, profile: MemoryProfile) -> None: - assert profile.total_delta_mb == pytest.approx(230.0) - - def test_to_dict(self, profile: MemoryProfile) -> None: - d = profile.to_dict() - assert d["rss_baseline_mb"] == 100.0 - assert d["rss_after_compile_mb"] == 320.0 - assert d["rss_after_inference_mb"] == 330.0 - assert d["model_load_delta_mb"] == 220.0 - assert d["inference_alloc_delta_mb"] == 10.0 - assert d["total_delta_mb"] == 230.0 - assert d["device_local_mb"] == 52.0 - - -class TestMemoryTracker: - """Test MemoryTracker snapshot collection.""" - - def test_full_workflow(self) -> None: - tracker = MemoryTracker() - tracker.snapshot_baseline() - tracker.snapshot_post_compile() - tracker.snapshot_post_inference() - profile = tracker.profile() - - assert profile is not None - assert profile.rss_baseline_mb > 0 - assert profile.rss_after_inference_mb > 0 - - def test_incomplete_returns_none(self) -> None: - tracker = MemoryTracker() - tracker.snapshot_baseline() - assert tracker.profile() is None - - def test_deltas_nonnegative_with_allocation(self) -> None: - tracker = MemoryTracker() - tracker.snapshot_baseline() - - # Allocate ~5 MB to ensure RSS grows - _data = [bytearray(1024 * 1024) for _ in range(5)] - - tracker.snapshot_post_compile() - tracker.snapshot_post_inference() - profile = tracker.profile() - - assert profile is not None - assert profile.model_load_delta_mb >= 0 + def test_increases_after_allocation(self) -> None: + before = get_rss_mb() + _data = [bytearray(1024 * 1024) for _ in range(10)] # ~10 MB + after = get_rss_mb() + assert after >= before assert _data is not None From b927dc21fd40f932f222f1916f26d1d729fb4954 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 15 Jun 2026 21:28:15 +0800 Subject: [PATCH 11/27] simplify: reduce _resolve_adapter_luid branching --- src/winml/modelkit/commands/perf.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 64ad27e5c..3b39089c2 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -420,11 +420,7 @@ def _generate_inputs(self) -> None: ) def _resolve_adapter_luid(self) -> str | None: - """Resolve the adapter LUID for device memory queries. - - Uses the same resolution logic as HWMonitor: device kind + EP name. - Returns None on non-Windows or when no adapter is available. - """ + """Resolve adapter LUID for device memory queries.""" import sys if sys.platform != "win32": @@ -432,25 +428,21 @@ def _resolve_adapter_luid(self) -> str | None: assert self._model is not None device = self._model.device or self.config.device - ep_name = self._model.ep_name - if device == "cpu": return None try: from ..sysinfo.pdh_adapters import resolve_adapter_luid - if device == "npu": - return resolve_adapter_luid("npu", ep_name=ep_name) - if device == "gpu": - return resolve_adapter_luid("gpu", ep_name=ep_name) - # "auto" — try NPU first, then GPU - luid = resolve_adapter_luid("npu", ep_name=ep_name) - if luid: - return luid - return resolve_adapter_luid("gpu", ep_name=ep_name) + ep_name = self._model.ep_name + # For "auto" or unknown, try npu then gpu + for kind in [device] if device in ("npu", "gpu") else ["npu", "gpu"]: + luid = resolve_adapter_luid(kind, ep_name=ep_name) + if luid: + return luid + return None except Exception: - logger.debug("Could not resolve adapter LUID for memory query", exc_info=True) + logger.debug("Could not resolve adapter LUID", exc_info=True) return None def _run_benchmark(self) -> PerfStats: From b1a9c086442985ecbeee2f661eb4d892ee6c819b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 15 Jun 2026 21:44:10 +0800 Subject: [PATCH 12/27] fix: take baseline after model load, before compile Aligns with mem_ov.py: baseline excludes Python library imports and model build pipeline. model_load_delta renamed to compile_delta since it now only measures ORT session compilation. --- src/winml/modelkit/commands/perf.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 3b39089c2..327bb3768 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -297,20 +297,20 @@ def run(self) -> BenchmarkResult: """ import gc - # Memory measurement: warmup EP, then take RSS at phase boundaries. + # [1] Load model (imports optimizer libs, builds ONNX graph, etc.) + logger.info("Loading model: %s", self.config.model_id) + self._load_model() + assert self._model is not None + + # Memory measurement: baseline is taken AFTER model load but BEFORE + # compile, so model_load_delta measures only ORT session compilation + # (matching mem_ov.py which takes baseline after all imports). if self.config.memory: from ..session.monitor.memory_tracker import get_device_memory_mb, get_rss_mb - from ..session.session import WinMLSession - WinMLSession._init_winml_eps_once() gc.collect() rss_baseline = get_rss_mb() - # [1] Load model - logger.info("Loading model: %s", self.config.model_id) - self._load_model() - assert self._model is not None - # [2] Compile session self._model._session.compile() @@ -346,8 +346,8 @@ def run(self) -> BenchmarkResult: "rss_baseline_mb": round(rss_baseline, 2), "rss_after_compile_mb": round(rss_after_compile, 2), "rss_after_inference_mb": round(rss_after_inference, 2), - "model_load_delta_mb": round(rss_after_compile - rss_baseline, 2), - "inference_alloc_delta_mb": round(rss_after_inference - rss_after_compile, 2), + "compile_delta_mb": round(rss_after_compile - rss_baseline, 2), + "inference_delta_mb": round(rss_after_inference - rss_after_compile, 2), "total_delta_mb": round(rss_after_inference - rss_baseline, 2), "device_local_mb": round(device_mb, 2), } @@ -947,8 +947,8 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: f"[bold]Memory:[/bold] {mem['rss_after_inference_mb']:.1f} MB (process){dev_str}" ) console.print( - f" [dim]model load: {mem['model_load_delta_mb']:+.1f} MB | " - f"inference: {mem['inference_alloc_delta_mb']:+.1f} MB | " + f" [dim]compile: {mem['compile_delta_mb']:+.1f} MB | " + f"inference: {mem['inference_delta_mb']:+.1f} MB | " f"total: {mem['total_delta_mb']:+.1f} MB[/dim]" ) From e5b8408da238e00f6c3d3b435a590170a8417c44 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 15 Jun 2026 21:45:58 +0800 Subject: [PATCH 13/27] fix: pre-import libs + warmup EP before baseline Baseline now taken after heavy Python imports (WinMLAutoModel etc.) and EP DLL loading, so model_load_delta measures only model-specific work: reading ONNX, graph optimization, ORT session compilation. --- src/winml/modelkit/commands/perf.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 327bb3768..25f88968b 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -297,21 +297,21 @@ def run(self) -> BenchmarkResult: """ import gc - # [1] Load model (imports optimizer libs, builds ONNX graph, etc.) - logger.info("Loading model: %s", self.config.model_id) - self._load_model() - assert self._model is not None - - # Memory measurement: baseline is taken AFTER model load but BEFORE - # compile, so model_load_delta measures only ORT session compilation - # (matching mem_ov.py which takes baseline after all imports). + # Memory: pre-import heavy dependencies and warmup EP so their + # one-time costs are excluded from model_load_delta. if self.config.memory: + from ..models import WinMLAutoModel # noqa: F401 (triggers heavy imports) from ..session.monitor.memory_tracker import get_device_memory_mb, get_rss_mb + from ..session.session import WinMLSession + WinMLSession._init_winml_eps_once() gc.collect() rss_baseline = get_rss_mb() - # [2] Compile session + # [1] Load model + compile + logger.info("Loading model: %s", self.config.model_id) + self._load_model() + assert self._model is not None self._model._session.compile() if self.config.memory: @@ -327,7 +327,7 @@ def run(self) -> BenchmarkResult: ep_name=self._model.ep_name, ) - # [3] Generate inputs + run benchmark + # [2] Generate inputs + run benchmark logger.info("Generating benchmark inputs") self._generate_inputs() @@ -346,13 +346,13 @@ def run(self) -> BenchmarkResult: "rss_baseline_mb": round(rss_baseline, 2), "rss_after_compile_mb": round(rss_after_compile, 2), "rss_after_inference_mb": round(rss_after_inference, 2), - "compile_delta_mb": round(rss_after_compile - rss_baseline, 2), + "model_load_delta_mb": round(rss_after_compile - rss_baseline, 2), "inference_delta_mb": round(rss_after_inference - rss_after_compile, 2), "total_delta_mb": round(rss_after_inference - rss_baseline, 2), "device_local_mb": round(device_mb, 2), } - # [4] Collect results + # [3] Collect results logger.info("Collecting results") return self._collect_results(stats) @@ -947,7 +947,7 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: f"[bold]Memory:[/bold] {mem['rss_after_inference_mb']:.1f} MB (process){dev_str}" ) console.print( - f" [dim]compile: {mem['compile_delta_mb']:+.1f} MB | " + f" [dim]model load: {mem['model_load_delta_mb']:+.1f} MB | " f"inference: {mem['inference_delta_mb']:+.1f} MB | " f"total: {mem['total_delta_mb']:+.1f} MB[/dim]" ) From 4eb603f1d1a988c88d14df815ff8bc7c75143728 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 15 Jun 2026 21:51:24 +0800 Subject: [PATCH 14/27] fix: baseline right before compile() for accurate model memory Takes RSS baseline after _load_model() completes but before compile(). This naturally excludes all Python imports, EP DLLs, and build pipeline overhead without fragile pre-import hacks. model_load_delta now measures only ORT session compilation (actual model weights in memory). --- src/winml/modelkit/commands/perf.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 25f88968b..912953e03 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -297,21 +297,21 @@ def run(self) -> BenchmarkResult: """ import gc - # Memory: pre-import heavy dependencies and warmup EP so their - # one-time costs are excluded from model_load_delta. + # [1] Load model (build pipeline: optimize, cache, etc.) + logger.info("Loading model: %s", self.config.model_id) + self._load_model() + assert self._model is not None + + # Memory: baseline right before compile() — excludes all Python lib + # imports, EP DLLs, and build pipeline overhead. Measures only ORT + # session compilation (model weights loaded into memory). if self.config.memory: - from ..models import WinMLAutoModel # noqa: F401 (triggers heavy imports) from ..session.monitor.memory_tracker import get_device_memory_mb, get_rss_mb - from ..session.session import WinMLSession - WinMLSession._init_winml_eps_once() gc.collect() rss_baseline = get_rss_mb() - # [1] Load model + compile - logger.info("Loading model: %s", self.config.model_id) - self._load_model() - assert self._model is not None + # [2] Compile session (ORT loads model weights into memory here) self._model._session.compile() if self.config.memory: @@ -327,7 +327,7 @@ def run(self) -> BenchmarkResult: ep_name=self._model.ep_name, ) - # [2] Generate inputs + run benchmark + # [3] Generate inputs + run benchmark logger.info("Generating benchmark inputs") self._generate_inputs() @@ -352,7 +352,7 @@ def run(self) -> BenchmarkResult: "device_local_mb": round(device_mb, 2), } - # [3] Collect results + # [4] Collect results logger.info("Collecting results") return self._collect_results(stats) From b5795cb57e74233c63dd5bde8d572308d3892a21 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 10:39:59 +0800 Subject: [PATCH 15/27] fix: address PR review comments - Add comment explaining device memory query position (only grows) - Inline adapter_luid call to reduce intermediate variable --- src/winml/modelkit/commands/perf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 912953e03..fcc3aa144 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -338,10 +338,11 @@ def run(self) -> BenchmarkResult: ) stats = self._run_benchmark() + # Device memory queried once at the end — captures peak allocation + # after inference (device memory only grows, never shrinks mid-session). if self.config.memory: rss_after_inference = get_rss_mb() - adapter_luid = self._resolve_adapter_luid() - device_mb = get_device_memory_mb(adapter_luid) + device_mb = get_device_memory_mb(self._resolve_adapter_luid()) self._memory = { "rss_baseline_mb": round(rss_baseline, 2), "rss_after_compile_mb": round(rss_after_compile, 2), From e4e176aea5fef0ed43adc9578ef9f96d4c1cc0af Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 11:54:40 +0800 Subject: [PATCH 16/27] feat: query device memory after compile and inference Shows device memory growth per phase so users can see how much the model load vs inference contributes to device allocation. --- src/winml/modelkit/commands/perf.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index fcc3aa144..712f24fa0 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -317,6 +317,8 @@ def run(self) -> BenchmarkResult: if self.config.memory: gc.collect() rss_after_compile = get_rss_mb() + adapter_luid = self._resolve_adapter_luid() + device_after_compile = get_device_memory_mb(adapter_luid) # Print model info before benchmark starts _print_model_info( @@ -338,11 +340,9 @@ def run(self) -> BenchmarkResult: ) stats = self._run_benchmark() - # Device memory queried once at the end — captures peak allocation - # after inference (device memory only grows, never shrinks mid-session). if self.config.memory: rss_after_inference = get_rss_mb() - device_mb = get_device_memory_mb(self._resolve_adapter_luid()) + device_after_inference = get_device_memory_mb(adapter_luid) self._memory = { "rss_baseline_mb": round(rss_baseline, 2), "rss_after_compile_mb": round(rss_after_compile, 2), @@ -350,7 +350,8 @@ def run(self) -> BenchmarkResult: "model_load_delta_mb": round(rss_after_compile - rss_baseline, 2), "inference_delta_mb": round(rss_after_inference - rss_after_compile, 2), "total_delta_mb": round(rss_after_inference - rss_baseline, 2), - "device_local_mb": round(device_mb, 2), + "device_after_compile_mb": round(device_after_compile, 2), + "device_after_inference_mb": round(device_after_inference, 2), } # [4] Collect results @@ -941,8 +942,9 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: # Memory section (only when --memory is enabled) if result.memory_profile: mem = result.memory_profile - dev_mb = mem.get("device_local_mb", 0) - dev_str = f" | {dev_mb:.1f} MB (device)" if dev_mb > 0 else "" + dev_compile = mem.get("device_after_compile_mb", 0) + dev_inference = mem.get("device_after_inference_mb", 0) + dev_str = f" | {dev_inference:.1f} MB (device)" if dev_inference > 0 else "" console.print() console.print( f"[bold]Memory:[/bold] {mem['rss_after_inference_mb']:.1f} MB (process){dev_str}" @@ -952,6 +954,11 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: f"inference: {mem['inference_delta_mb']:+.1f} MB | " f"total: {mem['total_delta_mb']:+.1f} MB[/dim]" ) + if dev_compile > 0 or dev_inference > 0: + console.print( + f" [dim]device: compile {dev_compile:.1f} MB → " + f"inference {dev_inference:.1f} MB[/dim]" + ) console.print() From 155547fd598b2a687d22c1c5c7033947eac029a8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 13:07:28 +0800 Subject: [PATCH 17/27] Remove device memory measurement (unreliable for NPU) Device memory uses WDDM PDH counters which only work reliably for GPU (DML). For NPU/QNN they report tiny numbers that don't reflect actual device usage. Simplify to process RSS only. --- src/winml/modelkit/commands/perf.py | 45 +------------------ .../session/monitor/memory_tracker.py | 38 +--------------- 2 files changed, 4 insertions(+), 79 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 712f24fa0..2c4ca35ff 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -306,7 +306,7 @@ def run(self) -> BenchmarkResult: # imports, EP DLLs, and build pipeline overhead. Measures only ORT # session compilation (model weights loaded into memory). if self.config.memory: - from ..session.monitor.memory_tracker import get_device_memory_mb, get_rss_mb + from ..session.monitor.memory_tracker import get_rss_mb gc.collect() rss_baseline = get_rss_mb() @@ -317,8 +317,6 @@ def run(self) -> BenchmarkResult: if self.config.memory: gc.collect() rss_after_compile = get_rss_mb() - adapter_luid = self._resolve_adapter_luid() - device_after_compile = get_device_memory_mb(adapter_luid) # Print model info before benchmark starts _print_model_info( @@ -342,7 +340,6 @@ def run(self) -> BenchmarkResult: if self.config.memory: rss_after_inference = get_rss_mb() - device_after_inference = get_device_memory_mb(adapter_luid) self._memory = { "rss_baseline_mb": round(rss_baseline, 2), "rss_after_compile_mb": round(rss_after_compile, 2), @@ -350,8 +347,6 @@ def run(self) -> BenchmarkResult: "model_load_delta_mb": round(rss_after_compile - rss_baseline, 2), "inference_delta_mb": round(rss_after_inference - rss_after_compile, 2), "total_delta_mb": round(rss_after_inference - rss_baseline, 2), - "device_after_compile_mb": round(device_after_compile, 2), - "device_after_inference_mb": round(device_after_inference, 2), } # [4] Collect results @@ -421,32 +416,6 @@ def _generate_inputs(self) -> None: batch_size=self.config.batch_size, ) - def _resolve_adapter_luid(self) -> str | None: - """Resolve adapter LUID for device memory queries.""" - import sys - - if sys.platform != "win32": - return None - - assert self._model is not None - device = self._model.device or self.config.device - if device == "cpu": - return None - - try: - from ..sysinfo.pdh_adapters import resolve_adapter_luid - - ep_name = self._model.ep_name - # For "auto" or unknown, try npu then gpu - for kind in [device] if device in ("npu", "gpu") else ["npu", "gpu"]: - luid = resolve_adapter_luid(kind, ep_name=ep_name) - if luid: - return luid - return None - except Exception: - logger.debug("Could not resolve adapter LUID", exc_info=True) - return None - def _run_benchmark(self) -> PerfStats: """Execute benchmark iterations with timing.""" if self.config.monitor: @@ -942,23 +911,13 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: # Memory section (only when --memory is enabled) if result.memory_profile: mem = result.memory_profile - dev_compile = mem.get("device_after_compile_mb", 0) - dev_inference = mem.get("device_after_inference_mb", 0) - dev_str = f" | {dev_inference:.1f} MB (device)" if dev_inference > 0 else "" console.print() - console.print( - f"[bold]Memory:[/bold] {mem['rss_after_inference_mb']:.1f} MB (process){dev_str}" - ) + console.print(f"[bold]Memory:[/bold] {mem['rss_after_inference_mb']:.1f} MB (process)") console.print( f" [dim]model load: {mem['model_load_delta_mb']:+.1f} MB | " f"inference: {mem['inference_delta_mb']:+.1f} MB | " f"total: {mem['total_delta_mb']:+.1f} MB[/dim]" ) - if dev_compile > 0 or dev_inference > 0: - console.print( - f" [dim]device: compile {dev_compile:.1f} MB → " - f"inference {dev_inference:.1f} MB[/dim]" - ) console.print() diff --git a/src/winml/modelkit/session/monitor/memory_tracker.py b/src/winml/modelkit/session/monitor/memory_tracker.py index 80cc67bd5..5820bf2f3 100644 --- a/src/winml/modelkit/session/monitor/memory_tracker.py +++ b/src/winml/modelkit/session/monitor/memory_tracker.py @@ -2,49 +2,15 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. # -------------------------------------------------------------------------- -"""Process memory helpers for perf benchmarking.""" +"""Process memory helper for perf benchmarking.""" from __future__ import annotations -import logging import os -import sys import psutil -logger = logging.getLogger(__name__) - -_MB = 1024 * 1024 - - def get_rss_mb() -> float: """Return current RSS in MB for this process.""" - return psutil.Process(os.getpid()).memory_info().rss / _MB - - -def get_device_memory_mb(luid: str | None) -> float: - """Single-shot PDH query for device local memory in MB.""" - if luid is None or sys.platform != "win32": - return 0.0 - try: - from ._pdh import PdhQuery - - pid = os.getpid() - query = PdhQuery() - query.open() - ok = query.add_counter( - "local", - rf"\GPU Process Memory(pid_{pid}_luid_{luid}_phys_0)\Local Usage", - fmt="large", - ) - if not ok: - query.close() - return 0.0 - query.prime() - values = query.collect() - query.close() - return (values.get("local") or 0) / _MB - except Exception: - logger.debug("Device memory query failed", exc_info=True) - return 0.0 + return psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024) From 7e68fefca8215bc7ed7065fe88e1c601b029d746 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 13:10:20 +0800 Subject: [PATCH 18/27] Remove Device Mem line from hardware monitor display The WDDM PDH device memory counters are GPU-specific and don't provide meaningful data for NPU workloads. --- src/winml/modelkit/commands/perf.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 2c4ca35ff..aec520f38 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -886,7 +886,6 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: console.print("[bold]Hardware (during benchmark)[/bold]") cpu = result.hw_monitor.get("cpu", {}) ram = result.hw_monitor.get("ram", {}) - dev_mem = result.hw_monitor.get("device_memory", {}) # to_dict() emits both "npu" (always) and "gpu" (when monitoring GPU). # device_kind is None when CPU/RAM-only — drop the adapter line entirely # rather than printing zeroed "NPU: 0.0% avg". @@ -899,10 +898,6 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: f"CPU: {cpu.get('mean_pct', 0):.1f}% avg | " f"Mem: {ram.get('used_mb', 0):.0f} MB" ) - console.print( - f" Device Mem: {dev_mem.get('local_peak_mb', 0):.0f}/" - f"{dev_mem.get('shared_peak_mb', 0):.0f} MB (local/shared)" - ) else: console.print( f" CPU: {cpu.get('mean_pct', 0):.1f}% avg | Mem: {ram.get('used_mb', 0):.0f} MB" From c9c9fecff4a80055ba45ad575d92ad54067c8685 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 13:18:07 +0800 Subject: [PATCH 19/27] Remove device memory from live display and hw_monitor Remove Device Mem from live chart, hw_monitor properties, and to_dict() output. Update tests accordingly. --- src/winml/modelkit/commands/_live_chart.py | 12 ++---------- src/winml/modelkit/commands/perf.py | 2 -- .../modelkit/session/monitor/hw_monitor.py | 19 ------------------- .../modelkit/session/monitor/live_display.py | 5 +---- tests/unit/session/test_ep_monitor.py | 14 +++----------- 5 files changed, 6 insertions(+), 46 deletions(-) diff --git a/src/winml/modelkit/commands/_live_chart.py b/src/winml/modelkit/commands/_live_chart.py index 3f5aa1125..d9a2eb275 100644 --- a/src/winml/modelkit/commands/_live_chart.py +++ b/src/winml/modelkit/commands/_live_chart.py @@ -85,8 +85,6 @@ def update( iteration: int, latency_ms: float, util_samples: list[float], - memory_local_mb: float = 0.0, - memory_shared_mb: float = 0.0, cpu_pct: float = 0.0, ram_mb: float = 0.0, cpu_samples: list[float] | None = None, @@ -101,8 +99,6 @@ def update( iteration, latency_ms, util_samples, - memory_local_mb, - memory_shared_mb, cpu_pct, ram_mb, ) @@ -221,8 +217,6 @@ def _render_status( iteration: int, latency_ms: float, util_samples: list[float], - memory_local_mb: float = 0.0, - memory_shared_mb: float = 0.0, cpu_pct: float = 0.0, ram_mb: float = 0.0, ) -> str: @@ -250,14 +244,12 @@ def _render_status( row1 = f" {pct_cell:<30}| {progress} | Device: {self._device}" # Row 2: Hardware (pad each cell to fixed width, spaces before divider). - # CPU-only mode drops the adapter cell + device-memory cell since we - # have no live values to populate them with. + # CPU-only mode drops the adapter cell since we have no live values. cpu_cell = f"CPU: {cpu_pct:.1f}%" ram_cell = f"Mem: {ram_mb:.0f} MB" if self._show_adapter: adapter_cell = f"{self._adapter_label}: {mean_util:.1f}% avg ({current_util:.1f}% now)" - mem_cell = f"Device Mem: {memory_local_mb:.0f}/{memory_shared_mb:.0f} MB (local/shared)" - row2 = f" {adapter_cell:<30}| {cpu_cell:<12}| {ram_cell} | {mem_cell}" + row2 = f" {adapter_cell:<30}| {cpu_cell:<12}| {ram_cell}" else: row2 = f" {cpu_cell:<12}| {ram_cell}" diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index aec520f38..d80be3fc0 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -1028,8 +1028,6 @@ def _run_monitored_loop( iteration=i + 1, latency_ms=latest_latency, util_samples=hw.utilization_samples, - memory_local_mb=hw.peak_memory_local_mb, - memory_shared_mb=hw.peak_memory_shared_mb, cpu_pct=hw.mean_cpu_pct, ram_mb=hw.ram_used_mb, cpu_samples=hw.cpu_samples, diff --git a/src/winml/modelkit/session/monitor/hw_monitor.py b/src/winml/modelkit/session/monitor/hw_monitor.py index 12740ee0d..7a391f727 100644 --- a/src/winml/modelkit/session/monitor/hw_monitor.py +++ b/src/winml/modelkit/session/monitor/hw_monitor.py @@ -113,21 +113,6 @@ def peak_utilization_pct(self) -> float: """Peak adapter (NPU/GPU) utilization % during monitoring period.""" return self._pdh.peak_utilization_pct - @property - def peak_memory_mb(self) -> float: - """Peak device memory (local preferred, shared fallback) in MB.""" - return self._pdh.peak_memory_mb - - @property - def peak_memory_local_mb(self) -> float: - """Peak dedicated device memory in MB.""" - return self._pdh.peak_memory_local_mb - - @property - def peak_memory_shared_mb(self) -> float: - """Peak shared system memory used by device in MB.""" - return self._pdh.peak_memory_shared_mb - # --- CPU metrics --- @property @@ -189,10 +174,6 @@ def to_dict(self) -> dict[str, Any]: "used_mb": round(self._pdh.ram_used_mb, 2), "peak_mb": round(self._pdh.peak_ram_used_mb, 2), }, - "device_memory": { - "local_peak_mb": round(self._pdh.peak_memory_local_mb, 2), - "shared_peak_mb": round(self._pdh.peak_memory_shared_mb, 2), - }, "running_time_ns": self._pdh.running_time_delta_ns, } if kind in ("npu", "gpu"): diff --git a/src/winml/modelkit/session/monitor/live_display.py b/src/winml/modelkit/session/monitor/live_display.py index 6b0e92207..94f803b7f 100644 --- a/src/winml/modelkit/session/monitor/live_display.py +++ b/src/winml/modelkit/session/monitor/live_display.py @@ -225,13 +225,10 @@ def _render_status( adapter_mean = sum(adapter_samples) / len(adapter_samples) if adapter_samples else 0.0 adapter_now = adapter_samples[-1] if adapter_samples else 0.0 - mem_local = self._hw.peak_memory_local_mb - mem_shared = self._hw.peak_memory_shared_mb adapter_cell = f"{self._adapter_label}: {adapter_mean:.1f}% avg ({adapter_now:.1f}% now)" - mem_cell = f"Device Mem: {mem_local:.0f}/{mem_shared:.0f} MB" - return f" {adapter_cell:<32}| {cpu_cell:<12}| {ram_cell:<16}| {mem_cell}" + return f" {adapter_cell:<32}| {cpu_cell:<12}| {ram_cell}" @property def hw(self) -> HWMonitor: diff --git a/tests/unit/session/test_ep_monitor.py b/tests/unit/session/test_ep_monitor.py index 9adc3a001..8632f8cb5 100644 --- a/tests/unit/session/test_ep_monitor.py +++ b/tests/unit/session/test_ep_monitor.py @@ -520,7 +520,6 @@ def test_context_manager_lifecycle(self): assert isinstance(hw.mean_utilization_pct, float) assert isinstance(hw.peak_utilization_pct, float) - assert isinstance(hw.peak_memory_mb, float) @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only") def test_to_dict_structure(self): @@ -557,10 +556,8 @@ def test_to_dict_structure(self): else: assert "npu" not in d assert "gpu" not in d - # Device memory + running time - assert "device_memory" in d - assert "local_peak_mb" in d["device_memory"] - assert "shared_peak_mb" in d["device_memory"] + # Running time + assert "device_memory" not in d assert "running_time_ns" in d @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only") @@ -1431,8 +1428,6 @@ def test_render_status_warmup_phase(self): iteration=5, latency_ms=1.0, util_samples=[50.0], - memory_local_mb=10.0, - memory_shared_mb=20.0, cpu_pct=5.0, ram_mb=8000.0, ) @@ -1447,8 +1442,6 @@ def test_render_status_benchmark_phase(self): iteration=50, latency_ms=2.0, util_samples=[80.0, 90.0], - memory_local_mb=31.0, - memory_shared_mb=43.0, cpu_pct=15.0, ram_mb=40000.0, ) @@ -1503,7 +1496,7 @@ def test_print_final_snapshot_is_noop(self): ) def test_cpu_only_status_omits_adapter_cell(self): - """device='cpu' → no adapter cell, no Device Mem cell — only CPU/RAM.""" + """device='cpu' → no adapter cell — only CPU/RAM.""" from winml.modelkit.commands._live_chart import LiveMonitorDisplay display = LiveMonitorDisplay(total_iterations=10, warmup=0, model_id="test", device="cpu") @@ -1554,4 +1547,3 @@ def test_adapter_kind_status_keeps_adapter_cell(self): ram_mb=8000.0, ) assert "GPU: 42.0% avg" in status - assert "Device Mem:" in status From 66b0c990a5c605308692048d0bd18159ea4ed769 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 13:25:52 +0800 Subject: [PATCH 20/27] Revert "Remove device memory from live display and hw_monitor" This reverts commit c9c9fecff4a80055ba45ad575d92ad54067c8685. --- src/winml/modelkit/commands/_live_chart.py | 12 ++++++++++-- src/winml/modelkit/commands/perf.py | 2 ++ .../modelkit/session/monitor/hw_monitor.py | 19 +++++++++++++++++++ .../modelkit/session/monitor/live_display.py | 5 ++++- tests/unit/session/test_ep_monitor.py | 14 +++++++++++--- 5 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/winml/modelkit/commands/_live_chart.py b/src/winml/modelkit/commands/_live_chart.py index d9a2eb275..3f5aa1125 100644 --- a/src/winml/modelkit/commands/_live_chart.py +++ b/src/winml/modelkit/commands/_live_chart.py @@ -85,6 +85,8 @@ def update( iteration: int, latency_ms: float, util_samples: list[float], + memory_local_mb: float = 0.0, + memory_shared_mb: float = 0.0, cpu_pct: float = 0.0, ram_mb: float = 0.0, cpu_samples: list[float] | None = None, @@ -99,6 +101,8 @@ def update( iteration, latency_ms, util_samples, + memory_local_mb, + memory_shared_mb, cpu_pct, ram_mb, ) @@ -217,6 +221,8 @@ def _render_status( iteration: int, latency_ms: float, util_samples: list[float], + memory_local_mb: float = 0.0, + memory_shared_mb: float = 0.0, cpu_pct: float = 0.0, ram_mb: float = 0.0, ) -> str: @@ -244,12 +250,14 @@ def _render_status( row1 = f" {pct_cell:<30}| {progress} | Device: {self._device}" # Row 2: Hardware (pad each cell to fixed width, spaces before divider). - # CPU-only mode drops the adapter cell since we have no live values. + # CPU-only mode drops the adapter cell + device-memory cell since we + # have no live values to populate them with. cpu_cell = f"CPU: {cpu_pct:.1f}%" ram_cell = f"Mem: {ram_mb:.0f} MB" if self._show_adapter: adapter_cell = f"{self._adapter_label}: {mean_util:.1f}% avg ({current_util:.1f}% now)" - row2 = f" {adapter_cell:<30}| {cpu_cell:<12}| {ram_cell}" + mem_cell = f"Device Mem: {memory_local_mb:.0f}/{memory_shared_mb:.0f} MB (local/shared)" + row2 = f" {adapter_cell:<30}| {cpu_cell:<12}| {ram_cell} | {mem_cell}" else: row2 = f" {cpu_cell:<12}| {ram_cell}" diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index d80be3fc0..aec520f38 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -1028,6 +1028,8 @@ def _run_monitored_loop( iteration=i + 1, latency_ms=latest_latency, util_samples=hw.utilization_samples, + memory_local_mb=hw.peak_memory_local_mb, + memory_shared_mb=hw.peak_memory_shared_mb, cpu_pct=hw.mean_cpu_pct, ram_mb=hw.ram_used_mb, cpu_samples=hw.cpu_samples, diff --git a/src/winml/modelkit/session/monitor/hw_monitor.py b/src/winml/modelkit/session/monitor/hw_monitor.py index 7a391f727..12740ee0d 100644 --- a/src/winml/modelkit/session/monitor/hw_monitor.py +++ b/src/winml/modelkit/session/monitor/hw_monitor.py @@ -113,6 +113,21 @@ def peak_utilization_pct(self) -> float: """Peak adapter (NPU/GPU) utilization % during monitoring period.""" return self._pdh.peak_utilization_pct + @property + def peak_memory_mb(self) -> float: + """Peak device memory (local preferred, shared fallback) in MB.""" + return self._pdh.peak_memory_mb + + @property + def peak_memory_local_mb(self) -> float: + """Peak dedicated device memory in MB.""" + return self._pdh.peak_memory_local_mb + + @property + def peak_memory_shared_mb(self) -> float: + """Peak shared system memory used by device in MB.""" + return self._pdh.peak_memory_shared_mb + # --- CPU metrics --- @property @@ -174,6 +189,10 @@ def to_dict(self) -> dict[str, Any]: "used_mb": round(self._pdh.ram_used_mb, 2), "peak_mb": round(self._pdh.peak_ram_used_mb, 2), }, + "device_memory": { + "local_peak_mb": round(self._pdh.peak_memory_local_mb, 2), + "shared_peak_mb": round(self._pdh.peak_memory_shared_mb, 2), + }, "running_time_ns": self._pdh.running_time_delta_ns, } if kind in ("npu", "gpu"): diff --git a/src/winml/modelkit/session/monitor/live_display.py b/src/winml/modelkit/session/monitor/live_display.py index 94f803b7f..6b0e92207 100644 --- a/src/winml/modelkit/session/monitor/live_display.py +++ b/src/winml/modelkit/session/monitor/live_display.py @@ -225,10 +225,13 @@ def _render_status( adapter_mean = sum(adapter_samples) / len(adapter_samples) if adapter_samples else 0.0 adapter_now = adapter_samples[-1] if adapter_samples else 0.0 + mem_local = self._hw.peak_memory_local_mb + mem_shared = self._hw.peak_memory_shared_mb adapter_cell = f"{self._adapter_label}: {adapter_mean:.1f}% avg ({adapter_now:.1f}% now)" + mem_cell = f"Device Mem: {mem_local:.0f}/{mem_shared:.0f} MB" - return f" {adapter_cell:<32}| {cpu_cell:<12}| {ram_cell}" + return f" {adapter_cell:<32}| {cpu_cell:<12}| {ram_cell:<16}| {mem_cell}" @property def hw(self) -> HWMonitor: diff --git a/tests/unit/session/test_ep_monitor.py b/tests/unit/session/test_ep_monitor.py index 8632f8cb5..9adc3a001 100644 --- a/tests/unit/session/test_ep_monitor.py +++ b/tests/unit/session/test_ep_monitor.py @@ -520,6 +520,7 @@ def test_context_manager_lifecycle(self): assert isinstance(hw.mean_utilization_pct, float) assert isinstance(hw.peak_utilization_pct, float) + assert isinstance(hw.peak_memory_mb, float) @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only") def test_to_dict_structure(self): @@ -556,8 +557,10 @@ def test_to_dict_structure(self): else: assert "npu" not in d assert "gpu" not in d - # Running time - assert "device_memory" not in d + # Device memory + running time + assert "device_memory" in d + assert "local_peak_mb" in d["device_memory"] + assert "shared_peak_mb" in d["device_memory"] assert "running_time_ns" in d @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only") @@ -1428,6 +1431,8 @@ def test_render_status_warmup_phase(self): iteration=5, latency_ms=1.0, util_samples=[50.0], + memory_local_mb=10.0, + memory_shared_mb=20.0, cpu_pct=5.0, ram_mb=8000.0, ) @@ -1442,6 +1447,8 @@ def test_render_status_benchmark_phase(self): iteration=50, latency_ms=2.0, util_samples=[80.0, 90.0], + memory_local_mb=31.0, + memory_shared_mb=43.0, cpu_pct=15.0, ram_mb=40000.0, ) @@ -1496,7 +1503,7 @@ def test_print_final_snapshot_is_noop(self): ) def test_cpu_only_status_omits_adapter_cell(self): - """device='cpu' → no adapter cell — only CPU/RAM.""" + """device='cpu' → no adapter cell, no Device Mem cell — only CPU/RAM.""" from winml.modelkit.commands._live_chart import LiveMonitorDisplay display = LiveMonitorDisplay(total_iterations=10, warmup=0, model_id="test", device="cpu") @@ -1547,3 +1554,4 @@ def test_adapter_kind_status_keeps_adapter_cell(self): ram_mb=8000.0, ) assert "GPU: 42.0% avg" in status + assert "Device Mem:" in status From fccd5e1bdf8d0402599ba3314a7cab2ebc124eb2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 13:31:12 +0800 Subject: [PATCH 21/27] Remove dim styling from memory breakdown line --- src/winml/modelkit/commands/perf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index aec520f38..ff0589e5a 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -909,9 +909,9 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: console.print() console.print(f"[bold]Memory:[/bold] {mem['rss_after_inference_mb']:.1f} MB (process)") console.print( - f" [dim]model load: {mem['model_load_delta_mb']:+.1f} MB | " + f" model load: {mem['model_load_delta_mb']:+.1f} MB | " f"inference: {mem['inference_delta_mb']:+.1f} MB | " - f"total: {mem['total_delta_mb']:+.1f} MB[/dim]" + f"total: {mem['total_delta_mb']:+.1f} MB" ) console.print() From b4b3e9ff63c1b5284ffa67ac791bb58aee4995c9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 13:35:49 +0800 Subject: [PATCH 22/27] =?UTF-8?q?Rename=20Mem=20=E2=86=92=20RAM=20and=20De?= =?UTF-8?q?vice=20Mem=20=E2=86=92=20VRAM=20for=20clarity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/winml/modelkit/commands/_live_chart.py | 4 ++-- src/winml/modelkit/commands/perf.py | 4 ++-- src/winml/modelkit/session/monitor/live_display.py | 4 ++-- tests/unit/session/test_ep_monitor.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/winml/modelkit/commands/_live_chart.py b/src/winml/modelkit/commands/_live_chart.py index 3f5aa1125..e75628195 100644 --- a/src/winml/modelkit/commands/_live_chart.py +++ b/src/winml/modelkit/commands/_live_chart.py @@ -253,10 +253,10 @@ def _render_status( # CPU-only mode drops the adapter cell + device-memory cell since we # have no live values to populate them with. cpu_cell = f"CPU: {cpu_pct:.1f}%" - ram_cell = f"Mem: {ram_mb:.0f} MB" + ram_cell = f"RAM: {ram_mb:.0f} MB" if self._show_adapter: adapter_cell = f"{self._adapter_label}: {mean_util:.1f}% avg ({current_util:.1f}% now)" - mem_cell = f"Device Mem: {memory_local_mb:.0f}/{memory_shared_mb:.0f} MB (local/shared)" + mem_cell = f"VRAM: {memory_local_mb:.0f}/{memory_shared_mb:.0f} MB (local/shared)" row2 = f" {adapter_cell:<30}| {cpu_cell:<12}| {ram_cell} | {mem_cell}" else: row2 = f" {cpu_cell:<12}| {ram_cell}" diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index ff0589e5a..9f91abbef 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -896,11 +896,11 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: f" {device_kind.upper()}: {adapter.get('mean_pct', 0):.1f}% avg, " f"{adapter.get('peak_pct', 0):.1f}% peak | " f"CPU: {cpu.get('mean_pct', 0):.1f}% avg | " - f"Mem: {ram.get('used_mb', 0):.0f} MB" + f"RAM: {ram.get('used_mb', 0):.0f} MB" ) else: console.print( - f" CPU: {cpu.get('mean_pct', 0):.1f}% avg | Mem: {ram.get('used_mb', 0):.0f} MB" + f" CPU: {cpu.get('mean_pct', 0):.1f}% avg | RAM: {ram.get('used_mb', 0):.0f} MB" ) # Memory section (only when --memory is enabled) diff --git a/src/winml/modelkit/session/monitor/live_display.py b/src/winml/modelkit/session/monitor/live_display.py index 6b0e92207..fb7c0cfe4 100644 --- a/src/winml/modelkit/session/monitor/live_display.py +++ b/src/winml/modelkit/session/monitor/live_display.py @@ -218,7 +218,7 @@ def _render_status( cpu_now = cpu_samples[-1] if cpu_samples else 0.0 ram_mb = self._hw.ram_used_mb cpu_cell = f"CPU: {cpu_now:.1f}%" - ram_cell = f"Mem: {ram_mb:.0f} MB" + ram_cell = f"RAM: {ram_mb:.0f} MB" if not self._show_adapter: return f" {cpu_cell:<12}| {ram_cell}" @@ -229,7 +229,7 @@ def _render_status( mem_shared = self._hw.peak_memory_shared_mb adapter_cell = f"{self._adapter_label}: {adapter_mean:.1f}% avg ({adapter_now:.1f}% now)" - mem_cell = f"Device Mem: {mem_local:.0f}/{mem_shared:.0f} MB" + mem_cell = f"VRAM: {mem_local:.0f}/{mem_shared:.0f} MB" return f" {adapter_cell:<32}| {cpu_cell:<12}| {ram_cell:<16}| {mem_cell}" diff --git a/tests/unit/session/test_ep_monitor.py b/tests/unit/session/test_ep_monitor.py index 9adc3a001..dbbfe98fe 100644 --- a/tests/unit/session/test_ep_monitor.py +++ b/tests/unit/session/test_ep_monitor.py @@ -1516,11 +1516,11 @@ def test_cpu_only_status_omits_adapter_cell(self): ) # Adapter line / device-memory line gone; CPU + Mem remain. assert "CPU: 12.5%" in status - assert "Mem: 8000 MB" in status + assert "RAM: 8000 MB" in status assert "NPU:" not in status assert "GPU:" not in status assert "Adapter:" not in status - assert "Device Mem:" not in status + assert "VRAM:" not in status def test_cpu_only_chart_legend_omits_adapter_swatch(self): """The chart legend should advertise only CPU when no adapter polled.""" @@ -1554,4 +1554,4 @@ def test_adapter_kind_status_keeps_adapter_cell(self): ram_mb=8000.0, ) assert "GPU: 42.0% avg" in status - assert "Device Mem:" in status + assert "VRAM:" in status From e6778120224017fb1cf0283011d5f997ce1c6795 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 14:22:18 +0800 Subject: [PATCH 23/27] Add VRAM phase measurement alongside RAM Query GPU Process Memory PDH counters at same phase boundaries (baseline, after compile, after inference) to report VRAM deltas for model load and inference separately. --- src/winml/modelkit/commands/perf.py | 59 ++++++++++++++++--- .../session/monitor/memory_tracker.py | 38 ++++++++++++ 2 files changed, 89 insertions(+), 8 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 9f91abbef..7a52052e7 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -306,10 +306,12 @@ def run(self) -> BenchmarkResult: # imports, EP DLLs, and build pipeline overhead. Measures only ORT # session compilation (model weights loaded into memory). if self.config.memory: - from ..session.monitor.memory_tracker import get_rss_mb + from ..session.monitor.memory_tracker import get_rss_mb, get_vram_mb + adapter_luid = self._resolve_adapter_luid() gc.collect() rss_baseline = get_rss_mb() + vram_baseline = get_vram_mb(adapter_luid) # [2] Compile session (ORT loads model weights into memory here) self._model._session.compile() @@ -317,6 +319,7 @@ def run(self) -> BenchmarkResult: if self.config.memory: gc.collect() rss_after_compile = get_rss_mb() + vram_after_compile = get_vram_mb(adapter_luid) # Print model info before benchmark starts _print_model_info( @@ -340,13 +343,20 @@ def run(self) -> BenchmarkResult: if self.config.memory: rss_after_inference = get_rss_mb() + vram_after_inference = get_vram_mb(adapter_luid) self._memory = { "rss_baseline_mb": round(rss_baseline, 2), "rss_after_compile_mb": round(rss_after_compile, 2), "rss_after_inference_mb": round(rss_after_inference, 2), - "model_load_delta_mb": round(rss_after_compile - rss_baseline, 2), - "inference_delta_mb": round(rss_after_inference - rss_after_compile, 2), - "total_delta_mb": round(rss_after_inference - rss_baseline, 2), + "rss_model_load_delta_mb": round(rss_after_compile - rss_baseline, 2), + "rss_inference_delta_mb": round(rss_after_inference - rss_after_compile, 2), + "rss_total_delta_mb": round(rss_after_inference - rss_baseline, 2), + "vram_baseline_mb": round(vram_baseline, 2), + "vram_after_compile_mb": round(vram_after_compile, 2), + "vram_after_inference_mb": round(vram_after_inference, 2), + "vram_model_load_delta_mb": round(vram_after_compile - vram_baseline, 2), + "vram_inference_delta_mb": round(vram_after_inference - vram_after_compile, 2), + "vram_total_delta_mb": round(vram_after_inference - vram_baseline, 2), } # [4] Collect results @@ -416,6 +426,31 @@ def _generate_inputs(self) -> None: batch_size=self.config.batch_size, ) + def _resolve_adapter_luid(self) -> str | None: + """Resolve adapter LUID for VRAM queries.""" + import sys + + if sys.platform != "win32": + return None + + assert self._model is not None + device = self._model.device or self.config.device + if device == "cpu": + return None + + try: + from ..sysinfo.pdh_adapters import resolve_adapter_luid + + ep_name = self._model.ep_name + for kind in [device] if device in ("npu", "gpu") else ["npu", "gpu"]: + luid = resolve_adapter_luid(kind, ep_name=ep_name) + if luid: + return luid + return None + except Exception: + logger.debug("Could not resolve adapter LUID", exc_info=True) + return None + def _run_benchmark(self) -> PerfStats: """Execute benchmark iterations with timing.""" if self.config.monitor: @@ -907,12 +942,20 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: if result.memory_profile: mem = result.memory_profile console.print() - console.print(f"[bold]Memory:[/bold] {mem['rss_after_inference_mb']:.1f} MB (process)") + console.print(f"[bold]Memory:[/bold] RAM: {mem['rss_after_inference_mb']:.1f} MB") console.print( - f" model load: {mem['model_load_delta_mb']:+.1f} MB | " - f"inference: {mem['inference_delta_mb']:+.1f} MB | " - f"total: {mem['total_delta_mb']:+.1f} MB" + f" model load: {mem['rss_model_load_delta_mb']:+.1f} MB | " + f"inference: {mem['rss_inference_delta_mb']:+.1f} MB | " + f"total: {mem['rss_total_delta_mb']:+.1f} MB" ) + vram_total = mem.get("vram_after_inference_mb", 0) + if vram_total > 0: + console.print(f" VRAM: {vram_total:.1f} MB") + console.print( + f" model load: {mem['vram_model_load_delta_mb']:+.1f} MB | " + f"inference: {mem['vram_inference_delta_mb']:+.1f} MB | " + f"total: {mem['vram_total_delta_mb']:+.1f} MB" + ) console.print() diff --git a/src/winml/modelkit/session/monitor/memory_tracker.py b/src/winml/modelkit/session/monitor/memory_tracker.py index 5820bf2f3..0fac374cc 100644 --- a/src/winml/modelkit/session/monitor/memory_tracker.py +++ b/src/winml/modelkit/session/monitor/memory_tracker.py @@ -6,11 +6,49 @@ from __future__ import annotations +import logging import os +import sys import psutil +logger = logging.getLogger(__name__) + + def get_rss_mb() -> float: """Return current RSS in MB for this process.""" return psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024) + + +def get_vram_mb(adapter_luid: str | None) -> float: + """Return current VRAM usage (local + shared) in MB via PDH. + + Returns 0.0 on non-Windows, if no adapter_luid is provided, or on failure. + """ + if sys.platform != "win32" or not adapter_luid: + return 0.0 + + try: + from ._pdh import PdhQuery + + pid = os.getpid() + q = PdhQuery() + q.open() + q.add_counter( + "local", + rf"\GPU Process Memory(pid_{pid}_luid_{adapter_luid}_phys_0)\Local Usage", + ) + q.add_counter( + "shared", + rf"\GPU Process Memory(pid_{pid}_luid_{adapter_luid}_phys_0)\Shared Usage", + ) + # Memory counters are absolute (not rate-based), single collect suffices. + values = q.collect() + q.close() + local = values.get("local") or 0 + shared = values.get("shared") or 0 + return (local + shared) / (1024 * 1024) + except Exception: + logger.debug("VRAM query failed", exc_info=True) + return 0.0 From 4b5db1da92668e70a633f90785601380a48505a5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 14:38:02 +0800 Subject: [PATCH 24/27] Compact memory display to single line per metric --- src/winml/modelkit/commands/perf.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 7a52052e7..f2f02beb8 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -942,17 +942,18 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: if result.memory_profile: mem = result.memory_profile console.print() - console.print(f"[bold]Memory:[/bold] RAM: {mem['rss_after_inference_mb']:.1f} MB") + console.print("[bold]Memory:[/bold]") console.print( - f" model load: {mem['rss_model_load_delta_mb']:+.1f} MB | " + f" RAM: {mem['rss_after_inference_mb']:.1f} MB -> " + f"model load: {mem['rss_model_load_delta_mb']:+.1f} MB | " f"inference: {mem['rss_inference_delta_mb']:+.1f} MB | " f"total: {mem['rss_total_delta_mb']:+.1f} MB" ) vram_total = mem.get("vram_after_inference_mb", 0) if vram_total > 0: - console.print(f" VRAM: {vram_total:.1f} MB") console.print( - f" model load: {mem['vram_model_load_delta_mb']:+.1f} MB | " + f" VRAM: {vram_total:.1f} MB -> " + f"model load: {mem['vram_model_load_delta_mb']:+.1f} MB | " f"inference: {mem['vram_inference_delta_mb']:+.1f} MB | " f"total: {mem['vram_total_delta_mb']:+.1f} MB" ) From 62a0064b6224aee3dc35756335573e881bb312e4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 14:42:30 +0800 Subject: [PATCH 25/27] Split VRAM into local/shared in memory display --- src/winml/modelkit/commands/perf.py | 40 ++++++++++++------- .../session/monitor/memory_tracker.py | 16 ++++---- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index f2f02beb8..c79f3d4b5 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -311,7 +311,7 @@ def run(self) -> BenchmarkResult: adapter_luid = self._resolve_adapter_luid() gc.collect() rss_baseline = get_rss_mb() - vram_baseline = get_vram_mb(adapter_luid) + vram_local_baseline, vram_shared_baseline = get_vram_mb(adapter_luid) # [2] Compile session (ORT loads model weights into memory here) self._model._session.compile() @@ -319,7 +319,7 @@ def run(self) -> BenchmarkResult: if self.config.memory: gc.collect() rss_after_compile = get_rss_mb() - vram_after_compile = get_vram_mb(adapter_luid) + vram_local_compile, vram_shared_compile = get_vram_mb(adapter_luid) # Print model info before benchmark starts _print_model_info( @@ -343,7 +343,7 @@ def run(self) -> BenchmarkResult: if self.config.memory: rss_after_inference = get_rss_mb() - vram_after_inference = get_vram_mb(adapter_luid) + vram_local_infer, vram_shared_infer = get_vram_mb(adapter_luid) self._memory = { "rss_baseline_mb": round(rss_baseline, 2), "rss_after_compile_mb": round(rss_after_compile, 2), @@ -351,12 +351,18 @@ def run(self) -> BenchmarkResult: "rss_model_load_delta_mb": round(rss_after_compile - rss_baseline, 2), "rss_inference_delta_mb": round(rss_after_inference - rss_after_compile, 2), "rss_total_delta_mb": round(rss_after_inference - rss_baseline, 2), - "vram_baseline_mb": round(vram_baseline, 2), - "vram_after_compile_mb": round(vram_after_compile, 2), - "vram_after_inference_mb": round(vram_after_inference, 2), - "vram_model_load_delta_mb": round(vram_after_compile - vram_baseline, 2), - "vram_inference_delta_mb": round(vram_after_inference - vram_after_compile, 2), - "vram_total_delta_mb": round(vram_after_inference - vram_baseline, 2), + "vram_local_after_inference_mb": round(vram_local_infer, 2), + "vram_shared_after_inference_mb": round(vram_shared_infer, 2), + "vram_local_model_load_delta_mb": round( + vram_local_compile - vram_local_baseline, 2 + ), + "vram_local_inference_delta_mb": round(vram_local_infer - vram_local_compile, 2), + "vram_local_total_delta_mb": round(vram_local_infer - vram_local_baseline, 2), + "vram_shared_model_load_delta_mb": round( + vram_shared_compile - vram_shared_baseline, 2 + ), + "vram_shared_inference_delta_mb": round(vram_shared_infer - vram_shared_compile, 2), + "vram_shared_total_delta_mb": round(vram_shared_infer - vram_shared_baseline, 2), } # [4] Collect results @@ -949,13 +955,17 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: f"inference: {mem['rss_inference_delta_mb']:+.1f} MB | " f"total: {mem['rss_total_delta_mb']:+.1f} MB" ) - vram_total = mem.get("vram_after_inference_mb", 0) - if vram_total > 0: + vram_local = mem.get("vram_local_after_inference_mb", 0) + vram_shared = mem.get("vram_shared_after_inference_mb", 0) + if vram_local > 0 or vram_shared > 0: console.print( - f" VRAM: {vram_total:.1f} MB -> " - f"model load: {mem['vram_model_load_delta_mb']:+.1f} MB | " - f"inference: {mem['vram_inference_delta_mb']:+.1f} MB | " - f"total: {mem['vram_total_delta_mb']:+.1f} MB" + f" VRAM: {vram_local:.1f}/{vram_shared:.1f} MB (local/shared) -> " + f"model load: {mem['vram_local_model_load_delta_mb']:+.1f}/" + f"{mem['vram_shared_model_load_delta_mb']:+.1f} MB | " + f"inference: {mem['vram_local_inference_delta_mb']:+.1f}/" + f"{mem['vram_shared_inference_delta_mb']:+.1f} MB | " + f"total: {mem['vram_local_total_delta_mb']:+.1f}/" + f"{mem['vram_shared_total_delta_mb']:+.1f} MB" ) console.print() diff --git a/src/winml/modelkit/session/monitor/memory_tracker.py b/src/winml/modelkit/session/monitor/memory_tracker.py index 0fac374cc..16de1d878 100644 --- a/src/winml/modelkit/session/monitor/memory_tracker.py +++ b/src/winml/modelkit/session/monitor/memory_tracker.py @@ -21,13 +21,13 @@ def get_rss_mb() -> float: return psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024) -def get_vram_mb(adapter_luid: str | None) -> float: - """Return current VRAM usage (local + shared) in MB via PDH. +def get_vram_mb(adapter_luid: str | None) -> tuple[float, float]: + """Return current VRAM usage as (local_mb, shared_mb) via PDH. - Returns 0.0 on non-Windows, if no adapter_luid is provided, or on failure. + Returns (0.0, 0.0) on non-Windows, if no adapter_luid, or on failure. """ if sys.platform != "win32" or not adapter_luid: - return 0.0 + return 0.0, 0.0 try: from ._pdh import PdhQuery @@ -46,9 +46,9 @@ def get_vram_mb(adapter_luid: str | None) -> float: # Memory counters are absolute (not rate-based), single collect suffices. values = q.collect() q.close() - local = values.get("local") or 0 - shared = values.get("shared") or 0 - return (local + shared) / (1024 * 1024) + local = (values.get("local") or 0) / (1024 * 1024) + shared = (values.get("shared") or 0) / (1024 * 1024) + return local, shared except Exception: logger.debug("VRAM query failed", exc_info=True) - return 0.0 + return 0.0, 0.0 From de15995f2c245501ae3b92f3d0c3a3485771c077 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 15:03:46 +0800 Subject: [PATCH 26/27] Fix CodeQL: initialize memory variables before conditional blocks --- src/winml/modelkit/commands/perf.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 72454f147..c920fe7d7 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -392,6 +392,12 @@ def _run_single(self) -> BenchmarkResult: assert self._model is not None + # Initialize memory tracking variables + adapter_luid: str | None = None + rss_baseline = rss_after_compile = 0.0 + vram_local_baseline = vram_shared_baseline = 0.0 + vram_local_compile = vram_shared_compile = 0.0 + # Memory: baseline right before compile() — excludes all Python lib # imports, EP DLLs, and build pipeline overhead. Measures only ORT # session compilation (model weights loaded into memory). From 11e573cfafea25791cf43d6ce8769373fd1394b7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 16 Jun 2026 15:13:18 +0800 Subject: [PATCH 27/27] Fix type error: use self._single in _resolve_adapter_luid --- src/winml/modelkit/commands/perf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index c920fe7d7..d5d8b506e 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -537,14 +537,14 @@ def _resolve_adapter_luid(self) -> str | None: return None assert self._model is not None - device = self._model.device or self.config.device + device = self._single.device or self.config.device if device == "cpu": return None try: from ..sysinfo.pdh_adapters import resolve_adapter_luid - ep_name = self._model.ep_name + ep_name = self._single.ep_name for kind in [device] if device in ("npu", "gpu") else ["npu", "gpu"]: luid = resolve_adapter_luid(kind, ep_name=ep_name) if luid: