From 63378616a6390a00ac6eff4bc327522df553997b Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Wed, 15 Jul 2026 15:15:39 -0700 Subject: [PATCH 01/64] feat(sana): add SANA-WM integration shim Add a workspace integration package for the upstream NVlabs/Sana SANA-WM bidirectional world model. The package registers the `sana-wm-bidirectional` runner, resolves upstream Sana assets and Hugging Face checkpoints, loads images, prompts, camera trajectories, and intrinsics, then writes generated videos through the upstream pipeline. Port CPU-safe camera trajectory and intrinsics helpers so the integration can be checked without running GPU inference. Add smoke and camera tests covering runner registration, config derivation, action parsing, and geometry transforms. Signed-off-by: Aidan Foster --- integrations/sana/README.md | 77 +++ integrations/sana/pyproject.toml | 50 ++ integrations/sana/sana_wm/__init__.py | 23 + integrations/sana/sana_wm/camera.py | 513 ++++++++++++++++++ integrations/sana/sana_wm/config.py | 56 ++ integrations/sana/sana_wm/constants.py | 49 ++ integrations/sana/sana_wm/placeholder.py | 77 +++ integrations/sana/sana_wm/runner.py | 435 +++++++++++++++ .../sana/tests/parity_check/README.md | 21 + integrations/sana/tests/test_camera.py | 142 +++++ integrations/sana/tests/test_smoke.py | 105 ++++ pyproject.toml | 2 + uv.lock | 21 + 13 files changed, 1571 insertions(+) create mode 100644 integrations/sana/README.md create mode 100644 integrations/sana/pyproject.toml create mode 100644 integrations/sana/sana_wm/__init__.py create mode 100644 integrations/sana/sana_wm/camera.py create mode 100644 integrations/sana/sana_wm/config.py create mode 100644 integrations/sana/sana_wm/constants.py create mode 100644 integrations/sana/sana_wm/placeholder.py create mode 100644 integrations/sana/sana_wm/runner.py create mode 100644 integrations/sana/tests/parity_check/README.md create mode 100644 integrations/sana/tests/test_camera.py create mode 100644 integrations/sana/tests/test_smoke.py diff --git a/integrations/sana/README.md b/integrations/sana/README.md new file mode 100644 index 000000000..88d2c2911 --- /dev/null +++ b/integrations/sana/README.md @@ -0,0 +1,77 @@ + + +# `sana_wm` + +FlashDreams runner shim for +[SANA-WM](https://huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional), +the 2.6B bidirectional camera-controlled world model from the +[NVlabs/Sana](https://github.com/NVlabs/Sana) repository. + +This first slice follows the dependency-isolated integration pattern: +FlashDreams owns the CLI registration, config surface, and CPU-tested +camera/action helpers, while Stage-1 DiT, LTX2 VAE/refiner, Pi3X, and +custom attention kernels are imported from an installed or local upstream +Sana checkout at runtime. + +## Runner + +| slug | description | +| --- | --- | +| `sana-wm-bidirectional` | SANA-WM bidirectional I2V world model using upstream NVlabs/Sana Stage-1 + LTX-2 refiner. | + +The FlashDreams package is named `sana_wm` rather than `sana` so it does +not shadow upstream's own top-level `sana` package. + +## Setup + +Clone Sana next to FlashDreams or pass its path explicitly: + +```bash +git clone https://github.com/NVlabs/Sana ../Sana +bash ../Sana/environment_setup.sh sana +``` + +The runner auto-detects `../Sana`. You can also set `SANA_ROOT` or pass +`--upstream-sana-root /path/to/Sana`. + +## Run + +```bash +uv run flashdreams-run sana-wm-bidirectional \ + --image-path ../Sana/asset/sana_wm/demo_0.png \ + --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ + --action "w-100,dw-60,w-100,aw-60" \ + --num-frames 321 \ + --output-dir outputs/sana_wm +``` + +To use an explicit trajectory: + +```bash +uv run flashdreams-run sana-wm-bidirectional \ + --image-path ../Sana/asset/sana_wm/demo_0.png \ + --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ + --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ + --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --num-frames 321 \ + --output-dir outputs/sana_wm +``` + +`--no-refiner` skips the LTX-2 refiner and decodes Stage-1 latents +directly, matching upstream's fast debugging path. Full generation still +downloads the public Hugging Face artefacts on first use. + +## Tests + +CPU-safe tests cover import, runner config, action parsing, intrinsics +handling, frame snapping, and SANA-WM camera-conditioning tensor shapes: + +```bash +uv run --extra dev pytest integrations/sana/tests +``` + +Full upstream parity and generated-video checks are heavyweight GPU +workflows and should live under `tests/parity_check/`. diff --git a/integrations/sana/pyproject.toml b/integrations/sana/pyproject.toml new file mode 100644 index 000000000..d0baed354 --- /dev/null +++ b/integrations/sana/pyproject.toml @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "flashdreams-sana-wm" +version = "0.1.0" +description = "SANA-WM bidirectional world-model runner shim for flashdreams." +readme = "README.md" +requires-python = ">=3.10" +# The upstream SANA repository owns the Stage-1 DiT, LTX2 VAE/refiner, +# Pi3X intrinsics estimator, and custom attention kernels. Those heavy +# dependencies are intentionally installed from the upstream SANA checkout, +# not pulled into every FlashDreams workspace sync. +dependencies = [ + "flashdreams", +] + +[tool.uv.sources] +flashdreams = { workspace = true } + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", +] + +[project.entry-points."flashdreams.runner_configs"] +"sana-wm-bidirectional" = "sana_wm.config:RUNNER_SANA_WM_BIDIRECTIONAL" + +[tool.setuptools.packages.find] +include = ["sana_wm*"] +exclude = ["tests"] + +[tool.uv] +managed = true diff --git a/integrations/sana/sana_wm/__init__.py b/integrations/sana/sana_wm/__init__.py new file mode 100644 index 000000000..f4b7dbd28 --- /dev/null +++ b/integrations/sana/sana_wm/__init__.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SANA-WM bidirectional runner shim for FlashDreams.""" + +from sana_wm.config import RUNNER_CONFIGS, RUNNER_SANA_WM_BIDIRECTIONAL + +__all__ = [ + "RUNNER_CONFIGS", + "RUNNER_SANA_WM_BIDIRECTIONAL", +] diff --git a/integrations/sana/sana_wm/camera.py b/integrations/sana/sana_wm/camera.py new file mode 100644 index 000000000..97daf41ba --- /dev/null +++ b/integrations/sana/sana_wm/camera.py @@ -0,0 +1,513 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SANA-WM camera-pose DSL, intrinsics, and Plucker conditioning helpers.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import torch + +from sana_wm.constants import ( + DEFAULT_VIDEO_HEIGHT, + DEFAULT_VIDEO_WIDTH, + SANA_WM_VAE_SPATIAL_COMPRESSION, + SANA_WM_VAE_TEMPORAL_COMPRESSION, +) + +FPS = 16 +"""Camera-control integration frame rate.""" + +DEFAULT_TRANSLATION_SPEED = 0.025 +"""Default per-frame translation magnitude used by upstream SANA-WM.""" + +DEFAULT_ROTATION_SPEED_DEG = 0.6 +"""Default per-frame rotation magnitude in degrees.""" + +DEFAULT_PITCH_LIMIT_DEG = 60.0 +"""Maximum absolute camera pitch in degrees.""" + +TAU_PRESS = 0.45 +"""Velocity smoothing time constant when a key is pressed.""" + +TAU_COAST = 1.0 +"""Velocity smoothing time constant when controls are released.""" + +DSL_KEY_TO_CONTROL: dict[str, str] = { + "w": "forward", + "s": "back", + "a": "yaw_left", + "d": "yaw_right", + "i": "pitch_up", + "k": "pitch_down", + "j": "strafe_left", + "l": "strafe_right", +} +"""Mapping from SANA-WM action DSL letters to canonical controls.""" + +ALLOWED_ACTION_KEYS = frozenset(DSL_KEY_TO_CONTROL) +"""Action DSL keys accepted in ``-`` segments.""" + + +@dataclass +class VelocityState: + """Per-frame camera velocity in OpenCV camera coordinates.""" + + tx: float = 0.0 + """Forward translation velocity.""" + + sx: float = 0.0 + """Rightward strafe velocity.""" + + yaw: float = 0.0 + """Positive yaw-right angular velocity in radians.""" + + pitch: float = 0.0 + """Positive pitch-up angular velocity in radians.""" + + def snap_to(self, target: "VelocityState") -> None: + """Copy all velocity components from ``target``.""" + self.tx, self.sx = target.tx, target.sx + self.yaw, self.pitch = target.yaw, target.pitch + + def step_toward(self, target: "VelocityState", dt: float) -> None: + """Ease velocity components toward ``target`` over one timestep.""" + for attr in ("tx", "sx", "yaw", "pitch"): + cur = getattr(self, attr) + tgt = getattr(target, attr) + tau = TAU_PRESS if abs(tgt) > 1e-12 else TAU_COAST + alpha = 1.0 - math.exp(-dt / tau) + setattr(self, attr, cur + alpha * (tgt - cur)) + + +class CameraPoseIntegrator: + """Integrate camera velocity into camera-to-world poses.""" + + def __init__( + self, pitch_limit_rad: float = math.radians(DEFAULT_PITCH_LIMIT_DEG) + ) -> None: + self.pose = np.eye(4, dtype=np.float64) + self.pitch = 0.0 + self.pitch_limit = float(pitch_limit_rad) + + def step(self, velocity: VelocityState) -> np.ndarray: + """Advance one frame and return the current camera-to-world pose.""" + new_pitch = max( + -self.pitch_limit, min(self.pitch_limit, self.pitch + velocity.pitch) + ) + pitch_step = new_pitch - self.pitch + self.pitch = new_pitch + + rotation = self.pose[:3, :3] + rotation_new = _rot_y(velocity.yaw) @ rotation @ _rot_x(pitch_step) + + forward = rotation_new[:, 2].copy() + forward[1] = 0.0 + right = rotation_new[:, 0].copy() + right[1] = 0.0 + forward_norm = float(np.linalg.norm(forward)) + right_norm = float(np.linalg.norm(right)) + if forward_norm > 0: + forward /= forward_norm + 1e-6 + if right_norm > 0: + right /= right_norm + 1e-6 + + translation = self.pose[:3, 3] + forward * velocity.tx + right * velocity.sx + self.pose = np.eye(4, dtype=np.float64) + self.pose[:3, :3] = rotation_new + self.pose[:3, 3] = translation + return self.pose.copy() + + +def _rot_x(angle_rad: float) -> np.ndarray: + c, s = math.cos(angle_rad), math.sin(angle_rad) + return np.array([[1.0, 0.0, 0.0], [0.0, c, -s], [0.0, s, c]], dtype=np.float64) + + +def _rot_y(angle_rad: float) -> np.ndarray: + c, s = math.cos(angle_rad), math.sin(angle_rad) + return np.array([[c, 0.0, s], [0.0, 1.0, 0.0], [-s, 0.0, c]], dtype=np.float64) + + +def controls_to_target_velocity( + controls: set[str], + *, + translation_speed: float = DEFAULT_TRANSLATION_SPEED, + rotation_speed_rad: float | None = None, +) -> VelocityState: + """Map canonical control tokens to a target velocity.""" + if rotation_speed_rad is None: + rotation_speed_rad = math.radians(DEFAULT_ROTATION_SPEED_DEG) + forward = (1.0 if "forward" in controls else 0.0) - ( + 1.0 if "back" in controls else 0.0 + ) + strafe = (1.0 if "strafe_right" in controls else 0.0) - ( + 1.0 if "strafe_left" in controls else 0.0 + ) + yaw = (1.0 if "yaw_right" in controls else 0.0) - ( + 1.0 if "yaw_left" in controls else 0.0 + ) + pitch = (1.0 if "pitch_up" in controls else 0.0) - ( + 1.0 if "pitch_down" in controls else 0.0 + ) + return VelocityState( + tx=forward * translation_speed, + sx=strafe * translation_speed, + yaw=yaw * rotation_speed_rad, + pitch=pitch * rotation_speed_rad, + ) + + +def parse_action_string(action: str) -> list[list[str]]: + """Expand a SANA-WM action string into per-frame held keys. + + Args: + action: Comma-separated ``-`` segments. ``none-N`` + holds the pose for ``N`` frames. + + Returns: + Per-frame lists of DSL keys. + + Raises: + ValueError: The action string is empty or contains an invalid segment. + """ + cleaned = "".join(action.replace("，", ",").split()) + if not cleaned: + raise ValueError("action string is empty") + per_frame: list[list[str]] = [] + for segment in cleaned.split(","): + if not segment or "-" not in segment: + raise ValueError( + f"Invalid action segment {segment!r}: expected '-'." + ) + keys_part, duration_part = segment.rsplit("-", 1) + if not duration_part.isdigit() or int(duration_part) <= 0: + raise ValueError(f"Action segment {segment!r} has a non-positive duration.") + keys_lower = keys_part.lower() + if keys_lower == "none": + keys: list[str] = [] + else: + bad = sorted({key for key in keys_lower if key not in ALLOWED_ACTION_KEYS}) + if bad: + raise ValueError( + f"Action segment {segment!r} contains unknown keys {bad}; " + f"allowed: {''.join(sorted(ALLOWED_ACTION_KEYS))}." + ) + keys = sorted(set(keys_lower)) + per_frame.extend([keys] * int(duration_part)) + return per_frame + + +def action_string_to_c2w( + action: str, + *, + translation_speed: float = DEFAULT_TRANSLATION_SPEED, + rotation_speed_deg: float = DEFAULT_ROTATION_SPEED_DEG, + pitch_limit_deg: float = DEFAULT_PITCH_LIMIT_DEG, + smooth: bool = True, +) -> np.ndarray: + """Roll out a camera-to-world trajectory from an action string. + + Args: + action: Comma-separated ``-`` segments. + translation_speed: Per-frame translation magnitude. + rotation_speed_deg: Per-frame angular magnitude in degrees. + pitch_limit_deg: Maximum absolute pitch in degrees. + smooth: Whether to use upstream's velocity smoothing model. + + Returns: + ``[F + 1, 4, 4]`` camera-to-world matrices in OpenCV convention. + """ + per_frame = parse_action_string(action) + integrator = CameraPoseIntegrator(math.radians(pitch_limit_deg)) + velocity = VelocityState() + poses = [integrator.pose.copy()] + last_controls: set[str] = set() + dt = 1.0 / FPS + rotation_speed_rad = math.radians(rotation_speed_deg) + + for keys in per_frame: + controls = {DSL_KEY_TO_CONTROL[key] for key in keys} + target = controls_to_target_velocity( + controls, + translation_speed=translation_speed, + rotation_speed_rad=rotation_speed_rad, + ) + if smooth: + if controls - last_controls: + velocity.snap_to(target) + else: + velocity.step_toward(target, dt) + last_controls = controls + else: + velocity = target + poses.append(integrator.step(velocity)) + + return np.stack(poses, axis=0).astype(np.float32) + + +def fit_intrinsics_sequence(arr: np.ndarray, num_frames: int) -> np.ndarray: + """Return ``arr`` fitted to ``num_frames`` along axis 0.""" + arr = np.asarray(arr, dtype=np.float32) + if arr.shape[0] == num_frames: + return arr.copy() + if arr.shape[0] > num_frames: + return arr[:num_frames].copy() + if arr.shape[0] == 1: + return np.broadcast_to(arr[:1], (num_frames, *arr.shape[1:])).copy() + + old_t = np.linspace(0.0, 1.0, arr.shape[0], dtype=np.float32) + new_t = np.linspace(0.0, 1.0, num_frames, dtype=np.float32) + flat = arr.reshape(arr.shape[0], -1) + fitted = np.empty((num_frames, flat.shape[1]), dtype=np.float32) + for idx in range(flat.shape[1]): + fitted[:, idx] = np.interp(new_t, old_t, flat[:, idx]).astype(np.float32) + return fitted.reshape((num_frames, *arr.shape[1:])) + + +def load_intrinsics(path: Path, num_frames: int) -> np.ndarray: + """Return ``[num_frames, 4]`` intrinsics as ``[fx, fy, cx, cy]``. + + Args: + path: ``.npy`` file shaped ``[3, 3]``, ``[F, 3, 3]``, ``[4]``, or + ``[F, 4]``. + num_frames: Number of frames required by the rollout. + + Returns: + Per-frame vector intrinsics. + + Raises: + ValueError: The file shape is unsupported. + """ + arr = np.load(path).astype(np.float32) + if arr.shape == (4,): + return np.broadcast_to(arr, (num_frames, 4)).copy() + if arr.shape == (3, 3): + vector = np.array( + [arr[0, 0], arr[1, 1], arr[0, 2], arr[1, 2]], dtype=np.float32 + ) + return np.broadcast_to(vector, (num_frames, 4)).copy() + if arr.ndim == 2 and arr.shape[1] == 4: + return fit_intrinsics_sequence(arr, num_frames) + if arr.ndim == 3 and arr.shape[1:] == (3, 3): + matrix = fit_intrinsics_sequence(arr, num_frames) + return np.stack( + [matrix[:, 0, 0], matrix[:, 1, 1], matrix[:, 0, 2], matrix[:, 1, 2]], + axis=1, + ) + raise ValueError( + f"Unsupported intrinsics shape {arr.shape} for num_frames={num_frames}; " + "expected (3,3), (F,3,3), (4,), or (F,4)." + ) + + +def snap_num_frames( + num_frames: int, + *, + stride: int = SANA_WM_VAE_TEMPORAL_COMPRESSION, + upper_bound: int | None = None, +) -> int: + """Snap frame count to the nearest ``stride * k + 1`` value.""" + if num_frames < 1: + return 1 + if (num_frames - 1) % stride == 0: + return num_frames + floor_cand = num_frames - ((num_frames - 1) % stride) + ceil_cand = floor_cand + stride + snapped = ( + floor_cand + if (num_frames - floor_cand) < (ceil_cand - num_frames) + else ceil_cand + ) + if upper_bound is not None and snapped > upper_bound: + snapped = floor_cand + return max(snapped, 1) + + +def transform_intrinsics_for_crop( + intrinsics_vec4: np.ndarray, + src_size: tuple[int, int], + resized_size: tuple[int, int], + crop_offset: tuple[int, int], +) -> np.ndarray: + """Adjust ``[fx, fy, cx, cy]`` to match resize-then-center-crop pixels.""" + src_w, src_h = src_size + resized_w, resized_h = resized_size + crop_left, crop_top = crop_offset + sx, sy = resized_w / src_w, resized_h / src_h + out = intrinsics_vec4.copy() + out[..., 0] *= sx + out[..., 2] = out[..., 2] * sx - crop_left + out[..., 1] *= sy + out[..., 3] = out[..., 3] * sy - crop_top + return out + + +def resize_center_crop_geometry( + src_size: tuple[int, int], + *, + target_h: int = DEFAULT_VIDEO_HEIGHT, + target_w: int = DEFAULT_VIDEO_WIDTH, +) -> tuple[tuple[int, int], tuple[int, int]]: + """Return resize dimensions and center-crop offset for an input image size. + + Args: + src_size: Source image size as ``(width, height)``. + target_h: Target crop height. + target_w: Target crop width. + + Returns: + ``(resized_size, crop_offset)`` where sizes are ``(width, height)`` + and offsets are ``(left, top)``. + """ + src_w, src_h = src_size + scale = max(target_h / src_h, target_w / src_w) + resized_w = max(target_w, int(round(src_w * scale))) + resized_h = max(target_h, int(round(src_h * scale))) + left = (resized_w - target_w) // 2 + top = (resized_h - target_h) // 2 + return (resized_w, resized_h), (left, top) + + +def get_pose_inverse(transform: torch.Tensor) -> torch.Tensor: + """Return inverse homogeneous transforms using rotation transpose.""" + rotation = transform[..., :3, :3] + translation = transform[..., :3, 3] + rotation_inv = rotation.transpose(-1, -2) + translation_inv = -torch.matmul(rotation_inv, translation.unsqueeze(-1)).squeeze(-1) + out = torch.eye(4, dtype=transform.dtype, device=transform.device).repeat( + transform.shape[:-2] + (1, 1) + ) + out[..., :3, :3] = rotation_inv + out[..., :3, 3] = translation_inv + return out + + +def compute_raymap( + intrinsics: torch.Tensor, + poses: torch.Tensor, + height: int, + width: int, + *, + use_plucker: bool = True, +) -> torch.Tensor: + """Compute world-space ray directions and moments for camera poses. + + Args: + intrinsics: ``[T, 4]`` tensor of ``[fx, fy, cx, cy]``. + poses: ``[T, 4, 4]`` camera-to-world matrices. + height: Raymap height. + width: Raymap width. + use_plucker: ``True`` returns ``[direction, moment]``; otherwise + returns ``[origin, direction]``. + + Returns: + ``[T, height, width, 6]`` raymap tensor. + """ + num_frames = intrinsics.shape[0] + device, dtype = intrinsics.device, intrinsics.dtype + y_grid, x_grid = torch.meshgrid( + torch.arange(height, device=device, dtype=dtype), + torch.arange(width, device=device, dtype=dtype), + indexing="ij", + ) + x_grid = x_grid[None].expand(num_frames, -1, -1) + y_grid = y_grid[None].expand(num_frames, -1, -1) + + fx = intrinsics[:, 0].view(num_frames, 1, 1) + fy = intrinsics[:, 1].view(num_frames, 1, 1) + cx = intrinsics[:, 2].view(num_frames, 1, 1) + cy = intrinsics[:, 3].view(num_frames, 1, 1) + + dirs_cam = torch.stack( + [(x_grid - cx) / fx, (y_grid - cy) / fy, torch.ones_like(x_grid)], + dim=-1, + ) + rotation = poses[:, :3, :3] + translation = poses[:, :3, 3] + dirs_world = torch.einsum("tij,thwj->thwi", rotation, dirs_cam) + dirs_world = dirs_world / torch.norm(dirs_world, dim=-1, keepdim=True) + origins = translation.view(num_frames, 1, 1, 3).expand_as(dirs_world) + if use_plucker: + return torch.cat([dirs_world, torch.cross(origins, dirs_world, dim=-1)], dim=-1) + return torch.cat([origins, dirs_world], dim=-1) + + +def prepare_camera( + poses_c2w: np.ndarray, + intrinsics_vec4: np.ndarray, + *, + target_size: tuple[int, int] = (DEFAULT_VIDEO_HEIGHT, DEFAULT_VIDEO_WIDTH), + vae_stride: tuple[int, int, int] = ( + SANA_WM_VAE_TEMPORAL_COMPRESSION, + SANA_WM_VAE_SPATIAL_COMPRESSION, + SANA_WM_VAE_SPATIAL_COMPRESSION, + ), +) -> dict[str, torch.Tensor]: + """Build SANA-WM raymap and chunk-Plucker conditioning tensors.""" + num_frames = poses_c2w.shape[0] + vae_time_stride, vae_spatial_stride = vae_stride[0], vae_stride[-1] + pixel_h, pixel_w = target_size + latent_h = pixel_h // vae_spatial_stride + latent_w = pixel_w // vae_spatial_stride + latent_frames = (num_frames - 1) // vae_time_stride + 1 + + poses = torch.from_numpy(poses_c2w).float() + first_inv = get_pose_inverse(poses[0:1]).squeeze(0) + poses_rel = torch.matmul(first_inv, poses[1:]) + poses = torch.cat([torch.eye(4).unsqueeze(0), poses_rel], dim=0) + + intrinsics = torch.from_numpy(intrinsics_vec4).float() + intrinsics_latent = intrinsics.clone() + intrinsics_latent[:, [0, 2]] *= latent_w / float(pixel_w) + intrinsics_latent[:, [1, 3]] *= latent_h / float(pixel_h) + + time_indices = torch.arange(0, num_frames, vae_time_stride) + if len(time_indices) > latent_frames: + time_indices = time_indices[:latent_frames] + raymap = torch.cat( + [ + poses[time_indices].reshape(len(time_indices), -1), + intrinsics_latent[time_indices], + ], + dim=-1, + ) + + chunks = [] + for start in time_indices - (vae_time_stride - 1): + chunk_start = max(0, int(start)) + chunk_end = chunk_start + vae_time_stride + chunk_poses = poses[chunk_start:chunk_end] + chunk_intrinsics = intrinsics_latent[chunk_start:chunk_end] + if chunk_poses.shape[0] < vae_time_stride: + pad = vae_time_stride - chunk_poses.shape[0] + chunk_poses = torch.cat([chunk_poses, chunk_poses[-1:].repeat(pad, 1, 1)]) + chunk_intrinsics = torch.cat( + [chunk_intrinsics, chunk_intrinsics[-1:].repeat(pad, 1)] + ) + plucker = compute_raymap( + chunk_intrinsics, + chunk_poses, + latent_h, + latent_w, + use_plucker=True, + ) + chunks.append(plucker.permute(0, 3, 1, 2).reshape(-1, latent_h, latent_w)) + chunk_plucker = torch.stack(chunks).permute(1, 0, 2, 3) + return {"raymap": raymap, "chunk_plucker": chunk_plucker} diff --git a/integrations/sana/sana_wm/config.py b/integrations/sana/sana_wm/config.py new file mode 100644 index 000000000..ffa72922e --- /dev/null +++ b/integrations/sana/sana_wm/config.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Static configs for the SANA-WM bidirectional runner.""" + +from __future__ import annotations + +from flashdreams.infra.diffusion.model import DiffusionModelConfig +from flashdreams.infra.diffusion.scheduler import FlowMatchSchedulerConfig +from flashdreams.infra.pipeline import StreamInferencePipelineConfig +from flashdreams.infra.runner import RunnerConfig +from sana_wm.placeholder import SanaWMPlaceholderTransformerConfig +from sana_wm.runner import SanaWMRunnerConfig + +PIPELINE_SANA_WM_BIDIRECTIONAL = StreamInferencePipelineConfig( + name="sana-wm-bidirectional", + diffusion_model=DiffusionModelConfig( + transformer=SanaWMPlaceholderTransformerConfig(), + scheduler=FlowMatchSchedulerConfig(), + seed=42, + ), +) +"""Schema placeholder for the upstream-delegating SANA-WM runner.""" + +RUNNER_SANA_WM_BIDIRECTIONAL = SanaWMRunnerConfig( + runner_name=PIPELINE_SANA_WM_BIDIRECTIONAL.name, + description=( + "SANA-WM bidirectional I2V world model (upstream NVlabs/Sana " + "Stage-1 + LTX-2 refiner runner)." + ), + pipeline=PIPELINE_SANA_WM_BIDIRECTIONAL, +) +"""Default SANA-WM bidirectional runner config.""" + +RUNNER_CONFIGS: dict[str, RunnerConfig] = { + cfg.runner_name: cfg for cfg in (RUNNER_SANA_WM_BIDIRECTIONAL,) +} +"""SANA-WM runner configs keyed by ``runner_name``.""" + +__all__ = [ + "PIPELINE_SANA_WM_BIDIRECTIONAL", + "RUNNER_CONFIGS", + "RUNNER_SANA_WM_BIDIRECTIONAL", +] diff --git a/integrations/sana/sana_wm/constants.py b/integrations/sana/sana_wm/constants.py new file mode 100644 index 000000000..5251099ca --- /dev/null +++ b/integrations/sana/sana_wm/constants.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Constants for the public SANA-WM bidirectional release.""" + +SANA_WM_HF_REPO = "Efficient-Large-Model/SANA-WM_bidirectional" +"""Hugging Face repository containing SANA-WM bidirectional artefacts.""" + +SANA_WM_MODEL_PATH = f"hf://{SANA_WM_HF_REPO}/dit/sana_wm_1600m_720p.safetensors" +"""Default Stage-1 SANA-WM DiT checkpoint.""" + +SANA_WM_CONFIG_PATH = f"hf://{SANA_WM_HF_REPO}/config.yaml" +"""Default upstream inference YAML.""" + +SANA_WM_REFINER_ROOT = f"hf://{SANA_WM_HF_REPO}/refiner" +"""Default LTX-2 refiner root.""" + +SANA_WM_REFINER_GEMMA_ROOT = f"hf://{SANA_WM_HF_REPO}/refiner/text_encoder" +"""Default Gemma text-encoder root used by the refiner.""" + +DEFAULT_VIDEO_HEIGHT = 704 +"""SANA-WM bidirectional output height in pixels.""" + +DEFAULT_VIDEO_WIDTH = 1280 +"""SANA-WM bidirectional output width in pixels.""" + +SANA_WM_VAE_TEMPORAL_COMPRESSION = 8 +"""Temporal compression ratio of the LTX2 VAE used by SANA-WM.""" + +SANA_WM_VAE_SPATIAL_COMPRESSION = 32 +"""Spatial compression ratio of the LTX2 VAE used by SANA-WM.""" + +DEFAULT_FPS = 16 +"""Frame rate used by the public SANA-WM examples.""" + +DEFAULT_ACTION = "w-100,dw-60,w-100,aw-60" +"""Default SANA-WM demo action string from upstream ``demo_0``.""" diff --git a/integrations/sana/sana_wm/placeholder.py b/integrations/sana/sana_wm/placeholder.py new file mode 100644 index 000000000..ef46f1690 --- /dev/null +++ b/integrations/sana/sana_wm/placeholder.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Placeholder FlashDreams pipeline objects for the upstream SANA-WM runner.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import torch +import torch.nn as nn +from torch import Tensor + +from flashdreams.infra.diffusion.transformer import ( + Transformer, + TransformerAutoregressiveCache, + TransformerConfig, +) + + +@dataclass(kw_only=True) +class SanaWMPlaceholderTransformerConfig(TransformerConfig): + """Transformer config used only to satisfy the runner config schema.""" + + _target: type["SanaWMPlaceholderTransformer"] = field( + default_factory=lambda: SanaWMPlaceholderTransformer + ) + + +class SanaWMPlaceholderTransformer(Transformer[TransformerAutoregressiveCache]): + """Non-executable transformer for the upstream-delegating SANA-WM runner.""" + + def __init__(self, config: SanaWMPlaceholderTransformerConfig) -> None: + super().__init__(config) + self._dummy = nn.Parameter(torch.empty(0)) + + @property + def latent_shape(self) -> tuple[int, ...]: + """Raise because SANA-WM generation is delegated to upstream Sana.""" + raise RuntimeError( + "SANA-WM bidirectional uses the custom SanaWMRunner path; " + "the placeholder FlashDreams pipeline is not executable." + ) + + def predict_flow( + self, + noisy_latent: Tensor, + timestep: Tensor, + cache: TransformerAutoregressiveCache, + input: Any = None, + ) -> Tensor: + """Raise because SANA-WM generation is delegated to upstream Sana.""" + raise RuntimeError( + "SANA-WM bidirectional uses the custom SanaWMRunner path; " + "the placeholder FlashDreams pipeline is not executable." + ) + + def patchify_and_maybe_split_cp(self, x: Any) -> Any: + """Return ``x`` unchanged for schema-only construction.""" + return x + + def unpatchify_and_maybe_gather_cp(self, x: Tensor) -> Tensor: + """Return ``x`` unchanged for schema-only construction.""" + return x diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py new file mode 100644 index 000000000..1a999673c --- /dev/null +++ b/integrations/sana/sana_wm/runner.py @@ -0,0 +1,435 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SANA-WM bidirectional runner that delegates execution to upstream Sana.""" + +from __future__ import annotations + +import importlib +import os +import sys +from dataclasses import dataclass, field +from pathlib import Path +from types import ModuleType +from typing import Literal + +import numpy as np +import torch +from loguru import logger + +from flashdreams.core.io.disk import preflight_runtime_write_paths +from flashdreams.infra.runner import Runner, RunnerConfig +from sana_wm.constants import ( + DEFAULT_ACTION, + DEFAULT_FPS, + DEFAULT_VIDEO_HEIGHT, + DEFAULT_VIDEO_WIDTH, + SANA_WM_CONFIG_PATH, + SANA_WM_MODEL_PATH, + SANA_WM_REFINER_GEMMA_ROOT, + SANA_WM_REFINER_ROOT, + SANA_WM_VAE_TEMPORAL_COMPRESSION, +) + +SamplingAlgo = Literal[ + "auto", + "flow_euler_ltx", + "flow_euler", + "flow_dpm-solver", + "chunk_flow_euler", + "self_forcing", +] +"""Sampling algorithms exposed by upstream SANA-WM inference.""" + + +@dataclass(kw_only=True) +class SanaWMRunnerConfig(RunnerConfig): + """Runner config for the SANA-WM bidirectional release.""" + + _target: type["SanaWMRunner"] = field(default_factory=lambda: SanaWMRunner) + + upstream_sana_root: Path | None = None + """Path to a cloned NVlabs/Sana checkout. ``None`` auto-detects + ``$SANA_ROOT`` / ``../Sana`` or imports an installed Sana package.""" + + device: str = "auto" + """Torch device for upstream SANA-WM. ``"auto"`` picks CUDA when available.""" + + image_path: Path | None = None + """Path to the first-frame RGB image. Required at ``run()`` time.""" + + prompt: str = "" + """Inline text prompt. A non-empty value wins over ``prompt_path``.""" + + prompt_path: Path | None = None + """Fallback prompt file read when ``prompt`` is empty.""" + + camera_path: Path | None = None + """Optional ``.npy`` camera-to-world trajectory shaped ``[F, 4, 4]``.""" + + intrinsics_path: Path | None = None + """Optional ``.npy`` intrinsics shaped ``[3, 3]``, ``[F, 3, 3]``, + ``[4]``, or ``[F, 4]``. When absent, upstream estimates intrinsics + with Pi3X.""" + + action: str | None = DEFAULT_ACTION + """Action DSL used when ``camera_path`` is not provided.""" + + translation_speed: float = 0.025 + """Per-frame action translation speed.""" + + rotation_speed_deg: float = 0.6 + """Per-frame action rotation speed in degrees.""" + + name: str = "sana_wm" + """Output filename stem passed to upstream ``write_video``.""" + + num_frames: int = 161 + """Requested output frames before LTX2-VAE stride snapping.""" + + fps: int = DEFAULT_FPS + """Output video frame rate.""" + + step: int = 60 + """Stage-1 DiT sampling steps.""" + + cfg_scale: float = 5.0 + """Classifier-free guidance scale for Stage 1.""" + + flow_shift: float | None = None + """Optional scheduler flow-shift override.""" + + sampling_algo: SamplingAlgo = "auto" + """Stage-1 sampler. ``"auto"`` follows upstream config defaults.""" + + chunk_interval_k: float | None = None + """ChunkFlowEuler interval ratio override.""" + + num_cached_blocks: int = 2 + """Self-forcing sampler cached-block count.""" + + sink_token: bool = False + """Whether self-forcing keeps chunk 0 as a permanent sink.""" + + num_frame_per_block: int = 3 + """Self-forcing latent frames per AR block.""" + + denoising_step_list: str = "" + """Comma-separated self-forcing timesteps ending in ``0``.""" + + save_stage1: bool = False + """Also decode the unrefined Stage-1 latent when the refiner is enabled.""" + + negative_prompt: str = "" + """Negative prompt used when ``cfg_scale > 1``.""" + + seed: int = 42 + """Stage-1 random seed.""" + + no_action_overlay: bool = False + """Skip upstream action-overlay compositing on generated videos.""" + + config_path: str = SANA_WM_CONFIG_PATH + """Upstream inference YAML path or ``hf://`` URI.""" + + model_path: str = SANA_WM_MODEL_PATH + """Stage-1 checkpoint path or ``hf://`` URI.""" + + no_refiner: bool = False + """Skip the LTX-2 refiner and decode Stage-1 latents directly.""" + + refiner_root: str = SANA_WM_REFINER_ROOT + """LTX-2 refiner root path or ``hf://`` URI.""" + + refiner_gemma_root: str = SANA_WM_REFINER_GEMMA_ROOT + """Gemma text-encoder root for the LTX-2 refiner.""" + + refiner_seed: int = 42 + """Refiner random seed.""" + + sink_size: int = 1 + """Number of sink latent frames used by the refiner.""" + + refiner_block_size: int | None = None + """Optional refiner AR block size; ``None`` uses sink-bidirectional mode.""" + + refiner_kv_max_frames: int = 11 + """Maximum refiner KV context frames in AR mode.""" + + offload_vae: bool = False + """Move the VAE to CPU between encode/decode phases.""" + + offload_refiner: bool = False + """Build and release the refiner only around refinement.""" + + offload_text_encoder: bool = False + """Move the Stage-1 text encoder to CPU between prompt encodes.""" + + +class SanaWMRunner(Runner[SanaWMRunnerConfig, object]): + """CLI driver for upstream SANA-WM bidirectional inference.""" + + config: SanaWMRunnerConfig + + def __init__(self, config: SanaWMRunnerConfig) -> None: + self.config = config + self.local_rank = int(os.environ.get("LOCAL_RANK", "0")) + if torch.distributed.is_initialized(): + self.world_size = torch.distributed.get_world_size() + self.global_rank = torch.distributed.get_rank() + else: + self.world_size = int(os.environ.get("WORLD_SIZE", "1")) + self.global_rank = int(os.environ.get("RANK", "0")) + self.is_rank_zero = self.global_rank == 0 + preflight_runtime_write_paths(output_dir=config.output_dir) + + def _resolve_prompt(self) -> str: + """Resolve the prompt from inline text or ``prompt_path``.""" + if self.config.prompt: + return self.config.prompt + if self.config.prompt_path is None: + raise ValueError("SanaWMRunner requires --prompt or --prompt-path.") + prompt = self.config.prompt_path.read_text( + encoding="utf-8", errors="replace" + ).strip() + if not prompt: + raise ValueError(f"Prompt file is empty: {self.config.prompt_path}") + return prompt + + def _resolve_device(self) -> torch.device: + """Return the device used by upstream SANA-WM.""" + if self.config.device == "auto": + if torch.cuda.is_available(): + return torch.device(f"cuda:{self.local_rank}") + return torch.device("cpu") + if self.config.device == "cuda" and torch.cuda.is_available(): + return torch.device(f"cuda:{self.local_rank}") + return torch.device(self.config.device) + + def _resolve_trajectory(self, upstream: ModuleType) -> np.ndarray: + """Load or roll out the camera-to-world trajectory.""" + if self.config.camera_path is not None: + c2w = np.load(self.config.camera_path).astype(np.float32) + if c2w.ndim != 3 or c2w.shape[1:] != (4, 4): + raise ValueError( + f"--camera-path must be a [F, 4, 4] .npy; got {c2w.shape}." + ) + return c2w + if not self.config.action: + raise ValueError("SanaWMRunner requires --camera-path or --action.") + return upstream.action_string_to_c2w( + self.config.action, + translation_speed=self.config.translation_speed, + rotation_speed_deg=self.config.rotation_speed_deg, + ) + + def _denoising_steps(self) -> list[int] | None: + """Parse optional self-forcing denoising timesteps.""" + if not self.config.denoising_step_list: + return None + steps = [ + int(item.strip()) + for item in self.config.denoising_step_list.split(",") + if item.strip() + ] + if not steps or steps[-1] != 0: + raise ValueError("--denoising-step-list must end with 0.") + return steps + + def run(self) -> None: + """Run upstream SANA-WM bidirectional inference and write outputs.""" + cfg = self.config + if cfg.image_path is None: + raise ValueError("SanaWMRunner requires --image-path.") + + upstream = _import_upstream_sana(cfg.upstream_sana_root) + device = self._resolve_device() + prompt = self._resolve_prompt() + + image = upstream.Image.open(cfg.image_path).convert("RGB") + c2w_full = self._resolve_trajectory(upstream) + num_frames = min(cfg.num_frames, c2w_full.shape[0]) + snapped = upstream._snap_num_frames( + num_frames, + stride=SANA_WM_VAE_TEMPORAL_COMPRESSION, + upper_bound=c2w_full.shape[0], + ) + if snapped != cfg.num_frames and self.is_rank_zero: + logger.warning( + "SANA-WM requires num_frames = 8k+1; requested {} snapped to {} " + "(trajectory has {} frames).", + cfg.num_frames, + snapped, + c2w_full.shape[0], + ) + num_frames = snapped + c2w = c2w_full[:num_frames] + + cropped, src_size, resized_size, crop_offset = upstream.resize_and_center_crop( + image, + target_h=DEFAULT_VIDEO_HEIGHT, + target_w=DEFAULT_VIDEO_WIDTH, + ) + if cfg.intrinsics_path is None: + intrinsics_src = np.broadcast_to( + upstream.estimate_intrinsics_with_pi3x( + image, device, upstream.get_root_logger() + ), + (num_frames, 4), + ).copy() + else: + intrinsics_src = upstream.load_intrinsics(cfg.intrinsics_path, num_frames) + intrinsics_vec4 = upstream.transform_intrinsics_for_crop( + intrinsics_src, src_size, resized_size, crop_offset + ) + + inference_config = upstream.pyrallis.parse( + config_class=upstream.InferenceConfig, + config_path=upstream.resolve_hf_path(cfg.config_path), + args=[], + ) + refiner = ( + None + if cfg.no_refiner + else upstream.RefinerSettings( + root=cfg.refiner_root, + gemma_root=cfg.refiner_gemma_root, + sink_size=cfg.sink_size, + seed=cfg.refiner_seed, + block_size=cfg.refiner_block_size, + kv_max_frames=cfg.refiner_kv_max_frames, + ) + ) + pipeline = upstream.SanaWMPipeline( + config=inference_config, + model_path=upstream.resolve_hf_path(cfg.model_path), + device=device, + refiner=refiner, + offload_vae=cfg.offload_vae, + offload_refiner=cfg.offload_refiner, + offload_text_encoder=cfg.offload_text_encoder, + logger=upstream.get_root_logger(), + ) + + sampling_algo = cfg.sampling_algo + if sampling_algo == "auto": + sampling_algo = ( + inference_config.scheduler.vis_sampler + if inference_config.scheduler.vis_sampler + in {"chunk_flow_euler", "self_forcing"} + else "flow_euler_ltx" + ) + params = upstream.GenerationParams( + num_frames=num_frames, + fps=cfg.fps, + step=cfg.step, + cfg_scale=cfg.cfg_scale, + flow_shift=cfg.flow_shift, + seed=cfg.seed, + negative_prompt=cfg.negative_prompt, + sampling_algo=sampling_algo, + chunk_interval_k=cfg.chunk_interval_k, + num_cached_blocks=cfg.num_cached_blocks, + sink_token=cfg.sink_token, + num_frame_per_block=cfg.num_frame_per_block, + denoising_step_list=self._denoising_steps(), + save_stage1=cfg.save_stage1, + ) + + result = pipeline.generate(cropped, prompt, c2w, intrinsics_vec4, params) + video_hwc = result["video"] + if not cfg.no_action_overlay: + video_hwc = upstream.apply_overlay(video_hwc, result["c2w"]) + + if not self.is_rank_zero: + return + upstream.write_video( + cfg.output_dir, + cfg.name, + video_hwc, + params.fps, + upstream.get_root_logger(), + ) + + stage1_video = result.get("stage1_video") + if stage1_video is not None: + stage1_hwc = stage1_video + if not cfg.no_action_overlay: + stage1_hwc = upstream.apply_overlay(stage1_hwc, result["stage1_c2w"]) + upstream.write_video( + cfg.output_dir, + f"{cfg.name}_stage1", + stage1_hwc, + params.fps, + upstream.get_root_logger(), + ) + + +def _candidate_upstream_roots(explicit_root: Path | None) -> list[Path]: + """Return candidate NVlabs/Sana checkout roots in priority order.""" + roots: list[Path] = [] + if explicit_root is not None: + roots.append(explicit_root) + for env_name in ("SANA_ROOT", "SANA_REPO", "SANA_WM_UPSTREAM_ROOT"): + value = os.environ.get(env_name) + if value: + roots.append(Path(value)) + repo_root = Path(__file__).resolve().parents[3] + roots.append(repo_root.parent / "Sana") + roots.append(Path.cwd().parent / "Sana") + return roots + + +def _is_upstream_sana_root(path: Path) -> bool: + """Return ``True`` when ``path`` looks like an NVlabs/Sana checkout.""" + return ( + path.expanduser() / "inference_video_scripts" / "wm" / "inference_sana_wm.py" + ).is_file() + + +def _import_upstream_sana(explicit_root: Path | None) -> ModuleType: + """Import upstream SANA-WM inference after adding a checkout to ``sys.path``.""" + for root in _candidate_upstream_roots(explicit_root): + candidate = root.expanduser().resolve() + if _is_upstream_sana_root(candidate): + candidate_str = str(candidate) + if candidate_str not in sys.path: + sys.path.insert(0, candidate_str) + break + + try: + return importlib.import_module("inference_video_scripts.wm.inference_sana_wm") + except ModuleNotFoundError as exc: + roots = ", ".join( + str(path) for path in _candidate_upstream_roots(explicit_root) + ) + raise ModuleNotFoundError( + "Could not import upstream SANA-WM inference. Clone NVlabs/Sana " + "next to this repo, pass --upstream-sana-root, or set SANA_ROOT; " + "then install the upstream Sana inference dependencies. " + f"Checked roots: {roots}. Original import error: {exc}" + ) from exc + except Exception as exc: + raise RuntimeError( + "Importing upstream SANA-WM inference failed. Ensure the upstream " + "Sana checkout dependencies are installed in the active environment." + ) from exc + + +__all__ = [ + "SamplingAlgo", + "SanaWMRunner", + "SanaWMRunnerConfig", +] diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md new file mode 100644 index 000000000..08d5d5abf --- /dev/null +++ b/integrations/sana/tests/parity_check/README.md @@ -0,0 +1,21 @@ + + +# SANA-WM parity check + +This directory is reserved for the opt-in SANA-WM upstream parity +harness. It should follow the existing integration pattern: + +- clone or reuse a pinned NVlabs/Sana checkout; +- install upstream-heavy dependencies in an isolated environment; +- run upstream `inference_video_scripts/wm/inference_sana_wm.py` and + `flashdreams-run sana-wm-bidirectional` on the same image, prompt, + camera, intrinsics, seed, and sampler settings; +- compare decoded MP4 frames and report mean / max `|Delta|`. + +Do not put the full checkpoint download, refiner execution, or MP4 +generation path in `ci_cpu`. Add a bounded `ci_gpu` gate only if CI +pre-provisions the required artefacts and the test skips cleanly when +they are absent. diff --git a/integrations/sana/tests/test_camera.py b/integrations/sana/tests/test_camera.py new file mode 100644 index 000000000..43d0b7275 --- /dev/null +++ b/integrations/sana/tests/test_camera.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CPU-safe tests for SANA-WM camera and intrinsics helpers.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest +import torch + +from sana_wm.camera import ( + action_string_to_c2w, + fit_intrinsics_sequence, + load_intrinsics, + prepare_camera, + resize_center_crop_geometry, + snap_num_frames, + transform_intrinsics_for_crop, +) + +pytestmark = pytest.mark.ci_cpu + + +def test_action_string_rolls_out_identity_plus_motion() -> None: + """Expand ``w-3`` to an identity frame plus three motion frames.""" + c2w = action_string_to_c2w("w-3", smooth=False) + + assert c2w.shape == (4, 4, 4) + np.testing.assert_allclose(c2w[0], np.eye(4), atol=1e-6) + assert np.all(np.diff(c2w[:, 2, 3]) > 0) + + +def test_action_string_rejects_unknown_keys() -> None: + """Reject invalid action DSL tokens.""" + with pytest.raises(ValueError, match="unknown keys"): + action_string_to_c2w("q-3") + + +def test_snap_num_frames_matches_ltx2_stride() -> None: + """Snap requested frames to nearest ``8k + 1`` value.""" + assert snap_num_frames(321) == 321 + assert snap_num_frames(322) == 321 + assert snap_num_frames(325) == 329 + assert snap_num_frames(325, upper_bound=326) == 321 + + +def test_fit_intrinsics_sequence_interpolates_short_sequences() -> None: + """Interpolate per-frame intrinsics when the source sequence is shorter.""" + source = np.array([[10.0, 20.0, 1.0, 2.0], [30.0, 40.0, 3.0, 4.0]]) + + fitted = fit_intrinsics_sequence(source, 3) + + np.testing.assert_allclose( + fitted, + np.array( + [ + [10.0, 20.0, 1.0, 2.0], + [20.0, 30.0, 2.0, 3.0], + [30.0, 40.0, 3.0, 4.0], + ], + dtype=np.float32, + ), + ) + + +@pytest.mark.parametrize( + ("array", "expected"), + [ + ( + np.array([100.0, 110.0, 50.0, 55.0], dtype=np.float32), + np.array([[100.0, 110.0, 50.0, 55.0]] * 3, dtype=np.float32), + ), + ( + np.array( + [[100.0, 0.0, 50.0], [0.0, 110.0, 55.0], [0.0, 0.0, 1.0]], + dtype=np.float32, + ), + np.array([[100.0, 110.0, 50.0, 55.0]] * 3, dtype=np.float32), + ), + ], +) +def test_load_intrinsics_accepts_static_shapes( + tmp_path: Path, array: np.ndarray, expected: np.ndarray +) -> None: + """Load static vector and matrix intrinsics as per-frame vectors.""" + path = tmp_path / "intrinsics.npy" + np.save(path, array) + + loaded = load_intrinsics(path, num_frames=3) + + np.testing.assert_allclose(loaded, expected) + + +def test_resize_crop_geometry_and_intrinsics_transform() -> None: + """Map source intrinsics through SANA-WM resize and center-crop geometry.""" + src_size = (640, 480) + resized_size, crop_offset = resize_center_crop_geometry(src_size) + intrinsics = np.array([[400.0, 420.0, 320.0, 240.0]], dtype=np.float32) + + transformed = transform_intrinsics_for_crop( + intrinsics, src_size, resized_size, crop_offset + ) + + assert resized_size == (1280, 960) + assert crop_offset == (0, 128) + np.testing.assert_allclose( + transformed, + np.array([[800.0, 840.0, 640.0, 352.0]], dtype=np.float32), + ) + + +def test_prepare_camera_shapes_for_sana_wm_resolution() -> None: + """Build raymap and chunk-Plucker tensors at 704x1280 SANA-WM shape.""" + num_frames = 17 + poses = np.broadcast_to(np.eye(4, dtype=np.float32), (num_frames, 4, 4)).copy() + poses[:, 2, 3] = np.linspace(0.0, 1.0, num_frames) + intrinsics = np.broadcast_to( + np.array([900.0, 900.0, 640.0, 352.0], dtype=np.float32), + (num_frames, 4), + ).copy() + + camera = prepare_camera(poses, intrinsics) + + assert camera["raymap"].shape == (3, 20) + assert camera["chunk_plucker"].shape == (48, 3, 22, 40) + assert camera["raymap"].dtype == torch.float32 + assert camera["chunk_plucker"].dtype == torch.float32 diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py new file mode 100644 index 000000000..c361941bc --- /dev/null +++ b/integrations/sana/tests/test_smoke.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CPU-safe smoke tests for the SANA-WM runner shim.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - Python < 3.11 fallback + import tomli as tomllib + +from sana_wm.config import ( + PIPELINE_SANA_WM_BIDIRECTIONAL, + RUNNER_CONFIGS, + RUNNER_SANA_WM_BIDIRECTIONAL, +) +from sana_wm.constants import ( + SANA_WM_CONFIG_PATH, + SANA_WM_HF_REPO, + SANA_WM_MODEL_PATH, +) +from sana_wm.runner import SanaWMRunner, SanaWMRunnerConfig + +from flashdreams.infra.config import derive_config + +pytestmark = pytest.mark.ci_cpu + +ENTRY_POINT_GROUP = "flashdreams.runner_configs" + + +def test_runner_config_is_registered() -> None: + """Expose one SANA-WM runner keyed by its public slug.""" + assert RUNNER_CONFIGS == {"sana-wm-bidirectional": RUNNER_SANA_WM_BIDIRECTIONAL} + + +def test_runner_name_mirrors_pipeline_name() -> None: + """Keep ``flashdreams-run `` aligned with the wrapped config slug.""" + assert RUNNER_SANA_WM_BIDIRECTIONAL.runner_name == ( + PIPELINE_SANA_WM_BIDIRECTIONAL.name + ) + + +def test_runner_has_description() -> None: + """Provide non-empty CLI help text for the runner registry.""" + assert RUNNER_SANA_WM_BIDIRECTIONAL.description.strip() + + +def test_hf_defaults_point_at_bidirectional_release() -> None: + """Pin every default SANA-WM artefact to the bidirectional HF repo.""" + assert SANA_WM_HF_REPO == "Efficient-Large-Model/SANA-WM_bidirectional" + assert SANA_WM_MODEL_PATH == ( + "hf://Efficient-Large-Model/SANA-WM_bidirectional/" + "dit/sana_wm_1600m_720p.safetensors" + ) + assert SANA_WM_CONFIG_PATH == ( + "hf://Efficient-Large-Model/SANA-WM_bidirectional/config.yaml" + ) + + +def test_runner_setup_does_not_import_upstream_sana() -> None: + """Construct the runner without touching upstream Sana dependencies.""" + cfg = derive_config( + RUNNER_SANA_WM_BIDIRECTIONAL, + image_path=Path("missing.png"), + prompt="demo", + ) + + runner = cfg.setup() + + assert isinstance(runner, SanaWMRunner) + assert runner.config.image_path == Path("missing.png") + + +def test_runner_config_type() -> None: + """Keep the exported literal on the SANA-WM runner config subclass.""" + assert isinstance(RUNNER_SANA_WM_BIDIRECTIONAL, SanaWMRunnerConfig) + + +def test_pyproject_entry_point_matches_runner_literal() -> None: + """Keep the package entry point aligned with ``RUNNER_CONFIGS``.""" + pyproject = tomllib.loads( + Path("integrations/sana/pyproject.toml").read_text(encoding="utf-8") + ) + entry_points = pyproject["project"]["entry-points"][ENTRY_POINT_GROUP] + + assert entry_points == { + "sana-wm-bidirectional": "sana_wm.config:RUNNER_SANA_WM_BIDIRECTIONAL" + } diff --git a/pyproject.toml b/pyproject.toml index 250048b84..49b04c628 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ extraPaths = [ "integrations/fastvideo_causal_wan22", "integrations/hy_worldplay", "integrations/lingbot", + "integrations/sana", "integrations/self_forcing", "integrations/wan21", "integrations/wan22", @@ -74,6 +75,7 @@ extra-paths = [ "integrations/fastvideo_causal_wan22", "integrations/hy_worldplay", "integrations/lingbot", + "integrations/sana", "integrations/self_forcing", "integrations/wan21", "integrations/wan22", diff --git a/uv.lock b/uv.lock index 2b0d03215..55413b69e 100644 --- a/uv.lock +++ b/uv.lock @@ -27,6 +27,7 @@ members = [ "flashdreams-hy-worldplay", "flashdreams-lingbot", "flashdreams-omnidreams", + "flashdreams-sana-wm", "flashdreams-self-forcing", "flashdreams-wan21", "flashdreams-wan22", @@ -1567,6 +1568,26 @@ requires-dist = [ ] provides-extras = ["interactive-drive", "dev"] +[[package]] +name = "flashdreams-sana-wm" +version = "0.1.0" +source = { editable = "integrations/sana" } +dependencies = [ + { name = "flashdreams" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "flashdreams", editable = "flashdreams" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, +] +provides-extras = ["dev"] + [[package]] name = "flashdreams-self-forcing" version = "0.1.0" From f504eac0f041b6de962b62a3c7efbe1f7b02baba Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Wed, 15 Jul 2026 16:23:10 -0700 Subject: [PATCH 02/64] docs(sana): document upstream env install flow Document that the SANA-WM integration is expected to run inside the upstream Sana environment created by `environment_setup.sh sana`. The setup notes keep Sana's pinned PyTorch and model dependencies intact, then install FlashDreams and the local integration in editable mode without dependency resolution. Include the expected checkout layout and call out Transformer Engine as an optional dependency for quantized inference while BF16 remains the default validated path. Signed-off-by: Aidan Foster --- integrations/sana/README.md | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/integrations/sana/README.md b/integrations/sana/README.md index 88d2c2911..6ae421109 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -31,16 +31,38 @@ Clone Sana next to FlashDreams or pass its path explicitly: ```bash git clone https://github.com/NVlabs/Sana ../Sana -bash ../Sana/environment_setup.sh sana +cd ../Sana +bash environment_setup.sh sana +conda activate sana ``` The runner auto-detects `../Sana`. You can also set `SANA_ROOT` or pass `--upstream-sana-root /path/to/Sana`. +Install the FlashDreams packages into the same environment without +letting pip resolve or upgrade dependencies. The upstream Sana environment +owns the model runtime stack and pins packages such as `transformers` and +`huggingface-hub`. + +```bash +cd ../flashdreams +python -m pip install --no-deps -e flashdreams -e integrations/sana +``` + +If FlashDreams was installed without `--no-deps` and pip upgraded Sana's +pinned packages, repair the env before running generation: + +```bash +python -m pip install --no-deps \ + "huggingface-hub==0.36.0" \ + "transformers==4.57.3" +python -m pip install --no-deps -e flashdreams -e integrations/sana +``` + ## Run ```bash -uv run flashdreams-run sana-wm-bidirectional \ +flashdreams-run sana-wm-bidirectional \ --image-path ../Sana/asset/sana_wm/demo_0.png \ --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ --action "w-100,dw-60,w-100,aw-60" \ @@ -51,7 +73,7 @@ uv run flashdreams-run sana-wm-bidirectional \ To use an explicit trajectory: ```bash -uv run flashdreams-run sana-wm-bidirectional \ +flashdreams-run sana-wm-bidirectional \ --image-path ../Sana/asset/sana_wm/demo_0.png \ --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ From 41ee2bb57b0c892c4a9fffb47f152451b0b2fe8b Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Wed, 15 Jul 2026 16:26:49 -0700 Subject: [PATCH 03/64] docs(sana): clarify boolean CLI values Update the SANA-WM README examples to pass explicit values for boolean Tyro options such as `--no-refiner True`. This matches the generated CLI contract and avoids the confusing "Missing value" error when users treat those fields as flag-style booleans. Signed-off-by: Aidan Foster --- integrations/sana/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/integrations/sana/README.md b/integrations/sana/README.md index 6ae421109..d22d06231 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -82,9 +82,11 @@ flashdreams-run sana-wm-bidirectional \ --output-dir outputs/sana_wm ``` -`--no-refiner` skips the LTX-2 refiner and decodes Stage-1 latents -directly, matching upstream's fast debugging path. Full generation still -downloads the public Hugging Face artefacts on first use. +Set `--no-refiner True` to skip the LTX-2 refiner and decode Stage-1 +latents directly, matching upstream's fast debugging path. FlashDreams' +current CLI expects explicit boolean values, so pass `True` or `False` +for boolean fields. Full generation still downloads the public Hugging +Face artefacts on first use. ## Tests From 7e6af884d939b399650e479fa70e5787c6920915 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Wed, 15 Jul 2026 17:07:19 -0700 Subject: [PATCH 04/64] docs(sana): record validated BF16 smoke tests Record the BF16 commands used to validate both the Stage-1-only and refined SANA-WM paths, including the expected output locations under `outputs/sana_wm*/sana_wm_generated.mp4`. Note the upstream warnings that appeared during successful runs so they are not mistaken for failures: the null-embed load warning, missing `pos_embed`, and the optional Apex RMSNorm fallback. These commands establish BF16 as the known-good baseline for the integration. Signed-off-by: Aidan Foster --- integrations/sana/README.md | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/integrations/sana/README.md b/integrations/sana/README.md index d22d06231..d18cb7a2a 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -88,6 +88,51 @@ current CLI expects explicit boolean values, so pass `True` or `False` for boolean fields. Full generation still downloads the public Hugging Face artefacts on first use. +## Validated BF16 smoke tests + +The following commands were validated on an NVIDIA RTX PRO 6000 Blackwell +with upstream Sana installed in the `sana` conda environment. They use +the default BF16 path and do not require Transformer Engine. + +Stage-1 plus VAE decode: + +```bash +flashdreams-run sana-wm-bidirectional \ + --upstream-sana-root ../Sana \ + --image-path ../Sana/asset/sana_wm/demo_0.png \ + --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ + --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ + --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --num-frames 161 \ + --output-dir outputs/sana_wm \ + --no-refiner True +``` + +Expected output: + +```text +outputs/sana_wm/sana_wm_generated.mp4 +``` + +Stage-1 plus LTX-2 refiner: + +```bash +flashdreams-run sana-wm-bidirectional \ + --upstream-sana-root ../Sana \ + --image-path ../Sana/asset/sana_wm/demo_0.png \ + --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ + --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ + --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --num-frames 161 \ + --output-dir outputs/sana_wm_refiner +``` + +Expected output: + +```text +outputs/sana_wm_refiner/sana_wm_generated.mp4 +``` + ## Tests CPU-safe tests cover import, runner config, action parsing, intrinsics From de39905ac8aee4d0093d819267f482ea322a425d Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 16 Jul 2026 08:21:19 -0700 Subject: [PATCH 05/64] feat(sana): add FP8 and FP4 precision options Add first-class `stage1_precision` and `refiner_precision` runner options for BF16, FP8, and FP4. The runner maps those options onto upstream SANA-WM Transformer Engine environment selectors inside a scoped environment so external NVFP4 variables cannot accidentally change the default BF16 path. Validate quantized precision requests before loading checkpoints. FP8 now requires a CUDA device, Hopper-or-newer GPU capability, CUDA 12.9 or newer as reported by PyTorch, and Transformer Engine's Float8BlockScaling recipe. FP4 requires a CUDA device, Blackwell GPU capability, and Transformer Engine's NVFP4 recipe. Extend the SANA README with precision requirements and an FP4 example command, and add CPU-safe tests for the env mapping, cleanup behavior, and early capability errors. Signed-off-by: Aidan Foster --- integrations/sana/README.md | 33 +++ integrations/sana/sana_wm/runner.py | 394 +++++++++++++++++++------- integrations/sana/tests/test_smoke.py | 117 +++++++- 3 files changed, 437 insertions(+), 107 deletions(-) diff --git a/integrations/sana/README.md b/integrations/sana/README.md index d18cb7a2a..705d721a3 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -133,6 +133,39 @@ Expected output: outputs/sana_wm_refiner/sana_wm_generated.mp4 ``` +## FP8 and FP4 precision + +The runner defaults to BF16. Quantized inference is opt-in: + +| option | hardware | dependency | +| --- | --- | --- | +| `--stage1-precision fp8` / `--refiner-precision fp8` | Hopper or newer (`sm_90+`) with CUDA 12.9+ | Transformer Engine | +| `--stage1-precision fp4` / `--refiner-precision fp4` | Blackwell (`sm_100+`) | Transformer Engine with NVFP4 recipes | + +The runner validates these requirements before loading checkpoints. If the +GPU or Transformer Engine install cannot support the requested precision, +it fails early with a targeted error instead of falling through to a deep +upstream import or model-build failure. + +Example Blackwell FP4 run: + +```bash +flashdreams-run sana-wm-bidirectional \ + --upstream-sana-root ../Sana \ + --image-path ../Sana/asset/sana_wm/demo_0.png \ + --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ + --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ + --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --num-frames 161 \ + --output-dir outputs/sana_wm_fp4 \ + --stage1-precision fp4 \ + --refiner-precision fp4 +``` + +Use `fp8` for Hopper-class GPUs when the PyTorch environment reports CUDA +12.9 or newer. Keep both precision flags at `bf16` if Transformer Engine is +unavailable or if maximum compatibility is preferred. + ## Tests CPU-safe tests cover import, runner config, action parsing, intrinsics diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index 1a999673c..56c5981b2 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -20,6 +20,8 @@ import importlib import os import sys +from collections.abc import Iterator, Mapping +from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path from types import ModuleType @@ -53,6 +55,9 @@ ] """Sampling algorithms exposed by upstream SANA-WM inference.""" +Precision = Literal["bf16", "fp8", "fp4"] +"""SANA-WM Stage-1/refiner precision modes.""" + @dataclass(kw_only=True) class SanaWMRunnerConfig(RunnerConfig): @@ -147,9 +152,20 @@ class SanaWMRunnerConfig(RunnerConfig): model_path: str = SANA_WM_MODEL_PATH """Stage-1 checkpoint path or ``hf://`` URI.""" + stage1_precision: Precision = "bf16" + """Stage-1 DiT compute precision. ``"bf16"`` is the default; ``"fp8"`` + requires Hopper or newer plus Transformer Engine; ``"fp4"`` requires + Blackwell plus Transformer Engine.""" + no_refiner: bool = False """Skip the LTX-2 refiner and decode Stage-1 latents directly.""" + refiner_precision: Precision = "bf16" + """LTX-2 refiner compute precision. ``"bf16"`` is the default; + ``"fp8"`` requires Hopper or newer plus Transformer Engine; ``"fp4"`` + requires Blackwell plus Transformer Engine. Ignored when + ``no_refiner`` is ``True``.""" + refiner_root: str = SANA_WM_REFINER_ROOT """LTX-2 refiner root path or ``hf://`` URI.""" @@ -254,128 +270,293 @@ def run(self) -> None: if cfg.image_path is None: raise ValueError("SanaWMRunner requires --image-path.") - upstream = _import_upstream_sana(cfg.upstream_sana_root) device = self._resolve_device() - prompt = self._resolve_prompt() - - image = upstream.Image.open(cfg.image_path).convert("RGB") - c2w_full = self._resolve_trajectory(upstream) - num_frames = min(cfg.num_frames, c2w_full.shape[0]) - snapped = upstream._snap_num_frames( - num_frames, - stride=SANA_WM_VAE_TEMPORAL_COMPRESSION, - upper_bound=c2w_full.shape[0], - ) - if snapped != cfg.num_frames and self.is_rank_zero: - logger.warning( - "SANA-WM requires num_frames = 8k+1; requested {} snapped to {} " - "(trajectory has {} frames).", - cfg.num_frames, - snapped, - c2w_full.shape[0], - ) - num_frames = snapped - c2w = c2w_full[:num_frames] - - cropped, src_size, resized_size, crop_offset = upstream.resize_and_center_crop( - image, - target_h=DEFAULT_VIDEO_HEIGHT, - target_w=DEFAULT_VIDEO_WIDTH, - ) - if cfg.intrinsics_path is None: - intrinsics_src = np.broadcast_to( - upstream.estimate_intrinsics_with_pi3x( - image, device, upstream.get_root_logger() - ), - (num_frames, 4), - ).copy() - else: - intrinsics_src = upstream.load_intrinsics(cfg.intrinsics_path, num_frames) - intrinsics_vec4 = upstream.transform_intrinsics_for_crop( - intrinsics_src, src_size, resized_size, crop_offset + _validate_precision_request( + device=device, + stage1_precision=cfg.stage1_precision, + refiner_precision=cfg.refiner_precision, + refiner_enabled=not cfg.no_refiner, ) - inference_config = upstream.pyrallis.parse( - config_class=upstream.InferenceConfig, - config_path=upstream.resolve_hf_path(cfg.config_path), - args=[], + precision_env = _precision_env_updates( + stage1_precision=cfg.stage1_precision, + refiner_precision=cfg.refiner_precision, + refiner_enabled=not cfg.no_refiner, ) - refiner = ( - None - if cfg.no_refiner - else upstream.RefinerSettings( - root=cfg.refiner_root, - gemma_root=cfg.refiner_gemma_root, - sink_size=cfg.sink_size, - seed=cfg.refiner_seed, - block_size=cfg.refiner_block_size, - kv_max_frames=cfg.refiner_kv_max_frames, + with _temporary_environment(precision_env): + upstream = _import_upstream_sana(cfg.upstream_sana_root) + prompt = self._resolve_prompt() + + image = upstream.Image.open(cfg.image_path).convert("RGB") + c2w_full = self._resolve_trajectory(upstream) + num_frames = min(cfg.num_frames, c2w_full.shape[0]) + snapped = upstream._snap_num_frames( + num_frames, + stride=SANA_WM_VAE_TEMPORAL_COMPRESSION, + upper_bound=c2w_full.shape[0], + ) + if snapped != cfg.num_frames and self.is_rank_zero: + logger.warning( + "SANA-WM requires num_frames = 8k+1; requested {} snapped to {} " + "(trajectory has {} frames).", + cfg.num_frames, + snapped, + c2w_full.shape[0], + ) + num_frames = snapped + c2w = c2w_full[:num_frames] + + cropped, src_size, resized_size, crop_offset = ( + upstream.resize_and_center_crop( + image, + target_h=DEFAULT_VIDEO_HEIGHT, + target_w=DEFAULT_VIDEO_WIDTH, + ) + ) + if cfg.intrinsics_path is None: + intrinsics_src = np.broadcast_to( + upstream.estimate_intrinsics_with_pi3x( + image, device, upstream.get_root_logger() + ), + (num_frames, 4), + ).copy() + else: + intrinsics_src = upstream.load_intrinsics( + cfg.intrinsics_path, num_frames + ) + intrinsics_vec4 = upstream.transform_intrinsics_for_crop( + intrinsics_src, src_size, resized_size, crop_offset ) - ) - pipeline = upstream.SanaWMPipeline( - config=inference_config, - model_path=upstream.resolve_hf_path(cfg.model_path), - device=device, - refiner=refiner, - offload_vae=cfg.offload_vae, - offload_refiner=cfg.offload_refiner, - offload_text_encoder=cfg.offload_text_encoder, - logger=upstream.get_root_logger(), - ) - sampling_algo = cfg.sampling_algo - if sampling_algo == "auto": - sampling_algo = ( - inference_config.scheduler.vis_sampler - if inference_config.scheduler.vis_sampler - in {"chunk_flow_euler", "self_forcing"} - else "flow_euler_ltx" + inference_config = upstream.pyrallis.parse( + config_class=upstream.InferenceConfig, + config_path=upstream.resolve_hf_path(cfg.config_path), + args=[], + ) + refiner = ( + None + if cfg.no_refiner + else upstream.RefinerSettings( + root=cfg.refiner_root, + gemma_root=cfg.refiner_gemma_root, + sink_size=cfg.sink_size, + seed=cfg.refiner_seed, + block_size=cfg.refiner_block_size, + kv_max_frames=cfg.refiner_kv_max_frames, + ) + ) + pipeline = upstream.SanaWMPipeline( + config=inference_config, + model_path=upstream.resolve_hf_path(cfg.model_path), + device=device, + refiner=refiner, + offload_vae=cfg.offload_vae, + offload_refiner=cfg.offload_refiner, + offload_text_encoder=cfg.offload_text_encoder, + logger=upstream.get_root_logger(), ) - params = upstream.GenerationParams( - num_frames=num_frames, - fps=cfg.fps, - step=cfg.step, - cfg_scale=cfg.cfg_scale, - flow_shift=cfg.flow_shift, - seed=cfg.seed, - negative_prompt=cfg.negative_prompt, - sampling_algo=sampling_algo, - chunk_interval_k=cfg.chunk_interval_k, - num_cached_blocks=cfg.num_cached_blocks, - sink_token=cfg.sink_token, - num_frame_per_block=cfg.num_frame_per_block, - denoising_step_list=self._denoising_steps(), - save_stage1=cfg.save_stage1, - ) - result = pipeline.generate(cropped, prompt, c2w, intrinsics_vec4, params) - video_hwc = result["video"] - if not cfg.no_action_overlay: - video_hwc = upstream.apply_overlay(video_hwc, result["c2w"]) - - if not self.is_rank_zero: - return - upstream.write_video( - cfg.output_dir, - cfg.name, - video_hwc, - params.fps, - upstream.get_root_logger(), - ) + sampling_algo = cfg.sampling_algo + if sampling_algo == "auto": + sampling_algo = ( + inference_config.scheduler.vis_sampler + if inference_config.scheduler.vis_sampler + in {"chunk_flow_euler", "self_forcing"} + else "flow_euler_ltx" + ) + params = upstream.GenerationParams( + num_frames=num_frames, + fps=cfg.fps, + step=cfg.step, + cfg_scale=cfg.cfg_scale, + flow_shift=cfg.flow_shift, + seed=cfg.seed, + negative_prompt=cfg.negative_prompt, + sampling_algo=sampling_algo, + chunk_interval_k=cfg.chunk_interval_k, + num_cached_blocks=cfg.num_cached_blocks, + sink_token=cfg.sink_token, + num_frame_per_block=cfg.num_frame_per_block, + denoising_step_list=self._denoising_steps(), + save_stage1=cfg.save_stage1, + ) - stage1_video = result.get("stage1_video") - if stage1_video is not None: - stage1_hwc = stage1_video + result = pipeline.generate(cropped, prompt, c2w, intrinsics_vec4, params) + video_hwc = result["video"] if not cfg.no_action_overlay: - stage1_hwc = upstream.apply_overlay(stage1_hwc, result["stage1_c2w"]) + video_hwc = upstream.apply_overlay(video_hwc, result["c2w"]) + + if not self.is_rank_zero: + return upstream.write_video( cfg.output_dir, - f"{cfg.name}_stage1", - stage1_hwc, + cfg.name, + video_hwc, params.fps, upstream.get_root_logger(), ) + stage1_video = result.get("stage1_video") + if stage1_video is not None: + stage1_hwc = stage1_video + if not cfg.no_action_overlay: + stage1_hwc = upstream.apply_overlay( + stage1_hwc, result["stage1_c2w"] + ) + upstream.write_video( + cfg.output_dir, + f"{cfg.name}_stage1", + stage1_hwc, + params.fps, + upstream.get_root_logger(), + ) + + +def _precision_env_updates( + *, + stage1_precision: Precision, + refiner_precision: Precision, + refiner_enabled: bool, +) -> dict[str, str | None]: + """Return upstream SANA-WM precision env overrides.""" + updates: dict[str, str | None] = {"SANA_WM_STAGE1_NVFP4": None} + if stage1_precision != "bf16": + updates.update( + { + "SANA_WM_STAGE1_NVFP4": "self_attn+cross+ffn", + "SANA_WM_STAGE1_NVFP4_SCOPE": "block", + "SANA_WM_STAGE1_LINEARIZE_FFN": None, + "SANA_WM_STAGE1_QUANT": _quant_env_value(stage1_precision), + } + ) + + updates["SANA_WM_REFINER_NVFP4"] = None + if refiner_enabled and refiner_precision != "bf16": + updates.update( + { + "SANA_WM_REFINER_NVFP4": "1", + "SANA_WM_REFINER_QUANT": _quant_env_value(refiner_precision), + } + ) + return updates + + +def _quant_env_value(precision: Precision) -> str: + """Return upstream Transformer Engine recipe selector for a precision.""" + if precision == "fp8": + return "fp8block" + if precision == "fp4": + return "nvfp4" + raise ValueError(f"{precision!r} does not use Transformer Engine quantization.") + + +@contextmanager +def _temporary_environment( + updates: Mapping[str, str | None], +) -> Iterator[None]: + """Temporarily set or unset environment variables.""" + previous = {key: os.environ.get(key) for key in updates} + try: + for key, value in updates.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + yield + finally: + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def _validate_precision_request( + *, + device: torch.device, + stage1_precision: Precision, + refiner_precision: Precision, + refiner_enabled: bool, +) -> None: + """Fail early when requested quantized precision cannot run.""" + active_precisions = [stage1_precision] + if refiner_enabled: + active_precisions.append(refiner_precision) + quantized = [precision for precision in active_precisions if precision != "bf16"] + if not quantized: + return + + if device.type != "cuda" or not torch.cuda.is_available(): + raise ValueError( + "SANA-WM fp8/fp4 precision requires a CUDA device; " + f"resolved device is {device}." + ) + + capability = torch.cuda.get_device_capability(device) + major, minor = capability + sm_name = f"sm_{major}{minor}" + if "fp8" in quantized and major < 9: + raise ValueError( + "SANA-WM fp8 precision requires a Hopper or newer GPU " + f"(sm_90+); detected {sm_name}." + ) + if "fp8" in quantized and not _torch_cuda_version_at_least(12, 9): + cuda_version = torch.version.cuda or "unknown" + raise ValueError( + "SANA-WM fp8 precision uses upstream Sana's Transformer Engine " + "Float8BlockScaling path, which requires CUDA 12.9 or newer. " + f"This PyTorch environment reports CUDA {cuda_version}." + ) + if "fp4" in quantized and major < 10: + raise ValueError( + "SANA-WM fp4/NVFP4 precision requires a Blackwell GPU " + f"(sm_100+); detected {sm_name}. Use bf16 or fp8 on this GPU." + ) + + _validate_transformer_engine(quantized) + + +def _torch_cuda_version_at_least(major: int, minor: int) -> bool: + """Return whether ``torch.version.cuda`` is at least ``major.minor``.""" + version = torch.version.cuda + if version is None: + return False + parts = version.split(".") + try: + current_major = int(parts[0]) + current_minor = int(parts[1]) if len(parts) > 1 else 0 + except (ValueError, IndexError): + return False + return (current_major, current_minor) >= (major, minor) + + +def _validate_transformer_engine(precisions: list[Precision]) -> None: + """Ensure Transformer Engine has the recipes required by precision modes.""" + required_recipes: set[str] = set() + if "fp8" in precisions: + required_recipes.add("Float8BlockScaling") + if "fp4" in precisions: + required_recipes.add("NVFP4BlockScaling") + try: + import transformer_engine.common.recipe as te_recipe + import transformer_engine.pytorch # noqa: F401 + except Exception as exc: + raise RuntimeError( + "SANA-WM fp8/fp4 precision requires NVIDIA Transformer Engine. " + "Install it in the upstream Sana environment, for example with " + "`pip install --no-build-isolation 'transformer_engine[pytorch]'`, " + "or run with the default bf16 precision." + ) from exc + + missing = sorted( + recipe for recipe in required_recipes if not hasattr(te_recipe, recipe) + ) + if missing: + raise RuntimeError( + "Installed Transformer Engine does not provide the recipe(s) " + f"required by the requested SANA-WM precision: {', '.join(missing)}." + ) + def _candidate_upstream_roots(explicit_root: Path | None) -> list[Path]: """Return candidate NVlabs/Sana checkout roots in priority order.""" @@ -429,6 +610,7 @@ def _import_upstream_sana(explicit_root: Path | None) -> ModuleType: __all__ = [ + "Precision", "SamplingAlgo", "SanaWMRunner", "SanaWMRunnerConfig", diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index c361941bc..6c7c032ed 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -17,9 +17,11 @@ from __future__ import annotations +import os from pathlib import Path import pytest +import torch try: import tomllib @@ -36,7 +38,13 @@ SANA_WM_HF_REPO, SANA_WM_MODEL_PATH, ) -from sana_wm.runner import SanaWMRunner, SanaWMRunnerConfig +from sana_wm.runner import ( + SanaWMRunner, + SanaWMRunnerConfig, + _precision_env_updates, + _temporary_environment, + _validate_precision_request, +) from flashdreams.infra.config import derive_config @@ -93,6 +101,113 @@ def test_runner_config_type() -> None: assert isinstance(RUNNER_SANA_WM_BIDIRECTIONAL, SanaWMRunnerConfig) +def test_runner_defaults_to_bf16_precision() -> None: + """Keep the default path on upstream SANA-WM's BF16 config.""" + assert RUNNER_SANA_WM_BIDIRECTIONAL.stage1_precision == "bf16" + assert RUNNER_SANA_WM_BIDIRECTIONAL.refiner_precision == "bf16" + + +def test_bf16_precision_clears_upstream_quant_env( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Prevent external NVFP4 env vars from changing the default runner path.""" + monkeypatch.setenv("SANA_WM_STAGE1_NVFP4", "1") + monkeypatch.setenv("SANA_WM_REFINER_NVFP4", "1") + updates = _precision_env_updates( + stage1_precision="bf16", + refiner_precision="bf16", + refiner_enabled=True, + ) + + with _temporary_environment(updates): + assert "SANA_WM_STAGE1_NVFP4" not in os.environ + assert "SANA_WM_REFINER_NVFP4" not in os.environ + + assert os.environ["SANA_WM_STAGE1_NVFP4"] == "1" + assert os.environ["SANA_WM_REFINER_NVFP4"] == "1" + + +def test_fp8_precision_sets_upstream_quant_env() -> None: + """Map FlashDreams fp8 fields to upstream SANA-WM env selectors.""" + updates = _precision_env_updates( + stage1_precision="fp8", + refiner_precision="fp8", + refiner_enabled=True, + ) + + assert updates["SANA_WM_STAGE1_NVFP4"] == "self_attn+cross+ffn" + assert updates["SANA_WM_STAGE1_QUANT"] == "fp8block" + assert updates["SANA_WM_STAGE1_LINEARIZE_FFN"] is None + assert updates["SANA_WM_REFINER_NVFP4"] == "1" + assert updates["SANA_WM_REFINER_QUANT"] == "fp8block" + + +def test_fp4_precision_sets_upstream_quant_env() -> None: + """Map FlashDreams fp4 fields to upstream SANA-WM NVFP4 env selectors.""" + updates = _precision_env_updates( + stage1_precision="fp4", + refiner_precision="fp4", + refiner_enabled=True, + ) + + assert updates["SANA_WM_STAGE1_QUANT"] == "nvfp4" + assert updates["SANA_WM_REFINER_QUANT"] == "nvfp4" + + +def test_quantized_precision_requires_cuda() -> None: + """Reject fp8/fp4 before importing upstream Sana on CPU-only devices.""" + with pytest.raises(ValueError, match="requires a CUDA device"): + _validate_precision_request( + device=torch.device("cpu"), + stage1_precision="fp8", + refiner_precision="bf16", + refiner_enabled=True, + ) + + +def test_fp8_precision_requires_hopper(monkeypatch: pytest.MonkeyPatch) -> None: + """Reject FP8 on pre-Hopper GPUs before checking Transformer Engine.""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (8, 9)) + + with pytest.raises(ValueError, match="requires a Hopper or newer GPU"): + _validate_precision_request( + device=torch.device("cuda:0"), + stage1_precision="fp8", + refiner_precision="bf16", + refiner_enabled=True, + ) + + +def test_fp8_precision_requires_cuda_129(monkeypatch: pytest.MonkeyPatch) -> None: + """Reject upstream FP8 block scaling when torch reports CUDA 12.8.""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) + monkeypatch.setattr(torch.version, "cuda", "12.8") + + with pytest.raises(ValueError, match="requires CUDA 12.9 or newer"): + _validate_precision_request( + device=torch.device("cuda:0"), + stage1_precision="fp8", + refiner_precision="bf16", + refiner_enabled=True, + ) + + +def test_fp4_precision_requires_blackwell(monkeypatch: pytest.MonkeyPatch) -> None: + """Reject NVFP4 on Hopper-class GPUs before checking Transformer Engine.""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (9, 0)) + + with pytest.raises(ValueError, match="requires a Blackwell GPU"): + _validate_precision_request( + device=torch.device("cuda:0"), + stage1_precision="fp4", + refiner_precision="bf16", + refiner_enabled=True, + ) + + def test_pyproject_entry_point_matches_runner_literal() -> None: """Keep the package entry point aligned with ``RUNNER_CONFIGS``.""" pyproject = tomllib.loads( From a710bd0043ee1035bbef44ec4af77d5f5e685518 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 16 Jul 2026 16:47:40 -0700 Subject: [PATCH 06/64] Add native SANA-WM Stage-1 FlashDreams integration Route the public SANA-WM runner through FlashDreams pipeline, diffusion-model, transformer, and scheduler boundaries instead of the upstream inference harness. Keep the upstream harness only as an explicit reference runner with separate documentation. Replace the vendored upstream diffusion package with a FlashDreams-owned Stage-1 DiT module, direct checkpoint loading, local YAML config parsing, direct diffusers LTX2 VAE encode/decode, and native runner wiring for Stage-1 BF16 execution. Add Transformer Engine-free native FP8 and FP4 Stage-1 Linear replacements using PyTorch scaled-mm and Triton NVFP4 activation quantization, with precision/backend validation before checkpoint loading. Keep the native scope honest: require no-refiner for native runs, keep action overlays and upstream TE quantization on the separate upstream-reference harness, and document the remaining refiner and GDN parity gaps separately from the native runner docs. Expand CPU-safe SANA tests to pin native/reference runner separation, config loading without upstream types, native Stage-1 checkpoint schema, Stage-1 forward shape, runtime release, VAE tiling safeguards, and native quant replacement behavior. Signed-off-by: Aidan Foster --- integrations/sana/README.md | 176 ++- integrations/sana/docs/upstream_reference.md | 103 ++ integrations/sana/pyproject.toml | 15 +- integrations/sana/sana_wm/__init__.py | 2 +- integrations/sana/sana_wm/_tools.py | 86 ++ integrations/sana/sana_wm/config.py | 73 +- integrations/sana/sana_wm/native_diffusion.py | 228 ++++ integrations/sana/sana_wm/native_quant.py | 564 +++++++++ .../sana/sana_wm/native_transformer.py | 1085 +++++++++++++++++ integrations/sana/sana_wm/runner.py | 403 +++++- integrations/sana/sana_wm/stage1_model.py | 514 ++++++++ .../{placeholder.py => upstream_reference.py} | 41 +- .../sana/tests/parity_check/README.md | 12 +- .../sana/tests/test_native_quant_cuda.py | 79 ++ integrations/sana/tests/test_smoke.py | 581 ++++++++- 15 files changed, 3785 insertions(+), 177 deletions(-) create mode 100644 integrations/sana/docs/upstream_reference.md create mode 100644 integrations/sana/sana_wm/_tools.py create mode 100644 integrations/sana/sana_wm/native_diffusion.py create mode 100644 integrations/sana/sana_wm/native_quant.py create mode 100644 integrations/sana/sana_wm/native_transformer.py create mode 100644 integrations/sana/sana_wm/stage1_model.py rename integrations/sana/sana_wm/{placeholder.py => upstream_reference.py} (55%) create mode 100644 integrations/sana/tests/test_native_quant_cuda.py diff --git a/integrations/sana/README.md b/integrations/sana/README.md index 705d721a3..aa0500e82 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -5,175 +5,135 @@ SPDX-License-Identifier: Apache-2.0 # `sana_wm` -FlashDreams runner shim for +FlashDreams SANA-WM integration for [SANA-WM](https://huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional), -the 2.6B bidirectional camera-controlled world model from the -[NVlabs/Sana](https://github.com/NVlabs/Sana) repository. +the 2.6B bidirectional camera-controlled world model released from NVlabs/Sana. -This first slice follows the dependency-isolated integration pattern: -FlashDreams owns the CLI registration, config surface, and CPU-tested -camera/action helpers, while Stage-1 DiT, LTX2 VAE/refiner, Pi3X, and -custom attention kernels are imported from an installed or local upstream -Sana checkout at runtime. +The `sana-wm-bidirectional` runner is the native FlashDreams path. It uses +FlashDreams config, runner, pipeline, diffusion-model, scheduler, transformer, +camera-conditioning, VAE decode, and output-writing boundaries. The Stage-1 +DiT module is implemented in this package and loads the public SANA-WM +checkpoint directly; it does not import or install a vendored upstream +`diffusion` package. + +Current native scope: + +| component | status | +| --- | --- | +| Stage-1 BF16 | Native FlashDreams path, GPU smoke-tested. | +| Stage-1 FP8 | Native PyTorch `_scaled_mm` backend, GPU smoke-tested. | +| Stage-1 FP4 | Native Triton quantization plus PyTorch `_scaled_mm`, GPU smoke-tested. | +| VAE decode | Native direct `diffusers` LTX2 VAE use with low-memory tiling. | +| LTX-2 refiner | Not native yet; the native runner requires `--no-refiner True`. | +| GDN attention parity | Not complete; smoke-tested, not quality-parity validated. | + +The separate `sana-wm-upstream-reference` runner is documented only in +`docs/upstream_reference.md`. Keep reference-harness details out of this +native integration README. ## Runner | slug | description | | --- | --- | -| `sana-wm-bidirectional` | SANA-WM bidirectional I2V world model using upstream NVlabs/Sana Stage-1 + LTX-2 refiner. | +| `sana-wm-bidirectional` | Native FlashDreams SANA-WM Stage-1 runner. | +| `sana-wm-native-bidirectional` | Alias for the native FlashDreams runner. | -The FlashDreams package is named `sana_wm` rather than `sana` so it does -not shadow upstream's own top-level `sana` package. +The FlashDreams package is named `sana_wm` rather than `sana` so it does not +shadow the upstream project name. ## Setup -Clone Sana next to FlashDreams or pass its path explicitly: +Install FlashDreams and the SANA integration into an environment with the +project's GPU runtime dependencies: ```bash -git clone https://github.com/NVlabs/Sana ../Sana -cd ../Sana -bash environment_setup.sh sana -conda activate sana +uv sync --package flashdreams-sana-wm --extra dev ``` -The runner auto-detects `../Sana`. You can also set `SANA_ROOT` or pass -`--upstream-sana-root /path/to/Sana`. - -Install the FlashDreams packages into the same environment without -letting pip resolve or upgrade dependencies. The upstream Sana environment -owns the model runtime stack and pins packages such as `transformers` and -`huggingface-hub`. +For an existing environment, install the local packages directly: ```bash -cd ../flashdreams -python -m pip install --no-deps -e flashdreams -e integrations/sana +python -m pip install -e flashdreams -e integrations/sana ``` -If FlashDreams was installed without `--no-deps` and pip upgraded Sana's -pinned packages, repair the env before running generation: - -```bash -python -m pip install --no-deps \ - "huggingface-hub==0.36.0" \ - "transformers==4.57.3" -python -m pip install --no-deps -e flashdreams -e integrations/sana -``` +The native runner does not require a local NVlabs/Sana checkout. The checkout +is only useful as a source of demo assets such as +`asset/sana_wm/demo_0.png`, `demo_0.txt`, `demo_0_pose.npy`, and +`demo_0_intrinsics.npy`. ## Run -```bash -flashdreams-run sana-wm-bidirectional \ - --image-path ../Sana/asset/sana_wm/demo_0.png \ - --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ - --action "w-100,dw-60,w-100,aw-60" \ - --num-frames 321 \ - --output-dir outputs/sana_wm -``` - -To use an explicit trajectory: +The native runner currently requires explicit intrinsics and no refiner: ```bash +PYTORCH_ALLOC_CONF=expandable_segments:True \ flashdreams-run sana-wm-bidirectional \ - --image-path ../Sana/asset/sana_wm/demo_0.png \ - --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ - --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ - --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ - --num-frames 321 \ - --output-dir outputs/sana_wm -``` - -Set `--no-refiner True` to skip the LTX-2 refiner and decode Stage-1 -latents directly, matching upstream's fast debugging path. FlashDreams' -current CLI expects explicit boolean values, so pass `True` or `False` -for boolean fields. Full generation still downloads the public Hugging -Face artefacts on first use. - -## Validated BF16 smoke tests - -The following commands were validated on an NVIDIA RTX PRO 6000 Blackwell -with upstream Sana installed in the `sana` conda environment. They use -the default BF16 path and do not require Transformer Engine. - -Stage-1 plus VAE decode: - -```bash -flashdreams-run sana-wm-bidirectional \ - --upstream-sana-root ../Sana \ --image-path ../Sana/asset/sana_wm/demo_0.png \ --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ --num-frames 161 \ - --output-dir outputs/sana_wm \ + --output-dir outputs/sana_wm_native_bf16 \ --no-refiner True ``` Expected output: ```text -outputs/sana_wm/sana_wm_generated.mp4 +outputs/sana_wm_native_bf16/sana_wm_generated.mp4 ``` -Stage-1 plus LTX-2 refiner: +## FP8 and FP4 + +The runner defaults to BF16. Quantized Stage-1 inference is opt-in and uses +native FlashDreams/PyTorch backends by default, not Transformer Engine. + +| option | hardware | backend | +| --- | --- | --- | +| `--stage1-precision fp8` | Hopper or newer (`sm_90+`) | native E4M3 `_scaled_mm` | +| `--stage1-precision fp4` | Blackwell (`sm_100+`) | native Triton NVFP4 plus `_scaled_mm` | +| `--quant-backend torch-fp8` | Hopper or newer (`sm_90+`) | force the native FP8 backend | +| `--quant-backend torch-fp4` | Blackwell (`sm_100+`) | force the native FP4 backend | + +FP8 smoke: ```bash +PYTORCH_ALLOC_CONF=expandable_segments:True \ flashdreams-run sana-wm-bidirectional \ - --upstream-sana-root ../Sana \ --image-path ../Sana/asset/sana_wm/demo_0.png \ --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ --num-frames 161 \ - --output-dir outputs/sana_wm_refiner -``` - -Expected output: - -```text -outputs/sana_wm_refiner/sana_wm_generated.mp4 + --output-dir outputs/sana_wm_native_fp8 \ + --no-refiner True \ + --stage1-precision fp8 ``` -## FP8 and FP4 precision - -The runner defaults to BF16. Quantized inference is opt-in: - -| option | hardware | dependency | -| --- | --- | --- | -| `--stage1-precision fp8` / `--refiner-precision fp8` | Hopper or newer (`sm_90+`) with CUDA 12.9+ | Transformer Engine | -| `--stage1-precision fp4` / `--refiner-precision fp4` | Blackwell (`sm_100+`) | Transformer Engine with NVFP4 recipes | - -The runner validates these requirements before loading checkpoints. If the -GPU or Transformer Engine install cannot support the requested precision, -it fails early with a targeted error instead of falling through to a deep -upstream import or model-build failure. - -Example Blackwell FP4 run: +FP4 smoke: ```bash +PYTORCH_ALLOC_CONF=expandable_segments:True \ flashdreams-run sana-wm-bidirectional \ - --upstream-sana-root ../Sana \ --image-path ../Sana/asset/sana_wm/demo_0.png \ --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ --num-frames 161 \ - --output-dir outputs/sana_wm_fp4 \ - --stage1-precision fp4 \ - --refiner-precision fp4 + --output-dir outputs/sana_wm_native_fp4 \ + --no-refiner True \ + --stage1-precision fp4 ``` -Use `fp8` for Hopper-class GPUs when the PyTorch environment reports CUDA -12.9 or newer. Keep both precision flags at `bf16` if Transformer Engine is -unavailable or if maximum compatibility is preferred. - ## Tests -CPU-safe tests cover import, runner config, action parsing, intrinsics -handling, frame snapping, and SANA-WM camera-conditioning tensor shapes: +CPU-safe tests cover import, config boundaries, action parsing, intrinsics, +camera conditioning, native Stage-1 checkpoint schema, native Stage-1 CPU +forward shape, VAE tiling, and native low-precision backend selection: ```bash -uv run --extra dev pytest integrations/sana/tests +uv run --extra dev pytest integrations/sana/tests/test_smoke.py ``` -Full upstream parity and generated-video checks are heavyweight GPU -workflows and should live under `tests/parity_check/`. +GPU generation and native-vs-reference quality parity checks are heavyweight +manual workflows and should stay out of `ci_cpu`. diff --git a/integrations/sana/docs/upstream_reference.md b/integrations/sana/docs/upstream_reference.md new file mode 100644 index 000000000..21a339fc0 --- /dev/null +++ b/integrations/sana/docs/upstream_reference.md @@ -0,0 +1,103 @@ + + +# SANA-WM Upstream Reference Harness + +This document covers the temporary upstream-reference runner. It exists only +for parity/debug work while the native FlashDreams integration is being +validated. Do not use this as user-facing documentation for the real SANA-WM +FlashDreams integration. + +## Runner + +| slug | description | +| --- | --- | +| `sana-wm-upstream-reference` | FlashDreams-packaged NVlabs/Sana reference harness. | + +This runner delegates generation to the SANA reference pipeline object. It is +expected to be removed once native generation is validated. + +## Setup + +The reference harness needs a local or importable NVlabs/Sana checkout: + +```bash +git clone https://github.com/NVlabs/Sana ../Sana +cd ../Sana +bash environment_setup.sh sana +conda activate sana +``` + +The runner auto-detects `../Sana`. You can also set `SANA_ROOT` or pass +`--upstream-sana-root /path/to/Sana`. + +## Run + +```bash +flashdreams-run sana-wm-upstream-reference \ + --upstream-sana-root ../Sana \ + --image-path ../Sana/asset/sana_wm/demo_0.png \ + --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ + --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ + --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --num-frames 321 \ + --output-dir outputs/sana_wm_reference +``` + +The upstream reference harness can estimate intrinsics with Pi3X when +`--intrinsics-path` is omitted. The native runner currently requires explicit +intrinsics instead. + +## Transformer Engine Reference Notes + +Upstream SANA does not use Transformer Engine here for positional embeddings. +Where the reference path uses Transformer Engine, it is as a quantized +`Linear`/GEMM backend: + +- Stage 1 replaces selected `torch.nn.Linear` layers with + `transformer_engine.pytorch.Linear`, then wraps transformer block or linear + forwards in `te.fp8_autocast(...)`. +- `SANA_WM_STAGE1_QUANT=nvfp4` selects `NVFP4BlockScaling`; `fp8block` selects + `Float8BlockScaling`. +- The default FlashDreams FP4 Stage-1 selector maps to upstream + `self_attn+cross+ffn`, covering self-attention, cross-attention, and MLP + linear layers. In the validated reference demo run, upstream reported 120 + converted linear layers and 20 wrapped transformer blocks. +- The refiner applies the same pattern to eligible LTX-2 transformer linear + layers after skipping input/output projection, audio-only, caption, and time + embedding modules. + +Use `--quant-backend upstream-te` only on this reference harness when comparing +native low-precision behavior against upstream Transformer Engine behavior. + +```bash +flashdreams-run sana-wm-upstream-reference \ + --upstream-sana-root ../Sana \ + --image-path ../Sana/asset/sana_wm/demo_0.png \ + --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ + --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ + --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --num-frames 161 \ + --output-dir outputs/sana_wm_reference_fp4 \ + --stage1-precision fp4 \ + --refiner-precision fp4 \ + --quant-backend upstream-te +``` + +FlashDreams already removed a separate Transformer Engine dependency for RoPE +via `flashdreams.core.attention.rope_kernel`, but that replacement is not a +drop-in for SANA FP4/FP8. Native SANA replaces eligible Stage-1 and refiner +`Linear` modules directly, including activation quantization/scales, weight +conversion, GEMM, and output rescaling. + +Relevant FlashDreams components for future hardening: + +- `flashdreams.core.attention.rope_kernel`: the prior narrow TE-replacement + pattern and test strategy. +- `integrations/omnidreams/omnidreams_singleview/python/cosmos_fp8_utils.py`: + FP8 weight quantization and prepared weight/scale aliases for Cosmos blocks. +- `integrations/omnidreams/omnidreams_singleview/src/dit_streaming/kernels/`: + CUDA/CUTLASS FP8 GEMM, BF16 GEMM, INT8 block quantization, and SageAttention3 + FP4 attention/cache kernels. diff --git a/integrations/sana/pyproject.toml b/integrations/sana/pyproject.toml index d0baed354..df7ebf34b 100644 --- a/integrations/sana/pyproject.toml +++ b/integrations/sana/pyproject.toml @@ -20,15 +20,18 @@ build-backend = "setuptools.build_meta" [project] name = "flashdreams-sana-wm" version = "0.1.0" -description = "SANA-WM bidirectional world-model runner shim for flashdreams." +description = "SANA-WM bidirectional world-model integration for flashdreams." readme = "README.md" requires-python = ">=3.10" -# The upstream SANA repository owns the Stage-1 DiT, LTX2 VAE/refiner, -# Pi3X intrinsics estimator, and custom attention kernels. Those heavy -# dependencies are intentionally installed from the upstream SANA checkout, -# not pulled into every FlashDreams workspace sync. dependencies = [ + "diffusers>=0.36", "flashdreams", + "imageio[ffmpeg]>=2.31", + "Pillow>=10", + "PyYAML>=6.0", + "safetensors>=0.5", + "torchvision>=0.26", + "transformers>=5.0,<6", ] [tool.uv.sources] @@ -41,6 +44,8 @@ dev = [ [project.entry-points."flashdreams.runner_configs"] "sana-wm-bidirectional" = "sana_wm.config:RUNNER_SANA_WM_BIDIRECTIONAL" +"sana-wm-upstream-reference" = "sana_wm.config:RUNNER_SANA_WM_UPSTREAM_REFERENCE" +"sana-wm-native-bidirectional" = "sana_wm.config:RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL" [tool.setuptools.packages.find] include = ["sana_wm*"] diff --git a/integrations/sana/sana_wm/__init__.py b/integrations/sana/sana_wm/__init__.py index f4b7dbd28..77dfd6e32 100644 --- a/integrations/sana/sana_wm/__init__.py +++ b/integrations/sana/sana_wm/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""SANA-WM bidirectional runner shim for FlashDreams.""" +"""SANA-WM native and upstream-reference runners for FlashDreams.""" from sana_wm.config import RUNNER_CONFIGS, RUNNER_SANA_WM_BIDIRECTIONAL diff --git a/integrations/sana/sana_wm/_tools.py b/integrations/sana/sana_wm/_tools.py new file mode 100644 index 000000000..7562bfcba --- /dev/null +++ b/integrations/sana/sana_wm/_tools.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Local file and checkpoint helpers for the SANA-WM integration.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +import torch + +HF_URI_SCHEME = "hf://" + + +def resolve_hf_path(path: str | Path) -> str: + """Resolve a local path or ``hf://owner/repo/subpath`` URI to a local path.""" + path_str = str(path) + if not path_str or Path(path_str).exists(): + return path_str + if not path_str.startswith(HF_URI_SCHEME): + return path_str + + from huggingface_hub import snapshot_download + + parts = path_str[len(HF_URI_SCHEME) :].split("/", 2) + if len(parts) < 2 or not parts[0] or not parts[1]: + raise ValueError( + f"Invalid HF path {path_str!r}; expected hf:///[/]." + ) + repo_id = f"{parts[0]}/{parts[1]}" + subpath = parts[2] if len(parts) > 2 else "" + allow_patterns = None + if subpath: + allow_patterns = [subpath, f"{subpath}/*", f"{subpath}/**"] + local_root = snapshot_download(repo_id=repo_id, allow_patterns=allow_patterns) + return os.path.join(local_root, subpath) if subpath else local_root + + +def find_model(model_name: str) -> dict[str, Any]: + """Load a SANA checkpoint from a local path or resolved HF artefact.""" + resolved = resolve_hf_path(model_name) + if not os.path.isfile(resolved): + raise FileNotFoundError(f"Could not find SANA checkpoint at {resolved}") + + if resolved.endswith(".safetensors"): + import safetensors.torch + + return {"state_dict": safetensors.torch.load_file(resolved, device="cpu")} + if resolved.endswith(".safetensors.index.json"): + import safetensors.torch + + with open(resolved, encoding="utf-8") as handle: + index = json.load(handle)["weight_map"] + state_dict = {} + for shard in sorted(set(index.values())): + shard_path = os.path.join(os.path.dirname(resolved), shard) + state_dict.update(safetensors.torch.load_file(shard_path, device="cpu")) + return {"state_dict": state_dict} + return torch.load(resolved, map_location=lambda storage, _loc: storage) + + +def hf_download_or_fpath(path: str | Path) -> str: + """Compatibility alias retained for local helper call sites.""" + return resolve_hf_path(path) + + +__all__ = [ + "find_model", + "hf_download_or_fpath", + "resolve_hf_path", +] diff --git a/integrations/sana/sana_wm/config.py b/integrations/sana/sana_wm/config.py index ffa72922e..faa9d8aa2 100644 --- a/integrations/sana/sana_wm/config.py +++ b/integrations/sana/sana_wm/config.py @@ -13,44 +13,99 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Static configs for the SANA-WM bidirectional runner.""" +"""Static configs for SANA-WM native and upstream-reference runners.""" from __future__ import annotations +from typing import cast + +from flashdreams.infra.config import derive_config from flashdreams.infra.diffusion.model import DiffusionModelConfig from flashdreams.infra.diffusion.scheduler import FlowMatchSchedulerConfig from flashdreams.infra.pipeline import StreamInferencePipelineConfig from flashdreams.infra.runner import RunnerConfig -from sana_wm.placeholder import SanaWMPlaceholderTransformerConfig +from sana_wm.native_diffusion import SanaWMDiffusionModelConfig +from sana_wm.native_transformer import SanaWMNativeTransformerConfig from sana_wm.runner import SanaWMRunnerConfig +from sana_wm.upstream_reference import SanaWMUpstreamReferenceTransformerConfig + + +def _reference_pipeline(name: str) -> StreamInferencePipelineConfig: + return StreamInferencePipelineConfig( + name=name, + diffusion_model=DiffusionModelConfig( + transformer=SanaWMUpstreamReferenceTransformerConfig(), + scheduler=FlowMatchSchedulerConfig(), + seed=42, + ), + ) + + +PIPELINE_SANA_WM_UPSTREAM_REFERENCE = _reference_pipeline("sana-wm-upstream-reference") +"""Schema marker for the upstream-delegating SANA-WM reference runner.""" PIPELINE_SANA_WM_BIDIRECTIONAL = StreamInferencePipelineConfig( name="sana-wm-bidirectional", - diffusion_model=DiffusionModelConfig( - transformer=SanaWMPlaceholderTransformerConfig(), + diffusion_model=SanaWMDiffusionModelConfig( + transformer=SanaWMNativeTransformerConfig(), scheduler=FlowMatchSchedulerConfig(), seed=42, ), ) -"""Schema placeholder for the upstream-delegating SANA-WM runner.""" +"""Native FlashDreams SANA-WM pipeline.""" + +PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL = cast( + StreamInferencePipelineConfig, + derive_config(PIPELINE_SANA_WM_BIDIRECTIONAL, name="sana-wm-native-bidirectional"), +) +"""Backward-compatible native SANA-WM pipeline alias.""" + +RUNNER_SANA_WM_UPSTREAM_REFERENCE = SanaWMRunnerConfig( + runner_name=PIPELINE_SANA_WM_UPSTREAM_REFERENCE.name, + description=( + "SANA-WM upstream reference harness (NVlabs/Sana Stage-1 + LTX-2 refiner)." + ), + pipeline=PIPELINE_SANA_WM_UPSTREAM_REFERENCE, +) +"""Explicit SANA-WM upstream reference runner config.""" RUNNER_SANA_WM_BIDIRECTIONAL = SanaWMRunnerConfig( runner_name=PIPELINE_SANA_WM_BIDIRECTIONAL.name, description=( - "SANA-WM bidirectional I2V world model (upstream NVlabs/Sana " - "Stage-1 + LTX-2 refiner runner)." + "SANA-WM bidirectional I2V native FlashDreams runner " + "(Stage-1 DiT)." ), pipeline=PIPELINE_SANA_WM_BIDIRECTIONAL, + execution_backend="native-flashdreams", + no_refiner=True, +) +"""SANA-WM native FlashDreams runner config.""" + +RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL = SanaWMRunnerConfig( + runner_name=PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL.name, + description="SANA-WM native FlashDreams pipeline alias.", + pipeline=PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL, + execution_backend="native-flashdreams", + no_refiner=True, ) -"""Default SANA-WM bidirectional runner config.""" +"""Backward-compatible native SANA-WM FlashDreams runner alias.""" RUNNER_CONFIGS: dict[str, RunnerConfig] = { - cfg.runner_name: cfg for cfg in (RUNNER_SANA_WM_BIDIRECTIONAL,) + cfg.runner_name: cfg + for cfg in ( + RUNNER_SANA_WM_BIDIRECTIONAL, + RUNNER_SANA_WM_UPSTREAM_REFERENCE, + RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL, + ) } """SANA-WM runner configs keyed by ``runner_name``.""" __all__ = [ "PIPELINE_SANA_WM_BIDIRECTIONAL", + "PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL", + "PIPELINE_SANA_WM_UPSTREAM_REFERENCE", "RUNNER_CONFIGS", "RUNNER_SANA_WM_BIDIRECTIONAL", + "RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL", + "RUNNER_SANA_WM_UPSTREAM_REFERENCE", ] diff --git a/integrations/sana/sana_wm/native_diffusion.py b/integrations/sana/sana_wm/native_diffusion.py new file mode 100644 index 000000000..be719ffab --- /dev/null +++ b/integrations/sana/sana_wm/native_diffusion.py @@ -0,0 +1,228 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SANA-WM-specific diffusion model for first-frame-pinned LTX Euler sampling.""" + +from __future__ import annotations + +import os +import time +from dataclasses import dataclass, field +from typing import Any, cast + +import torch +from loguru import logger +from torch import Tensor + +from flashdreams.infra.diffusion.model import DiffusionModel, DiffusionModelConfig +from sana_wm.native_transformer import ( + SanaWMNativeTransformer, + SanaWMNativeTransformerCache, +) + + +@dataclass(kw_only=True) +class SanaWMDiffusionModelConfig(DiffusionModelConfig): + """Diffusion model config for SANA-WM's LTX-style Stage-1 sampler.""" + + _target: type["SanaWMDiffusionModel"] = field( + default_factory=lambda: SanaWMDiffusionModel + ) + + +class SanaWMDiffusionModel(DiffusionModel[SanaWMNativeTransformerCache]): + """Run SANA-WM Stage-1 denoising behind the FlashDreams diffusion boundary.""" + + transformer: SanaWMNativeTransformer + + def __init__(self, config: SanaWMDiffusionModelConfig) -> None: + super().__init__(config) + if not isinstance(self.transformer, SanaWMNativeTransformer): + raise TypeError( + "SanaWMDiffusionModel requires SanaWMNativeTransformerConfig." + ) + + def generate( + self, + autoregressive_index: int, + cache: SanaWMNativeTransformerCache, + input: Any = None, + ) -> tuple[Tensor, "DiffusionModel.FinalState[SanaWMNativeTransformerCache]"]: + """Run SANA-WM's first-frame-pinned LTX Euler denoising loop.""" + del input + if autoregressive_index != 0: + raise ValueError("SANA-WM bidirectional native inference has one AR step.") + conditioning = cache.conditioning + if conditioning is None: + raise RuntimeError("SANA-WM native diffusion cache has no conditioning.") + + cache.start(autoregressive_index) + latents = self.transformer.initial_latents(conditioning) + if torch.cuda.is_available(): + torch.cuda.synchronize() + t0 = time.perf_counter() + clean_latent = self._sample_ltx_euler(latents, cache) + if torch.cuda.is_available(): + torch.cuda.synchronize() + logger.info( + "[timing] stage1 sample: {:.3f}s (latent shape {})", + time.perf_counter() - t0, + tuple(clean_latent.shape), + ) + + final_state = DiffusionModel.FinalState( + clean_latent=clean_latent, + autoregressive_index=autoregressive_index, + cache=cache, + input=None, + ) + return clean_latent, final_state + + def finalize( + self, + final_state: "DiffusionModel.FinalState[SanaWMNativeTransformerCache]", + ) -> None: + """Finalize the one-shot cache without an extra model forward.""" + final_state.cache.finalize(final_state.autoregressive_index) + + def _sample_ltx_euler( + self, + latents: Tensor, + cache: SanaWMNativeTransformerCache, + ) -> Tensor: + conditioning = cache.conditioning + assert conditioning is not None + + from diffusers import FlowMatchEulerDiscreteScheduler + from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3 import ( + retrieve_timesteps, + ) + from tqdm import tqdm + + scheduler = FlowMatchEulerDiscreteScheduler(shift=conditioning.flow_shift) + timesteps, _ = retrieve_timesteps( + scheduler, + conditioning.steps, + self.device, + None, + ) + do_cfg = conditioning.cfg_scale > 1.0 + condition_frame_info = dict( + cast( + dict[int, float], + cast(dict[str, object], conditioning.model_kwargs["data_info"]).get( + "condition_frame_info", + {}, + ), + ) + ) + condition_mask = torch.zeros_like(latents) + image_cond_noise_scale = 0.0 + for frame_idx, frame_weight in condition_frame_info.items(): + condition_mask[:, :, int(frame_idx)] = 1 + image_cond_noise_scale = max(image_cond_noise_scale, float(frame_weight)) + + prompt_embeds = conditioning.condition + if do_cfg: + if conditioning.uncondition is None: + raise RuntimeError("CFG was requested without negative prompt embeds.") + prompt_embeds = torch.cat( + [conditioning.uncondition, conditioning.condition], + dim=0, + ) + + init_latents = latents.clone() + generator = torch.Generator(device=self.device).manual_seed(conditioning.seed) + iterator = enumerate(timesteps) + if os.getenv("DPM_TQDM", "False") != "True": + iterator = tqdm(list(iterator)) + + for _, timestep_scalar in iterator: + if image_cond_noise_scale > 0: + latents = _add_noise_to_conditioning_latents( + t=timestep_scalar / 1000.0, + init_latents=init_latents, + latents=latents, + noise_scale=image_cond_noise_scale, + conditioning_mask=condition_mask, + generator=generator, + ) + + condition_mask_input = ( + torch.cat([condition_mask] * 2) if do_cfg else condition_mask + ) + latent_model_input = torch.cat([latents] * 2) if do_cfg else latents + timestep = timestep_scalar.expand(condition_mask_input.shape).float() + timestep = torch.min(timestep, (1 - condition_mask_input) * 1000.0) + + noise_pred = self.transformer.predict_flow( + noisy_latent=latent_model_input, + timestep=timestep[:, :1, :, 0, 0], + cache=cache, + input=prompt_embeds, + ) + + if do_cfg: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + conditioning.cfg_scale * ( + noise_pred_text - noise_pred_uncond + ) + timestep = timestep.chunk(2)[0] + + latents_dtype = latents.dtype + latents_shape = latents.shape + batch_size, channels, _frames, _height, _width = latents_shape + denoised_latents = scheduler.step( + -noise_pred.reshape(batch_size, channels, -1).transpose(1, 2), + timestep_scalar, + latents.reshape(batch_size, channels, -1).transpose(1, 2), + per_token_timesteps=timestep.reshape(batch_size, channels, -1)[:, 0], + return_dict=False, + )[0] + denoised_latents = denoised_latents.transpose(1, 2).reshape(latents_shape) + tokens_to_denoise_mask = timestep_scalar / 1000 - 1e-6 < ( + 1.0 - condition_mask + ) + latents = torch.where(tokens_to_denoise_mask, denoised_latents, latents) + if latents.dtype != latents_dtype: + latents = latents.to(latents_dtype) + + return latents.detach() + + +def _add_noise_to_conditioning_latents( + *, + t: Tensor, + init_latents: Tensor, + latents: Tensor, + noise_scale: float, + conditioning_mask: Tensor, + generator: torch.Generator, + eps: float = 1e-6, +) -> Tensor: + from diffusers.utils.torch_utils import randn_tensor + + noise = randn_tensor( + latents.shape, + generator=generator, + device=latents.device, + dtype=latents.dtype, + ) + need_to_noise = conditioning_mask > (1.0 - eps) + noised_latents = init_latents + noise_scale * noise * (t**2) + return torch.where(need_to_noise, noised_latents, latents) + + +__all__ = ["SanaWMDiffusionModel", "SanaWMDiffusionModelConfig"] diff --git a/integrations/sana/sana_wm/native_quant.py b/integrations/sana/sana_wm/native_quant.py new file mode 100644 index 000000000..50b878223 --- /dev/null +++ b/integrations/sana/sana_wm/native_quant.py @@ -0,0 +1,564 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native low-precision linear helpers for SANA-WM. + +Upstream SANA's FP8/FP4 path uses Transformer Engine by replacing selected +``torch.nn.Linear`` modules and entering ``te.fp8_autocast`` around the +corresponding transformer blocks. This module provides TE-free replacements for +the same replacement point: inference-only FP8 and NVFP4 Linear modules backed +by PyTorch's native ``torch._scaled_mm`` kernel, with Triton used for NVFP4 +activation quantization. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any, Literal + +import torch +import triton +import triton.language as tl +from torch import nn + +FP8_MAX_E4M3 = 448.0 +FP8_SCALE_EPS = 1.0e-12 +FP4_MAX_E2M1 = 6.0 +NVFP4_BLOCK_SIZE = 16 +NVFP4_SCALE_EPS = 1.5258789e-05 + + +@dataclass(frozen=True) +class TorchScaledMMFP8Recipe: + """Marker recipe used while patching upstream SANA's TE recipe hooks.""" + + name: str = "torch_scaled_mm_fp8" + precision: Literal["fp8"] = "fp8" + + +@dataclass(frozen=True) +class TorchScaledMMFP4Recipe: + """Marker recipe used while patching upstream SANA's NVFP4 hooks.""" + + name: str = "torch_scaled_mm_fp4" + precision: Literal["fp4"] = "fp4" + + +NativeQuantRecipe = TorchScaledMMFP8Recipe | TorchScaledMMFP4Recipe + + +@triton.jit +def _pack_fp32_to_fp4_pairs(values): + packed = tl.inline_asm_elementwise( + asm=""" + { + .reg .b8 byte0, byte1, byte2, byte3; + cvt.rn.satfinite.e2m1x2.f32 byte0, $5, $1; + cvt.rn.satfinite.e2m1x2.f32 byte1, $6, $2; + cvt.rn.satfinite.e2m1x2.f32 byte2, $7, $3; + cvt.rn.satfinite.e2m1x2.f32 byte3, $8, $4; + mov.b32 $0, {byte0, byte1, byte2, byte3}; + } + """, + constraints=("=r,r,r,r,r,r,r,r,r"), + args=values, + dtype=tl.uint8, + is_pure=True, + pack=4, + ) + return packed + + +@triton.jit +def _quantize_nvfp4_kernel( + input_ptr, + qdata_ptr, + scale_ptr, + stride_m, + stride_k, + rows: tl.constexpr, + cols: tl.constexpr, + mask_scales: tl.constexpr, +): + fp4_max = 6.0 + fp8_max = 448.0 + scale_eps = 1.5258789e-05 + + pid_k = tl.program_id(0) + pid_m = tl.program_id(1) + + offsets_m = pid_m * 128 + tl.arange(0, 128)[:, None] + offsets_k = pid_k * 64 + tl.arange(0, 64)[None, :] + mask = (offsets_m < rows) & (offsets_k < cols) + values = tl.load( + input_ptr + offsets_m * stride_m + offsets_k * stride_k, + mask=mask, + other=0.0, + ) + values = values.to(tl.float32).reshape(128, 4, 16) + block_amax = tl.max(tl.abs(values), axis=2) + scales_f32 = block_amax / fp4_max + scales_f32 = tl.clamp(scales_f32, scale_eps, fp8_max) + scales = scales_f32.to(tl.float8e4nv) + quantized = tl.div_rn(values, scales.to(tl.float32)[:, :, None]) + + if mask_scales: + scale_offsets_k = pid_k * 4 + tl.arange(0, 4)[None, :] + scale_mask = (offsets_m < rows) & (scale_offsets_k < tl.cdiv(cols, 16)) + scales = tl.where(scale_mask, scales, 0.0) + + packed_scales = scales.reshape(4, 32, 4).permute(1, 0, 2).reshape(32, 16) + scale_m = tl.arange(0, 32)[:, None] + scale_k = tl.arange(0, 16)[None, :] + tl.store( + scale_ptr + (pid_m * tl.num_programs(0) + pid_k) * (32 * 16) + scale_m * 16 + scale_k, + packed_scales, + ) + + packed = _pack_fp32_to_fp4_pairs(quantized.reshape(128, 32, 2).split()) + q_offsets_m = pid_m * 128 + tl.arange(0, 128)[:, None] + q_offsets_k = pid_k * 32 + tl.arange(0, 32)[None, :] + q_mask = (q_offsets_m < rows) & (q_offsets_k < cols // 2) + tl.store(qdata_ptr + q_offsets_m * (cols // 2) + q_offsets_k, packed, mask=q_mask) + + +def _ceil_div(value: int, divisor: int) -> int: + return (value + divisor - 1) // divisor + + +def _swizzled_nvfp4_scale_shape(rows: int, cols: int) -> tuple[int, int]: + return _ceil_div(rows, 128) * 32, _ceil_div(cols, 64) * 16 + + +def quantize_nvfp4_swizzled(input: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Quantize a 2D CUDA tensor to NVFP4 qdata plus swizzled E4M3 scales.""" + _require_fp4_dtype() + if input.dim() != 2: + raise ValueError(f"NVFP4 quantization requires a 2D tensor, got {input.dim()}D.") + if not input.is_cuda: + raise RuntimeError("NVFP4 quantization requires CUDA input.") + rows, cols = input.shape + if cols % NVFP4_BLOCK_SIZE != 0: + raise ValueError( + f"NVFP4 quantization requires last dim divisible by {NVFP4_BLOCK_SIZE}, got {cols}." + ) + input_2d = input.contiguous() + scale_rows, scale_cols = _swizzled_nvfp4_scale_shape(rows, cols) + qdata = torch.empty((rows, cols // 2), device=input.device, dtype=torch.uint8) + scales = torch.empty( + (scale_rows, scale_cols), + device=input.device, + dtype=torch.float8_e4m3fn, + ) + grid = (_ceil_div(cols, 64), _ceil_div(rows, 128)) + _quantize_nvfp4_kernel[grid]( + input_2d, + qdata, + scales, + input_2d.stride(0), + input_2d.stride(1), + rows, + cols, + mask_scales=(rows % 128 != 0 or cols % 64 != 0), + ) + return qdata, scales + + +class TorchScaledMMFP8Linear(nn.Module): + """Inference-only FP8 Linear using ``torch._scaled_mm``. + + The module stores the source weight as E4M3 FP8 with per-output-channel + scales. At runtime it quantizes the flattened activation rows to E4M3 with + per-row scales, then calls ``torch._scaled_mm`` and returns BF16 output. + This intentionally mirrors the shape contract of ``nn.Linear`` so it can + replace eligible upstream SANA linear layers without changing call sites. + """ + + in_features: int + out_features: int + + def __init__( + self, + *, + weight: torch.Tensor, + weight_fp8: torch.Tensor, + weight_scale: torch.Tensor, + bias: torch.Tensor | None, + out_dtype: torch.dtype, + ) -> None: + super().__init__() + if weight.shape != weight_fp8.shape: + raise ValueError( + "weight and weight_fp8 must have matching shape, got " + f"{tuple(weight.shape)} and {tuple(weight_fp8.shape)}." + ) + if weight_fp8.dim() != 2: + raise ValueError(f"weight_fp8 must be 2D, got {tuple(weight_fp8.shape)}.") + if weight_scale.shape != (weight_fp8.shape[0], 1): + raise ValueError( + "weight_scale must have shape " + f"({weight_fp8.shape[0]}, 1), got {tuple(weight_scale.shape)}." + ) + if bias is not None and bias.shape != (weight_fp8.shape[0],): + raise ValueError( + f"bias must have shape ({weight_fp8.shape[0]},), got {tuple(bias.shape)}." + ) + self.out_features = int(weight_fp8.shape[0]) + self.in_features = int(weight_fp8.shape[1]) + self.out_dtype = out_dtype + self.register_buffer("weight", weight.detach().contiguous()) + self.register_buffer("weight_fp8", weight_fp8.contiguous()) + self.register_buffer("weight_scale", weight_scale.to(torch.float32).contiguous()) + if bias is None: + self.register_buffer("bias", None) + else: + self.register_buffer("bias", bias.detach().to(out_dtype).contiguous()) + + @classmethod + def from_linear( + cls, + source: nn.Linear, + *, + out_dtype: torch.dtype, + ) -> "TorchScaledMMFP8Linear": + """Create an FP8 replacement from a source ``nn.Linear``.""" + _require_fp8_dtype() + weight_f32 = source.weight.detach().to(torch.float32) + weight_scale = ( + weight_f32.abs().amax(dim=1, keepdim=True).clamp_min(FP8_SCALE_EPS) + / FP8_MAX_E4M3 + ) + weight_fp8 = torch.clamp( + weight_f32 / weight_scale, + -FP8_MAX_E4M3, + FP8_MAX_E4M3, + ).to(torch.float8_e4m3fn) + bias = source.bias.detach() if source.bias is not None else None + replacement = cls( + weight=source.weight.detach().to(device=source.weight.device), + weight_fp8=weight_fp8.to(device=source.weight.device), + weight_scale=weight_scale.to(device=source.weight.device), + bias=bias.to(device=source.weight.device) if bias is not None else None, + out_dtype=out_dtype, + ) + replacement.train(source.training) + return replacement + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Apply the quantized linear projection.""" + if input.shape[-1] != self.in_features: + raise ValueError( + f"expected input last dim {self.in_features}, got {input.shape[-1]}." + ) + if not input.is_cuda: + raise RuntimeError("TorchScaledMMFP8Linear requires CUDA input.") + _require_scaled_mm() + _require_fp8_dtype() + + leading_shape = input.shape[:-1] + input_2d = input.reshape(-1, self.in_features) + if input_2d.numel() == 0: + return input.new_empty( + (*leading_shape, self.out_features), + dtype=self.out_dtype, + ) + + input_f32 = input_2d.to(torch.float32) + input_scale = ( + input_f32.abs().amax(dim=1, keepdim=True).clamp_min(FP8_SCALE_EPS) + / FP8_MAX_E4M3 + ) + input_fp8 = torch.clamp( + input_f32 / input_scale, + -FP8_MAX_E4M3, + FP8_MAX_E4M3, + ).to(torch.float8_e4m3fn) + input_fp8 = input_fp8.contiguous() + + # ``_scaled_mm`` expects the B operand as a column-major transpose view, + # so keep the transpose stride instead of forcing it contiguous. + output = torch._scaled_mm( + input_fp8, + self.weight_fp8.t(), + scale_a=input_scale.contiguous(), + scale_b=self.weight_scale.t().contiguous(), + out_dtype=self.out_dtype, + ) + if self.bias is not None: + output = output + self.bias.to(device=output.device, dtype=output.dtype) + return output.reshape(*leading_shape, self.out_features) + + +class TorchScaledMMFP4Linear(nn.Module): + """Inference-only NVFP4 Linear using Triton quantization and ``_scaled_mm``.""" + + in_features: int + out_features: int + + def __init__( + self, + *, + weight: torch.Tensor, + weight_qdata: torch.Tensor, + weight_scale: torch.Tensor, + bias: torch.Tensor | None, + out_dtype: torch.dtype, + ) -> None: + super().__init__() + expected_weight_shape = (weight_qdata.shape[0], weight_qdata.shape[1] * 2) + if weight.shape != expected_weight_shape: + raise ValueError( + "weight must match unpacked weight_qdata shape " + f"{expected_weight_shape}, got {tuple(weight.shape)}." + ) + if weight_qdata.dim() != 2: + raise ValueError(f"weight_qdata must be 2D, got {tuple(weight_qdata.shape)}.") + if weight_qdata.shape[1] * 2 % NVFP4_BLOCK_SIZE != 0: + raise ValueError( + "weight_qdata must represent a K dimension divisible by " + f"{NVFP4_BLOCK_SIZE}, got packed shape {tuple(weight_qdata.shape)}." + ) + if bias is not None and bias.shape != (weight_qdata.shape[0],): + raise ValueError( + f"bias must have shape ({weight_qdata.shape[0]},), got {tuple(bias.shape)}." + ) + self.out_features = int(weight_qdata.shape[0]) + self.in_features = int(weight_qdata.shape[1] * 2) + self.out_dtype = out_dtype + self.register_buffer("weight", weight.detach().contiguous()) + self.register_buffer("weight_qdata", weight_qdata.contiguous()) + self.register_buffer("weight_scale", weight_scale.contiguous()) + if bias is None: + self.register_buffer("bias", None) + else: + self.register_buffer("bias", bias.detach().to(out_dtype).contiguous()) + + @classmethod + def from_linear( + cls, + source: nn.Linear, + *, + out_dtype: torch.dtype, + ) -> "TorchScaledMMFP4Linear": + """Create an NVFP4 replacement from a source ``nn.Linear``.""" + _require_fp4_dtype() + weight_qdata, weight_scale = quantize_nvfp4_swizzled(source.weight.detach()) + bias = source.bias.detach() if source.bias is not None else None + replacement = cls( + weight=source.weight.detach().to(device=source.weight.device), + weight_qdata=weight_qdata, + weight_scale=weight_scale, + bias=bias.to(device=source.weight.device) if bias is not None else None, + out_dtype=out_dtype, + ) + replacement.train(source.training) + return replacement + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Apply the quantized linear projection.""" + if input.shape[-1] != self.in_features: + raise ValueError( + f"expected input last dim {self.in_features}, got {input.shape[-1]}." + ) + if not input.is_cuda: + raise RuntimeError("TorchScaledMMFP4Linear requires CUDA input.") + _require_scaled_mm() + _require_fp4_dtype() + + leading_shape = input.shape[:-1] + input_2d = input.reshape(-1, self.in_features) + if input_2d.numel() == 0: + return input.new_empty( + (*leading_shape, self.out_features), + dtype=self.out_dtype, + ) + + input_qdata, input_scale = quantize_nvfp4_swizzled(input_2d) + output = torch._scaled_mm( + input_qdata.view(torch.float4_e2m1fn_x2), + self.weight_qdata.t().view(torch.float4_e2m1fn_x2), + input_scale.view(torch.float8_e4m3fn), + self.weight_scale.view(torch.float8_e4m3fn), + bias=self.bias.to(device=input.device, dtype=self.out_dtype) + if self.bias is not None + else None, + out_dtype=self.out_dtype, + ) + return output.reshape(*leading_shape, self.out_features) + + +def replace_linear_with_torch_fp8( + module: nn.Module, + *, + recipe: Any, + params_dtype: torch.dtype, + skip_patterns: tuple[str, ...], + include_patterns: tuple[str, ...] | None = None, + prefix: str = "", +) -> tuple[int, int]: + """Replace eligible ``nn.Linear`` modules with ``TorchScaledMMFP8Linear``.""" + return _replace_linear_with_native_quant( + module, + recipe=TorchScaledMMFP8Recipe(), + params_dtype=params_dtype, + skip_patterns=skip_patterns, + include_patterns=include_patterns, + prefix=prefix, + ) + + +def replace_linear_with_torch_fp4( + module: nn.Module, + *, + recipe: Any, + params_dtype: torch.dtype, + skip_patterns: tuple[str, ...], + include_patterns: tuple[str, ...] | None = None, + prefix: str = "", +) -> tuple[int, int]: + """Replace eligible ``nn.Linear`` modules with ``TorchScaledMMFP4Linear``.""" + return _replace_linear_with_native_quant( + module, + recipe=TorchScaledMMFP4Recipe(), + params_dtype=params_dtype, + skip_patterns=skip_patterns, + include_patterns=include_patterns, + prefix=prefix, + ) + + +def replace_linear_with_native_quant( + module: nn.Module, + *, + recipe: NativeQuantRecipe, + params_dtype: torch.dtype, + skip_patterns: tuple[str, ...], + include_patterns: tuple[str, ...] | None = None, + prefix: str = "", +) -> tuple[int, int]: + """Replace eligible ``nn.Linear`` modules with the requested native backend. + + The signature intentionally matches upstream SANA's + ``_replace_linear_with_te_nvfp4`` helper so the runner can patch the backend + without modifying upstream source files. + """ + return _replace_linear_with_native_quant( + module, + recipe=recipe, + params_dtype=params_dtype, + skip_patterns=skip_patterns, + include_patterns=include_patterns, + prefix=prefix, + ) + + +def _replace_linear_with_native_quant( + module: nn.Module, + *, + recipe: NativeQuantRecipe, + params_dtype: torch.dtype, + skip_patterns: tuple[str, ...], + include_patterns: tuple[str, ...] | None, + prefix: str, +) -> tuple[int, int]: + converted = 0 + skipped = 0 + for name, child in list(module.named_children()): + child_prefix = f"{prefix}.{name}" if prefix else name + if _name_matches(skip_patterns, child_prefix): + skipped += 1 + continue + if isinstance(child, nn.Linear): + if include_patterns is not None and not _name_matches( + include_patterns, + child_prefix, + ): + skipped += 1 + continue + if child.in_features % 16 != 0 or child.out_features % 16 != 0: + skipped += 1 + continue + if recipe.precision == "fp4" and child.in_features % 32 != 0: + skipped += 1 + continue + out_dtype = ( + params_dtype + if params_dtype in {torch.bfloat16, torch.float16} + else torch.bfloat16 + ) + if recipe.precision == "fp8": + replacement = TorchScaledMMFP8Linear.from_linear( + child, + out_dtype=out_dtype, + ) + elif recipe.precision == "fp4": + replacement = TorchScaledMMFP4Linear.from_linear( + child, + out_dtype=out_dtype, + ) + else: + raise ValueError(f"Unsupported native quant recipe: {recipe!r}.") + setattr(module, name, replacement) + converted += 1 + continue + child_converted, child_skipped = _replace_linear_with_native_quant( + child, + recipe=recipe, + params_dtype=params_dtype, + skip_patterns=skip_patterns, + include_patterns=include_patterns, + prefix=child_prefix, + ) + converted += child_converted + skipped += child_skipped + return converted, skipped + + +def _name_matches(patterns: tuple[str, ...], name: str) -> bool: + return any(re.search(pattern, name) for pattern in patterns) + + +def _require_scaled_mm() -> None: + if not hasattr(torch, "_scaled_mm"): + raise RuntimeError("torch._scaled_mm is required for native SANA quantization.") + + +def _require_fp8_dtype() -> None: + if not hasattr(torch, "float8_e4m3fn"): + raise RuntimeError("torch.float8_e4m3fn is required for the native SANA FP8 backend.") + + +def _require_fp4_dtype() -> None: + if not hasattr(torch, "float4_e2m1fn_x2"): + raise RuntimeError( + "torch.float4_e2m1fn_x2 is required for the native SANA FP4 backend." + ) + if not hasattr(torch, "float8_e4m3fn"): + raise RuntimeError( + "torch.float8_e4m3fn scales are required for the native SANA FP4 backend." + ) + + +__all__ = [ + "TorchScaledMMFP4Linear", + "TorchScaledMMFP4Recipe", + "TorchScaledMMFP8Linear", + "TorchScaledMMFP8Recipe", + "quantize_nvfp4_swizzled", + "replace_linear_with_native_quant", + "replace_linear_with_torch_fp4", + "replace_linear_with_torch_fp8", +] diff --git a/integrations/sana/sana_wm/native_transformer.py b/integrations/sana/sana_wm/native_transformer.py new file mode 100644 index 000000000..d8be6efb1 --- /dev/null +++ b/integrations/sana/sana_wm/native_transformer.py @@ -0,0 +1,1085 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native FlashDreams adapter for the SANA-WM Stage-1 DiT.""" + +from __future__ import annotations + +import gc +import os +import time +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import SimpleNamespace +from typing import Any, Literal, cast + +import numpy as np +import torch +import torch.nn as nn +import yaml +from loguru import logger +from torch import Tensor + +from flashdreams.infra.diffusion.transformer import ( + Transformer, + TransformerAutoregressiveCache, + TransformerConfig, +) +from sana_wm._tools import find_model, resolve_hf_path +from sana_wm.constants import ( + DEFAULT_VIDEO_HEIGHT, + DEFAULT_VIDEO_WIDTH, + SANA_WM_CONFIG_PATH, + SANA_WM_MODEL_PATH, + SANA_WM_REFINER_GEMMA_ROOT, + SANA_WM_REFINER_ROOT, +) +from sana_wm.native_quant import ( + TorchScaledMMFP4Recipe, + TorchScaledMMFP8Recipe, + replace_linear_with_native_quant, +) +from sana_wm.stage1_model import SanaWMStage1Model + +Precision = Literal["bf16", "fp8", "fp4"] +QuantBackend = Literal["auto", "upstream-te", "torch-fp8", "torch-fp4", "native"] + +_STAGE1_QUANT_SKIP_DEFAULTS = ( + "^x_embedder", + "^raymap_embedder", + "^plucker_embedder", + "^t_embedder", + "^t_block", + "^y_embedder", + "^final_layer", + # SANA attention code accesses output_gate.weight directly; replacing it + # with a module that only has quantized buffers breaks that call site. + r"\.output_gate$", +) +_STAGE1_QUANT_INCLUDE_DEFAULTS = ( + r"^blocks\.\d+\.attn\.qkv$", + r"^blocks\.\d+\.attn\.proj$", + r"^blocks\.\d+\.attn\.beta_proj$", + r"^blocks\.\d+\.attn\.gate_proj$", + r"^blocks\.\d+\.cross_attn\.", + r"^blocks\.\d+\.mlp\.inverted_conv\.linear$", + r"^blocks\.\d+\.mlp\.point_conv\.linear$", +) + + +@dataclass(kw_only=True) +class SanaWMStage1Conditioning: + """Per-rollout inputs needed by the SANA-WM Stage-1 sampler.""" + + condition: Tensor + uncondition: Tensor | None + model_kwargs: dict[str, object] + first_latent: Tensor + latent_shape: tuple[int, int, int, int, int] + cfg_scale: float + flow_shift: float + steps: int + seed: int + + +@dataclass(kw_only=True) +class SanaWMNativeTransformerCache(TransformerAutoregressiveCache): + """AR cache for the one-shot SANA-WM Stage-1 rollout.""" + + conditioning: SanaWMStage1Conditioning | None = None + + +@dataclass(kw_only=True) +class SanaWMNativeTransformerConfig(TransformerConfig): + """Config for the native SANA-WM Stage-1 transformer adapter.""" + + _target: type["SanaWMNativeTransformer"] = field( + default_factory=lambda: SanaWMNativeTransformer + ) + + config_path: str = SANA_WM_CONFIG_PATH + """SANA-WM inference YAML path or ``hf://`` URI.""" + + checkpoint_path: str = SANA_WM_MODEL_PATH + """SANA-WM Stage-1 checkpoint path or ``hf://`` URI.""" + + stage1_precision: Precision = "bf16" + """Stage-1 precision requested by the runner.""" + + quant_backend: QuantBackend = "auto" + """Low-precision linear replacement backend for FP8/FP4 modes.""" + + refiner_root: str = SANA_WM_REFINER_ROOT + """LTX-2 refiner root path or ``hf://`` URI.""" + + refiner_gemma_root: str = SANA_WM_REFINER_GEMMA_ROOT + """Gemma text-encoder root for the refiner.""" + + refiner_precision: Precision = "bf16" + """Refiner precision requested by the runner.""" + + offload_vae: bool = False + """Move the VAE to CPU between encode/decode phases.""" + + offload_text_encoder: bool = False + """Move the Stage-1 text encoder to CPU between prompt encodes.""" + + offload_refiner: bool = False + """Release the LTX-2 refiner after each refinement.""" + + height: int = DEFAULT_VIDEO_HEIGHT + """Pixel height used by the public SANA-WM bidirectional release.""" + + width: int = DEFAULT_VIDEO_WIDTH + """Pixel width used by the public SANA-WM bidirectional release.""" + + vae_tile_sample_min_width: int = 256 + """Pixel width tile size used for native LTX-2 VAE decode.""" + + vae_tile_sample_stride_width: int = 224 + """Pixel width stride used for native LTX-2 VAE decode.""" + + vae_tile_sample_min_height: int = 256 + """Pixel height tile size used for native LTX-2 VAE decode.""" + + vae_tile_sample_stride_height: int = 192 + """Pixel height stride used for native LTX-2 VAE decode.""" + + vae_tile_sample_min_num_frames: int = 24 + """Pixel-frame temporal tile size used for native LTX-2 VAE decode.""" + + vae_tile_sample_stride_num_frames: int = 8 + """Pixel-frame temporal tile stride used for native LTX-2 VAE decode.""" + + vae_oom_retry_tile_sample_min_width: int = 128 + """Smaller pixel width tile size for one VAE decode OOM retry.""" + + vae_oom_retry_tile_sample_stride_width: int = 64 + """Smaller pixel width tile stride for one VAE decode OOM retry.""" + + vae_oom_retry_tile_sample_min_height: int = 128 + """Smaller pixel height tile size for one VAE decode OOM retry.""" + + vae_oom_retry_tile_sample_stride_height: int = 64 + """Smaller pixel height tile stride for one VAE decode OOM retry.""" + + +class SanaWMNativeTransformer(Transformer[SanaWMNativeTransformerCache]): + """Native FlashDreams adapter for the SANA-WM Stage-1 model call.""" + + def __init__(self, config: SanaWMNativeTransformerConfig) -> None: + super().__init__(config) + self.config: SanaWMNativeTransformerConfig = config + self._dummy = nn.Parameter(torch.empty(0)) + self._runtime_config: Any | None = None + self._model_path: str | None = None + self.weight_dtype: torch.dtype | None = None + self._model_built = False + self._vae_built = False + self._text_encoder_built = False + self._refiner_built = False + self._stage1_quantized = False + self._streaming_prompt_cache: dict[ + tuple[object, ...], tuple[Tensor, Tensor, Tensor, Tensor] + ] = {} + + @property + def latent_shape(self) -> tuple[int, ...]: + """Return the current rollout latent shape or the public default.""" + if self._runtime_config is None: + return (1, 128, 21, 22, 40) + cfg = self._ensure_runtime_config() + latent_t = (161 - 1) // int(cfg.vae.vae_stride[0]) + 1 + return ( + 1, + int(cfg.vae.vae_latent_dim), + latent_t, + self.config.height // int(cfg.vae.vae_stride[-1]), + self.config.width // int(cfg.vae.vae_stride[-1]), + ) + + def initialize_autoregressive_cache( + self, + *, + conditioning: SanaWMStage1Conditioning | None = None, + **_: Any, + ) -> SanaWMNativeTransformerCache: + """Build a cache containing per-rollout SANA-WM conditioning.""" + return SanaWMNativeTransformerCache(conditioning=conditioning) + + def predict_flow( + self, + noisy_latent: Tensor, + timestep: Tensor, + cache: SanaWMNativeTransformerCache, + input: Any = None, + ) -> Tensor: + """Execute one SANA-WM DiT flow prediction.""" + conditioning = _require_conditioning(cache) + prompt_embeds = cast(Tensor, input) if input is not None else conditioning.condition + self._ensure_model() + return self.model( + noisy_latent, + timestep, + prompt_embeds, + **conditioning.model_kwargs, + ) + + def finalize_kv_cache( + self, + noisy_latent: Tensor, + timestep: Tensor, + cache: SanaWMNativeTransformerCache, + input: Any = None, + ) -> None: + """SANA-WM bidirectional inference has no streaming KV cache to advance.""" + del noisy_latent, timestep, cache, input + + def patchify_and_maybe_split_cp(self, x: Any) -> Any: + """SANA-WM Stage-1 latents are already in model layout.""" + return x + + def unpatchify_and_maybe_gather_cp(self, x: Tensor) -> Tensor: + """SANA-WM Stage-1 latents are already in model layout.""" + return x + + def prepare_conditioning( + self, + *, + image: Any, + prompt: str, + camera: Mapping[str, Tensor], + num_frames: int, + fps: int, + steps: int, + cfg_scale: float, + flow_shift: float | None, + seed: int, + negative_prompt: str, + ) -> SanaWMStage1Conditioning: + """Encode first-frame, text, and camera tensors for one rollout.""" + del fps + cfg = self._ensure_runtime_config() + weight_dtype = self._ensure_weight_dtype() + self._ensure_vae() + if self.config.offload_vae: + self.vae.to(self.device) + + from torchvision import transforms as T + + img = (T.ToTensor()(image) * 2.0 - 1.0).unsqueeze(0).unsqueeze(2) + first_latent = self._vae_encode( + img.to(self.device, dtype=self.vae_dtype), + ).to(weight_dtype) + if self.config.offload_vae: + self.vae.to("cpu") + torch.cuda.empty_cache() + + cond, cond_mask, neg, neg_mask = self._encode_prompts( + prompt, + negative_prompt, + ) + cond, cond_mask, neg, neg_mask = self._pad_text_for_quant( + cond, + cond_mask, + neg, + neg_mask, + ) + + raymap = camera["raymap"].unsqueeze(0).to(self.device, dtype=weight_dtype) + chunk_plucker = camera["chunk_plucker"].unsqueeze(0).to( + self.device, + dtype=weight_dtype, + ) + if cfg_scale > 1.0: + mask_cfg = torch.cat([neg_mask, cond_mask], dim=0) + raymap_cfg = torch.cat([raymap, raymap], dim=0) + chunk_plucker_cfg = torch.cat([chunk_plucker, chunk_plucker], dim=0) + uncondition = neg + else: + mask_cfg = cond_mask + raymap_cfg = raymap + chunk_plucker_cfg = chunk_plucker + uncondition = None + + latent_t = (num_frames - 1) // int(cfg.vae.vae_stride[0]) + 1 + latent_h = self.config.height // int(cfg.vae.vae_stride[-1]) + latent_w = self.config.width // int(cfg.vae.vae_stride[-1]) + chunk_index = self._chunk_index(latent_t) + model_kwargs: dict[str, object] = { + "data_info": { + "img_hw": torch.tensor( + [[self.config.height, self.config.width]], + dtype=torch.float, + device=self.device, + ), + "condition_frame_info": {0: 0.0}, + }, + "mask": mask_cfg, + "camera_conditions": raymap_cfg, + "chunk_plucker": chunk_plucker_cfg, + } + if chunk_index is not None: + model_kwargs["chunk_index"] = chunk_index + + return SanaWMStage1Conditioning( + condition=cond, + uncondition=uncondition, + model_kwargs=model_kwargs, + first_latent=first_latent, + latent_shape=( + 1, + int(first_latent.shape[1]), + latent_t, + latent_h, + latent_w, + ), + cfg_scale=float(cfg_scale), + flow_shift=self._resolve_flow_shift(flow_shift), + steps=int(steps), + seed=int(seed), + ) + + def initial_latents(self, conditioning: SanaWMStage1Conditioning) -> Tensor: + """Draw Stage-1 initial noise and pin the encoded first frame.""" + generator = torch.Generator(device=self.device).manual_seed(conditioning.seed) + latents = torch.randn( + conditioning.latent_shape, + dtype=self._ensure_weight_dtype(), + device=self.device, + generator=generator, + ) + latents[:, :, :1] = conditioning.first_latent + return latents + + def decode_latents(self, latents: Tensor) -> np.ndarray: + """Decode SANA-WM VAE latents to ``uint8`` HWC video.""" + self._ensure_vae() + if self.config.offload_vae: + self.vae.to(self.device) + samples = latents.to(device=self.device, dtype=self.vae_dtype) + if torch.cuda.is_available(): + torch.cuda.synchronize() + t0 = time.perf_counter() + retry_decode = False + try: + decoded = self._vae_decode(samples) + except torch.OutOfMemoryError: + retry_decode = True + + if retry_decode: + if torch.cuda.is_available(): + torch.cuda.synchronize() + torch.cuda.empty_cache() + gc.collect() + logger.warning( + "[sana-vae] decode OOM; retrying with smaller tiles " + "width={} stride_width={} height={} stride_height={}", + self.config.vae_oom_retry_tile_sample_min_width, + self.config.vae_oom_retry_tile_sample_stride_width, + self.config.vae_oom_retry_tile_sample_min_height, + self.config.vae_oom_retry_tile_sample_stride_height, + ) + self._configure_vae_tiling( + tile_sample_min_width=( + self.config.vae_oom_retry_tile_sample_min_width + ), + tile_sample_stride_width=( + self.config.vae_oom_retry_tile_sample_stride_width + ), + tile_sample_min_height=( + self.config.vae_oom_retry_tile_sample_min_height + ), + tile_sample_stride_height=( + self.config.vae_oom_retry_tile_sample_stride_height + ), + ) + if torch.cuda.is_available(): + torch.cuda.empty_cache() + decoded = self._vae_decode(samples) + if torch.cuda.is_available(): + torch.cuda.synchronize() + logger.info( + "[timing] vae decode: {:.3f}s (latent T={} -> pixels {})", + time.perf_counter() - t0, + latents.shape[2], + tuple(decoded.shape) if isinstance(decoded, Tensor) else "list", + ) + if isinstance(decoded, list): + decoded = torch.stack(decoded, dim=0) + video = ( + torch.clamp(127.5 * decoded + 127.5, 0, 255) + .permute(0, 2, 3, 4, 1) + .to("cpu", dtype=torch.uint8) + .numpy()[0] + ) + if self.config.offload_vae: + self.vae.to("cpu") + del samples, decoded + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return video + + def release_stage1_runtime( + self, + cache: SanaWMNativeTransformerCache | None = None, + ) -> None: + """Release Stage-1-only tensors before VAE/refiner work.""" + free_before_gib: float | None = None + if torch.cuda.is_available(): + try: + free_before, _total = torch.cuda.mem_get_info() + free_before_gib = free_before / (1024**3) + except RuntimeError: + free_before_gib = None + if cache is not None: + cache.conditioning = None + self._streaming_prompt_cache.clear() + + for attr in ("model", "text_encoder"): + module = getattr(self, attr, None) + if module is None: + continue + try: + module.to("meta") + except Exception: + try: + module.to("cpu") + except Exception: + pass + setattr(self, attr, None) + + if hasattr(self, "tokenizer"): + self.tokenizer = None + self._model_built = False + self._text_encoder_built = False + self._stage1_quantized = False + if torch.cuda.is_available(): + torch.cuda.empty_cache() + try: + free_after, _total = torch.cuda.mem_get_info() + if free_before_gib is not None: + logger.info( + "[stage1] released Stage-1 runtime before decode/refine " + "(free CUDA memory: {:.2f} -> {:.2f} GiB)", + free_before_gib, + free_after / (1024**3), + ) + else: + logger.info("[stage1] released Stage-1 runtime before decode/refine") + except RuntimeError: + logger.info("[stage1] released Stage-1 runtime before decode/refine") + gc.collect() + + def refine_latents( + self, + *, + latents: Tensor, + prompt: str, + fps: int, + sink_size: int, + seed: int, + block_size: int | None, + kv_max_frames: int, + ) -> Tensor: + """Run the native LTX-2 refiner once that path is implemented.""" + self._ensure_refiner() + sigmas = torch.tensor( + self._stage2_sigmas(), + dtype=torch.float32, + device=self.device, + ) + logger.info( + "[refiner] {}-step Euler, start_sigma={:.4f}", + len(sigmas) - 1, + float(sigmas[0]), + ) + refined = self.refiner.refine_latents( + latents, + prompt, + fps=float(fps), + sink_size=int(sink_size), + seed=int(seed), + progress=True, + block_size=block_size, + kv_max_frames=int(kv_max_frames), + ) + if self.config.offload_refiner: + self._release_refiner() + return refined + + def _ensure_runtime_config(self) -> Any: + if self._runtime_config is not None: + return self._runtime_config + self._runtime_config = _load_inference_config(self.config.config_path) + return self._runtime_config + + def _ensure_weight_dtype(self) -> torch.dtype: + if self.weight_dtype is None: + self.weight_dtype = _get_weight_dtype( + self._ensure_runtime_config().model.mixed_precision + ) + return self.weight_dtype + + def _ensure_vae(self) -> None: + if self._vae_built: + return + cfg = self._ensure_runtime_config() + self.vae_dtype = _get_weight_dtype(cfg.vae.weight_dtype) + cfg.vae.vae_pretrained = resolve_hf_path(cfg.vae.vae_pretrained) + self.vae = _get_vae( + cfg.vae.vae_type, + cfg.vae.vae_pretrained, + device=self.device, + dtype=self.vae_dtype, + config=cfg.vae, + ) + if hasattr(self.vae, "enable_tiling"): + self._configure_vae_tiling() + self._vae_built = True + + def _ensure_text_encoder(self) -> None: + if self._text_encoder_built: + return + cfg = self._ensure_runtime_config() + self.tokenizer, self.text_encoder = _get_tokenizer_and_text_encoder( + name=cfg.text_encoder.text_encoder_name, + device=self.device, + ) + if self.config.offload_text_encoder: + self.text_encoder.to("cpu") + self._text_encoder_built = True + + def _ensure_model(self) -> None: + if self._model_built: + self._prepare_stage1_quant() + return + cfg = self._ensure_runtime_config() + weight_dtype = self._ensure_weight_dtype() + model = SanaWMStage1Model().to(self.device) + logger.info( + "[Sana] Loaded native {} ({:,} params)", + cfg.model.model, + sum(p.numel() for p in model.parameters()), + ) + self._model_path = resolve_hf_path(self.config.checkpoint_path) + state = find_model(self._model_path) + if "generator" in state: + state = state["generator"] + if "state_dict" not in state: + state = { + "state_dict": { + (k[len("model.") :] if k.startswith("model.") else k): v + for k, v in state.items() + } + } + missing, unexpected = model.load_state_dict(state["state_dict"], strict=True) + if missing: + logger.warning("[Sana] Missing keys: {}", missing) + if unexpected: + logger.warning("[Sana] Unexpected keys: {}", unexpected) + self.model = model.eval().to(weight_dtype) + self._model_built = True + self._prepare_stage1_quant() + + def _ensure_refiner(self) -> None: + raise RuntimeError( + "Native SANA-WM refiner execution is not implemented yet. " + "Use --no-refiner True for native Stage-1 GPU tests, or " + "sana-wm-upstream-reference for the temporary upstream refiner harness." + ) + + def _release_refiner(self) -> None: + if not self._refiner_built: + return + del self.refiner + self._refiner_built = False + if torch.cuda.is_available(): + torch.cuda.empty_cache() + gc.collect() + + def _configure_vae_tiling( + self, + *, + tile_sample_min_width: int | None = None, + tile_sample_stride_width: int | None = None, + tile_sample_min_height: int | None = None, + tile_sample_stride_height: int | None = None, + tile_sample_min_num_frames: int | None = None, + tile_sample_stride_num_frames: int | None = None, + ) -> None: + vae = self.vae + min_width = int(tile_sample_min_width or self.config.vae_tile_sample_min_width) + stride_width = int( + tile_sample_stride_width or self.config.vae_tile_sample_stride_width + ) + min_height = int( + tile_sample_min_height or self.config.vae_tile_sample_min_height + ) + stride_height = int( + tile_sample_stride_height or self.config.vae_tile_sample_stride_height + ) + spatial_ratio = int(getattr(vae, "spatial_compression_ratio", 1)) + stride_width = _avoid_degenerate_tile_tail( + sample_extent=self.config.width, + sample_tile_min=min_width, + sample_stride=stride_width, + compression_ratio=spatial_ratio, + ) + stride_height = _avoid_degenerate_tile_tail( + sample_extent=self.config.height, + sample_tile_min=min_height, + sample_stride=stride_height, + compression_ratio=spatial_ratio, + ) + min_frames = int( + tile_sample_min_num_frames + or self.config.vae_tile_sample_min_num_frames + ) + stride_frames = int( + tile_sample_stride_num_frames + or self.config.vae_tile_sample_stride_num_frames + ) + kwargs = { + "tile_sample_min_height": min_height, + "tile_sample_stride_height": stride_height, + "tile_sample_min_width": min_width, + "tile_sample_stride_width": stride_width, + "tile_sample_min_num_frames": min_frames, + "tile_sample_stride_num_frames": stride_frames, + } + if hasattr(vae, "enable_tiling"): + try: + vae.enable_tiling(**kwargs) + except TypeError: + vae.enable_tiling() + for name, value in kwargs.items(): + if hasattr(vae, name): + setattr(vae, name, value) + if hasattr(vae, "use_framewise_encoding"): + vae.use_framewise_encoding = True + if hasattr(vae, "use_framewise_decoding"): + vae.use_framewise_decoding = True + logger.info( + "[sana-vae] tiling width={} stride_width={} height={} " + "stride_height={} frames={} stride_frames={}", + min_width, + stride_width, + min_height, + stride_height, + min_frames, + stride_frames, + ) + + def _encode_prompts( + self, + prompt: str, + negative_prompt: str, + ) -> tuple[Tensor, Tensor, Tensor, Tensor]: + cfg = self._ensure_runtime_config() + self._ensure_text_encoder() + max_length = cfg.text_encoder.model_max_length + chi_prompt = "\n".join(cfg.text_encoder.chi_prompt or []) + if chi_prompt: + prompt = chi_prompt + prompt + max_length_all = len(self.tokenizer.encode(chi_prompt)) + max_length - 2 + else: + max_length_all = max_length + + key = ( + prompt, + negative_prompt, + str(self.device), + str(self._ensure_weight_dtype()), + self.config.stage1_precision, + self.config.quant_backend, + ) + if key in self._streaming_prompt_cache: + return self._streaming_prompt_cache[key] + + move_text_encoder = self.config.offload_text_encoder or ( + next(self.text_encoder.parameters()).device != self.device + ) + if move_text_encoder: + self.text_encoder.to(self.device) + + def encode(text: str, length: int) -> tuple[Tensor, Tensor]: + tokens = self.tokenizer( + [text], + max_length=length, + padding="max_length", + truncation=True, + return_tensors="pt", + ).to(self.device) + return self.text_encoder(tokens.input_ids, tokens.attention_mask)[0], ( + tokens.attention_mask + ) + + try: + cond, cond_mask = encode(prompt, max_length_all) + select = [0] + list(range(-max_length + 1, 0)) + cond = cond[:, None][:, :, select] + cond_mask = cond_mask[:, select] + neg, neg_mask = encode(negative_prompt, max_length) + result = (cond, cond_mask, neg[:, None], neg_mask) + self._streaming_prompt_cache.clear() + self._streaming_prompt_cache[key] = result + return result + finally: + if move_text_encoder: + self.text_encoder.to("cpu") + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + def _pad_text_for_quant( + self, + cond: Tensor, + cond_mask: Tensor, + neg: Tensor, + neg_mask: Tensor, + ) -> tuple[Tensor, Tensor, Tensor, Tensor]: + if self.config.stage1_precision == "bf16": + return cond, cond_mask, neg, neg_mask + multiple = int(os.environ.get("SANA_WM_STAGE1_NVFP4_TEXT_PAD_MULTIPLE", "8")) + if multiple <= 1: + return cond, cond_mask, neg, neg_mask + + def pad_pair(text: Tensor, mask: Tensor) -> tuple[Tensor, Tensor]: + pad = (-text.shape[-2]) % multiple + if pad == 0: + return text, mask + text_shape = list(text.shape) + text_shape[-2] = pad + mask_shape = list(mask.shape) + mask_shape[-1] = pad + return ( + torch.cat([text, text.new_zeros(text_shape)], dim=-2), + torch.cat([mask, mask.new_zeros(mask_shape)], dim=-1), + ) + + cond, cond_mask = pad_pair(cond, cond_mask) + neg, neg_mask = pad_pair(neg, neg_mask) + return cond, cond_mask, neg, neg_mask + + def _prepare_stage1_quant(self) -> None: + if self._stage1_quantized or self.config.stage1_precision == "bf16": + return + if self.config.quant_backend == "upstream-te": + raise ValueError( + "Native FlashDreams SANA execution does not use Transformer Engine; " + "select --quant-backend native, torch-fp8, or torch-fp4." + ) + if self.config.stage1_precision == "fp8": + recipe = TorchScaledMMFP8Recipe() + else: + recipe = TorchScaledMMFP4Recipe() + converted, skipped = replace_linear_with_native_quant( + self.model, + recipe=recipe, + params_dtype=self._ensure_weight_dtype(), + skip_patterns=_STAGE1_QUANT_SKIP_DEFAULTS, + include_patterns=_STAGE1_QUANT_INCLUDE_DEFAULTS, + ) + if converted <= 0: + raise RuntimeError( + f"SANA-WM native {self.config.stage1_precision} converted no " + f"Stage-1 Linear layers; skipped={skipped}." + ) + self._stage1_quantized = True + logger.info( + "[stage1-native-quant] precision={} converted {} Linear layers (skipped {})", + self.config.stage1_precision, + converted, + skipped, + ) + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + def _resolve_flow_shift(self, override: float | None) -> float: + cfg = self._ensure_runtime_config() + if override is not None: + return float(override) + if cfg.scheduler.inference_flow_shift is not None: + return float(cfg.scheduler.inference_flow_shift) + return float(cfg.scheduler.flow_shift) + + def _chunk_index(self, latent_frames: int) -> list[int] | None: + return _chunk_index_from_config( + self._ensure_runtime_config(), + num_frames=latent_frames, + ) + + def _vae_encode(self, images: Tensor) -> Tensor: + return _vae_encode_ltx2( + self._ensure_runtime_config().vae.vae_type, + self.vae, + images, + device=self.device, + ) + + def _vae_decode(self, latents: Tensor) -> Tensor: + return _vae_decode_ltx2( + self._ensure_runtime_config().vae.vae_type, + self.vae, + latents, + ) + + @staticmethod + def _stage2_sigmas() -> tuple[float, ...]: + return (0.909375, 0.725, 0.421875, 0.0) + + +def _require_conditioning( + cache: SanaWMNativeTransformerCache, +) -> SanaWMStage1Conditioning: + if cache.conditioning is None: + raise RuntimeError("SANA-WM native cache was initialized without conditioning.") + return cache.conditioning + + +def _avoid_degenerate_tile_tail( + *, + sample_extent: int, + sample_tile_min: int, + sample_stride: int, + compression_ratio: int, +) -> int: + """Choose a tile stride that avoids final latent tiles of size one.""" + if compression_ratio <= 1: + return sample_stride + latent_extent = max(1, sample_extent // compression_ratio) + latent_tile_min = max(1, sample_tile_min // compression_ratio) + requested_latent_stride = max(1, sample_stride // compression_ratio) + + def tail_size(latent_stride: int) -> int: + last_start = ((latent_extent - 1) // latent_stride) * latent_stride + return latent_extent - last_start + + candidates = range( + min(requested_latent_stride, latent_tile_min), + 1, + -1, + ) + for latent_stride in candidates: + if tail_size(latent_stride) > 1: + return latent_stride * compression_ratio + + for latent_stride in range( + requested_latent_stride + 1, + latent_tile_min + 1, + ): + if tail_size(latent_stride) > 1: + return latent_stride * compression_ratio + + return sample_stride + + +def _load_inference_config(config_path: str) -> Any: + with open(resolve_hf_path(config_path), encoding="utf-8") as handle: + raw = yaml.safe_load(handle) or {} + if not isinstance(raw, dict): + raise TypeError(f"SANA-WM config must be a mapping, got {type(raw).__name__}.") + raw.setdefault("work_dir", "") + return _to_namespace(raw) + + +def _get_vae(*args: Any, **kwargs: Any) -> nn.Module: + name, model_path = args[:2] + device = kwargs["device"] + dtype = kwargs["dtype"] + if "LTX2VAE_diffusers" not in str(name): + raise ValueError(f"Unsupported native SANA-WM VAE type: {name!r}") + from diffusers import AutoencoderKLLTX2Video + + return ( + AutoencoderKLLTX2Video.from_pretrained( + model_path, + subfolder="vae", + torch_dtype=dtype, + ) + .to(device) + .eval() + ) + + +def _get_tokenizer_and_text_encoder(*args: Any, **kwargs: Any) -> tuple[Any, nn.Module]: + name = kwargs.get("name", args[0] if args else "T5") + device = kwargs.get("device", "cuda") + model_id = _TEXT_ENCODER_MODEL_IDS.get(str(name)) + if model_id is None: + raise ValueError(f"Unsupported native SANA-WM text encoder: {name!r}") + from transformers import AutoModelForCausalLM, AutoTokenizer, T5EncoderModel, T5Tokenizer + + if "T5" in str(name): + tokenizer = T5Tokenizer.from_pretrained(model_id) + text_encoder = T5EncoderModel.from_pretrained( + model_id, + torch_dtype=torch.float16, + ).to(device) + return tokenizer, text_encoder.eval() + + tokenizer = AutoTokenizer.from_pretrained(model_id) + tokenizer.padding_side = "right" + text_encoder = ( + AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16) + .get_decoder() + .to(device) + .eval() + ) + return tokenizer, text_encoder + + +def _get_weight_dtype(value: str) -> torch.dtype: + normalized = str(value).lower() + if normalized in {"bf16", "bfloat16"}: + return torch.bfloat16 + if normalized in {"fp16", "float16", "half"}: + return torch.float16 + if normalized in {"fp32", "float32", "float"}: + return torch.float32 + raise ValueError(f"Unsupported dtype value: {value!r}") + + +class _ConfigNamespace(SimpleNamespace): + """Simple YAML-backed config object with dict-like ``get`` support.""" + + def get(self, name: str, default: Any = None) -> Any: + return getattr(self, name, default) + + +_TEXT_ENCODER_MODEL_IDS = { + "T5": "DeepFloyd/t5-v1_1-xxl", + "T5-small": "google/t5-v1_1-small", + "T5-base": "google/t5-v1_1-base", + "T5-large": "google/t5-v1_1-large", + "T5-xl": "google/t5-v1_1-xl", + "T5-xxl": "google/t5-v1_1-xxl", + "gemma-2b": "google/gemma-2b", + "gemma-2b-it": "google/gemma-2b-it", + "gemma-2-2b": "google/gemma-2-2b", + "gemma-2-2b-it": "Efficient-Large-Model/gemma-2-2b-it", + "gemma-2-9b": "google/gemma-2-9b", + "gemma-2-9b-it": "google/gemma-2-9b-it", +} + + +def _vae_encode_ltx2(name: str, vae: nn.Module, images: Tensor, *, device: torch.device) -> Tensor: + if "LTX2VAE_diffusers" not in name: + raise ValueError(f"Unsupported native SANA-WM VAE encode type: {name!r}") + dtype = images.dtype + vae_device = next(vae.parameters()).device + vae_dtype = next(vae.parameters()).dtype + posterior = vae.encode(images.to(device=vae_device, dtype=vae_dtype)).latent_dist + z = posterior.mode() + latents_mean = vae.latents_mean.view(1, -1, 1, 1, 1).to(z.device, z.dtype) + latents_std = vae.latents_std.view(1, -1, 1, 1, 1).to(z.device, z.dtype) + z = (z - latents_mean) * vae.config.scaling_factor / latents_std + return z.to(device=device, dtype=dtype) + + +def _vae_decode_ltx2(name: str, vae: nn.Module, latents: Tensor) -> Tensor: + if "LTX2VAE_diffusers" not in name: + raise ValueError(f"Unsupported native SANA-WM VAE decode type: {name!r}") + vae_device = next(vae.parameters()).device + vae_dtype = next(vae.parameters()).dtype + latents_mean = vae.latents_mean.view(1, -1, 1, 1, 1).to( + latents.device, + latents.dtype, + ) + latents_std = vae.latents_std.view(1, -1, 1, 1, 1).to( + latents.device, + latents.dtype, + ) + scaled = latents * latents_std / vae.config.scaling_factor + latents_mean + return vae.decode( + scaled.to(device=vae_device, dtype=vae_dtype), + temb=None, + return_dict=False, + )[0] + + +def _to_namespace(value: Any) -> Any: + if isinstance(value, dict): + return _ConfigNamespace( + **{str(key): _to_namespace(child) for key, child in value.items()} + ) + if isinstance(value, list): + return [_to_namespace(child) for child in value] + return value + + +def _chunk_index_from_config(config: Any, *, num_frames: int) -> list[int] | None: + model = getattr(config, "model", None) + if model is None: + return None + chunk_index = model.get("chunk_index", None) + chunk_size = model.get("chunk_size", None) + strategy = model.get("chunk_split_strategy", "uniform") + if chunk_index is not None: + if not isinstance(chunk_index, (list, tuple)): + raise TypeError( + f"chunk_index must be a list, got {type(chunk_index).__name__}" + ) + if len(chunk_index) == 0: + raise ValueError("chunk_index cannot be empty.") + return [int(index) for index in chunk_index] + if chunk_size is None: + return None + return _chunk_index_from_chunk_size( + num_frames, + int(chunk_size), + strategy=str(strategy), + ) + + +def _chunk_index_from_chunk_size( + num_frames: int, + chunk_size: int, + *, + strategy: str = "uniform", +) -> list[int]: + if num_frames <= 0: + raise ValueError(f"num_frames must be > 0, got {num_frames}.") + if chunk_size <= 0: + raise ValueError(f"chunk_size must be > 0, got {chunk_size}.") + normalized = strategy.lower() + if normalized in {"uniform", "default"}: + indices = list(range(0, num_frames, chunk_size)) + if len(indices) > 1 and (num_frames - indices[-1]) < chunk_size: + indices.pop() + return indices + if normalized in {"first_frame", "first_frame_alone", "first_frame_only"}: + if num_frames <= 1: + return [0] + indices = [0] + list(range(1, num_frames, chunk_size)) + if len(indices) > 2 and (num_frames - indices[-1]) < chunk_size: + indices.pop() + return indices + if normalized in {"first_plus_one", "first_chunk_plus_one"}: + if num_frames <= chunk_size + 1: + return [0] + indices = [0] + list(range(chunk_size + 1, num_frames, chunk_size)) + if len(indices) > 1 and (num_frames - indices[-1]) < chunk_size: + indices.pop() + return indices + raise ValueError(f"Unknown chunk_split_strategy {strategy!r}.") + + +__all__ = [ + "SanaWMNativeTransformer", + "SanaWMNativeTransformerCache", + "SanaWMNativeTransformerConfig", + "SanaWMStage1Conditioning", +] diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index 56c5981b2..09795c905 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""SANA-WM bidirectional runner that delegates execution to upstream Sana.""" +"""SANA-WM bidirectional runner and upstream-reference harness.""" from __future__ import annotations @@ -32,7 +32,17 @@ from loguru import logger from flashdreams.core.io.disk import preflight_runtime_write_paths +from flashdreams.infra.config import derive_config +from flashdreams.infra.pipeline import StreamInferencePipelineConfig from flashdreams.infra.runner import Runner, RunnerConfig +from sana_wm.camera import ( + action_string_to_c2w, + load_intrinsics, + prepare_camera, + resize_center_crop_geometry, + snap_num_frames, + transform_intrinsics_for_crop, +) from sana_wm.constants import ( DEFAULT_ACTION, DEFAULT_FPS, @@ -44,6 +54,7 @@ SANA_WM_REFINER_ROOT, SANA_WM_VAE_TEMPORAL_COMPRESSION, ) +from sana_wm.native_transformer import SanaWMNativeTransformer SamplingAlgo = Literal[ "auto", @@ -58,6 +69,15 @@ Precision = Literal["bf16", "fp8", "fp4"] """SANA-WM Stage-1/refiner precision modes.""" +QuantBackend = Literal["auto", "upstream-te", "torch-fp8", "torch-fp4", "native"] +"""Low-precision linear backend for FP8/FP4 SANA-WM paths.""" + +ResolvedQuantBackend = Literal["upstream-te", "torch-fp8", "torch-fp4", "native"] +"""Concrete low-precision linear backend selected after resolving ``auto``.""" + +ExecutionBackend = Literal["upstream-reference", "native-flashdreams"] +"""Execution path selected by a SANA-WM runner config.""" + @dataclass(kw_only=True) class SanaWMRunnerConfig(RunnerConfig): @@ -143,7 +163,7 @@ class SanaWMRunnerConfig(RunnerConfig): seed: int = 42 """Stage-1 random seed.""" - no_action_overlay: bool = False + no_action_overlay: bool = True """Skip upstream action-overlay compositing on generated videos.""" config_path: str = SANA_WM_CONFIG_PATH @@ -154,17 +174,26 @@ class SanaWMRunnerConfig(RunnerConfig): stage1_precision: Precision = "bf16" """Stage-1 DiT compute precision. ``"bf16"`` is the default; ``"fp8"`` - requires Hopper or newer plus Transformer Engine; ``"fp4"`` requires - Blackwell plus Transformer Engine.""" + requires Hopper or newer; ``"fp4"`` requires Blackwell.""" no_refiner: bool = False """Skip the LTX-2 refiner and decode Stage-1 latents directly.""" refiner_precision: Precision = "bf16" """LTX-2 refiner compute precision. ``"bf16"`` is the default; - ``"fp8"`` requires Hopper or newer plus Transformer Engine; ``"fp4"`` - requires Blackwell plus Transformer Engine. Ignored when - ``no_refiner`` is ``True``.""" + ``"fp8"`` requires Hopper or newer; ``"fp4"`` requires Blackwell. + Ignored when ``no_refiner`` is ``True``.""" + + quant_backend: QuantBackend = "auto" + """Backend for quantized linear layers. ``"auto"`` uses the native + FlashDreams/PyTorch low-precision path. ``"upstream-te"`` is accepted only + by the upstream reference harness. ``"torch-fp8"`` and ``"torch-fp4"`` + select one native ``torch._scaled_mm`` replacement explicitly; ``"native"`` + allows both native FP8 and native FP4.""" + + execution_backend: ExecutionBackend = "upstream-reference" + """Whether this config runs the upstream reference harness or the native + FlashDreams path.""" refiner_root: str = SANA_WM_REFINER_ROOT """LTX-2 refiner root path or ``hf://`` URI.""" @@ -195,7 +224,7 @@ class SanaWMRunnerConfig(RunnerConfig): class SanaWMRunner(Runner[SanaWMRunnerConfig, object]): - """CLI driver for upstream SANA-WM bidirectional inference.""" + """CLI driver for SANA-WM native and upstream-reference configs.""" config: SanaWMRunnerConfig @@ -234,7 +263,7 @@ def _resolve_device(self) -> torch.device: return torch.device(f"cuda:{self.local_rank}") return torch.device(self.config.device) - def _resolve_trajectory(self, upstream: ModuleType) -> np.ndarray: + def _resolve_trajectory(self) -> np.ndarray: """Load or roll out the camera-to-world trajectory.""" if self.config.camera_path is not None: c2w = np.load(self.config.camera_path).astype(np.float32) @@ -245,7 +274,7 @@ def _resolve_trajectory(self, upstream: ModuleType) -> np.ndarray: return c2w if not self.config.action: raise ValueError("SanaWMRunner requires --camera-path or --action.") - return upstream.action_string_to_c2w( + return action_string_to_c2w( self.config.action, translation_speed=self.config.translation_speed, rotation_speed_deg=self.config.rotation_speed_deg, @@ -265,18 +294,142 @@ def _denoising_steps(self) -> list[int] | None: return steps def run(self) -> None: - """Run upstream SANA-WM bidirectional inference and write outputs.""" + """Run SANA-WM bidirectional inference and write outputs.""" + if self.config.execution_backend == "native-flashdreams": + self._run_native() + return + self._run_upstream_reference() + + def _run_native(self) -> None: + """Run the native FlashDreams SANA-WM pipeline.""" cfg = self.config if cfg.image_path is None: raise ValueError("SanaWMRunner requires --image-path.") + if not cfg.no_refiner: + raise ValueError( + "Native SANA-WM currently supports Stage-1 execution only. " + "Use --no-refiner True, or use sana-wm-upstream-reference for " + "the temporary upstream refiner harness." + ) + if not cfg.no_action_overlay: + raise ValueError( + "Native SANA-WM does not include the upstream action overlay. " + "Use --no-action-overlay True." + ) device = self._resolve_device() + quant_backend: QuantBackend = ( + "native" if cfg.quant_backend == "auto" else cfg.quant_backend + ) _validate_precision_request( device=device, stage1_precision=cfg.stage1_precision, refiner_precision=cfg.refiner_precision, refiner_enabled=not cfg.no_refiner, + quant_backend=quant_backend, + ) + resolved_quant_backend = _resolve_quant_backend( + quant_backend, + _active_quantized_precisions( + stage1_precision=cfg.stage1_precision, + refiner_precision=cfg.refiner_precision, + refiner_enabled=not cfg.no_refiner, + ), + ) + precision_env = _precision_env_updates( + stage1_precision=cfg.stage1_precision, + refiner_precision=cfg.refiner_precision, + refiner_enabled=not cfg.no_refiner, ) + with _temporary_environment(precision_env): + prompt = self._resolve_prompt() + image, c2w, intrinsics_vec4, num_frames = self._prepare_native_inputs( + device + ) + pipeline_cfg = _native_pipeline_config( + cfg, + quant_backend=quant_backend, + ) + pipeline = pipeline_cfg.setup().to(device).eval() + transformer = pipeline.diffusion_model.transformer + if not isinstance(transformer, SanaWMNativeTransformer): + raise TypeError( + "Native SANA-WM runner resolved a non-SANA transformer: " + f"{type(transformer).__name__}." + ) + sampling_algo = self._native_sampling_algo() + if sampling_algo != "flow_euler_ltx": + raise ValueError( + "Native SANA-WM currently supports flow_euler_ltx only; " + f"got {sampling_algo!r}. Use sana-wm-upstream-reference " + "for chunk/self-forcing samplers." + ) + camera = prepare_camera( + c2w, + intrinsics_vec4, + target_size=(DEFAULT_VIDEO_HEIGHT, DEFAULT_VIDEO_WIDTH), + ) + conditioning = transformer.prepare_conditioning( + image=image, + prompt=prompt, + camera=camera, + num_frames=num_frames, + fps=cfg.fps, + steps=cfg.step, + cfg_scale=cfg.cfg_scale, + flow_shift=cfg.flow_shift, + seed=cfg.seed, + negative_prompt=cfg.negative_prompt, + ) + cache = pipeline.initialize_cache( + transformer_context={"conditioning": conditioning} + ) + sana_latent = pipeline.generate(0, cache) + pipeline.finalize(0, cache) + transformer.release_stage1_runtime(cache) + del conditioning, cache + + video_hwc = transformer.decode_latents(sana_latent) + if not self.is_rank_zero: + return + _write_video(cfg.output_dir, cfg.name, video_hwc, cfg.fps) + + def _run_upstream_reference(self) -> None: + """Run the explicit upstream SANA-WM reference harness.""" + cfg = self.config + if cfg.image_path is None: + raise ValueError("SanaWMRunner requires --image-path.") + + device = self._resolve_device() + _validate_precision_request( + device=device, + stage1_precision=cfg.stage1_precision, + refiner_precision=cfg.refiner_precision, + refiner_enabled=not cfg.no_refiner, + quant_backend=cfg.quant_backend, + ) + resolved_quant_backend = _resolve_quant_backend( + cfg.quant_backend, + _active_quantized_precisions( + stage1_precision=cfg.stage1_precision, + refiner_precision=cfg.refiner_precision, + refiner_enabled=not cfg.no_refiner, + ), + ) + if ( + _active_quantized_precisions( + stage1_precision=cfg.stage1_precision, + refiner_precision=cfg.refiner_precision, + refiner_enabled=not cfg.no_refiner, + ) + and resolved_quant_backend != "upstream-te" + ): + raise ValueError( + "The upstream reference runner uses upstream SANA's own " + "quantized path. Pass --quant-backend upstream-te for " + "FP8/FP4 reference comparisons, or use sana-wm-bidirectional " + "for the native FlashDreams low-precision backend." + ) precision_env = _precision_env_updates( stage1_precision=cfg.stage1_precision, @@ -288,7 +441,7 @@ def run(self) -> None: prompt = self._resolve_prompt() image = upstream.Image.open(cfg.image_path).convert("RGB") - c2w_full = self._resolve_trajectory(upstream) + c2w_full = self._resolve_trajectory() num_frames = min(cfg.num_frames, c2w_full.shape[0]) snapped = upstream._snap_num_frames( num_frames, @@ -411,6 +564,107 @@ def run(self) -> None: upstream.get_root_logger(), ) + def _prepare_native_inputs( + self, + device: torch.device, + ) -> tuple[object, np.ndarray, np.ndarray, int]: + """Load/crop the input image and prepare c2w/intrinsics for native run.""" + del device + from PIL import Image + + cfg = self.config + assert cfg.image_path is not None + image = Image.open(cfg.image_path).convert("RGB") + c2w_full = self._resolve_trajectory() + num_frames = min(cfg.num_frames, c2w_full.shape[0]) + snapped = snap_num_frames( + num_frames, + stride=SANA_WM_VAE_TEMPORAL_COMPRESSION, + upper_bound=c2w_full.shape[0], + ) + if snapped != cfg.num_frames and self.is_rank_zero: + logger.warning( + "SANA-WM requires num_frames = 8k+1; requested {} snapped to {} " + "(trajectory has {} frames).", + cfg.num_frames, + snapped, + c2w_full.shape[0], + ) + num_frames = snapped + c2w = c2w_full[:num_frames] + + resized_size, crop_offset = resize_center_crop_geometry( + image.size, + target_h=DEFAULT_VIDEO_HEIGHT, + target_w=DEFAULT_VIDEO_WIDTH, + ) + resized = image.resize(resized_size, Image.LANCZOS) + left, top = crop_offset + cropped = resized.crop( + ( + left, + top, + left + DEFAULT_VIDEO_WIDTH, + top + DEFAULT_VIDEO_HEIGHT, + ) + ) + if cfg.intrinsics_path is None: + raise ValueError( + "The native SANA-WM runner currently requires --intrinsics-path. " + "Use sana-wm-upstream-reference if you need Pi3X intrinsics estimation." + ) + intrinsics_src = load_intrinsics(cfg.intrinsics_path, num_frames) + intrinsics_vec4 = transform_intrinsics_for_crop( + intrinsics_src, + image.size, + resized_size, + crop_offset, + ) + return cropped, c2w, intrinsics_vec4, num_frames + + def _native_sampling_algo(self) -> SamplingAlgo: + """Resolve the native sampler selection.""" + if self.config.sampling_algo == "auto": + return "flow_euler_ltx" + return self.config.sampling_algo + + +def _native_pipeline_config( + cfg: SanaWMRunnerConfig, + *, + quant_backend: QuantBackend, +) -> StreamInferencePipelineConfig: + """Apply CLI runtime fields to the native SANA-WM pipeline literal.""" + return derive_config( + cfg.pipeline, + diffusion_model=dict( + seed=cfg.seed, + transformer=dict( + config_path=cfg.config_path, + checkpoint_path=cfg.model_path, + stage1_precision=cfg.stage1_precision, + quant_backend=quant_backend, + refiner_root=cfg.refiner_root, + refiner_gemma_root=cfg.refiner_gemma_root, + refiner_precision=cfg.refiner_precision, + offload_vae=cfg.offload_vae, + offload_refiner=cfg.offload_refiner, + offload_text_encoder=cfg.offload_text_encoder, + ), + ), + ) + + +def _write_video(output_dir: Path, name: str, video_hwc: np.ndarray, fps: int) -> Path: + """Write an HWC uint8 video to the runner output directory.""" + import imageio.v3 as iio + + output_dir.mkdir(parents=True, exist_ok=True) + video_path = output_dir / f"{name}_generated.mp4" + iio.imwrite(video_path, video_hwc, fps=fps) + logger.info("Saved {}", video_path) + return video_path + def _precision_env_updates( *, @@ -418,7 +672,7 @@ def _precision_env_updates( refiner_precision: Precision, refiner_enabled: bool, ) -> dict[str, str | None]: - """Return upstream SANA-WM precision env overrides.""" + """Return SANA-WM precision environment overrides.""" updates: dict[str, str | None] = {"SANA_WM_STAGE1_NVFP4": None} if stage1_precision != "bf16": updates.update( @@ -442,12 +696,12 @@ def _precision_env_updates( def _quant_env_value(precision: Precision) -> str: - """Return upstream Transformer Engine recipe selector for a precision.""" + """Return the SANA quantization recipe selector for a precision.""" if precision == "fp8": return "fp8block" if precision == "fp4": return "nvfp4" - raise ValueError(f"{precision!r} does not use Transformer Engine quantization.") + raise ValueError(f"{precision!r} does not use quantized SANA execution.") @contextmanager @@ -477,12 +731,14 @@ def _validate_precision_request( stage1_precision: Precision, refiner_precision: Precision, refiner_enabled: bool, + quant_backend: QuantBackend, ) -> None: """Fail early when requested quantized precision cannot run.""" - active_precisions = [stage1_precision] - if refiner_enabled: - active_precisions.append(refiner_precision) - quantized = [precision for precision in active_precisions if precision != "bf16"] + quantized = _active_quantized_precisions( + stage1_precision=stage1_precision, + refiner_precision=refiner_precision, + refiner_enabled=refiner_enabled, + ) if not quantized: return @@ -500,20 +756,58 @@ def _validate_precision_request( "SANA-WM fp8 precision requires a Hopper or newer GPU " f"(sm_90+); detected {sm_name}." ) - if "fp8" in quantized and not _torch_cuda_version_at_least(12, 9): - cuda_version = torch.version.cuda or "unknown" - raise ValueError( - "SANA-WM fp8 precision uses upstream Sana's Transformer Engine " - "Float8BlockScaling path, which requires CUDA 12.9 or newer. " - f"This PyTorch environment reports CUDA {cuda_version}." - ) if "fp4" in quantized and major < 10: raise ValueError( "SANA-WM fp4/NVFP4 precision requires a Blackwell GPU " f"(sm_100+); detected {sm_name}. Use bf16 or fp8 on this GPU." ) - _validate_transformer_engine(quantized) + resolved_backend = _resolve_quant_backend(quant_backend, quantized) + if resolved_backend == "torch-fp8": + _validate_torch_fp8_backend(quantized) + elif resolved_backend == "torch-fp4": + _validate_torch_fp4_backend(quantized) + elif resolved_backend == "native": + _validate_native_quant_backend(quantized) + elif resolved_backend == "upstream-te": + if "fp8" in quantized and not _torch_cuda_version_at_least(12, 9): + cuda_version = torch.version.cuda or "unknown" + raise ValueError( + "SANA-WM fp8 precision uses upstream Sana's Transformer Engine " + "Float8BlockScaling path, which requires CUDA 12.9 or newer. " + f"This PyTorch environment reports CUDA {cuda_version}. Use " + "--quant-backend torch-fp8 or --quant-backend native to try " + "the native PyTorch FP8 backend." + ) + _validate_transformer_engine(quantized) + else: + raise ValueError(f"Unsupported SANA-WM quant backend: {quant_backend!r}.") + + +def _active_quantized_precisions( + *, + stage1_precision: Precision, + refiner_precision: Precision, + refiner_enabled: bool, +) -> list[Precision]: + """Return active non-BF16 precision requests in execution order.""" + active_precisions = [stage1_precision] + if refiner_enabled: + active_precisions.append(refiner_precision) + return [precision for precision in active_precisions if precision != "bf16"] + + +def _resolve_quant_backend( + quant_backend: QuantBackend, + quantized_precisions: list[Precision], +) -> ResolvedQuantBackend: + """Resolve ``auto`` to the native low-precision backend.""" + del quantized_precisions + if quant_backend == "auto": + return "native" + if quant_backend in {"upstream-te", "torch-fp8", "torch-fp4", "native"}: + return quant_backend + raise ValueError(f"Unsupported SANA-WM quant backend: {quant_backend!r}.") def _torch_cuda_version_at_least(major: int, minor: int) -> bool: @@ -530,6 +824,59 @@ def _torch_cuda_version_at_least(major: int, minor: int) -> bool: return (current_major, current_minor) >= (major, minor) +def _validate_torch_fp8_backend(precisions: list[Precision]) -> None: + """Validate the native PyTorch scaled-MM backend request.""" + unsupported = sorted({precision for precision in precisions if precision != "fp8"}) + if unsupported: + raise ValueError( + "SANA-WM --quant-backend torch-fp8 currently supports fp8 only; " + f"unsupported requested precision(s): {', '.join(unsupported)}. " + "Use --quant-backend torch-fp4 or --quant-backend native for fp4." + ) + _validate_native_fp8_primitives() + + +def _validate_torch_fp4_backend(precisions: list[Precision]) -> None: + """Validate the native PyTorch NVFP4 backend request.""" + unsupported = sorted({precision for precision in precisions if precision != "fp4"}) + if unsupported: + raise ValueError( + "SANA-WM --quant-backend torch-fp4 currently supports fp4 only; " + f"unsupported requested precision(s): {', '.join(unsupported)}. " + "Use --quant-backend torch-fp8 or --quant-backend native for fp8." + ) + _validate_native_fp4_primitives() + + +def _validate_native_quant_backend(precisions: list[Precision]) -> None: + """Validate native PyTorch low-precision primitives.""" + if "fp8" in precisions: + _validate_native_fp8_primitives() + if "fp4" in precisions: + _validate_native_fp4_primitives() + + +def _validate_native_fp8_primitives() -> None: + if not hasattr(torch, "_scaled_mm") or not hasattr(torch, "float8_e4m3fn"): + raise RuntimeError( + "SANA-WM --quant-backend torch-fp8 requires PyTorch with " + "torch._scaled_mm and torch.float8_e4m3fn support." + ) + + +def _validate_native_fp4_primitives() -> None: + missing = [ + name + for name in ("_scaled_mm", "float4_e2m1fn_x2", "float8_e4m3fn") + if not hasattr(torch, name) + ] + if missing: + raise RuntimeError( + "SANA-WM native fp4 requires PyTorch with " + f"{', '.join(f'torch.{name}' for name in missing)} support." + ) + + def _validate_transformer_engine(precisions: list[Precision]) -> None: """Ensure Transformer Engine has the recipes required by precision modes.""" required_recipes: set[str] = set() @@ -610,7 +957,9 @@ def _import_upstream_sana(explicit_root: Path | None) -> ModuleType: __all__ = [ + "ExecutionBackend", "Precision", + "QuantBackend", "SamplingAlgo", "SanaWMRunner", "SanaWMRunnerConfig", diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py new file mode 100644 index 000000000..74f21cff2 --- /dev/null +++ b/integrations/sana/sana_wm/stage1_model.py @@ -0,0 +1,514 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""FlashDreams-owned SANA-WM Stage-1 DiT module definitions. + +This file intentionally models the public SANA-WM bidirectional Stage-1 +checkpoint schema without importing the upstream SANA ``diffusion`` package. +The module names and tensor shapes are checkpoint-facing API: keep them stable +unless the public checkpoint contract changes. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor + + +@dataclass(frozen=True) +class SanaWMStage1Spec: + """Static architecture values for the public SANA-WM bidirectional DiT.""" + + latent_channels: int = 128 + hidden_size: int = 2240 + text_dim: int = 2304 + timestep_dim: int = 256 + depth: int = 20 + num_heads: int = 20 + head_dim: int = 112 + max_text_length: int = 300 + latent_grid_size: tuple[int, int] = (22, 40) + mlp_ratio: int = 3 + conv_kernel_size: int = 4 + temporal_kernel_size: int = 3 + plucker_channels: int = 48 + raymap_channels: int = 3 + softmax_every_n: int = 4 + + @property + def mlp_inner_size(self) -> int: + """Return the GLUMBConv hidden expansion width.""" + return self.hidden_size * self.mlp_ratio * 2 + + @property + def gated_mlp_size(self) -> int: + """Return the width consumed by the pointwise output projection.""" + return self.hidden_size * self.mlp_ratio + + def block_uses_gdn(self, index: int) -> bool: + """Return whether a block has GDN convolution checkpoint tensors.""" + return (index + 1) % self.softmax_every_n != 0 + + +SANA_WM_STAGE1_SPEC = SanaWMStage1Spec() +"""Architecture spec for the public SANA-WM bidirectional Stage-1 checkpoint.""" + + +class RMSNorm(nn.Module): + """RMSNorm parameter container matching the SANA checkpoint schema.""" + + def __init__(self, hidden_size: int, eps: float = 1e-6) -> None: + super().__init__() + self.weight = nn.Parameter(torch.empty(hidden_size)) + self.eps = eps + + def forward(self, x: Tensor) -> Tensor: + """Apply RMS normalization using the stored scale parameter.""" + dtype = x.dtype + normed = x.float() * torch.rsqrt( + x.float().pow(2).mean(dim=-1, keepdim=True) + self.eps + ) + return (normed * self.weight).to(dtype=dtype) + + +class Conv3dProjector(nn.Module): + """Named 1x1x1 projection used by latent, ray, and Plucker embedders.""" + + def __init__(self, in_channels: int, hidden_size: int) -> None: + super().__init__() + self.proj = nn.Conv3d(in_channels, hidden_size, kernel_size=1) + + def forward(self, x: Tensor) -> Tensor: + """Project a 5D tensor to hidden channels.""" + return self.proj(x) + + +class TimestepEmbedder(nn.Module): + """Two-layer timestep embedder with checkpoint-compatible names.""" + + def __init__(self, spec: SanaWMStage1Spec) -> None: + super().__init__() + self.mlp = nn.Sequential( + nn.Linear(spec.timestep_dim, spec.hidden_size), + nn.SiLU(), + nn.Linear(spec.hidden_size, spec.hidden_size), + ) + + def forward(self, t: Tensor) -> Tensor: + """Project timestep features into the DiT hidden width.""" + t_freq = _timestep_embedding(t.flatten(), self.mlp[0].in_features) + return self.mlp(t_freq.to(device=t.device, dtype=self.mlp[0].weight.dtype)) + + +class TextProjection(nn.Module): + """Two-layer text projection matching ``y_embedder.y_proj`` keys.""" + + def __init__(self, spec: SanaWMStage1Spec) -> None: + super().__init__() + self.fc1 = nn.Linear(spec.text_dim, spec.hidden_size) + self.act = nn.SiLU() + self.fc2 = nn.Linear(spec.hidden_size, spec.hidden_size) + + def forward(self, y: Tensor) -> Tensor: + """Project text encoder activations into the DiT hidden width.""" + return self.fc2(self.act(self.fc1(y))) + + +class TextEmbedder(nn.Module): + """Text embedding/projection container matching SANA checkpoint keys.""" + + def __init__(self, spec: SanaWMStage1Spec) -> None: + super().__init__() + self.y_embedding = nn.Parameter(torch.empty(spec.max_text_length, spec.text_dim)) + self.y_proj = TextProjection(spec) + + def forward(self, y: Tensor | None) -> Tensor: + """Project text embeddings, using the learned null embedding when absent.""" + if y is None: + y = self.y_embedding.unsqueeze(0) + if y.ndim == 4 and y.shape[1] == 1: + y = y.squeeze(1) + return self.y_proj(y) + + +class FinalLayer(nn.Module): + """Final AdaLN and latent-channel projection container.""" + + def __init__(self, spec: SanaWMStage1Spec) -> None: + super().__init__() + self.norm_final = nn.LayerNorm( + spec.hidden_size, + elementwise_affine=False, + eps=1e-6, + ) + self.scale_shift_table = nn.Parameter(torch.empty(2, spec.hidden_size)) + self.linear = nn.Linear(spec.hidden_size, spec.latent_channels) + + def forward(self, x: Tensor, t: Tensor, *, frames: int) -> Tensor: + """Project hidden tokens back to latent channels.""" + if t.ndim > 2: + batch, tokens, channels = x.shape + shift, scale = ( + self.scale_shift_table[None, None, :, :] + t.transpose(1, 2) + ).chunk(2, dim=-2) + x = _modulate( + self.norm_final(x).reshape(batch, frames, -1, channels), + shift, + scale, + ).reshape(batch, tokens, channels) + else: + shift, scale = (self.scale_shift_table[None] + t[:, None]).chunk( + 2, + dim=1, + ) + x = _modulate(self.norm_final(x), shift, scale) + return self.linear(x) + + +class Conv2dContainer(nn.Module): + """Expose a convolution under a stable ``.conv`` attribute.""" + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int | tuple[int, int], + *, + groups: int = 1, + bias: bool = True, + padding: int | tuple[int, int] = 0, + ) -> None: + super().__init__() + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=kernel_size, + groups=groups, + bias=bias, + padding=padding, + ) + + def forward(self, x: Tensor) -> Tensor: + """Apply the contained convolution.""" + return self.conv(x) + + +class GLUMBConvTemp(nn.Module): + """Checkpoint-compatible GLUMBConvTemp feed-forward container.""" + + def __init__(self, spec: SanaWMStage1Spec) -> None: + super().__init__() + self.inverted_conv = Conv2dContainer(spec.hidden_size, spec.mlp_inner_size, 1) + self.depth_conv = Conv2dContainer( + spec.mlp_inner_size, + spec.mlp_inner_size, + 3, + groups=spec.mlp_inner_size, + padding=1, + ) + self.point_conv = Conv2dContainer( + spec.gated_mlp_size, + spec.hidden_size, + 1, + bias=False, + ) + self.t_conv = nn.Conv2d( + spec.hidden_size, + spec.hidden_size, + kernel_size=(spec.temporal_kernel_size, 1), + padding=(spec.temporal_kernel_size // 2, 0), + bias=False, + ) + + def forward(self, x: Tensor, *, frames: int, height: int, width: int) -> Tensor: + """Run spatial GLU plus temporal aggregation.""" + batch, tokens, channels = x.shape + x_2d = x.reshape(batch * frames, height, width, channels).permute(0, 3, 1, 2) + x_2d = self.inverted_conv(x_2d) + x_2d = self.depth_conv(x_2d) + value, gate = x_2d.chunk(2, dim=1) + x_2d = self.point_conv(value * F.silu(gate)) + + x_time = x_2d.view(batch, frames, channels, height * width).permute(0, 2, 1, 3) + x_time = x_time + self.t_conv(x_time) + return x_time.permute(0, 2, 3, 1).reshape(batch, tokens, channels) + + +class Stage1SelfAttention(nn.Module): + """Self/camera attention parameter container for one Stage-1 block.""" + + def __init__(self, spec: SanaWMStage1Spec, *, use_gdn_convs: bool) -> None: + super().__init__() + self.A_log = nn.Parameter(torch.empty(spec.num_heads)) + self.beta_proj = nn.Linear(spec.hidden_size, spec.num_heads) + self.dt_bias = nn.Parameter(torch.empty(spec.num_heads)) + self.gate_proj = nn.Linear(spec.hidden_size, spec.num_heads) + self.k_norm = RMSNorm(spec.hidden_size) + self.k_norm_cam = RMSNorm(spec.hidden_size) + self.k_proj_cam = nn.Linear(spec.hidden_size, spec.hidden_size) + self.out_proj_cam = nn.Linear(spec.hidden_size, spec.hidden_size) + self.output_gate = nn.Linear(spec.hidden_size, spec.hidden_size) + self.proj = nn.Linear(spec.hidden_size, spec.hidden_size) + self.q_norm = RMSNorm(spec.hidden_size) + self.q_norm_cam = RMSNorm(spec.hidden_size) + self.q_proj_cam = nn.Linear(spec.hidden_size, spec.hidden_size) + self.qkv = nn.Linear(spec.hidden_size, 3 * spec.hidden_size, bias=False) + self.recall_gate = nn.Parameter(torch.empty(1)) + self.v_proj_cam = nn.Linear(spec.hidden_size, spec.hidden_size) + if use_gdn_convs: + self.conv_k = nn.Conv1d( + spec.hidden_size, + spec.hidden_size, + kernel_size=spec.conv_kernel_size, + groups=spec.hidden_size, + bias=False, + ) + self.conv_k_cam = nn.Conv1d( + spec.hidden_size, + spec.hidden_size, + kernel_size=spec.conv_kernel_size, + groups=spec.hidden_size, + bias=False, + ) + + def forward(self, x: Tensor, *_args: object, **_kwargs: object) -> Tensor: + """Run a memory-bounded native self-attention approximation. + + The public checkpoint's GDN blocks are linear-recurrent attention + blocks. Until the native GDN scan is implemented, this path uses the + checkpoint QKV/value/projection tensors without materializing an + all-token softmax attention matrix. + """ + batch, tokens, channels = x.shape + qkv = self.qkv(x).view(batch, tokens, 3, channels) + q, _k, v = qkv.unbind(dim=2) + q = self.q_norm(q) + v = v * torch.sigmoid(q) + out = F.silu(self.output_gate(x).float()).to(dtype=v.dtype) * v + return self.proj(out.to(dtype=self.proj.weight.dtype)) + + +class Stage1CrossAttention(nn.Module): + """Cross-attention parameter container for one Stage-1 block.""" + + def __init__(self, spec: SanaWMStage1Spec) -> None: + super().__init__() + self.num_heads = spec.num_heads + self.head_dim = spec.hidden_size // spec.num_heads + self.k_norm = RMSNorm(spec.hidden_size) + self.kv_linear = nn.Linear(spec.hidden_size, 2 * spec.hidden_size) + self.proj = nn.Linear(spec.hidden_size, spec.hidden_size) + self.q_linear = nn.Linear(spec.hidden_size, spec.hidden_size) + self.q_norm = RMSNorm(spec.hidden_size) + + def forward( + self, + x: Tensor, + y: Tensor, + *, + mask: Tensor | None = None, + **_kwargs: object, + ) -> Tensor: + """Cross-attention execution is implemented in a later native pass.""" + batch, tokens, channels = x.shape + q = self.q_norm(self.q_linear(x)).view( + batch, + tokens, + self.num_heads, + self.head_dim, + ) + kv = self.kv_linear(y).view(batch, -1, 2, channels) + k, v = kv.unbind(dim=2) + k = self.k_norm(k).view(batch, -1, self.num_heads, self.head_dim) + v = v.view(batch, -1, self.num_heads, self.head_dim) + + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + attn_mask = None + if mask is not None: + attn_mask = (1 - mask.to(dtype=q.dtype)) * -10000.0 + attn_mask = attn_mask[:, None, None, :] + out = F.scaled_dot_product_attention( + q, + k, + v, + attn_mask=attn_mask, + dropout_p=0.0, + is_causal=False, + ) + out = out.transpose(1, 2).reshape(batch, tokens, channels) + return self.proj(out.to(dtype=self.proj.weight.dtype)) + + +class SanaWMStage1Block(nn.Module): + """One checkpoint-compatible SANA-WM Stage-1 transformer block.""" + + def __init__(self, spec: SanaWMStage1Spec, *, index: int) -> None: + super().__init__() + self.norm1 = nn.LayerNorm( + spec.hidden_size, + elementwise_affine=False, + eps=1e-6, + ) + self.norm2 = nn.LayerNorm( + spec.hidden_size, + elementwise_affine=False, + eps=1e-6, + ) + self.scale_shift_table = nn.Parameter(torch.empty(6, spec.hidden_size)) + self.attn = Stage1SelfAttention( + spec, + use_gdn_convs=spec.block_uses_gdn(index), + ) + self.cross_attn = Stage1CrossAttention(spec) + self.mlp = GLUMBConvTemp(spec) + self.plucker_proj = nn.Linear(spec.hidden_size, spec.hidden_size) + + def forward( + self, + x: Tensor, + y: Tensor, + t: Tensor, + *, + frames: int, + height: int, + width: int, + mask: Tensor | None = None, + plucker_emb: Tensor | None = None, + **kwargs: object, + ) -> Tensor: + """Run one native Stage-1 transformer block.""" + batch, tokens, channels = x.shape + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + self.scale_shift_table[None, None, :, :] + t.reshape(batch, frames, 6, -1) + ).chunk(6, dim=-2) + + x_norm = self.norm1(x).reshape(batch, frames, -1, channels) + attn_in = _modulate(x_norm, shift_msa, scale_msa).reshape(batch, tokens, channels) + attn_out = self.attn(attn_in, **kwargs).reshape(batch, frames, -1, channels) + x = x + (gate_msa * attn_out).reshape(batch, tokens, channels) + + if plucker_emb is not None: + x = x + self.plucker_proj(plucker_emb) + + x = x + self.cross_attn(x, y, mask=mask) + + x_norm = self.norm2(x).reshape(batch, frames, -1, channels) + mlp_in = _modulate(x_norm, shift_mlp, scale_mlp).reshape(batch, tokens, channels) + mlp_out = self.mlp(mlp_in, frames=frames, height=height, width=width) + mlp_out = mlp_out.reshape(batch, frames, -1, channels) + return x + (gate_mlp * mlp_out).reshape(batch, tokens, channels) + + +class SanaWMStage1Model(nn.Module): + """Checkpoint-compatible native Stage-1 SANA-WM DiT shell.""" + + def __init__(self, spec: SanaWMStage1Spec = SANA_WM_STAGE1_SPEC) -> None: + super().__init__() + self.spec = spec + self.x_embedder = Conv3dProjector(spec.latent_channels, spec.hidden_size) + self.t_embedder = TimestepEmbedder(spec) + self.t_block = nn.Sequential(nn.SiLU(), nn.Linear(spec.hidden_size, 6 * spec.hidden_size)) + self.y_embedder = TextEmbedder(spec) + self.attention_y_norm = RMSNorm(spec.hidden_size) + self.raymap_embedder = Conv3dProjector(spec.raymap_channels, spec.hidden_size) + self.plucker_embedder = Conv3dProjector(spec.plucker_channels, spec.hidden_size) + self.blocks = nn.ModuleList( + SanaWMStage1Block(spec, index=index) for index in range(spec.depth) + ) + num_pos_tokens = spec.latent_grid_size[0] * spec.latent_grid_size[1] // 2 + 44 + self.pos_embed = nn.Parameter(torch.empty(1, num_pos_tokens, spec.hidden_size)) + self.final_layer = FinalLayer(spec) + + def forward( + self, + x: Tensor, + timestep: Tensor, + y: Tensor, + *, + mask: Tensor | None = None, + chunk_plucker: Tensor | None = None, + **kwargs: object, + ) -> Tensor: + """Run the native SANA-WM Stage-1 DiT.""" + batch, _channels, frames, height, width = x.shape + x = self.x_embedder(x) + x = x.permute(0, 2, 3, 4, 1).reshape(batch, frames * height * width, -1) + + plucker_emb = None + if chunk_plucker is not None: + plucker_emb = self.plucker_embedder(chunk_plucker) + plucker_emb = plucker_emb.permute(0, 2, 3, 4, 1).reshape_as(x) + + y = self.y_embedder(y) + y = self.attention_y_norm(y) + if mask is not None and mask.ndim > 2: + mask = mask.squeeze(1).squeeze(1) + + timestep_embed = self.t_embedder(timestep.flatten()) + timestep_embed = timestep_embed.unflatten(0, timestep.shape) + block_t = self.t_block(timestep_embed).reshape(batch, 1, frames, 6 * self.spec.hidden_size) + + for block in self.blocks: + x = block( + x, + y, + block_t, + frames=frames, + height=height, + width=width, + mask=mask, + plucker_emb=plucker_emb, + **kwargs, + ) + + x = self.final_layer(x, timestep_embed, frames=frames) + return x.reshape(batch, frames, height, width, -1).permute(0, 4, 1, 2, 3) + + +def _modulate(x: Tensor, shift: Tensor, scale: Tensor) -> Tensor: + return x * (1 + scale) + shift + + +def _timestep_embedding(t: Tensor, dim: int, max_period: int = 10000) -> Tensor: + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=torch.float32, device=t.device) + / half + ) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + +__all__ = [ + "SANA_WM_STAGE1_SPEC", + "GLUMBConvTemp", + "RMSNorm", + "SanaWMStage1Block", + "SanaWMStage1Model", + "SanaWMStage1Spec", + "Stage1CrossAttention", + "Stage1SelfAttention", +] diff --git a/integrations/sana/sana_wm/placeholder.py b/integrations/sana/sana_wm/upstream_reference.py similarity index 55% rename from integrations/sana/sana_wm/placeholder.py rename to integrations/sana/sana_wm/upstream_reference.py index ef46f1690..6b87504e2 100644 --- a/integrations/sana/sana_wm/placeholder.py +++ b/integrations/sana/sana_wm/upstream_reference.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Placeholder FlashDreams pipeline objects for the upstream SANA-WM runner.""" +"""Schema-only FlashDreams objects for the upstream SANA-WM reference runner.""" from __future__ import annotations @@ -32,27 +32,34 @@ @dataclass(kw_only=True) -class SanaWMPlaceholderTransformerConfig(TransformerConfig): - """Transformer config used only to satisfy the runner config schema.""" +class SanaWMUpstreamReferenceTransformerConfig(TransformerConfig): + """Transformer config used by the upstream-reference runner. - _target: type["SanaWMPlaceholderTransformer"] = field( - default_factory=lambda: SanaWMPlaceholderTransformer + This is intentionally not an executable SANA DiT adapter. It exists so the + CLI can expose the upstream reference path through the normal runner config + registry without pretending that upstream execution is the native path. + """ + + _target: type["SanaWMUpstreamReferenceTransformer"] = field( + default_factory=lambda: SanaWMUpstreamReferenceTransformer ) -class SanaWMPlaceholderTransformer(Transformer[TransformerAutoregressiveCache]): - """Non-executable transformer for the upstream-delegating SANA-WM runner.""" +class SanaWMUpstreamReferenceTransformer( + Transformer[TransformerAutoregressiveCache] +): + """Non-executable marker for the upstream-delegating reference runner.""" - def __init__(self, config: SanaWMPlaceholderTransformerConfig) -> None: + def __init__(self, config: SanaWMUpstreamReferenceTransformerConfig) -> None: super().__init__(config) self._dummy = nn.Parameter(torch.empty(0)) @property def latent_shape(self) -> tuple[int, ...]: - """Raise because SANA-WM generation is delegated to upstream Sana.""" + """Raise because this config marks the upstream reference path.""" raise RuntimeError( - "SANA-WM bidirectional uses the custom SanaWMRunner path; " - "the placeholder FlashDreams pipeline is not executable." + "This SANA-WM runner is an upstream reference harness. Use " + "sana-wm-bidirectional for native FlashDreams execution." ) def predict_flow( @@ -62,10 +69,10 @@ def predict_flow( cache: TransformerAutoregressiveCache, input: Any = None, ) -> Tensor: - """Raise because SANA-WM generation is delegated to upstream Sana.""" + """Raise because this config marks the upstream reference path.""" raise RuntimeError( - "SANA-WM bidirectional uses the custom SanaWMRunner path; " - "the placeholder FlashDreams pipeline is not executable." + "This SANA-WM runner delegates generation to upstream NVlabs/Sana " + "instead of executing through Transformer.predict_flow." ) def patchify_and_maybe_split_cp(self, x: Any) -> Any: @@ -75,3 +82,9 @@ def patchify_and_maybe_split_cp(self, x: Any) -> Any: def unpatchify_and_maybe_gather_cp(self, x: Tensor) -> Tensor: """Return ``x`` unchanged for schema-only construction.""" return x + + +__all__ = [ + "SanaWMUpstreamReferenceTransformer", + "SanaWMUpstreamReferenceTransformerConfig", +] diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index 08d5d5abf..62b13519b 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -5,16 +5,20 @@ SPDX-License-Identifier: Apache-2.0 # SANA-WM parity check -This directory is reserved for the opt-in SANA-WM upstream parity +This directory is reserved for the opt-in SANA-WM native-vs-upstream parity harness. It should follow the existing integration pattern: - clone or reuse a pinned NVlabs/Sana checkout; - install upstream-heavy dependencies in an isolated environment; -- run upstream `inference_video_scripts/wm/inference_sana_wm.py` and - `flashdreams-run sana-wm-bidirectional` on the same image, prompt, - camera, intrinsics, seed, and sampler settings; +- run upstream `inference_video_scripts/wm/inference_sana_wm.py` and the + native `flashdreams-run sana-wm-bidirectional` path on the same image, + prompt, camera, intrinsics, seed, and sampler settings; - compare decoded MP4 frames and report mean / max `|Delta|`. +The temporary FlashDreams-packaged upstream-reference runner is documented in +`../../docs/upstream_reference.md`. Keep those wrapper details out of the native +integration README. + Do not put the full checkpoint download, refiner execution, or MP4 generation path in `ci_cpu`. Add a bounded `ci_gpu` gate only if CI pre-provisions the required artefacts and the test skips cleanly when diff --git a/integrations/sana/tests/test_native_quant_cuda.py b/integrations/sana/tests/test_native_quant_cuda.py new file mode 100644 index 000000000..c32e2e65a --- /dev/null +++ b/integrations/sana/tests/test_native_quant_cuda.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CUDA smoke tests for SANA-WM native low-precision linear replacements.""" + +from __future__ import annotations + +import pytest +import torch + +from sana_wm.native_quant import TorchScaledMMFP4Linear, TorchScaledMMFP8Linear + +pytestmark = pytest.mark.ci_gpu + + +def _require_native_quant_cuda() -> None: + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for native quantization smokes") + missing = [ + name + for name in ("_scaled_mm", "float8_e4m3fn", "float4_e2m1fn_x2") + if not hasattr(torch, name) + ] + if missing: + pytest.skip(f"PyTorch lacks native quantization primitive(s): {missing}") + + +def test_native_fp8_linear_runs_scaled_mm_on_cuda() -> None: + """Exercise the TE-free E4M3 FP8 Linear replacement on CUDA.""" + _require_native_quant_cuda() + source = torch.nn.Linear(32, 64, bias=True, device="cuda", dtype=torch.bfloat16) + quantized = TorchScaledMMFP8Linear.from_linear( + source, + out_dtype=torch.bfloat16, + ) + inputs = torch.randn(4, 5, 32, device="cuda", dtype=torch.bfloat16) + + output = quantized(inputs) + + assert quantized.weight.shape == source.weight.shape + assert quantized.bias is not None + assert output.shape == (4, 5, 64) + assert output.dtype == torch.bfloat16 + assert torch.isfinite(output.float()).all() + + +def test_native_fp4_linear_runs_scaled_mm_on_blackwell() -> None: + """Exercise the TE-free E2M1 NVFP4 Linear replacement on Blackwell CUDA.""" + _require_native_quant_cuda() + major, minor = torch.cuda.get_device_capability() + if major < 10: + pytest.skip(f"NVFP4 requires Blackwell-class CUDA, got sm_{major}{minor}") + + source = torch.nn.Linear(32, 64, bias=True, device="cuda", dtype=torch.bfloat16) + quantized = TorchScaledMMFP4Linear.from_linear( + source, + out_dtype=torch.bfloat16, + ) + inputs = torch.randn(4, 5, 32, device="cuda", dtype=torch.bfloat16) + + output = quantized(inputs) + + assert quantized.weight.shape == source.weight.shape + assert quantized.bias is not None + assert output.shape == (4, 5, 64) + assert output.dtype == torch.bfloat16 + assert torch.isfinite(output.float()).all() diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index 6c7c032ed..692795bad 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""CPU-safe smoke tests for the SANA-WM runner shim.""" +"""CPU-safe smoke tests for the SANA-WM native and reference configs.""" from __future__ import annotations @@ -30,8 +30,12 @@ from sana_wm.config import ( PIPELINE_SANA_WM_BIDIRECTIONAL, + PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL, + PIPELINE_SANA_WM_UPSTREAM_REFERENCE, RUNNER_CONFIGS, RUNNER_SANA_WM_BIDIRECTIONAL, + RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL, + RUNNER_SANA_WM_UPSTREAM_REFERENCE, ) from sana_wm.constants import ( SANA_WM_CONFIG_PATH, @@ -42,9 +46,30 @@ SanaWMRunner, SanaWMRunnerConfig, _precision_env_updates, + _resolve_quant_backend, _temporary_environment, _validate_precision_request, ) +from sana_wm.native_diffusion import SanaWMDiffusionModelConfig +from sana_wm.native_transformer import ( + SanaWMNativeTransformerCache, + SanaWMNativeTransformerConfig, + SanaWMStage1Conditioning, + _avoid_degenerate_tile_tail, + _load_inference_config, +) +from sana_wm.native_quant import ( + TorchScaledMMFP4Linear, + TorchScaledMMFP8Linear, + replace_linear_with_torch_fp4, + replace_linear_with_torch_fp8, +) +from sana_wm.stage1_model import ( + SANA_WM_STAGE1_SPEC, + SanaWMStage1Model, + SanaWMStage1Spec, +) +from sana_wm.upstream_reference import SanaWMUpstreamReferenceTransformerConfig from flashdreams.infra.config import derive_config @@ -54,8 +79,12 @@ def test_runner_config_is_registered() -> None: - """Expose one SANA-WM runner keyed by its public slug.""" - assert RUNNER_CONFIGS == {"sana-wm-bidirectional": RUNNER_SANA_WM_BIDIRECTIONAL} + """Expose SANA-WM native and upstream-reference runner slugs.""" + assert RUNNER_CONFIGS == { + "sana-wm-bidirectional": RUNNER_SANA_WM_BIDIRECTIONAL, + "sana-wm-upstream-reference": RUNNER_SANA_WM_UPSTREAM_REFERENCE, + "sana-wm-native-bidirectional": RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL, + } def test_runner_name_mirrors_pipeline_name() -> None: @@ -63,11 +92,313 @@ def test_runner_name_mirrors_pipeline_name() -> None: assert RUNNER_SANA_WM_BIDIRECTIONAL.runner_name == ( PIPELINE_SANA_WM_BIDIRECTIONAL.name ) + assert RUNNER_SANA_WM_UPSTREAM_REFERENCE.runner_name == ( + PIPELINE_SANA_WM_UPSTREAM_REFERENCE.name + ) + assert RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL.runner_name == ( + PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL.name + ) def test_runner_has_description() -> None: """Provide non-empty CLI help text for the runner registry.""" assert RUNNER_SANA_WM_BIDIRECTIONAL.description.strip() + assert RUNNER_SANA_WM_UPSTREAM_REFERENCE.description.strip() + assert RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL.description.strip() + + +def test_reference_and_native_pipelines_are_distinct() -> None: + """Keep the upstream harness separate from the native FlashDreams target.""" + reference_transformer = ( + PIPELINE_SANA_WM_UPSTREAM_REFERENCE.diffusion_model.transformer + ) + main_transformer = PIPELINE_SANA_WM_BIDIRECTIONAL.diffusion_model.transformer + native_transformer = ( + PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL.diffusion_model.transformer + ) + + assert isinstance(reference_transformer, SanaWMUpstreamReferenceTransformerConfig) + assert isinstance( + PIPELINE_SANA_WM_BIDIRECTIONAL.diffusion_model, + SanaWMDiffusionModelConfig, + ) + assert isinstance(main_transformer, SanaWMNativeTransformerConfig) + assert isinstance(native_transformer, SanaWMNativeTransformerConfig) + assert RUNNER_SANA_WM_BIDIRECTIONAL.execution_backend == "native-flashdreams" + assert RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL.execution_backend == ( + "native-flashdreams" + ) + + +def test_native_transformer_contract_shape_and_conditioning_guard() -> None: + """Pin the native SANA-WM Stage-1 boundary to the public model layout.""" + transformer_cfg = SanaWMNativeTransformerConfig() + transformer = transformer_cfg.setup() + + assert transformer.latent_shape == (1, 128, 21, 22, 40) + cache = transformer.initialize_autoregressive_cache() + + with pytest.raises(RuntimeError, match="without conditioning"): + transformer.predict_flow( + noisy_latent=torch.empty(transformer.latent_shape), + timestep=torch.tensor(1000.0), + cache=cache, + ) + + +def test_native_inference_config_loads_yaml_without_upstream_types( + tmp_path: Path, +) -> None: + """Parse SANA-WM YAML with local config objects, not SANA pyrallis types.""" + config_path = tmp_path / "config.yaml" + config_path.write_text( + """ +model: + model: SanaMSVideoCamCtrl_1600M_P1_D20 + mixed_precision: bf16 + chunk_split_strategy: first_chunk_plus_one +vae: + vae_type: LTX2VAE_diffusers + vae_pretrained: hf://example/model + weight_dtype: bfloat16 + vae_latent_dim: 128 + vae_stride: [8, 32, 32] +text_encoder: + text_encoder_name: gemma-2-2b-it + model_max_length: 300 + chi_prompt: ["prefix"] +scheduler: + flow_shift: 9.95 + inference_flow_shift: 9.8 +""", + encoding="utf-8", + ) + + cfg = _load_inference_config(str(config_path)) + + assert cfg.model.model == "SanaMSVideoCamCtrl_1600M_P1_D20" + assert cfg.model.get("missing", "fallback") == "fallback" + assert cfg.vae.vae_stride == [8, 32, 32] + assert cfg.text_encoder.chi_prompt == ["prefix"] + assert cfg.scheduler.inference_flow_shift == 9.8 + assert cfg.work_dir == "" + + +def test_native_stage1_model_matches_checkpoint_schema() -> None: + """Pin the FlashDreams-owned Stage-1 module to the public checkpoint schema.""" + state = SanaWMStage1Model().state_dict() + + assert len(state) == 872 + assert tuple(state["x_embedder.proj.weight"].shape) == (2240, 128, 1, 1, 1) + assert tuple(state["raymap_embedder.proj.weight"].shape) == (2240, 3, 1, 1, 1) + assert tuple(state["plucker_embedder.proj.weight"].shape) == ( + 2240, + 48, + 1, + 1, + 1, + ) + assert tuple(state["pos_embed"].shape) == (1, 484, 2240) + assert tuple(state["y_embedder.y_embedding"].shape) == (300, 2304) + assert tuple(state["blocks.0.attn.qkv.weight"].shape) == (6720, 2240) + assert tuple(state["blocks.0.attn.conv_k.weight"].shape) == (2240, 1, 4) + assert tuple(state["blocks.0.cross_attn.kv_linear.weight"].shape) == ( + 4480, + 2240, + ) + assert tuple(state["blocks.19.mlp.t_conv.weight"].shape) == (2240, 2240, 3, 1) + assert tuple(state["final_layer.linear.weight"].shape) == (128, 2240) + + for block_index in range(SANA_WM_STAGE1_SPEC.depth): + has_gdn_conv = f"blocks.{block_index}.attn.conv_k.weight" in state + assert has_gdn_conv is SANA_WM_STAGE1_SPEC.block_uses_gdn(block_index) + + +def test_native_stage1_forward_preserves_latent_shape() -> None: + """Exercise the native Stage-1 forward path on a small CPU-safe spec.""" + spec = SanaWMStage1Spec( + latent_channels=4, + hidden_size=16, + text_dim=12, + timestep_dim=8, + depth=2, + num_heads=4, + head_dim=4, + max_text_length=5, + latent_grid_size=(2, 2), + mlp_ratio=1, + conv_kernel_size=3, + temporal_kernel_size=3, + plucker_channels=6, + raymap_channels=3, + softmax_every_n=2, + ) + model = SanaWMStage1Model(spec) + latents = torch.randn(1, 4, 3, 2, 2) + timesteps = torch.ones(1, 1, 3) + text = torch.randn(1, 1, 5, 12) + mask = torch.ones(1, 5) + plucker = torch.randn(1, 6, 3, 2, 2) + + out = model(latents, timesteps, text, mask=mask, chunk_plucker=plucker) + + assert out.shape == latents.shape + + +def test_native_transformer_releases_stage1_runtime() -> None: + """Free Stage-1-only modules and conditioning before decode/refine.""" + transformer = SanaWMNativeTransformerConfig().setup() + transformer.model = torch.nn.Linear(1, 1) + transformer.text_encoder = torch.nn.Linear(1, 1) + transformer.tokenizer = object() + transformer._model_built = True + transformer._text_encoder_built = True + transformer._stage1_quantized = True + transformer._streaming_prompt_cache[("prompt",)] = ( + torch.empty(1), + torch.empty(1), + torch.empty(1), + torch.empty(1), + ) + cache = SanaWMNativeTransformerCache( + conditioning=SanaWMStage1Conditioning( + condition=torch.empty(1), + uncondition=None, + model_kwargs={}, + first_latent=torch.empty(1), + latent_shape=(1, 1, 1, 1, 1), + cfg_scale=1.0, + flow_shift=1.0, + steps=1, + seed=0, + ) + ) + + transformer.release_stage1_runtime(cache) + + assert cache.conditioning is None + assert transformer.model is None + assert transformer.text_encoder is None + assert transformer.tokenizer is None + assert transformer._streaming_prompt_cache == {} + assert transformer._model_built is False + assert transformer._text_encoder_built is False + assert transformer._stage1_quantized is False + + +def test_native_vae_tiling_uses_low_memory_tiles() -> None: + """Keep native VAE decode on smaller tiles than upstream defaults.""" + + class DummyVAE: + def __init__(self) -> None: + self.calls: list[dict[str, int]] = [] + self.tile_sample_min_height = 512 + self.tile_sample_stride_height = 448 + self.tile_sample_min_width = 512 + self.tile_sample_stride_width = 448 + self.tile_sample_min_num_frames = 96 + self.tile_sample_stride_num_frames = 64 + self.use_framewise_encoding = False + self.use_framewise_decoding = False + self.spatial_compression_ratio = 32 + + def enable_tiling(self, **kwargs: int) -> None: + self.calls.append(kwargs) + + transformer = SanaWMNativeTransformerConfig().setup() + transformer.vae = DummyVAE() + + transformer._configure_vae_tiling() + + assert transformer.vae.calls == [ + { + "tile_sample_min_height": 256, + "tile_sample_stride_height": 192, + "tile_sample_min_width": 256, + "tile_sample_stride_width": 224, + "tile_sample_min_num_frames": 24, + "tile_sample_stride_num_frames": 8, + } + ] + assert transformer.vae.tile_sample_min_height == 256 + assert transformer.vae.tile_sample_stride_height == 192 + assert transformer.vae.tile_sample_min_width == 256 + assert transformer.vae.tile_sample_stride_width == 224 + assert transformer.vae.tile_sample_min_num_frames == 24 + assert transformer.vae.tile_sample_stride_num_frames == 8 + assert transformer.vae.use_framewise_encoding is True + assert transformer.vae.use_framewise_decoding is True + + +def test_native_decode_retries_vae_oom_with_smaller_tiles( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Retry native VAE decode with smaller tiles after a CUDA OOM.""" + + class DummyVAE: + def __init__(self) -> None: + self.tile_sample_min_height = 256 + self.tile_sample_stride_height = 224 + self.tile_sample_min_width = 256 + self.tile_sample_stride_width = 224 + self.tile_sample_min_num_frames = 24 + self.tile_sample_stride_num_frames = 8 + self.use_framewise_encoding = False + self.use_framewise_decoding = False + self.spatial_compression_ratio = 32 + + def to(self, *_args: object, **_kwargs: object) -> "DummyVAE": + return self + + def enable_tiling(self, **kwargs: int) -> None: + for name, value in kwargs.items(): + setattr(self, name, value) + + transformer = SanaWMNativeTransformerConfig().setup() + transformer.vae = DummyVAE() + transformer.vae_dtype = torch.float32 + monkeypatch.setattr(transformer, "_ensure_vae", lambda: None) + calls = 0 + + def fake_decode(_samples: torch.Tensor) -> torch.Tensor: + nonlocal calls + calls += 1 + if calls == 1: + raise torch.OutOfMemoryError("test OOM") + return torch.zeros((1, 3, 1, 2, 2), dtype=torch.float32) + + monkeypatch.setattr(transformer, "_vae_decode", fake_decode) + + video = transformer.decode_latents(torch.zeros((1, 4, 1, 1, 1))) + + assert calls == 2 + assert video.shape == (1, 2, 2, 3) + assert transformer.vae.tile_sample_min_height == 128 + assert transformer.vae.tile_sample_stride_height == 64 + assert transformer.vae.tile_sample_min_width == 128 + assert transformer.vae.tile_sample_stride_width == 64 + + +def test_native_vae_tiling_avoids_degenerate_latent_tails() -> None: + """Avoid last spatial VAE tiles with size one after compression.""" + assert ( + _avoid_degenerate_tile_tail( + sample_extent=704, + sample_tile_min=256, + sample_stride=224, + compression_ratio=32, + ) + == 192 + ) + assert ( + _avoid_degenerate_tile_tail( + sample_extent=1280, + sample_tile_min=128, + sample_stride=112, + compression_ratio=32, + ) + == 64 + ) def test_hf_defaults_point_at_bidirectional_release() -> None: @@ -99,12 +430,22 @@ def test_runner_setup_does_not_import_upstream_sana() -> None: def test_runner_config_type() -> None: """Keep the exported literal on the SANA-WM runner config subclass.""" assert isinstance(RUNNER_SANA_WM_BIDIRECTIONAL, SanaWMRunnerConfig) + assert isinstance(RUNNER_SANA_WM_UPSTREAM_REFERENCE, SanaWMRunnerConfig) + assert isinstance(RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL, SanaWMRunnerConfig) def test_runner_defaults_to_bf16_precision() -> None: """Keep the default path on upstream SANA-WM's BF16 config.""" assert RUNNER_SANA_WM_BIDIRECTIONAL.stage1_precision == "bf16" assert RUNNER_SANA_WM_BIDIRECTIONAL.refiner_precision == "bf16" + assert RUNNER_SANA_WM_BIDIRECTIONAL.quant_backend == "auto" + + +def test_auto_quant_backend_resolves_to_native_backend() -> None: + """Keep default quantization on the TE-free native backend.""" + assert _resolve_quant_backend("auto", ["fp4"]) == "native" + assert _resolve_quant_backend("auto", ["fp8", "fp4"]) == "native" + assert _resolve_quant_backend("auto", []) == "native" def test_bf16_precision_clears_upstream_quant_env( @@ -142,8 +483,8 @@ def test_fp8_precision_sets_upstream_quant_env() -> None: assert updates["SANA_WM_REFINER_QUANT"] == "fp8block" -def test_fp4_precision_sets_upstream_quant_env() -> None: - """Map FlashDreams fp4 fields to upstream SANA-WM NVFP4 env selectors.""" +def test_fp4_precision_sets_sana_quant_env() -> None: + """Map FlashDreams fp4 fields to SANA-WM NVFP4 env selectors.""" updates = _precision_env_updates( stage1_precision="fp4", refiner_precision="fp4", @@ -155,18 +496,30 @@ def test_fp4_precision_sets_upstream_quant_env() -> None: def test_quantized_precision_requires_cuda() -> None: - """Reject fp8/fp4 before importing upstream Sana on CPU-only devices.""" + """Reject fp8/fp4 before loading checkpoints on CPU-only devices.""" with pytest.raises(ValueError, match="requires a CUDA device"): _validate_precision_request( device=torch.device("cpu"), stage1_precision="fp8", refiner_precision="bf16", refiner_enabled=True, + quant_backend="upstream-te", ) +def test_bf16_precision_does_not_require_cuda_or_quant_backend() -> None: + """Keep BF16 as the TE-free compatibility path.""" + _validate_precision_request( + device=torch.device("cpu"), + stage1_precision="bf16", + refiner_precision="bf16", + refiner_enabled=True, + quant_backend="auto", + ) + + def test_fp8_precision_requires_hopper(monkeypatch: pytest.MonkeyPatch) -> None: - """Reject FP8 on pre-Hopper GPUs before checking Transformer Engine.""" + """Reject FP8 on pre-Hopper GPUs before checking quantization backend.""" monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (8, 9)) @@ -176,6 +529,7 @@ def test_fp8_precision_requires_hopper(monkeypatch: pytest.MonkeyPatch) -> None: stage1_precision="fp8", refiner_precision="bf16", refiner_enabled=True, + quant_backend="upstream-te", ) @@ -191,11 +545,12 @@ def test_fp8_precision_requires_cuda_129(monkeypatch: pytest.MonkeyPatch) -> Non stage1_precision="fp8", refiner_precision="bf16", refiner_enabled=True, + quant_backend="upstream-te", ) def test_fp4_precision_requires_blackwell(monkeypatch: pytest.MonkeyPatch) -> None: - """Reject NVFP4 on Hopper-class GPUs before checking Transformer Engine.""" + """Reject NVFP4 on Hopper-class GPUs before checking quantization backend.""" monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (9, 0)) @@ -205,9 +560,197 @@ def test_fp4_precision_requires_blackwell(monkeypatch: pytest.MonkeyPatch) -> No stage1_precision="fp4", refiner_precision="bf16", refiner_enabled=True, + quant_backend="upstream-te", ) +def test_torch_fp8_backend_skips_transformer_engine_cuda_129_gate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Allow native FP8 validation in the CUDA 12.8 env that blocks TE FP8.""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) + monkeypatch.setattr(torch.version, "cuda", "12.8") + monkeypatch.setattr(torch, "_scaled_mm", object(), raising=False) + monkeypatch.setattr(torch, "float8_e4m3fn", torch.uint8, raising=False) + + _validate_precision_request( + device=torch.device("cuda:0"), + stage1_precision="fp8", + refiner_precision="bf16", + refiner_enabled=True, + quant_backend="torch-fp8", + ) + + +def test_auto_fp8_backend_uses_native_primitives( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Route default FP8 to the TE-free native backend.""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) + monkeypatch.setattr(torch.version, "cuda", "12.8") + monkeypatch.setattr(torch, "_scaled_mm", object(), raising=False) + monkeypatch.setattr(torch, "float8_e4m3fn", torch.uint8, raising=False) + + _validate_precision_request( + device=torch.device("cuda:0"), + stage1_precision="fp8", + refiner_precision="bf16", + refiner_enabled=True, + quant_backend="auto", + ) + + +def test_native_runner_reaches_normal_input_validation() -> None: + """Do not silently delegate the native runner to the upstream harness.""" + cfg = derive_config( + RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL, + image_path=Path("missing.png"), + prompt="demo", + ) + runner = cfg.setup() + + with pytest.raises(FileNotFoundError, match="missing.png"): + runner.run() + + +def test_torch_fp8_backend_rejects_fp4(monkeypatch: pytest.MonkeyPatch) -> None: + """Do not silently route FP4 to a backend without an NVFP4 kernel.""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) + + with pytest.raises(ValueError, match="supports fp8 only"): + _validate_precision_request( + device=torch.device("cuda:0"), + stage1_precision="fp4", + refiner_precision="bf16", + refiner_enabled=True, + quant_backend="torch-fp8", + ) + + +def test_torch_fp4_backend_skips_transformer_engine(monkeypatch: pytest.MonkeyPatch) -> None: + """Allow native FP4 validation without importing Transformer Engine.""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) + monkeypatch.setattr(torch, "_scaled_mm", object(), raising=False) + monkeypatch.setattr(torch, "float4_e2m1fn_x2", torch.uint8, raising=False) + monkeypatch.setattr(torch, "float8_e4m3fn", torch.uint8, raising=False) + + _validate_precision_request( + device=torch.device("cuda:0"), + stage1_precision="fp4", + refiner_precision="bf16", + refiner_enabled=True, + quant_backend="torch-fp4", + ) + + +def test_torch_fp4_backend_rejects_fp8(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep precision-specific native backends explicit.""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) + + with pytest.raises(ValueError, match="supports fp4 only"): + _validate_precision_request( + device=torch.device("cuda:0"), + stage1_precision="fp8", + refiner_precision="bf16", + refiner_enabled=True, + quant_backend="torch-fp4", + ) + + +def test_native_backend_allows_mixed_fp8_fp4_without_cuda_129( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Allow mixed native FP8/FP4 requests in the CUDA 12.8 env that blocks TE FP8.""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) + monkeypatch.setattr(torch.version, "cuda", "12.8") + monkeypatch.setattr(torch, "_scaled_mm", object(), raising=False) + monkeypatch.setattr(torch, "float4_e2m1fn_x2", torch.uint8, raising=False) + monkeypatch.setattr(torch, "float8_e4m3fn", torch.uint8, raising=False) + + _validate_precision_request( + device=torch.device("cuda:0"), + stage1_precision="fp8", + refiner_precision="fp4", + refiner_enabled=True, + quant_backend="native", + ) + + +def test_torch_fp8_linear_replaces_matching_modules() -> None: + """Provide a TE-free replacement for eligible upstream Linear modules.""" + if not hasattr(torch, "float8_e4m3fn"): + pytest.skip("torch.float8_e4m3fn is required for native FP8 replacement") + + module = torch.nn.Sequential( + torch.nn.Linear(16, 32, bias=True), + torch.nn.Sequential(torch.nn.Linear(16, 32, bias=False)), + torch.nn.Linear(15, 32, bias=False), + ) + + converted, skipped = replace_linear_with_torch_fp8( + module, + recipe=None, + params_dtype=torch.bfloat16, + skip_patterns=(), + include_patterns=("^0$", "^1\\.0$", "^2$"), + ) + + assert converted == 2 + assert skipped == 1 + assert isinstance(module[0], TorchScaledMMFP8Linear) + assert isinstance(module[1][0], TorchScaledMMFP8Linear) + assert isinstance(module[2], torch.nn.Linear) + assert module[0].weight.shape == (32, 16) + assert module[0].bias is not None + + +def test_torch_fp4_linear_replaces_matching_modules( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Route eligible FP4 modules through the native replacement helper.""" + module = torch.nn.Sequential( + torch.nn.Linear(32, 64, bias=True), + torch.nn.Sequential(torch.nn.Linear(32, 64, bias=False)), + torch.nn.Linear(16, 64, bias=False), + ) + + @classmethod + def from_linear( + cls: type[TorchScaledMMFP4Linear], + source: torch.nn.Linear, + *, + out_dtype: torch.dtype, + ) -> TorchScaledMMFP4Linear: + del out_dtype + instance = cls.__new__(cls) + torch.nn.Module.__init__(instance) + instance.in_features = source.in_features + instance.out_features = source.out_features + return instance + + monkeypatch.setattr(TorchScaledMMFP4Linear, "from_linear", from_linear) + + converted, skipped = replace_linear_with_torch_fp4( + module, + recipe=None, + params_dtype=torch.bfloat16, + skip_patterns=(), + include_patterns=("^0$", "^1\\.0$", "^2$"), + ) + + assert converted == 2 + assert skipped == 1 + assert isinstance(module[0], TorchScaledMMFP4Linear) + assert isinstance(module[1][0], TorchScaledMMFP4Linear) + assert isinstance(module[2], torch.nn.Linear) + + def test_pyproject_entry_point_matches_runner_literal() -> None: """Keep the package entry point aligned with ``RUNNER_CONFIGS``.""" pyproject = tomllib.loads( @@ -216,5 +759,25 @@ def test_pyproject_entry_point_matches_runner_literal() -> None: entry_points = pyproject["project"]["entry-points"][ENTRY_POINT_GROUP] assert entry_points == { - "sana-wm-bidirectional": "sana_wm.config:RUNNER_SANA_WM_BIDIRECTIONAL" + "sana-wm-bidirectional": "sana_wm.config:RUNNER_SANA_WM_BIDIRECTIONAL", + "sana-wm-upstream-reference": ( + "sana_wm.config:RUNNER_SANA_WM_UPSTREAM_REFERENCE" + ), + "sana-wm-native-bidirectional": ( + "sana_wm.config:RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL" + ), } + + +def test_pyproject_excludes_upstream_diffusion_package() -> None: + """Do not install the vendored upstream SANA ``diffusion`` package.""" + pyproject = tomllib.loads( + Path("integrations/sana/pyproject.toml").read_text(encoding="utf-8") + ) + + packages = pyproject["tool"]["setuptools"]["packages"]["find"] + dependencies = set(pyproject["project"]["dependencies"]) + + assert packages["include"] == ["sana_wm*"] + assert "diffusers>=0.36" in dependencies + assert "transformers>=5.0,<6" in dependencies From 5c87bfddbce0b9c6b05d4e4121befcb53c4ab0e9 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 16 Jul 2026 18:36:20 -0700 Subject: [PATCH 07/64] Reduce native SANA-WM Stage-1 memory use Run classifier-free guidance as sequential negative and positive Stage-1 forwards instead of doubling the full model batch. This reduces peak activation memory for long native SANA-WM rollouts. Release large Stage-1 block temporaries earlier and make the VAE decode OOM retry reduce temporal tile size as well as spatial tile size. Add CPU coverage for sequential CFG kwargs and the lower-memory VAE retry configuration. Signed-off-by: Aidan Foster --- integrations/sana/sana_wm/native_diffusion.py | 72 ++++++++++++------- .../sana/sana_wm/native_transformer.py | 36 +++++++--- integrations/sana/sana_wm/stage1_model.py | 8 ++- integrations/sana/tests/test_smoke.py | 29 ++++++++ 4 files changed, 109 insertions(+), 36 deletions(-) diff --git a/integrations/sana/sana_wm/native_diffusion.py b/integrations/sana/sana_wm/native_diffusion.py index be719ffab..35270d4a0 100644 --- a/integrations/sana/sana_wm/native_diffusion.py +++ b/integrations/sana/sana_wm/native_diffusion.py @@ -134,14 +134,10 @@ def _sample_ltx_euler( condition_mask[:, :, int(frame_idx)] = 1 image_cond_noise_scale = max(image_cond_noise_scale, float(frame_weight)) - prompt_embeds = conditioning.condition - if do_cfg: - if conditioning.uncondition is None: - raise RuntimeError("CFG was requested without negative prompt embeds.") - prompt_embeds = torch.cat( - [conditioning.uncondition, conditioning.condition], - dim=0, - ) + condition_kwargs = _condition_model_kwargs(conditioning.model_kwargs) + uncondition_kwargs = _uncondition_model_kwargs(conditioning.model_kwargs) + if do_cfg and conditioning.uncondition is None: + raise RuntimeError("CFG was requested without negative prompt embeds.") init_latents = latents.clone() generator = torch.Generator(device=self.device).manual_seed(conditioning.seed) @@ -160,26 +156,36 @@ def _sample_ltx_euler( generator=generator, ) - condition_mask_input = ( - torch.cat([condition_mask] * 2) if do_cfg else condition_mask - ) - latent_model_input = torch.cat([latents] * 2) if do_cfg else latents - timestep = timestep_scalar.expand(condition_mask_input.shape).float() - timestep = torch.min(timestep, (1 - condition_mask_input) * 1000.0) - - noise_pred = self.transformer.predict_flow( - noisy_latent=latent_model_input, - timestep=timestep[:, :1, :, 0, 0], - cache=cache, - input=prompt_embeds, - ) - + timestep = timestep_scalar.expand(condition_mask.shape).float() + timestep = torch.min(timestep, (1 - condition_mask) * 1000.0) if do_cfg: - noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + assert conditioning.uncondition is not None + noise_pred_uncond = self.transformer.predict_flow( + noisy_latent=latents, + timestep=timestep[:, :1, :, 0, 0], + cache=cache, + input=conditioning.uncondition, + model_kwargs=uncondition_kwargs, + ) + noise_pred_text = self.transformer.predict_flow( + noisy_latent=latents, + timestep=timestep[:, :1, :, 0, 0], + cache=cache, + input=conditioning.condition, + model_kwargs=condition_kwargs, + ) noise_pred = noise_pred_uncond + conditioning.cfg_scale * ( noise_pred_text - noise_pred_uncond ) - timestep = timestep.chunk(2)[0] + del noise_pred_uncond, noise_pred_text + else: + noise_pred = self.transformer.predict_flow( + noisy_latent=latents, + timestep=timestep[:, :1, :, 0, 0], + cache=cache, + input=conditioning.condition, + model_kwargs=condition_kwargs, + ) latents_dtype = latents.dtype latents_shape = latents.shape @@ -202,6 +208,24 @@ def _sample_ltx_euler( return latents.detach() +def _condition_model_kwargs(model_kwargs: dict[str, object]) -> dict[str, object]: + """Return model kwargs for the positive prompt branch.""" + return { + key: value + for key, value in model_kwargs.items() + if key != "negative_mask" + } + + +def _uncondition_model_kwargs(model_kwargs: dict[str, object]) -> dict[str, object]: + """Return model kwargs for the negative prompt branch.""" + kwargs = _condition_model_kwargs(model_kwargs) + negative_mask = model_kwargs.get("negative_mask") + if negative_mask is not None: + kwargs["mask"] = negative_mask + return kwargs + + def _add_noise_to_conditioning_latents( *, t: Tensor, diff --git a/integrations/sana/sana_wm/native_transformer.py b/integrations/sana/sana_wm/native_transformer.py index d8be6efb1..59ebf538f 100644 --- a/integrations/sana/sana_wm/native_transformer.py +++ b/integrations/sana/sana_wm/native_transformer.py @@ -175,6 +175,12 @@ class SanaWMNativeTransformerConfig(TransformerConfig): vae_oom_retry_tile_sample_stride_height: int = 64 """Smaller pixel height tile stride for one VAE decode OOM retry.""" + vae_oom_retry_tile_sample_min_num_frames: int = 16 + """Smaller temporal tile size for one VAE decode OOM retry.""" + + vae_oom_retry_tile_sample_stride_num_frames: int = 4 + """Smaller temporal tile stride for one VAE decode OOM retry.""" + class SanaWMNativeTransformer(Transformer[SanaWMNativeTransformerCache]): """Native FlashDreams adapter for the SANA-WM Stage-1 model call.""" @@ -225,16 +231,18 @@ def predict_flow( timestep: Tensor, cache: SanaWMNativeTransformerCache, input: Any = None, + model_kwargs: dict[str, object] | None = None, ) -> Tensor: """Execute one SANA-WM DiT flow prediction.""" conditioning = _require_conditioning(cache) prompt_embeds = cast(Tensor, input) if input is not None else conditioning.condition + kwargs = conditioning.model_kwargs if model_kwargs is None else model_kwargs self._ensure_model() return self.model( noisy_latent, timestep, prompt_embeds, - **conditioning.model_kwargs, + **kwargs, ) def finalize_kv_cache( @@ -303,15 +311,11 @@ def prepare_conditioning( self.device, dtype=weight_dtype, ) + model_kwargs_extra: dict[str, object] = {} if cfg_scale > 1.0: - mask_cfg = torch.cat([neg_mask, cond_mask], dim=0) - raymap_cfg = torch.cat([raymap, raymap], dim=0) - chunk_plucker_cfg = torch.cat([chunk_plucker, chunk_plucker], dim=0) + model_kwargs_extra["negative_mask"] = neg_mask uncondition = neg else: - mask_cfg = cond_mask - raymap_cfg = raymap - chunk_plucker_cfg = chunk_plucker uncondition = None latent_t = (num_frames - 1) // int(cfg.vae.vae_stride[0]) + 1 @@ -327,9 +331,10 @@ def prepare_conditioning( ), "condition_frame_info": {0: 0.0}, }, - "mask": mask_cfg, - "camera_conditions": raymap_cfg, - "chunk_plucker": chunk_plucker_cfg, + "mask": cond_mask, + "camera_conditions": raymap, + "chunk_plucker": chunk_plucker, + **model_kwargs_extra, } if chunk_index is not None: model_kwargs["chunk_index"] = chunk_index @@ -386,11 +391,14 @@ def decode_latents(self, latents: Tensor) -> np.ndarray: gc.collect() logger.warning( "[sana-vae] decode OOM; retrying with smaller tiles " - "width={} stride_width={} height={} stride_height={}", + "width={} stride_width={} height={} stride_height={} " + "frames={} stride_frames={}", self.config.vae_oom_retry_tile_sample_min_width, self.config.vae_oom_retry_tile_sample_stride_width, self.config.vae_oom_retry_tile_sample_min_height, self.config.vae_oom_retry_tile_sample_stride_height, + self.config.vae_oom_retry_tile_sample_min_num_frames, + self.config.vae_oom_retry_tile_sample_stride_num_frames, ) self._configure_vae_tiling( tile_sample_min_width=( @@ -405,6 +413,12 @@ def decode_latents(self, latents: Tensor) -> np.ndarray: tile_sample_stride_height=( self.config.vae_oom_retry_tile_sample_stride_height ), + tile_sample_min_num_frames=( + self.config.vae_oom_retry_tile_sample_min_num_frames + ), + tile_sample_stride_num_frames=( + self.config.vae_oom_retry_tile_sample_stride_num_frames + ), ) if torch.cuda.is_available(): torch.cuda.empty_cache() diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index 74f21cff2..10d0733ec 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -403,8 +403,10 @@ def forward( x_norm = self.norm1(x).reshape(batch, frames, -1, channels) attn_in = _modulate(x_norm, shift_msa, scale_msa).reshape(batch, tokens, channels) + del x_norm, shift_msa, scale_msa attn_out = self.attn(attn_in, **kwargs).reshape(batch, frames, -1, channels) x = x + (gate_msa * attn_out).reshape(batch, tokens, channels) + del attn_in, attn_out, gate_msa if plucker_emb is not None: x = x + self.plucker_proj(plucker_emb) @@ -413,9 +415,13 @@ def forward( x_norm = self.norm2(x).reshape(batch, frames, -1, channels) mlp_in = _modulate(x_norm, shift_mlp, scale_mlp).reshape(batch, tokens, channels) + del x_norm, shift_mlp, scale_mlp mlp_out = self.mlp(mlp_in, frames=frames, height=height, width=width) + del mlp_in mlp_out = mlp_out.reshape(batch, frames, -1, channels) - return x + (gate_mlp * mlp_out).reshape(batch, tokens, channels) + out = x + (gate_mlp * mlp_out).reshape(batch, tokens, channels) + del gate_mlp, mlp_out + return out class SanaWMStage1Model(nn.Module): diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index 692795bad..282f12229 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -50,6 +50,10 @@ _temporary_environment, _validate_precision_request, ) +from sana_wm.native_diffusion import ( + _condition_model_kwargs, + _uncondition_model_kwargs, +) from sana_wm.native_diffusion import SanaWMDiffusionModelConfig from sana_wm.native_transformer import ( SanaWMNativeTransformerCache, @@ -377,6 +381,31 @@ def fake_decode(_samples: torch.Tensor) -> torch.Tensor: assert transformer.vae.tile_sample_stride_height == 64 assert transformer.vae.tile_sample_min_width == 128 assert transformer.vae.tile_sample_stride_width == 64 + assert transformer.vae.tile_sample_min_num_frames == 16 + assert transformer.vae.tile_sample_stride_num_frames == 4 + + +def test_cfg_model_kwargs_do_not_duplicate_camera_tensors() -> None: + """Run CFG as sequential branches instead of doubling Stage-1 activations.""" + camera = torch.zeros((1, 3, 3, 2, 2)) + cond_mask = torch.ones((1, 8)) + neg_mask = torch.zeros((1, 8)) + kwargs = { + "mask": cond_mask, + "negative_mask": neg_mask, + "camera_conditions": camera, + "chunk_plucker": torch.zeros((1, 6, 3, 2, 2)), + } + + cond_kwargs = _condition_model_kwargs(kwargs) + neg_kwargs = _uncondition_model_kwargs(kwargs) + + assert cond_kwargs["mask"] is cond_mask + assert neg_kwargs["mask"] is neg_mask + assert cond_kwargs["camera_conditions"] is camera + assert neg_kwargs["camera_conditions"] is camera + assert "negative_mask" not in cond_kwargs + assert "negative_mask" not in neg_kwargs def test_native_vae_tiling_avoids_degenerate_latent_tails() -> None: From c00b3c58f357172a3d3f240f7da2e0ce1e19bbf9 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 16 Jul 2026 18:43:19 -0700 Subject: [PATCH 08/64] Fix SANA-WM VAE retry temporal stride Keep retry temporal tiling compatible with LTX2's temporal compression ratio so diffusers does not derive a zero latent stride during OOM retry. Update the SANA smoke coverage for the clamped retry stride and sync the uv lock metadata for the integration dependencies declared by the native SANA package. Signed-off-by: Aidan Foster --- .../sana/sana_wm/native_transformer.py | 6 ++- integrations/sana/tests/test_smoke.py | 3 +- uv.lock | 40 +++++++++---------- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/integrations/sana/sana_wm/native_transformer.py b/integrations/sana/sana_wm/native_transformer.py index 59ebf538f..2b9775f77 100644 --- a/integrations/sana/sana_wm/native_transformer.py +++ b/integrations/sana/sana_wm/native_transformer.py @@ -178,7 +178,7 @@ class SanaWMNativeTransformerConfig(TransformerConfig): vae_oom_retry_tile_sample_min_num_frames: int = 16 """Smaller temporal tile size for one VAE decode OOM retry.""" - vae_oom_retry_tile_sample_stride_num_frames: int = 4 + vae_oom_retry_tile_sample_stride_num_frames: int = 8 """Smaller temporal tile stride for one VAE decode OOM retry.""" @@ -666,6 +666,10 @@ def _configure_vae_tiling( tile_sample_stride_num_frames or self.config.vae_tile_sample_stride_num_frames ) + temporal_ratio = int(getattr(vae, "temporal_compression_ratio", 1)) + if temporal_ratio > 1: + min_frames = max(min_frames, temporal_ratio) + stride_frames = max(stride_frames, temporal_ratio) kwargs = { "tile_sample_min_height": min_height, "tile_sample_stride_height": stride_height, diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index 282f12229..ae960fd71 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -350,6 +350,7 @@ def __init__(self) -> None: self.use_framewise_encoding = False self.use_framewise_decoding = False self.spatial_compression_ratio = 32 + self.temporal_compression_ratio = 8 def to(self, *_args: object, **_kwargs: object) -> "DummyVAE": return self @@ -382,7 +383,7 @@ def fake_decode(_samples: torch.Tensor) -> torch.Tensor: assert transformer.vae.tile_sample_min_width == 128 assert transformer.vae.tile_sample_stride_width == 64 assert transformer.vae.tile_sample_min_num_frames == 16 - assert transformer.vae.tile_sample_stride_num_frames == 4 + assert transformer.vae.tile_sample_stride_num_frames == 8 def test_cfg_model_kwargs_do_not_duplicate_camera_tensors() -> None: diff --git a/uv.lock b/uv.lock index 55413b69e..407fb184b 100644 --- a/uv.lock +++ b/uv.lock @@ -1573,7 +1573,16 @@ name = "flashdreams-sana-wm" version = "0.1.0" source = { editable = "integrations/sana" } dependencies = [ + { name = "diffusers" }, { name = "flashdreams" }, + { name = "imageio", extra = ["ffmpeg"] }, + { name = "pillow" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torchvision", version = "0.26.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform != 'win32' and extra == 'group-11-flashdreams-cuda12') or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "torchvision", version = "0.27.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'win32' and extra == 'extra-11-flashdreams-dev') or (sys_platform != 'win32' and extra != 'group-11-flashdreams-cuda12') or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "torchvision", version = "0.27.1+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "sys_platform == 'win32' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "transformers" }, ] [package.optional-dependencies] @@ -1583,8 +1592,15 @@ dev = [ [package.metadata] requires-dist = [ + { name = "diffusers", specifier = ">=0.36" }, { name = "flashdreams", editable = "flashdreams" }, + { name = "imageio", extras = ["ffmpeg"], specifier = ">=2.31" }, + { name = "pillow", specifier = ">=10" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "safetensors", specifier = ">=0.5" }, + { name = "torchvision", specifier = ">=0.26" }, + { name = "transformers", specifier = ">=5.0,<6" }, ] provides-extras = ["dev"] @@ -3221,26 +3237,10 @@ name = "numpy" version = "2.4.6" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32' and extra != 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13'", - "python_full_version == '3.11.*' and sys_platform == 'win32' and extra != 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13'", - "python_full_version >= '3.14' and sys_platform != 'win32' and extra != 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13'", - "python_full_version == '3.13.*' and sys_platform != 'win32' and extra != 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13'", - "python_full_version == '3.12.*' and sys_platform != 'win32' and extra != 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13'", - "python_full_version == '3.11.*' and sys_platform != 'win32' and extra != 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13'", - "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32' and extra != 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", - "python_full_version == '3.11.*' and sys_platform == 'win32' and extra != 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", - "python_full_version >= '3.14' and sys_platform != 'win32' and extra != 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'win32' and extra != 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", - "python_full_version == '3.11.*' and sys_platform != 'win32' and extra != 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", - "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32' and extra != 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", - "python_full_version == '3.11.*' and sys_platform == 'win32' and extra != 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", - "python_full_version >= '3.14' and sys_platform != 'win32' and extra != 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", - "python_full_version == '3.13.*' and sys_platform != 'win32' and extra != 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", - "python_full_version == '3.12.*' and sys_platform != 'win32' and extra != 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", - "python_full_version == '3.11.*' and sys_platform != 'win32' and extra != 'group-11-flashdreams-cuda12' and extra != 'group-11-flashdreams-cuda13'", + "python_full_version >= '3.12' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version >= '3.12' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'win32'", ] sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } wheels = [ From 7de18d69548dd7f98651c62fd49fb1cae6a23563 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 16 Jul 2026 18:48:36 -0700 Subject: [PATCH 09/64] Run native SANA VAE decode in inference mode Wrap native SANA conditioning and VAE decode in torch.inference_mode so full-length runs do not retain autograd activations during first-frame encode, prompt encode, or LTX2 VAE decode. Extend the VAE OOM retry smoke test to assert both decode attempts run under inference mode. Signed-off-by: Aidan Foster --- integrations/sana/sana_wm/native_transformer.py | 2 ++ integrations/sana/tests/test_smoke.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/integrations/sana/sana_wm/native_transformer.py b/integrations/sana/sana_wm/native_transformer.py index 2b9775f77..b847761dc 100644 --- a/integrations/sana/sana_wm/native_transformer.py +++ b/integrations/sana/sana_wm/native_transformer.py @@ -263,6 +263,7 @@ def unpatchify_and_maybe_gather_cp(self, x: Tensor) -> Tensor: """SANA-WM Stage-1 latents are already in model layout.""" return x + @torch.inference_mode() def prepare_conditioning( self, *, @@ -369,6 +370,7 @@ def initial_latents(self, conditioning: SanaWMStage1Conditioning) -> Tensor: latents[:, :, :1] = conditioning.first_latent return latents + @torch.inference_mode() def decode_latents(self, latents: Tensor) -> np.ndarray: """Decode SANA-WM VAE latents to ``uint8`` HWC video.""" self._ensure_vae() diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index ae960fd71..2b839d7d7 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -364,10 +364,12 @@ def enable_tiling(self, **kwargs: int) -> None: transformer.vae_dtype = torch.float32 monkeypatch.setattr(transformer, "_ensure_vae", lambda: None) calls = 0 + inference_modes: list[bool] = [] def fake_decode(_samples: torch.Tensor) -> torch.Tensor: nonlocal calls calls += 1 + inference_modes.append(torch.is_inference_mode_enabled()) if calls == 1: raise torch.OutOfMemoryError("test OOM") return torch.zeros((1, 3, 1, 2, 2), dtype=torch.float32) @@ -377,6 +379,7 @@ def fake_decode(_samples: torch.Tensor) -> torch.Tensor: video = transformer.decode_latents(torch.zeros((1, 4, 1, 1, 1))) assert calls == 2 + assert inference_modes == [True, True] assert video.shape == (1, 2, 2, 3) assert transformer.vae.tile_sample_min_height == 128 assert transformer.vae.tile_sample_stride_height == 64 From 46b6e660bac13ae862d4c4aa722ad4b614a59543 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 16 Jul 2026 19:52:32 -0700 Subject: [PATCH 10/64] Implement native SANA-WM refinement Replace the Stage-1 placeholder attention path with checkpoint-compatible GDN, softmax, camera UCPE, output-gate, and FFN behavior. Add a native LTX-2 refiner that loads diffusers transformer/connectors and Gemma text conditioning directly from the SANA-WM checkpoint, then wire the native runner to refine by default before VAE decode. Keep BF16/FP8/FP4 on FlashDreams/PyTorch native paths, add accelerate for low-memory refiner loading, update docs, and cover the new refiner construction plus latent packing in CPU tests. Validation: uv run --package flashdreams-sana-wm pytest integrations/sana/tests/test_smoke.py integrations/sana/tests/test_camera.py; uv run --package flashdreams-sana-wm flashdreams-run --no-instantiate sana-wm-bidirectional; no-instantiate checks for fp8, fp4, and --no-refiner True. --- integrations/sana/README.md | 40 +- integrations/sana/pyproject.toml | 1 + integrations/sana/sana_wm/config.py | 4 +- integrations/sana/sana_wm/native_refiner.py | 601 +++++++++++++ .../sana/sana_wm/native_transformer.py | 30 +- integrations/sana/sana_wm/runner.py | 45 +- integrations/sana/sana_wm/stage1_model.py | 791 +++++++++++++++++- integrations/sana/tests/test_smoke.py | 157 +++- uv.lock | 23 + 9 files changed, 1628 insertions(+), 64 deletions(-) create mode 100644 integrations/sana/sana_wm/native_refiner.py diff --git a/integrations/sana/README.md b/integrations/sana/README.md index aa0500e82..db4830004 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -13,19 +13,18 @@ The `sana-wm-bidirectional` runner is the native FlashDreams path. It uses FlashDreams config, runner, pipeline, diffusion-model, scheduler, transformer, camera-conditioning, VAE decode, and output-writing boundaries. The Stage-1 DiT module is implemented in this package and loads the public SANA-WM -checkpoint directly; it does not import or install a vendored upstream +checkpoint directly; it does not import or install the NVlabs/Sana `diffusion` package. Current native scope: | component | status | | --- | --- | -| Stage-1 BF16 | Native FlashDreams path, GPU smoke-tested. | -| Stage-1 FP8 | Native PyTorch `_scaled_mm` backend, GPU smoke-tested. | -| Stage-1 FP4 | Native Triton quantization plus PyTorch `_scaled_mm`, GPU smoke-tested. | +| Stage-1 BF16 | Native FlashDreams DiT execution. | +| Stage-1 FP8 | Native PyTorch `_scaled_mm` backend. | +| Stage-1 FP4 | Native Triton quantization plus PyTorch `_scaled_mm`. | | VAE decode | Native direct `diffusers` LTX2 VAE use with low-memory tiling. | -| LTX-2 refiner | Not native yet; the native runner requires `--no-refiner True`. | -| GDN attention parity | Not complete; smoke-tested, not quality-parity validated. | +| LTX-2 refiner | Native direct `diffusers` LTX-2 transformer and Gemma connector use. | The separate `sana-wm-upstream-reference` runner is documented only in `docs/upstream_reference.md`. Keep reference-harness details out of this @@ -35,7 +34,7 @@ native integration README. | slug | description | | --- | --- | -| `sana-wm-bidirectional` | Native FlashDreams SANA-WM Stage-1 runner. | +| `sana-wm-bidirectional` | Native FlashDreams SANA-WM Stage-1 + LTX-2 refiner runner. | | `sana-wm-native-bidirectional` | Alias for the native FlashDreams runner. | The FlashDreams package is named `sana_wm` rather than `sana` so it does not @@ -50,12 +49,6 @@ project's GPU runtime dependencies: uv sync --package flashdreams-sana-wm --extra dev ``` -For an existing environment, install the local packages directly: - -```bash -python -m pip install -e flashdreams -e integrations/sana -``` - The native runner does not require a local NVlabs/Sana checkout. The checkout is only useful as a source of demo assets such as `asset/sana_wm/demo_0.png`, `demo_0.txt`, `demo_0_pose.npy`, and @@ -63,18 +56,17 @@ is only useful as a source of demo assets such as ## Run -The native runner currently requires explicit intrinsics and no refiner: +The native runner requires explicit intrinsics: ```bash PYTORCH_ALLOC_CONF=expandable_segments:True \ -flashdreams-run sana-wm-bidirectional \ +uv run flashdreams-run sana-wm-bidirectional \ --image-path ../Sana/asset/sana_wm/demo_0.png \ --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ --num-frames 161 \ - --output-dir outputs/sana_wm_native_bf16 \ - --no-refiner True + --output-dir outputs/sana_wm_native_bf16 ``` Expected output: @@ -83,6 +75,8 @@ Expected output: outputs/sana_wm_native_bf16/sana_wm_generated.mp4 ``` +For Stage-1-only diagnostics, add `--no-refiner True`. + ## FP8 and FP4 The runner defaults to BF16. Quantized Stage-1 inference is opt-in and uses @@ -99,30 +93,30 @@ FP8 smoke: ```bash PYTORCH_ALLOC_CONF=expandable_segments:True \ -flashdreams-run sana-wm-bidirectional \ +uv run flashdreams-run sana-wm-bidirectional \ --image-path ../Sana/asset/sana_wm/demo_0.png \ --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ --num-frames 161 \ --output-dir outputs/sana_wm_native_fp8 \ - --no-refiner True \ - --stage1-precision fp8 + --stage1-precision fp8 \ + --refiner-precision fp8 ``` FP4 smoke: ```bash PYTORCH_ALLOC_CONF=expandable_segments:True \ -flashdreams-run sana-wm-bidirectional \ +uv run flashdreams-run sana-wm-bidirectional \ --image-path ../Sana/asset/sana_wm/demo_0.png \ --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ --num-frames 161 \ --output-dir outputs/sana_wm_native_fp4 \ - --no-refiner True \ - --stage1-precision fp4 + --stage1-precision fp4 \ + --refiner-precision fp4 ``` ## Tests diff --git a/integrations/sana/pyproject.toml b/integrations/sana/pyproject.toml index df7ebf34b..780bf446b 100644 --- a/integrations/sana/pyproject.toml +++ b/integrations/sana/pyproject.toml @@ -24,6 +24,7 @@ description = "SANA-WM bidirectional world-model integration for flashdreams." readme = "README.md" requires-python = ">=3.10" dependencies = [ + "accelerate>=1.0", "diffusers>=0.36", "flashdreams", "imageio[ffmpeg]>=2.31", diff --git a/integrations/sana/sana_wm/config.py b/integrations/sana/sana_wm/config.py index faa9d8aa2..73734b634 100644 --- a/integrations/sana/sana_wm/config.py +++ b/integrations/sana/sana_wm/config.py @@ -73,11 +73,10 @@ def _reference_pipeline(name: str) -> StreamInferencePipelineConfig: runner_name=PIPELINE_SANA_WM_BIDIRECTIONAL.name, description=( "SANA-WM bidirectional I2V native FlashDreams runner " - "(Stage-1 DiT)." + "(Stage-1 DiT + LTX-2 refiner)." ), pipeline=PIPELINE_SANA_WM_BIDIRECTIONAL, execution_backend="native-flashdreams", - no_refiner=True, ) """SANA-WM native FlashDreams runner config.""" @@ -86,7 +85,6 @@ def _reference_pipeline(name: str) -> StreamInferencePipelineConfig: description="SANA-WM native FlashDreams pipeline alias.", pipeline=PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL, execution_backend="native-flashdreams", - no_refiner=True, ) """Backward-compatible native SANA-WM FlashDreams runner alias.""" diff --git a/integrations/sana/sana_wm/native_refiner.py b/integrations/sana/sana_wm/native_refiner.py new file mode 100644 index 000000000..084940608 --- /dev/null +++ b/integrations/sana/sana_wm/native_refiner.py @@ -0,0 +1,601 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native LTX-2 refiner used by the SANA-WM integration.""" + +from __future__ import annotations + +import gc +from pathlib import Path +from typing import Literal + +import torch +import torch.nn as nn +import torch.nn.functional as F +from loguru import logger +from torch import Tensor + +from sana_wm.native_quant import ( + TorchScaledMMFP4Recipe, + TorchScaledMMFP8Recipe, + replace_linear_with_native_quant, +) + +Precision = Literal["bf16", "fp8", "fp4"] +QuantBackend = Literal["auto", "upstream-te", "torch-fp8", "torch-fp4", "native"] + +_REFINER_QUANT_SKIP_DEFAULTS = ( + r"^proj_in$", + r"^proj_out$", + r"(^|\.)audio_", + r"audio_to_video", + r"video_to_audio", + r"av_cross_attn", + r"caption_projection", + r"time_embed", +) + + +class SanaWMLTX2Refiner(nn.Module): + """Run the SANA-WM LTX-2 latent refiner without importing a Sana checkout.""" + + def __init__( + self, + *, + refiner_root: str | Path, + gemma_root: str | Path, + dtype: torch.dtype, + device: torch.device | str, + precision: Precision = "bf16", + quant_backend: QuantBackend = "native", + text_max_sequence_length: int = 1024, + ) -> None: + super().__init__() + self.refiner_root = Path(refiner_root) + self.gemma_root = Path(gemma_root) + self.dtype = dtype + self.device = torch.device(device) + self.precision = precision + self.quant_backend = quant_backend + self.text_max_sequence_length = int(text_max_sequence_length) + self._quantized = False + self.transformer, self.connectors = self._load_diffusers_components() + + def refine_latents( + self, + sana_latent: Tensor, + prompt: str, + *, + fps: float, + sink_size: int = 1, + seed: int = 42, + progress: bool = True, + block_size: int | None = None, + kv_max_frames: int = 11, + sigmas: tuple[float, ...] = (0.909375, 0.725, 0.421875, 0.0), + ) -> Tensor: + """Refine Stage-1 VAE latents with the sink-bidirectional LTX-2 path.""" + del kv_max_frames + if block_size is not None: + logger.warning( + "[refiner] --refiner-block-size={} requested; running the " + "sink-bidirectional LTX-2 path used by the bidirectional model.", + block_size, + ) + if sana_latent.shape[2] <= sink_size: + raise ValueError( + f"Stage-1 latent has {sana_latent.shape[2]} frames but " + f"sink_size={sink_size}." + ) + + prompt_embeds, prompt_attention_mask = self._encode_prompt(prompt) + self.transformer.to(self.device) + self.transformer.eval() + self._prepare_quantization() + + z = sana_latent.to(device=self.device, dtype=self.dtype) + sigmas_t = torch.tensor(sigmas, dtype=torch.float32, device=self.device) + start_sigma = float(sigmas_t[0]) + sink = z[:, :, :sink_size].contiguous() + current = z[:, :, sink_size:].contiguous() + generator = torch.Generator(device=self.device).manual_seed(int(seed)) + eps = torch.randn( + current.shape, + generator=generator, + device=self.device, + dtype=self.dtype, + ) + noisy = (1.0 - start_sigma) * current + start_sigma * eps + + iterator = range(len(sigmas_t) - 1) + if progress: + from tqdm.auto import tqdm + + iterator = tqdm(iterator, desc="refiner", unit="step") + + for step_index in iterator: + sigma = sigmas_t[step_index] + denoised = self._predict_current_x0( + sink=sink, + noisy_current=noisy, + prompt_embeds=prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + sigma=sigma, + fps=fps, + ) + noisy_tokens = _pack_latents( + noisy, + patch_size=self.transformer.config.patch_size, + patch_size_t=self.transformer.config.patch_size_t, + ) + velocity = (noisy_tokens.float() - denoised.float()) / sigma.float() + next_tokens = ( + noisy_tokens.float() + + velocity * (sigmas_t[step_index + 1] - sigma).float() + ) + noisy = _unpack_latents( + next_tokens.to(self.dtype), + num_frames=noisy.shape[2], + height=noisy.shape[3], + width=noisy.shape[4], + patch_size=self.transformer.config.patch_size, + patch_size_t=self.transformer.config.patch_size_t, + ) + + return torch.cat([sink, noisy], dim=2) + + def _load_diffusers_components(self) -> tuple[nn.Module, nn.Module]: + from diffusers.models.transformers.transformer_ltx2 import ( + LTX2VideoTransformer3DModel, + ) + from diffusers.pipelines.ltx2 import LTX2TextConnectors + + transformer = LTX2VideoTransformer3DModel.from_pretrained( + self.refiner_root, + subfolder="transformer", + torch_dtype=self.dtype, + ).eval() + connectors = LTX2TextConnectors.from_pretrained( + self.refiner_root, + subfolder="connectors", + torch_dtype=self.dtype, + ).eval() + return transformer, connectors + + def _prepare_quantization(self) -> None: + if self._quantized or self.precision == "bf16": + return + if self.quant_backend == "upstream-te": + raise ValueError( + "Native SANA-WM refiner execution does not use Transformer Engine; " + "select --quant-backend native, torch-fp8, or torch-fp4." + ) + recipe = ( + TorchScaledMMFP8Recipe() + if self.precision == "fp8" + else TorchScaledMMFP4Recipe() + ) + converted, skipped = replace_linear_with_native_quant( + self.transformer, + recipe=recipe, + params_dtype=self.dtype, + skip_patterns=_REFINER_QUANT_SKIP_DEFAULTS, + ) + if converted <= 0: + raise RuntimeError( + f"SANA-WM native refiner {self.precision} converted no Linear " + f"layers; skipped={skipped}." + ) + self._quantized = True + + @torch.inference_mode() + def _encode_prompt(self, prompt: str) -> tuple[Tensor, Tensor]: + from transformers import AutoTokenizer, Gemma3ForConditionalGeneration + + tokenizer = AutoTokenizer.from_pretrained(self.gemma_root) + tokenizer.padding_side = "left" + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + text_inputs = tokenizer( + [prompt.strip()], + padding="max_length", + max_length=self.text_max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + input_ids = text_inputs.input_ids.to(self.device) + attention_mask = text_inputs.attention_mask.to(self.device) + + text_encoder = Gemma3ForConditionalGeneration.from_pretrained( + self.gemma_root, + torch_dtype=self.dtype, + low_cpu_mem_usage=True, + ).eval() + text_encoder.to(self.device) + text_backbone = getattr(text_encoder, "model", text_encoder) + outputs = text_backbone( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + ) + hidden_states = torch.stack(outputs.hidden_states, dim=-1) + sequence_lengths = attention_mask.sum(dim=-1) + del text_encoder, text_backbone, outputs + _empty_cuda_cache() + + prompt_embeds = _pack_text_embeds( + hidden_states, + sequence_lengths, + device=self.device, + padding_side=tokenizer.padding_side, + ).to(dtype=self.dtype) + del hidden_states + _empty_cuda_cache() + + self.connectors.to(self.device) + connector_prompt_embeds, _, connector_attention_mask = self.connectors( + prompt_embeds, + attention_mask, + ) + self.connectors.to("cpu") + del prompt_embeds, attention_mask + _empty_cuda_cache() + return ( + connector_prompt_embeds.to(device=self.device, dtype=self.dtype), + connector_attention_mask.to(device=self.device), + ) + + def _predict_current_x0( + self, + *, + sink: Tensor, + noisy_current: Tensor, + prompt_embeds: Tensor, + prompt_attention_mask: Tensor, + sigma: Tensor, + fps: float, + ) -> Tensor: + full_latent = torch.cat([sink, noisy_current], dim=2) + batch_size, _, num_frames, height, width = full_latent.shape + latent_tokens = _pack_latents( + full_latent, + patch_size=self.transformer.config.patch_size, + patch_size_t=self.transformer.config.patch_size_t, + ) + n_context_tokens = _pack_latents( + sink, + patch_size=self.transformer.config.patch_size, + patch_size_t=self.transformer.config.patch_size_t, + ).shape[1] + + raw_timestep = torch.zeros( + batch_size, + latent_tokens.shape[1], + 1, + dtype=torch.float32, + device=self.device, + ) + raw_timestep[:, n_context_tokens:, 0] = sigma.float() + model_timestep = ( + raw_timestep.squeeze(-1) + * float(self.transformer.config.timestep_scale_multiplier) + ) + + velocity = self._forward_video_only( + hidden_states=latent_tokens, + encoder_hidden_states=prompt_embeds, + timestep=model_timestep, + encoder_attention_mask=prompt_attention_mask, + num_frames=num_frames, + height=height, + width=width, + fps=fps, + n_context_tokens=n_context_tokens, + ) + denoised = latent_tokens.float() - velocity.float() * raw_timestep + return denoised[:, n_context_tokens:, :].to(self.dtype) + + def _forward_video_only( + self, + *, + hidden_states: Tensor, + encoder_hidden_states: Tensor, + timestep: Tensor, + encoder_attention_mask: Tensor | None, + num_frames: int, + height: int, + width: int, + fps: float, + n_context_tokens: int, + ) -> Tensor: + transformer = self.transformer + batch_size = hidden_states.size(0) + encoder_attention_mask = _prepare_encoder_attention_mask( + encoder_attention_mask, + hidden_states.dtype, + ) + video_coords = transformer.rope.prepare_video_coords( + batch_size, + num_frames, + height, + width, + hidden_states.device, + fps=fps, + ) + video_rotary_emb = transformer.rope(video_coords, device=hidden_states.device) + + hidden_states = transformer.proj_in(hidden_states) + temb, embedded_timestep = transformer.time_embed( + timestep.flatten(), + batch_size=batch_size, + hidden_dtype=hidden_states.dtype, + ) + temb = temb.view(batch_size, -1, temb.size(-1)) + embedded_timestep = embedded_timestep.view( + batch_size, + -1, + embedded_timestep.size(-1), + ) + + encoder_hidden_states = transformer.caption_projection(encoder_hidden_states) + encoder_hidden_states = encoder_hidden_states.view( + batch_size, + -1, + hidden_states.size(-1), + ) + + for block in transformer.transformer_blocks: + hidden_states = _forward_video_block( + block=block, + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + video_rotary_emb=video_rotary_emb, + encoder_attention_mask=encoder_attention_mask, + n_context_tokens=n_context_tokens, + ) + + scale_shift_values = ( + transformer.scale_shift_table[None, None] + + embedded_timestep[:, :, None] + ) + shift, scale = scale_shift_values[:, :, 0], scale_shift_values[:, :, 1] + hidden_states = transformer.norm_out(hidden_states) + hidden_states = hidden_states * (1 + scale) + shift + return transformer.proj_out(hidden_states) + + +def _forward_video_block( + *, + block: nn.Module, + hidden_states: Tensor, + encoder_hidden_states: Tensor, + temb: Tensor, + video_rotary_emb: tuple[Tensor, Tensor], + encoder_attention_mask: Tensor | None, + n_context_tokens: int, +) -> Tensor: + batch_size = hidden_states.size(0) + norm_hidden_states = block.norm1(hidden_states) + num_ada_params = block.scale_shift_table.shape[0] + ada_values = block.scale_shift_table[None, None].to(temb.device) + temb.reshape( + batch_size, + temb.size(1), + num_ada_params, + -1, + ) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + ada_values.unbind(dim=2) + ) + norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa + + attn_hidden_states = _streaming_self_attention( + attn=block.attn1, + hidden_states=norm_hidden_states, + query_rotary_emb=video_rotary_emb, + n_context_tokens=n_context_tokens, + ) + hidden_states = hidden_states + attn_hidden_states * gate_msa + + norm_hidden_states = block.norm2(hidden_states) + attn_hidden_states = block.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + query_rotary_emb=None, + attention_mask=encoder_attention_mask, + ) + hidden_states = hidden_states + attn_hidden_states + + norm_hidden_states = block.norm3(hidden_states) * (1 + scale_mlp) + shift_mlp + return hidden_states + block.ff(norm_hidden_states) * gate_mlp + + +def _streaming_self_attention( + *, + attn: nn.Module, + hidden_states: Tensor, + query_rotary_emb: tuple[Tensor, Tensor], + n_context_tokens: int, +) -> Tensor: + from diffusers.models.transformers.transformer_ltx2 import ( + apply_interleaved_rotary_emb, + apply_split_rotary_emb, + ) + + gate_logits = ( + attn.to_gate_logits(hidden_states) + if attn.to_gate_logits is not None + else None + ) + query = attn.to_q(hidden_states) + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + query = attn.norm_q(query) + key = attn.norm_k(key) + + if attn.rope_type == "interleaved": + query = apply_interleaved_rotary_emb(query, query_rotary_emb) + key = apply_interleaved_rotary_emb(key, query_rotary_emb) + elif attn.rope_type == "split": + query = apply_split_rotary_emb(query, query_rotary_emb) + key = apply_split_rotary_emb(key, query_rotary_emb) + else: + raise ValueError(f"Unsupported LTX-2 RoPE type: {attn.rope_type}") + + query = query.unflatten(2, (attn.heads, -1)) + key = key.unflatten(2, (attn.heads, -1)) + value = value.unflatten(2, (attn.heads, -1)) + + if n_context_tokens <= 0 or n_context_tokens >= query.shape[1]: + hidden_states = _refiner_attention(query, key, value) + else: + context_hidden_states = _refiner_attention( + query[:, :n_context_tokens], + key[:, :n_context_tokens], + value[:, :n_context_tokens], + ) + current_hidden_states = _refiner_attention( + query[:, n_context_tokens:], + key, + value, + ) + hidden_states = torch.cat( + [context_hidden_states, current_hidden_states], + dim=1, + ) + + hidden_states = hidden_states.flatten(2, 3).to(query.dtype) + if gate_logits is not None: + hidden_states = hidden_states.unflatten(2, (attn.heads, -1)) + gates = (2.0 * torch.sigmoid(gate_logits)).unsqueeze(-1) + hidden_states = hidden_states * gates + hidden_states = hidden_states.flatten(2, 3) + hidden_states = attn.to_out[0](hidden_states) + return attn.to_out[1](hidden_states) + + +def _refiner_attention(query: Tensor, key: Tensor, value: Tensor) -> Tensor: + hidden_states = F.scaled_dot_product_attention( + query.transpose(1, 2), + key.transpose(1, 2), + value.transpose(1, 2), + attn_mask=None, + dropout_p=0.0, + is_causal=False, + ) + return hidden_states.transpose(1, 2) + + +def _pack_text_embeds( + text_hidden_states: Tensor, + sequence_lengths: Tensor, + *, + device: torch.device | str, + padding_side: str = "left", + scale_factor: int = 8, + eps: float = 1e-6, +) -> Tensor: + batch_size, seq_len, hidden_dim, _ = text_hidden_states.shape + original_dtype = text_hidden_states.dtype + token_indices = torch.arange(seq_len, device=device).unsqueeze(0) + if padding_side == "right": + mask = token_indices < sequence_lengths[:, None] + elif padding_side == "left": + start_indices = seq_len - sequence_lengths[:, None] + mask = token_indices >= start_indices + else: + raise ValueError( + f"padding_side must be 'left' or 'right', got {padding_side!r}." + ) + mask = mask[:, :, None, None] + + masked = text_hidden_states.masked_fill(~mask, 0.0) + denom = (sequence_lengths * hidden_dim).view(batch_size, 1, 1, 1) + masked_mean = masked.sum(dim=(1, 2), keepdim=True) / (denom + eps) + x_min = text_hidden_states.masked_fill(~mask, float("inf")).amin( + dim=(1, 2), + keepdim=True, + ) + x_max = text_hidden_states.masked_fill(~mask, float("-inf")).amax( + dim=(1, 2), + keepdim=True, + ) + normalized = (text_hidden_states - masked_mean) / (x_max - x_min + eps) + normalized = normalized * scale_factor + normalized = normalized.flatten(2) + mask_flat = mask.squeeze(-1).expand(-1, -1, normalized.shape[-1]) + normalized = normalized.masked_fill(~mask_flat, 0.0) + return normalized.to(dtype=original_dtype) + + +def _pack_latents(latents: Tensor, patch_size: int = 1, patch_size_t: int = 1) -> Tensor: + batch_size, _, num_frames, height, width = latents.shape + post_patch_num_frames = num_frames // patch_size_t + post_patch_height = height // patch_size + post_patch_width = width // patch_size + latents = latents.reshape( + batch_size, + -1, + post_patch_num_frames, + patch_size_t, + post_patch_height, + patch_size, + post_patch_width, + patch_size, + ) + latents = latents.permute(0, 2, 4, 6, 1, 3, 5, 7) + return latents.flatten(4, 7).flatten(1, 3) + + +def _unpack_latents( + latents: Tensor, + *, + num_frames: int, + height: int, + width: int, + patch_size: int = 1, + patch_size_t: int = 1, +) -> Tensor: + batch_size = latents.size(0) + latents = latents.reshape( + batch_size, + num_frames, + height, + width, + -1, + patch_size_t, + patch_size, + patch_size, + ) + latents = latents.permute(0, 4, 1, 5, 2, 6, 3, 7) + return latents.flatten(6, 7).flatten(4, 5).flatten(2, 3) + + +def _prepare_encoder_attention_mask(mask: Tensor | None, dtype: torch.dtype) -> Tensor | None: + if mask is None: + return None + if mask.ndim != 2: + return mask + if bool(torch.all(mask)): + return None + return ((1 - mask.to(dtype)) * -10000.0).unsqueeze(1) + + +def _empty_cuda_cache() -> None: + if torch.cuda.is_available(): + torch.cuda.empty_cache() + gc.collect() diff --git a/integrations/sana/sana_wm/native_transformer.py b/integrations/sana/sana_wm/native_transformer.py index b847761dc..bebf59e7f 100644 --- a/integrations/sana/sana_wm/native_transformer.py +++ b/integrations/sana/sana_wm/native_transformer.py @@ -499,6 +499,10 @@ def release_stage1_runtime( logger.info("[stage1] released Stage-1 runtime before decode/refine") gc.collect() + def release_refiner_runtime(self) -> None: + """Release native refiner tensors before VAE decode.""" + self._release_refiner() + def refine_latents( self, *, @@ -510,7 +514,7 @@ def refine_latents( block_size: int | None, kv_max_frames: int, ) -> Tensor: - """Run the native LTX-2 refiner once that path is implemented.""" + """Run the native LTX-2 refiner.""" self._ensure_refiner() sigmas = torch.tensor( self._stage2_sigmas(), @@ -611,11 +615,27 @@ def _ensure_model(self) -> None: self._prepare_stage1_quant() def _ensure_refiner(self) -> None: - raise RuntimeError( - "Native SANA-WM refiner execution is not implemented yet. " - "Use --no-refiner True for native Stage-1 GPU tests, or " - "sana-wm-upstream-reference for the temporary upstream refiner harness." + if self._refiner_built: + return + from sana_wm.native_refiner import SanaWMLTX2Refiner + + compute_dtype = ( + torch.bfloat16 + if self.config.refiner_precision in {"fp8", "fp4"} + else _get_weight_dtype(self.config.refiner_precision) + ) + quant_backend: QuantBackend = ( + "native" if self.config.quant_backend == "auto" else self.config.quant_backend + ) + self.refiner = SanaWMLTX2Refiner( + refiner_root=resolve_hf_path(self.config.refiner_root), + gemma_root=resolve_hf_path(self.config.refiner_gemma_root), + dtype=compute_dtype, + device=self.device, + precision=self.config.refiner_precision, + quant_backend=quant_backend, ) + self._refiner_built = True def _release_refiner(self) -> None: if not self._refiner_built: diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index 09795c905..f0266c667 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -305,12 +305,6 @@ def _run_native(self) -> None: cfg = self.config if cfg.image_path is None: raise ValueError("SanaWMRunner requires --image-path.") - if not cfg.no_refiner: - raise ValueError( - "Native SANA-WM currently supports Stage-1 execution only. " - "Use --no-refiner True, or use sana-wm-upstream-reference for " - "the temporary upstream refiner harness." - ) if not cfg.no_action_overlay: raise ValueError( "Native SANA-WM does not include the upstream action overlay. " @@ -360,9 +354,8 @@ def _run_native(self) -> None: sampling_algo = self._native_sampling_algo() if sampling_algo != "flow_euler_ltx": raise ValueError( - "Native SANA-WM currently supports flow_euler_ltx only; " - f"got {sampling_algo!r}. Use sana-wm-upstream-reference " - "for chunk/self-forcing samplers." + "Native SANA-WM requires flow_euler_ltx for the " + f"bidirectional runner; got {sampling_algo!r}." ) camera = prepare_camera( c2w, @@ -389,10 +382,38 @@ def _run_native(self) -> None: transformer.release_stage1_runtime(cache) del conditioning, cache - video_hwc = transformer.decode_latents(sana_latent) + stage1_video_hwc: np.ndarray | None = None + output_latent = sana_latent + if not cfg.no_refiner: + output_latent = transformer.refine_latents( + latents=sana_latent, + prompt=prompt, + fps=cfg.fps, + sink_size=cfg.sink_size, + seed=cfg.refiner_seed, + block_size=cfg.refiner_block_size, + kv_max_frames=cfg.refiner_kv_max_frames, + ) + transformer.release_refiner_runtime() + elif cfg.save_stage1: + logger.info( + "Native SANA-WM is already running without the refiner; " + "--save-stage1 does not create an extra output." + ) + + video_hwc = transformer.decode_latents(output_latent) + if self.is_rank_zero and cfg.save_stage1 and not cfg.no_refiner: + stage1_video_hwc = transformer.decode_latents(sana_latent) if not self.is_rank_zero: return _write_video(cfg.output_dir, cfg.name, video_hwc, cfg.fps) + if stage1_video_hwc is not None: + _write_video( + cfg.output_dir, + f"{cfg.name}_stage1", + stage1_video_hwc, + cfg.fps, + ) def _run_upstream_reference(self) -> None: """Run the explicit upstream SANA-WM reference harness.""" @@ -829,7 +850,7 @@ def _validate_torch_fp8_backend(precisions: list[Precision]) -> None: unsupported = sorted({precision for precision in precisions if precision != "fp8"}) if unsupported: raise ValueError( - "SANA-WM --quant-backend torch-fp8 currently supports fp8 only; " + "SANA-WM --quant-backend torch-fp8 accepts fp8 only; " f"unsupported requested precision(s): {', '.join(unsupported)}. " "Use --quant-backend torch-fp4 or --quant-backend native for fp4." ) @@ -841,7 +862,7 @@ def _validate_torch_fp4_backend(precisions: list[Precision]) -> None: unsupported = sorted({precision for precision in precisions if precision != "fp4"}) if unsupported: raise ValueError( - "SANA-WM --quant-backend torch-fp4 currently supports fp4 only; " + "SANA-WM --quant-backend torch-fp4 accepts fp4 only; " f"unsupported requested precision(s): {', '.join(unsupported)}. " "Use --quant-backend torch-fp8 or --quant-backend native for fp8." ) diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index 10d0733ec..f657c0b0e 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -23,6 +23,7 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass import math @@ -78,6 +79,7 @@ def __init__(self, hidden_size: int, eps: float = 1e-6) -> None: super().__init__() self.weight = nn.Parameter(torch.empty(hidden_size)) self.eps = eps + nn.init.ones_(self.weight) def forward(self, x: Tensor) -> Tensor: """Apply RMS normalization using the stored scale parameter.""" @@ -236,12 +238,13 @@ def __init__(self, spec: SanaWMStage1Spec) -> None: padding=(spec.temporal_kernel_size // 2, 0), bias=False, ) + nn.init.zeros_(self.t_conv.weight) def forward(self, x: Tensor, *, frames: int, height: int, width: int) -> Tensor: """Run spatial GLU plus temporal aggregation.""" batch, tokens, channels = x.shape x_2d = x.reshape(batch * frames, height, width, channels).permute(0, 3, 1, 2) - x_2d = self.inverted_conv(x_2d) + x_2d = F.silu(self.inverted_conv(x_2d)) x_2d = self.depth_conv(x_2d) value, gate = x_2d.chunk(2, dim=1) x_2d = self.point_conv(value * F.silu(gate)) @@ -256,6 +259,11 @@ class Stage1SelfAttention(nn.Module): def __init__(self, spec: SanaWMStage1Spec, *, use_gdn_convs: bool) -> None: super().__init__() + self.heads = spec.num_heads + self.dim = spec.head_dim + self.eps = 1e-6 + self.use_gdn_convs = use_gdn_convs + self.patch_size = (1, 1, 1) self.A_log = nn.Parameter(torch.empty(spec.num_heads)) self.beta_proj = nn.Linear(spec.hidden_size, spec.num_heads) self.dt_bias = nn.Parameter(torch.empty(spec.num_heads)) @@ -287,22 +295,242 @@ def __init__(self, spec: SanaWMStage1Spec, *, use_gdn_convs: bool) -> None: groups=spec.hidden_size, bias=False, ) + nn.init.zeros_(self.A_log) + nn.init.constant_(self.dt_bias, -5.0) + nn.init.zeros_(self.recall_gate) - def forward(self, x: Tensor, *_args: object, **_kwargs: object) -> Tensor: - """Run a memory-bounded native self-attention approximation. + def forward( + self, + x: Tensor, + *_args: object, + HW: tuple[int, int, int] | None = None, + rotary_emb: Tensor | None = None, + camera_conditions: Tensor | None = None, + apply_output_gate: bool = True, + **kwargs: object, + ) -> Tensor: + """Run native SANA-WM self/camera attention.""" + if HW is None: + raise ValueError("SANA-WM Stage-1 attention requires HW=(T, H, W).") + batch, tokens, channels = x.shape + if channels != self.heads * self.dim: + raise ValueError( + f"channels={channels} != heads*dim={self.heads * self.dim}" + ) - The public checkpoint's GDN blocks are linear-recurrent attention - blocks. Until the native GDN scan is implemented, this path uses the - checkpoint QKV/value/projection tensors without materializing an - all-token softmax attention matrix. - """ + precomputed_gates = self._compute_frame_gates(x, HW) + if self.use_gdn_convs: + main_raw = self._forward_gdn_main( + x, + HW=HW, + rotary_emb=rotary_emb, + precomputed_gates=precomputed_gates, + ) + else: + main_raw = self._forward_softmax_main( + x, + HW=HW, + rotary_emb=rotary_emb, + ) + + cam_contrib: Tensor | int = 0 + if camera_conditions is not None: + if self.use_gdn_convs: + cam_raw = self._forward_gdn_camera( + x, + HW=HW, + rotary_emb=rotary_emb, + camera_conditions=camera_conditions, + precomputed_gates=precomputed_gates, + ) + else: + cam_raw = self._forward_softmax_camera( + x, + HW=HW, + rotary_emb=rotary_emb, + camera_conditions=camera_conditions, + ) + cam_contrib = self.out_proj_cam(cam_raw) + + combined = main_raw + cam_contrib + if apply_output_gate: + combined = _apply_output_gate(combined, x, self.output_gate) + combined = self.proj(combined.to(dtype=self.proj.weight.dtype)) + del kwargs + return combined + + def _forward_gdn_main( + self, + x: Tensor, + *, + HW: tuple[int, int, int], + rotary_emb: Tensor | None, + precomputed_gates: tuple[Tensor, Tensor], + ) -> Tensor: batch, tokens, channels = x.shape - qkv = self.qkv(x).view(batch, tokens, 3, channels) - q, _k, v = qkv.unbind(dim=2) - q = self.q_norm(q) - v = v * torch.sigmoid(q) - out = F.silu(self.output_gate(x).float()).to(dtype=v.dtype) * v - return self.proj(out.to(dtype=self.proj.weight.dtype)) + qkv = self.qkv(x).reshape(batch, tokens, 3, self.heads, self.dim) + if hasattr(self, "conv_k"): + k_raw = qkv[:, :, 1].reshape(batch, tokens, channels) + k_conv = _apply_bidirectional_temporal_conv(k_raw, self.conv_k, HW) + qkv[:, :, 1] = k_conv.reshape(batch, tokens, self.heads, self.dim) + + q = self.q_norm(qkv[:, :, 0].reshape(batch, tokens, channels)).reshape( + batch, + tokens, + self.heads, + self.dim, + ) + k = self.k_norm(qkv[:, :, 1].reshape(batch, tokens, channels)).reshape( + batch, + tokens, + self.heads, + self.dim, + ) + v = qkv[:, :, 2] + q = F.relu(q.float()) + k = F.relu(k.float()) * _gdn_key_scale(self.dim, HW) + v = v.float() + q_rot = _apply_complex_rope(q, rotary_emb) + k_rot = _apply_complex_rope(k, rotary_emb) + beta, decay = precomputed_gates + out = _bidirectional_gdn_scan( + q=q, + k=k, + q_rot=q_rot, + k_rot=k_rot, + v=v, + beta=beta, + decay=decay, + HW=HW, + eps=self.eps, + ) + return out.reshape(batch, tokens, channels).to(dtype=x.dtype) + + def _forward_softmax_main( + self, + x: Tensor, + *, + HW: tuple[int, int, int], + rotary_emb: Tensor | None, + ) -> Tensor: + del HW + batch, tokens, channels = x.shape + qkv = self.qkv(x).reshape(batch, tokens, 3, self.heads, self.dim) + q, k, v = qkv.unbind(dim=2) + q = self.q_norm(q.reshape(batch, tokens, channels)).reshape( + batch, + tokens, + self.heads, + self.dim, + ) + k = self.k_norm(k.reshape(batch, tokens, channels)).reshape( + batch, + tokens, + self.heads, + self.dim, + ) + q = _apply_complex_rope(q, rotary_emb) + k = _apply_complex_rope(k, rotary_emb) + out = F.scaled_dot_product_attention( + q.transpose(1, 2), + k.transpose(1, 2), + v.transpose(1, 2), + dropout_p=0.0, + is_causal=False, + ) + return out.transpose(1, 2).reshape(batch, tokens, channels).to(dtype=x.dtype) + + def _forward_gdn_camera( + self, + x: Tensor, + *, + HW: tuple[int, int, int], + rotary_emb: Tensor | None, + camera_conditions: Tensor, + precomputed_gates: tuple[Tensor, Tensor], + ) -> Tensor: + q_cam, k_cam, v_cam = _camera_qkv(self, x, HW) + q_trans, k_trans, v_trans, inflation_sq, output_projector = _prepare_ucpe_qkv( + q_cam, + k_cam, + v_cam, + camera_conditions=camera_conditions, + HW=HW, + rotary_emb=rotary_emb, + q_norm_weight=self.q_norm_cam.weight, + k_norm_weight=self.k_norm_cam.weight, + norm_eps=self.q_norm_cam.eps, + ) + beta, decay = precomputed_gates + frame_inflation = inflation_sq.reshape( + x.shape[0], + self.heads, + HW[0], + HW[1] * HW[2], + ).mean(dim=-1) + beta = beta / frame_inflation.unsqueeze(-1).clamp_min(1.0) + out = _bidirectional_numerator_scan( + q=q_trans, + k=k_trans, + v=v_trans, + beta=beta, + decay=decay, + HW=HW, + ) + out = output_projector(out) + return out.reshape(x.shape[0], x.shape[1], -1).to(dtype=x.dtype) + + def _forward_softmax_camera( + self, + x: Tensor, + *, + HW: tuple[int, int, int], + rotary_emb: Tensor | None, + camera_conditions: Tensor, + ) -> Tensor: + q_cam, k_cam, v_cam = _camera_qkv(self, x, HW) + q_trans, k_trans, v_trans, output_projector = _prepare_ucpe_qkv_softmax( + q_cam, + k_cam, + v_cam, + camera_conditions=camera_conditions, + HW=HW, + rotary_emb=rotary_emb, + q_norm_weight=self.q_norm_cam.weight, + k_norm_weight=self.k_norm_cam.weight, + norm_eps=self.q_norm_cam.eps, + ) + out = F.scaled_dot_product_attention( + q_trans.transpose(1, 2), + k_trans.transpose(1, 2), + v_trans.transpose(1, 2), + dropout_p=0.0, + is_causal=False, + ) + return output_projector(out.transpose(1, 2)).reshape( + x.shape[0], + x.shape[1], + -1, + ).to(dtype=x.dtype) + + def _compute_frame_gates( + self, + x: Tensor, + HW: tuple[int, int, int], + ) -> tuple[Tensor, Tensor]: + batch, tokens, channels = x.shape + frames, height, width = HW + spatial = height * width + if tokens != frames * spatial: + raise ValueError(f"tokens={tokens} != T*H*W={frames * spatial}") + beta = self.beta_proj(x).sigmoid() + beta = beta.reshape(batch, frames, spatial, self.heads).permute(0, 3, 1, 2) + x_frame = x.reshape(batch, frames, spatial, channels).mean(dim=2) + gate = self.gate_proj(x_frame).float() + dt = self.dt_bias.float().view(1, 1, -1) + a_val = self.A_log.float().exp().view(1, 1, -1) + decay = (-a_val * F.softplus(gate + dt)).exp().transpose(1, 2) + return beta.float(), decay.float() class Stage1CrossAttention(nn.Module): @@ -326,7 +554,7 @@ def forward( mask: Tensor | None = None, **_kwargs: object, ) -> Tensor: - """Cross-attention execution is implemented in a later native pass.""" + """Run text cross-attention with checkpoint-compatible Q/K norms.""" batch, tokens, channels = x.shape q = self.q_norm(self.q_linear(x)).view( batch, @@ -464,6 +692,18 @@ def forward( plucker_emb = self.plucker_embedder(chunk_plucker) plucker_emb = plucker_emb.permute(0, 2, 3, 4, 1).reshape_as(x) + rotary_emb = _wan_rope_complex( + self.spec.head_dim, + frames, + height, + width, + x.device, + ) + camera_conditions = kwargs.get("camera_conditions") + block_kwargs = { + key: value for key, value in kwargs.items() if key != "camera_conditions" + } + y = self.y_embedder(y) y = self.attention_y_norm(y) if mask is not None and mask.ndim > 2: @@ -483,7 +723,14 @@ def forward( width=width, mask=mask, plucker_emb=plucker_emb, - **kwargs, + HW=(frames, height, width), + rotary_emb=rotary_emb, + camera_conditions=( + camera_conditions.to(device=x.device, dtype=x.dtype) + if isinstance(camera_conditions, Tensor) + else None + ), + **block_kwargs, ) x = self.final_layer(x, timestep_embed, frames=frames) @@ -508,6 +755,518 @@ def _timestep_embedding(t: Tensor, dim: int, max_period: int = 10000) -> Tensor: return embedding +def _gdn_key_scale(head_dim: int, HW: tuple[int, int, int]) -> float: + return (head_dim**-0.5) * ((HW[1] * HW[2]) ** -0.5) + + +def _apply_output_gate(out: Tensor, gate_x: Tensor, gate: nn.Linear) -> Tensor: + gate_values = F.silu(F.linear(gate_x, gate.weight, gate.bias).float()) + return out * gate_values.to(dtype=out.dtype) + + +def _apply_bidirectional_temporal_conv( + x: Tensor, + conv: nn.Conv1d, + HW: tuple[int, int, int], +) -> Tensor: + batch, tokens, channels = x.shape + frames, height, width = HW + spatial = height * width + if tokens != frames * spatial: + raise ValueError(f"tokens={tokens} != T*H*W={frames * spatial}") + temporal = ( + x.reshape(batch, frames, spatial, channels) + .permute(0, 2, 3, 1) + .reshape(batch * spatial, channels, frames) + ) + y_fwd = _causal_depthwise_conv(temporal, conv) + y_bwd = _causal_depthwise_conv(temporal.flip(-1), conv).flip(-1) + center = conv.weight[:, 0, -1].view(1, channels, 1) + out = y_fwd + y_bwd - temporal * center + return ( + out.reshape(batch, spatial, channels, frames) + .permute(0, 3, 1, 2) + .reshape(batch, tokens, channels) + .to(dtype=x.dtype) + ) + + +def _causal_depthwise_conv(x: Tensor, conv: nn.Conv1d) -> Tensor: + kernel = int(conv.kernel_size[0]) + padded = F.pad(x, (kernel - 1, 0)) + return F.conv1d( + padded, + conv.weight.to(dtype=x.dtype), + bias=None, + stride=1, + padding=0, + dilation=1, + groups=conv.groups, + ) + + +def _wan_rope_complex( + head_dim: int, + frames: int, + height: int, + width: int, + device: torch.device, +) -> Tensor: + t_size = head_dim // 2 - 2 * (head_dim // 6) + h_size = head_dim // 6 + w_size = head_dim // 6 + freqs_t = _axis_rope_complex(frames, t_size, device) + freqs_h = _axis_rope_complex(height, h_size, device) + freqs_w = _axis_rope_complex(width, w_size, device) + expanded_t = freqs_t[:, None, None, :].expand(frames, height, width, t_size) + expanded_h = freqs_h[None, :, None, :].expand(frames, height, width, h_size) + expanded_w = freqs_w[None, None, :, :].expand(frames, height, width, w_size) + freqs = torch.cat([expanded_t, expanded_h, expanded_w], dim=-1) + return freqs.reshape(1, 1, frames * height * width, -1) + + +def _axis_rope_complex(length: int, complex_dims: int, device: torch.device) -> Tensor: + if complex_dims == 0: + return torch.empty(length, 0, dtype=torch.complex64, device=device) + dim = complex_dims * 2 + exponent = torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim + freqs = 1.0 / (10000.0**exponent) + positions = torch.arange(length, dtype=torch.float32, device=device) + angles = positions[:, None] * freqs[None] + return torch.polar(torch.ones_like(angles), angles) + + +def _apply_complex_rope(x: Tensor, rotary_emb: Tensor | None) -> Tensor: + if rotary_emb is None: + return x + batch, tokens, heads, dim = x.shape + freqs = rotary_emb.squeeze(0).squeeze(0) + if freqs.shape[0] != tokens or freqs.shape[1] != dim // 2: + raise ValueError( + f"RoPE shape {tuple(freqs.shape)} is incompatible with {(tokens, dim)}." + ) + x_float = x.float() + x_complex = torch.view_as_complex( + x_float.reshape(batch, tokens, heads, dim // 2, 2) + ) + rotated = torch.view_as_real(x_complex * freqs[None, :, None, :]).flatten(-2) + return rotated.to(dtype=x.dtype) + + +def _slice_rope_for_camera(rotary_emb: Tensor | None, head_dim: int) -> Tensor | None: + if rotary_emb is None: + return None + orig_t_size = head_dim // 2 - 2 * (head_dim // 6) + orig_h_size = head_dim // 6 + new_head_dim = head_dim // 2 + new_t_size = new_head_dim // 2 - 2 * (new_head_dim // 6) + new_h_size = new_head_dim // 6 + new_w_size = new_head_dim // 6 + t_part = rotary_emb[..., :new_t_size] + h_part = rotary_emb[..., orig_t_size : orig_t_size + new_h_size] + w_part = rotary_emb[ + ..., + orig_t_size + + orig_h_size : orig_t_size + + orig_h_size + + new_w_size, + ] + return torch.cat([t_part, h_part, w_part], dim=-1) + + +def _bidirectional_gdn_scan( + *, + q: Tensor, + k: Tensor, + q_rot: Tensor, + k_rot: Tensor, + v: Tensor, + beta: Tensor, + decay: Tensor, + HW: tuple[int, int, int], + eps: float, +) -> Tensor: + m_hist, z_hist = _gdn_histories( + k=k, + k_rot=k_rot, + v=v, + beta=beta, + decay=decay, + HW=HW, + include_denominator=True, + ) + batch, tokens, heads, dim = q.shape + frames, height, width = HW + spatial = height * width + q = q.reshape(batch, frames, spatial, heads, dim).permute(0, 3, 1, 2, 4) + q_rot = q_rot.reshape(batch, frames, spatial, heads, dim).permute( + 0, + 3, + 1, + 2, + 4, + ) + num = torch.einsum("bhfsd,bhfde->bhfse", q_rot.float(), m_hist) + den = torch.einsum("bhfsd,bhfd->bhfs", q.float(), z_hist) + out = num / (den[..., None] + eps) + return out.permute(0, 2, 3, 1, 4).reshape(batch, tokens, heads, dim) + + +def _bidirectional_numerator_scan( + *, + q: Tensor, + k: Tensor, + v: Tensor, + beta: Tensor, + decay: Tensor, + HW: tuple[int, int, int], +) -> Tensor: + m_hist, _z_hist = _gdn_histories( + k=k, + k_rot=k, + v=v, + beta=beta, + decay=decay, + HW=HW, + include_denominator=False, + ) + batch, tokens, heads, dim = q.shape + frames, height, width = HW + spatial = height * width + q = q.reshape(batch, frames, spatial, heads, dim).permute(0, 3, 1, 2, 4) + out = torch.einsum("bhfsd,bhfde->bhfse", q.float(), m_hist) + return out.permute(0, 2, 3, 1, 4).reshape(batch, tokens, heads, dim) + + +def _gdn_histories( + *, + k: Tensor, + k_rot: Tensor, + v: Tensor, + beta: Tensor, + decay: Tensor, + HW: tuple[int, int, int], + include_denominator: bool, +) -> tuple[Tensor, Tensor | None]: + batch, tokens, heads, dim = k.shape + frames, height, width = HW + spatial = height * width + k = k.reshape(batch, frames, spatial, heads, dim).permute(0, 3, 1, 2, 4).float() + k_rot = ( + k_rot.reshape(batch, frames, spatial, heads, dim) + .permute(0, 3, 1, 2, 4) + .float() + ) + v = v.reshape(batch, frames, spatial, heads, dim).permute(0, 3, 1, 2, 4).float() + eye = torch.eye(dim, dtype=torch.float32, device=k.device).view(1, 1, dim, dim) + m = torch.zeros(batch, heads, dim, dim, dtype=torch.float32, device=k.device) + z = torch.zeros(batch, heads, dim, dtype=torch.float32, device=k.device) + m_hist = torch.empty( + batch, + heads, + frames, + dim, + dim, + dtype=torch.float32, + device=k.device, + ) + z_hist = ( + torch.empty(batch, heads, frames, dim, dtype=torch.float32, device=k.device) + if include_denominator + else None + ) + for frame in range(frames): + beta_f = beta[:, :, frame] + decay_f = decay[:, :, frame] + p_kv = torch.einsum( + "bhsd,bhse->bhde", + k_rot[:, :, frame], + beta_f[..., None] * k_rot[:, :, frame], + ) + a_val = torch.einsum( + "bhsd,bhse->bhde", + k_rot[:, :, frame], + beta_f[..., None] * v[:, :, frame], + ) + m = decay_f[..., None, None] * torch.einsum("bhde,bhef->bhdf", eye - p_kv, m) + m = m + a_val + m_hist[:, :, frame] = m + if include_denominator and z_hist is not None: + p_z = torch.einsum( + "bhsd,bhse->bhde", + k[:, :, frame], + beta_f[..., None] * k[:, :, frame], + ) + b_z = (beta_f[..., None] * k[:, :, frame]).sum(dim=2) + z = decay_f[..., None] * torch.einsum("bhde,bhe->bhd", eye - p_z, z) + z = z + b_z + z_hist[:, :, frame] = z + + m = torch.zeros_like(m) + z = torch.zeros_like(z) + for src in range(frames - 1, 0, -1): + dst = src - 1 + beta_f = beta[:, :, src] + decay_f = decay[:, :, src] + p_kv = torch.einsum( + "bhsd,bhse->bhde", + k_rot[:, :, src], + beta_f[..., None] * k_rot[:, :, src], + ) + a_val = torch.einsum( + "bhsd,bhse->bhde", + k_rot[:, :, src], + beta_f[..., None] * v[:, :, src], + ) + m = decay_f[..., None, None] * torch.einsum("bhde,bhef->bhdf", eye - p_kv, m) + m = m + a_val + m_hist[:, :, dst] += m + if include_denominator and z_hist is not None: + p_z = torch.einsum( + "bhsd,bhse->bhde", + k[:, :, src], + beta_f[..., None] * k[:, :, src], + ) + b_z = (beta_f[..., None] * k[:, :, src]).sum(dim=2) + z = decay_f[..., None] * torch.einsum("bhde,bhe->bhd", eye - p_z, z) + z = z + b_z + z_hist[:, :, dst] += z + return m_hist, z_hist + + +def _camera_qkv( + module: Stage1SelfAttention, + x: Tensor, + HW: tuple[int, int, int], +) -> tuple[Tensor, Tensor, Tensor]: + batch, tokens, _channels = x.shape + q_raw = module.q_proj_cam(x).reshape(batch, tokens, module.heads, module.dim) + k_raw_flat = module.k_proj_cam(x) + if hasattr(module, "conv_k_cam"): + k_raw_flat = _apply_bidirectional_temporal_conv(k_raw_flat, module.conv_k_cam, HW) + k_raw = k_raw_flat.reshape(batch, tokens, module.heads, module.dim) + v_raw = module.v_proj_cam(x).reshape(batch, tokens, module.heads, module.dim) + return q_raw, k_raw, v_raw + + +def _prepare_ucpe_qkv( + q_raw: Tensor, + k_raw: Tensor, + v_raw: Tensor, + *, + camera_conditions: Tensor, + HW: tuple[int, int, int], + rotary_emb: Tensor | None, + q_norm_weight: Tensor, + k_norm_weight: Tensor, + norm_eps: float, +) -> tuple[Tensor, Tensor, Tensor, Tensor, Callable[[Tensor], Tensor]]: + batch, tokens, heads, dim = q_raw.shape + q_inv = _inv_rms(q_raw, norm_eps) + k_inv = _inv_rms(k_raw, norm_eps) + q_weight = q_norm_weight.float().view(heads, dim) + k_weight = k_norm_weight.float().view(heads, dim) + q_norm = F.relu(q_raw.float() * q_inv[:, :, None, None] * q_weight[None, None]) + k_norm = F.relu(k_raw.float() * k_inv[:, :, None, None] * k_weight[None, None]) + k_norm = k_norm * _gdn_key_scale(dim, HW) + v_float = v_raw.float() + raymats = _camera_ray_mats(camera_conditions.float(), HW) + matrix_q = raymats.reshape(batch, tokens, 4, 4).transpose(-1, -2).contiguous() + matrix_kv = _invert_se3(raymats.reshape(batch, tokens, 4, 4)).contiguous() + rope_cam = _slice_rope_for_camera(rotary_emb, dim) + + q_trans, _q_pre, _q_post = _ucpe_transform( + q_norm, + matrix_q, + rope_cam, + inverse_rope=False, + ) + k_trans, k_pre_sq, k_post_sq = _ucpe_transform( + k_norm, + matrix_kv, + rope_cam, + inverse_rope=False, + ) + v_trans, _v_pre, _v_post = _ucpe_transform( + v_float, + matrix_kv, + rope_cam, + inverse_rope=False, + ) + inflation_sq = k_post_sq.clamp_min(1e-12) / k_pre_sq.clamp_min(1e-12) + + def output_projector(out: Tensor) -> Tensor: + projected, _pre, _post = _ucpe_transform( + out.float(), + raymats.reshape(batch, tokens, 4, 4), + rope_cam, + inverse_rope=True, + ) + return projected.to(dtype=out.dtype) + + return q_trans, k_trans, v_trans, inflation_sq, output_projector + + +def _prepare_ucpe_qkv_softmax( + q_raw: Tensor, + k_raw: Tensor, + v_raw: Tensor, + *, + camera_conditions: Tensor, + HW: tuple[int, int, int], + rotary_emb: Tensor | None, + q_norm_weight: Tensor, + k_norm_weight: Tensor, + norm_eps: float, +) -> tuple[Tensor, Tensor, Tensor, Callable[[Tensor], Tensor]]: + batch, tokens, heads, dim = q_raw.shape + q_inv = _inv_rms(q_raw, norm_eps) + k_inv = _inv_rms(k_raw, norm_eps) + q_weight = q_norm_weight.float().view(heads, dim) + k_weight = k_norm_weight.float().view(heads, dim) + q_norm = q_raw.float() * q_inv[:, :, None, None] * q_weight[None, None] + k_norm = k_raw.float() * k_inv[:, :, None, None] * k_weight[None, None] + v_float = v_raw.float() + raymats = _camera_ray_mats(camera_conditions.float(), HW) + matrix_q = raymats.reshape(batch, tokens, 4, 4).transpose(-1, -2).contiguous() + matrix_kv = _invert_se3(raymats.reshape(batch, tokens, 4, 4)).contiguous() + rope_cam = _slice_rope_for_camera(rotary_emb, dim) + q_trans, _q_pre, _q_post = _ucpe_transform( + q_norm, + matrix_q, + rope_cam, + inverse_rope=False, + ) + k_trans, _k_pre, _k_post = _ucpe_transform( + k_norm, + matrix_kv, + rope_cam, + inverse_rope=False, + ) + v_trans, _v_pre, _v_post = _ucpe_transform( + v_float, + matrix_kv, + rope_cam, + inverse_rope=False, + ) + + def output_projector(out: Tensor) -> Tensor: + projected, _pre, _post = _ucpe_transform( + out.float(), + raymats.reshape(batch, tokens, 4, 4), + rope_cam, + inverse_rope=True, + ) + return projected.to(dtype=out.dtype) + + return q_trans, k_trans, v_trans, output_projector + + +def _inv_rms(x: Tensor, eps: float) -> Tensor: + batch, tokens, heads, dim = x.shape + channels = heads * dim + return torch.rsqrt(x.float().pow(2).sum(dim=(-1, -2)) / channels + eps) + + +def _ucpe_transform( + x: Tensor, + matrix: Tensor, + rotary_emb: Tensor | None, + *, + inverse_rope: bool, +) -> tuple[Tensor, Tensor, Tensor]: + batch, tokens, heads, dim = x.shape + half = dim // 2 + if half % 4 != 0: + raise ValueError(f"UCPE requires head_dim/2 divisible by 4, got {half}.") + first = x[..., :half].reshape(batch, tokens, heads, half // 4, 4) + first_out = torch.einsum("bnij,bnhgj->bnhgi", matrix.float(), first.float()) + first_out = first_out.reshape(batch, tokens, heads, half) + second = x[..., half:] + if inverse_rope and rotary_emb is not None: + second_out = _apply_complex_rope(second, rotary_emb.conj()) + else: + second_out = _apply_complex_rope(second, rotary_emb) + out = torch.cat([first_out, second_out], dim=-1) + pre_sq = x.float().pow(2).sum(dim=-1).transpose(1, 2).contiguous() + post_sq = out.float().pow(2).sum(dim=-1).transpose(1, 2).contiguous() + return out, pre_sq, post_sq + + +def _camera_ray_mats( + camera_conditions: Tensor, + HW: tuple[int, int, int], +) -> Tensor: + batch, frames = camera_conditions.shape[:2] + latent_frames, height, width = HW + if frames != latent_frames: + raise ValueError( + f"camera_conditions has {frames} frames but latent grid has {latent_frames}." + ) + c2w = camera_conditions[..., :16].reshape(batch, frames, 4, 4) + fx = camera_conditions[..., 16] + fy = camera_conditions[..., 17] + cx = camera_conditions[..., 18] + cy = camera_conditions[..., 19] + y_grid, x_grid = torch.meshgrid( + torch.arange(height, dtype=torch.float32, device=camera_conditions.device), + torch.arange(width, dtype=torch.float32, device=camera_conditions.device), + indexing="ij", + ) + x = (x_grid.view(1, 1, height, width) - cx[:, :, None, None]) / fx[ + :, + :, + None, + None, + ].clamp_min(1e-6) + y = (y_grid.view(1, 1, height, width) - cy[:, :, None, None]) / fy[ + :, + :, + None, + None, + ].clamp_min(1e-6) + dirs_cam = torch.stack([x, y, torch.ones_like(x)], dim=-1) + dirs_cam = F.normalize(dirs_cam, dim=-1, eps=1e-6) + rotation = c2w[..., :3, :3] + translation = c2w[..., :3, 3] + dirs_world = torch.einsum("btij,bthwj->bthwi", rotation, dirs_cam) + cam_y = rotation[..., :, 1].view(batch, frames, 1, 1, 3).expand_as(dirs_world) + z_ray = F.normalize(dirs_world, dim=-1, eps=1e-6) + x_ray = F.normalize(torch.cross(cam_y, z_ray, dim=-1), dim=-1, eps=1e-6) + y_ray = F.normalize(torch.cross(z_ray, x_ray, dim=-1), dim=-1, eps=1e-6) + ray_to_world = torch.stack([x_ray, y_ray, z_ray], dim=-1) + world_to_ray = ray_to_world.transpose(-1, -2) + origin = translation.view(batch, frames, 1, 1, 3).expand_as(dirs_world) + trans = -torch.einsum("bthwij,bthwj->bthwi", world_to_ray, origin) + mats = torch.zeros( + batch, + frames, + height, + width, + 4, + 4, + dtype=torch.float32, + device=camera_conditions.device, + ) + mats[..., :3, :3] = world_to_ray + mats[..., :3, 3] = trans + mats[..., 3, 3] = 1.0 + return mats + + +def _invert_se3(transforms: Tensor) -> Tensor: + rotation_inv = transforms[..., :3, :3].transpose(-1, -2) + out = torch.zeros_like(transforms) + out[..., :3, :3] = rotation_inv + out[..., :3, 3] = -torch.einsum( + "...ij,...j->...i", + rotation_inv, + transforms[..., :3, 3], + ) + out[..., 3, 3] = 1.0 + return out + + __all__ = [ "SANA_WM_STAGE1_SPEC", "GLUMBConvTemp", diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index 2b839d7d7..eae39bf05 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -23,6 +23,9 @@ import pytest import torch +import sana_wm.native_refiner as native_refiner +import sana_wm.native_transformer as native_transformer_module + try: import tomllib except ModuleNotFoundError: # pragma: no cover - Python < 3.11 fallback @@ -62,6 +65,7 @@ _avoid_degenerate_tile_tail, _load_inference_config, ) +from sana_wm.native_refiner import SanaWMLTX2Refiner, _pack_latents, _unpack_latents from sana_wm.native_quant import ( TorchScaledMMFP4Linear, TorchScaledMMFP8Linear, @@ -72,6 +76,7 @@ SANA_WM_STAGE1_SPEC, SanaWMStage1Model, SanaWMStage1Spec, + Stage1SelfAttention, ) from sana_wm.upstream_reference import SanaWMUpstreamReferenceTransformerConfig @@ -249,6 +254,41 @@ def test_native_stage1_forward_preserves_latent_shape() -> None: assert out.shape == latents.shape +def test_stage1_self_attention_uses_camera_conditions() -> None: + """Changing camera conditions must affect native camera attention.""" + torch.manual_seed(0) + spec = SanaWMStage1Spec( + latent_channels=4, + hidden_size=16, + text_dim=12, + timestep_dim=8, + depth=1, + num_heads=2, + head_dim=8, + max_text_length=5, + latent_grid_size=(2, 2), + mlp_ratio=1, + conv_kernel_size=3, + temporal_kernel_size=3, + plucker_channels=6, + raymap_channels=3, + softmax_every_n=2, + ) + attn = Stage1SelfAttention(spec, use_gdn_convs=True).eval() + hidden = torch.randn(1, 8, 16) + camera = torch.zeros(1, 2, 20) + camera[..., :16] = torch.eye(4).flatten() + camera[..., 16:] = torch.tensor([1.0, 1.0, 0.5, 0.5]) + shifted_camera = camera.clone() + shifted_camera[..., 3] = 0.25 + + base = attn(hidden, HW=(2, 2, 2), camera_conditions=camera) + shifted = attn(hidden, HW=(2, 2, 2), camera_conditions=shifted_camera) + + assert base.shape == hidden.shape + assert not torch.allclose(base, shifted) + + def test_native_transformer_releases_stage1_runtime() -> None: """Free Stage-1-only modules and conditioning before decode/refine.""" transformer = SanaWMNativeTransformerConfig().setup() @@ -366,7 +406,7 @@ def enable_tiling(self, **kwargs: int) -> None: calls = 0 inference_modes: list[bool] = [] - def fake_decode(_samples: torch.Tensor) -> torch.Tensor: + def decode_once(_samples: torch.Tensor) -> torch.Tensor: nonlocal calls calls += 1 inference_modes.append(torch.is_inference_mode_enabled()) @@ -374,7 +414,7 @@ def fake_decode(_samples: torch.Tensor) -> torch.Tensor: raise torch.OutOfMemoryError("test OOM") return torch.zeros((1, 3, 1, 2, 2), dtype=torch.float32) - monkeypatch.setattr(transformer, "_vae_decode", fake_decode) + monkeypatch.setattr(transformer, "_vae_decode", decode_once) video = transformer.decode_latents(torch.zeros((1, 4, 1, 1, 1))) @@ -472,6 +512,112 @@ def test_runner_defaults_to_bf16_precision() -> None: assert RUNNER_SANA_WM_BIDIRECTIONAL.stage1_precision == "bf16" assert RUNNER_SANA_WM_BIDIRECTIONAL.refiner_precision == "bf16" assert RUNNER_SANA_WM_BIDIRECTIONAL.quant_backend == "auto" + assert RUNNER_SANA_WM_BIDIRECTIONAL.no_refiner is False + + +def test_native_refiner_is_flashdreams_owned( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Build the native refiner adapter instead of raising or importing Sana.""" + calls: dict[str, object] = {} + + class DummyRefiner: + def __init__(self, **kwargs: object) -> None: + calls.update(kwargs) + + monkeypatch.setattr(native_refiner, "SanaWMLTX2Refiner", DummyRefiner) + monkeypatch.setattr( + native_transformer_module, + "resolve_hf_path", + lambda value: f"/resolved/{value}", + ) + transformer = SanaWMNativeTransformerConfig().setup() + + transformer._ensure_refiner() + + assert transformer._refiner_built is True + assert isinstance(transformer.refiner, DummyRefiner) + assert calls["refiner_root"] == ( + "/resolved/hf://Efficient-Large-Model/SANA-WM_bidirectional/refiner" + ) + assert calls["gemma_root"] == ( + "/resolved/hf://Efficient-Large-Model/SANA-WM_bidirectional/refiner/text_encoder" + ) + assert calls["dtype"] is torch.bfloat16 + assert calls["precision"] == "bf16" + assert calls["quant_backend"] == "native" + + +def test_refiner_latent_pack_round_trips() -> None: + """Preserve LTX-2 latent layout across token packing and unpacking.""" + latents = torch.arange(1 * 4 * 3 * 2 * 2, dtype=torch.float32).reshape( + 1, + 4, + 3, + 2, + 2, + ) + + packed = _pack_latents(latents, patch_size=1, patch_size_t=1) + unpacked = _unpack_latents( + packed, + num_frames=3, + height=2, + width=2, + patch_size=1, + patch_size_t=1, + ) + + torch.testing.assert_close(unpacked, latents) + + +def test_refiner_block_size_request_still_executes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Do not turn native refiner block-size requests into a hard failure.""" + + class DummyTransformer(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.config = type("Config", (), {"patch_size": 1, "patch_size_t": 1})() + + refiner = SanaWMLTX2Refiner.__new__(SanaWMLTX2Refiner) + torch.nn.Module.__init__(refiner) + refiner.device = torch.device("cpu") + refiner.dtype = torch.float32 + refiner.transformer = DummyTransformer() + monkeypatch.setattr(refiner, "_prepare_quantization", lambda: None) + monkeypatch.setattr( + refiner, + "_encode_prompt", + lambda _prompt: (torch.zeros((1, 1, 1)), torch.ones((1, 1))), + ) + + def predict_current_x0( + *, + sink: torch.Tensor, + noisy_current: torch.Tensor, + prompt_embeds: torch.Tensor, + prompt_attention_mask: torch.Tensor, + sigma: torch.Tensor, + fps: float, + ) -> torch.Tensor: + del sink, prompt_embeds, prompt_attention_mask, sigma, fps + return torch.zeros_like(_pack_latents(noisy_current)) + + monkeypatch.setattr(refiner, "_predict_current_x0", predict_current_x0) + latents = torch.zeros((1, 2, 3, 1, 1)) + + refined = refiner.refine_latents( + latents, + "demo", + fps=16.0, + block_size=1, + progress=False, + sigmas=(0.5, 0.0), + ) + + assert refined.shape == latents.shape def test_auto_quant_backend_resolves_to_native_backend() -> None: @@ -653,7 +799,7 @@ def test_torch_fp8_backend_rejects_fp4(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) - with pytest.raises(ValueError, match="supports fp8 only"): + with pytest.raises(ValueError, match="accepts fp8 only"): _validate_precision_request( device=torch.device("cuda:0"), stage1_precision="fp4", @@ -685,7 +831,7 @@ def test_torch_fp4_backend_rejects_fp8(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) - with pytest.raises(ValueError, match="supports fp4 only"): + with pytest.raises(ValueError, match="accepts fp4 only"): _validate_precision_request( device=torch.device("cuda:0"), stage1_precision="fp8", @@ -803,7 +949,7 @@ def test_pyproject_entry_point_matches_runner_literal() -> None: def test_pyproject_excludes_upstream_diffusion_package() -> None: - """Do not install the vendored upstream SANA ``diffusion`` package.""" + """Do not install the NVlabs/Sana ``diffusion`` package.""" pyproject = tomllib.loads( Path("integrations/sana/pyproject.toml").read_text(encoding="utf-8") ) @@ -811,6 +957,7 @@ def test_pyproject_excludes_upstream_diffusion_package() -> None: packages = pyproject["tool"]["setuptools"]["packages"]["find"] dependencies = set(pyproject["project"]["dependencies"]) + assert "accelerate>=1.0" in dependencies assert packages["include"] == ["sana_wm*"] assert "diffusers>=0.36" in dependencies assert "transformers>=5.0,<6" in dependencies diff --git a/uv.lock b/uv.lock index 407fb184b..4a76b98c6 100644 --- a/uv.lock +++ b/uv.lock @@ -77,6 +77,27 @@ test = [ { name = "tomli", specifier = ">=2.0" }, ] +[[package]] +name = "accelerate" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform != 'win32' and extra == 'group-11-flashdreams-cuda12') or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "torch", version = "2.12.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'win32' and extra == 'extra-11-flashdreams-dev') or (sys_platform != 'win32' and extra != 'group-11-flashdreams-cuda12') or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "torch", version = "2.12.1+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "sys_platform == 'win32' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8d/75/94cd5d389649578aca399e5aa822637eec18319a1dadc400ffe2f9a7493f/accelerate-1.14.0.tar.gz", hash = "sha256:41b9c4377a54e0b460a959b0defa1b736e4ca0a2373252d9a539964c2afe3c8d", size = 412167, upload-time = "2026-06-11T13:45:52.326Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/db/253133d7e7cb40d3af384bb2f5c0b4a2b7fdcffbc95c688cc67a20a3c103/accelerate-1.14.0-py3-none-any.whl", hash = "sha256:e94390c2863b873be18f623f9df48a0d8fe5eff13ea7f1a00092b0a7904888c6", size = 389246, upload-time = "2026-06-11T13:45:50.477Z" }, +] + [[package]] name = "accessible-pygments" version = "0.0.5" @@ -1573,6 +1594,7 @@ name = "flashdreams-sana-wm" version = "0.1.0" source = { editable = "integrations/sana" } dependencies = [ + { name = "accelerate" }, { name = "diffusers" }, { name = "flashdreams" }, { name = "imageio", extra = ["ffmpeg"] }, @@ -1592,6 +1614,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "accelerate", specifier = ">=1.0" }, { name = "diffusers", specifier = ">=0.36" }, { name = "flashdreams", editable = "flashdreams" }, { name = "imageio", extras = ["ffmpeg"], specifier = ">=2.31" }, From 0f77ccfb33933bda2c5fd001e0fc9091d73d353a Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 16 Jul 2026 19:58:18 -0700 Subject: [PATCH 11/64] Run native SANA refiner under inference mode The native LTX-2 refiner prompt encoder returns inference tensors. Keep the whole refiner denoise path inside torch.inference_mode so diffusers Linear calls do not try to save those tensors for autograd. Add a CPU regression assertion that the refiner prediction callback executes with inference mode enabled. Validation: uv run --package flashdreams-sana-wm pytest integrations/sana/tests/test_smoke.py integrations/sana/tests/test_camera.py. --- integrations/sana/sana_wm/native_refiner.py | 1 + integrations/sana/tests/test_smoke.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/integrations/sana/sana_wm/native_refiner.py b/integrations/sana/sana_wm/native_refiner.py index 084940608..4ea247c9a 100644 --- a/integrations/sana/sana_wm/native_refiner.py +++ b/integrations/sana/sana_wm/native_refiner.py @@ -73,6 +73,7 @@ def __init__( self._quantized = False self.transformer, self.connectors = self._load_diffusers_components() + @torch.inference_mode() def refine_latents( self, sana_latent: Tensor, diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index eae39bf05..ddb05d8f6 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -586,6 +586,7 @@ def __init__(self) -> None: refiner.device = torch.device("cpu") refiner.dtype = torch.float32 refiner.transformer = DummyTransformer() + inference_modes: list[bool] = [] monkeypatch.setattr(refiner, "_prepare_quantization", lambda: None) monkeypatch.setattr( refiner, @@ -603,6 +604,7 @@ def predict_current_x0( fps: float, ) -> torch.Tensor: del sink, prompt_embeds, prompt_attention_mask, sigma, fps + inference_modes.append(torch.is_inference_mode_enabled()) return torch.zeros_like(_pack_latents(noisy_current)) monkeypatch.setattr(refiner, "_predict_current_x0", predict_current_x0) @@ -618,6 +620,7 @@ def predict_current_x0( ) assert refined.shape == latents.shape + assert inference_modes == [True] def test_auto_quant_backend_resolves_to_native_backend() -> None: From 52617e89673ecd3e8eab3e49f7917696db356d95 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 17 Jul 2026 09:28:38 -0700 Subject: [PATCH 12/64] Clean up SANA-WM integration surface Signed-off-by: Aidan Foster --- integrations/sana/README.md | 67 ++- integrations/sana/docs/upstream_reference.md | 103 ---- integrations/sana/pyproject.toml | 2 - integrations/sana/sana_wm/__init__.py | 2 +- integrations/sana/sana_wm/camera.py | 4 +- integrations/sana/sana_wm/config.py | 71 +-- integrations/sana/sana_wm/constants.py | 4 +- .../{native_diffusion.py => diffusion.py} | 26 +- .../sana_wm/{native_quant.py => quant.py} | 53 +- .../sana_wm/{native_refiner.py => refiner.py} | 19 +- integrations/sana/sana_wm/runner.py | 470 +++--------------- integrations/sana/sana_wm/stage1_model.py | 12 +- .../{native_transformer.py => transformer.py} | 87 ++-- .../sana/sana_wm/upstream_reference.py | 90 ---- .../sana/tests/parity_check/README.md | 25 - ...ative_quant_cuda.py => test_quant_cuda.py} | 18 +- integrations/sana/tests/test_smoke.py | 199 +++----- 17 files changed, 269 insertions(+), 983 deletions(-) delete mode 100644 integrations/sana/docs/upstream_reference.md rename integrations/sana/sana_wm/{native_diffusion.py => diffusion.py} (91%) rename integrations/sana/sana_wm/{native_quant.py => quant.py} (91%) rename integrations/sana/sana_wm/{native_refiner.py => refiner.py} (96%) rename integrations/sana/sana_wm/{native_transformer.py => transformer.py} (92%) delete mode 100644 integrations/sana/sana_wm/upstream_reference.py delete mode 100644 integrations/sana/tests/parity_check/README.md rename integrations/sana/tests/{test_native_quant_cuda.py => test_quant_cuda.py} (81%) diff --git a/integrations/sana/README.md b/integrations/sana/README.md index db4830004..a8a9f18d5 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -9,36 +9,28 @@ FlashDreams SANA-WM integration for [SANA-WM](https://huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional), the 2.6B bidirectional camera-controlled world model released from NVlabs/Sana. -The `sana-wm-bidirectional` runner is the native FlashDreams path. It uses -FlashDreams config, runner, pipeline, diffusion-model, scheduler, transformer, -camera-conditioning, VAE decode, and output-writing boundaries. The Stage-1 -DiT module is implemented in this package and loads the public SANA-WM -checkpoint directly; it does not import or install the NVlabs/Sana -`diffusion` package. +The `sana-wm-bidirectional` runner uses FlashDreams config, runner, pipeline, +diffusion-model, scheduler, transformer, camera-conditioning, VAE decode, and +output-writing boundaries. The Stage-1 DiT module in this package loads the +public SANA-WM checkpoint directly. -Current native scope: +Current scope: | component | status | | --- | --- | -| Stage-1 BF16 | Native FlashDreams DiT execution. | -| Stage-1 FP8 | Native PyTorch `_scaled_mm` backend. | -| Stage-1 FP4 | Native Triton quantization plus PyTorch `_scaled_mm`. | -| VAE decode | Native direct `diffusers` LTX2 VAE use with low-memory tiling. | -| LTX-2 refiner | Native direct `diffusers` LTX-2 transformer and Gemma connector use. | - -The separate `sana-wm-upstream-reference` runner is documented only in -`docs/upstream_reference.md`. Keep reference-harness details out of this -native integration README. +| Stage-1 BF16 | FlashDreams DiT execution. | +| Stage-1 FP8 | PyTorch `_scaled_mm` backend. | +| Stage-1 FP4 | Triton quantization plus PyTorch `_scaled_mm`. | +| VAE decode | Direct `diffusers` LTX2 VAE use with low-memory tiling. | +| LTX-2 refiner | Direct `diffusers` LTX-2 transformer and Gemma connector use. | ## Runner | slug | description | | --- | --- | -| `sana-wm-bidirectional` | Native FlashDreams SANA-WM Stage-1 + LTX-2 refiner runner. | -| `sana-wm-native-bidirectional` | Alias for the native FlashDreams runner. | +| `sana-wm-bidirectional` | SANA-WM Stage-1 + LTX-2 refiner runner. | -The FlashDreams package is named `sana_wm` rather than `sana` so it does not -shadow the upstream project name. +The FlashDreams package is named `sana_wm`. ## Setup @@ -49,14 +41,13 @@ project's GPU runtime dependencies: uv sync --package flashdreams-sana-wm --extra dev ``` -The native runner does not require a local NVlabs/Sana checkout. The checkout -is only useful as a source of demo assets such as -`asset/sana_wm/demo_0.png`, `demo_0.txt`, `demo_0_pose.npy`, and -`demo_0_intrinsics.npy`. +The examples below use the public demo image, prompt, camera, and intrinsics +files from the SANA-WM release. Equivalent inputs with the same shapes work as +well. ## Run -The native runner requires explicit intrinsics: +The runner requires explicit intrinsics: ```bash PYTORCH_ALLOC_CONF=expandable_segments:True \ @@ -66,13 +57,13 @@ uv run flashdreams-run sana-wm-bidirectional \ --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ --num-frames 161 \ - --output-dir outputs/sana_wm_native_bf16 + --output-dir outputs/sana_wm_bf16 ``` Expected output: ```text -outputs/sana_wm_native_bf16/sana_wm_generated.mp4 +outputs/sana_wm_bf16/sana_wm_generated.mp4 ``` For Stage-1-only diagnostics, add `--no-refiner True`. @@ -80,14 +71,14 @@ For Stage-1-only diagnostics, add `--no-refiner True`. ## FP8 and FP4 The runner defaults to BF16. Quantized Stage-1 inference is opt-in and uses -native FlashDreams/PyTorch backends by default, not Transformer Engine. +Torch-based backends by default. | option | hardware | backend | | --- | --- | --- | -| `--stage1-precision fp8` | Hopper or newer (`sm_90+`) | native E4M3 `_scaled_mm` | -| `--stage1-precision fp4` | Blackwell (`sm_100+`) | native Triton NVFP4 plus `_scaled_mm` | -| `--quant-backend torch-fp8` | Hopper or newer (`sm_90+`) | force the native FP8 backend | -| `--quant-backend torch-fp4` | Blackwell (`sm_100+`) | force the native FP4 backend | +| `--stage1-precision fp8` | Hopper or newer (`sm_90+`) | E4M3 `_scaled_mm` | +| `--stage1-precision fp4` | Blackwell (`sm_100+`) | Triton NVFP4 plus `_scaled_mm` | +| `--quant-backend torch-fp8` | Hopper or newer (`sm_90+`) | force FP8 | +| `--quant-backend torch-fp4` | Blackwell (`sm_100+`) | force FP4 | FP8 smoke: @@ -99,7 +90,7 @@ uv run flashdreams-run sana-wm-bidirectional \ --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ --num-frames 161 \ - --output-dir outputs/sana_wm_native_fp8 \ + --output-dir outputs/sana_wm_fp8 \ --stage1-precision fp8 \ --refiner-precision fp8 ``` @@ -114,7 +105,7 @@ uv run flashdreams-run sana-wm-bidirectional \ --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ --num-frames 161 \ - --output-dir outputs/sana_wm_native_fp4 \ + --output-dir outputs/sana_wm_fp4 \ --stage1-precision fp4 \ --refiner-precision fp4 ``` @@ -122,12 +113,12 @@ uv run flashdreams-run sana-wm-bidirectional \ ## Tests CPU-safe tests cover import, config boundaries, action parsing, intrinsics, -camera conditioning, native Stage-1 checkpoint schema, native Stage-1 CPU -forward shape, VAE tiling, and native low-precision backend selection: +camera conditioning, Stage-1 checkpoint schema, Stage-1 CPU forward shape, VAE +tiling, and low-precision backend selection: ```bash uv run --extra dev pytest integrations/sana/tests/test_smoke.py ``` -GPU generation and native-vs-reference quality parity checks are heavyweight -manual workflows and should stay out of `ci_cpu`. +GPU generation checks are heavyweight manual workflows and should stay out of +`ci_cpu`. diff --git a/integrations/sana/docs/upstream_reference.md b/integrations/sana/docs/upstream_reference.md deleted file mode 100644 index 21a339fc0..000000000 --- a/integrations/sana/docs/upstream_reference.md +++ /dev/null @@ -1,103 +0,0 @@ - - -# SANA-WM Upstream Reference Harness - -This document covers the temporary upstream-reference runner. It exists only -for parity/debug work while the native FlashDreams integration is being -validated. Do not use this as user-facing documentation for the real SANA-WM -FlashDreams integration. - -## Runner - -| slug | description | -| --- | --- | -| `sana-wm-upstream-reference` | FlashDreams-packaged NVlabs/Sana reference harness. | - -This runner delegates generation to the SANA reference pipeline object. It is -expected to be removed once native generation is validated. - -## Setup - -The reference harness needs a local or importable NVlabs/Sana checkout: - -```bash -git clone https://github.com/NVlabs/Sana ../Sana -cd ../Sana -bash environment_setup.sh sana -conda activate sana -``` - -The runner auto-detects `../Sana`. You can also set `SANA_ROOT` or pass -`--upstream-sana-root /path/to/Sana`. - -## Run - -```bash -flashdreams-run sana-wm-upstream-reference \ - --upstream-sana-root ../Sana \ - --image-path ../Sana/asset/sana_wm/demo_0.png \ - --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ - --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ - --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ - --num-frames 321 \ - --output-dir outputs/sana_wm_reference -``` - -The upstream reference harness can estimate intrinsics with Pi3X when -`--intrinsics-path` is omitted. The native runner currently requires explicit -intrinsics instead. - -## Transformer Engine Reference Notes - -Upstream SANA does not use Transformer Engine here for positional embeddings. -Where the reference path uses Transformer Engine, it is as a quantized -`Linear`/GEMM backend: - -- Stage 1 replaces selected `torch.nn.Linear` layers with - `transformer_engine.pytorch.Linear`, then wraps transformer block or linear - forwards in `te.fp8_autocast(...)`. -- `SANA_WM_STAGE1_QUANT=nvfp4` selects `NVFP4BlockScaling`; `fp8block` selects - `Float8BlockScaling`. -- The default FlashDreams FP4 Stage-1 selector maps to upstream - `self_attn+cross+ffn`, covering self-attention, cross-attention, and MLP - linear layers. In the validated reference demo run, upstream reported 120 - converted linear layers and 20 wrapped transformer blocks. -- The refiner applies the same pattern to eligible LTX-2 transformer linear - layers after skipping input/output projection, audio-only, caption, and time - embedding modules. - -Use `--quant-backend upstream-te` only on this reference harness when comparing -native low-precision behavior against upstream Transformer Engine behavior. - -```bash -flashdreams-run sana-wm-upstream-reference \ - --upstream-sana-root ../Sana \ - --image-path ../Sana/asset/sana_wm/demo_0.png \ - --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ - --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ - --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ - --num-frames 161 \ - --output-dir outputs/sana_wm_reference_fp4 \ - --stage1-precision fp4 \ - --refiner-precision fp4 \ - --quant-backend upstream-te -``` - -FlashDreams already removed a separate Transformer Engine dependency for RoPE -via `flashdreams.core.attention.rope_kernel`, but that replacement is not a -drop-in for SANA FP4/FP8. Native SANA replaces eligible Stage-1 and refiner -`Linear` modules directly, including activation quantization/scales, weight -conversion, GEMM, and output rescaling. - -Relevant FlashDreams components for future hardening: - -- `flashdreams.core.attention.rope_kernel`: the prior narrow TE-replacement - pattern and test strategy. -- `integrations/omnidreams/omnidreams_singleview/python/cosmos_fp8_utils.py`: - FP8 weight quantization and prepared weight/scale aliases for Cosmos blocks. -- `integrations/omnidreams/omnidreams_singleview/src/dit_streaming/kernels/`: - CUDA/CUTLASS FP8 GEMM, BF16 GEMM, INT8 block quantization, and SageAttention3 - FP4 attention/cache kernels. diff --git a/integrations/sana/pyproject.toml b/integrations/sana/pyproject.toml index 780bf446b..2d63b6f77 100644 --- a/integrations/sana/pyproject.toml +++ b/integrations/sana/pyproject.toml @@ -45,8 +45,6 @@ dev = [ [project.entry-points."flashdreams.runner_configs"] "sana-wm-bidirectional" = "sana_wm.config:RUNNER_SANA_WM_BIDIRECTIONAL" -"sana-wm-upstream-reference" = "sana_wm.config:RUNNER_SANA_WM_UPSTREAM_REFERENCE" -"sana-wm-native-bidirectional" = "sana_wm.config:RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL" [tool.setuptools.packages.find] include = ["sana_wm*"] diff --git a/integrations/sana/sana_wm/__init__.py b/integrations/sana/sana_wm/__init__.py index 77dfd6e32..c59615ca6 100644 --- a/integrations/sana/sana_wm/__init__.py +++ b/integrations/sana/sana_wm/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""SANA-WM native and upstream-reference runners for FlashDreams.""" +"""SANA-WM runner for FlashDreams.""" from sana_wm.config import RUNNER_CONFIGS, RUNNER_SANA_WM_BIDIRECTIONAL diff --git a/integrations/sana/sana_wm/camera.py b/integrations/sana/sana_wm/camera.py index 97daf41ba..af8aa406e 100644 --- a/integrations/sana/sana_wm/camera.py +++ b/integrations/sana/sana_wm/camera.py @@ -35,7 +35,7 @@ """Camera-control integration frame rate.""" DEFAULT_TRANSLATION_SPEED = 0.025 -"""Default per-frame translation magnitude used by upstream SANA-WM.""" +"""Default per-frame translation magnitude used by SANA-WM.""" DEFAULT_ROTATION_SPEED_DEG = 0.6 """Default per-frame rotation magnitude in degrees.""" @@ -229,7 +229,7 @@ def action_string_to_c2w( translation_speed: Per-frame translation magnitude. rotation_speed_deg: Per-frame angular magnitude in degrees. pitch_limit_deg: Maximum absolute pitch in degrees. - smooth: Whether to use upstream's velocity smoothing model. + smooth: Whether to use the SANA-WM velocity smoothing model. Returns: ``[F + 1, 4, 4]`` camera-to-world matrices in OpenCV convention. diff --git a/integrations/sana/sana_wm/config.py b/integrations/sana/sana_wm/config.py index 73734b634..5d9a176c8 100644 --- a/integrations/sana/sana_wm/config.py +++ b/integrations/sana/sana_wm/config.py @@ -13,97 +13,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Static configs for SANA-WM native and upstream-reference runners.""" +"""Static configs for the SANA-WM runner.""" from __future__ import annotations -from typing import cast - -from flashdreams.infra.config import derive_config -from flashdreams.infra.diffusion.model import DiffusionModelConfig from flashdreams.infra.diffusion.scheduler import FlowMatchSchedulerConfig from flashdreams.infra.pipeline import StreamInferencePipelineConfig from flashdreams.infra.runner import RunnerConfig -from sana_wm.native_diffusion import SanaWMDiffusionModelConfig -from sana_wm.native_transformer import SanaWMNativeTransformerConfig +from sana_wm.diffusion import SanaWMDiffusionModelConfig from sana_wm.runner import SanaWMRunnerConfig -from sana_wm.upstream_reference import SanaWMUpstreamReferenceTransformerConfig - - -def _reference_pipeline(name: str) -> StreamInferencePipelineConfig: - return StreamInferencePipelineConfig( - name=name, - diffusion_model=DiffusionModelConfig( - transformer=SanaWMUpstreamReferenceTransformerConfig(), - scheduler=FlowMatchSchedulerConfig(), - seed=42, - ), - ) +from sana_wm.transformer import SanaWMTransformerConfig -PIPELINE_SANA_WM_UPSTREAM_REFERENCE = _reference_pipeline("sana-wm-upstream-reference") -"""Schema marker for the upstream-delegating SANA-WM reference runner.""" - PIPELINE_SANA_WM_BIDIRECTIONAL = StreamInferencePipelineConfig( name="sana-wm-bidirectional", diffusion_model=SanaWMDiffusionModelConfig( - transformer=SanaWMNativeTransformerConfig(), + transformer=SanaWMTransformerConfig(), scheduler=FlowMatchSchedulerConfig(), seed=42, ), ) -"""Native FlashDreams SANA-WM pipeline.""" - -PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL = cast( - StreamInferencePipelineConfig, - derive_config(PIPELINE_SANA_WM_BIDIRECTIONAL, name="sana-wm-native-bidirectional"), -) -"""Backward-compatible native SANA-WM pipeline alias.""" - -RUNNER_SANA_WM_UPSTREAM_REFERENCE = SanaWMRunnerConfig( - runner_name=PIPELINE_SANA_WM_UPSTREAM_REFERENCE.name, - description=( - "SANA-WM upstream reference harness (NVlabs/Sana Stage-1 + LTX-2 refiner)." - ), - pipeline=PIPELINE_SANA_WM_UPSTREAM_REFERENCE, -) -"""Explicit SANA-WM upstream reference runner config.""" +"""FlashDreams SANA-WM pipeline.""" RUNNER_SANA_WM_BIDIRECTIONAL = SanaWMRunnerConfig( runner_name=PIPELINE_SANA_WM_BIDIRECTIONAL.name, - description=( - "SANA-WM bidirectional I2V native FlashDreams runner " - "(Stage-1 DiT + LTX-2 refiner)." - ), + description="SANA-WM bidirectional I2V runner (Stage-1 DiT + LTX-2 refiner).", pipeline=PIPELINE_SANA_WM_BIDIRECTIONAL, - execution_backend="native-flashdreams", -) -"""SANA-WM native FlashDreams runner config.""" - -RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL = SanaWMRunnerConfig( - runner_name=PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL.name, - description="SANA-WM native FlashDreams pipeline alias.", - pipeline=PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL, - execution_backend="native-flashdreams", ) -"""Backward-compatible native SANA-WM FlashDreams runner alias.""" +"""SANA-WM runner config.""" RUNNER_CONFIGS: dict[str, RunnerConfig] = { - cfg.runner_name: cfg - for cfg in ( - RUNNER_SANA_WM_BIDIRECTIONAL, - RUNNER_SANA_WM_UPSTREAM_REFERENCE, - RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL, - ) + cfg.runner_name: cfg for cfg in (RUNNER_SANA_WM_BIDIRECTIONAL,) } """SANA-WM runner configs keyed by ``runner_name``.""" __all__ = [ "PIPELINE_SANA_WM_BIDIRECTIONAL", - "PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL", - "PIPELINE_SANA_WM_UPSTREAM_REFERENCE", "RUNNER_CONFIGS", "RUNNER_SANA_WM_BIDIRECTIONAL", - "RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL", - "RUNNER_SANA_WM_UPSTREAM_REFERENCE", ] diff --git a/integrations/sana/sana_wm/constants.py b/integrations/sana/sana_wm/constants.py index 5251099ca..d45f0c974 100644 --- a/integrations/sana/sana_wm/constants.py +++ b/integrations/sana/sana_wm/constants.py @@ -22,7 +22,7 @@ """Default Stage-1 SANA-WM DiT checkpoint.""" SANA_WM_CONFIG_PATH = f"hf://{SANA_WM_HF_REPO}/config.yaml" -"""Default upstream inference YAML.""" +"""Default inference YAML.""" SANA_WM_REFINER_ROOT = f"hf://{SANA_WM_HF_REPO}/refiner" """Default LTX-2 refiner root.""" @@ -46,4 +46,4 @@ """Frame rate used by the public SANA-WM examples.""" DEFAULT_ACTION = "w-100,dw-60,w-100,aw-60" -"""Default SANA-WM demo action string from upstream ``demo_0``.""" +"""Default SANA-WM demo action string.""" diff --git a/integrations/sana/sana_wm/native_diffusion.py b/integrations/sana/sana_wm/diffusion.py similarity index 91% rename from integrations/sana/sana_wm/native_diffusion.py rename to integrations/sana/sana_wm/diffusion.py index 35270d4a0..2c26149b3 100644 --- a/integrations/sana/sana_wm/native_diffusion.py +++ b/integrations/sana/sana_wm/diffusion.py @@ -27,9 +27,9 @@ from torch import Tensor from flashdreams.infra.diffusion.model import DiffusionModel, DiffusionModelConfig -from sana_wm.native_transformer import ( - SanaWMNativeTransformer, - SanaWMNativeTransformerCache, +from sana_wm.transformer import ( + SanaWMTransformer, + SanaWMTransformerCache, ) @@ -42,31 +42,31 @@ class SanaWMDiffusionModelConfig(DiffusionModelConfig): ) -class SanaWMDiffusionModel(DiffusionModel[SanaWMNativeTransformerCache]): +class SanaWMDiffusionModel(DiffusionModel[SanaWMTransformerCache]): """Run SANA-WM Stage-1 denoising behind the FlashDreams diffusion boundary.""" - transformer: SanaWMNativeTransformer + transformer: SanaWMTransformer def __init__(self, config: SanaWMDiffusionModelConfig) -> None: super().__init__(config) - if not isinstance(self.transformer, SanaWMNativeTransformer): + if not isinstance(self.transformer, SanaWMTransformer): raise TypeError( - "SanaWMDiffusionModel requires SanaWMNativeTransformerConfig." + "SanaWMDiffusionModel requires SanaWMTransformerConfig." ) def generate( self, autoregressive_index: int, - cache: SanaWMNativeTransformerCache, + cache: SanaWMTransformerCache, input: Any = None, - ) -> tuple[Tensor, "DiffusionModel.FinalState[SanaWMNativeTransformerCache]"]: + ) -> tuple[Tensor, "DiffusionModel.FinalState[SanaWMTransformerCache]"]: """Run SANA-WM's first-frame-pinned LTX Euler denoising loop.""" del input if autoregressive_index != 0: - raise ValueError("SANA-WM bidirectional native inference has one AR step.") + raise ValueError("SANA-WM bidirectional inference has one AR step.") conditioning = cache.conditioning if conditioning is None: - raise RuntimeError("SANA-WM native diffusion cache has no conditioning.") + raise RuntimeError("SANA-WM diffusion cache has no conditioning.") cache.start(autoregressive_index) latents = self.transformer.initial_latents(conditioning) @@ -92,7 +92,7 @@ def generate( def finalize( self, - final_state: "DiffusionModel.FinalState[SanaWMNativeTransformerCache]", + final_state: "DiffusionModel.FinalState[SanaWMTransformerCache]", ) -> None: """Finalize the one-shot cache without an extra model forward.""" final_state.cache.finalize(final_state.autoregressive_index) @@ -100,7 +100,7 @@ def finalize( def _sample_ltx_euler( self, latents: Tensor, - cache: SanaWMNativeTransformerCache, + cache: SanaWMTransformerCache, ) -> Tensor: conditioning = cache.conditioning assert conditioning is not None diff --git a/integrations/sana/sana_wm/native_quant.py b/integrations/sana/sana_wm/quant.py similarity index 91% rename from integrations/sana/sana_wm/native_quant.py rename to integrations/sana/sana_wm/quant.py index 50b878223..3ff3ee0e1 100644 --- a/integrations/sana/sana_wm/native_quant.py +++ b/integrations/sana/sana_wm/quant.py @@ -13,15 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Native low-precision linear helpers for SANA-WM. - -Upstream SANA's FP8/FP4 path uses Transformer Engine by replacing selected -``torch.nn.Linear`` modules and entering ``te.fp8_autocast`` around the -corresponding transformer blocks. This module provides TE-free replacements for -the same replacement point: inference-only FP8 and NVFP4 Linear modules backed -by PyTorch's native ``torch._scaled_mm`` kernel, with Triton used for NVFP4 -activation quantization. -""" +"""Low-precision linear helpers for SANA-WM.""" from __future__ import annotations @@ -43,7 +35,7 @@ @dataclass(frozen=True) class TorchScaledMMFP8Recipe: - """Marker recipe used while patching upstream SANA's TE recipe hooks.""" + """FP8 recipe marker for SANA-WM Linear replacement.""" name: str = "torch_scaled_mm_fp8" precision: Literal["fp8"] = "fp8" @@ -51,13 +43,13 @@ class TorchScaledMMFP8Recipe: @dataclass(frozen=True) class TorchScaledMMFP4Recipe: - """Marker recipe used while patching upstream SANA's NVFP4 hooks.""" + """FP4 recipe marker for SANA-WM Linear replacement.""" name: str = "torch_scaled_mm_fp4" precision: Literal["fp4"] = "fp4" -NativeQuantRecipe = TorchScaledMMFP8Recipe | TorchScaledMMFP4Recipe +QuantRecipe = TorchScaledMMFP8Recipe | TorchScaledMMFP4Recipe @triton.jit @@ -184,7 +176,7 @@ class TorchScaledMMFP8Linear(nn.Module): scales. At runtime it quantizes the flattened activation rows to E4M3 with per-row scales, then calls ``torch._scaled_mm`` and returns BF16 output. This intentionally mirrors the shape contract of ``nn.Linear`` so it can - replace eligible upstream SANA linear layers without changing call sites. + replace eligible SANA-WM linear layers without changing call sites. """ in_features: int @@ -410,7 +402,7 @@ def replace_linear_with_torch_fp8( prefix: str = "", ) -> tuple[int, int]: """Replace eligible ``nn.Linear`` modules with ``TorchScaledMMFP8Linear``.""" - return _replace_linear_with_native_quant( + return _replace_linear_with_quant( module, recipe=TorchScaledMMFP8Recipe(), params_dtype=params_dtype, @@ -430,7 +422,7 @@ def replace_linear_with_torch_fp4( prefix: str = "", ) -> tuple[int, int]: """Replace eligible ``nn.Linear`` modules with ``TorchScaledMMFP4Linear``.""" - return _replace_linear_with_native_quant( + return _replace_linear_with_quant( module, recipe=TorchScaledMMFP4Recipe(), params_dtype=params_dtype, @@ -440,22 +432,17 @@ def replace_linear_with_torch_fp4( ) -def replace_linear_with_native_quant( +def replace_linear_with_quant( module: nn.Module, *, - recipe: NativeQuantRecipe, + recipe: QuantRecipe, params_dtype: torch.dtype, skip_patterns: tuple[str, ...], include_patterns: tuple[str, ...] | None = None, prefix: str = "", ) -> tuple[int, int]: - """Replace eligible ``nn.Linear`` modules with the requested native backend. - - The signature intentionally matches upstream SANA's - ``_replace_linear_with_te_nvfp4`` helper so the runner can patch the backend - without modifying upstream source files. - """ - return _replace_linear_with_native_quant( + """Replace eligible ``nn.Linear`` modules with the requested backend.""" + return _replace_linear_with_quant( module, recipe=recipe, params_dtype=params_dtype, @@ -465,10 +452,10 @@ def replace_linear_with_native_quant( ) -def _replace_linear_with_native_quant( +def _replace_linear_with_quant( module: nn.Module, *, - recipe: NativeQuantRecipe, + recipe: QuantRecipe, params_dtype: torch.dtype, skip_patterns: tuple[str, ...], include_patterns: tuple[str, ...] | None, @@ -510,11 +497,11 @@ def _replace_linear_with_native_quant( out_dtype=out_dtype, ) else: - raise ValueError(f"Unsupported native quant recipe: {recipe!r}.") + raise ValueError(f"Unsupported SANA-WM quant recipe: {recipe!r}.") setattr(module, name, replacement) converted += 1 continue - child_converted, child_skipped = _replace_linear_with_native_quant( + child_converted, child_skipped = _replace_linear_with_quant( child, recipe=recipe, params_dtype=params_dtype, @@ -533,22 +520,22 @@ def _name_matches(patterns: tuple[str, ...], name: str) -> bool: def _require_scaled_mm() -> None: if not hasattr(torch, "_scaled_mm"): - raise RuntimeError("torch._scaled_mm is required for native SANA quantization.") + raise RuntimeError("torch._scaled_mm is required for SANA-WM quantization.") def _require_fp8_dtype() -> None: if not hasattr(torch, "float8_e4m3fn"): - raise RuntimeError("torch.float8_e4m3fn is required for the native SANA FP8 backend.") + raise RuntimeError("torch.float8_e4m3fn is required for the SANA-WM FP8 backend.") def _require_fp4_dtype() -> None: if not hasattr(torch, "float4_e2m1fn_x2"): raise RuntimeError( - "torch.float4_e2m1fn_x2 is required for the native SANA FP4 backend." + "torch.float4_e2m1fn_x2 is required for the SANA-WM FP4 backend." ) if not hasattr(torch, "float8_e4m3fn"): raise RuntimeError( - "torch.float8_e4m3fn scales are required for the native SANA FP4 backend." + "torch.float8_e4m3fn scales are required for the SANA-WM FP4 backend." ) @@ -558,7 +545,7 @@ def _require_fp4_dtype() -> None: "TorchScaledMMFP8Linear", "TorchScaledMMFP8Recipe", "quantize_nvfp4_swizzled", - "replace_linear_with_native_quant", + "replace_linear_with_quant", "replace_linear_with_torch_fp4", "replace_linear_with_torch_fp8", ] diff --git a/integrations/sana/sana_wm/native_refiner.py b/integrations/sana/sana_wm/refiner.py similarity index 96% rename from integrations/sana/sana_wm/native_refiner.py rename to integrations/sana/sana_wm/refiner.py index 4ea247c9a..c29844d93 100644 --- a/integrations/sana/sana_wm/native_refiner.py +++ b/integrations/sana/sana_wm/refiner.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Native LTX-2 refiner used by the SANA-WM integration.""" +"""LTX-2 refiner used by the SANA-WM integration.""" from __future__ import annotations @@ -27,14 +27,14 @@ from loguru import logger from torch import Tensor -from sana_wm.native_quant import ( +from sana_wm.quant import ( TorchScaledMMFP4Recipe, TorchScaledMMFP8Recipe, - replace_linear_with_native_quant, + replace_linear_with_quant, ) Precision = Literal["bf16", "fp8", "fp4"] -QuantBackend = Literal["auto", "upstream-te", "torch-fp8", "torch-fp4", "native"] +QuantBackend = Literal["auto", "torch", "torch-fp8", "torch-fp4"] _REFINER_QUANT_SKIP_DEFAULTS = ( r"^proj_in$", @@ -59,7 +59,7 @@ def __init__( dtype: torch.dtype, device: torch.device | str, precision: Precision = "bf16", - quant_backend: QuantBackend = "native", + quant_backend: QuantBackend = "torch", text_max_sequence_length: int = 1024, ) -> None: super().__init__() @@ -178,17 +178,12 @@ def _load_diffusers_components(self) -> tuple[nn.Module, nn.Module]: def _prepare_quantization(self) -> None: if self._quantized or self.precision == "bf16": return - if self.quant_backend == "upstream-te": - raise ValueError( - "Native SANA-WM refiner execution does not use Transformer Engine; " - "select --quant-backend native, torch-fp8, or torch-fp4." - ) recipe = ( TorchScaledMMFP8Recipe() if self.precision == "fp8" else TorchScaledMMFP4Recipe() ) - converted, skipped = replace_linear_with_native_quant( + converted, skipped = replace_linear_with_quant( self.transformer, recipe=recipe, params_dtype=self.dtype, @@ -196,7 +191,7 @@ def _prepare_quantization(self) -> None: ) if converted <= 0: raise RuntimeError( - f"SANA-WM native refiner {self.precision} converted no Linear " + f"SANA-WM refiner {self.precision} converted no Linear " f"layers; skipped={skipped}." ) self._quantized = True diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index f0266c667..02d1163d0 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -13,18 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""SANA-WM bidirectional runner and upstream-reference harness.""" +"""SANA-WM bidirectional runner.""" from __future__ import annotations -import importlib import os -import sys from collections.abc import Iterator, Mapping from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path -from types import ModuleType from typing import Literal import numpy as np @@ -54,30 +51,20 @@ SANA_WM_REFINER_ROOT, SANA_WM_VAE_TEMPORAL_COMPRESSION, ) -from sana_wm.native_transformer import SanaWMNativeTransformer - -SamplingAlgo = Literal[ - "auto", - "flow_euler_ltx", - "flow_euler", - "flow_dpm-solver", - "chunk_flow_euler", - "self_forcing", -] -"""Sampling algorithms exposed by upstream SANA-WM inference.""" +from sana_wm.transformer import SanaWMTransformer + +SamplingAlgo = Literal["auto", "flow_euler_ltx"] +"""Sampling algorithms exposed by the SANA-WM runner.""" Precision = Literal["bf16", "fp8", "fp4"] """SANA-WM Stage-1/refiner precision modes.""" -QuantBackend = Literal["auto", "upstream-te", "torch-fp8", "torch-fp4", "native"] +QuantBackend = Literal["auto", "torch", "torch-fp8", "torch-fp4"] """Low-precision linear backend for FP8/FP4 SANA-WM paths.""" -ResolvedQuantBackend = Literal["upstream-te", "torch-fp8", "torch-fp4", "native"] +ResolvedQuantBackend = Literal["torch", "torch-fp8", "torch-fp4"] """Concrete low-precision linear backend selected after resolving ``auto``.""" -ExecutionBackend = Literal["upstream-reference", "native-flashdreams"] -"""Execution path selected by a SANA-WM runner config.""" - @dataclass(kw_only=True) class SanaWMRunnerConfig(RunnerConfig): @@ -85,12 +72,8 @@ class SanaWMRunnerConfig(RunnerConfig): _target: type["SanaWMRunner"] = field(default_factory=lambda: SanaWMRunner) - upstream_sana_root: Path | None = None - """Path to a cloned NVlabs/Sana checkout. ``None`` auto-detects - ``$SANA_ROOT`` / ``../Sana`` or imports an installed Sana package.""" - device: str = "auto" - """Torch device for upstream SANA-WM. ``"auto"`` picks CUDA when available.""" + """Torch device for SANA-WM. ``"auto"`` picks CUDA when available.""" image_path: Path | None = None """Path to the first-frame RGB image. Required at ``run()`` time.""" @@ -106,8 +89,7 @@ class SanaWMRunnerConfig(RunnerConfig): intrinsics_path: Path | None = None """Optional ``.npy`` intrinsics shaped ``[3, 3]``, ``[F, 3, 3]``, - ``[4]``, or ``[F, 4]``. When absent, upstream estimates intrinsics - with Pi3X.""" + ``[4]``, or ``[F, 4]``.""" action: str | None = DEFAULT_ACTION """Action DSL used when ``camera_path`` is not provided.""" @@ -119,7 +101,7 @@ class SanaWMRunnerConfig(RunnerConfig): """Per-frame action rotation speed in degrees.""" name: str = "sana_wm" - """Output filename stem passed to upstream ``write_video``.""" + """Output filename stem.""" num_frames: int = 161 """Requested output frames before LTX2-VAE stride snapping.""" @@ -137,22 +119,7 @@ class SanaWMRunnerConfig(RunnerConfig): """Optional scheduler flow-shift override.""" sampling_algo: SamplingAlgo = "auto" - """Stage-1 sampler. ``"auto"`` follows upstream config defaults.""" - - chunk_interval_k: float | None = None - """ChunkFlowEuler interval ratio override.""" - - num_cached_blocks: int = 2 - """Self-forcing sampler cached-block count.""" - - sink_token: bool = False - """Whether self-forcing keeps chunk 0 as a permanent sink.""" - - num_frame_per_block: int = 3 - """Self-forcing latent frames per AR block.""" - - denoising_step_list: str = "" - """Comma-separated self-forcing timesteps ending in ``0``.""" + """Stage-1 sampler. ``"auto"`` uses ``"flow_euler_ltx"``.""" save_stage1: bool = False """Also decode the unrefined Stage-1 latent when the refiner is enabled.""" @@ -163,11 +130,8 @@ class SanaWMRunnerConfig(RunnerConfig): seed: int = 42 """Stage-1 random seed.""" - no_action_overlay: bool = True - """Skip upstream action-overlay compositing on generated videos.""" - config_path: str = SANA_WM_CONFIG_PATH - """Upstream inference YAML path or ``hf://`` URI.""" + """SANA-WM inference YAML path or ``hf://`` URI.""" model_path: str = SANA_WM_MODEL_PATH """Stage-1 checkpoint path or ``hf://`` URI.""" @@ -185,15 +149,9 @@ class SanaWMRunnerConfig(RunnerConfig): Ignored when ``no_refiner`` is ``True``.""" quant_backend: QuantBackend = "auto" - """Backend for quantized linear layers. ``"auto"`` uses the native - FlashDreams/PyTorch low-precision path. ``"upstream-te"`` is accepted only - by the upstream reference harness. ``"torch-fp8"`` and ``"torch-fp4"`` - select one native ``torch._scaled_mm`` replacement explicitly; ``"native"`` - allows both native FP8 and native FP4.""" - - execution_backend: ExecutionBackend = "upstream-reference" - """Whether this config runs the upstream reference harness or the native - FlashDreams path.""" + """Backend for quantized linear layers. ``"auto"`` and ``"torch"`` allow + both FP8 and FP4 Torch replacements. ``"torch-fp8"`` and ``"torch-fp4"`` + select one ``torch._scaled_mm`` replacement explicitly.""" refiner_root: str = SANA_WM_REFINER_ROOT """LTX-2 refiner root path or ``hf://`` URI.""" @@ -224,7 +182,7 @@ class SanaWMRunnerConfig(RunnerConfig): class SanaWMRunner(Runner[SanaWMRunnerConfig, object]): - """CLI driver for SANA-WM native and upstream-reference configs.""" + """CLI driver for SANA-WM configs.""" config: SanaWMRunnerConfig @@ -254,7 +212,7 @@ def _resolve_prompt(self) -> str: return prompt def _resolve_device(self) -> torch.device: - """Return the device used by upstream SANA-WM.""" + """Return the device used by SANA-WM.""" if self.config.device == "auto": if torch.cuda.is_available(): return torch.device(f"cuda:{self.local_rank}") @@ -280,55 +238,27 @@ def _resolve_trajectory(self) -> np.ndarray: rotation_speed_deg=self.config.rotation_speed_deg, ) - def _denoising_steps(self) -> list[int] | None: - """Parse optional self-forcing denoising timesteps.""" - if not self.config.denoising_step_list: - return None - steps = [ - int(item.strip()) - for item in self.config.denoising_step_list.split(",") - if item.strip() - ] - if not steps or steps[-1] != 0: - raise ValueError("--denoising-step-list must end with 0.") - return steps - def run(self) -> None: """Run SANA-WM bidirectional inference and write outputs.""" - if self.config.execution_backend == "native-flashdreams": - self._run_native() - return - self._run_upstream_reference() - - def _run_native(self) -> None: - """Run the native FlashDreams SANA-WM pipeline.""" cfg = self.config if cfg.image_path is None: raise ValueError("SanaWMRunner requires --image-path.") - if not cfg.no_action_overlay: - raise ValueError( - "Native SANA-WM does not include the upstream action overlay. " - "Use --no-action-overlay True." - ) device = self._resolve_device() - quant_backend: QuantBackend = ( - "native" if cfg.quant_backend == "auto" else cfg.quant_backend + quant_backend = _resolve_quant_backend( + cfg.quant_backend, + _active_quantized_precisions( + stage1_precision=cfg.stage1_precision, + refiner_precision=cfg.refiner_precision, + refiner_enabled=not cfg.no_refiner, + ), ) _validate_precision_request( device=device, stage1_precision=cfg.stage1_precision, refiner_precision=cfg.refiner_precision, refiner_enabled=not cfg.no_refiner, - quant_backend=quant_backend, - ) - resolved_quant_backend = _resolve_quant_backend( - quant_backend, - _active_quantized_precisions( - stage1_precision=cfg.stage1_precision, - refiner_precision=cfg.refiner_precision, - refiner_enabled=not cfg.no_refiner, - ), + quant_backend=cfg.quant_backend, ) precision_env = _precision_env_updates( stage1_precision=cfg.stage1_precision, @@ -337,24 +267,22 @@ def _run_native(self) -> None: ) with _temporary_environment(precision_env): prompt = self._resolve_prompt() - image, c2w, intrinsics_vec4, num_frames = self._prepare_native_inputs( - device - ) - pipeline_cfg = _native_pipeline_config( + image, c2w, intrinsics_vec4, num_frames = self._prepare_inputs() + pipeline_cfg = _pipeline_config( cfg, quant_backend=quant_backend, ) pipeline = pipeline_cfg.setup().to(device).eval() transformer = pipeline.diffusion_model.transformer - if not isinstance(transformer, SanaWMNativeTransformer): + if not isinstance(transformer, SanaWMTransformer): raise TypeError( - "Native SANA-WM runner resolved a non-SANA transformer: " + "SANA-WM runner resolved a non-SANA transformer: " f"{type(transformer).__name__}." ) - sampling_algo = self._native_sampling_algo() + sampling_algo = self._sampling_algo() if sampling_algo != "flow_euler_ltx": raise ValueError( - "Native SANA-WM requires flow_euler_ltx for the " + "SANA-WM requires flow_euler_ltx for the " f"bidirectional runner; got {sampling_algo!r}." ) camera = prepare_camera( @@ -397,7 +325,7 @@ def _run_native(self) -> None: transformer.release_refiner_runtime() elif cfg.save_stage1: logger.info( - "Native SANA-WM is already running without the refiner; " + "SANA-WM is already running without the refiner; " "--save-stage1 does not create an extra output." ) @@ -415,182 +343,8 @@ def _run_native(self) -> None: cfg.fps, ) - def _run_upstream_reference(self) -> None: - """Run the explicit upstream SANA-WM reference harness.""" - cfg = self.config - if cfg.image_path is None: - raise ValueError("SanaWMRunner requires --image-path.") - - device = self._resolve_device() - _validate_precision_request( - device=device, - stage1_precision=cfg.stage1_precision, - refiner_precision=cfg.refiner_precision, - refiner_enabled=not cfg.no_refiner, - quant_backend=cfg.quant_backend, - ) - resolved_quant_backend = _resolve_quant_backend( - cfg.quant_backend, - _active_quantized_precisions( - stage1_precision=cfg.stage1_precision, - refiner_precision=cfg.refiner_precision, - refiner_enabled=not cfg.no_refiner, - ), - ) - if ( - _active_quantized_precisions( - stage1_precision=cfg.stage1_precision, - refiner_precision=cfg.refiner_precision, - refiner_enabled=not cfg.no_refiner, - ) - and resolved_quant_backend != "upstream-te" - ): - raise ValueError( - "The upstream reference runner uses upstream SANA's own " - "quantized path. Pass --quant-backend upstream-te for " - "FP8/FP4 reference comparisons, or use sana-wm-bidirectional " - "for the native FlashDreams low-precision backend." - ) - - precision_env = _precision_env_updates( - stage1_precision=cfg.stage1_precision, - refiner_precision=cfg.refiner_precision, - refiner_enabled=not cfg.no_refiner, - ) - with _temporary_environment(precision_env): - upstream = _import_upstream_sana(cfg.upstream_sana_root) - prompt = self._resolve_prompt() - - image = upstream.Image.open(cfg.image_path).convert("RGB") - c2w_full = self._resolve_trajectory() - num_frames = min(cfg.num_frames, c2w_full.shape[0]) - snapped = upstream._snap_num_frames( - num_frames, - stride=SANA_WM_VAE_TEMPORAL_COMPRESSION, - upper_bound=c2w_full.shape[0], - ) - if snapped != cfg.num_frames and self.is_rank_zero: - logger.warning( - "SANA-WM requires num_frames = 8k+1; requested {} snapped to {} " - "(trajectory has {} frames).", - cfg.num_frames, - snapped, - c2w_full.shape[0], - ) - num_frames = snapped - c2w = c2w_full[:num_frames] - - cropped, src_size, resized_size, crop_offset = ( - upstream.resize_and_center_crop( - image, - target_h=DEFAULT_VIDEO_HEIGHT, - target_w=DEFAULT_VIDEO_WIDTH, - ) - ) - if cfg.intrinsics_path is None: - intrinsics_src = np.broadcast_to( - upstream.estimate_intrinsics_with_pi3x( - image, device, upstream.get_root_logger() - ), - (num_frames, 4), - ).copy() - else: - intrinsics_src = upstream.load_intrinsics( - cfg.intrinsics_path, num_frames - ) - intrinsics_vec4 = upstream.transform_intrinsics_for_crop( - intrinsics_src, src_size, resized_size, crop_offset - ) - - inference_config = upstream.pyrallis.parse( - config_class=upstream.InferenceConfig, - config_path=upstream.resolve_hf_path(cfg.config_path), - args=[], - ) - refiner = ( - None - if cfg.no_refiner - else upstream.RefinerSettings( - root=cfg.refiner_root, - gemma_root=cfg.refiner_gemma_root, - sink_size=cfg.sink_size, - seed=cfg.refiner_seed, - block_size=cfg.refiner_block_size, - kv_max_frames=cfg.refiner_kv_max_frames, - ) - ) - pipeline = upstream.SanaWMPipeline( - config=inference_config, - model_path=upstream.resolve_hf_path(cfg.model_path), - device=device, - refiner=refiner, - offload_vae=cfg.offload_vae, - offload_refiner=cfg.offload_refiner, - offload_text_encoder=cfg.offload_text_encoder, - logger=upstream.get_root_logger(), - ) - - sampling_algo = cfg.sampling_algo - if sampling_algo == "auto": - sampling_algo = ( - inference_config.scheduler.vis_sampler - if inference_config.scheduler.vis_sampler - in {"chunk_flow_euler", "self_forcing"} - else "flow_euler_ltx" - ) - params = upstream.GenerationParams( - num_frames=num_frames, - fps=cfg.fps, - step=cfg.step, - cfg_scale=cfg.cfg_scale, - flow_shift=cfg.flow_shift, - seed=cfg.seed, - negative_prompt=cfg.negative_prompt, - sampling_algo=sampling_algo, - chunk_interval_k=cfg.chunk_interval_k, - num_cached_blocks=cfg.num_cached_blocks, - sink_token=cfg.sink_token, - num_frame_per_block=cfg.num_frame_per_block, - denoising_step_list=self._denoising_steps(), - save_stage1=cfg.save_stage1, - ) - - result = pipeline.generate(cropped, prompt, c2w, intrinsics_vec4, params) - video_hwc = result["video"] - if not cfg.no_action_overlay: - video_hwc = upstream.apply_overlay(video_hwc, result["c2w"]) - - if not self.is_rank_zero: - return - upstream.write_video( - cfg.output_dir, - cfg.name, - video_hwc, - params.fps, - upstream.get_root_logger(), - ) - - stage1_video = result.get("stage1_video") - if stage1_video is not None: - stage1_hwc = stage1_video - if not cfg.no_action_overlay: - stage1_hwc = upstream.apply_overlay( - stage1_hwc, result["stage1_c2w"] - ) - upstream.write_video( - cfg.output_dir, - f"{cfg.name}_stage1", - stage1_hwc, - params.fps, - upstream.get_root_logger(), - ) - - def _prepare_native_inputs( - self, - device: torch.device, - ) -> tuple[object, np.ndarray, np.ndarray, int]: - """Load/crop the input image and prepare c2w/intrinsics for native run.""" - del device + def _prepare_inputs(self) -> tuple[object, np.ndarray, np.ndarray, int]: + """Load/crop the input image and prepare c2w/intrinsics.""" from PIL import Image cfg = self.config @@ -630,10 +384,7 @@ def _prepare_native_inputs( ) ) if cfg.intrinsics_path is None: - raise ValueError( - "The native SANA-WM runner currently requires --intrinsics-path. " - "Use sana-wm-upstream-reference if you need Pi3X intrinsics estimation." - ) + raise ValueError("SanaWMRunner requires --intrinsics-path.") intrinsics_src = load_intrinsics(cfg.intrinsics_path, num_frames) intrinsics_vec4 = transform_intrinsics_for_crop( intrinsics_src, @@ -643,19 +394,19 @@ def _prepare_native_inputs( ) return cropped, c2w, intrinsics_vec4, num_frames - def _native_sampling_algo(self) -> SamplingAlgo: - """Resolve the native sampler selection.""" + def _sampling_algo(self) -> SamplingAlgo: + """Resolve the sampler selection.""" if self.config.sampling_algo == "auto": return "flow_euler_ltx" return self.config.sampling_algo -def _native_pipeline_config( +def _pipeline_config( cfg: SanaWMRunnerConfig, *, - quant_backend: QuantBackend, + quant_backend: ResolvedQuantBackend, ) -> StreamInferencePipelineConfig: - """Apply CLI runtime fields to the native SANA-WM pipeline literal.""" + """Apply CLI runtime fields to the SANA-WM pipeline literal.""" return derive_config( cfg.pipeline, diffusion_model=dict( @@ -788,19 +539,8 @@ def _validate_precision_request( _validate_torch_fp8_backend(quantized) elif resolved_backend == "torch-fp4": _validate_torch_fp4_backend(quantized) - elif resolved_backend == "native": - _validate_native_quant_backend(quantized) - elif resolved_backend == "upstream-te": - if "fp8" in quantized and not _torch_cuda_version_at_least(12, 9): - cuda_version = torch.version.cuda or "unknown" - raise ValueError( - "SANA-WM fp8 precision uses upstream Sana's Transformer Engine " - "Float8BlockScaling path, which requires CUDA 12.9 or newer. " - f"This PyTorch environment reports CUDA {cuda_version}. Use " - "--quant-backend torch-fp8 or --quant-backend native to try " - "the native PyTorch FP8 backend." - ) - _validate_transformer_engine(quantized) + elif resolved_backend == "torch": + _validate_quant_backend(quantized) else: raise ValueError(f"Unsupported SANA-WM quant backend: {quant_backend!r}.") @@ -822,62 +562,48 @@ def _resolve_quant_backend( quant_backend: QuantBackend, quantized_precisions: list[Precision], ) -> ResolvedQuantBackend: - """Resolve ``auto`` to the native low-precision backend.""" + """Resolve ``auto`` to the Torch low-precision backend.""" del quantized_precisions if quant_backend == "auto": - return "native" - if quant_backend in {"upstream-te", "torch-fp8", "torch-fp4", "native"}: + return "torch" + if quant_backend in {"torch", "torch-fp8", "torch-fp4"}: return quant_backend raise ValueError(f"Unsupported SANA-WM quant backend: {quant_backend!r}.") -def _torch_cuda_version_at_least(major: int, minor: int) -> bool: - """Return whether ``torch.version.cuda`` is at least ``major.minor``.""" - version = torch.version.cuda - if version is None: - return False - parts = version.split(".") - try: - current_major = int(parts[0]) - current_minor = int(parts[1]) if len(parts) > 1 else 0 - except (ValueError, IndexError): - return False - return (current_major, current_minor) >= (major, minor) - - def _validate_torch_fp8_backend(precisions: list[Precision]) -> None: - """Validate the native PyTorch scaled-MM backend request.""" + """Validate the PyTorch scaled-MM backend request.""" unsupported = sorted({precision for precision in precisions if precision != "fp8"}) if unsupported: raise ValueError( "SANA-WM --quant-backend torch-fp8 accepts fp8 only; " f"unsupported requested precision(s): {', '.join(unsupported)}. " - "Use --quant-backend torch-fp4 or --quant-backend native for fp4." + "Use --quant-backend torch-fp4 or --quant-backend torch for fp4." ) - _validate_native_fp8_primitives() + _validate_torch_fp8_primitives() def _validate_torch_fp4_backend(precisions: list[Precision]) -> None: - """Validate the native PyTorch NVFP4 backend request.""" + """Validate the PyTorch NVFP4 backend request.""" unsupported = sorted({precision for precision in precisions if precision != "fp4"}) if unsupported: raise ValueError( "SANA-WM --quant-backend torch-fp4 accepts fp4 only; " f"unsupported requested precision(s): {', '.join(unsupported)}. " - "Use --quant-backend torch-fp8 or --quant-backend native for fp8." + "Use --quant-backend torch-fp8 or --quant-backend torch for fp8." ) - _validate_native_fp4_primitives() + _validate_torch_fp4_primitives() -def _validate_native_quant_backend(precisions: list[Precision]) -> None: - """Validate native PyTorch low-precision primitives.""" +def _validate_quant_backend(precisions: list[Precision]) -> None: + """Validate PyTorch low-precision primitives.""" if "fp8" in precisions: - _validate_native_fp8_primitives() + _validate_torch_fp8_primitives() if "fp4" in precisions: - _validate_native_fp4_primitives() + _validate_torch_fp4_primitives() -def _validate_native_fp8_primitives() -> None: +def _validate_torch_fp8_primitives() -> None: if not hasattr(torch, "_scaled_mm") or not hasattr(torch, "float8_e4m3fn"): raise RuntimeError( "SANA-WM --quant-backend torch-fp8 requires PyTorch with " @@ -885,7 +611,7 @@ def _validate_native_fp8_primitives() -> None: ) -def _validate_native_fp4_primitives() -> None: +def _validate_torch_fp4_primitives() -> None: missing = [ name for name in ("_scaled_mm", "float4_e2m1fn_x2", "float8_e4m3fn") @@ -893,92 +619,12 @@ def _validate_native_fp4_primitives() -> None: ] if missing: raise RuntimeError( - "SANA-WM native fp4 requires PyTorch with " + "SANA-WM fp4 requires PyTorch with " f"{', '.join(f'torch.{name}' for name in missing)} support." ) -def _validate_transformer_engine(precisions: list[Precision]) -> None: - """Ensure Transformer Engine has the recipes required by precision modes.""" - required_recipes: set[str] = set() - if "fp8" in precisions: - required_recipes.add("Float8BlockScaling") - if "fp4" in precisions: - required_recipes.add("NVFP4BlockScaling") - try: - import transformer_engine.common.recipe as te_recipe - import transformer_engine.pytorch # noqa: F401 - except Exception as exc: - raise RuntimeError( - "SANA-WM fp8/fp4 precision requires NVIDIA Transformer Engine. " - "Install it in the upstream Sana environment, for example with " - "`pip install --no-build-isolation 'transformer_engine[pytorch]'`, " - "or run with the default bf16 precision." - ) from exc - - missing = sorted( - recipe for recipe in required_recipes if not hasattr(te_recipe, recipe) - ) - if missing: - raise RuntimeError( - "Installed Transformer Engine does not provide the recipe(s) " - f"required by the requested SANA-WM precision: {', '.join(missing)}." - ) - - -def _candidate_upstream_roots(explicit_root: Path | None) -> list[Path]: - """Return candidate NVlabs/Sana checkout roots in priority order.""" - roots: list[Path] = [] - if explicit_root is not None: - roots.append(explicit_root) - for env_name in ("SANA_ROOT", "SANA_REPO", "SANA_WM_UPSTREAM_ROOT"): - value = os.environ.get(env_name) - if value: - roots.append(Path(value)) - repo_root = Path(__file__).resolve().parents[3] - roots.append(repo_root.parent / "Sana") - roots.append(Path.cwd().parent / "Sana") - return roots - - -def _is_upstream_sana_root(path: Path) -> bool: - """Return ``True`` when ``path`` looks like an NVlabs/Sana checkout.""" - return ( - path.expanduser() / "inference_video_scripts" / "wm" / "inference_sana_wm.py" - ).is_file() - - -def _import_upstream_sana(explicit_root: Path | None) -> ModuleType: - """Import upstream SANA-WM inference after adding a checkout to ``sys.path``.""" - for root in _candidate_upstream_roots(explicit_root): - candidate = root.expanduser().resolve() - if _is_upstream_sana_root(candidate): - candidate_str = str(candidate) - if candidate_str not in sys.path: - sys.path.insert(0, candidate_str) - break - - try: - return importlib.import_module("inference_video_scripts.wm.inference_sana_wm") - except ModuleNotFoundError as exc: - roots = ", ".join( - str(path) for path in _candidate_upstream_roots(explicit_root) - ) - raise ModuleNotFoundError( - "Could not import upstream SANA-WM inference. Clone NVlabs/Sana " - "next to this repo, pass --upstream-sana-root, or set SANA_ROOT; " - "then install the upstream Sana inference dependencies. " - f"Checked roots: {roots}. Original import error: {exc}" - ) from exc - except Exception as exc: - raise RuntimeError( - "Importing upstream SANA-WM inference failed. Ensure the upstream " - "Sana checkout dependencies are installed in the active environment." - ) from exc - - __all__ = [ - "ExecutionBackend", "Precision", "QuantBackend", "SamplingAlgo", diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index f657c0b0e..87cbfe21f 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -13,10 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""FlashDreams-owned SANA-WM Stage-1 DiT module definitions. +"""SANA-WM Stage-1 DiT module definitions. -This file intentionally models the public SANA-WM bidirectional Stage-1 -checkpoint schema without importing the upstream SANA ``diffusion`` package. The module names and tensor shapes are checkpoint-facing API: keep them stable unless the public checkpoint contract changes. """ @@ -309,7 +307,7 @@ def forward( apply_output_gate: bool = True, **kwargs: object, ) -> Tensor: - """Run native SANA-WM self/camera attention.""" + """Run SANA-WM self/camera attention.""" if HW is None: raise ValueError("SANA-WM Stage-1 attention requires HW=(T, H, W).") batch, tokens, channels = x.shape @@ -623,7 +621,7 @@ def forward( plucker_emb: Tensor | None = None, **kwargs: object, ) -> Tensor: - """Run one native Stage-1 transformer block.""" + """Run one Stage-1 transformer block.""" batch, tokens, channels = x.shape shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( self.scale_shift_table[None, None, :, :] + t.reshape(batch, frames, 6, -1) @@ -653,7 +651,7 @@ def forward( class SanaWMStage1Model(nn.Module): - """Checkpoint-compatible native Stage-1 SANA-WM DiT shell.""" + """Checkpoint-compatible Stage-1 SANA-WM DiT shell.""" def __init__(self, spec: SanaWMStage1Spec = SANA_WM_STAGE1_SPEC) -> None: super().__init__() @@ -682,7 +680,7 @@ def forward( chunk_plucker: Tensor | None = None, **kwargs: object, ) -> Tensor: - """Run the native SANA-WM Stage-1 DiT.""" + """Run the SANA-WM Stage-1 DiT.""" batch, _channels, frames, height, width = x.shape x = self.x_embedder(x) x = x.permute(0, 2, 3, 4, 1).reshape(batch, frames * height * width, -1) diff --git a/integrations/sana/sana_wm/native_transformer.py b/integrations/sana/sana_wm/transformer.py similarity index 92% rename from integrations/sana/sana_wm/native_transformer.py rename to integrations/sana/sana_wm/transformer.py index bebf59e7f..56b8482fc 100644 --- a/integrations/sana/sana_wm/native_transformer.py +++ b/integrations/sana/sana_wm/transformer.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Native FlashDreams adapter for the SANA-WM Stage-1 DiT.""" +"""FlashDreams adapter for the SANA-WM Stage-1 DiT.""" from __future__ import annotations @@ -46,15 +46,15 @@ SANA_WM_REFINER_GEMMA_ROOT, SANA_WM_REFINER_ROOT, ) -from sana_wm.native_quant import ( +from sana_wm.quant import ( TorchScaledMMFP4Recipe, TorchScaledMMFP8Recipe, - replace_linear_with_native_quant, + replace_linear_with_quant, ) from sana_wm.stage1_model import SanaWMStage1Model Precision = Literal["bf16", "fp8", "fp4"] -QuantBackend = Literal["auto", "upstream-te", "torch-fp8", "torch-fp4", "native"] +QuantBackend = Literal["auto", "torch", "torch-fp8", "torch-fp4"] _STAGE1_QUANT_SKIP_DEFAULTS = ( "^x_embedder", @@ -95,18 +95,18 @@ class SanaWMStage1Conditioning: @dataclass(kw_only=True) -class SanaWMNativeTransformerCache(TransformerAutoregressiveCache): +class SanaWMTransformerCache(TransformerAutoregressiveCache): """AR cache for the one-shot SANA-WM Stage-1 rollout.""" conditioning: SanaWMStage1Conditioning | None = None @dataclass(kw_only=True) -class SanaWMNativeTransformerConfig(TransformerConfig): - """Config for the native SANA-WM Stage-1 transformer adapter.""" +class SanaWMTransformerConfig(TransformerConfig): + """Config for the SANA-WM Stage-1 transformer adapter.""" - _target: type["SanaWMNativeTransformer"] = field( - default_factory=lambda: SanaWMNativeTransformer + _target: type["SanaWMTransformer"] = field( + default_factory=lambda: SanaWMTransformer ) config_path: str = SANA_WM_CONFIG_PATH @@ -146,22 +146,22 @@ class SanaWMNativeTransformerConfig(TransformerConfig): """Pixel width used by the public SANA-WM bidirectional release.""" vae_tile_sample_min_width: int = 256 - """Pixel width tile size used for native LTX-2 VAE decode.""" + """Pixel width tile size used for LTX-2 VAE decode.""" vae_tile_sample_stride_width: int = 224 - """Pixel width stride used for native LTX-2 VAE decode.""" + """Pixel width stride used for LTX-2 VAE decode.""" vae_tile_sample_min_height: int = 256 - """Pixel height tile size used for native LTX-2 VAE decode.""" + """Pixel height tile size used for LTX-2 VAE decode.""" vae_tile_sample_stride_height: int = 192 - """Pixel height stride used for native LTX-2 VAE decode.""" + """Pixel height stride used for LTX-2 VAE decode.""" vae_tile_sample_min_num_frames: int = 24 - """Pixel-frame temporal tile size used for native LTX-2 VAE decode.""" + """Pixel-frame temporal tile size used for LTX-2 VAE decode.""" vae_tile_sample_stride_num_frames: int = 8 - """Pixel-frame temporal tile stride used for native LTX-2 VAE decode.""" + """Pixel-frame temporal tile stride used for LTX-2 VAE decode.""" vae_oom_retry_tile_sample_min_width: int = 128 """Smaller pixel width tile size for one VAE decode OOM retry.""" @@ -182,12 +182,12 @@ class SanaWMNativeTransformerConfig(TransformerConfig): """Smaller temporal tile stride for one VAE decode OOM retry.""" -class SanaWMNativeTransformer(Transformer[SanaWMNativeTransformerCache]): - """Native FlashDreams adapter for the SANA-WM Stage-1 model call.""" +class SanaWMTransformer(Transformer[SanaWMTransformerCache]): + """FlashDreams adapter for the SANA-WM Stage-1 model call.""" - def __init__(self, config: SanaWMNativeTransformerConfig) -> None: + def __init__(self, config: SanaWMTransformerConfig) -> None: super().__init__(config) - self.config: SanaWMNativeTransformerConfig = config + self.config: SanaWMTransformerConfig = config self._dummy = nn.Parameter(torch.empty(0)) self._runtime_config: Any | None = None self._model_path: str | None = None @@ -221,15 +221,15 @@ def initialize_autoregressive_cache( *, conditioning: SanaWMStage1Conditioning | None = None, **_: Any, - ) -> SanaWMNativeTransformerCache: + ) -> SanaWMTransformerCache: """Build a cache containing per-rollout SANA-WM conditioning.""" - return SanaWMNativeTransformerCache(conditioning=conditioning) + return SanaWMTransformerCache(conditioning=conditioning) def predict_flow( self, noisy_latent: Tensor, timestep: Tensor, - cache: SanaWMNativeTransformerCache, + cache: SanaWMTransformerCache, input: Any = None, model_kwargs: dict[str, object] | None = None, ) -> Tensor: @@ -249,7 +249,7 @@ def finalize_kv_cache( self, noisy_latent: Tensor, timestep: Tensor, - cache: SanaWMNativeTransformerCache, + cache: SanaWMTransformerCache, input: Any = None, ) -> None: """SANA-WM bidirectional inference has no streaming KV cache to advance.""" @@ -450,7 +450,7 @@ def decode_latents(self, latents: Tensor) -> np.ndarray: def release_stage1_runtime( self, - cache: SanaWMNativeTransformerCache | None = None, + cache: SanaWMTransformerCache | None = None, ) -> None: """Release Stage-1-only tensors before VAE/refiner work.""" free_before_gib: float | None = None @@ -500,7 +500,7 @@ def release_stage1_runtime( gc.collect() def release_refiner_runtime(self) -> None: - """Release native refiner tensors before VAE decode.""" + """Release refiner tensors before VAE decode.""" self._release_refiner() def refine_latents( @@ -514,7 +514,7 @@ def refine_latents( block_size: int | None, kv_max_frames: int, ) -> Tensor: - """Run the native LTX-2 refiner.""" + """Run the LTX-2 refiner.""" self._ensure_refiner() sigmas = torch.tensor( self._stage2_sigmas(), @@ -590,7 +590,7 @@ def _ensure_model(self) -> None: weight_dtype = self._ensure_weight_dtype() model = SanaWMStage1Model().to(self.device) logger.info( - "[Sana] Loaded native {} ({:,} params)", + "[Sana] Loaded {} ({:,} params)", cfg.model.model, sum(p.numel() for p in model.parameters()), ) @@ -617,7 +617,7 @@ def _ensure_model(self) -> None: def _ensure_refiner(self) -> None: if self._refiner_built: return - from sana_wm.native_refiner import SanaWMLTX2Refiner + from sana_wm.refiner import SanaWMLTX2Refiner compute_dtype = ( torch.bfloat16 @@ -625,7 +625,7 @@ def _ensure_refiner(self) -> None: else _get_weight_dtype(self.config.refiner_precision) ) quant_backend: QuantBackend = ( - "native" if self.config.quant_backend == "auto" else self.config.quant_backend + "torch" if self.config.quant_backend == "auto" else self.config.quant_backend ) self.refiner = SanaWMLTX2Refiner( refiner_root=resolve_hf_path(self.config.refiner_root), @@ -816,16 +816,11 @@ def pad_pair(text: Tensor, mask: Tensor) -> tuple[Tensor, Tensor]: def _prepare_stage1_quant(self) -> None: if self._stage1_quantized or self.config.stage1_precision == "bf16": return - if self.config.quant_backend == "upstream-te": - raise ValueError( - "Native FlashDreams SANA execution does not use Transformer Engine; " - "select --quant-backend native, torch-fp8, or torch-fp4." - ) if self.config.stage1_precision == "fp8": recipe = TorchScaledMMFP8Recipe() else: recipe = TorchScaledMMFP4Recipe() - converted, skipped = replace_linear_with_native_quant( + converted, skipped = replace_linear_with_quant( self.model, recipe=recipe, params_dtype=self._ensure_weight_dtype(), @@ -834,12 +829,12 @@ def _prepare_stage1_quant(self) -> None: ) if converted <= 0: raise RuntimeError( - f"SANA-WM native {self.config.stage1_precision} converted no " + f"SANA-WM {self.config.stage1_precision} converted no " f"Stage-1 Linear layers; skipped={skipped}." ) self._stage1_quantized = True logger.info( - "[stage1-native-quant] precision={} converted {} Linear layers (skipped {})", + "[stage1-quant] precision={} converted {} Linear layers (skipped {})", self.config.stage1_precision, converted, skipped, @@ -882,10 +877,10 @@ def _stage2_sigmas() -> tuple[float, ...]: def _require_conditioning( - cache: SanaWMNativeTransformerCache, + cache: SanaWMTransformerCache, ) -> SanaWMStage1Conditioning: if cache.conditioning is None: - raise RuntimeError("SANA-WM native cache was initialized without conditioning.") + raise RuntimeError("SANA-WM cache was initialized without conditioning.") return cache.conditioning @@ -940,7 +935,7 @@ def _get_vae(*args: Any, **kwargs: Any) -> nn.Module: device = kwargs["device"] dtype = kwargs["dtype"] if "LTX2VAE_diffusers" not in str(name): - raise ValueError(f"Unsupported native SANA-WM VAE type: {name!r}") + raise ValueError(f"Unsupported SANA-WM VAE type: {name!r}") from diffusers import AutoencoderKLLTX2Video return ( @@ -959,7 +954,7 @@ def _get_tokenizer_and_text_encoder(*args: Any, **kwargs: Any) -> tuple[Any, nn. device = kwargs.get("device", "cuda") model_id = _TEXT_ENCODER_MODEL_IDS.get(str(name)) if model_id is None: - raise ValueError(f"Unsupported native SANA-WM text encoder: {name!r}") + raise ValueError(f"Unsupported SANA-WM text encoder: {name!r}") from transformers import AutoModelForCausalLM, AutoTokenizer, T5EncoderModel, T5Tokenizer if "T5" in str(name): @@ -1017,7 +1012,7 @@ def get(self, name: str, default: Any = None) -> Any: def _vae_encode_ltx2(name: str, vae: nn.Module, images: Tensor, *, device: torch.device) -> Tensor: if "LTX2VAE_diffusers" not in name: - raise ValueError(f"Unsupported native SANA-WM VAE encode type: {name!r}") + raise ValueError(f"Unsupported SANA-WM VAE encode type: {name!r}") dtype = images.dtype vae_device = next(vae.parameters()).device vae_dtype = next(vae.parameters()).dtype @@ -1031,7 +1026,7 @@ def _vae_encode_ltx2(name: str, vae: nn.Module, images: Tensor, *, device: torch def _vae_decode_ltx2(name: str, vae: nn.Module, latents: Tensor) -> Tensor: if "LTX2VAE_diffusers" not in name: - raise ValueError(f"Unsupported native SANA-WM VAE decode type: {name!r}") + raise ValueError(f"Unsupported SANA-WM VAE decode type: {name!r}") vae_device = next(vae.parameters()).device vae_dtype = next(vae.parameters()).dtype latents_mean = vae.latents_mean.view(1, -1, 1, 1, 1).to( @@ -1118,8 +1113,8 @@ def _chunk_index_from_chunk_size( __all__ = [ - "SanaWMNativeTransformer", - "SanaWMNativeTransformerCache", - "SanaWMNativeTransformerConfig", + "SanaWMTransformer", + "SanaWMTransformerCache", + "SanaWMTransformerConfig", "SanaWMStage1Conditioning", ] diff --git a/integrations/sana/sana_wm/upstream_reference.py b/integrations/sana/sana_wm/upstream_reference.py deleted file mode 100644 index 6b87504e2..000000000 --- a/integrations/sana/sana_wm/upstream_reference.py +++ /dev/null @@ -1,90 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Schema-only FlashDreams objects for the upstream SANA-WM reference runner.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any - -import torch -import torch.nn as nn -from torch import Tensor - -from flashdreams.infra.diffusion.transformer import ( - Transformer, - TransformerAutoregressiveCache, - TransformerConfig, -) - - -@dataclass(kw_only=True) -class SanaWMUpstreamReferenceTransformerConfig(TransformerConfig): - """Transformer config used by the upstream-reference runner. - - This is intentionally not an executable SANA DiT adapter. It exists so the - CLI can expose the upstream reference path through the normal runner config - registry without pretending that upstream execution is the native path. - """ - - _target: type["SanaWMUpstreamReferenceTransformer"] = field( - default_factory=lambda: SanaWMUpstreamReferenceTransformer - ) - - -class SanaWMUpstreamReferenceTransformer( - Transformer[TransformerAutoregressiveCache] -): - """Non-executable marker for the upstream-delegating reference runner.""" - - def __init__(self, config: SanaWMUpstreamReferenceTransformerConfig) -> None: - super().__init__(config) - self._dummy = nn.Parameter(torch.empty(0)) - - @property - def latent_shape(self) -> tuple[int, ...]: - """Raise because this config marks the upstream reference path.""" - raise RuntimeError( - "This SANA-WM runner is an upstream reference harness. Use " - "sana-wm-bidirectional for native FlashDreams execution." - ) - - def predict_flow( - self, - noisy_latent: Tensor, - timestep: Tensor, - cache: TransformerAutoregressiveCache, - input: Any = None, - ) -> Tensor: - """Raise because this config marks the upstream reference path.""" - raise RuntimeError( - "This SANA-WM runner delegates generation to upstream NVlabs/Sana " - "instead of executing through Transformer.predict_flow." - ) - - def patchify_and_maybe_split_cp(self, x: Any) -> Any: - """Return ``x`` unchanged for schema-only construction.""" - return x - - def unpatchify_and_maybe_gather_cp(self, x: Tensor) -> Tensor: - """Return ``x`` unchanged for schema-only construction.""" - return x - - -__all__ = [ - "SanaWMUpstreamReferenceTransformer", - "SanaWMUpstreamReferenceTransformerConfig", -] diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md deleted file mode 100644 index 62b13519b..000000000 --- a/integrations/sana/tests/parity_check/README.md +++ /dev/null @@ -1,25 +0,0 @@ - - -# SANA-WM parity check - -This directory is reserved for the opt-in SANA-WM native-vs-upstream parity -harness. It should follow the existing integration pattern: - -- clone or reuse a pinned NVlabs/Sana checkout; -- install upstream-heavy dependencies in an isolated environment; -- run upstream `inference_video_scripts/wm/inference_sana_wm.py` and the - native `flashdreams-run sana-wm-bidirectional` path on the same image, - prompt, camera, intrinsics, seed, and sampler settings; -- compare decoded MP4 frames and report mean / max `|Delta|`. - -The temporary FlashDreams-packaged upstream-reference runner is documented in -`../../docs/upstream_reference.md`. Keep those wrapper details out of the native -integration README. - -Do not put the full checkpoint download, refiner execution, or MP4 -generation path in `ci_cpu`. Add a bounded `ci_gpu` gate only if CI -pre-provisions the required artefacts and the test skips cleanly when -they are absent. diff --git a/integrations/sana/tests/test_native_quant_cuda.py b/integrations/sana/tests/test_quant_cuda.py similarity index 81% rename from integrations/sana/tests/test_native_quant_cuda.py rename to integrations/sana/tests/test_quant_cuda.py index c32e2e65a..9034c0c7f 100644 --- a/integrations/sana/tests/test_native_quant_cuda.py +++ b/integrations/sana/tests/test_quant_cuda.py @@ -13,33 +13,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""CUDA smoke tests for SANA-WM native low-precision linear replacements.""" +"""CUDA smoke tests for SANA-WM low-precision linear replacements.""" from __future__ import annotations import pytest import torch -from sana_wm.native_quant import TorchScaledMMFP4Linear, TorchScaledMMFP8Linear +from sana_wm.quant import TorchScaledMMFP4Linear, TorchScaledMMFP8Linear pytestmark = pytest.mark.ci_gpu -def _require_native_quant_cuda() -> None: +def _require_quant_cuda() -> None: if not torch.cuda.is_available(): - pytest.skip("CUDA is required for native quantization smokes") + pytest.skip("CUDA is required for quantization smokes") missing = [ name for name in ("_scaled_mm", "float8_e4m3fn", "float4_e2m1fn_x2") if not hasattr(torch, name) ] if missing: - pytest.skip(f"PyTorch lacks native quantization primitive(s): {missing}") + pytest.skip(f"PyTorch lacks quantization primitive(s): {missing}") -def test_native_fp8_linear_runs_scaled_mm_on_cuda() -> None: +def test_fp8_linear_runs_scaled_mm_on_cuda() -> None: """Exercise the TE-free E4M3 FP8 Linear replacement on CUDA.""" - _require_native_quant_cuda() + _require_quant_cuda() source = torch.nn.Linear(32, 64, bias=True, device="cuda", dtype=torch.bfloat16) quantized = TorchScaledMMFP8Linear.from_linear( source, @@ -56,9 +56,9 @@ def test_native_fp8_linear_runs_scaled_mm_on_cuda() -> None: assert torch.isfinite(output.float()).all() -def test_native_fp4_linear_runs_scaled_mm_on_blackwell() -> None: +def test_fp4_linear_runs_scaled_mm_on_blackwell() -> None: """Exercise the TE-free E2M1 NVFP4 Linear replacement on Blackwell CUDA.""" - _require_native_quant_cuda() + _require_quant_cuda() major, minor = torch.cuda.get_device_capability() if major < 10: pytest.skip(f"NVFP4 requires Blackwell-class CUDA, got sm_{major}{minor}") diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index ddb05d8f6..5cb0a8dfd 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""CPU-safe smoke tests for the SANA-WM native and reference configs.""" +"""CPU-safe smoke tests for the SANA-WM configs.""" from __future__ import annotations @@ -23,8 +23,8 @@ import pytest import torch -import sana_wm.native_refiner as native_refiner -import sana_wm.native_transformer as native_transformer_module +import sana_wm.refiner as refiner_module +import sana_wm.transformer as transformer_module try: import tomllib @@ -33,12 +33,8 @@ from sana_wm.config import ( PIPELINE_SANA_WM_BIDIRECTIONAL, - PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL, - PIPELINE_SANA_WM_UPSTREAM_REFERENCE, RUNNER_CONFIGS, RUNNER_SANA_WM_BIDIRECTIONAL, - RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL, - RUNNER_SANA_WM_UPSTREAM_REFERENCE, ) from sana_wm.constants import ( SANA_WM_CONFIG_PATH, @@ -53,20 +49,20 @@ _temporary_environment, _validate_precision_request, ) -from sana_wm.native_diffusion import ( +from sana_wm.diffusion import ( _condition_model_kwargs, _uncondition_model_kwargs, ) -from sana_wm.native_diffusion import SanaWMDiffusionModelConfig -from sana_wm.native_transformer import ( - SanaWMNativeTransformerCache, - SanaWMNativeTransformerConfig, +from sana_wm.diffusion import SanaWMDiffusionModelConfig +from sana_wm.transformer import ( + SanaWMTransformerCache, + SanaWMTransformerConfig, SanaWMStage1Conditioning, _avoid_degenerate_tile_tail, _load_inference_config, ) -from sana_wm.native_refiner import SanaWMLTX2Refiner, _pack_latents, _unpack_latents -from sana_wm.native_quant import ( +from sana_wm.refiner import SanaWMLTX2Refiner, _pack_latents, _unpack_latents +from sana_wm.quant import ( TorchScaledMMFP4Linear, TorchScaledMMFP8Linear, replace_linear_with_torch_fp4, @@ -78,7 +74,6 @@ SanaWMStage1Spec, Stage1SelfAttention, ) -from sana_wm.upstream_reference import SanaWMUpstreamReferenceTransformerConfig from flashdreams.infra.config import derive_config @@ -88,11 +83,9 @@ def test_runner_config_is_registered() -> None: - """Expose SANA-WM native and upstream-reference runner slugs.""" + """Expose the SANA-WM runner slug.""" assert RUNNER_CONFIGS == { "sana-wm-bidirectional": RUNNER_SANA_WM_BIDIRECTIONAL, - "sana-wm-upstream-reference": RUNNER_SANA_WM_UPSTREAM_REFERENCE, - "sana-wm-native-bidirectional": RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL, } @@ -101,47 +94,27 @@ def test_runner_name_mirrors_pipeline_name() -> None: assert RUNNER_SANA_WM_BIDIRECTIONAL.runner_name == ( PIPELINE_SANA_WM_BIDIRECTIONAL.name ) - assert RUNNER_SANA_WM_UPSTREAM_REFERENCE.runner_name == ( - PIPELINE_SANA_WM_UPSTREAM_REFERENCE.name - ) - assert RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL.runner_name == ( - PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL.name - ) def test_runner_has_description() -> None: """Provide non-empty CLI help text for the runner registry.""" assert RUNNER_SANA_WM_BIDIRECTIONAL.description.strip() - assert RUNNER_SANA_WM_UPSTREAM_REFERENCE.description.strip() - assert RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL.description.strip() -def test_reference_and_native_pipelines_are_distinct() -> None: - """Keep the upstream harness separate from the native FlashDreams target.""" - reference_transformer = ( - PIPELINE_SANA_WM_UPSTREAM_REFERENCE.diffusion_model.transformer - ) - main_transformer = PIPELINE_SANA_WM_BIDIRECTIONAL.diffusion_model.transformer - native_transformer = ( - PIPELINE_SANA_WM_NATIVE_BIDIRECTIONAL.diffusion_model.transformer - ) +def test_pipeline_uses_sana_diffusion_model() -> None: + """Keep the public runner wired to the SANA-WM diffusion model.""" + transformer = PIPELINE_SANA_WM_BIDIRECTIONAL.diffusion_model.transformer - assert isinstance(reference_transformer, SanaWMUpstreamReferenceTransformerConfig) assert isinstance( PIPELINE_SANA_WM_BIDIRECTIONAL.diffusion_model, SanaWMDiffusionModelConfig, ) - assert isinstance(main_transformer, SanaWMNativeTransformerConfig) - assert isinstance(native_transformer, SanaWMNativeTransformerConfig) - assert RUNNER_SANA_WM_BIDIRECTIONAL.execution_backend == "native-flashdreams" - assert RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL.execution_backend == ( - "native-flashdreams" - ) + assert isinstance(transformer, SanaWMTransformerConfig) -def test_native_transformer_contract_shape_and_conditioning_guard() -> None: - """Pin the native SANA-WM Stage-1 boundary to the public model layout.""" - transformer_cfg = SanaWMNativeTransformerConfig() +def test_transformer_contract_shape_and_conditioning_guard() -> None: + """Pin the SANA-WM Stage-1 boundary to the public model layout.""" + transformer_cfg = SanaWMTransformerConfig() transformer = transformer_cfg.setup() assert transformer.latent_shape == (1, 128, 21, 22, 40) @@ -155,10 +128,10 @@ def test_native_transformer_contract_shape_and_conditioning_guard() -> None: ) -def test_native_inference_config_loads_yaml_without_upstream_types( +def test_inference_config_loads_yaml( tmp_path: Path, ) -> None: - """Parse SANA-WM YAML with local config objects, not SANA pyrallis types.""" + """Parse SANA-WM YAML with local config objects.""" config_path = tmp_path / "config.yaml" config_path.write_text( """ @@ -193,8 +166,8 @@ def test_native_inference_config_loads_yaml_without_upstream_types( assert cfg.work_dir == "" -def test_native_stage1_model_matches_checkpoint_schema() -> None: - """Pin the FlashDreams-owned Stage-1 module to the public checkpoint schema.""" +def test_stage1_model_matches_checkpoint_schema() -> None: + """Pin the Stage-1 module to the public checkpoint schema.""" state = SanaWMStage1Model().state_dict() assert len(state) == 872 @@ -223,8 +196,8 @@ def test_native_stage1_model_matches_checkpoint_schema() -> None: assert has_gdn_conv is SANA_WM_STAGE1_SPEC.block_uses_gdn(block_index) -def test_native_stage1_forward_preserves_latent_shape() -> None: - """Exercise the native Stage-1 forward path on a small CPU-safe spec.""" +def test_stage1_forward_preserves_latent_shape() -> None: + """Exercise the Stage-1 forward path on a small CPU-safe spec.""" spec = SanaWMStage1Spec( latent_channels=4, hidden_size=16, @@ -255,7 +228,7 @@ def test_native_stage1_forward_preserves_latent_shape() -> None: def test_stage1_self_attention_uses_camera_conditions() -> None: - """Changing camera conditions must affect native camera attention.""" + """Changing camera conditions must affect camera attention.""" torch.manual_seed(0) spec = SanaWMStage1Spec( latent_channels=4, @@ -289,9 +262,9 @@ def test_stage1_self_attention_uses_camera_conditions() -> None: assert not torch.allclose(base, shifted) -def test_native_transformer_releases_stage1_runtime() -> None: +def test_transformer_releases_stage1_runtime() -> None: """Free Stage-1-only modules and conditioning before decode/refine.""" - transformer = SanaWMNativeTransformerConfig().setup() + transformer = SanaWMTransformerConfig().setup() transformer.model = torch.nn.Linear(1, 1) transformer.text_encoder = torch.nn.Linear(1, 1) transformer.tokenizer = object() @@ -304,7 +277,7 @@ def test_native_transformer_releases_stage1_runtime() -> None: torch.empty(1), torch.empty(1), ) - cache = SanaWMNativeTransformerCache( + cache = SanaWMTransformerCache( conditioning=SanaWMStage1Conditioning( condition=torch.empty(1), uncondition=None, @@ -330,8 +303,8 @@ def test_native_transformer_releases_stage1_runtime() -> None: assert transformer._stage1_quantized is False -def test_native_vae_tiling_uses_low_memory_tiles() -> None: - """Keep native VAE decode on smaller tiles than upstream defaults.""" +def test_vae_tiling_uses_low_memory_tiles() -> None: + """Keep VAE decode on the configured low-memory tiles.""" class DummyVAE: def __init__(self) -> None: @@ -349,7 +322,7 @@ def __init__(self) -> None: def enable_tiling(self, **kwargs: int) -> None: self.calls.append(kwargs) - transformer = SanaWMNativeTransformerConfig().setup() + transformer = SanaWMTransformerConfig().setup() transformer.vae = DummyVAE() transformer._configure_vae_tiling() @@ -374,10 +347,10 @@ def enable_tiling(self, **kwargs: int) -> None: assert transformer.vae.use_framewise_decoding is True -def test_native_decode_retries_vae_oom_with_smaller_tiles( +def test_decode_retries_vae_oom_with_smaller_tiles( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Retry native VAE decode with smaller tiles after a CUDA OOM.""" + """Retry VAE decode with smaller tiles after a CUDA OOM.""" class DummyVAE: def __init__(self) -> None: @@ -399,7 +372,7 @@ def enable_tiling(self, **kwargs: int) -> None: for name, value in kwargs.items(): setattr(self, name, value) - transformer = SanaWMNativeTransformerConfig().setup() + transformer = SanaWMTransformerConfig().setup() transformer.vae = DummyVAE() transformer.vae_dtype = torch.float32 monkeypatch.setattr(transformer, "_ensure_vae", lambda: None) @@ -452,7 +425,7 @@ def test_cfg_model_kwargs_do_not_duplicate_camera_tensors() -> None: assert "negative_mask" not in neg_kwargs -def test_native_vae_tiling_avoids_degenerate_latent_tails() -> None: +def test_vae_tiling_avoids_degenerate_latent_tails() -> None: """Avoid last spatial VAE tiles with size one after compression.""" assert ( _avoid_degenerate_tile_tail( @@ -486,8 +459,8 @@ def test_hf_defaults_point_at_bidirectional_release() -> None: ) -def test_runner_setup_does_not_import_upstream_sana() -> None: - """Construct the runner without touching upstream Sana dependencies.""" +def test_runner_setup_preserves_cli_fields() -> None: + """Construct the runner and preserve CLI override fields.""" cfg = derive_config( RUNNER_SANA_WM_BIDIRECTIONAL, image_path=Path("missing.png"), @@ -503,35 +476,33 @@ def test_runner_setup_does_not_import_upstream_sana() -> None: def test_runner_config_type() -> None: """Keep the exported literal on the SANA-WM runner config subclass.""" assert isinstance(RUNNER_SANA_WM_BIDIRECTIONAL, SanaWMRunnerConfig) - assert isinstance(RUNNER_SANA_WM_UPSTREAM_REFERENCE, SanaWMRunnerConfig) - assert isinstance(RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL, SanaWMRunnerConfig) def test_runner_defaults_to_bf16_precision() -> None: - """Keep the default path on upstream SANA-WM's BF16 config.""" + """Keep the default runner on BF16 precision.""" assert RUNNER_SANA_WM_BIDIRECTIONAL.stage1_precision == "bf16" assert RUNNER_SANA_WM_BIDIRECTIONAL.refiner_precision == "bf16" assert RUNNER_SANA_WM_BIDIRECTIONAL.quant_backend == "auto" assert RUNNER_SANA_WM_BIDIRECTIONAL.no_refiner is False -def test_native_refiner_is_flashdreams_owned( +def test_refiner_is_flashdreams_owned( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Build the native refiner adapter instead of raising or importing Sana.""" + """Build the refiner adapter.""" calls: dict[str, object] = {} class DummyRefiner: def __init__(self, **kwargs: object) -> None: calls.update(kwargs) - monkeypatch.setattr(native_refiner, "SanaWMLTX2Refiner", DummyRefiner) + monkeypatch.setattr(refiner_module, "SanaWMLTX2Refiner", DummyRefiner) monkeypatch.setattr( - native_transformer_module, + transformer_module, "resolve_hf_path", lambda value: f"/resolved/{value}", ) - transformer = SanaWMNativeTransformerConfig().setup() + transformer = SanaWMTransformerConfig().setup() transformer._ensure_refiner() @@ -545,7 +516,7 @@ def __init__(self, **kwargs: object) -> None: ) assert calls["dtype"] is torch.bfloat16 assert calls["precision"] == "bf16" - assert calls["quant_backend"] == "native" + assert calls["quant_backend"] == "torch" def test_refiner_latent_pack_round_trips() -> None: @@ -574,7 +545,7 @@ def test_refiner_latent_pack_round_trips() -> None: def test_refiner_block_size_request_still_executes( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Do not turn native refiner block-size requests into a hard failure.""" + """Do not turn refiner block-size requests into a hard failure.""" class DummyTransformer(torch.nn.Module): def __init__(self) -> None: @@ -623,14 +594,14 @@ def predict_current_x0( assert inference_modes == [True] -def test_auto_quant_backend_resolves_to_native_backend() -> None: - """Keep default quantization on the TE-free native backend.""" - assert _resolve_quant_backend("auto", ["fp4"]) == "native" - assert _resolve_quant_backend("auto", ["fp8", "fp4"]) == "native" - assert _resolve_quant_backend("auto", []) == "native" +def test_auto_quant_backend_resolves_to_torch_backend() -> None: + """Keep default quantization on the Torch backend.""" + assert _resolve_quant_backend("auto", ["fp4"]) == "torch" + assert _resolve_quant_backend("auto", ["fp8", "fp4"]) == "torch" + assert _resolve_quant_backend("auto", []) == "torch" -def test_bf16_precision_clears_upstream_quant_env( +def test_bf16_precision_clears_quant_env( monkeypatch: pytest.MonkeyPatch, ) -> None: """Prevent external NVFP4 env vars from changing the default runner path.""" @@ -650,8 +621,8 @@ def test_bf16_precision_clears_upstream_quant_env( assert os.environ["SANA_WM_REFINER_NVFP4"] == "1" -def test_fp8_precision_sets_upstream_quant_env() -> None: - """Map FlashDreams fp8 fields to upstream SANA-WM env selectors.""" +def test_fp8_precision_sets_quant_env() -> None: + """Map FlashDreams fp8 fields to SANA-WM env selectors.""" updates = _precision_env_updates( stage1_precision="fp8", refiner_precision="fp8", @@ -685,7 +656,7 @@ def test_quantized_precision_requires_cuda() -> None: stage1_precision="fp8", refiner_precision="bf16", refiner_enabled=True, - quant_backend="upstream-te", + quant_backend="torch", ) @@ -711,23 +682,7 @@ def test_fp8_precision_requires_hopper(monkeypatch: pytest.MonkeyPatch) -> None: stage1_precision="fp8", refiner_precision="bf16", refiner_enabled=True, - quant_backend="upstream-te", - ) - - -def test_fp8_precision_requires_cuda_129(monkeypatch: pytest.MonkeyPatch) -> None: - """Reject upstream FP8 block scaling when torch reports CUDA 12.8.""" - monkeypatch.setattr(torch.cuda, "is_available", lambda: True) - monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) - monkeypatch.setattr(torch.version, "cuda", "12.8") - - with pytest.raises(ValueError, match="requires CUDA 12.9 or newer"): - _validate_precision_request( - device=torch.device("cuda:0"), - stage1_precision="fp8", - refiner_precision="bf16", - refiner_enabled=True, - quant_backend="upstream-te", + quant_backend="torch", ) @@ -742,14 +697,14 @@ def test_fp4_precision_requires_blackwell(monkeypatch: pytest.MonkeyPatch) -> No stage1_precision="fp4", refiner_precision="bf16", refiner_enabled=True, - quant_backend="upstream-te", + quant_backend="torch", ) -def test_torch_fp8_backend_skips_transformer_engine_cuda_129_gate( +def test_torch_fp8_backend_validates_primitives( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Allow native FP8 validation in the CUDA 12.8 env that blocks TE FP8.""" + """Allow FP8 validation with the required PyTorch primitives.""" monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) monkeypatch.setattr(torch.version, "cuda", "12.8") @@ -765,10 +720,10 @@ def test_torch_fp8_backend_skips_transformer_engine_cuda_129_gate( ) -def test_auto_fp8_backend_uses_native_primitives( +def test_auto_fp8_backend_uses_torch_primitives( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Route default FP8 to the TE-free native backend.""" + """Route default FP8 to the Torch backend.""" monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) monkeypatch.setattr(torch.version, "cuda", "12.8") @@ -784,10 +739,10 @@ def test_auto_fp8_backend_uses_native_primitives( ) -def test_native_runner_reaches_normal_input_validation() -> None: - """Do not silently delegate the native runner to the upstream harness.""" +def test_runner_reaches_normal_input_validation() -> None: + """Reach normal file input validation.""" cfg = derive_config( - RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL, + RUNNER_SANA_WM_BIDIRECTIONAL, image_path=Path("missing.png"), prompt="demo", ) @@ -812,8 +767,8 @@ def test_torch_fp8_backend_rejects_fp4(monkeypatch: pytest.MonkeyPatch) -> None: ) -def test_torch_fp4_backend_skips_transformer_engine(monkeypatch: pytest.MonkeyPatch) -> None: - """Allow native FP4 validation without importing Transformer Engine.""" +def test_torch_fp4_backend_validates_primitives(monkeypatch: pytest.MonkeyPatch) -> None: + """Allow FP4 validation with the required PyTorch primitives.""" monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) monkeypatch.setattr(torch, "_scaled_mm", object(), raising=False) @@ -830,7 +785,7 @@ def test_torch_fp4_backend_skips_transformer_engine(monkeypatch: pytest.MonkeyPa def test_torch_fp4_backend_rejects_fp8(monkeypatch: pytest.MonkeyPatch) -> None: - """Keep precision-specific native backends explicit.""" + """Keep precision-specific backends explicit.""" monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) @@ -844,10 +799,10 @@ def test_torch_fp4_backend_rejects_fp8(monkeypatch: pytest.MonkeyPatch) -> None: ) -def test_native_backend_allows_mixed_fp8_fp4_without_cuda_129( +def test_torch_backend_allows_mixed_fp8_fp4( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Allow mixed native FP8/FP4 requests in the CUDA 12.8 env that blocks TE FP8.""" + """Allow mixed FP8/FP4 requests on the Torch backend.""" monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) monkeypatch.setattr(torch.version, "cuda", "12.8") @@ -860,14 +815,14 @@ def test_native_backend_allows_mixed_fp8_fp4_without_cuda_129( stage1_precision="fp8", refiner_precision="fp4", refiner_enabled=True, - quant_backend="native", + quant_backend="torch", ) def test_torch_fp8_linear_replaces_matching_modules() -> None: - """Provide a TE-free replacement for eligible upstream Linear modules.""" + """Provide a Torch replacement for eligible Linear modules.""" if not hasattr(torch, "float8_e4m3fn"): - pytest.skip("torch.float8_e4m3fn is required for native FP8 replacement") + pytest.skip("torch.float8_e4m3fn is required for FP8 replacement") module = torch.nn.Sequential( torch.nn.Linear(16, 32, bias=True), @@ -895,7 +850,7 @@ def test_torch_fp8_linear_replaces_matching_modules() -> None: def test_torch_fp4_linear_replaces_matching_modules( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Route eligible FP4 modules through the native replacement helper.""" + """Route eligible FP4 modules through the replacement helper.""" module = torch.nn.Sequential( torch.nn.Linear(32, 64, bias=True), torch.nn.Sequential(torch.nn.Linear(32, 64, bias=False)), @@ -942,17 +897,11 @@ def test_pyproject_entry_point_matches_runner_literal() -> None: assert entry_points == { "sana-wm-bidirectional": "sana_wm.config:RUNNER_SANA_WM_BIDIRECTIONAL", - "sana-wm-upstream-reference": ( - "sana_wm.config:RUNNER_SANA_WM_UPSTREAM_REFERENCE" - ), - "sana-wm-native-bidirectional": ( - "sana_wm.config:RUNNER_SANA_WM_NATIVE_BIDIRECTIONAL" - ), } -def test_pyproject_excludes_upstream_diffusion_package() -> None: - """Do not install the NVlabs/Sana ``diffusion`` package.""" +def test_pyproject_package_selection() -> None: + """Keep the package metadata aligned with the integration package.""" pyproject = tomllib.loads( Path("integrations/sana/pyproject.toml").read_text(encoding="utf-8") ) From 26a66be62b119daebcec7254472a833abe2773e8 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 17 Jul 2026 11:25:56 -0700 Subject: [PATCH 13/64] Harden SANA-WM FlashDreams integration boundaries Move SANA-WM prompt, first-frame, camera, decode, and refiner orchestration into explicit pipeline components instead of keeping it inside the transformer or runner. The public runner now drives the normal StreamInferencePipeline contract, with a Sana conditioning encoder, Stage-1 transformer, LTX-style scheduler, and video decoder/refiner boundary. Keep the native Stage-1 DiT implementation but make SanaWMDiffusionModelConfig instantiate the shared FlashDreams DiffusionModel. Add generic infra hooks for transformer-provided initial noise and scheduler access to per-step encoder context so Sana can pin the first-frame latent and run per-token LTX Euler sampling without owning a custom diffusion loop. Replace direct diffusers FlowMatch Euler use with a SanaWMLTXEulerScheduler that matches the LTX per-token behavior and keeps the scheduler-specific timestep handling isolated behind the FlashDreams scheduler interface. Restore batched CFG as the default Stage-1 path and remove manual early temporary deletions added during the earlier OOM investigation. Keep inference-mode execution and default-false offload/tiling fallback paths where they remain useful inference hygiene or explicit user controls. Expand CPU coverage for runner registration, component config routing, prompt/first-frame/camera conditioning shapes, scheduler parity, base DiffusionModel instantiation, CFG batching, VAE decode/refiner boundaries, and Stage-1 checkpoint shape coverage. Signed-off-by: Aidan Foster --- .../flashdreams/infra/diffusion/model/base.py | 21 +- .../infra/diffusion/scheduler/base.py | 6 +- .../infra/diffusion/scheduler/fm.py | 3 + .../infra/diffusion/scheduler/fm_euler.py | 3 + .../infra/diffusion/scheduler/fm_unipc.py | 3 + .../infra/diffusion/transformer/base.py | 31 + integrations/sana/sana_wm/conditioning.py | 552 +++++++++++++++ integrations/sana/sana_wm/config.py | 8 +- integrations/sana/sana_wm/decoder.py | 488 +++++++++++++ integrations/sana/sana_wm/diffusion.py | 231 +------ integrations/sana/sana_wm/runner.py | 127 ++-- integrations/sana/sana_wm/scheduler.py | 310 +++++++++ integrations/sana/sana_wm/stage1_model.py | 8 +- integrations/sana/sana_wm/transformer.py | 648 +++--------------- integrations/sana/tests/test_smoke.py | 566 +++++++++++++-- 15 files changed, 2089 insertions(+), 916 deletions(-) create mode 100644 integrations/sana/sana_wm/conditioning.py create mode 100644 integrations/sana/sana_wm/decoder.py create mode 100644 integrations/sana/sana_wm/scheduler.py diff --git a/flashdreams/flashdreams/infra/diffusion/model/base.py b/flashdreams/flashdreams/infra/diffusion/model/base.py index 02dbb909c..67cef7532 100644 --- a/flashdreams/flashdreams/infra/diffusion/model/base.py +++ b/flashdreams/flashdreams/infra/diffusion/model/base.py @@ -173,19 +173,19 @@ def generate( self.latent_shape, device=self.device, dtype=self.dtype ) dummy_latent = self.transformer.unpatchify_and_maybe_gather_cp(dummy_latent) - initial_noise = torch.randn( - dummy_latent.shape, - device=self.device, - dtype=self.dtype, - generator=self.rng, + initial_noise = self.transformer.initial_noise( + latent_shape=tuple(dummy_latent.shape), + rng=self.rng, + cache=cache, + input=input, ) else: - initial_noise = torch.randn( - self.latent_shape, - device=self.device, - dtype=self.dtype, - generator=self.rng, + initial_noise = self.transformer.initial_noise( + latent_shape=self.latent_shape, + rng=self.rng, + cache=cache, + input=input, ) def predict_flow(noisy_latent: Tensor, timestep: Tensor) -> Tensor: @@ -212,6 +212,7 @@ def predict_flow(noisy_latent: Tensor, timestep: Tensor) -> Tensor: initial_noise=initial_noise, predict_flow=predict_flow, rng=self.rng, + context=input, ) if self.config.noise_in_unpatchified_shape: diff --git a/flashdreams/flashdreams/infra/diffusion/scheduler/base.py b/flashdreams/flashdreams/infra/diffusion/scheduler/base.py index 46511d93b..4bb0f1dd6 100644 --- a/flashdreams/flashdreams/infra/diffusion/scheduler/base.py +++ b/flashdreams/flashdreams/infra/diffusion/scheduler/base.py @@ -19,7 +19,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Protocol +from typing import Any, Protocol import torch import torch.nn as nn @@ -69,6 +69,7 @@ def sample( initial_noise: Tensor, predict_flow: FlowPredictor, rng: torch.Generator | None = None, + context: Any = None, ) -> Tensor: """Run the full denoising loop and return the clean latent. @@ -83,6 +84,9 @@ def sample( ``num_inference_steps`` times. rng: Generator on the same device. Used by self-forcing renoise loops; pure ODE solvers ignore it. + context: Optional per-AR-step encoder output. Most schedulers + ignore it; schedulers with token-local constraints can use it + without taking over the diffusion-model orchestration. Returns: Clean latent with the same shape, device, and dtype as diff --git a/flashdreams/flashdreams/infra/diffusion/scheduler/fm.py b/flashdreams/flashdreams/infra/diffusion/scheduler/fm.py index baaabf78b..75a6cbc4e 100644 --- a/flashdreams/flashdreams/infra/diffusion/scheduler/fm.py +++ b/flashdreams/flashdreams/infra/diffusion/scheduler/fm.py @@ -18,6 +18,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from typing import Any import torch from torch import Tensor @@ -212,6 +213,7 @@ def sample( initial_noise: Tensor, predict_flow: FlowPredictor, rng: torch.Generator | None = None, + context: Any = None, ) -> Tensor: """Run the self-forcing flow-match denoising loop. @@ -220,6 +222,7 @@ def sample( sigma before the network forward. Schedule arithmetic auto-promotes to fp32; the result is cast back to ``initial_noise.dtype``. """ + del context input_dtype = initial_noise.dtype sigmas = self.denoising_sigmas timesteps = self.denoising_step_list diff --git a/flashdreams/flashdreams/infra/diffusion/scheduler/fm_euler.py b/flashdreams/flashdreams/infra/diffusion/scheduler/fm_euler.py index 76498d987..2723e0bd0 100644 --- a/flashdreams/flashdreams/infra/diffusion/scheduler/fm_euler.py +++ b/flashdreams/flashdreams/infra/diffusion/scheduler/fm_euler.py @@ -29,6 +29,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from typing import Any import numpy as np import torch @@ -186,6 +187,7 @@ def sample( initial_noise: Tensor, predict_flow: FlowPredictor, rng: torch.Generator | None = None, + context: Any = None, ) -> Tensor: """Run the explicit Euler denoising loop. @@ -199,6 +201,7 @@ def sample( ``rng`` is unused (deterministic ODE) but accepted for interface conformance. """ + del context input_dtype = initial_noise.dtype N = self.config.num_inference_steps diff --git a/flashdreams/flashdreams/infra/diffusion/scheduler/fm_unipc.py b/flashdreams/flashdreams/infra/diffusion/scheduler/fm_unipc.py index a9ce940a7..ff5ca162d 100644 --- a/flashdreams/flashdreams/infra/diffusion/scheduler/fm_unipc.py +++ b/flashdreams/flashdreams/infra/diffusion/scheduler/fm_unipc.py @@ -18,6 +18,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from typing import Any import numpy as np import torch @@ -331,6 +332,7 @@ def sample( initial_noise: Tensor, predict_flow: FlowPredictor, rng: torch.Generator | None = None, + context: Any = None, ) -> Tensor: """Run the order-2 UniPC predictor-corrector denoising loop. @@ -340,6 +342,7 @@ def sample( fp32; the result is cast back to ``initial_noise.dtype``. ``rng`` is unused (deterministic ODE) but accepted for interface conformance. """ + del context input_dtype = initial_noise.dtype N = self.timesteps.shape[0] diff --git a/flashdreams/flashdreams/infra/diffusion/transformer/base.py b/flashdreams/flashdreams/infra/diffusion/transformer/base.py index c904e3d3a..1ff10749c 100644 --- a/flashdreams/flashdreams/infra/diffusion/transformer/base.py +++ b/flashdreams/flashdreams/infra/diffusion/transformer/base.py @@ -169,6 +169,37 @@ def initialize_autoregressive_cache(self, **context: Any) -> TransformerCacheT: """ return cast("TransformerCacheT", TransformerAutoregressiveCache()) + def initial_noise( + self, + *, + latent_shape: tuple[int, ...], + rng: torch.Generator | None, + cache: TransformerCacheT, + input: Any = None, + ) -> Tensor: + """Draw the starting latent for one denoising loop. + + Default is Gaussian noise in ``latent_shape``. Model integrations can + override this to seed from encoder outputs, pin I2V frames, or attach + per-step state to the cache while still using ``DiffusionModel``. + + Args: + latent_shape: Shape requested by the diffusion model. + rng: Per-model generator on ``self.device``, or ``None``. + cache: Per-rollout AR cache. + input: Same patchified encoder output passed to ``predict_flow``. + + Returns: + Initial latent tensor for scheduler sampling. + """ + del cache, input + return torch.randn( + latent_shape, + device=self.device, + dtype=self.dtype, + generator=rng, + ) + def postprocess_clean_latent( self, clean_latent: Tensor, diff --git a/integrations/sana/sana_wm/conditioning.py b/integrations/sana/sana_wm/conditioning.py new file mode 100644 index 000000000..6be23039b --- /dev/null +++ b/integrations/sana/sana_wm/conditioning.py @@ -0,0 +1,552 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SANA-WM prompt, first-frame, and camera conditioning components.""" + +from __future__ import annotations + +import os +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any + +import numpy as np +import torch +import torch.nn as nn +from torch import Tensor + +from flashdreams.infra.encoder import ( + Encoder, + EncoderConfig, + StreamingEncoder, + StreamingEncoderCache, +) +from sana_wm.camera import prepare_camera +from sana_wm.constants import ( + DEFAULT_VIDEO_HEIGHT, + DEFAULT_VIDEO_WIDTH, + SANA_WM_CONFIG_PATH, +) +from sana_wm.transformer import ( + QuantBackend, + SanaWMStage1Conditioning, + _chunk_index_from_config, + _get_tokenizer_and_text_encoder, + _get_vae, + _get_weight_dtype, + _load_inference_config, + _vae_encode_ltx2, +) + + +@dataclass(kw_only=True) +class SanaWMTextPromptRequest: + """Raw text prompt inputs for SANA-WM Stage 1.""" + + prompt: str + negative_prompt: str = "" + + +@dataclass(kw_only=True) +class SanaWMTextConditioning: + """Encoded positive and negative prompt tensors.""" + + condition: Tensor + condition_mask: Tensor + negative: Tensor + negative_mask: Tensor + + +@dataclass(kw_only=True) +class SanaWMCameraRequest: + """Raw camera trajectory inputs for SANA-WM conditioning.""" + + poses_c2w: np.ndarray + intrinsics_vec4: np.ndarray + + +@dataclass(kw_only=True) +class SanaWMI2VConditioning: + """Encoded first-frame and camera tensors for Stage-1 diffusion.""" + + first_latent: Tensor + camera: dict[str, Tensor] + num_frames: int + + +@dataclass(kw_only=True) +class SanaWMI2VConditioningRequest: + """Raw one-shot SANA-WM I2V rollout inputs.""" + + image: Any + prompt: str + poses_c2w: np.ndarray + intrinsics_vec4: np.ndarray + num_frames: int + fps: int + steps: int + cfg_scale: float + flow_shift: float | None + seed: int + negative_prompt: str = "" + + +@dataclass(kw_only=True) +class SanaWMTextPromptEncoderConfig(EncoderConfig): + """Config for the Stage-1 prompt encoder component.""" + + _target: type["SanaWMTextPromptEncoder"] = field( + default_factory=lambda: SanaWMTextPromptEncoder + ) + + config_path: str = SANA_WM_CONFIG_PATH + """SANA-WM inference YAML path or ``hf://`` URI.""" + + stage1_precision: str = "bf16" + """Stage-1 precision; quantized paths pad text tokens for scaled-MM.""" + + quant_backend: QuantBackend = "auto" + """Low-precision backend selector, included in the prompt cache key.""" + + offload_text_encoder: bool = False + """Move the text encoder back to CPU after prompt encoding.""" + + +class SanaWMTextPromptEncoder(Encoder): + """Encode SANA-WM Stage-1 positive and negative prompts.""" + + config: SanaWMTextPromptEncoderConfig + + def __init__(self, config: SanaWMTextPromptEncoderConfig) -> None: + super().__init__(config) + self.config = config + self._dummy = nn.Parameter(torch.empty(0)) + self._runtime_config: Any | None = None + self._text_encoder_built = False + self._prompt_cache: dict[ + tuple[object, ...], tuple[Tensor, Tensor, Tensor, Tensor] + ] = {} + + @property + def device(self) -> torch.device: + return self._dummy.device + + def forward(self, input: SanaWMTextPromptRequest) -> SanaWMTextConditioning: + """Encode prompt strings into Stage-1 text embeddings and masks.""" + cond, cond_mask, neg, neg_mask = self._encode_prompts( + input.prompt, + input.negative_prompt, + ) + cond, cond_mask, neg, neg_mask = self._pad_text_for_quant( + cond, + cond_mask, + neg, + neg_mask, + ) + return SanaWMTextConditioning( + condition=cond, + condition_mask=cond_mask, + negative=neg, + negative_mask=neg_mask, + ) + + def release_runtime(self) -> None: + """Release prompt encoder tensors.""" + self._prompt_cache.clear() + for attr in ("text_encoder", "tokenizer"): + if hasattr(self, attr): + setattr(self, attr, None) + self._text_encoder_built = False + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + def _ensure_runtime_config(self) -> Any: + if self._runtime_config is None: + self._runtime_config = _load_inference_config(self.config.config_path) + return self._runtime_config + + def _ensure_weight_dtype(self) -> torch.dtype: + return _get_weight_dtype(self._ensure_runtime_config().model.mixed_precision) + + def _ensure_text_encoder(self) -> None: + if self._text_encoder_built: + return + cfg = self._ensure_runtime_config() + self.tokenizer, self.text_encoder = _get_tokenizer_and_text_encoder( + name=cfg.text_encoder.text_encoder_name, + device=self.device, + ) + if self.config.offload_text_encoder: + self.text_encoder.to("cpu") + self._text_encoder_built = True + + def _encode_prompts( + self, + prompt: str, + negative_prompt: str, + ) -> tuple[Tensor, Tensor, Tensor, Tensor]: + cfg = self._ensure_runtime_config() + self._ensure_text_encoder() + max_length = cfg.text_encoder.model_max_length + chi_prompt = "\n".join(cfg.text_encoder.chi_prompt or []) + if chi_prompt: + prompt = chi_prompt + prompt + max_length_all = len(self.tokenizer.encode(chi_prompt)) + max_length - 2 + else: + max_length_all = max_length + + key = ( + prompt, + negative_prompt, + str(self.device), + str(self._ensure_weight_dtype()), + self.config.stage1_precision, + self.config.quant_backend, + ) + if key in self._prompt_cache: + return self._prompt_cache[key] + + move_text_encoder = self.config.offload_text_encoder or ( + _module_device(self.text_encoder) != self.device + ) + if move_text_encoder: + self.text_encoder.to(self.device) + + def encode(text: str, length: int) -> tuple[Tensor, Tensor]: + tokens = self.tokenizer( + [text], + max_length=length, + padding="max_length", + truncation=True, + return_tensors="pt", + ).to(self.device) + return self.text_encoder(tokens.input_ids, tokens.attention_mask)[0], ( + tokens.attention_mask + ) + + try: + cond, cond_mask = encode(prompt, max_length_all) + select = [0] + list(range(-max_length + 1, 0)) + cond = cond[:, None][:, :, select] + cond_mask = cond_mask[:, select] + neg, neg_mask = encode(negative_prompt, max_length) + result = (cond, cond_mask, neg[:, None], neg_mask) + self._prompt_cache.clear() + self._prompt_cache[key] = result + return result + finally: + if move_text_encoder: + self.text_encoder.to("cpu") + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + def _pad_text_for_quant( + self, + cond: Tensor, + cond_mask: Tensor, + neg: Tensor, + neg_mask: Tensor, + ) -> tuple[Tensor, Tensor, Tensor, Tensor]: + if self.config.stage1_precision == "bf16": + return cond, cond_mask, neg, neg_mask + multiple = int(os.environ.get("SANA_WM_STAGE1_NVFP4_TEXT_PAD_MULTIPLE", "8")) + if multiple <= 1: + return cond, cond_mask, neg, neg_mask + + def pad_pair(text: Tensor, mask: Tensor) -> tuple[Tensor, Tensor]: + pad = (-text.shape[-2]) % multiple + if pad == 0: + return text, mask + text_shape = list(text.shape) + text_shape[-2] = pad + mask_shape = list(mask.shape) + mask_shape[-1] = pad + return ( + torch.cat([text, text.new_zeros(text_shape)], dim=-2), + torch.cat([mask, mask.new_zeros(mask_shape)], dim=-1), + ) + + cond, cond_mask = pad_pair(cond, cond_mask) + neg, neg_mask = pad_pair(neg, neg_mask) + return cond, cond_mask, neg, neg_mask + + +@dataclass(kw_only=True) +class SanaWMFirstFrameEncoderConfig(EncoderConfig): + """Config for the first-frame VAE encoder component.""" + + _target: type["SanaWMFirstFrameEncoder"] = field( + default_factory=lambda: SanaWMFirstFrameEncoder + ) + + config_path: str = SANA_WM_CONFIG_PATH + """SANA-WM inference YAML path or ``hf://`` URI.""" + + offload_vae: bool = False + """Move the VAE to CPU after first-frame encoding.""" + + +class SanaWMFirstFrameEncoder(Encoder): + """Encode the input first frame with the LTX-2 VAE.""" + + config: SanaWMFirstFrameEncoderConfig + + def __init__(self, config: SanaWMFirstFrameEncoderConfig) -> None: + super().__init__(config) + self.config = config + self._dummy = nn.Parameter(torch.empty(0)) + self._runtime_config: Any | None = None + self._vae_built = False + + @property + def device(self) -> torch.device: + return self._dummy.device + + def forward(self, input: Any) -> Tensor: + """Encode a PIL-like RGB image into a single latent frame.""" + from torchvision import transforms as T + + cfg = self._ensure_runtime_config() + weight_dtype = _get_weight_dtype(cfg.model.mixed_precision) + self._ensure_vae() + if self.config.offload_vae: + self.vae.to(self.device) + + image = (T.ToTensor()(input) * 2.0 - 1.0).unsqueeze(0).unsqueeze(2) + latent = _vae_encode_ltx2( + cfg.vae.vae_type, + self.vae, + image.to(self.device, dtype=self.vae_dtype), + device=self.device, + ).to(weight_dtype) + if self.config.offload_vae: + self.vae.to("cpu") + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return latent + + def _ensure_runtime_config(self) -> Any: + if self._runtime_config is None: + self._runtime_config = _load_inference_config(self.config.config_path) + return self._runtime_config + + def _ensure_vae(self) -> None: + if self._vae_built: + return + cfg = self._ensure_runtime_config() + self.vae_dtype = _get_weight_dtype(cfg.vae.weight_dtype) + from sana_wm._tools import resolve_hf_path + + cfg.vae.vae_pretrained = resolve_hf_path(cfg.vae.vae_pretrained) + self.vae = _get_vae( + cfg.vae.vae_type, + cfg.vae.vae_pretrained, + device=self.device, + dtype=self.vae_dtype, + config=cfg.vae, + ) + self._vae_built = True + + +@dataclass(kw_only=True) +class SanaWMCameraConditioningEncoderConfig(EncoderConfig): + """Config for the camera/raymap conditioning component.""" + + _target: type["SanaWMCameraConditioningEncoder"] = field( + default_factory=lambda: SanaWMCameraConditioningEncoder + ) + + height: int = DEFAULT_VIDEO_HEIGHT + width: int = DEFAULT_VIDEO_WIDTH + + +class SanaWMCameraConditioningEncoder(Encoder): + """Build SANA-WM raymap and chunk-Plucker camera tensors.""" + + config: SanaWMCameraConditioningEncoderConfig + + def __init__(self, config: SanaWMCameraConditioningEncoderConfig) -> None: + super().__init__(config) + self.config = config + + def forward(self, input: SanaWMCameraRequest) -> dict[str, Tensor]: + """Encode raw poses and intrinsics into Stage-1 camera tensors.""" + return prepare_camera( + input.poses_c2w, + input.intrinsics_vec4, + target_size=(self.config.height, self.config.width), + ) + + +@dataclass(kw_only=True) +class SanaWMConditioningEncoderConfig(EncoderConfig): + """Config for the pipeline-level SANA-WM conditioning encoder.""" + + _target: type["SanaWMConditioningEncoder"] = field( + default_factory=lambda: SanaWMConditioningEncoder + ) + + config_path: str = SANA_WM_CONFIG_PATH + """SANA-WM inference YAML path or ``hf://`` URI.""" + + text_encoder: SanaWMTextPromptEncoderConfig = field( + default_factory=SanaWMTextPromptEncoderConfig + ) + first_frame_encoder: SanaWMFirstFrameEncoderConfig = field( + default_factory=SanaWMFirstFrameEncoderConfig + ) + camera_encoder: SanaWMCameraConditioningEncoderConfig = field( + default_factory=SanaWMCameraConditioningEncoderConfig + ) + height: int = DEFAULT_VIDEO_HEIGHT + width: int = DEFAULT_VIDEO_WIDTH + + +class SanaWMConditioningEncoder(StreamingEncoder[StreamingEncoderCache]): + """Prepare all per-rollout inputs for the Sana Stage-1 sampler.""" + + config: SanaWMConditioningEncoderConfig + + def __init__(self, config: SanaWMConditioningEncoderConfig) -> None: + super().__init__(config) + self.config = config + self.text_encoder = config.text_encoder.setup() + self.first_frame_encoder = config.first_frame_encoder.setup() + self.camera_encoder = config.camera_encoder.setup() + self._runtime_config: Any | None = None + + def initialize_autoregressive_cache(self, **_context: Any) -> StreamingEncoderCache: + """Return an empty cache for the one-shot conditioning encoder.""" + return StreamingEncoderCache() + + @torch.inference_mode() + def forward( + self, + input: SanaWMI2VConditioningRequest, + autoregressive_index: int = 0, + cache: StreamingEncoderCache | None = None, + ) -> SanaWMStage1Conditioning: + """Encode raw rollout inputs into Stage-1 conditioning.""" + del cache + if autoregressive_index != 0: + raise ValueError("SANA-WM bidirectional inference has one AR step.") + + cfg = self._ensure_runtime_config() + text = self.text_encoder( + SanaWMTextPromptRequest( + prompt=input.prompt, + negative_prompt=input.negative_prompt, + ) + ) + first_latent = self.first_frame_encoder(input.image) + weight_dtype = _get_weight_dtype(cfg.model.mixed_precision) + camera = self.camera_encoder( + SanaWMCameraRequest( + poses_c2w=input.poses_c2w, + intrinsics_vec4=input.intrinsics_vec4, + ) + ) + raymap = camera["raymap"].unsqueeze(0).to( + first_latent.device, + dtype=weight_dtype, + ) + chunk_plucker = camera["chunk_plucker"].unsqueeze(0).to( + first_latent.device, + dtype=weight_dtype, + ) + + model_kwargs_extra: dict[str, object] = {} + if input.cfg_scale > 1.0: + model_kwargs_extra["negative_mask"] = text.negative_mask + uncondition = text.negative + else: + uncondition = None + + vae_stride = cfg.vae.vae_stride + latent_t = (input.num_frames - 1) // int(vae_stride[0]) + 1 + latent_h = self.config.height // int(vae_stride[-1]) + latent_w = self.config.width // int(vae_stride[-1]) + chunk_index = _chunk_index_from_config(cfg, num_frames=latent_t) + model_kwargs: dict[str, object] = { + "data_info": { + "img_hw": torch.tensor( + [[self.config.height, self.config.width]], + dtype=torch.float, + device=first_latent.device, + ), + "condition_frame_info": {0: 0.0}, + }, + "mask": text.condition_mask, + "camera_conditions": raymap, + "chunk_plucker": chunk_plucker, + **model_kwargs_extra, + } + if chunk_index is not None: + model_kwargs["chunk_index"] = chunk_index + + return SanaWMStage1Conditioning( + condition=text.condition, + uncondition=uncondition, + model_kwargs=model_kwargs, + first_latent=first_latent, + latent_shape=( + 1, + int(first_latent.shape[1]), + latent_t, + latent_h, + latent_w, + ), + cfg_scale=float(input.cfg_scale), + flow_shift=self._resolve_flow_shift(input.flow_shift), + steps=int(input.steps), + seed=int(input.seed), + ) + + def _ensure_runtime_config(self) -> Any: + if self._runtime_config is None: + self._runtime_config = _load_inference_config(self.config.config_path) + return self._runtime_config + + def _resolve_flow_shift(self, override: float | None) -> float: + cfg = self._ensure_runtime_config() + if override is not None: + return float(override) + if cfg.scheduler.inference_flow_shift is not None: + return float(cfg.scheduler.inference_flow_shift) + return float(cfg.scheduler.flow_shift) + + +def _module_device(module: nn.Module) -> torch.device: + try: + return next(module.parameters()).device + except StopIteration: + return torch.device("cpu") + + +__all__ = [ + "SanaWMCameraConditioningEncoder", + "SanaWMCameraConditioningEncoderConfig", + "SanaWMCameraRequest", + "SanaWMConditioningEncoder", + "SanaWMConditioningEncoderConfig", + "SanaWMFirstFrameEncoder", + "SanaWMFirstFrameEncoderConfig", + "SanaWMI2VConditioning", + "SanaWMI2VConditioningRequest", + "SanaWMTextConditioning", + "SanaWMTextPromptEncoder", + "SanaWMTextPromptEncoderConfig", + "SanaWMTextPromptRequest", +] diff --git a/integrations/sana/sana_wm/config.py b/integrations/sana/sana_wm/config.py index 5d9a176c8..e08b2c679 100644 --- a/integrations/sana/sana_wm/config.py +++ b/integrations/sana/sana_wm/config.py @@ -17,21 +17,25 @@ from __future__ import annotations -from flashdreams.infra.diffusion.scheduler import FlowMatchSchedulerConfig from flashdreams.infra.pipeline import StreamInferencePipelineConfig from flashdreams.infra.runner import RunnerConfig +from sana_wm.conditioning import SanaWMConditioningEncoderConfig +from sana_wm.decoder import SanaWMVideoDecoderConfig from sana_wm.diffusion import SanaWMDiffusionModelConfig from sana_wm.runner import SanaWMRunnerConfig +from sana_wm.scheduler import SanaWMLTXEulerSchedulerConfig from sana_wm.transformer import SanaWMTransformerConfig PIPELINE_SANA_WM_BIDIRECTIONAL = StreamInferencePipelineConfig( name="sana-wm-bidirectional", + encoder=SanaWMConditioningEncoderConfig(), diffusion_model=SanaWMDiffusionModelConfig( transformer=SanaWMTransformerConfig(), - scheduler=FlowMatchSchedulerConfig(), + scheduler=SanaWMLTXEulerSchedulerConfig(), seed=42, ), + decoder=SanaWMVideoDecoderConfig(), ) """FlashDreams SANA-WM pipeline.""" diff --git a/integrations/sana/sana_wm/decoder.py b/integrations/sana/sana_wm/decoder.py new file mode 100644 index 000000000..698e1d6aa --- /dev/null +++ b/integrations/sana/sana_wm/decoder.py @@ -0,0 +1,488 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SANA-WM latent refiner and VAE decoder components.""" + +from __future__ import annotations + +import gc +import time +from dataclasses import dataclass, field +from typing import Any + +import numpy as np +import torch +import torch.nn as nn +from loguru import logger +from torch import Tensor + +from flashdreams.infra.config import InstantiateConfig +from flashdreams.infra.decoder import ( + DecoderConfig, + StreamingDecoder, + StreamingDecoderCache, +) +from sana_wm._tools import resolve_hf_path +from sana_wm.constants import ( + DEFAULT_VIDEO_HEIGHT, + DEFAULT_VIDEO_WIDTH, + SANA_WM_CONFIG_PATH, + SANA_WM_REFINER_GEMMA_ROOT, + SANA_WM_REFINER_ROOT, + SANA_WM_VAE_SPATIAL_COMPRESSION, + SANA_WM_VAE_TEMPORAL_COMPRESSION, +) +from sana_wm.transformer import ( + Precision, + QuantBackend, + _avoid_degenerate_tile_tail, + _get_vae, + _get_weight_dtype, + _load_inference_config, + _vae_decode_ltx2, +) + + +@dataclass(kw_only=True) +class SanaWMDecodedVideo: + """Decoded SANA-WM video outputs.""" + + video_hwc: np.ndarray + stage1_video_hwc: np.ndarray | None = None + + +@dataclass(kw_only=True) +class SanaWMVideoDecoderCache(StreamingDecoderCache): + """Per-rollout settings for the SANA-WM decoder/refiner stage.""" + + prompt: str = "" + fps: int = 16 + save_stage1: bool = False + refiner_seed: int = 42 + sink_size: int = 1 + refiner_block_size: int | None = None + refiner_kv_max_frames: int = 11 + + +@dataclass(kw_only=True) +class SanaWMLTX2VAEDecoderConfig(InstantiateConfig): + """Config for the LTX-2 VAE latent-to-video decoder.""" + + _target: type["SanaWMLTX2VAEDecoder"] = field( + default_factory=lambda: SanaWMLTX2VAEDecoder + ) + + config_path: str = SANA_WM_CONFIG_PATH + """SANA-WM inference YAML path or ``hf://`` URI.""" + + height: int = DEFAULT_VIDEO_HEIGHT + width: int = DEFAULT_VIDEO_WIDTH + + offload_vae: bool = False + """Move the VAE to CPU between decode calls.""" + + vae_tile_sample_min_width: int = 256 + vae_tile_sample_stride_width: int = 224 + vae_tile_sample_min_height: int = 256 + vae_tile_sample_stride_height: int = 192 + vae_tile_sample_min_num_frames: int = 24 + vae_tile_sample_stride_num_frames: int = 8 + + vae_oom_retry_tile_sample_min_width: int = 128 + vae_oom_retry_tile_sample_stride_width: int = 64 + vae_oom_retry_tile_sample_min_height: int = 128 + vae_oom_retry_tile_sample_stride_height: int = 64 + vae_oom_retry_tile_sample_min_num_frames: int = 16 + vae_oom_retry_tile_sample_stride_num_frames: int = 8 + + +class SanaWMLTX2VAEDecoder(nn.Module): + """Decode SANA-WM LTX-2 VAE latents into HWC uint8 video.""" + + config: SanaWMLTX2VAEDecoderConfig + + def __init__(self, config: SanaWMLTX2VAEDecoderConfig) -> None: + super().__init__() + self.config = config + self._dummy = nn.Parameter(torch.empty(0)) + self._runtime_config: Any | None = None + self._vae_built = False + + @property + def device(self) -> torch.device: + return self._dummy.device + + @torch.inference_mode() + def decode_latents(self, latents: Tensor) -> np.ndarray: + """Decode VAE latents to ``uint8`` HWC video.""" + self._ensure_vae() + if self.config.offload_vae: + self.vae.to(self.device) + samples = latents.to(device=self.device, dtype=self.vae_dtype) + if torch.cuda.is_available(): + torch.cuda.synchronize() + t0 = time.perf_counter() + retry_decode = False + try: + decoded = self._vae_decode(samples) + except torch.OutOfMemoryError: + retry_decode = True + + if retry_decode: + if torch.cuda.is_available(): + torch.cuda.synchronize() + torch.cuda.empty_cache() + gc.collect() + logger.warning( + "[sana-vae] decode OOM; retrying with smaller tiles " + "width={} stride_width={} height={} stride_height={} " + "frames={} stride_frames={}", + self.config.vae_oom_retry_tile_sample_min_width, + self.config.vae_oom_retry_tile_sample_stride_width, + self.config.vae_oom_retry_tile_sample_min_height, + self.config.vae_oom_retry_tile_sample_stride_height, + self.config.vae_oom_retry_tile_sample_min_num_frames, + self.config.vae_oom_retry_tile_sample_stride_num_frames, + ) + self._configure_vae_tiling( + tile_sample_min_width=self.config.vae_oom_retry_tile_sample_min_width, + tile_sample_stride_width=( + self.config.vae_oom_retry_tile_sample_stride_width + ), + tile_sample_min_height=self.config.vae_oom_retry_tile_sample_min_height, + tile_sample_stride_height=( + self.config.vae_oom_retry_tile_sample_stride_height + ), + tile_sample_min_num_frames=( + self.config.vae_oom_retry_tile_sample_min_num_frames + ), + tile_sample_stride_num_frames=( + self.config.vae_oom_retry_tile_sample_stride_num_frames + ), + ) + if torch.cuda.is_available(): + torch.cuda.empty_cache() + decoded = self._vae_decode(samples) + if torch.cuda.is_available(): + torch.cuda.synchronize() + logger.info( + "[timing] vae decode: {:.3f}s (latent T={} -> pixels {})", + time.perf_counter() - t0, + latents.shape[2], + tuple(decoded.shape) if isinstance(decoded, Tensor) else "list", + ) + if isinstance(decoded, list): + decoded = torch.stack(decoded, dim=0) + video = ( + torch.clamp(127.5 * decoded + 127.5, 0, 255) + .permute(0, 2, 3, 4, 1) + .to("cpu", dtype=torch.uint8) + .numpy()[0] + ) + if self.config.offload_vae: + self.vae.to("cpu") + del samples, decoded + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return video + + def _ensure_runtime_config(self) -> Any: + if self._runtime_config is None: + self._runtime_config = _load_inference_config(self.config.config_path) + return self._runtime_config + + def _ensure_vae(self) -> None: + if self._vae_built: + return + cfg = self._ensure_runtime_config() + self.vae_dtype = _get_weight_dtype(cfg.vae.weight_dtype) + cfg.vae.vae_pretrained = resolve_hf_path(cfg.vae.vae_pretrained) + self.vae = _get_vae( + cfg.vae.vae_type, + cfg.vae.vae_pretrained, + device=self.device, + dtype=self.vae_dtype, + config=cfg.vae, + ) + if hasattr(self.vae, "enable_tiling"): + self._configure_vae_tiling() + self._vae_built = True + + def _configure_vae_tiling( + self, + *, + tile_sample_min_width: int | None = None, + tile_sample_stride_width: int | None = None, + tile_sample_min_height: int | None = None, + tile_sample_stride_height: int | None = None, + tile_sample_min_num_frames: int | None = None, + tile_sample_stride_num_frames: int | None = None, + ) -> None: + vae = self.vae + min_width = int(tile_sample_min_width or self.config.vae_tile_sample_min_width) + stride_width = int( + tile_sample_stride_width or self.config.vae_tile_sample_stride_width + ) + min_height = int(tile_sample_min_height or self.config.vae_tile_sample_min_height) + stride_height = int( + tile_sample_stride_height or self.config.vae_tile_sample_stride_height + ) + spatial_ratio = int(getattr(vae, "spatial_compression_ratio", 1)) + stride_width = _avoid_degenerate_tile_tail( + sample_extent=self.config.width, + sample_tile_min=min_width, + sample_stride=stride_width, + compression_ratio=spatial_ratio, + ) + stride_height = _avoid_degenerate_tile_tail( + sample_extent=self.config.height, + sample_tile_min=min_height, + sample_stride=stride_height, + compression_ratio=spatial_ratio, + ) + min_frames = int( + tile_sample_min_num_frames + or self.config.vae_tile_sample_min_num_frames + ) + stride_frames = int( + tile_sample_stride_num_frames + or self.config.vae_tile_sample_stride_num_frames + ) + temporal_ratio = int(getattr(vae, "temporal_compression_ratio", 1)) + if temporal_ratio > 1: + min_frames = max(min_frames, temporal_ratio) + stride_frames = max(stride_frames, temporal_ratio) + kwargs = { + "tile_sample_min_height": min_height, + "tile_sample_stride_height": stride_height, + "tile_sample_min_width": min_width, + "tile_sample_stride_width": stride_width, + "tile_sample_min_num_frames": min_frames, + "tile_sample_stride_num_frames": stride_frames, + } + if hasattr(vae, "enable_tiling"): + try: + vae.enable_tiling(**kwargs) + except TypeError: + vae.enable_tiling() + for name, value in kwargs.items(): + if hasattr(vae, name): + setattr(vae, name, value) + if hasattr(vae, "use_framewise_encoding"): + vae.use_framewise_encoding = True + if hasattr(vae, "use_framewise_decoding"): + vae.use_framewise_decoding = True + logger.info( + "[sana-vae] tiling width={} stride_width={} height={} " + "stride_height={} frames={} stride_frames={}", + min_width, + stride_width, + min_height, + stride_height, + min_frames, + stride_frames, + ) + + def _vae_decode(self, latents: Tensor) -> Tensor: + return _vae_decode_ltx2( + self._ensure_runtime_config().vae.vae_type, + self.vae, + latents, + ) + + +@dataclass(kw_only=True) +class SanaWMLTX2LatentRefinerConfig(InstantiateConfig): + """Config for the optional LTX-2 latent refiner component.""" + + _target: type["SanaWMLTX2LatentRefiner"] = field( + default_factory=lambda: SanaWMLTX2LatentRefiner + ) + + refiner_root: str = SANA_WM_REFINER_ROOT + refiner_gemma_root: str = SANA_WM_REFINER_GEMMA_ROOT + refiner_precision: Precision = "bf16" + quant_backend: QuantBackend = "torch" + offload_refiner: bool = False + + +class SanaWMLTX2LatentRefiner(nn.Module): + """Run the optional LTX-2 refinement stage over Stage-1 latents.""" + + config: SanaWMLTX2LatentRefinerConfig + + def __init__(self, config: SanaWMLTX2LatentRefinerConfig) -> None: + super().__init__() + self.config = config + self._dummy = nn.Parameter(torch.empty(0)) + self._refiner_built = False + + @property + def device(self) -> torch.device: + return self._dummy.device + + @torch.inference_mode() + def refine_latents( + self, + *, + latents: Tensor, + prompt: str, + fps: int, + sink_size: int, + seed: int, + block_size: int | None, + kv_max_frames: int, + ) -> Tensor: + """Run the LTX-2 refiner.""" + self._ensure_refiner() + refined = self.refiner.refine_latents( + latents, + prompt, + fps=float(fps), + sink_size=int(sink_size), + seed=int(seed), + progress=True, + block_size=block_size, + kv_max_frames=int(kv_max_frames), + ) + if self.config.offload_refiner: + self.release_runtime() + return refined + + def release_runtime(self) -> None: + """Release refiner tensors.""" + if not self._refiner_built: + return + del self.refiner + self._refiner_built = False + if torch.cuda.is_available(): + torch.cuda.empty_cache() + gc.collect() + + def _ensure_refiner(self) -> None: + if self._refiner_built: + return + from sana_wm.refiner import SanaWMLTX2Refiner + + compute_dtype = ( + torch.bfloat16 + if self.config.refiner_precision in {"fp8", "fp4"} + else _get_weight_dtype(self.config.refiner_precision) + ) + self.refiner = SanaWMLTX2Refiner( + refiner_root=resolve_hf_path(self.config.refiner_root), + gemma_root=resolve_hf_path(self.config.refiner_gemma_root), + dtype=compute_dtype, + device=self.device, + precision=self.config.refiner_precision, + quant_backend=self.config.quant_backend, + ) + self._refiner_built = True + + +@dataclass(kw_only=True) +class SanaWMVideoDecoderConfig(DecoderConfig): + """Config for the SANA-WM latent refiner plus VAE decode boundary.""" + + _target: type["SanaWMVideoDecoder"] = field( + default_factory=lambda: SanaWMVideoDecoder + ) + + vae_decoder: SanaWMLTX2VAEDecoderConfig = field( + default_factory=SanaWMLTX2VAEDecoderConfig + ) + refiner: SanaWMLTX2LatentRefinerConfig | None = field( + default_factory=SanaWMLTX2LatentRefinerConfig + ) + + +class SanaWMVideoDecoder(StreamingDecoder[SanaWMVideoDecoderCache]): + """Decode Stage-1 latents, optionally through the LTX-2 refiner.""" + + config: SanaWMVideoDecoderConfig + + def __init__(self, config: SanaWMVideoDecoderConfig) -> None: + super().__init__(config) + self.config = config + self.vae_decoder = config.vae_decoder.setup() + self.refiner = config.refiner.setup() if config.refiner is not None else None + + def initialize_autoregressive_cache( + self, + **context: Any, + ) -> SanaWMVideoDecoderCache: + """Build per-rollout decode/refiner settings.""" + return SanaWMVideoDecoderCache(**context) + + @torch.inference_mode() + def forward( + self, + input: Tensor, + autoregressive_index: int = 0, + cache: SanaWMVideoDecoderCache | None = None, + ) -> SanaWMDecodedVideo: + """Refine/decode one SANA-WM latent rollout.""" + if autoregressive_index != 0: + raise ValueError("SANA-WM bidirectional inference has one AR step.") + cache = cache or SanaWMVideoDecoderCache() + stage1_latent = input + output_latent = input + if self.refiner is not None: + output_latent = self.refiner.refine_latents( + latents=stage1_latent, + prompt=cache.prompt, + fps=cache.fps, + sink_size=cache.sink_size, + seed=cache.refiner_seed, + block_size=cache.refiner_block_size, + kv_max_frames=cache.refiner_kv_max_frames, + ) + self.refiner.release_runtime() + elif cache.save_stage1: + logger.info( + "SANA-WM is already running without the refiner; " + "--save-stage1 does not create an extra output." + ) + + video_hwc = self.vae_decoder.decode_latents(output_latent) + stage1_video_hwc = None + if cache.save_stage1 and self.refiner is not None: + stage1_video_hwc = self.vae_decoder.decode_latents(stage1_latent) + return SanaWMDecodedVideo( + video_hwc=video_hwc, + stage1_video_hwc=stage1_video_hwc, + ) + + @property + def spatial_compression_ratio(self) -> int: + """Pixel side divided by latent side.""" + return SANA_WM_VAE_SPATIAL_COMPRESSION + + @property + def temporal_compression_ratio(self) -> int: + """Pixel frame compression ratio after the first latent.""" + return SANA_WM_VAE_TEMPORAL_COMPRESSION + + +__all__ = [ + "SanaWMDecodedVideo", + "SanaWMLTX2LatentRefiner", + "SanaWMLTX2LatentRefinerConfig", + "SanaWMLTX2VAEDecoder", + "SanaWMLTX2VAEDecoderConfig", + "SanaWMVideoDecoder", + "SanaWMVideoDecoderCache", + "SanaWMVideoDecoderConfig", +] diff --git a/integrations/sana/sana_wm/diffusion.py b/integrations/sana/sana_wm/diffusion.py index 2c26149b3..6b67f01b5 100644 --- a/integrations/sana/sana_wm/diffusion.py +++ b/integrations/sana/sana_wm/diffusion.py @@ -13,240 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""SANA-WM-specific diffusion model for first-frame-pinned LTX Euler sampling.""" +"""SANA-WM diffusion config bound to the shared FlashDreams diffusion model.""" from __future__ import annotations -import os -import time from dataclasses import dataclass, field -from typing import Any, cast - -import torch -from loguru import logger -from torch import Tensor from flashdreams.infra.diffusion.model import DiffusionModel, DiffusionModelConfig -from sana_wm.transformer import ( - SanaWMTransformer, - SanaWMTransformerCache, -) @dataclass(kw_only=True) class SanaWMDiffusionModelConfig(DiffusionModelConfig): - """Diffusion model config for SANA-WM's LTX-style Stage-1 sampler.""" - - _target: type["SanaWMDiffusionModel"] = field( - default_factory=lambda: SanaWMDiffusionModel - ) - - -class SanaWMDiffusionModel(DiffusionModel[SanaWMTransformerCache]): - """Run SANA-WM Stage-1 denoising behind the FlashDreams diffusion boundary.""" - - transformer: SanaWMTransformer - - def __init__(self, config: SanaWMDiffusionModelConfig) -> None: - super().__init__(config) - if not isinstance(self.transformer, SanaWMTransformer): - raise TypeError( - "SanaWMDiffusionModel requires SanaWMTransformerConfig." - ) - - def generate( - self, - autoregressive_index: int, - cache: SanaWMTransformerCache, - input: Any = None, - ) -> tuple[Tensor, "DiffusionModel.FinalState[SanaWMTransformerCache]"]: - """Run SANA-WM's first-frame-pinned LTX Euler denoising loop.""" - del input - if autoregressive_index != 0: - raise ValueError("SANA-WM bidirectional inference has one AR step.") - conditioning = cache.conditioning - if conditioning is None: - raise RuntimeError("SANA-WM diffusion cache has no conditioning.") - - cache.start(autoregressive_index) - latents = self.transformer.initial_latents(conditioning) - if torch.cuda.is_available(): - torch.cuda.synchronize() - t0 = time.perf_counter() - clean_latent = self._sample_ltx_euler(latents, cache) - if torch.cuda.is_available(): - torch.cuda.synchronize() - logger.info( - "[timing] stage1 sample: {:.3f}s (latent shape {})", - time.perf_counter() - t0, - tuple(clean_latent.shape), - ) - - final_state = DiffusionModel.FinalState( - clean_latent=clean_latent, - autoregressive_index=autoregressive_index, - cache=cache, - input=None, - ) - return clean_latent, final_state - - def finalize( - self, - final_state: "DiffusionModel.FinalState[SanaWMTransformerCache]", - ) -> None: - """Finalize the one-shot cache without an extra model forward.""" - final_state.cache.finalize(final_state.autoregressive_index) - - def _sample_ltx_euler( - self, - latents: Tensor, - cache: SanaWMTransformerCache, - ) -> Tensor: - conditioning = cache.conditioning - assert conditioning is not None - - from diffusers import FlowMatchEulerDiscreteScheduler - from diffusers.pipelines.stable_diffusion_3.pipeline_stable_diffusion_3 import ( - retrieve_timesteps, - ) - from tqdm import tqdm - - scheduler = FlowMatchEulerDiscreteScheduler(shift=conditioning.flow_shift) - timesteps, _ = retrieve_timesteps( - scheduler, - conditioning.steps, - self.device, - None, - ) - do_cfg = conditioning.cfg_scale > 1.0 - condition_frame_info = dict( - cast( - dict[int, float], - cast(dict[str, object], conditioning.model_kwargs["data_info"]).get( - "condition_frame_info", - {}, - ), - ) - ) - condition_mask = torch.zeros_like(latents) - image_cond_noise_scale = 0.0 - for frame_idx, frame_weight in condition_frame_info.items(): - condition_mask[:, :, int(frame_idx)] = 1 - image_cond_noise_scale = max(image_cond_noise_scale, float(frame_weight)) - - condition_kwargs = _condition_model_kwargs(conditioning.model_kwargs) - uncondition_kwargs = _uncondition_model_kwargs(conditioning.model_kwargs) - if do_cfg and conditioning.uncondition is None: - raise RuntimeError("CFG was requested without negative prompt embeds.") - - init_latents = latents.clone() - generator = torch.Generator(device=self.device).manual_seed(conditioning.seed) - iterator = enumerate(timesteps) - if os.getenv("DPM_TQDM", "False") != "True": - iterator = tqdm(list(iterator)) - - for _, timestep_scalar in iterator: - if image_cond_noise_scale > 0: - latents = _add_noise_to_conditioning_latents( - t=timestep_scalar / 1000.0, - init_latents=init_latents, - latents=latents, - noise_scale=image_cond_noise_scale, - conditioning_mask=condition_mask, - generator=generator, - ) - - timestep = timestep_scalar.expand(condition_mask.shape).float() - timestep = torch.min(timestep, (1 - condition_mask) * 1000.0) - if do_cfg: - assert conditioning.uncondition is not None - noise_pred_uncond = self.transformer.predict_flow( - noisy_latent=latents, - timestep=timestep[:, :1, :, 0, 0], - cache=cache, - input=conditioning.uncondition, - model_kwargs=uncondition_kwargs, - ) - noise_pred_text = self.transformer.predict_flow( - noisy_latent=latents, - timestep=timestep[:, :1, :, 0, 0], - cache=cache, - input=conditioning.condition, - model_kwargs=condition_kwargs, - ) - noise_pred = noise_pred_uncond + conditioning.cfg_scale * ( - noise_pred_text - noise_pred_uncond - ) - del noise_pred_uncond, noise_pred_text - else: - noise_pred = self.transformer.predict_flow( - noisy_latent=latents, - timestep=timestep[:, :1, :, 0, 0], - cache=cache, - input=conditioning.condition, - model_kwargs=condition_kwargs, - ) - - latents_dtype = latents.dtype - latents_shape = latents.shape - batch_size, channels, _frames, _height, _width = latents_shape - denoised_latents = scheduler.step( - -noise_pred.reshape(batch_size, channels, -1).transpose(1, 2), - timestep_scalar, - latents.reshape(batch_size, channels, -1).transpose(1, 2), - per_token_timesteps=timestep.reshape(batch_size, channels, -1)[:, 0], - return_dict=False, - )[0] - denoised_latents = denoised_latents.transpose(1, 2).reshape(latents_shape) - tokens_to_denoise_mask = timestep_scalar / 1000 - 1e-6 < ( - 1.0 - condition_mask - ) - latents = torch.where(tokens_to_denoise_mask, denoised_latents, latents) - if latents.dtype != latents_dtype: - latents = latents.to(latents_dtype) - - return latents.detach() - - -def _condition_model_kwargs(model_kwargs: dict[str, object]) -> dict[str, object]: - """Return model kwargs for the positive prompt branch.""" - return { - key: value - for key, value in model_kwargs.items() - if key != "negative_mask" - } - - -def _uncondition_model_kwargs(model_kwargs: dict[str, object]) -> dict[str, object]: - """Return model kwargs for the negative prompt branch.""" - kwargs = _condition_model_kwargs(model_kwargs) - negative_mask = model_kwargs.get("negative_mask") - if negative_mask is not None: - kwargs["mask"] = negative_mask - return kwargs - + """Diffusion model config for SANA-WM's Stage-1 sampler. -def _add_noise_to_conditioning_latents( - *, - t: Tensor, - init_latents: Tensor, - latents: Tensor, - noise_scale: float, - conditioning_mask: Tensor, - generator: torch.Generator, - eps: float = 1e-6, -) -> Tensor: - from diffusers.utils.torch_utils import randn_tensor + Sana-specific behavior lives in the configured transformer and scheduler; + this config intentionally instantiates the common FlashDreams + :class:`DiffusionModel` instead of a custom ``generate`` implementation. + """ - noise = randn_tensor( - latents.shape, - generator=generator, - device=latents.device, - dtype=latents.dtype, - ) - need_to_noise = conditioning_mask > (1.0 - eps) - noised_latents = init_latents + noise_scale * noise * (t**2) - return torch.where(need_to_noise, noised_latents, latents) + _target: type[DiffusionModel] = field(default_factory=lambda: DiffusionModel) -__all__ = ["SanaWMDiffusionModel", "SanaWMDiffusionModelConfig"] +__all__ = ["SanaWMDiffusionModelConfig"] diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index 02d1163d0..48cada5f6 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -35,11 +35,11 @@ from sana_wm.camera import ( action_string_to_c2w, load_intrinsics, - prepare_camera, resize_center_crop_geometry, snap_num_frames, transform_intrinsics_for_crop, ) +from sana_wm.conditioning import SanaWMI2VConditioningRequest from sana_wm.constants import ( DEFAULT_ACTION, DEFAULT_FPS, @@ -51,7 +51,7 @@ SANA_WM_REFINER_ROOT, SANA_WM_VAE_TEMPORAL_COMPRESSION, ) -from sana_wm.transformer import SanaWMTransformer +from sana_wm.decoder import SanaWMDecodedVideo SamplingAlgo = Literal["auto", "flow_euler_ltx"] """Sampling algorithms exposed by the SANA-WM runner.""" @@ -273,73 +273,54 @@ def run(self) -> None: quant_backend=quant_backend, ) pipeline = pipeline_cfg.setup().to(device).eval() - transformer = pipeline.diffusion_model.transformer - if not isinstance(transformer, SanaWMTransformer): - raise TypeError( - "SANA-WM runner resolved a non-SANA transformer: " - f"{type(transformer).__name__}." - ) sampling_algo = self._sampling_algo() if sampling_algo != "flow_euler_ltx": raise ValueError( "SANA-WM requires flow_euler_ltx for the " f"bidirectional runner; got {sampling_algo!r}." ) - camera = prepare_camera( - c2w, - intrinsics_vec4, - target_size=(DEFAULT_VIDEO_HEIGHT, DEFAULT_VIDEO_WIDTH), - ) - conditioning = transformer.prepare_conditioning( - image=image, - prompt=prompt, - camera=camera, - num_frames=num_frames, - fps=cfg.fps, - steps=cfg.step, - cfg_scale=cfg.cfg_scale, - flow_shift=cfg.flow_shift, - seed=cfg.seed, - negative_prompt=cfg.negative_prompt, - ) cache = pipeline.initialize_cache( - transformer_context={"conditioning": conditioning} + decoder_context={ + "prompt": prompt, + "fps": cfg.fps, + "save_stage1": cfg.save_stage1, + "refiner_seed": cfg.refiner_seed, + "sink_size": cfg.sink_size, + "refiner_block_size": cfg.refiner_block_size, + "refiner_kv_max_frames": cfg.refiner_kv_max_frames, + } ) - sana_latent = pipeline.generate(0, cache) - pipeline.finalize(0, cache) - transformer.release_stage1_runtime(cache) - del conditioning, cache - - stage1_video_hwc: np.ndarray | None = None - output_latent = sana_latent - if not cfg.no_refiner: - output_latent = transformer.refine_latents( - latents=sana_latent, + decoded = pipeline.generate( + 0, + cache, + input=SanaWMI2VConditioningRequest( + image=image, prompt=prompt, + poses_c2w=c2w, + intrinsics_vec4=intrinsics_vec4, + num_frames=num_frames, fps=cfg.fps, - sink_size=cfg.sink_size, - seed=cfg.refiner_seed, - block_size=cfg.refiner_block_size, - kv_max_frames=cfg.refiner_kv_max_frames, - ) - transformer.release_refiner_runtime() - elif cfg.save_stage1: - logger.info( - "SANA-WM is already running without the refiner; " - "--save-stage1 does not create an extra output." + steps=cfg.step, + cfg_scale=cfg.cfg_scale, + flow_shift=cfg.flow_shift, + seed=cfg.seed, + negative_prompt=cfg.negative_prompt, + ), + ) + pipeline.finalize(0, cache) + if not isinstance(decoded, SanaWMDecodedVideo): + raise TypeError( + "SANA-WM pipeline decoder returned " + f"{type(decoded).__name__}, expected SanaWMDecodedVideo." ) - - video_hwc = transformer.decode_latents(output_latent) - if self.is_rank_zero and cfg.save_stage1 and not cfg.no_refiner: - stage1_video_hwc = transformer.decode_latents(sana_latent) if not self.is_rank_zero: return - _write_video(cfg.output_dir, cfg.name, video_hwc, cfg.fps) - if stage1_video_hwc is not None: + _write_video(cfg.output_dir, cfg.name, decoded.video_hwc, cfg.fps) + if decoded.stage1_video_hwc is not None: _write_video( cfg.output_dir, f"{cfg.name}_stage1", - stage1_video_hwc, + decoded.stage1_video_hwc, cfg.fps, ) @@ -416,13 +397,43 @@ def _pipeline_config( checkpoint_path=cfg.model_path, stage1_precision=cfg.stage1_precision, quant_backend=quant_backend, - refiner_root=cfg.refiner_root, - refiner_gemma_root=cfg.refiner_gemma_root, - refiner_precision=cfg.refiner_precision, - offload_vae=cfg.offload_vae, - offload_refiner=cfg.offload_refiner, + ), + ), + encoder=dict( + config_path=cfg.config_path, + text_encoder=dict( + config_path=cfg.config_path, + stage1_precision=cfg.stage1_precision, + quant_backend=quant_backend, offload_text_encoder=cfg.offload_text_encoder, ), + first_frame_encoder=dict( + config_path=cfg.config_path, + offload_vae=cfg.offload_vae, + ), + camera_encoder=dict( + height=DEFAULT_VIDEO_HEIGHT, + width=DEFAULT_VIDEO_WIDTH, + ), + height=DEFAULT_VIDEO_HEIGHT, + width=DEFAULT_VIDEO_WIDTH, + ), + decoder=dict( + vae_decoder=dict( + config_path=cfg.config_path, + offload_vae=cfg.offload_vae, + ), + refiner=( + None + if cfg.no_refiner + else dict( + refiner_root=cfg.refiner_root, + refiner_gemma_root=cfg.refiner_gemma_root, + refiner_precision=cfg.refiner_precision, + quant_backend=quant_backend, + offload_refiner=cfg.offload_refiner, + ) + ), ), ) diff --git a/integrations/sana/sana_wm/scheduler.py b/integrations/sana/sana_wm/scheduler.py new file mode 100644 index 000000000..4bf1ed288 --- /dev/null +++ b/integrations/sana/sana_wm/scheduler.py @@ -0,0 +1,310 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SANA-WM LTX-style Euler scheduler boundary.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import Any, cast + +import numpy as np +import torch +from torch import Tensor + +from flashdreams.infra.diffusion.scheduler import ( + FlowPredictor, + Scheduler, + SchedulerConfig, +) +from sana_wm.transformer import SanaWMStage1Conditioning + + +@dataclass(kw_only=True) +class SanaWMLTXEulerSchedulerConfig(SchedulerConfig): + """Config for SANA-WM's LTX-style flow-matching Euler scheduler.""" + + _target: type["SanaWMLTXEulerScheduler"] = field( + default_factory=lambda: SanaWMLTXEulerScheduler + ) + + num_inference_steps: int = 60 + """Default number of Euler steps when using the generic scheduler API.""" + + shift: float = 9.8 + """Default flow-match schedule shift when using the generic scheduler API.""" + + num_train_timesteps: int = 1000 + """Training timestep scale used by the public SANA-WM release.""" + + +class SanaWMLTXEulerScheduler(Scheduler): + """Euler scheduler with SANA-WM per-token timestep support.""" + + config: SanaWMLTXEulerSchedulerConfig + + def __init__(self, config: SanaWMLTXEulerSchedulerConfig) -> None: + super().__init__(config) + self.config = config + + def timesteps( + self, + *, + num_inference_steps: int, + shift: float, + device: torch.device | str, + ) -> Tensor: + """Return diffusers-compatible FlowMatch Euler timesteps.""" + steps = int(num_inference_steps) + if steps <= 0: + raise ValueError(f"num_inference_steps must be > 0, got {steps}.") + num_train_timesteps = int(self.config.num_train_timesteps) + init_timesteps = np.linspace( + 1, + num_train_timesteps, + num_train_timesteps, + dtype=np.float32, + )[::-1].copy() + init_sigmas = init_timesteps / num_train_timesteps + init_sigmas = shift * init_sigmas / (1.0 + (shift - 1.0) * init_sigmas) + sigma_min = float(init_sigmas[-1]) + sigma_max = float(init_sigmas[0]) + timesteps = np.linspace( + sigma_max * num_train_timesteps, + sigma_min * num_train_timesteps, + steps, + ) + sigmas = timesteps / num_train_timesteps + sigmas = shift * sigmas / (1.0 + (shift - 1.0) * sigmas) + sigmas = np.concatenate([sigmas, [0.0]]).astype(np.float32) + return torch.from_numpy(sigmas * num_train_timesteps).to(device=device) + + def step_ltx( + self, + *, + model_output: Tensor, + timestep: Tensor, + next_timestep: Tensor, + sample: Tensor, + per_token_timesteps: Tensor | None = None, + schedule_timesteps: Tensor | None = None, + ) -> Tensor: + """Apply one SANA-WM LTX Euler step in token layout. + + Args: + model_output: Flow tensor in ``[B, N, C]`` token layout. + timestep: Current scalar training-scale timestep. + next_timestep: Next scalar training-scale timestep. + sample: Current latent in ``[B, N, C]`` token layout. + per_token_timesteps: Optional ``[B, N]`` timestep table. Tokens + with timestep ``0`` stay at sigma zero, which matches the + first-frame-pinning branch used by SANA-WM. + schedule_timesteps: Optional full scheduler timestep table used + to find the next lower sigma for each per-token timestep. + + Returns: + Updated sample in ``[B, N, C]`` layout. + """ + num_train_timesteps = float(self.config.num_train_timesteps) + sample = sample.to(torch.float32) + if per_token_timesteps is None: + sigma = timestep.to(device=sample.device, dtype=sample.dtype) + sigma_next = next_timestep.to(device=sample.device, dtype=sample.dtype) + dt = (sigma_next - sigma) / num_train_timesteps + return (sample + dt * model_output).to(model_output.dtype) + + per_token_sigmas = ( + per_token_timesteps.to(device=sample.device, dtype=sample.dtype) + / num_train_timesteps + ) + if schedule_timesteps is not None: + sigmas = ( + schedule_timesteps.to(device=sample.device, dtype=sample.dtype) + / num_train_timesteps + ) + sigmas = sigmas[:, None, None] + lower_mask = sigmas < per_token_sigmas[None] - 1e-6 + lower_sigmas = (lower_mask * sigmas).max(dim=0).values + else: + next_sigma_scalar = ( + next_timestep.to(device=sample.device, dtype=sample.dtype) + / num_train_timesteps + ) + lower_sigmas = torch.where( + per_token_sigmas > next_sigma_scalar + 1e-6, + next_sigma_scalar.expand_as(per_token_sigmas), + torch.zeros_like(per_token_sigmas), + ) + dt = per_token_sigmas - lower_sigmas + return sample + dt.unsqueeze(-1) * model_output + + def sample( + self, + initial_noise: Tensor, + predict_flow: FlowPredictor, + rng: torch.Generator | None = None, + context: Any = None, + ) -> Tensor: + """Run the generic scalar-timestep Euler loop.""" + del rng + if isinstance(context, SanaWMStage1Conditioning): + return self._sample_conditioned( + initial_noise=initial_noise, + predict_flow=predict_flow, + conditioning=context, + ) + + timesteps = self.timesteps( + num_inference_steps=self.config.num_inference_steps, + shift=self.config.shift, + device=initial_noise.device, + ) + noisy = initial_noise + for index, timestep in enumerate(timesteps[:-1]): + flow = predict_flow(noisy, timestep.to(dtype=initial_noise.dtype)) + noisy = self.step_ltx( + model_output=flow, + timestep=timestep, + next_timestep=timesteps[index + 1], + sample=noisy, + ) + return noisy.to(initial_noise.dtype) + + def _sample_conditioned( + self, + *, + initial_noise: Tensor, + predict_flow: FlowPredictor, + conditioning: SanaWMStage1Conditioning, + ) -> Tensor: + """Run SANA-WM's first-frame-pinned LTX Euler denoising loop.""" + latents = initial_noise + timesteps = self.timesteps( + num_inference_steps=conditioning.steps, + shift=conditioning.flow_shift, + device=latents.device, + ) + condition_frame_info = dict( + cast( + dict[int, float], + cast(dict[str, object], conditioning.model_kwargs["data_info"]).get( + "condition_frame_info", + {}, + ), + ) + ) + condition_mask = torch.zeros_like(latents) + image_cond_noise_scale = 0.0 + for frame_idx, frame_weight in condition_frame_info.items(): + condition_mask[:, :, int(frame_idx)] = 1 + image_cond_noise_scale = max(image_cond_noise_scale, float(frame_weight)) + + init_latents = latents.clone() + generator = torch.Generator(device=latents.device).manual_seed( + conditioning.seed + ) + iterator = enumerate(timesteps[:-1]) + if os.getenv("DPM_TQDM", "False") == "True": + from tqdm import tqdm + + iterator = tqdm(list(iterator)) + + for step_index, timestep_scalar in iterator: + if image_cond_noise_scale > 0: + latents = _add_noise_to_conditioning_latents( + t=timestep_scalar / self.config.num_train_timesteps, + init_latents=init_latents, + latents=latents, + noise_scale=image_cond_noise_scale, + conditioning_mask=condition_mask, + generator=generator, + ) + + timestep = timestep_scalar.expand(condition_mask.shape).float() + timestep = torch.min( + timestep, + (1 - condition_mask) * float(self.config.num_train_timesteps), + ) + noise_pred = predict_flow(latents, timestep[:, :1, :, 0, 0]) + + latents_dtype = latents.dtype + latents_shape = latents.shape + batch_size, channels, _frames, _height, _width = latents_shape + denoised_latents = self.step_ltx( + model_output=-noise_pred.reshape( + batch_size, + channels, + -1, + ).transpose(1, 2), + timestep=timestep_scalar, + next_timestep=timesteps[step_index + 1], + sample=latents.reshape(batch_size, channels, -1).transpose(1, 2), + per_token_timesteps=timestep.reshape(batch_size, channels, -1)[:, 0], + schedule_timesteps=timesteps, + ) + denoised_latents = denoised_latents.transpose(1, 2).reshape(latents_shape) + tokens_to_denoise_mask = ( + timestep_scalar / self.config.num_train_timesteps - 1e-6 + ) < (1.0 - condition_mask) + latents = torch.where(tokens_to_denoise_mask, denoised_latents, latents) + if latents.dtype != latents_dtype: + latents = latents.to(latents_dtype) + + return latents.detach() + + def add_noise( + self, + clean_input: Tensor, + timestep: Tensor, + rng: torch.Generator | None = None, + ) -> Tensor: + """Apply forward flow-match corruption at ``timestep``.""" + sigma = timestep.to(device=clean_input.device, dtype=clean_input.dtype) + sigma = sigma / float(self.config.num_train_timesteps) + noise = torch.randn( + clean_input.shape, + generator=rng, + device=clean_input.device, + dtype=clean_input.dtype, + ) + return ((1.0 - sigma) * clean_input + sigma * noise).to(clean_input.dtype) + + +def _add_noise_to_conditioning_latents( + *, + t: Tensor, + init_latents: Tensor, + latents: Tensor, + noise_scale: float, + conditioning_mask: Tensor, + generator: torch.Generator, + eps: float = 1e-6, +) -> Tensor: + noise = torch.randn( + latents.shape, + generator=generator, + device=latents.device, + dtype=latents.dtype, + ) + need_to_noise = conditioning_mask > (1.0 - eps) + noised_latents = init_latents + noise_scale * noise * (t**2) + return torch.where(need_to_noise, noised_latents, latents) + + +__all__ = [ + "SanaWMLTXEulerScheduler", + "SanaWMLTXEulerSchedulerConfig", +] diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index 87cbfe21f..e6615e9a9 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -629,10 +629,8 @@ def forward( x_norm = self.norm1(x).reshape(batch, frames, -1, channels) attn_in = _modulate(x_norm, shift_msa, scale_msa).reshape(batch, tokens, channels) - del x_norm, shift_msa, scale_msa attn_out = self.attn(attn_in, **kwargs).reshape(batch, frames, -1, channels) x = x + (gate_msa * attn_out).reshape(batch, tokens, channels) - del attn_in, attn_out, gate_msa if plucker_emb is not None: x = x + self.plucker_proj(plucker_emb) @@ -641,13 +639,9 @@ def forward( x_norm = self.norm2(x).reshape(batch, frames, -1, channels) mlp_in = _modulate(x_norm, shift_mlp, scale_mlp).reshape(batch, tokens, channels) - del x_norm, shift_mlp, scale_mlp mlp_out = self.mlp(mlp_in, frames=frames, height=height, width=width) - del mlp_in mlp_out = mlp_out.reshape(batch, frames, -1, channels) - out = x + (gate_mlp * mlp_out).reshape(batch, tokens, channels) - del gate_mlp, mlp_out - return out + return x + (gate_mlp * mlp_out).reshape(batch, tokens, channels) class SanaWMStage1Model(nn.Module): diff --git a/integrations/sana/sana_wm/transformer.py b/integrations/sana/sana_wm/transformer.py index 56b8482fc..335db4e9c 100644 --- a/integrations/sana/sana_wm/transformer.py +++ b/integrations/sana/sana_wm/transformer.py @@ -18,14 +18,10 @@ from __future__ import annotations import gc -import os -import time -from collections.abc import Mapping from dataclasses import dataclass, field from types import SimpleNamespace from typing import Any, Literal, cast -import numpy as np import torch import torch.nn as nn import yaml @@ -43,8 +39,6 @@ DEFAULT_VIDEO_WIDTH, SANA_WM_CONFIG_PATH, SANA_WM_MODEL_PATH, - SANA_WM_REFINER_GEMMA_ROOT, - SANA_WM_REFINER_ROOT, ) from sana_wm.quant import ( TorchScaledMMFP4Recipe, @@ -121,66 +115,12 @@ class SanaWMTransformerConfig(TransformerConfig): quant_backend: QuantBackend = "auto" """Low-precision linear replacement backend for FP8/FP4 modes.""" - refiner_root: str = SANA_WM_REFINER_ROOT - """LTX-2 refiner root path or ``hf://`` URI.""" - - refiner_gemma_root: str = SANA_WM_REFINER_GEMMA_ROOT - """Gemma text-encoder root for the refiner.""" - - refiner_precision: Precision = "bf16" - """Refiner precision requested by the runner.""" - - offload_vae: bool = False - """Move the VAE to CPU between encode/decode phases.""" - - offload_text_encoder: bool = False - """Move the Stage-1 text encoder to CPU between prompt encodes.""" - - offload_refiner: bool = False - """Release the LTX-2 refiner after each refinement.""" - height: int = DEFAULT_VIDEO_HEIGHT """Pixel height used by the public SANA-WM bidirectional release.""" width: int = DEFAULT_VIDEO_WIDTH """Pixel width used by the public SANA-WM bidirectional release.""" - vae_tile_sample_min_width: int = 256 - """Pixel width tile size used for LTX-2 VAE decode.""" - - vae_tile_sample_stride_width: int = 224 - """Pixel width stride used for LTX-2 VAE decode.""" - - vae_tile_sample_min_height: int = 256 - """Pixel height tile size used for LTX-2 VAE decode.""" - - vae_tile_sample_stride_height: int = 192 - """Pixel height stride used for LTX-2 VAE decode.""" - - vae_tile_sample_min_num_frames: int = 24 - """Pixel-frame temporal tile size used for LTX-2 VAE decode.""" - - vae_tile_sample_stride_num_frames: int = 8 - """Pixel-frame temporal tile stride used for LTX-2 VAE decode.""" - - vae_oom_retry_tile_sample_min_width: int = 128 - """Smaller pixel width tile size for one VAE decode OOM retry.""" - - vae_oom_retry_tile_sample_stride_width: int = 64 - """Smaller pixel width tile stride for one VAE decode OOM retry.""" - - vae_oom_retry_tile_sample_min_height: int = 128 - """Smaller pixel height tile size for one VAE decode OOM retry.""" - - vae_oom_retry_tile_sample_stride_height: int = 64 - """Smaller pixel height tile stride for one VAE decode OOM retry.""" - - vae_oom_retry_tile_sample_min_num_frames: int = 16 - """Smaller temporal tile size for one VAE decode OOM retry.""" - - vae_oom_retry_tile_sample_stride_num_frames: int = 8 - """Smaller temporal tile stride for one VAE decode OOM retry.""" - class SanaWMTransformer(Transformer[SanaWMTransformerCache]): """FlashDreams adapter for the SANA-WM Stage-1 model call.""" @@ -193,13 +133,7 @@ def __init__(self, config: SanaWMTransformerConfig) -> None: self._model_path: str | None = None self.weight_dtype: torch.dtype | None = None self._model_built = False - self._vae_built = False - self._text_encoder_built = False - self._refiner_built = False self._stage1_quantized = False - self._streaming_prompt_cache: dict[ - tuple[object, ...], tuple[Tensor, Tensor, Tensor, Tensor] - ] = {} @property def latent_shape(self) -> tuple[int, ...]: @@ -234,15 +168,24 @@ def predict_flow( model_kwargs: dict[str, object] | None = None, ) -> Tensor: """Execute one SANA-WM DiT flow prediction.""" + if isinstance(input, SanaWMStage1Conditioning): + cache.conditioning = input + return self._predict_conditioned( + noisy_latent=noisy_latent, + timestep=timestep, + conditioning=input, + ) + conditioning = _require_conditioning(cache) - prompt_embeds = cast(Tensor, input) if input is not None else conditioning.condition + prompt_embeds = ( + cast(Tensor, input) if input is not None else conditioning.condition + ) kwargs = conditioning.model_kwargs if model_kwargs is None else model_kwargs - self._ensure_model() - return self.model( + return self._predict_with_prompt( noisy_latent, timestep, prompt_embeds, - **kwargs, + kwargs, ) def finalize_kv_cache( @@ -263,100 +206,33 @@ def unpatchify_and_maybe_gather_cp(self, x: Tensor) -> Tensor: """SANA-WM Stage-1 latents are already in model layout.""" return x - @torch.inference_mode() - def prepare_conditioning( + def initial_noise( self, *, - image: Any, - prompt: str, - camera: Mapping[str, Tensor], - num_frames: int, - fps: int, - steps: int, - cfg_scale: float, - flow_shift: float | None, - seed: int, - negative_prompt: str, - ) -> SanaWMStage1Conditioning: - """Encode first-frame, text, and camera tensors for one rollout.""" - del fps - cfg = self._ensure_runtime_config() - weight_dtype = self._ensure_weight_dtype() - self._ensure_vae() - if self.config.offload_vae: - self.vae.to(self.device) - - from torchvision import transforms as T - - img = (T.ToTensor()(image) * 2.0 - 1.0).unsqueeze(0).unsqueeze(2) - first_latent = self._vae_encode( - img.to(self.device, dtype=self.vae_dtype), - ).to(weight_dtype) - if self.config.offload_vae: - self.vae.to("cpu") - torch.cuda.empty_cache() - - cond, cond_mask, neg, neg_mask = self._encode_prompts( - prompt, - negative_prompt, - ) - cond, cond_mask, neg, neg_mask = self._pad_text_for_quant( - cond, - cond_mask, - neg, - neg_mask, - ) - - raymap = camera["raymap"].unsqueeze(0).to(self.device, dtype=weight_dtype) - chunk_plucker = camera["chunk_plucker"].unsqueeze(0).to( - self.device, - dtype=weight_dtype, - ) - model_kwargs_extra: dict[str, object] = {} - if cfg_scale > 1.0: - model_kwargs_extra["negative_mask"] = neg_mask - uncondition = neg + latent_shape: tuple[int, ...], + rng: torch.Generator | None, + cache: SanaWMTransformerCache, + input: Any = None, + ) -> Tensor: + """Draw SANA-WM's first-frame-pinned Stage-1 initial latent.""" + del latent_shape, rng + if isinstance(input, SanaWMStage1Conditioning): + cache.conditioning = input + conditioning = input else: - uncondition = None - - latent_t = (num_frames - 1) // int(cfg.vae.vae_stride[0]) + 1 - latent_h = self.config.height // int(cfg.vae.vae_stride[-1]) - latent_w = self.config.width // int(cfg.vae.vae_stride[-1]) - chunk_index = self._chunk_index(latent_t) - model_kwargs: dict[str, object] = { - "data_info": { - "img_hw": torch.tensor( - [[self.config.height, self.config.width]], - dtype=torch.float, - device=self.device, - ), - "condition_frame_info": {0: 0.0}, - }, - "mask": cond_mask, - "camera_conditions": raymap, - "chunk_plucker": chunk_plucker, - **model_kwargs_extra, - } - if chunk_index is not None: - model_kwargs["chunk_index"] = chunk_index - - return SanaWMStage1Conditioning( - condition=cond, - uncondition=uncondition, - model_kwargs=model_kwargs, - first_latent=first_latent, - latent_shape=( - 1, - int(first_latent.shape[1]), - latent_t, - latent_h, - latent_w, - ), - cfg_scale=float(cfg_scale), - flow_shift=self._resolve_flow_shift(flow_shift), - steps=int(steps), - seed=int(seed), - ) + conditioning = _require_conditioning(cache) + return self.initial_latents(conditioning) + + def postprocess_clean_latent( + self, + clean_latent: Tensor, + cache: SanaWMTransformerCache, + input: Any = None, + ) -> Tensor: + """Release Stage-1 runtime before the pipeline enters decode/refine.""" + del input + self.release_stage1_runtime(cache) + return clean_latent def initial_latents(self, conditioning: SanaWMStage1Conditioning) -> Tensor: """Draw Stage-1 initial noise and pin the encoded first frame.""" @@ -370,84 +246,6 @@ def initial_latents(self, conditioning: SanaWMStage1Conditioning) -> Tensor: latents[:, :, :1] = conditioning.first_latent return latents - @torch.inference_mode() - def decode_latents(self, latents: Tensor) -> np.ndarray: - """Decode SANA-WM VAE latents to ``uint8`` HWC video.""" - self._ensure_vae() - if self.config.offload_vae: - self.vae.to(self.device) - samples = latents.to(device=self.device, dtype=self.vae_dtype) - if torch.cuda.is_available(): - torch.cuda.synchronize() - t0 = time.perf_counter() - retry_decode = False - try: - decoded = self._vae_decode(samples) - except torch.OutOfMemoryError: - retry_decode = True - - if retry_decode: - if torch.cuda.is_available(): - torch.cuda.synchronize() - torch.cuda.empty_cache() - gc.collect() - logger.warning( - "[sana-vae] decode OOM; retrying with smaller tiles " - "width={} stride_width={} height={} stride_height={} " - "frames={} stride_frames={}", - self.config.vae_oom_retry_tile_sample_min_width, - self.config.vae_oom_retry_tile_sample_stride_width, - self.config.vae_oom_retry_tile_sample_min_height, - self.config.vae_oom_retry_tile_sample_stride_height, - self.config.vae_oom_retry_tile_sample_min_num_frames, - self.config.vae_oom_retry_tile_sample_stride_num_frames, - ) - self._configure_vae_tiling( - tile_sample_min_width=( - self.config.vae_oom_retry_tile_sample_min_width - ), - tile_sample_stride_width=( - self.config.vae_oom_retry_tile_sample_stride_width - ), - tile_sample_min_height=( - self.config.vae_oom_retry_tile_sample_min_height - ), - tile_sample_stride_height=( - self.config.vae_oom_retry_tile_sample_stride_height - ), - tile_sample_min_num_frames=( - self.config.vae_oom_retry_tile_sample_min_num_frames - ), - tile_sample_stride_num_frames=( - self.config.vae_oom_retry_tile_sample_stride_num_frames - ), - ) - if torch.cuda.is_available(): - torch.cuda.empty_cache() - decoded = self._vae_decode(samples) - if torch.cuda.is_available(): - torch.cuda.synchronize() - logger.info( - "[timing] vae decode: {:.3f}s (latent T={} -> pixels {})", - time.perf_counter() - t0, - latents.shape[2], - tuple(decoded.shape) if isinstance(decoded, Tensor) else "list", - ) - if isinstance(decoded, list): - decoded = torch.stack(decoded, dim=0) - video = ( - torch.clamp(127.5 * decoded + 127.5, 0, 255) - .permute(0, 2, 3, 4, 1) - .to("cpu", dtype=torch.uint8) - .numpy()[0] - ) - if self.config.offload_vae: - self.vae.to("cpu") - del samples, decoded - if torch.cuda.is_available(): - torch.cuda.empty_cache() - return video - def release_stage1_runtime( self, cache: SanaWMTransformerCache | None = None, @@ -462,9 +260,8 @@ def release_stage1_runtime( free_before_gib = None if cache is not None: cache.conditioning = None - self._streaming_prompt_cache.clear() - for attr in ("model", "text_encoder"): + for attr in ("model",): module = getattr(self, attr, None) if module is None: continue @@ -477,10 +274,7 @@ def release_stage1_runtime( pass setattr(self, attr, None) - if hasattr(self, "tokenizer"): - self.tokenizer = None self._model_built = False - self._text_encoder_built = False self._stage1_quantized = False if torch.cuda.is_available(): torch.cuda.empty_cache() @@ -494,51 +288,55 @@ def release_stage1_runtime( free_after / (1024**3), ) else: - logger.info("[stage1] released Stage-1 runtime before decode/refine") + logger.info( + "[stage1] released Stage-1 runtime before decode/refine" + ) except RuntimeError: logger.info("[stage1] released Stage-1 runtime before decode/refine") gc.collect() - def release_refiner_runtime(self) -> None: - """Release refiner tensors before VAE decode.""" - self._release_refiner() - - def refine_latents( + def _predict_conditioned( self, *, - latents: Tensor, - prompt: str, - fps: int, - sink_size: int, - seed: int, - block_size: int | None, - kv_max_frames: int, + noisy_latent: Tensor, + timestep: Tensor, + conditioning: SanaWMStage1Conditioning, ) -> Tensor: - """Run the LTX-2 refiner.""" - self._ensure_refiner() - sigmas = torch.tensor( - self._stage2_sigmas(), - dtype=torch.float32, - device=self.device, + if conditioning.cfg_scale <= 1.0: + return self._predict_with_prompt( + noisy_latent, + timestep, + conditioning.condition, + _condition_model_kwargs(conditioning.model_kwargs), + ) + if conditioning.uncondition is None: + raise RuntimeError("CFG was requested without negative prompt embeds.") + + noise_pred = self._predict_with_prompt( + torch.cat([noisy_latent, noisy_latent], dim=0), + torch.cat([timestep, timestep], dim=0), + torch.cat([conditioning.uncondition, conditioning.condition], dim=0), + _batched_cfg_model_kwargs(conditioning.model_kwargs), ) - logger.info( - "[refiner] {}-step Euler, start_sigma={:.4f}", - len(sigmas) - 1, - float(sigmas[0]), + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2, dim=0) + return noise_pred_uncond + conditioning.cfg_scale * ( + noise_pred_text - noise_pred_uncond ) - refined = self.refiner.refine_latents( - latents, - prompt, - fps=float(fps), - sink_size=int(sink_size), - seed=int(seed), - progress=True, - block_size=block_size, - kv_max_frames=int(kv_max_frames), + + def _predict_with_prompt( + self, + noisy_latent: Tensor, + timestep: Tensor, + prompt_embeds: Tensor, + model_kwargs: dict[str, object], + ) -> Tensor: + self._ensure_model() + return self.model( + noisy_latent, + timestep, + prompt_embeds, + **model_kwargs, ) - if self.config.offload_refiner: - self._release_refiner() - return refined def _ensure_runtime_config(self) -> Any: if self._runtime_config is not None: @@ -553,35 +351,6 @@ def _ensure_weight_dtype(self) -> torch.dtype: ) return self.weight_dtype - def _ensure_vae(self) -> None: - if self._vae_built: - return - cfg = self._ensure_runtime_config() - self.vae_dtype = _get_weight_dtype(cfg.vae.weight_dtype) - cfg.vae.vae_pretrained = resolve_hf_path(cfg.vae.vae_pretrained) - self.vae = _get_vae( - cfg.vae.vae_type, - cfg.vae.vae_pretrained, - device=self.device, - dtype=self.vae_dtype, - config=cfg.vae, - ) - if hasattr(self.vae, "enable_tiling"): - self._configure_vae_tiling() - self._vae_built = True - - def _ensure_text_encoder(self) -> None: - if self._text_encoder_built: - return - cfg = self._ensure_runtime_config() - self.tokenizer, self.text_encoder = _get_tokenizer_and_text_encoder( - name=cfg.text_encoder.text_encoder_name, - device=self.device, - ) - if self.config.offload_text_encoder: - self.text_encoder.to("cpu") - self._text_encoder_built = True - def _ensure_model(self) -> None: if self._model_built: self._prepare_stage1_quant() @@ -614,205 +383,6 @@ def _ensure_model(self) -> None: self._model_built = True self._prepare_stage1_quant() - def _ensure_refiner(self) -> None: - if self._refiner_built: - return - from sana_wm.refiner import SanaWMLTX2Refiner - - compute_dtype = ( - torch.bfloat16 - if self.config.refiner_precision in {"fp8", "fp4"} - else _get_weight_dtype(self.config.refiner_precision) - ) - quant_backend: QuantBackend = ( - "torch" if self.config.quant_backend == "auto" else self.config.quant_backend - ) - self.refiner = SanaWMLTX2Refiner( - refiner_root=resolve_hf_path(self.config.refiner_root), - gemma_root=resolve_hf_path(self.config.refiner_gemma_root), - dtype=compute_dtype, - device=self.device, - precision=self.config.refiner_precision, - quant_backend=quant_backend, - ) - self._refiner_built = True - - def _release_refiner(self) -> None: - if not self._refiner_built: - return - del self.refiner - self._refiner_built = False - if torch.cuda.is_available(): - torch.cuda.empty_cache() - gc.collect() - - def _configure_vae_tiling( - self, - *, - tile_sample_min_width: int | None = None, - tile_sample_stride_width: int | None = None, - tile_sample_min_height: int | None = None, - tile_sample_stride_height: int | None = None, - tile_sample_min_num_frames: int | None = None, - tile_sample_stride_num_frames: int | None = None, - ) -> None: - vae = self.vae - min_width = int(tile_sample_min_width or self.config.vae_tile_sample_min_width) - stride_width = int( - tile_sample_stride_width or self.config.vae_tile_sample_stride_width - ) - min_height = int( - tile_sample_min_height or self.config.vae_tile_sample_min_height - ) - stride_height = int( - tile_sample_stride_height or self.config.vae_tile_sample_stride_height - ) - spatial_ratio = int(getattr(vae, "spatial_compression_ratio", 1)) - stride_width = _avoid_degenerate_tile_tail( - sample_extent=self.config.width, - sample_tile_min=min_width, - sample_stride=stride_width, - compression_ratio=spatial_ratio, - ) - stride_height = _avoid_degenerate_tile_tail( - sample_extent=self.config.height, - sample_tile_min=min_height, - sample_stride=stride_height, - compression_ratio=spatial_ratio, - ) - min_frames = int( - tile_sample_min_num_frames - or self.config.vae_tile_sample_min_num_frames - ) - stride_frames = int( - tile_sample_stride_num_frames - or self.config.vae_tile_sample_stride_num_frames - ) - temporal_ratio = int(getattr(vae, "temporal_compression_ratio", 1)) - if temporal_ratio > 1: - min_frames = max(min_frames, temporal_ratio) - stride_frames = max(stride_frames, temporal_ratio) - kwargs = { - "tile_sample_min_height": min_height, - "tile_sample_stride_height": stride_height, - "tile_sample_min_width": min_width, - "tile_sample_stride_width": stride_width, - "tile_sample_min_num_frames": min_frames, - "tile_sample_stride_num_frames": stride_frames, - } - if hasattr(vae, "enable_tiling"): - try: - vae.enable_tiling(**kwargs) - except TypeError: - vae.enable_tiling() - for name, value in kwargs.items(): - if hasattr(vae, name): - setattr(vae, name, value) - if hasattr(vae, "use_framewise_encoding"): - vae.use_framewise_encoding = True - if hasattr(vae, "use_framewise_decoding"): - vae.use_framewise_decoding = True - logger.info( - "[sana-vae] tiling width={} stride_width={} height={} " - "stride_height={} frames={} stride_frames={}", - min_width, - stride_width, - min_height, - stride_height, - min_frames, - stride_frames, - ) - - def _encode_prompts( - self, - prompt: str, - negative_prompt: str, - ) -> tuple[Tensor, Tensor, Tensor, Tensor]: - cfg = self._ensure_runtime_config() - self._ensure_text_encoder() - max_length = cfg.text_encoder.model_max_length - chi_prompt = "\n".join(cfg.text_encoder.chi_prompt or []) - if chi_prompt: - prompt = chi_prompt + prompt - max_length_all = len(self.tokenizer.encode(chi_prompt)) + max_length - 2 - else: - max_length_all = max_length - - key = ( - prompt, - negative_prompt, - str(self.device), - str(self._ensure_weight_dtype()), - self.config.stage1_precision, - self.config.quant_backend, - ) - if key in self._streaming_prompt_cache: - return self._streaming_prompt_cache[key] - - move_text_encoder = self.config.offload_text_encoder or ( - next(self.text_encoder.parameters()).device != self.device - ) - if move_text_encoder: - self.text_encoder.to(self.device) - - def encode(text: str, length: int) -> tuple[Tensor, Tensor]: - tokens = self.tokenizer( - [text], - max_length=length, - padding="max_length", - truncation=True, - return_tensors="pt", - ).to(self.device) - return self.text_encoder(tokens.input_ids, tokens.attention_mask)[0], ( - tokens.attention_mask - ) - - try: - cond, cond_mask = encode(prompt, max_length_all) - select = [0] + list(range(-max_length + 1, 0)) - cond = cond[:, None][:, :, select] - cond_mask = cond_mask[:, select] - neg, neg_mask = encode(negative_prompt, max_length) - result = (cond, cond_mask, neg[:, None], neg_mask) - self._streaming_prompt_cache.clear() - self._streaming_prompt_cache[key] = result - return result - finally: - if move_text_encoder: - self.text_encoder.to("cpu") - if torch.cuda.is_available(): - torch.cuda.empty_cache() - - def _pad_text_for_quant( - self, - cond: Tensor, - cond_mask: Tensor, - neg: Tensor, - neg_mask: Tensor, - ) -> tuple[Tensor, Tensor, Tensor, Tensor]: - if self.config.stage1_precision == "bf16": - return cond, cond_mask, neg, neg_mask - multiple = int(os.environ.get("SANA_WM_STAGE1_NVFP4_TEXT_PAD_MULTIPLE", "8")) - if multiple <= 1: - return cond, cond_mask, neg, neg_mask - - def pad_pair(text: Tensor, mask: Tensor) -> tuple[Tensor, Tensor]: - pad = (-text.shape[-2]) % multiple - if pad == 0: - return text, mask - text_shape = list(text.shape) - text_shape[-2] = pad - mask_shape = list(mask.shape) - mask_shape[-1] = pad - return ( - torch.cat([text, text.new_zeros(text_shape)], dim=-2), - torch.cat([mask, mask.new_zeros(mask_shape)], dim=-1), - ) - - cond, cond_mask = pad_pair(cond, cond_mask) - neg, neg_mask = pad_pair(neg, neg_mask) - return cond, cond_mask, neg, neg_mask - def _prepare_stage1_quant(self) -> None: if self._stage1_quantized or self.config.stage1_precision == "bf16": return @@ -842,39 +412,6 @@ def _prepare_stage1_quant(self) -> None: if torch.cuda.is_available(): torch.cuda.empty_cache() - def _resolve_flow_shift(self, override: float | None) -> float: - cfg = self._ensure_runtime_config() - if override is not None: - return float(override) - if cfg.scheduler.inference_flow_shift is not None: - return float(cfg.scheduler.inference_flow_shift) - return float(cfg.scheduler.flow_shift) - - def _chunk_index(self, latent_frames: int) -> list[int] | None: - return _chunk_index_from_config( - self._ensure_runtime_config(), - num_frames=latent_frames, - ) - - def _vae_encode(self, images: Tensor) -> Tensor: - return _vae_encode_ltx2( - self._ensure_runtime_config().vae.vae_type, - self.vae, - images, - device=self.device, - ) - - def _vae_decode(self, latents: Tensor) -> Tensor: - return _vae_decode_ltx2( - self._ensure_runtime_config().vae.vae_type, - self.vae, - latents, - ) - - @staticmethod - def _stage2_sigmas() -> tuple[float, ...]: - return (0.909375, 0.725, 0.421875, 0.0) - def _require_conditioning( cache: SanaWMTransformerCache, @@ -884,6 +421,30 @@ def _require_conditioning( return cache.conditioning +def _condition_model_kwargs(model_kwargs: dict[str, object]) -> dict[str, object]: + """Return model kwargs for the positive prompt branch.""" + return { + key: value + for key, value in model_kwargs.items() + if key != "negative_mask" + } + + +def _batched_cfg_model_kwargs(model_kwargs: dict[str, object]) -> dict[str, object]: + """Return model kwargs for a single batched negative/positive CFG forward.""" + kwargs = _condition_model_kwargs(model_kwargs) + mask = kwargs.get("mask") + negative_mask = model_kwargs.get("negative_mask") + if isinstance(mask, Tensor): + mask_uncond = negative_mask if isinstance(negative_mask, Tensor) else mask + kwargs["mask"] = torch.cat([mask_uncond, mask], dim=0) + for key in ("camera_conditions", "chunk_plucker"): + value = kwargs.get(key) + if isinstance(value, Tensor): + kwargs[key] = torch.cat([value, value], dim=0) + return kwargs + + def _avoid_degenerate_tile_tail( *, sample_extent: int, @@ -955,7 +516,12 @@ def _get_tokenizer_and_text_encoder(*args: Any, **kwargs: Any) -> tuple[Any, nn. model_id = _TEXT_ENCODER_MODEL_IDS.get(str(name)) if model_id is None: raise ValueError(f"Unsupported SANA-WM text encoder: {name!r}") - from transformers import AutoModelForCausalLM, AutoTokenizer, T5EncoderModel, T5Tokenizer + from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + T5EncoderModel, + T5Tokenizer, + ) if "T5" in str(name): tokenizer = T5Tokenizer.from_pretrained(model_id) diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index 5cb0a8dfd..c626a5b26 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -19,12 +19,15 @@ import os from pathlib import Path +from types import SimpleNamespace +import numpy as np import pytest import torch +import sana_wm.conditioning as conditioning_module +import sana_wm.decoder as decoder_module import sana_wm.refiner as refiner_module -import sana_wm.transformer as transformer_module try: import tomllib @@ -44,16 +47,31 @@ from sana_wm.runner import ( SanaWMRunner, SanaWMRunnerConfig, + _pipeline_config, _precision_env_updates, _resolve_quant_backend, _temporary_environment, _validate_precision_request, ) -from sana_wm.diffusion import ( - _condition_model_kwargs, - _uncondition_model_kwargs, +from sana_wm.conditioning import ( + SanaWMCameraConditioningEncoderConfig, + SanaWMCameraRequest, + SanaWMConditioningEncoderConfig, + SanaWMFirstFrameEncoderConfig, + SanaWMTextPromptEncoderConfig, + SanaWMTextPromptRequest, +) +from sana_wm.decoder import ( + SanaWMDecodedVideo, + SanaWMLTX2LatentRefinerConfig, + SanaWMLTX2VAEDecoderConfig, + SanaWMVideoDecoderConfig, ) from sana_wm.diffusion import SanaWMDiffusionModelConfig +from sana_wm.scheduler import ( + SanaWMLTXEulerScheduler, + SanaWMLTXEulerSchedulerConfig, +) from sana_wm.transformer import ( SanaWMTransformerCache, SanaWMTransformerConfig, @@ -76,6 +94,7 @@ ) from flashdreams.infra.config import derive_config +from flashdreams.infra.diffusion.model import DiffusionModel pytestmark = pytest.mark.ci_cpu @@ -102,14 +121,164 @@ def test_runner_has_description() -> None: def test_pipeline_uses_sana_diffusion_model() -> None: - """Keep the public runner wired to the SANA-WM diffusion model.""" + """Keep the public runner wired to explicit SANA-WM boundaries.""" + pipeline = PIPELINE_SANA_WM_BIDIRECTIONAL transformer = PIPELINE_SANA_WM_BIDIRECTIONAL.diffusion_model.transformer - assert isinstance( - PIPELINE_SANA_WM_BIDIRECTIONAL.diffusion_model, - SanaWMDiffusionModelConfig, - ) + assert isinstance(pipeline.encoder, SanaWMConditioningEncoderConfig) + assert isinstance(pipeline.decoder, SanaWMVideoDecoderConfig) + assert isinstance(pipeline.diffusion_model, SanaWMDiffusionModelConfig) + assert pipeline.diffusion_model._target is DiffusionModel assert isinstance(transformer, SanaWMTransformerConfig) + assert isinstance(pipeline.diffusion_model.scheduler, SanaWMLTXEulerSchedulerConfig) + assert isinstance(pipeline.decoder.refiner, SanaWMLTX2LatentRefinerConfig) + + +def test_sana_diffusion_config_instantiates_base_model() -> None: + """Use FlashDreams' shared diffusion model rather than a Sana-only runner.""" + model = PIPELINE_SANA_WM_BIDIRECTIONAL.diffusion_model.setup() + + assert type(model) is DiffusionModel + assert isinstance(model.transformer.config, SanaWMTransformerConfig) + assert isinstance(model.scheduler, SanaWMLTXEulerScheduler) + + +def test_runner_pipeline_config_routes_runtime_fields_to_components() -> None: + """Apply CLI overrides to the component that owns each runtime field.""" + cfg = derive_config( + RUNNER_SANA_WM_BIDIRECTIONAL, + config_path="local_config.yaml", + model_path="local_model.safetensors", + stage1_precision="fp8", + no_refiner=True, + offload_vae=True, + offload_text_encoder=True, + ) + + pipeline = _pipeline_config(cfg, quant_backend="torch-fp8") + + assert isinstance(pipeline.encoder, SanaWMConditioningEncoderConfig) + assert isinstance(pipeline.decoder, SanaWMVideoDecoderConfig) + assert pipeline.encoder.config_path == "local_config.yaml" + assert pipeline.encoder.text_encoder.config_path == "local_config.yaml" + assert pipeline.encoder.text_encoder.stage1_precision == "fp8" + assert pipeline.encoder.text_encoder.quant_backend == "torch-fp8" + assert pipeline.encoder.text_encoder.offload_text_encoder is True + assert pipeline.encoder.first_frame_encoder.offload_vae is True + assert pipeline.decoder.vae_decoder.config_path == "local_config.yaml" + assert pipeline.decoder.vae_decoder.offload_vae is True + assert pipeline.decoder.refiner is None + assert pipeline.diffusion_model.transformer.checkpoint_path == ( + "local_model.safetensors" + ) + + +def test_sana_ltx_scheduler_step_pins_zero_timestep_tokens() -> None: + """Keep first-frame tokens fixed in the per-token LTX Euler step.""" + scheduler = SanaWMLTXEulerSchedulerConfig(num_inference_steps=4).setup() + + timesteps = scheduler.timesteps( + num_inference_steps=4, + shift=5.0, + device=torch.device("cpu"), + ) + sample = torch.ones((1, 2, 1)) + model_output = torch.ones_like(sample) + stepped = scheduler.step_ltx( + model_output=model_output, + timestep=torch.tensor(1000.0), + next_timestep=torch.tensor(500.0), + sample=sample, + per_token_timesteps=torch.tensor([[1000.0, 0.0]]), + schedule_timesteps=torch.tensor([1000.0, 500.0, 0.0]), + ) + + assert isinstance(scheduler, SanaWMLTXEulerScheduler) + assert timesteps.shape == (5,) + assert float(timesteps[-1]) == 0.0 + torch.testing.assert_close( + stepped, + torch.tensor([[[1.5], [1.0]]]), + ) + + +def test_sana_ltx_scheduler_matches_diffusers_per_token_step() -> None: + """Match diffusers FlowMatch Euler for SANA-WM's per-token branch.""" + from diffusers import FlowMatchEulerDiscreteScheduler + + ours = SanaWMLTXEulerSchedulerConfig(num_inference_steps=4).setup() + schedule_timesteps = ours.timesteps( + num_inference_steps=4, + shift=5.0, + device=torch.device("cpu"), + ) + upstream = FlowMatchEulerDiscreteScheduler(shift=5.0) + upstream.set_timesteps(4, device=torch.device("cpu")) + torch.testing.assert_close(schedule_timesteps[:-1], upstream.timesteps) + torch.testing.assert_close(schedule_timesteps / 1000.0, upstream.sigmas) + + sample = torch.tensor([[[1.0, -0.5], [0.25, 0.5], [-1.0, 0.75]]]) + model_output = torch.tensor([[[0.2, 0.4], [-0.3, 0.1], [0.5, -0.2]]]) + per_token_timesteps = torch.stack( + [ + torch.tensor( + [ + float(upstream.timesteps[0]), + 0.0, + float(upstream.timesteps[0]), + ] + ) + ] + ) + + expected = upstream.step( + model_output, + upstream.timesteps[0], + sample, + per_token_timesteps=per_token_timesteps, + return_dict=False, + )[0] + actual = ours.step_ltx( + model_output=model_output, + timestep=schedule_timesteps[0], + next_timestep=schedule_timesteps[1], + sample=sample, + per_token_timesteps=per_token_timesteps, + schedule_timesteps=schedule_timesteps, + ) + + torch.testing.assert_close(actual, expected) + + +def test_sana_scheduler_context_path_keeps_conditioned_frame_fixed() -> None: + """Let the scheduler own SANA-WM's per-token first-frame constraint.""" + scheduler = SanaWMLTXEulerSchedulerConfig(num_inference_steps=1).setup() + initial_noise = torch.zeros((1, 1, 2, 1, 1), dtype=torch.float32) + initial_noise[:, :, 0] = 5.0 + conditioning = SanaWMStage1Conditioning( + condition=torch.ones((1, 1, 1, 1)), + uncondition=None, + model_kwargs={"data_info": {"condition_frame_info": {0: 0.0}}}, + first_latent=torch.empty((1, 1, 1, 1, 1)), + latent_shape=tuple(initial_noise.shape), + cfg_scale=1.0, + flow_shift=5.0, + steps=1, + seed=0, + ) + + def predict_flow(noisy_latent: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor: + assert timestep.shape == (1, 1, 2) + return -torch.ones_like(noisy_latent) + + sampled = scheduler.sample( + initial_noise=initial_noise, + predict_flow=predict_flow, + context=conditioning, + ) + + torch.testing.assert_close(sampled[:, :, 0], initial_noise[:, :, 0]) + assert torch.all(sampled[:, :, 1] > initial_noise[:, :, 1]) def test_transformer_contract_shape_and_conditioning_guard() -> None: @@ -128,6 +297,127 @@ def test_transformer_contract_shape_and_conditioning_guard() -> None: ) +def test_transformer_initial_noise_uses_conditioning_payload() -> None: + """Generate first-frame-pinned noise through the shared transformer hook.""" + transformer = SanaWMTransformerConfig().setup() + transformer.weight_dtype = torch.float32 + first_latent = torch.full((1, 1, 1, 1, 1), 7.0) + conditioning = SanaWMStage1Conditioning( + condition=torch.empty((1, 1, 1, 1)), + uncondition=None, + model_kwargs={}, + first_latent=first_latent, + latent_shape=(1, 1, 2, 1, 1), + cfg_scale=1.0, + flow_shift=1.0, + steps=1, + seed=123, + ) + cache = transformer.initialize_autoregressive_cache() + + noise = transformer.initial_noise( + latent_shape=(1, 1, 2, 1, 1), + rng=None, + cache=cache, + input=conditioning, + ) + + assert cache.conditioning is conditioning + assert noise.shape == (1, 1, 2, 1, 1) + torch.testing.assert_close(noise[:, :, :1], first_latent) + + +def test_transformer_predict_flow_applies_cfg_from_conditioning_input() -> None: + """Keep CFG inside the transformer boundary used by base diffusion.""" + + class DummyModel(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.calls: list[dict[str, torch.Tensor]] = [] + + def forward( + self, + noisy_latent: torch.Tensor, + timestep: torch.Tensor, + prompt_embeds: torch.Tensor, + *, + mask: torch.Tensor, + camera_conditions: torch.Tensor, + chunk_plucker: torch.Tensor, + **_kwargs: object, + ) -> torch.Tensor: + assert "negative_mask" not in _kwargs + self.calls.append( + { + "noisy_latent": noisy_latent, + "timestep": timestep, + "prompt_embeds": prompt_embeds, + "mask": mask, + "camera_conditions": camera_conditions, + "chunk_plucker": chunk_plucker, + "data_info": _kwargs["data_info"], + } + ) + branch_values = prompt_embeds.flatten(1).mean(dim=1).reshape( + -1, + 1, + 1, + 1, + 1, + ) + return torch.ones_like(noisy_latent) * branch_values + + transformer = SanaWMTransformerConfig().setup() + dummy_model = DummyModel() + transformer.model = dummy_model + transformer._model_built = True + cond_mask = torch.ones((1, 1)) + neg_mask = torch.zeros((1, 1)) + camera = torch.zeros((1, 3, 1, 1, 1)) + chunk_plucker = torch.ones((1, 6, 1, 1, 1)) + conditioning = SanaWMStage1Conditioning( + condition=torch.ones((1, 1, 1, 1)), + uncondition=torch.zeros((1, 1, 1, 1)), + model_kwargs={ + "mask": cond_mask, + "negative_mask": neg_mask, + "camera_conditions": camera, + "chunk_plucker": chunk_plucker, + "data_info": {"condition_frame_info": {0: 0.0}}, + }, + first_latent=torch.empty((1, 1, 1, 1, 1)), + latent_shape=(1, 1, 1, 1, 1), + cfg_scale=2.0, + flow_shift=1.0, + steps=1, + seed=0, + ) + + out = transformer.predict_flow( + noisy_latent=torch.zeros((1, 1, 1, 1, 1)), + timestep=torch.zeros((1, 1, 1)), + cache=SanaWMTransformerCache(), + input=conditioning, + ) + + torch.testing.assert_close(out, torch.full((1, 1, 1, 1, 1), 2.0)) + assert len(dummy_model.calls) == 1 + call = dummy_model.calls[0] + assert call["noisy_latent"].shape == (2, 1, 1, 1, 1) + assert call["timestep"].shape == (2, 1, 1) + assert call["prompt_embeds"].shape == (2, 1, 1, 1) + torch.testing.assert_close(call["mask"], torch.cat([neg_mask, cond_mask], dim=0)) + torch.testing.assert_close( + call["camera_conditions"], + torch.cat([camera, camera], dim=0), + ) + torch.testing.assert_close( + call["chunk_plucker"], + torch.cat([chunk_plucker, chunk_plucker], dim=0), + ) + assert call["data_info"] == {"condition_frame_info": {0: 0.0}} + + def test_inference_config_loads_yaml( tmp_path: Path, ) -> None: @@ -166,6 +456,170 @@ def test_inference_config_loads_yaml( assert cfg.work_dir == "" +def test_text_prompt_encoder_outputs_padded_prompt_schema( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Encode prompts behind the explicit text component boundary.""" + + class DummyTokens: + def __init__(self, max_length: int) -> None: + self.input_ids = torch.arange(max_length).reshape(1, max_length) + self.attention_mask = torch.ones((1, max_length), dtype=torch.long) + + def to(self, _device: torch.device) -> "DummyTokens": + return self + + class DummyTokenizer: + def encode(self, _text: str) -> list[int]: + return [1, 2] + + def __call__( + self, + *_args: object, + max_length: int, + **_kwargs: object, + ) -> DummyTokens: + return DummyTokens(max_length) + + class DummyTextEncoder(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.weight = torch.nn.Parameter(torch.empty(0)) + + def forward( + self, + input_ids: torch.Tensor, + attention_mask: torch.Tensor, + ) -> tuple[torch.Tensor]: + del attention_mask + hidden = torch.ones((1, input_ids.shape[1], 12), dtype=torch.float32) + return (hidden,) + + monkeypatch.setenv("SANA_WM_STAGE1_NVFP4_TEXT_PAD_MULTIPLE", "8") + monkeypatch.setattr( + conditioning_module, + "_load_inference_config", + lambda _path: SimpleNamespace( + model=SimpleNamespace(mixed_precision="bf16"), + text_encoder=SimpleNamespace( + text_encoder_name="T5-small", + model_max_length=5, + chi_prompt=["prefix"], + ), + ), + ) + monkeypatch.setattr( + conditioning_module, + "_get_tokenizer_and_text_encoder", + lambda **_kwargs: (DummyTokenizer(), DummyTextEncoder()), + ) + encoder = SanaWMTextPromptEncoderConfig( + config_path="dummy.yaml", + stage1_precision="fp4", + ).setup() + + encoded = encoder( + SanaWMTextPromptRequest( + prompt="drive forward", + negative_prompt="low quality", + ) + ) + + assert encoded.condition.shape == (1, 1, 8, 12) + assert encoded.condition_mask.shape == (1, 8) + assert encoded.negative.shape == (1, 1, 8, 12) + assert encoded.negative_mask.shape == (1, 8) + + +def test_first_frame_encoder_outputs_latent_shape( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Encode the first frame behind the named I2V component.""" + from PIL import Image + + class DummyLatentDist: + def mode(self) -> torch.Tensor: + return torch.ones((1, 4, 1, 2, 3), dtype=torch.float32) + + class DummyPosterior: + latent_dist = DummyLatentDist() + + class DummyVAE(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.weight = torch.nn.Parameter(torch.empty(0)) + self.latents_mean = torch.zeros(4) + self.latents_std = torch.ones(4) + self.config = SimpleNamespace(scaling_factor=1.0) + + def encode(self, _images: torch.Tensor) -> DummyPosterior: + return DummyPosterior() + + monkeypatch.setattr( + conditioning_module, + "_load_inference_config", + lambda _path: SimpleNamespace( + model=SimpleNamespace(mixed_precision="bf16"), + vae=SimpleNamespace( + weight_dtype="float32", + vae_type="LTX2VAE_diffusers", + vae_pretrained="local", + ), + ), + ) + monkeypatch.setattr( + conditioning_module, + "_get_vae", + lambda *_args, **_kwargs: DummyVAE(), + ) + encoder = SanaWMFirstFrameEncoderConfig(config_path="dummy.yaml").setup() + + latent = encoder(Image.new("RGB", (4, 4))) + + assert latent.shape == (1, 4, 1, 2, 3) + assert latent.dtype == torch.bfloat16 + + +def test_camera_conditioning_encoder_outputs_sana_schema() -> None: + """Build raymap/chunk-Plucker tensors behind the camera component.""" + poses = np.broadcast_to(np.eye(4, dtype=np.float32), (17, 4, 4)).copy() + intrinsics = np.broadcast_to( + np.array([900.0, 900.0, 640.0, 352.0], dtype=np.float32), + (17, 4), + ).copy() + encoder = SanaWMCameraConditioningEncoderConfig().setup() + + camera = encoder( + SanaWMCameraRequest( + poses_c2w=poses, + intrinsics_vec4=intrinsics, + ) + ) + + assert camera["raymap"].shape == (3, 20) + assert camera["chunk_plucker"].shape == (48, 3, 22, 40) + + +def test_video_decoder_returns_structured_video(monkeypatch: pytest.MonkeyPatch) -> None: + """Decode Stage-1 latents through the explicit decoder component.""" + decoder = SanaWMVideoDecoderConfig(refiner=None).setup() + monkeypatch.setattr( + decoder.vae_decoder, + "decode_latents", + lambda _latents: np.zeros((1, 2, 2, 3), dtype=np.uint8), + ) + + decoded = decoder( + torch.zeros((1, 4, 1, 1, 1)), + autoregressive_index=0, + cache=None, + ) + + assert isinstance(decoded, SanaWMDecodedVideo) + assert decoded.video_hwc.shape == (1, 2, 2, 3) + assert decoded.stage1_video_hwc is None + + def test_stage1_model_matches_checkpoint_schema() -> None: """Pin the Stage-1 module to the public checkpoint schema.""" state = SanaWMStage1Model().state_dict() @@ -266,17 +720,8 @@ def test_transformer_releases_stage1_runtime() -> None: """Free Stage-1-only modules and conditioning before decode/refine.""" transformer = SanaWMTransformerConfig().setup() transformer.model = torch.nn.Linear(1, 1) - transformer.text_encoder = torch.nn.Linear(1, 1) - transformer.tokenizer = object() transformer._model_built = True - transformer._text_encoder_built = True transformer._stage1_quantized = True - transformer._streaming_prompt_cache[("prompt",)] = ( - torch.empty(1), - torch.empty(1), - torch.empty(1), - torch.empty(1), - ) cache = SanaWMTransformerCache( conditioning=SanaWMStage1Conditioning( condition=torch.empty(1), @@ -295,11 +740,7 @@ def test_transformer_releases_stage1_runtime() -> None: assert cache.conditioning is None assert transformer.model is None - assert transformer.text_encoder is None - assert transformer.tokenizer is None - assert transformer._streaming_prompt_cache == {} assert transformer._model_built is False - assert transformer._text_encoder_built is False assert transformer._stage1_quantized is False @@ -322,12 +763,12 @@ def __init__(self) -> None: def enable_tiling(self, **kwargs: int) -> None: self.calls.append(kwargs) - transformer = SanaWMTransformerConfig().setup() - transformer.vae = DummyVAE() + decoder = SanaWMLTX2VAEDecoderConfig().setup() + decoder.vae = DummyVAE() - transformer._configure_vae_tiling() + decoder._configure_vae_tiling() - assert transformer.vae.calls == [ + assert decoder.vae.calls == [ { "tile_sample_min_height": 256, "tile_sample_stride_height": 192, @@ -337,14 +778,14 @@ def enable_tiling(self, **kwargs: int) -> None: "tile_sample_stride_num_frames": 8, } ] - assert transformer.vae.tile_sample_min_height == 256 - assert transformer.vae.tile_sample_stride_height == 192 - assert transformer.vae.tile_sample_min_width == 256 - assert transformer.vae.tile_sample_stride_width == 224 - assert transformer.vae.tile_sample_min_num_frames == 24 - assert transformer.vae.tile_sample_stride_num_frames == 8 - assert transformer.vae.use_framewise_encoding is True - assert transformer.vae.use_framewise_decoding is True + assert decoder.vae.tile_sample_min_height == 256 + assert decoder.vae.tile_sample_stride_height == 192 + assert decoder.vae.tile_sample_min_width == 256 + assert decoder.vae.tile_sample_stride_width == 224 + assert decoder.vae.tile_sample_min_num_frames == 24 + assert decoder.vae.tile_sample_stride_num_frames == 8 + assert decoder.vae.use_framewise_encoding is True + assert decoder.vae.use_framewise_decoding is True def test_decode_retries_vae_oom_with_smaller_tiles( @@ -372,10 +813,10 @@ def enable_tiling(self, **kwargs: int) -> None: for name, value in kwargs.items(): setattr(self, name, value) - transformer = SanaWMTransformerConfig().setup() - transformer.vae = DummyVAE() - transformer.vae_dtype = torch.float32 - monkeypatch.setattr(transformer, "_ensure_vae", lambda: None) + decoder = SanaWMLTX2VAEDecoderConfig().setup() + decoder.vae = DummyVAE() + decoder.vae_dtype = torch.float32 + monkeypatch.setattr(decoder, "_ensure_vae", lambda: None) calls = 0 inference_modes: list[bool] = [] @@ -387,42 +828,19 @@ def decode_once(_samples: torch.Tensor) -> torch.Tensor: raise torch.OutOfMemoryError("test OOM") return torch.zeros((1, 3, 1, 2, 2), dtype=torch.float32) - monkeypatch.setattr(transformer, "_vae_decode", decode_once) + monkeypatch.setattr(decoder, "_vae_decode", decode_once) - video = transformer.decode_latents(torch.zeros((1, 4, 1, 1, 1))) + video = decoder.decode_latents(torch.zeros((1, 4, 1, 1, 1))) assert calls == 2 assert inference_modes == [True, True] assert video.shape == (1, 2, 2, 3) - assert transformer.vae.tile_sample_min_height == 128 - assert transformer.vae.tile_sample_stride_height == 64 - assert transformer.vae.tile_sample_min_width == 128 - assert transformer.vae.tile_sample_stride_width == 64 - assert transformer.vae.tile_sample_min_num_frames == 16 - assert transformer.vae.tile_sample_stride_num_frames == 8 - - -def test_cfg_model_kwargs_do_not_duplicate_camera_tensors() -> None: - """Run CFG as sequential branches instead of doubling Stage-1 activations.""" - camera = torch.zeros((1, 3, 3, 2, 2)) - cond_mask = torch.ones((1, 8)) - neg_mask = torch.zeros((1, 8)) - kwargs = { - "mask": cond_mask, - "negative_mask": neg_mask, - "camera_conditions": camera, - "chunk_plucker": torch.zeros((1, 6, 3, 2, 2)), - } - - cond_kwargs = _condition_model_kwargs(kwargs) - neg_kwargs = _uncondition_model_kwargs(kwargs) - - assert cond_kwargs["mask"] is cond_mask - assert neg_kwargs["mask"] is neg_mask - assert cond_kwargs["camera_conditions"] is camera - assert neg_kwargs["camera_conditions"] is camera - assert "negative_mask" not in cond_kwargs - assert "negative_mask" not in neg_kwargs + assert decoder.vae.tile_sample_min_height == 128 + assert decoder.vae.tile_sample_stride_height == 64 + assert decoder.vae.tile_sample_min_width == 128 + assert decoder.vae.tile_sample_stride_width == 64 + assert decoder.vae.tile_sample_min_num_frames == 16 + assert decoder.vae.tile_sample_stride_num_frames == 8 def test_vae_tiling_avoids_degenerate_latent_tails() -> None: @@ -498,16 +916,16 @@ def __init__(self, **kwargs: object) -> None: monkeypatch.setattr(refiner_module, "SanaWMLTX2Refiner", DummyRefiner) monkeypatch.setattr( - transformer_module, + decoder_module, "resolve_hf_path", lambda value: f"/resolved/{value}", ) - transformer = SanaWMTransformerConfig().setup() + refiner = SanaWMLTX2LatentRefinerConfig().setup() - transformer._ensure_refiner() + refiner._ensure_refiner() - assert transformer._refiner_built is True - assert isinstance(transformer.refiner, DummyRefiner) + assert refiner._refiner_built is True + assert isinstance(refiner.refiner, DummyRefiner) assert calls["refiner_root"] == ( "/resolved/hf://Efficient-Large-Model/SANA-WM_bidirectional/refiner" ) From 80ef8155e41c1bce4b5bef092d613c3e6bc274ab Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 17 Jul 2026 13:32:13 -0700 Subject: [PATCH 14/64] Keep SANA-WM models resident across reuse without inflating peak memory Make the SANA-WM pipeline reuse-friendly: the Stage-1 DiT, LTX-2 refiner, and Gemma text encoder are no longer torn down and rebuilt on every generate(). Previously each rollout unconditionally released Stage-1 before decode, released the refiner after refine, and reloaded the ~20 GB Gemma encoder from disk per prompt, so a reused pipeline re-paid a full checkpoint reload and re-quantization (~10 s bf16, ~17 s fp8) plus an ~8-15 s Gemma reload on every call. - transformer: gate the Stage-1 teardown behind a new offload_stage1 flag (default keeps it resident); still drop per-rollout conditioning. - decoder: drop the unconditional refiner release (refine_latents already releases when offload_refiner is set). - refiner: cache the Gemma tokenizer/encoder in CPU RAM instead of reloading per call, and move it to the GPU only for the encode forward so it never inflates peak memory during denoise/decode. Fix deprecated torch_dtype and drop dead _empty_cuda_cache/gc. - runner: expose --offload-stage1 and route it to the transformer. - Add [timing] instrumentation for the Stage-1 build, refiner build, and Gemma load. Measured on Blackwell: warm-reuse pass drops from ~26 s to ~5 s (fp8); full-res fp8 demo_0 peaks at ~54/98 GB with no OOM. Co-Authored-By: Claude Opus 4.8 (1M context) --- integrations/sana/sana_wm/decoder.py | 10 ++- integrations/sana/sana_wm/refiner.py | 78 ++++++++++++++++-------- integrations/sana/sana_wm/runner.py | 6 ++ integrations/sana/sana_wm/transformer.py | 24 +++++++- integrations/sana/tests/test_smoke.py | 54 ++++++++++++++++ 5 files changed, 142 insertions(+), 30 deletions(-) diff --git a/integrations/sana/sana_wm/decoder.py b/integrations/sana/sana_wm/decoder.py index 698e1d6aa..f3367f6d7 100644 --- a/integrations/sana/sana_wm/decoder.py +++ b/integrations/sana/sana_wm/decoder.py @@ -376,6 +376,7 @@ def _ensure_refiner(self) -> None: return from sana_wm.refiner import SanaWMLTX2Refiner + t0 = time.perf_counter() compute_dtype = ( torch.bfloat16 if self.config.refiner_precision in {"fp8", "fp4"} @@ -390,6 +391,11 @@ def _ensure_refiner(self) -> None: quant_backend=self.config.quant_backend, ) self._refiner_built = True + logger.info( + "[timing] refiner build: {:.3f}s (precision={})", + time.perf_counter() - t0, + self.config.refiner_precision, + ) @dataclass(kw_only=True) @@ -449,7 +455,9 @@ def forward( block_size=cache.refiner_block_size, kv_max_frames=cache.refiner_kv_max_frames, ) - self.refiner.release_runtime() + # refine_latents() already releases the refiner when offload_refiner + # is set; keep it resident otherwise so a reused pipeline avoids a + # full refiner rebuild + re-quantization on the next call. elif cache.save_stage1: logger.info( "SANA-WM is already running without the refiner; " diff --git a/integrations/sana/sana_wm/refiner.py b/integrations/sana/sana_wm/refiner.py index c29844d93..8cfce1f96 100644 --- a/integrations/sana/sana_wm/refiner.py +++ b/integrations/sana/sana_wm/refiner.py @@ -17,7 +17,7 @@ from __future__ import annotations -import gc +import time from pathlib import Path from typing import Literal @@ -71,6 +71,7 @@ def __init__( self.quant_backend = quant_backend self.text_max_sequence_length = int(text_max_sequence_length) self._quantized = False + self._text_encoder_built = False self.transformer, self.connectors = self._load_diffusers_components() @torch.inference_mode() @@ -196,14 +197,44 @@ def _prepare_quantization(self) -> None: ) self._quantized = True - @torch.inference_mode() - def _encode_prompt(self, prompt: str) -> tuple[Tensor, Tensor]: + def _ensure_text_encoder(self) -> None: + """Load the Gemma tokenizer + encoder once and cache them in CPU RAM. + + The encoder is ~20 GB; reloading it from disk on every refine call + dominated repeated-use latency, so the built module is kept cached and + a reused refiner pays the load once. It stays on the CPU between calls + and is moved to the GPU only for the encode forward (see + :meth:`_encode_prompt`) so it never inflates peak memory during the + denoise/decode phases. When ``offload_refiner`` is set the whole + refiner (and this encoder) is released between runs by + ``release_runtime``. + """ + if self._text_encoder_built: + return from transformers import AutoTokenizer, Gemma3ForConditionalGeneration + t0 = time.perf_counter() tokenizer = AutoTokenizer.from_pretrained(self.gemma_root) tokenizer.padding_side = "left" if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token + self.tokenizer = tokenizer + # Built on CPU; moved to the GPU on demand for each encode call only. + self.text_encoder = Gemma3ForConditionalGeneration.from_pretrained( + self.gemma_root, + dtype=self.dtype, + low_cpu_mem_usage=True, + ).eval() + self._text_encoder_built = True + logger.info( + "[timing] refiner text-encoder build: {:.3f}s", + time.perf_counter() - t0, + ) + + @torch.inference_mode() + def _encode_prompt(self, prompt: str) -> tuple[Tensor, Tensor]: + self._ensure_text_encoder() + tokenizer = self.tokenizer text_inputs = tokenizer( [prompt.strip()], @@ -216,22 +247,23 @@ def _encode_prompt(self, prompt: str) -> tuple[Tensor, Tensor]: input_ids = text_inputs.input_ids.to(self.device) attention_mask = text_inputs.attention_mask.to(self.device) - text_encoder = Gemma3ForConditionalGeneration.from_pretrained( - self.gemma_root, - torch_dtype=self.dtype, - low_cpu_mem_usage=True, - ).eval() - text_encoder.to(self.device) - text_backbone = getattr(text_encoder, "model", text_encoder) - outputs = text_backbone( - input_ids=input_ids, - attention_mask=attention_mask, - output_hidden_states=True, - ) - hidden_states = torch.stack(outputs.hidden_states, dim=-1) - sequence_lengths = attention_mask.sum(dim=-1) - del text_encoder, text_backbone, outputs - _empty_cuda_cache() + # Pull the cached encoder onto the GPU only for the forward, then park + # it back on the CPU so it is absent during denoise/decode. + self.text_encoder.to(self.device) + try: + text_backbone = getattr(self.text_encoder, "model", self.text_encoder) + outputs = text_backbone( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + ) + hidden_states = torch.stack(outputs.hidden_states, dim=-1) + sequence_lengths = attention_mask.sum(dim=-1) + del outputs + finally: + self.text_encoder.to("cpu") + if torch.cuda.is_available(): + torch.cuda.empty_cache() prompt_embeds = _pack_text_embeds( hidden_states, @@ -240,7 +272,6 @@ def _encode_prompt(self, prompt: str) -> tuple[Tensor, Tensor]: padding_side=tokenizer.padding_side, ).to(dtype=self.dtype) del hidden_states - _empty_cuda_cache() self.connectors.to(self.device) connector_prompt_embeds, _, connector_attention_mask = self.connectors( @@ -249,7 +280,6 @@ def _encode_prompt(self, prompt: str) -> tuple[Tensor, Tensor]: ) self.connectors.to("cpu") del prompt_embeds, attention_mask - _empty_cuda_cache() return ( connector_prompt_embeds.to(device=self.device, dtype=self.dtype), connector_attention_mask.to(device=self.device), @@ -589,9 +619,3 @@ def _prepare_encoder_attention_mask(mask: Tensor | None, dtype: torch.dtype) -> if bool(torch.all(mask)): return None return ((1 - mask.to(dtype)) * -10000.0).unsqueeze(1) - - -def _empty_cuda_cache() -> None: - if torch.cuda.is_available(): - torch.cuda.empty_cache() - gc.collect() diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index 48cada5f6..deb66530f 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -174,6 +174,11 @@ class SanaWMRunnerConfig(RunnerConfig): offload_vae: bool = False """Move the VAE to CPU between encode/decode phases.""" + offload_stage1: bool = False + """Tear down the Stage-1 DiT after sampling to free memory for decode/ + refine. Default keeps it resident (fastest); enable on memory-constrained + GPUs.""" + offload_refiner: bool = False """Build and release the refiner only around refinement.""" @@ -397,6 +402,7 @@ def _pipeline_config( checkpoint_path=cfg.model_path, stage1_precision=cfg.stage1_precision, quant_backend=quant_backend, + offload_stage1=cfg.offload_stage1, ), ), encoder=dict( diff --git a/integrations/sana/sana_wm/transformer.py b/integrations/sana/sana_wm/transformer.py index 335db4e9c..dd5b1ac19 100644 --- a/integrations/sana/sana_wm/transformer.py +++ b/integrations/sana/sana_wm/transformer.py @@ -18,6 +18,7 @@ from __future__ import annotations import gc +import time from dataclasses import dataclass, field from types import SimpleNamespace from typing import Any, Literal, cast @@ -121,6 +122,11 @@ class SanaWMTransformerConfig(TransformerConfig): width: int = DEFAULT_VIDEO_WIDTH """Pixel width used by the public SANA-WM bidirectional release.""" + offload_stage1: bool = False + """Tear down the Stage-1 DiT after sampling, before decode/refine. Default + keeps it resident (fastest when memory is available); enable only to free + Stage-1 memory for the VAE/refiner on memory-constrained GPUs.""" + class SanaWMTransformer(Transformer[SanaWMTransformerCache]): """FlashDreams adapter for the SANA-WM Stage-1 model call.""" @@ -229,9 +235,17 @@ def postprocess_clean_latent( cache: SanaWMTransformerCache, input: Any = None, ) -> Tensor: - """Release Stage-1 runtime before the pipeline enters decode/refine.""" + """Release Stage-1 runtime before decode/refine only when offloading. + + By default the Stage-1 DiT stays resident so a reused pipeline never + reloads and re-quantizes it; ``offload_stage1`` restores the old + free-before-decode behavior for memory-constrained GPUs. + """ del input - self.release_stage1_runtime(cache) + if self.config.offload_stage1: + self.release_stage1_runtime(cache) + elif cache is not None: + cache.conditioning = None return clean_latent def initial_latents(self, conditioning: SanaWMStage1Conditioning) -> Tensor: @@ -355,6 +369,7 @@ def _ensure_model(self) -> None: if self._model_built: self._prepare_stage1_quant() return + t0 = time.perf_counter() cfg = self._ensure_runtime_config() weight_dtype = self._ensure_weight_dtype() model = SanaWMStage1Model().to(self.device) @@ -382,6 +397,11 @@ def _ensure_model(self) -> None: self.model = model.eval().to(weight_dtype) self._model_built = True self._prepare_stage1_quant() + logger.info( + "[timing] stage1 build+load+quant: {:.3f}s (precision={})", + time.perf_counter() - t0, + self.config.stage1_precision, + ) def _prepare_stage1_quant(self) -> None: if self._stage1_quantized or self.config.stage1_precision == "bf16": diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index c626a5b26..6c9c76fcf 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -153,6 +153,7 @@ def test_runner_pipeline_config_routes_runtime_fields_to_components() -> None: no_refiner=True, offload_vae=True, offload_text_encoder=True, + offload_stage1=True, ) pipeline = _pipeline_config(cfg, quant_backend="torch-fp8") @@ -171,6 +172,7 @@ def test_runner_pipeline_config_routes_runtime_fields_to_components() -> None: assert pipeline.diffusion_model.transformer.checkpoint_path == ( "local_model.safetensors" ) + assert pipeline.diffusion_model.transformer.offload_stage1 is True def test_sana_ltx_scheduler_step_pins_zero_timestep_tokens() -> None: @@ -744,6 +746,58 @@ def test_transformer_releases_stage1_runtime() -> None: assert transformer._stage1_quantized is False +def _stage1_conditioning_cache() -> SanaWMTransformerCache: + return SanaWMTransformerCache( + conditioning=SanaWMStage1Conditioning( + condition=torch.empty(1), + uncondition=None, + model_kwargs={}, + first_latent=torch.empty(1), + latent_shape=(1, 1, 1, 1, 1), + cfg_scale=1.0, + flow_shift=1.0, + steps=1, + seed=0, + ) + ) + + +def test_postprocess_keeps_stage1_resident_by_default() -> None: + """Default keeps the Stage-1 DiT resident so a reused pipeline never reloads.""" + transformer = SanaWMTransformerConfig().setup() + assert transformer.config.offload_stage1 is False + model = torch.nn.Linear(1, 1) + transformer.model = model + transformer._model_built = True + transformer._stage1_quantized = True + cache = _stage1_conditioning_cache() + latent = torch.zeros(1) + + result = transformer.postprocess_clean_latent(latent, cache) + + assert result is latent + assert transformer.model is model + assert transformer._model_built is True + assert transformer._stage1_quantized is True + # Per-rollout conditioning is still dropped even when the model stays warm. + assert cache.conditioning is None + + +def test_postprocess_releases_stage1_when_offloading() -> None: + """offload_stage1 restores the free-before-decode behavior.""" + transformer = SanaWMTransformerConfig(offload_stage1=True).setup() + transformer.model = torch.nn.Linear(1, 1) + transformer._model_built = True + transformer._stage1_quantized = True + cache = _stage1_conditioning_cache() + + transformer.postprocess_clean_latent(torch.zeros(1), cache) + + assert transformer.model is None + assert transformer._model_built is False + assert cache.conditioning is None + + def test_vae_tiling_uses_low_memory_tiles() -> None: """Keep VAE decode on the configured low-memory tiles.""" From 4e4a8cd23e5edf1fa2b86171ccc4c21cb1f67fd1 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 17 Jul 2026 14:06:44 -0700 Subject: [PATCH 15/64] Derive SANA-WM intrinsics from the first frame when none provided Make --intrinsics-path optional. When omitted, intrinsics are derived from the first-frame size with a centered principal point and a focal length set by a horizontal field of view (new --intrinsics-hfov-deg, default 90 to match the public demo intrinsics). This lets a run be driven by just an image and a prompt (with --action supplying the camera trajectory). - camera: add default_intrinsics_vec4(), returning [F, 4] intrinsics in source-image pixels so they flow through transform_intrinsics_for_crop identically to a loaded file. - runner: derive intrinsics when intrinsics_path is None, log the derivation, and expose --intrinsics-hfov-deg; explicit files behave as before. - README: document optional intrinsics and a minimal image+prompt run. - tests: cover the helper (centering, FOV math, bad-FOV rejection) and the runner derive path. Co-Authored-By: Claude Opus 4.8 (1M context) --- integrations/sana/README.md | 17 ++++++++++-- integrations/sana/sana_wm/camera.py | 36 ++++++++++++++++++++++++++ integrations/sana/sana_wm/runner.py | 24 ++++++++++++++--- integrations/sana/tests/test_camera.py | 21 +++++++++++++++ integrations/sana/tests/test_smoke.py | 26 +++++++++++++++++++ 5 files changed, 119 insertions(+), 5 deletions(-) diff --git a/integrations/sana/README.md b/integrations/sana/README.md index a8a9f18d5..48e3c0b2a 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -47,8 +47,6 @@ well. ## Run -The runner requires explicit intrinsics: - ```bash PYTORCH_ALLOC_CONF=expandable_segments:True \ uv run flashdreams-run sana-wm-bidirectional \ @@ -66,6 +64,21 @@ Expected output: outputs/sana_wm_bf16/sana_wm_generated.mp4 ``` +`--intrinsics-path` is optional. When omitted, intrinsics are derived from the +first-frame size assuming a centered principal point and a horizontal field of +view of `--intrinsics-hfov-deg` (default `90`, matching the demo intrinsics). +Likewise `--camera-path` may be replaced by an `--action` DSL string. A minimal +run therefore needs only an image and a prompt: + +```bash +uv run flashdreams-run sana-wm-bidirectional \ + --image-path my_frame.png \ + --prompt "a scene description; describe the world's own motion" \ + --action "w-100,dw-60,w-101" \ + --num-frames 161 \ + --output-dir outputs/mine +``` + For Stage-1-only diagnostics, add `--no-refiner True`. ## FP8 and FP4 diff --git a/integrations/sana/sana_wm/camera.py b/integrations/sana/sana_wm/camera.py index af8aa406e..363534100 100644 --- a/integrations/sana/sana_wm/camera.py +++ b/integrations/sana/sana_wm/camera.py @@ -317,6 +317,42 @@ def load_intrinsics(path: Path, num_frames: int) -> np.ndarray: ) +def default_intrinsics_vec4( + src_size: tuple[int, int], + num_frames: int, + *, + hfov_deg: float = 90.0, +) -> np.ndarray: + """Derive ``[fx, fy, cx, cy]`` intrinsics from an image size. + + Used when the caller does not supply an intrinsics file. Assumes square + pixels and a principal point at the image center, with focal length set by + a horizontal field of view. The public SANA-WM demo intrinsics correspond + to roughly ``90`` degrees; lower values are narrower/more zoomed-in. The + returned intrinsics are in the source image's pixel coordinates, matching + :func:`load_intrinsics`, so they flow through + :func:`transform_intrinsics_for_crop` identically. + + Args: + src_size: Source image size as ``(width, height)``. + num_frames: Number of frames required by the rollout. + hfov_deg: Horizontal field of view in degrees. + + Returns: + ``[num_frames, 4]`` intrinsics as ``[fx, fy, cx, cy]``. + """ + src_w, src_h = src_size + if src_w <= 0 or src_h <= 0: + raise ValueError(f"src_size must be positive, got {src_size}.") + if not 0.0 < hfov_deg < 180.0: + raise ValueError(f"hfov_deg must be in (0, 180), got {hfov_deg}.") + focal = 0.5 * src_w / math.tan(math.radians(hfov_deg) / 2.0) + vector = np.array( + [focal, focal, src_w / 2.0, src_h / 2.0], dtype=np.float32 + ) + return np.broadcast_to(vector, (num_frames, 4)).copy() + + def snap_num_frames( num_frames: int, *, diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index deb66530f..ea0e01dd3 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -34,6 +34,7 @@ from flashdreams.infra.runner import Runner, RunnerConfig from sana_wm.camera import ( action_string_to_c2w, + default_intrinsics_vec4, load_intrinsics, resize_center_crop_geometry, snap_num_frames, @@ -89,7 +90,14 @@ class SanaWMRunnerConfig(RunnerConfig): intrinsics_path: Path | None = None """Optional ``.npy`` intrinsics shaped ``[3, 3]``, ``[F, 3, 3]``, - ``[4]``, or ``[F, 4]``.""" + ``[4]``, or ``[F, 4]``. When omitted, intrinsics are derived from the + first-frame size using ``intrinsics_hfov_deg`` with a centered principal + point.""" + + intrinsics_hfov_deg: float = 90.0 + """Horizontal field of view in degrees used to derive intrinsics when + ``intrinsics_path`` is not provided. The public demo intrinsics correspond + to ~90 degrees; lower values are narrower/more zoomed-in.""" action: str | None = DEFAULT_ACTION """Action DSL used when ``camera_path`` is not provided.""" @@ -370,8 +378,18 @@ def _prepare_inputs(self) -> tuple[object, np.ndarray, np.ndarray, int]: ) ) if cfg.intrinsics_path is None: - raise ValueError("SanaWMRunner requires --intrinsics-path.") - intrinsics_src = load_intrinsics(cfg.intrinsics_path, num_frames) + intrinsics_src = default_intrinsics_vec4( + image.size, num_frames, hfov_deg=cfg.intrinsics_hfov_deg + ) + if self.is_rank_zero: + logger.info( + "No --intrinsics-path provided; deriving intrinsics from " + "image size {} at hfov={} deg (principal point centered).", + image.size, + cfg.intrinsics_hfov_deg, + ) + else: + intrinsics_src = load_intrinsics(cfg.intrinsics_path, num_frames) intrinsics_vec4 = transform_intrinsics_for_crop( intrinsics_src, image.size, diff --git a/integrations/sana/tests/test_camera.py b/integrations/sana/tests/test_camera.py index 43d0b7275..ba7a52ed8 100644 --- a/integrations/sana/tests/test_camera.py +++ b/integrations/sana/tests/test_camera.py @@ -25,6 +25,7 @@ from sana_wm.camera import ( action_string_to_c2w, + default_intrinsics_vec4, fit_intrinsics_sequence, load_intrinsics, prepare_camera, @@ -36,6 +37,26 @@ pytestmark = pytest.mark.ci_cpu +def test_default_intrinsics_vec4_centers_and_matches_hfov() -> None: + vec = default_intrinsics_vec4((1000, 500), num_frames=3, hfov_deg=90.0) + assert vec.shape == (3, 4) + fx, fy, cx, cy = vec[0] + # 90 deg hfov with square pixels -> fx = 0.5 * W / tan(45) = 0.5 * W. + assert fx == pytest.approx(500.0) + assert fy == pytest.approx(500.0) + # Principal point at the image center. + assert cx == pytest.approx(500.0) + assert cy == pytest.approx(250.0) + # Same intrinsics for every frame. + assert np.allclose(vec, vec[0]) + + +def test_default_intrinsics_vec4_rejects_bad_fov() -> None: + for bad in (0.0, 180.0, 200.0): + with pytest.raises(ValueError): + default_intrinsics_vec4((640, 360), num_frames=1, hfov_deg=bad) + + def test_action_string_rolls_out_identity_plus_motion() -> None: """Expand ``w-3`` to an identity frame plus three motion frames.""" c2w = action_string_to_c2w("w-3", smooth=False) diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index 6c9c76fcf..f39c3eaf3 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -945,6 +945,32 @@ def test_runner_setup_preserves_cli_fields() -> None: assert runner.config.image_path == Path("missing.png") +def test_runner_derives_intrinsics_when_omitted(tmp_path: Path) -> None: + """Omitting --intrinsics-path derives per-frame intrinsics from the frame.""" + from PIL import Image + + image_path = tmp_path / "frame.png" + Image.new("RGB", (640, 360), color=(10, 20, 30)).save(image_path) + cfg = derive_config( + RUNNER_SANA_WM_BIDIRECTIONAL, + image_path=image_path, + prompt="demo", + intrinsics_path=None, + action="w-40", + num_frames=25, + ) + runner = cfg.setup() + + _image, c2w, intrinsics_vec4, num_frames = runner._prepare_inputs() + + assert num_frames == 25 + assert c2w.shape[0] == num_frames + assert intrinsics_vec4.shape == (num_frames, 4) + # Derived focal lengths are finite and positive after the crop transform. + assert np.all(np.isfinite(intrinsics_vec4)) + assert np.all(intrinsics_vec4[:, :2] > 0) + + def test_runner_config_type() -> None: """Keep the exported literal on the SANA-WM runner config subclass.""" assert isinstance(RUNNER_SANA_WM_BIDIRECTIONAL, SanaWMRunnerConfig) From 5449238e58025be25a106a1e5cd4d29ce42c3464 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 17 Jul 2026 14:19:19 -0700 Subject: [PATCH 16/64] Make SANA integration obey FlashDreams contracts Move SANA-WM first-frame timestep pinning and zero-flow behavior out of the scheduler context path and into the SANA transformer. The shared diffusion model now calls schedulers through the generic initial_noise/predict_flow/rng interface again, and the SANA LTX Euler scheduler only owns timestep construction and update math. Make the decoder/refiner boundary conform to FlashDreams video decoder contracts by subclassing StreamingVideoDecoder and publishing the SANA 8k+1 temporal sizing. Remove runner, cache, and refiner AR block-size/KV-frame knobs that were accepted but ignored, leaving the supported sink-bidirectional LTX-2 refiner path explicit. Route runner step and flow_shift overrides into the scheduler config instead of through SANA conditioning, and update SANA smoke coverage for scheduler isolation, transformer first-frame pinning, decoder sizing, CFG, and refiner behavior. Tests:\n- uv run --package flashdreams-sana-wm --extra dev pytest integrations/sana/tests Signed-off-by: Aidan Foster --- .../flashdreams/infra/diffusion/model/base.py | 1 - .../infra/diffusion/scheduler/base.py | 6 +- .../infra/diffusion/scheduler/fm.py | 3 - .../infra/diffusion/scheduler/fm_euler.py | 3 - .../infra/diffusion/scheduler/fm_unipc.py | 3 - integrations/sana/sana_wm/decoder.py | 43 +++++-- integrations/sana/sana_wm/refiner.py | 9 -- integrations/sana/sana_wm/runner.py | 12 +- integrations/sana/sana_wm/scheduler.py | 114 ------------------ integrations/sana/sana_wm/transformer.py | 93 +++++++++++++- integrations/sana/tests/test_smoke.py | 74 +++++++++--- 11 files changed, 185 insertions(+), 176 deletions(-) diff --git a/flashdreams/flashdreams/infra/diffusion/model/base.py b/flashdreams/flashdreams/infra/diffusion/model/base.py index 67cef7532..57b4e5171 100644 --- a/flashdreams/flashdreams/infra/diffusion/model/base.py +++ b/flashdreams/flashdreams/infra/diffusion/model/base.py @@ -212,7 +212,6 @@ def predict_flow(noisy_latent: Tensor, timestep: Tensor) -> Tensor: initial_noise=initial_noise, predict_flow=predict_flow, rng=self.rng, - context=input, ) if self.config.noise_in_unpatchified_shape: diff --git a/flashdreams/flashdreams/infra/diffusion/scheduler/base.py b/flashdreams/flashdreams/infra/diffusion/scheduler/base.py index 4bb0f1dd6..46511d93b 100644 --- a/flashdreams/flashdreams/infra/diffusion/scheduler/base.py +++ b/flashdreams/flashdreams/infra/diffusion/scheduler/base.py @@ -19,7 +19,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Any, Protocol +from typing import Protocol import torch import torch.nn as nn @@ -69,7 +69,6 @@ def sample( initial_noise: Tensor, predict_flow: FlowPredictor, rng: torch.Generator | None = None, - context: Any = None, ) -> Tensor: """Run the full denoising loop and return the clean latent. @@ -84,9 +83,6 @@ def sample( ``num_inference_steps`` times. rng: Generator on the same device. Used by self-forcing renoise loops; pure ODE solvers ignore it. - context: Optional per-AR-step encoder output. Most schedulers - ignore it; schedulers with token-local constraints can use it - without taking over the diffusion-model orchestration. Returns: Clean latent with the same shape, device, and dtype as diff --git a/flashdreams/flashdreams/infra/diffusion/scheduler/fm.py b/flashdreams/flashdreams/infra/diffusion/scheduler/fm.py index 75a6cbc4e..baaabf78b 100644 --- a/flashdreams/flashdreams/infra/diffusion/scheduler/fm.py +++ b/flashdreams/flashdreams/infra/diffusion/scheduler/fm.py @@ -18,7 +18,6 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any import torch from torch import Tensor @@ -213,7 +212,6 @@ def sample( initial_noise: Tensor, predict_flow: FlowPredictor, rng: torch.Generator | None = None, - context: Any = None, ) -> Tensor: """Run the self-forcing flow-match denoising loop. @@ -222,7 +220,6 @@ def sample( sigma before the network forward. Schedule arithmetic auto-promotes to fp32; the result is cast back to ``initial_noise.dtype``. """ - del context input_dtype = initial_noise.dtype sigmas = self.denoising_sigmas timesteps = self.denoising_step_list diff --git a/flashdreams/flashdreams/infra/diffusion/scheduler/fm_euler.py b/flashdreams/flashdreams/infra/diffusion/scheduler/fm_euler.py index 2723e0bd0..76498d987 100644 --- a/flashdreams/flashdreams/infra/diffusion/scheduler/fm_euler.py +++ b/flashdreams/flashdreams/infra/diffusion/scheduler/fm_euler.py @@ -29,7 +29,6 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any import numpy as np import torch @@ -187,7 +186,6 @@ def sample( initial_noise: Tensor, predict_flow: FlowPredictor, rng: torch.Generator | None = None, - context: Any = None, ) -> Tensor: """Run the explicit Euler denoising loop. @@ -201,7 +199,6 @@ def sample( ``rng`` is unused (deterministic ODE) but accepted for interface conformance. """ - del context input_dtype = initial_noise.dtype N = self.config.num_inference_steps diff --git a/flashdreams/flashdreams/infra/diffusion/scheduler/fm_unipc.py b/flashdreams/flashdreams/infra/diffusion/scheduler/fm_unipc.py index ff5ca162d..a9ce940a7 100644 --- a/flashdreams/flashdreams/infra/diffusion/scheduler/fm_unipc.py +++ b/flashdreams/flashdreams/infra/diffusion/scheduler/fm_unipc.py @@ -18,7 +18,6 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any import numpy as np import torch @@ -332,7 +331,6 @@ def sample( initial_noise: Tensor, predict_flow: FlowPredictor, rng: torch.Generator | None = None, - context: Any = None, ) -> Tensor: """Run the order-2 UniPC predictor-corrector denoising loop. @@ -342,7 +340,6 @@ def sample( fp32; the result is cast back to ``initial_noise.dtype``. ``rng`` is unused (deterministic ODE) but accepted for interface conformance. """ - del context input_dtype = initial_noise.dtype N = self.timesteps.shape[0] diff --git a/integrations/sana/sana_wm/decoder.py b/integrations/sana/sana_wm/decoder.py index f3367f6d7..34c53b8df 100644 --- a/integrations/sana/sana_wm/decoder.py +++ b/integrations/sana/sana_wm/decoder.py @@ -31,8 +31,8 @@ from flashdreams.infra.config import InstantiateConfig from flashdreams.infra.decoder import ( DecoderConfig, - StreamingDecoder, StreamingDecoderCache, + StreamingVideoDecoder, ) from sana_wm._tools import resolve_hf_path from sana_wm.constants import ( @@ -72,8 +72,6 @@ class SanaWMVideoDecoderCache(StreamingDecoderCache): save_stage1: bool = False refiner_seed: int = 42 sink_size: int = 1 - refiner_block_size: int | None = None - refiner_kv_max_frames: int = 11 @dataclass(kw_only=True) @@ -342,8 +340,6 @@ def refine_latents( fps: int, sink_size: int, seed: int, - block_size: int | None, - kv_max_frames: int, ) -> Tensor: """Run the LTX-2 refiner.""" self._ensure_refiner() @@ -354,8 +350,6 @@ def refine_latents( sink_size=int(sink_size), seed=int(seed), progress=True, - block_size=block_size, - kv_max_frames=int(kv_max_frames), ) if self.config.offload_refiner: self.release_runtime() @@ -414,7 +408,7 @@ class SanaWMVideoDecoderConfig(DecoderConfig): ) -class SanaWMVideoDecoder(StreamingDecoder[SanaWMVideoDecoderCache]): +class SanaWMVideoDecoder(StreamingVideoDecoder[SanaWMVideoDecoderCache]): """Decode Stage-1 latents, optionally through the LTX-2 refiner.""" config: SanaWMVideoDecoderConfig @@ -452,8 +446,6 @@ def forward( fps=cache.fps, sink_size=cache.sink_size, seed=cache.refiner_seed, - block_size=cache.refiner_block_size, - kv_max_frames=cache.refiner_kv_max_frames, ) # refine_latents() already releases the refiner when offload_refiner # is set; keep it resident otherwise so a reused pipeline avoids a @@ -483,6 +475,37 @@ def temporal_compression_ratio(self) -> int: """Pixel frame compression ratio after the first latent.""" return SANA_WM_VAE_TEMPORAL_COMPRESSION + def get_output_temporal_size( + self, + autoregressive_index: int, + input_temporal_size: int, + ) -> int: + """Return decoded pixel frames for a SANA-WM latent sequence.""" + if autoregressive_index != 0: + raise ValueError("SANA-WM bidirectional inference has one AR step.") + if input_temporal_size <= 0: + raise ValueError( + f"input_temporal_size must be positive, got {input_temporal_size}." + ) + return 1 + (input_temporal_size - 1) * self.temporal_compression_ratio + + def get_input_temporal_size( + self, + autoregressive_index: int, + output_temporal_size: int, + ) -> int: + """Return latent frames required to decode ``output_temporal_size`` pixels.""" + if autoregressive_index != 0: + raise ValueError("SANA-WM bidirectional inference has one AR step.") + ratio = self.temporal_compression_ratio + remainder = (output_temporal_size - 1) % ratio + if output_temporal_size <= 0 or remainder != 0: + raise ValueError( + "SANA-WM output frame count must be positive and equal to 8k+1; " + f"got {output_temporal_size}." + ) + return ((output_temporal_size - 1) // ratio) + 1 + __all__ = [ "SanaWMDecodedVideo", diff --git a/integrations/sana/sana_wm/refiner.py b/integrations/sana/sana_wm/refiner.py index 8cfce1f96..86e465c1e 100644 --- a/integrations/sana/sana_wm/refiner.py +++ b/integrations/sana/sana_wm/refiner.py @@ -84,18 +84,9 @@ def refine_latents( sink_size: int = 1, seed: int = 42, progress: bool = True, - block_size: int | None = None, - kv_max_frames: int = 11, sigmas: tuple[float, ...] = (0.909375, 0.725, 0.421875, 0.0), ) -> Tensor: """Refine Stage-1 VAE latents with the sink-bidirectional LTX-2 path.""" - del kv_max_frames - if block_size is not None: - logger.warning( - "[refiner] --refiner-block-size={} requested; running the " - "sink-bidirectional LTX-2 path used by the bidirectional model.", - block_size, - ) if sana_latent.shape[2] <= sink_size: raise ValueError( f"Stage-1 latent has {sana_latent.shape[2]} frames but " diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index ea0e01dd3..6debabab4 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -173,12 +173,6 @@ class SanaWMRunnerConfig(RunnerConfig): sink_size: int = 1 """Number of sink latent frames used by the refiner.""" - refiner_block_size: int | None = None - """Optional refiner AR block size; ``None`` uses sink-bidirectional mode.""" - - refiner_kv_max_frames: int = 11 - """Maximum refiner KV context frames in AR mode.""" - offload_vae: bool = False """Move the VAE to CPU between encode/decode phases.""" @@ -299,8 +293,6 @@ def run(self) -> None: "save_stage1": cfg.save_stage1, "refiner_seed": cfg.refiner_seed, "sink_size": cfg.sink_size, - "refiner_block_size": cfg.refiner_block_size, - "refiner_kv_max_frames": cfg.refiner_kv_max_frames, } ) decoded = pipeline.generate( @@ -411,10 +403,14 @@ def _pipeline_config( quant_backend: ResolvedQuantBackend, ) -> StreamInferencePipelineConfig: """Apply CLI runtime fields to the SANA-WM pipeline literal.""" + scheduler_updates: dict[str, object] = {"num_inference_steps": cfg.step} + if cfg.flow_shift is not None: + scheduler_updates["shift"] = cfg.flow_shift return derive_config( cfg.pipeline, diffusion_model=dict( seed=cfg.seed, + scheduler=scheduler_updates, transformer=dict( config_path=cfg.config_path, checkpoint_path=cfg.model_path, diff --git a/integrations/sana/sana_wm/scheduler.py b/integrations/sana/sana_wm/scheduler.py index 4bf1ed288..168c377db 100644 --- a/integrations/sana/sana_wm/scheduler.py +++ b/integrations/sana/sana_wm/scheduler.py @@ -17,9 +17,7 @@ from __future__ import annotations -import os from dataclasses import dataclass, field -from typing import Any, cast import numpy as np import torch @@ -30,7 +28,6 @@ Scheduler, SchedulerConfig, ) -from sana_wm.transformer import SanaWMStage1Conditioning @dataclass(kw_only=True) @@ -156,17 +153,9 @@ def sample( initial_noise: Tensor, predict_flow: FlowPredictor, rng: torch.Generator | None = None, - context: Any = None, ) -> Tensor: """Run the generic scalar-timestep Euler loop.""" del rng - if isinstance(context, SanaWMStage1Conditioning): - return self._sample_conditioned( - initial_noise=initial_noise, - predict_flow=predict_flow, - conditioning=context, - ) - timesteps = self.timesteps( num_inference_steps=self.config.num_inference_steps, shift=self.config.shift, @@ -183,88 +172,6 @@ def sample( ) return noisy.to(initial_noise.dtype) - def _sample_conditioned( - self, - *, - initial_noise: Tensor, - predict_flow: FlowPredictor, - conditioning: SanaWMStage1Conditioning, - ) -> Tensor: - """Run SANA-WM's first-frame-pinned LTX Euler denoising loop.""" - latents = initial_noise - timesteps = self.timesteps( - num_inference_steps=conditioning.steps, - shift=conditioning.flow_shift, - device=latents.device, - ) - condition_frame_info = dict( - cast( - dict[int, float], - cast(dict[str, object], conditioning.model_kwargs["data_info"]).get( - "condition_frame_info", - {}, - ), - ) - ) - condition_mask = torch.zeros_like(latents) - image_cond_noise_scale = 0.0 - for frame_idx, frame_weight in condition_frame_info.items(): - condition_mask[:, :, int(frame_idx)] = 1 - image_cond_noise_scale = max(image_cond_noise_scale, float(frame_weight)) - - init_latents = latents.clone() - generator = torch.Generator(device=latents.device).manual_seed( - conditioning.seed - ) - iterator = enumerate(timesteps[:-1]) - if os.getenv("DPM_TQDM", "False") == "True": - from tqdm import tqdm - - iterator = tqdm(list(iterator)) - - for step_index, timestep_scalar in iterator: - if image_cond_noise_scale > 0: - latents = _add_noise_to_conditioning_latents( - t=timestep_scalar / self.config.num_train_timesteps, - init_latents=init_latents, - latents=latents, - noise_scale=image_cond_noise_scale, - conditioning_mask=condition_mask, - generator=generator, - ) - - timestep = timestep_scalar.expand(condition_mask.shape).float() - timestep = torch.min( - timestep, - (1 - condition_mask) * float(self.config.num_train_timesteps), - ) - noise_pred = predict_flow(latents, timestep[:, :1, :, 0, 0]) - - latents_dtype = latents.dtype - latents_shape = latents.shape - batch_size, channels, _frames, _height, _width = latents_shape - denoised_latents = self.step_ltx( - model_output=-noise_pred.reshape( - batch_size, - channels, - -1, - ).transpose(1, 2), - timestep=timestep_scalar, - next_timestep=timesteps[step_index + 1], - sample=latents.reshape(batch_size, channels, -1).transpose(1, 2), - per_token_timesteps=timestep.reshape(batch_size, channels, -1)[:, 0], - schedule_timesteps=timesteps, - ) - denoised_latents = denoised_latents.transpose(1, 2).reshape(latents_shape) - tokens_to_denoise_mask = ( - timestep_scalar / self.config.num_train_timesteps - 1e-6 - ) < (1.0 - condition_mask) - latents = torch.where(tokens_to_denoise_mask, denoised_latents, latents) - if latents.dtype != latents_dtype: - latents = latents.to(latents_dtype) - - return latents.detach() - def add_noise( self, clean_input: Tensor, @@ -283,27 +190,6 @@ def add_noise( return ((1.0 - sigma) * clean_input + sigma * noise).to(clean_input.dtype) -def _add_noise_to_conditioning_latents( - *, - t: Tensor, - init_latents: Tensor, - latents: Tensor, - noise_scale: float, - conditioning_mask: Tensor, - generator: torch.Generator, - eps: float = 1e-6, -) -> Tensor: - noise = torch.randn( - latents.shape, - generator=generator, - device=latents.device, - dtype=latents.dtype, - ) - need_to_noise = conditioning_mask > (1.0 - eps) - noised_latents = init_latents + noise_scale * noise * (t**2) - return torch.where(need_to_noise, noised_latents, latents) - - __all__ = [ "SanaWMLTXEulerScheduler", "SanaWMLTXEulerSchedulerConfig", diff --git a/integrations/sana/sana_wm/transformer.py b/integrations/sana/sana_wm/transformer.py index dd5b1ac19..e9e0abd72 100644 --- a/integrations/sana/sana_wm/transformer.py +++ b/integrations/sana/sana_wm/transformer.py @@ -316,26 +316,34 @@ def _predict_conditioned( timestep: Tensor, conditioning: SanaWMStage1Conditioning, ) -> Tensor: + model_timestep = _conditioned_frame_timestep( + noisy_latent=noisy_latent, + timestep=timestep, + conditioning=conditioning, + num_train_timesteps=1000, + ) if conditioning.cfg_scale <= 1.0: - return self._predict_with_prompt( + flow = self._predict_with_prompt( noisy_latent, - timestep, + model_timestep, conditioning.condition, _condition_model_kwargs(conditioning.model_kwargs), ) + return _zero_conditioned_frame_flow(flow, conditioning) if conditioning.uncondition is None: raise RuntimeError("CFG was requested without negative prompt embeds.") noise_pred = self._predict_with_prompt( torch.cat([noisy_latent, noisy_latent], dim=0), - torch.cat([timestep, timestep], dim=0), + torch.cat([model_timestep, model_timestep], dim=0), torch.cat([conditioning.uncondition, conditioning.condition], dim=0), _batched_cfg_model_kwargs(conditioning.model_kwargs), ) noise_pred_uncond, noise_pred_text = noise_pred.chunk(2, dim=0) - return noise_pred_uncond + conditioning.cfg_scale * ( + flow = noise_pred_uncond + conditioning.cfg_scale * ( noise_pred_text - noise_pred_uncond ) + return _zero_conditioned_frame_flow(flow, conditioning) def _predict_with_prompt( self, @@ -465,6 +473,83 @@ def _batched_cfg_model_kwargs(model_kwargs: dict[str, object]) -> dict[str, obje return kwargs +def _conditioned_frame_timestep( + *, + noisy_latent: Tensor, + timestep: Tensor, + conditioning: SanaWMStage1Conditioning, + num_train_timesteps: int, +) -> Tensor: + """Expand scheduler timesteps and pin conditioned frames to timestep zero.""" + batch, _channels, frames, _height, _width = noisy_latent.shape + if timestep.ndim == 0: + frame_timestep = timestep.reshape(1, 1, 1).expand(batch, 1, frames) + elif timestep.ndim == 1: + frame_timestep = timestep.reshape(batch, 1, 1).expand(batch, 1, frames) + elif timestep.ndim == 3: + frame_timestep = timestep + elif timestep.ndim == noisy_latent.ndim: + frame_timestep = timestep[:, :1, :, 0, 0] + else: + raise ValueError( + "SANA-WM timestep must be scalar, [B], [B, 1, T], or latent-shaped; " + f"got shape={tuple(timestep.shape)}." + ) + if frame_timestep.shape != (batch, 1, frames): + raise ValueError( + "SANA-WM timestep shape is incompatible with the latent: " + f"got {tuple(frame_timestep.shape)}, expected {(batch, 1, frames)}." + ) + + frame_timestep = frame_timestep.to(device=noisy_latent.device, dtype=torch.float32) + condition_frame_mask = _condition_frame_mask( + conditioning, + batch=batch, + frames=frames, + device=noisy_latent.device, + ) + max_timestep = (1.0 - condition_frame_mask) * float(num_train_timesteps) + return torch.minimum(frame_timestep, max_timestep) + + +def _zero_conditioned_frame_flow( + flow: Tensor, + conditioning: SanaWMStage1Conditioning, +) -> Tensor: + """Prevent scalar scheduler updates from moving pinned condition frames.""" + batch, _channels, frames, _height, _width = flow.shape + condition_frame_mask = _condition_frame_mask( + conditioning, + batch=batch, + frames=frames, + device=flow.device, + ).to(dtype=torch.bool) + return torch.where(condition_frame_mask[:, :, :, None, None], 0, flow) + + +def _condition_frame_mask( + conditioning: SanaWMStage1Conditioning, + *, + batch: int, + frames: int, + device: torch.device, +) -> Tensor: + data_info = conditioning.model_kwargs.get("data_info", {}) + condition_frame_info = ( + data_info.get("condition_frame_info", {}) + if isinstance(data_info, dict) + else {} + ) + mask = torch.zeros(batch, 1, frames, dtype=torch.float32, device=device) + if not isinstance(condition_frame_info, dict): + return mask + for frame_idx in condition_frame_info: + index = int(frame_idx) + if 0 <= index < frames: + mask[:, :, index] = 1.0 + return mask + + def _avoid_degenerate_tile_tail( *, sample_extent: int, diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index f39c3eaf3..6c774ec0e 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -94,6 +94,7 @@ ) from flashdreams.infra.config import derive_config +from flashdreams.infra.decoder import StreamingVideoDecoder from flashdreams.infra.diffusion.model import DiffusionModel pytestmark = pytest.mark.ci_cpu @@ -134,6 +135,21 @@ def test_pipeline_uses_sana_diffusion_model() -> None: assert isinstance(pipeline.decoder.refiner, SanaWMLTX2LatentRefinerConfig) +def test_sana_decoder_uses_video_decoder_contract() -> None: + """Expose SANA-WM VAE temporal sizing through the FlashDreams decoder API.""" + decoder = SanaWMVideoDecoderConfig().setup() + + assert isinstance(decoder, StreamingVideoDecoder) + assert decoder.spatial_compression_ratio == 32 + assert decoder.temporal_compression_ratio == 8 + assert decoder.get_output_temporal_size(0, 21) == 161 + assert decoder.get_input_temporal_size(0, 161) == 21 + with pytest.raises(ValueError, match="8k\\+1"): + decoder.get_input_temporal_size(0, 160) + with pytest.raises(ValueError, match="one AR step"): + decoder.get_output_temporal_size(1, 21) + + def test_sana_diffusion_config_instantiates_base_model() -> None: """Use FlashDreams' shared diffusion model rather than a Sana-only runner.""" model = PIPELINE_SANA_WM_BIDIRECTIONAL.diffusion_model.setup() @@ -150,6 +166,8 @@ def test_runner_pipeline_config_routes_runtime_fields_to_components() -> None: config_path="local_config.yaml", model_path="local_model.safetensors", stage1_precision="fp8", + step=7, + flow_shift=6.5, no_refiner=True, offload_vae=True, offload_text_encoder=True, @@ -173,6 +191,8 @@ def test_runner_pipeline_config_routes_runtime_fields_to_components() -> None: "local_model.safetensors" ) assert pipeline.diffusion_model.transformer.offload_stage1 is True + assert pipeline.diffusion_model.scheduler.num_inference_steps == 7 + assert pipeline.diffusion_model.scheduler.shift == 6.5 def test_sana_ltx_scheduler_step_pins_zero_timestep_tokens() -> None: @@ -252,9 +272,10 @@ def test_sana_ltx_scheduler_matches_diffusers_per_token_step() -> None: torch.testing.assert_close(actual, expected) -def test_sana_scheduler_context_path_keeps_conditioned_frame_fixed() -> None: - """Let the scheduler own SANA-WM's per-token first-frame constraint.""" +def test_sana_transformer_keeps_conditioned_frame_fixed_with_generic_scheduler() -> None: + """Keep SANA conditioning out of the scheduler and inside the transformer.""" scheduler = SanaWMLTXEulerSchedulerConfig(num_inference_steps=1).setup() + transformer = SanaWMTransformerConfig().setup() initial_noise = torch.zeros((1, 1, 2, 1, 1), dtype=torch.float32) initial_noise[:, :, 0] = 5.0 conditioning = SanaWMStage1Conditioning( @@ -269,14 +290,33 @@ def test_sana_scheduler_context_path_keeps_conditioned_frame_fixed() -> None: seed=0, ) + class DummyModel(torch.nn.Module): + def forward( + self, + noisy_latent: torch.Tensor, + timestep: torch.Tensor, + _prompt_embeds: torch.Tensor, + **_kwargs: object, + ) -> torch.Tensor: + assert timestep.shape == (1, 1, 2) + torch.testing.assert_close(timestep[:, :, 0], torch.zeros((1, 1))) + assert torch.all(timestep[:, :, 1] > 0) + return -torch.ones_like(noisy_latent) + + transformer.model = DummyModel() + transformer._model_built = True + def predict_flow(noisy_latent: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor: - assert timestep.shape == (1, 1, 2) - return -torch.ones_like(noisy_latent) + return transformer.predict_flow( + noisy_latent=noisy_latent, + timestep=timestep, + cache=SanaWMTransformerCache(), + input=conditioning, + ) sampled = scheduler.sample( initial_noise=initial_noise, predict_flow=predict_flow, - context=conditioning, ) torch.testing.assert_close(sampled[:, :, 0], initial_noise[:, :, 0]) @@ -375,8 +415,8 @@ def forward( transformer._model_built = True cond_mask = torch.ones((1, 1)) neg_mask = torch.zeros((1, 1)) - camera = torch.zeros((1, 3, 1, 1, 1)) - chunk_plucker = torch.ones((1, 6, 1, 1, 1)) + camera = torch.zeros((1, 3, 2, 1, 1)) + chunk_plucker = torch.ones((1, 6, 2, 1, 1)) conditioning = SanaWMStage1Conditioning( condition=torch.ones((1, 1, 1, 1)), uncondition=torch.zeros((1, 1, 1, 1)), @@ -388,7 +428,7 @@ def forward( "data_info": {"condition_frame_info": {0: 0.0}}, }, first_latent=torch.empty((1, 1, 1, 1, 1)), - latent_shape=(1, 1, 1, 1, 1), + latent_shape=(1, 1, 2, 1, 1), cfg_scale=2.0, flow_shift=1.0, steps=1, @@ -396,17 +436,20 @@ def forward( ) out = transformer.predict_flow( - noisy_latent=torch.zeros((1, 1, 1, 1, 1)), - timestep=torch.zeros((1, 1, 1)), + noisy_latent=torch.zeros((1, 1, 2, 1, 1)), + timestep=torch.full((1, 1, 2), 1000.0), cache=SanaWMTransformerCache(), input=conditioning, ) - torch.testing.assert_close(out, torch.full((1, 1, 1, 1, 1), 2.0)) + torch.testing.assert_close( + out, + torch.tensor([[[[[0.0]], [[2.0]]]]]), + ) assert len(dummy_model.calls) == 1 call = dummy_model.calls[0] - assert call["noisy_latent"].shape == (2, 1, 1, 1, 1) - assert call["timestep"].shape == (2, 1, 1) + assert call["noisy_latent"].shape == (2, 1, 2, 1, 1) + assert call["timestep"].shape == (2, 1, 2) assert call["prompt_embeds"].shape == (2, 1, 1, 1) torch.testing.assert_close(call["mask"], torch.cat([neg_mask, cond_mask], dim=0)) torch.testing.assert_close( @@ -1040,10 +1083,10 @@ def test_refiner_latent_pack_round_trips() -> None: torch.testing.assert_close(unpacked, latents) -def test_refiner_block_size_request_still_executes( +def test_refiner_sink_bidirectional_path_runs_under_inference_mode( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Do not turn refiner block-size requests into a hard failure.""" + """Keep the supported LTX-2 refiner path explicit and inference-only.""" class DummyTransformer(torch.nn.Module): def __init__(self) -> None: @@ -1083,7 +1126,6 @@ def predict_current_x0( latents, "demo", fps=16.0, - block_size=1, progress=False, sigmas=(0.5, 0.0), ) From 5a82a1e697c06b44bf983321469464d51442ae7f Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 17 Jul 2026 14:34:02 -0700 Subject: [PATCH 17/64] Fit SANA trajectories to requested frame count Make the SANA-WM runner choose the snapped rollout length from --num-frames before preparing camera conditioning. Action-derived trajectories now repeat or truncate to that snapped length, so a short action prompt no longer caps the generated video. Fit explicit --camera-path trajectories to the same frame count instead of treating the file length as an upper bound. This matches the existing intrinsics fitting behavior and prevents a 321-frame pose file from truncating a longer requested rollout. Document the updated behavior and add CPU coverage for action repetition, explicit camera-path fitting, and runner input preparation. Tests: - uv run --package flashdreams-sana-wm --extra dev pytest integrations/sana/tests Signed-off-by: Aidan Foster --- integrations/sana/README.md | 6 ++- integrations/sana/sana_wm/camera.py | 61 ++++++++++++++++++++++++++ integrations/sana/sana_wm/runner.py | 34 +++++++++----- integrations/sana/tests/test_camera.py | 32 ++++++++++++++ integrations/sana/tests/test_smoke.py | 30 ++++++++++++- 5 files changed, 148 insertions(+), 15 deletions(-) diff --git a/integrations/sana/README.md b/integrations/sana/README.md index 48e3c0b2a..142108a5d 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -67,8 +67,10 @@ outputs/sana_wm_bf16/sana_wm_generated.mp4 `--intrinsics-path` is optional. When omitted, intrinsics are derived from the first-frame size assuming a centered principal point and a horizontal field of view of `--intrinsics-hfov-deg` (default `90`, matching the demo intrinsics). -Likewise `--camera-path` may be replaced by an `--action` DSL string. A minimal -run therefore needs only an image and a prompt: +Likewise `--camera-path` may be replaced by an `--action` DSL string, which is +repeated or truncated to the requested snapped frame count. Explicit camera +paths are fitted to the same frame count instead of capping the video length. A +minimal run therefore needs only an image and a prompt: ```bash uv run flashdreams-run sana-wm-bidirectional \ diff --git a/integrations/sana/sana_wm/camera.py b/integrations/sana/sana_wm/camera.py index 363534100..063345910 100644 --- a/integrations/sana/sana_wm/camera.py +++ b/integrations/sana/sana_wm/camera.py @@ -221,6 +221,7 @@ def action_string_to_c2w( rotation_speed_deg: float = DEFAULT_ROTATION_SPEED_DEG, pitch_limit_deg: float = DEFAULT_PITCH_LIMIT_DEG, smooth: bool = True, + num_frames: int | None = None, ) -> np.ndarray: """Roll out a camera-to-world trajectory from an action string. @@ -230,11 +231,25 @@ def action_string_to_c2w( rotation_speed_deg: Per-frame angular magnitude in degrees. pitch_limit_deg: Maximum absolute pitch in degrees. smooth: Whether to use the SANA-WM velocity smoothing model. + num_frames: Optional output frame count. When provided, the action + prompt is repeated or truncated to produce exactly this many + camera poses. Returns: ``[F + 1, 4, 4]`` camera-to-world matrices in OpenCV convention. """ per_frame = parse_action_string(action) + if num_frames is not None: + if num_frames < 1: + raise ValueError(f"num_frames must be positive, got {num_frames}.") + target_steps = num_frames - 1 + if target_steps == 0: + per_frame = [] + elif len(per_frame) < target_steps: + repeats = math.ceil(target_steps / len(per_frame)) + per_frame = (per_frame * repeats)[:target_steps] + else: + per_frame = per_frame[:target_steps] integrator = CameraPoseIntegrator(math.radians(pitch_limit_deg)) velocity = VelocityState() poses = [integrator.pose.copy()] @@ -262,6 +277,52 @@ def action_string_to_c2w( return np.stack(poses, axis=0).astype(np.float32) +def fit_camera_trajectory(c2w: np.ndarray, num_frames: int) -> np.ndarray: + """Return a camera-to-world trajectory fitted to ``num_frames`` poses. + + Translations are interpolated linearly. Rotations are interpolated in + matrix space and projected back to the nearest proper rotation, avoiding a + new SciPy dependency for runner-time camera fitting. + """ + c2w = np.asarray(c2w, dtype=np.float32) + if c2w.ndim != 3 or c2w.shape[1:] != (4, 4): + raise ValueError(f"camera trajectory must be [F, 4, 4], got {c2w.shape}.") + if num_frames < 1: + raise ValueError(f"num_frames must be positive, got {num_frames}.") + if c2w.shape[0] == num_frames: + return c2w.copy() + if c2w.shape[0] == 1: + return np.broadcast_to(c2w[:1], (num_frames, 4, 4)).copy() + + old_t = np.linspace(0.0, 1.0, c2w.shape[0], dtype=np.float32) + new_t = np.linspace(0.0, 1.0, num_frames, dtype=np.float32) + fitted = np.broadcast_to(np.eye(4, dtype=np.float32), (num_frames, 4, 4)).copy() + + for axis in range(3): + fitted[:, axis, 3] = np.interp( + new_t, + old_t, + c2w[:, axis, 3], + ).astype(np.float32) + + rotations = c2w[:, :3, :3].reshape(c2w.shape[0], 9) + interp_rotations = np.empty((num_frames, 9), dtype=np.float32) + for idx in range(9): + interp_rotations[:, idx] = np.interp( + new_t, + old_t, + rotations[:, idx], + ).astype(np.float32) + for frame_idx, rotation in enumerate(interp_rotations.reshape(num_frames, 3, 3)): + u, _, vh = np.linalg.svd(rotation.astype(np.float64), full_matrices=False) + projected = u @ vh + if np.linalg.det(projected) < 0.0: + u[:, -1] *= -1.0 + projected = u @ vh + fitted[frame_idx, :3, :3] = projected.astype(np.float32) + return fitted + + def fit_intrinsics_sequence(arr: np.ndarray, num_frames: int) -> np.ndarray: """Return ``arr`` fitted to ``num_frames`` along axis 0.""" arr = np.asarray(arr, dtype=np.float32) diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index 6debabab4..a3bc1a46d 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -35,6 +35,7 @@ from sana_wm.camera import ( action_string_to_c2w, default_intrinsics_vec4, + fit_camera_trajectory, load_intrinsics, resize_center_crop_geometry, snap_num_frames, @@ -100,7 +101,8 @@ class SanaWMRunnerConfig(RunnerConfig): to ~90 degrees; lower values are narrower/more zoomed-in.""" action: str | None = DEFAULT_ACTION - """Action DSL used when ``camera_path`` is not provided.""" + """Action DSL used to derive camera motion when ``camera_path`` is not + provided. The action is repeated or truncated to match ``num_frames``.""" translation_speed: float = 0.025 """Per-frame action translation speed.""" @@ -228,21 +230,34 @@ def _resolve_device(self) -> torch.device: return torch.device(f"cuda:{self.local_rank}") return torch.device(self.config.device) - def _resolve_trajectory(self) -> np.ndarray: - """Load or roll out the camera-to-world trajectory.""" + def _resolve_trajectory(self, *, num_frames: int) -> np.ndarray: + """Load, fit, or roll out the camera-to-world trajectory.""" if self.config.camera_path is not None: c2w = np.load(self.config.camera_path).astype(np.float32) if c2w.ndim != 3 or c2w.shape[1:] != (4, 4): raise ValueError( f"--camera-path must be a [F, 4, 4] .npy; got {c2w.shape}." ) - return c2w + if c2w.shape[0] != num_frames and self.is_rank_zero: + logger.info( + "Fitting --camera-path trajectory from {} to {} frames.", + c2w.shape[0], + num_frames, + ) + return fit_camera_trajectory(c2w, num_frames) if not self.config.action: raise ValueError("SanaWMRunner requires --camera-path or --action.") + if self.is_rank_zero: + logger.info( + "No --camera-path provided; deriving a {}-frame trajectory " + "from --action.", + num_frames, + ) return action_string_to_c2w( self.config.action, translation_speed=self.config.translation_speed, rotation_speed_deg=self.config.rotation_speed_deg, + num_frames=num_frames, ) def run(self) -> None: @@ -336,23 +351,18 @@ def _prepare_inputs(self) -> tuple[object, np.ndarray, np.ndarray, int]: cfg = self.config assert cfg.image_path is not None image = Image.open(cfg.image_path).convert("RGB") - c2w_full = self._resolve_trajectory() - num_frames = min(cfg.num_frames, c2w_full.shape[0]) snapped = snap_num_frames( - num_frames, + cfg.num_frames, stride=SANA_WM_VAE_TEMPORAL_COMPRESSION, - upper_bound=c2w_full.shape[0], ) if snapped != cfg.num_frames and self.is_rank_zero: logger.warning( - "SANA-WM requires num_frames = 8k+1; requested {} snapped to {} " - "(trajectory has {} frames).", + "SANA-WM requires num_frames = 8k+1; requested {} snapped to {}.", cfg.num_frames, snapped, - c2w_full.shape[0], ) num_frames = snapped - c2w = c2w_full[:num_frames] + c2w = self._resolve_trajectory(num_frames=num_frames) resized_size, crop_offset = resize_center_crop_geometry( image.size, diff --git a/integrations/sana/tests/test_camera.py b/integrations/sana/tests/test_camera.py index ba7a52ed8..e54d14eb9 100644 --- a/integrations/sana/tests/test_camera.py +++ b/integrations/sana/tests/test_camera.py @@ -26,6 +26,7 @@ from sana_wm.camera import ( action_string_to_c2w, default_intrinsics_vec4, + fit_camera_trajectory, fit_intrinsics_sequence, load_intrinsics, prepare_camera, @@ -66,6 +67,19 @@ def test_action_string_rolls_out_identity_plus_motion() -> None: assert np.all(np.diff(c2w[:, 2, 3]) > 0) +def test_action_string_repeats_to_requested_frame_count() -> None: + """Use action prompts to derive the requested number of camera poses.""" + c2w = action_string_to_c2w("w-1", smooth=False, num_frames=5) + + assert c2w.shape == (5, 4, 4) + np.testing.assert_allclose(c2w[0], np.eye(4), atol=1e-6) + np.testing.assert_allclose( + c2w[:, 2, 3], + [0.0, 0.025, 0.05, 0.075, 0.1], + atol=1e-6, + ) + + def test_action_string_rejects_unknown_keys() -> None: """Reject invalid action DSL tokens.""" with pytest.raises(ValueError, match="unknown keys"): @@ -99,6 +113,24 @@ def test_fit_intrinsics_sequence_interpolates_short_sequences() -> None: ) +def test_fit_camera_trajectory_interpolates_short_sequences() -> None: + """Fit explicit camera paths to the requested rollout length.""" + source = np.broadcast_to(np.eye(4, dtype=np.float32), (2, 4, 4)).copy() + source[1, 2, 3] = 2.0 + + fitted = fit_camera_trajectory(source, 5) + + assert fitted.shape == (5, 4, 4) + np.testing.assert_allclose(fitted[:, 2, 3], [0.0, 0.5, 1.0, 1.5, 2.0]) + np.testing.assert_allclose( + fitted[:, 3], + np.broadcast_to(np.array([0.0, 0.0, 0.0, 1.0]), (5, 4)), + ) + for rotation in fitted[:, :3, :3]: + np.testing.assert_allclose(rotation.T @ rotation, np.eye(3), atol=1e-5) + assert np.linalg.det(rotation) == pytest.approx(1.0, abs=1e-5) + + @pytest.mark.parametrize( ("array", "expected"), [ diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index 6c774ec0e..ad41e950b 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -999,7 +999,7 @@ def test_runner_derives_intrinsics_when_omitted(tmp_path: Path) -> None: image_path=image_path, prompt="demo", intrinsics_path=None, - action="w-40", + action="w-4", num_frames=25, ) runner = cfg.setup() @@ -1014,6 +1014,34 @@ def test_runner_derives_intrinsics_when_omitted(tmp_path: Path) -> None: assert np.all(intrinsics_vec4[:, :2] > 0) +def test_runner_fits_camera_path_to_requested_frames(tmp_path: Path) -> None: + """Do not let short explicit trajectories cap the output frame count.""" + from PIL import Image + + image_path = tmp_path / "frame.png" + camera_path = tmp_path / "camera.npy" + Image.new("RGB", (640, 360), color=(10, 20, 30)).save(image_path) + c2w = np.broadcast_to(np.eye(4, dtype=np.float32), (3, 4, 4)).copy() + c2w[:, 2, 3] = [0.0, 1.0, 2.0] + np.save(camera_path, c2w) + cfg = derive_config( + RUNNER_SANA_WM_BIDIRECTIONAL, + image_path=image_path, + prompt="demo", + intrinsics_path=None, + camera_path=camera_path, + num_frames=25, + ) + runner = cfg.setup() + + _image, fitted_c2w, intrinsics_vec4, num_frames = runner._prepare_inputs() + + assert num_frames == 25 + assert fitted_c2w.shape[0] == num_frames + assert intrinsics_vec4.shape == (num_frames, 4) + np.testing.assert_allclose(fitted_c2w[[0, -1], 2, 3], [0.0, 2.0]) + + def test_runner_config_type() -> None: """Keep the exported literal on the SANA-WM runner config subclass.""" assert isinstance(RUNNER_SANA_WM_BIDIRECTIONAL, SanaWMRunnerConfig) From 3f4bbfdf7df12839ef4105d798ef72ab4e1bdffc Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Mon, 20 Jul 2026 16:10:22 -0700 Subject: [PATCH 18/64] Adapt runner for refactor --- integrations/sana/sana_wm/runner.py | 42 +++++++++++++---------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index a3bc1a46d..ff7e23ed3 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -28,10 +28,10 @@ import torch from loguru import logger -from flashdreams.core.io.disk import preflight_runtime_write_paths from flashdreams.infra.config import derive_config from flashdreams.infra.pipeline import StreamInferencePipelineConfig from flashdreams.infra.runner import Runner, RunnerConfig +from flashdreams.infra.runner_io import ensure_output_dir, resolve_prompt_value, runner_artifact_path from sana_wm.camera import ( action_string_to_c2w, default_intrinsics_vec4, @@ -110,9 +110,6 @@ class SanaWMRunnerConfig(RunnerConfig): rotation_speed_deg: float = 0.6 """Per-frame action rotation speed in degrees.""" - name: str = "sana_wm" - """Output filename stem.""" - num_frames: int = 161 """Requested output frames before LTX2-VAE stride snapping.""" @@ -205,20 +202,15 @@ def __init__(self, config: SanaWMRunnerConfig) -> None: self.world_size = int(os.environ.get("WORLD_SIZE", "1")) self.global_rank = int(os.environ.get("RANK", "0")) self.is_rank_zero = self.global_rank == 0 - preflight_runtime_write_paths(output_dir=config.output_dir) def _resolve_prompt(self) -> str: """Resolve the prompt from inline text or ``prompt_path``.""" - if self.config.prompt: - return self.config.prompt - if self.config.prompt_path is None: + cfg = self.config + if cfg.prompt: + return resolve_prompt_value(cfg.prompt) + if cfg.prompt_path is None: raise ValueError("SanaWMRunner requires --prompt or --prompt-path.") - prompt = self.config.prompt_path.read_text( - encoding="utf-8", errors="replace" - ).strip() - if not prompt: - raise ValueError(f"Prompt file is empty: {self.config.prompt_path}") - return prompt + return resolve_prompt_value(cfg.prompt_path) def _resolve_device(self) -> torch.device: """Return the device used by SANA-WM.""" @@ -335,11 +327,15 @@ def run(self) -> None: ) if not self.is_rank_zero: return - _write_video(cfg.output_dir, cfg.name, decoded.video_hwc, cfg.fps) + ensure_output_dir(cfg.output_dir) + _write_video( + runner_artifact_path(cfg.output_dir, cfg.runner_name, "mp4"), + decoded.video_hwc, + cfg.fps, + ) if decoded.stage1_video_hwc is not None: _write_video( - cfg.output_dir, - f"{cfg.name}_stage1", + runner_artifact_path(cfg.output_dir, f"{cfg.runner_name}_stage1", "mp4"), decoded.stage1_video_hwc, cfg.fps, ) @@ -468,15 +464,13 @@ def _pipeline_config( ) -def _write_video(output_dir: Path, name: str, video_hwc: np.ndarray, fps: int) -> Path: - """Write an HWC uint8 video to the runner output directory.""" +def _write_video(path: Path, video_hwc: np.ndarray, fps: int) -> Path: + """Write an HWC uint8 video to ``path``.""" import imageio.v3 as iio - output_dir.mkdir(parents=True, exist_ok=True) - video_path = output_dir / f"{name}_generated.mp4" - iio.imwrite(video_path, video_hwc, fps=fps) - logger.info("Saved {}", video_path) - return video_path + iio.imwrite(path, video_hwc, fps=fps) + logger.info("Saved {}", path) + return path def _precision_env_updates( From e26d882657456d9d02ad1acb841e156b4ff06868 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Tue, 21 Jul 2026 11:06:38 -0700 Subject: [PATCH 19/64] Prune dead env-var code --- integrations/sana/sana_wm/runner.py | 173 ++++++++------------------ integrations/sana/tests/test_smoke.py | 50 -------- 2 files changed, 53 insertions(+), 170 deletions(-) diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index ff7e23ed3..bce53ea96 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -18,8 +18,6 @@ from __future__ import annotations import os -from collections.abc import Iterator, Mapping -from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path from typing import Literal @@ -274,71 +272,65 @@ def run(self) -> None: refiner_enabled=not cfg.no_refiner, quant_backend=cfg.quant_backend, ) - precision_env = _precision_env_updates( - stage1_precision=cfg.stage1_precision, - refiner_precision=cfg.refiner_precision, - refiner_enabled=not cfg.no_refiner, + prompt = self._resolve_prompt() + image, c2w, intrinsics_vec4, num_frames = self._prepare_inputs() + pipeline_cfg = _pipeline_config( + cfg, + quant_backend=quant_backend, ) - with _temporary_environment(precision_env): - prompt = self._resolve_prompt() - image, c2w, intrinsics_vec4, num_frames = self._prepare_inputs() - pipeline_cfg = _pipeline_config( - cfg, - quant_backend=quant_backend, - ) - pipeline = pipeline_cfg.setup().to(device).eval() - sampling_algo = self._sampling_algo() - if sampling_algo != "flow_euler_ltx": - raise ValueError( - "SANA-WM requires flow_euler_ltx for the " - f"bidirectional runner; got {sampling_algo!r}." - ) - cache = pipeline.initialize_cache( - decoder_context={ - "prompt": prompt, - "fps": cfg.fps, - "save_stage1": cfg.save_stage1, - "refiner_seed": cfg.refiner_seed, - "sink_size": cfg.sink_size, - } + pipeline = pipeline_cfg.setup().to(device).eval() + sampling_algo = self._sampling_algo() + if sampling_algo != "flow_euler_ltx": + raise ValueError( + "SANA-WM requires flow_euler_ltx for the " + f"bidirectional runner; got {sampling_algo!r}." ) - decoded = pipeline.generate( - 0, - cache, - input=SanaWMI2VConditioningRequest( - image=image, - prompt=prompt, - poses_c2w=c2w, - intrinsics_vec4=intrinsics_vec4, - num_frames=num_frames, - fps=cfg.fps, - steps=cfg.step, - cfg_scale=cfg.cfg_scale, - flow_shift=cfg.flow_shift, - seed=cfg.seed, - negative_prompt=cfg.negative_prompt, - ), + cache = pipeline.initialize_cache( + decoder_context={ + "prompt": prompt, + "fps": cfg.fps, + "save_stage1": cfg.save_stage1, + "refiner_seed": cfg.refiner_seed, + "sink_size": cfg.sink_size, + } + ) + decoded = pipeline.generate( + 0, + cache, + input=SanaWMI2VConditioningRequest( + image=image, + prompt=prompt, + poses_c2w=c2w, + intrinsics_vec4=intrinsics_vec4, + num_frames=num_frames, + fps=cfg.fps, + steps=cfg.step, + cfg_scale=cfg.cfg_scale, + flow_shift=cfg.flow_shift, + seed=cfg.seed, + negative_prompt=cfg.negative_prompt, + ), + ) + pipeline.finalize(0, cache) + if not isinstance(decoded, SanaWMDecodedVideo): + raise TypeError( + "SANA-WM pipeline decoder returned " + f"{type(decoded).__name__}, expected SanaWMDecodedVideo." ) - pipeline.finalize(0, cache) - if not isinstance(decoded, SanaWMDecodedVideo): - raise TypeError( - "SANA-WM pipeline decoder returned " - f"{type(decoded).__name__}, expected SanaWMDecodedVideo." - ) - if not self.is_rank_zero: - return - ensure_output_dir(cfg.output_dir) + if not self.is_rank_zero: + return + ensure_output_dir(cfg.output_dir) + _write_video( + runner_artifact_path(cfg.output_dir, cfg.runner_name, "mp4"), + decoded.video_hwc, + cfg.fps, + ) + if decoded.stage1_video_hwc is not None: _write_video( - runner_artifact_path(cfg.output_dir, cfg.runner_name, "mp4"), - decoded.video_hwc, + runner_artifact_path(cfg.output_dir, f"{cfg.runner_name}_stage1", "mp4"), + decoded.stage1_video_hwc, cfg.fps, ) - if decoded.stage1_video_hwc is not None: - _write_video( - runner_artifact_path(cfg.output_dir, f"{cfg.runner_name}_stage1", "mp4"), - decoded.stage1_video_hwc, - cfg.fps, - ) def _prepare_inputs(self) -> tuple[object, np.ndarray, np.ndarray, int]: """Load/crop the input image and prepare c2w/intrinsics.""" @@ -473,65 +465,6 @@ def _write_video(path: Path, video_hwc: np.ndarray, fps: int) -> Path: return path -def _precision_env_updates( - *, - stage1_precision: Precision, - refiner_precision: Precision, - refiner_enabled: bool, -) -> dict[str, str | None]: - """Return SANA-WM precision environment overrides.""" - updates: dict[str, str | None] = {"SANA_WM_STAGE1_NVFP4": None} - if stage1_precision != "bf16": - updates.update( - { - "SANA_WM_STAGE1_NVFP4": "self_attn+cross+ffn", - "SANA_WM_STAGE1_NVFP4_SCOPE": "block", - "SANA_WM_STAGE1_LINEARIZE_FFN": None, - "SANA_WM_STAGE1_QUANT": _quant_env_value(stage1_precision), - } - ) - - updates["SANA_WM_REFINER_NVFP4"] = None - if refiner_enabled and refiner_precision != "bf16": - updates.update( - { - "SANA_WM_REFINER_NVFP4": "1", - "SANA_WM_REFINER_QUANT": _quant_env_value(refiner_precision), - } - ) - return updates - - -def _quant_env_value(precision: Precision) -> str: - """Return the SANA quantization recipe selector for a precision.""" - if precision == "fp8": - return "fp8block" - if precision == "fp4": - return "nvfp4" - raise ValueError(f"{precision!r} does not use quantized SANA execution.") - - -@contextmanager -def _temporary_environment( - updates: Mapping[str, str | None], -) -> Iterator[None]: - """Temporarily set or unset environment variables.""" - previous = {key: os.environ.get(key) for key in updates} - try: - for key, value in updates.items(): - if value is None: - os.environ.pop(key, None) - else: - os.environ[key] = value - yield - finally: - for key, value in previous.items(): - if value is None: - os.environ.pop(key, None) - else: - os.environ[key] = value - - def _validate_precision_request( *, device: torch.device, diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index ad41e950b..7630a625e 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -17,7 +17,6 @@ from __future__ import annotations -import os from pathlib import Path from types import SimpleNamespace @@ -48,9 +47,7 @@ SanaWMRunner, SanaWMRunnerConfig, _pipeline_config, - _precision_env_updates, _resolve_quant_backend, - _temporary_environment, _validate_precision_request, ) from sana_wm.conditioning import ( @@ -1169,53 +1166,6 @@ def test_auto_quant_backend_resolves_to_torch_backend() -> None: assert _resolve_quant_backend("auto", []) == "torch" -def test_bf16_precision_clears_quant_env( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Prevent external NVFP4 env vars from changing the default runner path.""" - monkeypatch.setenv("SANA_WM_STAGE1_NVFP4", "1") - monkeypatch.setenv("SANA_WM_REFINER_NVFP4", "1") - updates = _precision_env_updates( - stage1_precision="bf16", - refiner_precision="bf16", - refiner_enabled=True, - ) - - with _temporary_environment(updates): - assert "SANA_WM_STAGE1_NVFP4" not in os.environ - assert "SANA_WM_REFINER_NVFP4" not in os.environ - - assert os.environ["SANA_WM_STAGE1_NVFP4"] == "1" - assert os.environ["SANA_WM_REFINER_NVFP4"] == "1" - - -def test_fp8_precision_sets_quant_env() -> None: - """Map FlashDreams fp8 fields to SANA-WM env selectors.""" - updates = _precision_env_updates( - stage1_precision="fp8", - refiner_precision="fp8", - refiner_enabled=True, - ) - - assert updates["SANA_WM_STAGE1_NVFP4"] == "self_attn+cross+ffn" - assert updates["SANA_WM_STAGE1_QUANT"] == "fp8block" - assert updates["SANA_WM_STAGE1_LINEARIZE_FFN"] is None - assert updates["SANA_WM_REFINER_NVFP4"] == "1" - assert updates["SANA_WM_REFINER_QUANT"] == "fp8block" - - -def test_fp4_precision_sets_sana_quant_env() -> None: - """Map FlashDreams fp4 fields to SANA-WM NVFP4 env selectors.""" - updates = _precision_env_updates( - stage1_precision="fp4", - refiner_precision="fp4", - refiner_enabled=True, - ) - - assert updates["SANA_WM_STAGE1_QUANT"] == "nvfp4" - assert updates["SANA_WM_REFINER_QUANT"] == "nvfp4" - - def test_quantized_precision_requires_cuda() -> None: """Reject fp8/fp4 before loading checkpoints on CPU-only devices.""" with pytest.raises(ValueError, match="requires a CUDA device"): From 2fe96afe06f63ee85b2bcc81b32fc030e2e49032 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Wed, 22 Jul 2026 11:37:48 -0700 Subject: [PATCH 20/64] Add WIP model card --- docs/source/index.rst | 8 ++ docs/source/models/index.rst | 9 +++ docs/source/models/sana_wm.rst | 144 +++++++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 docs/source/models/sana_wm.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 16530b788..49971cc80 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -261,6 +261,14 @@ invocation, the checkpoint source, and the per-implementation knobs. Bidirectional Cosmos-Predict2 reference implementations (T2V / I2V, 2B). + .. grid-item-card:: SANA-WM (bidirectional) + :class-card: fd-feature + :link: models/sana_wm + :link-type: doc + + Bidirectional camera-controlled world model (Stage-1 DiT + LTX-2 + refiner, 2.6B). + .. Master toctree: one flat entry per top-level navbar item. Order here = order in the navbar. diff --git a/docs/source/models/index.rst b/docs/source/models/index.rst index 78d767002..3a68a014a 100644 --- a/docs/source/models/index.rst +++ b/docs/source/models/index.rst @@ -28,6 +28,7 @@ Models flashvsr hy_worldplay lingbot_world + sana_wm wan21 FlashDreams runs a growing family of world and video models (text-to-video, @@ -172,6 +173,14 @@ uses, and the settings you can tune. Bidirectional Cosmos-Predict2 reference implementations (T2V / I2V, 2B). + .. grid-item-card:: SANA-WM + :class-card: fd-feature + :link: /models/sana_wm + :link-type: doc + + Bidirectional camera-controlled world model (Stage-1 DiT + LTX-2 + refiner, 2.6B). + .. container:: fd-eyebrow Super-resolution diff --git a/docs/source/models/sana_wm.rst b/docs/source/models/sana_wm.rst new file mode 100644 index 000000000..f8cf6bb75 --- /dev/null +++ b/docs/source/models/sana_wm.rst @@ -0,0 +1,144 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 +.. +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. + +SANA-WM +=================================== + +.. container:: fd-cta-row + + .. button-link:: https://nvlabs.github.io/Sana/ + :color: primary + + Project page + + .. button-link:: https://arxiv.org/abs/2410.10629 + :color: primary + + arXiv paper + + .. button-link:: https://huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional + :color: primary + + Model page + + .. button-link:: https://github.com/NVlabs/Sana + :color: primary + + Official code + +SANA-WM is a 2.6B bidirectional, camera-controlled world model from the +`NVlabs/Sana `_ family. Given a first frame, a +text prompt, and a camera trajectory it renders a video clip in a single +bidirectional pass. FlashDreams runs it through the ``sana-wm-bidirectional`` +runner, which pairs a native Stage-1 DiT (loading the public SANA-WM checkpoint +directly) with an LTX-2 refiner for the final decode. + +Requirements +------------ + +- **PyTorch**: >= 2.9. +- **Precision**: BF16 by default. FP8 Stage-1/refiner inference is available on + Hopper or newer GPUs (``sm_90+``) and FP4 on Blackwell (``sm_100+``); both + lower the memory footprint relative to the BF16 default. + +Installation +------------ + +.. code-block:: bash + + # from the repo root + uv sync --package flashdreams-sana-wm --extra dev + +Running the method +------------------ + +To run SANA-WM, launch the ``sana-wm-bidirectional`` runner with a first-frame +image, a prompt, and a camera trajectory. Set +``PYTORCH_ALLOC_CONF=expandable_segments:True`` to keep the allocator from +fragmenting across the Stage-1 + refiner handoff: + +.. code-block:: bash + + PYTORCH_ALLOC_CONF=expandable_segments:True \ + uv run flashdreams-run sana-wm-bidirectional \ + --image-path ../Sana/asset/sana_wm/demo_0.png \ + --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ + --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ + --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --num-frames 161 \ + --output-dir outputs/sana_wm_bf16 + +The demo image, prompt, camera, and intrinsics files ship with the SANA-WM +release; any inputs with matching shapes work as well. + +Optional inputs and knobs +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``--intrinsics-path`` is optional. When omitted, intrinsics are derived from + the first-frame size, assuming a centered principal point and a horizontal + field of view of ``--intrinsics-hfov-deg`` (default ``90``, matching the demo + intrinsics). +- ``--camera-path`` can be replaced by an ``--action`` DSL string, so a minimal + run needs only an image and a prompt: + + .. code-block:: bash + + uv run flashdreams-run sana-wm-bidirectional \ + --image-path my_frame.png \ + --prompt "a scene description; describe the world's own motion" \ + --action "w-100,dw-60,w-101" \ + --num-frames 161 \ + --output-dir outputs/mine + +- ``--no-refiner True`` runs Stage-1 only, for diagnostics. +- Quantized Stage-1/refiner inference is opt-in via ``--stage1-precision`` / + ``--refiner-precision`` (``fp8`` on ``sm_90+``, ``fp4`` on ``sm_100+``), with + ``--quant-backend`` (``torch-fp8`` / ``torch-fp4``) to force a backend. + +To inspect all supported CLI arguments and their default values, run: + +.. code-block:: bash + + uv run flashdreams-run sana-wm-bidirectional --help + +What to expect +-------------- + +- **Model checkpoint**: pulled from + ``huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional`` on first run and + cached under ``$HF_HOME``. The LTX-2 refiner and VAE weights are fetched from + their respective ``diffusers`` repositories on first use. +- **First launch**: a few minutes for the download and warmup; subsequent + launches reuse the caches. +- **Outputs**: ``outputs//sana_wm_generated.mp4``. + +See :doc:`/developer_guides/inference_pipeline_overview` for what one pass does +end-to-end. + +Citation +-------- + +If you use SANA-WM, please cite the original SANA work: + +.. code-block:: bibtex + + @misc{xie2024sana, + title={SANA: Efficient High-Resolution Image Synthesis with Linear Diffusion Transformers}, + author={Enze Xie and Junsong Chen and Junyu Chen and Han Cai and Haotian Tang and Yujun Lin and Zhekai Zhang and Muyang Li and Ligeng Zhu and Yao Lu and Song Han}, + year={2024}, + eprint={2410.10629}, + archivePrefix={arXiv}, + primaryClass={cs.CV} + } From ec6031e3d60a6e766a8ae8afb6c66fe0560f2efe Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Wed, 22 Jul 2026 16:50:56 -0700 Subject: [PATCH 21/64] Fix FP4 quality using NVFP4 scaling --- integrations/sana/README.md | 16 +- integrations/sana/sana_wm/quant.py | 166 +++++++++++++++++++-- integrations/sana/sana_wm/refiner.py | 14 ++ integrations/sana/sana_wm/stage1_model.py | 61 +++++++- integrations/sana/sana_wm/transformer.py | 30 +++- integrations/sana/tests/test_quant_cuda.py | 32 ++++ integrations/sana/tests/test_smoke.py | 85 ++++++++++- 7 files changed, 377 insertions(+), 27 deletions(-) diff --git a/integrations/sana/README.md b/integrations/sana/README.md index 142108a5d..3da123d38 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -61,7 +61,7 @@ uv run flashdreams-run sana-wm-bidirectional \ Expected output: ```text -outputs/sana_wm_bf16/sana_wm_generated.mp4 +outputs/sana_wm_bf16/sana-wm-bidirectional.mp4 ``` `--intrinsics-path` is optional. When omitted, intrinsics are derived from the @@ -83,10 +83,10 @@ uv run flashdreams-run sana-wm-bidirectional \ For Stage-1-only diagnostics, add `--no-refiner True`. -## FP8 and FP4 +## BF16, FP8, and FP4 -The runner defaults to BF16. Quantized Stage-1 inference is opt-in and uses -Torch-based backends by default. +The runner defaults to BF16. Quantized Stage-1 and refiner inference are opt-in +and use Torch-based backends by default. | option | hardware | backend | | --- | --- | --- | @@ -95,6 +95,14 @@ Torch-based backends by default. | `--quant-backend torch-fp8` | Hopper or newer (`sm_90+`) | force FP8 | | `--quant-backend torch-fp4` | Blackwell (`sm_100+`) | force FP4 | +Stage-1 FP8 and FP4 follow the upstream precision CLI scope: self-attention, +cross-attention, and linearized FFN pointwise projections. This integration uses +Torch/Triton scaled-MM replacements instead of TransformerEngine. FP4 uses +NVFP4 W4A4 GEMMs with tiled 16-wide Hadamard rotation enabled, two-level +per-tensor-plus-block scaling, and 2D 16x16 block scaling for weights, matching +the upstream recipe behavior that keeps Stage-1 camera/action conditioning +coherent. + FP8 smoke: ```bash diff --git a/integrations/sana/sana_wm/quant.py b/integrations/sana/sana_wm/quant.py index 3ff3ee0e1..d285e73f1 100644 --- a/integrations/sana/sana_wm/quant.py +++ b/integrations/sana/sana_wm/quant.py @@ -30,7 +30,27 @@ FP8_SCALE_EPS = 1.0e-12 FP4_MAX_E2M1 = 6.0 NVFP4_BLOCK_SIZE = 16 -NVFP4_SCALE_EPS = 1.5258789e-05 +NVFP4_E4M3_SCALE_EPS = 0.015625 +NVFP4_GLOBAL_SCALE_EPS = 1.0e-12 +_RHT16_SCALE = 0.25 +_RHT16_SIGNS = ( + 1.0, + -1.0, + -1.0, + 1.0, + -1.0, + 1.0, + -1.0, + 1.0, + 1.0, + 1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 1.0, + 1.0, +) @dataclass(frozen=True) @@ -47,6 +67,9 @@ class TorchScaledMMFP4Recipe: name: str = "torch_scaled_mm_fp4" precision: Literal["fp4"] = "fp4" + use_rht: bool = True + use_global_scale: bool = True + weight_scale_2d: bool = True QuantRecipe = TorchScaledMMFP8Recipe | TorchScaledMMFP4Recipe @@ -79,15 +102,21 @@ def _quantize_nvfp4_kernel( input_ptr, qdata_ptr, scale_ptr, + global_scale_ptr, stride_m, stride_k, rows: tl.constexpr, cols: tl.constexpr, mask_scales: tl.constexpr, + scale_2d: tl.constexpr, + has_global_scale: tl.constexpr, ): fp4_max = 6.0 fp8_max = 448.0 - scale_eps = 1.5258789e-05 + scale_eps = 0.015625 + global_scale = 1.0 + if has_global_scale: + global_scale = tl.load(global_scale_ptr).to(tl.float32) pid_k = tl.program_id(0) pid_m = tl.program_id(1) @@ -100,12 +129,31 @@ def _quantize_nvfp4_kernel( mask=mask, other=0.0, ) - values = values.to(tl.float32).reshape(128, 4, 16) - block_amax = tl.max(tl.abs(values), axis=2) - scales_f32 = block_amax / fp4_max - scales_f32 = tl.clamp(scales_f32, scale_eps, fp8_max) - scales = scales_f32.to(tl.float8e4nv) - quantized = tl.div_rn(values, scales.to(tl.float32)[:, :, None]) + values = values.to(tl.float32) + if scale_2d: + values_4d = values.reshape(8, 16, 4, 16) + block_amax = tl.max(tl.max(tl.abs(values_4d), axis=3), axis=1) + scales_f32 = block_amax / fp4_max + if has_global_scale: + scales_f32 = scales_f32 / global_scale + scales_f32 = tl.clamp(scales_f32, scale_eps, fp8_max) + scales_tile = scales_f32.to(tl.float8e4nv) + scales = tl.broadcast_to(tl.expand_dims(scales_tile, 1), (8, 16, 4)) + scales = scales.reshape(128, 4) + else: + values_3d = values.reshape(128, 4, 16) + block_amax = tl.max(tl.abs(values_3d), axis=2) + scales_f32 = block_amax / fp4_max + if has_global_scale: + scales_f32 = scales_f32 / global_scale + scales_f32 = tl.clamp(scales_f32, scale_eps, fp8_max) + scales = scales_f32.to(tl.float8e4nv) + + values = values.reshape(128, 4, 16) + scale_product = scales.to(tl.float32) + if has_global_scale: + scale_product = scale_product * global_scale + quantized = tl.div_rn(values, scale_product[:, :, None]) if mask_scales: scale_offsets_k = pid_k * 4 + tl.arange(0, 4)[None, :] @@ -135,7 +183,12 @@ def _swizzled_nvfp4_scale_shape(rows: int, cols: int) -> tuple[int, int]: return _ceil_div(rows, 128) * 32, _ceil_div(cols, 64) * 16 -def quantize_nvfp4_swizzled(input: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: +def quantize_nvfp4_swizzled( + input: torch.Tensor, + *, + global_scale: torch.Tensor | None = None, + scale_2d: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: """Quantize a 2D CUDA tensor to NVFP4 qdata plus swizzled E4M3 scales.""" _require_fp4_dtype() if input.dim() != 2: @@ -160,15 +213,45 @@ def quantize_nvfp4_swizzled(input: torch.Tensor) -> tuple[torch.Tensor, torch.Te input_2d, qdata, scales, + global_scale if global_scale is not None else input_2d, input_2d.stride(0), input_2d.stride(1), rows, cols, mask_scales=(rows % 128 != 0 or cols % 64 != 0), + scale_2d=scale_2d, + has_global_scale=global_scale is not None, ) return qdata, scales +def nvfp4_global_scale(input: torch.Tensor) -> torch.Tensor: + """Return the per-tensor FP32 global scale for hierarchical NVFP4.""" + amax = input.detach().abs().amax().to(torch.float32) + scale = amax / (FP8_MAX_E4M3 * FP4_MAX_E2M1) + return scale.clamp_min(NVFP4_GLOBAL_SCALE_EPS).reshape(()) + + +def apply_rht16(input: torch.Tensor) -> torch.Tensor: + """Apply the tiled 16-wide random Hadamard transform used by NVFP4.""" + if input.shape[-1] % 16 != 0: + raise ValueError( + f"RHT16 requires the last dimension to be divisible by 16, got {input.shape[-1]}." + ) + original_shape = input.shape + work = input.to(torch.float32).reshape(-1, 16) + signs = torch.tensor(_RHT16_SIGNS, device=input.device, dtype=work.dtype) + work = work * signs + width = 1 + while width < 16: + work = work.reshape(-1, 16 // (2 * width), 2, width) + left = work[:, :, 0, :] + right = work[:, :, 1, :] + work = torch.stack((left + right, left - right), dim=2).reshape(-1, 16) + width *= 2 + return (work * _RHT16_SCALE).reshape(original_shape).contiguous() + + class TorchScaledMMFP8Linear(nn.Module): """Inference-only FP8 Linear using ``torch._scaled_mm``. @@ -306,8 +389,10 @@ def __init__( weight: torch.Tensor, weight_qdata: torch.Tensor, weight_scale: torch.Tensor, + weight_global_scale: torch.Tensor | None, bias: torch.Tensor | None, out_dtype: torch.dtype, + use_rht: bool, ) -> None: super().__init__() expected_weight_shape = (weight_qdata.shape[0], weight_qdata.shape[1] * 2) @@ -330,9 +415,17 @@ def __init__( self.out_features = int(weight_qdata.shape[0]) self.in_features = int(weight_qdata.shape[1] * 2) self.out_dtype = out_dtype + self.use_rht = use_rht self.register_buffer("weight", weight.detach().contiguous()) self.register_buffer("weight_qdata", weight_qdata.contiguous()) self.register_buffer("weight_scale", weight_scale.contiguous()) + if weight_global_scale is None: + self.register_buffer("weight_global_scale", None) + else: + self.register_buffer( + "weight_global_scale", + weight_global_scale.detach().to(torch.float32).reshape(()), + ) if bias is None: self.register_buffer("bias", None) else: @@ -344,17 +437,32 @@ def from_linear( source: nn.Linear, *, out_dtype: torch.dtype, + use_rht: bool = True, + use_global_scale: bool = True, + weight_scale_2d: bool = True, ) -> "TorchScaledMMFP4Linear": """Create an NVFP4 replacement from a source ``nn.Linear``.""" _require_fp4_dtype() - weight_qdata, weight_scale = quantize_nvfp4_swizzled(source.weight.detach()) + weight_for_quant = source.weight.detach() + if use_rht: + weight_for_quant = apply_rht16(weight_for_quant) + weight_global_scale = ( + nvfp4_global_scale(weight_for_quant) if use_global_scale else None + ) + weight_qdata, weight_scale = quantize_nvfp4_swizzled( + weight_for_quant, + global_scale=weight_global_scale, + scale_2d=weight_scale_2d, + ) bias = source.bias.detach() if source.bias is not None else None replacement = cls( weight=source.weight.detach().to(device=source.weight.device), weight_qdata=weight_qdata, weight_scale=weight_scale, + weight_global_scale=weight_global_scale, bias=bias.to(device=source.weight.device) if bias is not None else None, out_dtype=out_dtype, + use_rht=use_rht, ) replacement.train(source.training) return replacement @@ -378,17 +486,42 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: dtype=self.out_dtype, ) - input_qdata, input_scale = quantize_nvfp4_swizzled(input_2d) + input_for_quant = apply_rht16(input_2d) if self.use_rht else input_2d + input_global_scale = ( + nvfp4_global_scale(input_for_quant) + if self.weight_global_scale is not None + else None + ) + input_qdata, input_scale = quantize_nvfp4_swizzled( + input_for_quant, + global_scale=input_global_scale, + ) + add_bias_after_scale = ( + self.bias is not None + and self.weight_global_scale is not None + and input_global_scale is not None + ) output = torch._scaled_mm( input_qdata.view(torch.float4_e2m1fn_x2), self.weight_qdata.t().view(torch.float4_e2m1fn_x2), input_scale.view(torch.float8_e4m3fn), self.weight_scale.view(torch.float8_e4m3fn), - bias=self.bias.to(device=input.device, dtype=self.out_dtype) - if self.bias is not None - else None, + bias=None + if add_bias_after_scale + else ( + self.bias.to(device=input.device, dtype=self.out_dtype) + if self.bias is not None + else None + ), out_dtype=self.out_dtype, ) + if input_global_scale is not None and self.weight_global_scale is not None: + output = output * ( + input_global_scale.to(device=output.device, dtype=output.dtype) + * self.weight_global_scale.to(device=output.device, dtype=output.dtype) + ) + if add_bias_after_scale: + output = output + self.bias.to(device=output.device, dtype=output.dtype) return output.reshape(*leading_shape, self.out_features) @@ -495,6 +628,9 @@ def _replace_linear_with_quant( replacement = TorchScaledMMFP4Linear.from_linear( child, out_dtype=out_dtype, + use_rht=recipe.use_rht, + use_global_scale=recipe.use_global_scale, + weight_scale_2d=recipe.weight_scale_2d, ) else: raise ValueError(f"Unsupported SANA-WM quant recipe: {recipe!r}.") @@ -544,6 +680,8 @@ def _require_fp4_dtype() -> None: "TorchScaledMMFP4Recipe", "TorchScaledMMFP8Linear", "TorchScaledMMFP8Recipe", + "apply_rht16", + "nvfp4_global_scale", "quantize_nvfp4_swizzled", "replace_linear_with_quant", "replace_linear_with_torch_fp4", diff --git a/integrations/sana/sana_wm/refiner.py b/integrations/sana/sana_wm/refiner.py index 86e465c1e..0d7c61107 100644 --- a/integrations/sana/sana_wm/refiner.py +++ b/integrations/sana/sana_wm/refiner.py @@ -187,6 +187,20 @@ def _prepare_quantization(self) -> None: f"layers; skipped={skipped}." ) self._quantized = True + recipe_detail = "" + if isinstance(recipe, TorchScaledMMFP4Recipe): + recipe_detail = ( + f" rht={recipe.use_rht}" + f" global_scale={recipe.use_global_scale}" + f" weight_scale_2d={recipe.weight_scale_2d}" + ) + logger.info( + "[refiner-quant] precision={}{} converted {} Linear layers (skipped {})", + self.precision, + recipe_detail, + converted, + skipped, + ) def _ensure_text_encoder(self) -> None: """Load the Gemma tokenizer + encoder once and cache them in CPU RAM. diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index e6615e9a9..522b2b44c 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -252,6 +252,62 @@ def forward(self, x: Tensor, *, frames: int, height: int, width: int) -> Tensor: return x_time.permute(0, 2, 3, 1).reshape(batch, tokens, channels) +class _LinearizedPointwiseConv2d(nn.Module): + """Run an existing 1x1 ``Conv2dContainer`` through a Linear module.""" + + def __init__(self, conv_layer: nn.Module) -> None: + super().__init__() + conv = getattr(conv_layer, "conv", None) + if not isinstance(conv, nn.Conv2d): + raise ValueError("expected Conv2dContainer.conv to be nn.Conv2d.") + if conv.kernel_size != (1, 1) or conv.stride != (1, 1) or conv.padding != (0, 0): + raise ValueError("only exact 1x1 pointwise Conv2d can be linearized.") + if conv.dilation != (1, 1) or conv.groups != 1: + raise ValueError("grouped or dilated pointwise Conv2d cannot be linearized.") + self.linear = nn.Linear( + conv.in_channels, + conv.out_channels, + bias=conv.bias is not None, + device=conv.weight.device, + dtype=conv.weight.dtype, + ) + with torch.no_grad(): + self.linear.weight.copy_(conv.weight.flatten(1)) + if conv.bias is not None: + self.linear.bias.copy_(conv.bias) + + def forward(self, x: Tensor) -> Tensor: + """Apply the pointwise projection while preserving NCHW layout.""" + if x.dim() != 4: + raise ValueError(f"expected NCHW input, got shape {tuple(x.shape)}.") + batch, _channels, height, width = x.shape + x_nhwc = x.permute(0, 2, 3, 1).reshape(batch * height * width, -1) + y = self.linear(x_nhwc) + return y.reshape(batch, height, width, -1).permute(0, 3, 1, 2).contiguous() + + +def linearize_stage1_ffn_for_quant(module: nn.Module) -> tuple[int, int]: + """Expose Stage-1 pointwise FFN convolutions as Linears for quantization.""" + converted = 0 + skipped = 0 + for child in module.modules(): + if not isinstance(child, GLUMBConvTemp): + continue + for attr in ("inverted_conv", "point_conv"): + pointwise = getattr(child, attr) + if isinstance(pointwise, _LinearizedPointwiseConv2d): + continue + try: + replacement = _LinearizedPointwiseConv2d(pointwise) + except ValueError: + skipped += 1 + continue + replacement.train(pointwise.training) + setattr(child, attr, replacement) + converted += 1 + return converted, skipped + + class Stage1SelfAttention(nn.Module): """Self/camera attention parameter container for one Stage-1 block.""" @@ -751,8 +807,8 @@ def _gdn_key_scale(head_dim: int, HW: tuple[int, int, int]) -> float: return (head_dim**-0.5) * ((HW[1] * HW[2]) ** -0.5) -def _apply_output_gate(out: Tensor, gate_x: Tensor, gate: nn.Linear) -> Tensor: - gate_values = F.silu(F.linear(gate_x, gate.weight, gate.bias).float()) +def _apply_output_gate(out: Tensor, gate_x: Tensor, gate: nn.Module) -> Tensor: + gate_values = F.silu(gate(gate_x).float()) return out * gate_values.to(dtype=out.dtype) @@ -1268,4 +1324,5 @@ def _invert_se3(transforms: Tensor) -> Tensor: "SanaWMStage1Spec", "Stage1CrossAttention", "Stage1SelfAttention", + "linearize_stage1_ffn_for_quant", ] diff --git a/integrations/sana/sana_wm/transformer.py b/integrations/sana/sana_wm/transformer.py index e9e0abd72..0b07b2810 100644 --- a/integrations/sana/sana_wm/transformer.py +++ b/integrations/sana/sana_wm/transformer.py @@ -46,7 +46,7 @@ TorchScaledMMFP8Recipe, replace_linear_with_quant, ) -from sana_wm.stage1_model import SanaWMStage1Model +from sana_wm.stage1_model import SanaWMStage1Model, linearize_stage1_ffn_for_quant Precision = Literal["bf16", "fp8", "fp4"] QuantBackend = Literal["auto", "torch", "torch-fp8", "torch-fp4"] @@ -59,15 +59,13 @@ "^t_block", "^y_embedder", "^final_layer", - # SANA attention code accesses output_gate.weight directly; replacing it - # with a module that only has quantized buffers breaks that call site. - r"\.output_gate$", ) _STAGE1_QUANT_INCLUDE_DEFAULTS = ( r"^blocks\.\d+\.attn\.qkv$", r"^blocks\.\d+\.attn\.proj$", r"^blocks\.\d+\.attn\.beta_proj$", r"^blocks\.\d+\.attn\.gate_proj$", + r"^blocks\.\d+\.attn\.output_gate$", r"^blocks\.\d+\.cross_attn\.", r"^blocks\.\d+\.mlp\.inverted_conv\.linear$", r"^blocks\.\d+\.mlp\.point_conv\.linear$", @@ -418,12 +416,19 @@ def _prepare_stage1_quant(self) -> None: recipe = TorchScaledMMFP8Recipe() else: recipe = TorchScaledMMFP4Recipe() + linearized, linearize_skipped = linearize_stage1_ffn_for_quant(self.model) + if linearized > 0 or linearize_skipped > 0: + logger.info( + "[stage1-quant] linearized {} FFN pointwise convs (skipped {})", + linearized, + linearize_skipped, + ) converted, skipped = replace_linear_with_quant( self.model, recipe=recipe, params_dtype=self._ensure_weight_dtype(), skip_patterns=_STAGE1_QUANT_SKIP_DEFAULTS, - include_patterns=_STAGE1_QUANT_INCLUDE_DEFAULTS, + include_patterns=_stage1_quant_include_patterns(), ) if converted <= 0: raise RuntimeError( @@ -431,9 +436,17 @@ def _prepare_stage1_quant(self) -> None: f"Stage-1 Linear layers; skipped={skipped}." ) self._stage1_quantized = True + recipe_detail = "" + if isinstance(recipe, TorchScaledMMFP4Recipe): + recipe_detail = ( + f" rht={recipe.use_rht}" + f" global_scale={recipe.use_global_scale}" + f" weight_scale_2d={recipe.weight_scale_2d}" + ) logger.info( - "[stage1-quant] precision={} converted {} Linear layers (skipped {})", + "[stage1-quant] precision={}{} converted {} Linear layers (skipped {})", self.config.stage1_precision, + recipe_detail, converted, skipped, ) @@ -441,6 +454,11 @@ def _prepare_stage1_quant(self) -> None: torch.cuda.empty_cache() +def _stage1_quant_include_patterns() -> tuple[str, ...]: + """Return Stage-1 Linear names eligible for FP8/FP4 quantization.""" + return _STAGE1_QUANT_INCLUDE_DEFAULTS + + def _require_conditioning( cache: SanaWMTransformerCache, ) -> SanaWMStage1Conditioning: diff --git a/integrations/sana/tests/test_quant_cuda.py b/integrations/sana/tests/test_quant_cuda.py index 9034c0c7f..ff6265760 100644 --- a/integrations/sana/tests/test_quant_cuda.py +++ b/integrations/sana/tests/test_quant_cuda.py @@ -77,3 +77,35 @@ def test_fp4_linear_runs_scaled_mm_on_blackwell() -> None: assert output.shape == (4, 5, 64) assert output.dtype == torch.bfloat16 assert torch.isfinite(output.float()).all() + + +def test_quantized_linears_roughly_track_source_layer() -> None: + """Catch gross scale/layout regressions beyond expected FP4/FP8 noise.""" + _require_quant_cuda() + major, minor = torch.cuda.get_device_capability() + if major < 10: + pytest.skip(f"NVFP4 requires Blackwell-class CUDA, got sm_{major}{minor}") + + torch.manual_seed(123) + source = torch.nn.Linear(32, 64, bias=True, device="cuda", dtype=torch.bfloat16) + inputs = torch.randn(4, 5, 32, device="cuda", dtype=torch.bfloat16) + reference = source(inputs).float() + + fp8 = TorchScaledMMFP8Linear.from_linear( + source, + out_dtype=torch.bfloat16, + ) + fp4 = TorchScaledMMFP4Linear.from_linear( + source, + out_dtype=torch.bfloat16, + ) + + fp8_error = (fp8(inputs).float() - reference).abs() + fp4_error = (fp4(inputs).float() - reference).abs() + fp8_rel_mae = fp8_error.mean() / reference.abs().mean().clamp_min(1e-6) + fp4_rel_mae = fp4_error.mean() / reference.abs().mean().clamp_min(1e-6) + + assert fp8_error.max().item() <= 0.10 + assert fp8_rel_mae.item() <= 0.06 + assert fp4_error.max().item() <= 0.35 + assert fp4_rel_mae.item() <= 0.20 diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index 7630a625e..e079e56ed 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -75,19 +75,24 @@ SanaWMStage1Conditioning, _avoid_degenerate_tile_tail, _load_inference_config, + _stage1_quant_include_patterns, ) from sana_wm.refiner import SanaWMLTX2Refiner, _pack_latents, _unpack_latents from sana_wm.quant import ( + apply_rht16, + nvfp4_global_scale, TorchScaledMMFP4Linear, TorchScaledMMFP8Linear, replace_linear_with_torch_fp4, replace_linear_with_torch_fp8, ) from sana_wm.stage1_model import ( + GLUMBConvTemp, SANA_WM_STAGE1_SPEC, SanaWMStage1Model, SanaWMStage1Spec, Stage1SelfAttention, + linearize_stage1_ffn_for_quant, ) from flashdreams.infra.config import derive_config @@ -1052,6 +1057,81 @@ def test_runner_defaults_to_bf16_precision() -> None: assert RUNNER_SANA_WM_BIDIRECTIONAL.no_refiner is False +def test_stage1_quant_scope_matches_upstream_precision_cli() -> None: + """Keep FP8 and FP4 on upstream's self-attn + cross-attn + FFN scope.""" + patterns = _stage1_quant_include_patterns() + + assert patterns is not None + assert r"^blocks\.\d+\.attn\.qkv$" in patterns + assert r"^blocks\.\d+\.attn\.output_gate$" in patterns + assert r"^blocks\.\d+\.cross_attn\." in patterns + assert r"^blocks\.\d+\.mlp\.inverted_conv\.linear$" in patterns + assert r"^blocks\.\d+\.mlp\.point_conv\.linear$" in patterns + + +def test_stage1_ffn_linearization_preserves_forward() -> None: + """Expose pointwise FFN convs as Linear modules without changing math.""" + torch.manual_seed(0) + spec = SanaWMStage1Spec( + latent_channels=4, + hidden_size=16, + text_dim=12, + timestep_dim=8, + depth=1, + num_heads=2, + head_dim=8, + max_text_length=5, + latent_grid_size=(2, 2), + mlp_ratio=1, + conv_kernel_size=3, + temporal_kernel_size=3, + plucker_channels=6, + raymap_channels=3, + softmax_every_n=2, + ) + mlp = GLUMBConvTemp(spec).eval() + inputs = torch.randn(1, 8, 16) + reference = mlp(inputs, frames=2, height=2, width=2) + + converted, skipped = linearize_stage1_ffn_for_quant(mlp) + output = mlp(inputs, frames=2, height=2, width=2) + + assert converted == 2 + assert skipped == 0 + assert hasattr(mlp.inverted_conv, "linear") + assert hasattr(mlp.point_conv, "linear") + torch.testing.assert_close(output, reference) + + +def test_fp4_rht16_is_orthogonal() -> None: + """The tiled Hadamard rotation must preserve dot products before quantization.""" + inputs = torch.randn(3, 4, 32) + weights = torch.randn(5, 32) + + rotated_inputs = apply_rht16(inputs) + rotated_weights = apply_rht16(weights) + + torch.testing.assert_close( + rotated_inputs.reshape(-1, 32) @ rotated_weights.t(), + inputs.reshape(-1, 32) @ weights.t(), + atol=1.0e-5, + rtol=1.0e-5, + ) + + +def test_fp4_rht16_rejects_unaligned_last_dimension() -> None: + """NVFP4 RHT is tiled in groups of sixteen values.""" + with pytest.raises(ValueError, match="divisible by 16"): + apply_rht16(torch.randn(2, 15)) + + +def test_fp4_global_scale_uses_nvfp4_dynamic_range() -> None: + """The hierarchical FP4 global scale should target E4M3 x E2M1 range.""" + inputs = torch.tensor([-2688.0, 0.0, 1344.0]) + + assert nvfp4_global_scale(inputs).item() == pytest.approx(1.0) + + def test_refiner_is_flashdreams_owned( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -1381,8 +1461,11 @@ def from_linear( source: torch.nn.Linear, *, out_dtype: torch.dtype, + use_rht: bool = True, + use_global_scale: bool = True, + weight_scale_2d: bool = True, ) -> TorchScaledMMFP4Linear: - del out_dtype + del out_dtype, use_rht, use_global_scale, weight_scale_2d instance = cls.__new__(cls) torch.nn.Module.__init__(instance) instance.in_features = source.in_features From 8c6a6e4c484991d42f705e9e1cbe081eb4776d4c Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 23 Jul 2026 17:06:38 -0700 Subject: [PATCH 22/64] Add benchmarking scripts --- docs/source/models/sana_wm.rst | 5 +- integrations/sana/README.md | 26 +- .../sana/tests/parity_check/.gitignore | 9 + .../parity_check/BENCHMARK_FINALIZATION.md | 61 + .../sana/tests/parity_check/PERF_HANDOFF.md | 68 + .../sana/tests/parity_check/README.md | 97 + integrations/sana/tests/parity_check/bench.sh | 263 +++ .../sana/tests/parity_check/bench_summary.py | 327 ++++ .../tests/parity_check/bench_sweep_summary.py | 128 ++ .../sana/tests/parity_check/changes.patch | 215 +++ .../parity_check/compat/mmcv/__init__.py | 116 ++ .../tests/parity_check/compat/mmcv/runner.py | 35 + .../compat/mmcv/utils/__init__.py | 13 + .../parity_check/compat/mmcv/utils/logging.py | 8 + .../sana/tests/parity_check/diff_parity.py | 79 + .../sana/tests/parity_check/pyproject.toml | 47 + integrations/sana/tests/parity_check/run.sh | 185 ++ .../sana/tests/parity_check/run_native.py | 319 ++++ integrations/sana/tests/parity_check/uv.lock | 1676 +++++++++++++++++ 19 files changed, 3669 insertions(+), 8 deletions(-) create mode 100644 integrations/sana/tests/parity_check/.gitignore create mode 100644 integrations/sana/tests/parity_check/BENCHMARK_FINALIZATION.md create mode 100644 integrations/sana/tests/parity_check/PERF_HANDOFF.md create mode 100644 integrations/sana/tests/parity_check/README.md create mode 100644 integrations/sana/tests/parity_check/bench.sh create mode 100644 integrations/sana/tests/parity_check/bench_summary.py create mode 100644 integrations/sana/tests/parity_check/bench_sweep_summary.py create mode 100644 integrations/sana/tests/parity_check/changes.patch create mode 100644 integrations/sana/tests/parity_check/compat/mmcv/__init__.py create mode 100644 integrations/sana/tests/parity_check/compat/mmcv/runner.py create mode 100644 integrations/sana/tests/parity_check/compat/mmcv/utils/__init__.py create mode 100644 integrations/sana/tests/parity_check/compat/mmcv/utils/logging.py create mode 100644 integrations/sana/tests/parity_check/diff_parity.py create mode 100644 integrations/sana/tests/parity_check/pyproject.toml create mode 100644 integrations/sana/tests/parity_check/run.sh create mode 100644 integrations/sana/tests/parity_check/run_native.py create mode 100644 integrations/sana/tests/parity_check/uv.lock diff --git a/docs/source/models/sana_wm.rst b/docs/source/models/sana_wm.rst index f8cf6bb75..b58ccb795 100644 --- a/docs/source/models/sana_wm.rst +++ b/docs/source/models/sana_wm.rst @@ -65,13 +65,10 @@ Running the method ------------------ To run SANA-WM, launch the ``sana-wm-bidirectional`` runner with a first-frame -image, a prompt, and a camera trajectory. Set -``PYTORCH_ALLOC_CONF=expandable_segments:True`` to keep the allocator from -fragmenting across the Stage-1 + refiner handoff: +image, a prompt, and a camera trajectory: .. code-block:: bash - PYTORCH_ALLOC_CONF=expandable_segments:True \ uv run flashdreams-run sana-wm-bidirectional \ --image-path ../Sana/asset/sana_wm/demo_0.png \ --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ diff --git a/integrations/sana/README.md b/integrations/sana/README.md index 3da123d38..16f64236b 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -21,7 +21,7 @@ Current scope: | Stage-1 BF16 | FlashDreams DiT execution. | | Stage-1 FP8 | PyTorch `_scaled_mm` backend. | | Stage-1 FP4 | Triton quantization plus PyTorch `_scaled_mm`. | -| VAE decode | Direct `diffusers` LTX2 VAE use with low-memory tiling. | +| VAE decode | Direct `diffusers` LTX2 VAE use with upstream-matched tiling. | | LTX-2 refiner | Direct `diffusers` LTX-2 transformer and Gemma connector use. | ## Runner @@ -48,7 +48,6 @@ well. ## Run ```bash -PYTORCH_ALLOC_CONF=expandable_segments:True \ uv run flashdreams-run sana-wm-bidirectional \ --image-path ../Sana/asset/sana_wm/demo_0.png \ --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ @@ -106,7 +105,6 @@ coherent. FP8 smoke: ```bash -PYTORCH_ALLOC_CONF=expandable_segments:True \ uv run flashdreams-run sana-wm-bidirectional \ --image-path ../Sana/asset/sana_wm/demo_0.png \ --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ @@ -121,7 +119,6 @@ uv run flashdreams-run sana-wm-bidirectional \ FP4 smoke: ```bash -PYTORCH_ALLOC_CONF=expandable_segments:True \ uv run flashdreams-run sana-wm-bidirectional \ --image-path ../Sana/asset/sana_wm/demo_0.png \ --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ @@ -133,6 +130,27 @@ uv run flashdreams-run sana-wm-bidirectional \ --refiner-precision fp4 ``` +## Parity and benchmark + +The upstream comparison harness lives under `tests/parity_check/`, matching the +pattern used by the other benchmarked FlashDreams integrations. + +```bash +cd integrations/sana/tests/parity_check + +# Upstream + FlashDreams parity artifacts and frame diff. +bash run.sh + +# Matched upstream-vs-FlashDreams benchmark report. +DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh +``` + +`bench.sh` writes `outputs/bench/bench.md` for review and `outputs/bench/perf.md` +for model-card chart data. Both use the same benchmark metric: post-load +generation latency per generated frame. Stage-1 DiT, conditioning/encode, VAE +decode, optional refiner, and memory rows are reported as the breakdown of that +metric. + ## Tests CPU-safe tests cover import, config boundaries, action parsing, intrinsics, diff --git a/integrations/sana/tests/parity_check/.gitignore b/integrations/sana/tests/parity_check/.gitignore new file mode 100644 index 000000000..2df75f55e --- /dev/null +++ b/integrations/sana/tests/parity_check/.gitignore @@ -0,0 +1,9 @@ +/Sana/ +/.venv/ +/outputs/ +__pycache__/ +*.pyc +*.npy +*.mp4 +*.json +*.log diff --git a/integrations/sana/tests/parity_check/BENCHMARK_FINALIZATION.md b/integrations/sana/tests/parity_check/BENCHMARK_FINALIZATION.md new file mode 100644 index 000000000..d6c6f2654 --- /dev/null +++ b/integrations/sana/tests/parity_check/BENCHMARK_FINALIZATION.md @@ -0,0 +1,61 @@ + + +# SANA-WM Benchmark Finalization + +Use this after the perf optimization pass is done and the final BF16, FP8, and +FP4 numbers are ready for the model card. + +1. Run the precision sweep: + + ```bash + cd integrations/sana/tests/parity_check + DEVICE_LABEL="" BENCH_PRECISIONS=bf16,fp8,fp4 bash bench.sh + ``` + +2. Copy `outputs/bench/perf.md` to + `docs/source/_static/performance/sana_wm/perf-.md`. + +3. Add a `Profiling benchmark` section to `docs/source/models/sana_wm.rst` + between `What to expect` and `Citation`. + +4. The section should describe post-load generation latency per generated frame + for FlashDreams SANA-WM versus the official `NVlabs/Sana` implementation + under matched BF16, FP8, and FP4 settings. + +5. Use the standard benchmark chart block: + + ```rst + Profiling benchmark + ------------------- + + Here is the profiling benchmark on post-load generation latency per generated + frame for FlashDreams SANA-WM compared to the + `official SANA-WM implementation `_ under + matched BF16, FP8, and FP4 settings. + + .. raw:: html + +
+
+
+

+ This chart shows post-load generation latency per generated frame in milliseconds on a single GPU. + For the official SANA-WM implementation, see + this instruction. +

+
+
+ + ``` + +Keep the chart metric unchanged: post-load generation latency per generated +frame, in ms/frame. diff --git a/integrations/sana/tests/parity_check/PERF_HANDOFF.md b/integrations/sana/tests/parity_check/PERF_HANDOFF.md new file mode 100644 index 000000000..aaed1d654 --- /dev/null +++ b/integrations/sana/tests/parity_check/PERF_HANDOFF.md @@ -0,0 +1,68 @@ + + +# SANA-WM Perf Handoff + +This note is only for the next SANA-WM performance investigation. It should not +be copied into the model card or public README. + +## Observed Delta + +The best current BF16, no-refiner, 121-frame, 60-step run still has +FlashDreams slower than upstream: + +| artifact | official | FlashDreams | gap | +| --- | ---: | ---: | ---: | +| `outputs/bf16_step60_bidirectional_default/bench.md` | 640.89 ms/frame | 663.60 ms/frame | +22.71 ms/frame | + +Component medians from that run: + +| component | official | FlashDreams | +| --- | ---: | ---: | +| wall | 77.55 s | 80.30 s | +| Stage-1 DiT | 75,565.73 ms | 78,237.04 ms | +| VAE decode row | 1,340.43 ms | 1,584.82 ms | +| peak memory | 14.37 GiB | 17.98 GiB | + +The main target is the 60-step Stage-1 DiT path. The decode row also still has +a smaller gap. + +## Perf Theories Tested + +| theory | result | evidence | +| --- | --- | --- | +| FlashDreams was slower because VAE tiling used low-memory tiles. | Partly right. | Early probes showed FlashDreams VAE decode around 5.2 s. Switching to upstream-style tiles dropped it to about 1.5 s. This fixed a large initial problem, but not the remaining 60-step gap. | +| Remaining VAE gap is mostly raw VAE forward kernels. | Probably wrong. | Later internal VAE timing was close to upstream, while the harness decode row remained slower. The remaining cost is likely after the VAE forward: clamp, permute, CPU transfer, numpy materialization, or `torch.cuda.empty_cache()`. | +| Matching upstream's bidirectional chunk default would close the gap. | Correct for config parity, insufficient for speed. | `SanaWMStage1Spec.chunk_size=None` with `chunk_split_strategy="first_chunk_plus_one"` matches upstream. The 8-step probe became slightly favorable to FlashDreams, but the 60-step run stayed slower. | +| Softmax camera Q/K/V transforms staying in higher precision were slowing blocks. | Right. | After casting transformed softmax-camera Q/K/V back to BF16, targeted one-step block timings dropped from roughly 106-107 ms to roughly 49-52 ms for those blocks. This change is kept. | +| Forcing cuDNN SDPA would make the default comparison faster. | Wrong. | The forced-cuDNN probe was much slower and used about 80 GiB. Keep `FORCE_CUDNN_SDPA=1` only as a backend-isolation probe. | +| `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` helps this path. | Unproven. | No run showed a speed or memory benefit for SANA-WM. The harness and docs do not set allocator overrides. | +| Direct SDPA on head dim 112 is faster than padding to 128. | Not supported by clean evidence. | The direct-D112 probe had suspicious backend and memory behavior. Current code keeps padding to 128. | +| `torch.inference_mode()` could reduce framework overhead versus `no_grad`. | Untested. | Upstream wraps generation with `@torch.inference_mode()`. FlashDreams framework generation uses `@torch.no_grad()`. A harness-only test was started, then reverted before measurement. | + +## Next Probes + +1. Reproduce the 60-step BF16 no-refiner benchmark once before changing code, + using `bench.sh`, to confirm the current gap on the current branch. +2. Profile Stage-1 DiT without using profiler-perturbed timings as headline + numbers. Focus on block-level attention, cross-attention, FFN, camera QKV, + and Python/framework overhead. +3. Test `torch.inference_mode()` around the FlashDreams generation path and + compare against the same command with `no_grad`. +4. Split FlashDreams decode timing into VAE forward, clamp, permute, CPU copy, + numpy conversion, and `empty_cache`. +5. Test disabling or configuring `torch.cuda.empty_cache()` after decode. The + optimization goal allows spending more memory for speed. + +## Local Provenance + +These paths are under ignored `outputs/` directories and may not exist in a +fresh checkout: + +- `outputs/bf16_step60_bidirectional_default/bench.md` +- `outputs/bf16_step8_bidirectional_default/bench.md` +- `outputs/bf16_step60_softmax_bf16/bench.md` +- `outputs/bench_121_noalloc/bench.md` +- `outputs/fd_profile_step1_softmax_bf16/stats.json` diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md new file mode 100644 index 000000000..d0119a828 --- /dev/null +++ b/integrations/sana/tests/parity_check/README.md @@ -0,0 +1,97 @@ + + +# SANA-WM parity and benchmark harness + +This harness compares upstream +[`NVlabs/Sana`](https://github.com/NVlabs/Sana) SANA-WM against the in-tree +FlashDreams `sana-wm-bidirectional` integration on the same demo image, prompt, +camera trajectory, intrinsics, seed, resolution, and precision settings. + +All dependencies live in this directory's isolated `./.venv`. `run.sh` reuses or +clones the upstream checkout, pins it to `6298508`, and applies `changes.patch` +idempotently. The patch is instrumentation-only: it adds upstream CLI flags for +frame dumps, stats JSON, precision selection, cuDNN SDPA selection, and Stage-1 +`torch.compile` without changing the generation algorithm. A small +`compat/mmcv` shim covers the registry/logging imports needed by the inference +path. + +## Run parity + +```bash +cd integrations/sana/tests/parity_check +bash run.sh +``` + +Defaults: + +- `SANA_REPO=$HOME/dev/Sana` +- upstream pin `6298508` +- demo input `asset/sana_wm/demo_0.*` +- `NUM_FRAMES=121` +- `SEED=42` +- `NO_REFINER=1` to isolate Stage-1 plus SANA VAE decode +- `STAGE1_PRECISION=bf16` +- `REFINER_PRECISION=$STAGE1_PRECISION` +- `QUANT_BACKEND=auto` +- `FORCE_CUDNN_SDPA=0` + +Outputs are written under `outputs/parity/`: + +- `upstream/frames.npy` +- `upstream/stats.json` +- `flashdreams/frames.npy` +- `flashdreams/stats.json` +- `parity.json` + +Set `NO_REFINER=0` to compare the full Stage-1 + LTX-2 refiner path. Set +`COMPILE_STAGE1=1` to wrap each Stage-1 DiT with `torch.compile`; this is opt-in +because the pinned upstream SANA-WM stack can fail during TorchInductor Triton +compilation on current PyTorch/Triton builds. + +## Run benchmark + +```bash +cd integrations/sana/tests/parity_check +bash bench.sh +``` + +Benchmark defaults discard one warmup run and measure three additional runs: +`WARMUP_RUNS=1 MEASURED_RUNS=3 COMPILE_STAGE1=0 NO_REFINER=1 FORCE_CUDNN_SDPA=0`. +Outputs are under `outputs/bench/`: + +- `bench.json` - machine-readable inputs, medians, p90s, memory, and stage + timings. +- `bench.md` - human-readable report. +- `perf.md` - chart-ready model-card data using the same benchmark metric as + `bench.md`. + +The benchmark metric is post-load generation latency per generated frame. Model +construction, checkpoint loading, video writing, and frame dumps are outside the +timing boundary. With the default `NO_REFINER=1`, the timed work covers +conditioning, Stage-1 DiT, and SANA VAE decode. Set `NO_REFINER=0` to benchmark +the full Stage-1 + LTX-2 refiner path with the same timing boundary. + +`bench.md` also reports Stage-1 DiT, conditioning/encode, VAE decode, optional +refiner, and memory breakdowns. Those rows explain the benchmark result; they +are not a second benchmark metric. + +Set `FORCE_CUDNN_SDPA=1` only for backend-isolation probes. It is not the +default SANA-WM benchmark setting. + +Set `DEVICE_LABEL` when generating chart data for docs: + +```bash +DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh +``` + +Run the precision sweep used by the SANA-WM model card with: + +```bash +DEVICE_LABEL="RTX PRO 6000 Blackwell" BENCH_PRECISIONS=bf16,fp8,fp4 bash bench.sh +``` + +The scripts do not set allocator overrides or GPU wait loops. If GPU contention +matters in your environment, handle it outside the harness. diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh new file mode 100644 index 000000000..2769d7e59 --- /dev/null +++ b/integrations/sana/tests/parity_check/bench.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Stack-matched SANA-WM upstream vs FlashDreams benchmark. The chart metric is +# post-load generation latency per generated frame. Defaults to Stage-1-only so +# conditioning, Stage-1 DiT, and SANA VAE decode are measured without the LTX-2 +# refiner. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +_abspath() { + case "$1" in + /*) printf '%s\n' "$1" ;; + *) printf '%s/%s\n' "${PWD}" "$1" ;; + esac +} + +SANA_REPO="$(_abspath "${SANA_REPO:-${HOME}/dev/Sana}")" +PATCH_FILE="${SCRIPT_DIR}/changes.patch" +REPO_URL="https://github.com/NVlabs/Sana.git" +PIN_COMMIT="6298508" + +OUTPUT_DIR="$(_abspath "${OUTPUT_DIR:-${SCRIPT_DIR}/outputs/bench}")" +UPSTREAM_ROOT="${OUTPUT_DIR}/upstream" +NATIVE_ROOT="${OUTPUT_DIR}/flashdreams" +IMAGE_PATH="$(_abspath "${IMAGE_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.png}")" +PROMPT_PATH="$(_abspath "${PROMPT_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.txt}")" +CAMERA_PATH="$(_abspath "${CAMERA_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_pose.npy}")" +INTRINSICS_PATH="$(_abspath "${INTRINSICS_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_intrinsics.npy}")" +NUM_FRAMES="${NUM_FRAMES:-121}" +FPS="${FPS:-16}" +STEP="${STEP:-60}" +CFG_SCALE="${CFG_SCALE:-5.0}" +SEED="${SEED:-42}" +NO_REFINER="${NO_REFINER:-1}" +FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" +COMPILE_STAGE1="${COMPILE_STAGE1:-0}" +WARMUP_RUNS="${WARMUP_RUNS:-1}" +MEASURED_RUNS="${MEASURED_RUNS:-3}" +DEVICE_LABEL="${DEVICE_LABEL:-GPU}" +STAGE1_PRECISION="${STAGE1_PRECISION:-bf16}" +REFINER_PRECISION="${REFINER_PRECISION:-${STAGE1_PRECISION}}" +QUANT_BACKEND="${QUANT_BACKEND:-auto}" +CHART_LABEL="${CHART_LABEL:-${DEVICE_LABEL}}" +BENCH_PRECISIONS="${BENCH_PRECISIONS:-}" + +_is_true() { + case "${1,,}" in + 1|true|yes|on) return 0 ;; + *) return 1 ;; + esac +} + +if [[ -n "${BENCH_PRECISIONS}" ]]; then + mkdir -p "${OUTPUT_DIR}" + SWEEP_ITEMS=() + IFS=',' read -r -a PRECISION_LIST <<< "${BENCH_PRECISIONS}" + for PRECISION_RAW in "${PRECISION_LIST[@]}"; do + PRECISION="${PRECISION_RAW//[[:space:]]/}" + if [[ -z "${PRECISION}" ]]; then + continue + fi + case "${PRECISION}" in + bf16|fp8|fp4) ;; + *) + echo "[bench] ERROR: unsupported BENCH_PRECISIONS entry: ${PRECISION}" >&2 + exit 1 + ;; + esac + PRECISION_LABEL="${PRECISION^^}" + PRECISION_OUTPUT_DIR="${OUTPUT_DIR}/${PRECISION}" + echo "[bench] precision sweep row ${PRECISION_LABEL} -> ${PRECISION_OUTPUT_DIR}" + BENCH_PRECISIONS="" \ + OUTPUT_DIR="${PRECISION_OUTPUT_DIR}" \ + STAGE1_PRECISION="${PRECISION}" \ + REFINER_PRECISION="${PRECISION}" \ + QUANT_BACKEND="${QUANT_BACKEND}" \ + CHART_LABEL="${PRECISION_LABEL}" \ + bash "${SCRIPT_DIR}/bench.sh" + SWEEP_ITEMS+=(--item "${PRECISION_LABEL}:${PRECISION_OUTPUT_DIR}/bench.json") + done + if [[ "${#SWEEP_ITEMS[@]}" -eq 0 ]]; then + echo "[bench] ERROR: BENCH_PRECISIONS produced no precision rows." >&2 + exit 1 + fi + echo "[bench] aggregating precision sweep -> ${OUTPUT_DIR}/perf.md" + ( cd "${SCRIPT_DIR}" && \ + uv run python "${SCRIPT_DIR}/bench_sweep_summary.py" \ + "${SWEEP_ITEMS[@]}" \ + --output-json "${OUTPUT_DIR}/bench.json" \ + --output-md "${OUTPUT_DIR}/bench.md" \ + --output-chart-md "${OUTPUT_DIR}/perf.md" ) + echo "[bench] done." + echo " summary: ${OUTPUT_DIR}/bench.md" + echo " chart data: ${OUTPUT_DIR}/perf.md" + exit 0 +fi + +if [[ ! -d "${SANA_REPO}/.git" ]]; then + echo "[setup] cloning ${REPO_URL} -> ${SANA_REPO}" + git clone "${REPO_URL}" "${SANA_REPO}" +else + echo "[setup] repo already present at ${SANA_REPO}, skipping clone" +fi + +cd "${SANA_REPO}" +CURRENT_COMMIT="$(git rev-parse --short HEAD)" +if [[ "${CURRENT_COMMIT}" != "${PIN_COMMIT}" ]]; then + echo "[setup] checking out pinned commit ${PIN_COMMIT}" + git checkout "${PIN_COMMIT}" +else + echo "[setup] already at pinned commit ${PIN_COMMIT}, skipping checkout" +fi + +if git apply --reverse --check "${PATCH_FILE}" >/dev/null 2>&1; then + echo "[setup] patch already applied, skipping" +elif git apply --check "${PATCH_FILE}" >/dev/null 2>&1; then + echo "[setup] applying ${PATCH_FILE}" + git apply "${PATCH_FILE}" +else + echo "[setup] ERROR: ${PATCH_FILE} neither cleanly applies nor is already applied." >&2 + exit 1 +fi + +echo "[setup] ensuring Python deps via uv sync (isolated venv)" +( cd "${SCRIPT_DIR}" && uv sync ) +UPSTREAM_PYTHONPATH="${SCRIPT_DIR}/compat:${SANA_REPO}" +if [[ -n "${PYTHONPATH:-}" ]]; then + UPSTREAM_PYTHONPATH="${UPSTREAM_PYTHONPATH}:${PYTHONPATH}" +fi + +mkdir -p "${UPSTREAM_ROOT}" "${NATIVE_ROOT}" + +UPSTREAM_REFINER_ARGS=() +NATIVE_REFINER_ARGS=() +if _is_true "${NO_REFINER}"; then + UPSTREAM_REFINER_ARGS+=(--no_refiner) + NATIVE_REFINER_ARGS+=(--no-refiner) +fi +UPSTREAM_BACKEND_ARGS=() +NATIVE_BACKEND_ARGS=() +if _is_true "${FORCE_CUDNN_SDPA}"; then + UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) + NATIVE_BACKEND_ARGS+=(--force-cudnn-sdpa) +fi +UPSTREAM_COMPILE_ARGS=() +NATIVE_COMPILE_ARGS=() +if _is_true "${COMPILE_STAGE1}"; then + UPSTREAM_COMPILE_ARGS+=(--compile_stage1) + NATIVE_COMPILE_ARGS+=(--compile-stage1) +fi +UPSTREAM_PRECISION_ARGS=( + --stage1_precision "${STAGE1_PRECISION}" + --refiner_precision "${REFINER_PRECISION}" +) +NATIVE_PRECISION_ARGS=( + --stage1-precision "${STAGE1_PRECISION}" + --refiner-precision "${REFINER_PRECISION}" + --quant-backend "${QUANT_BACKEND}" +) + +TOTAL_RUNS=$(( WARMUP_RUNS + MEASURED_RUNS )) +for ((i = 0; i < TOTAL_RUNS; i++)); do + UPSTREAM_OUT="${UPSTREAM_ROOT}/run_${i}" + NATIVE_OUT="${NATIVE_ROOT}/run_${i}" + mkdir -p "${UPSTREAM_OUT}" "${NATIVE_OUT}" + + echo "[bench] upstream run ${i}/${TOTAL_RUNS}" + ( cd "${SCRIPT_DIR}" && \ + PYTHONPATH="${UPSTREAM_PYTHONPATH}" \ + uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm.py" \ + --image "${IMAGE_PATH}" \ + --prompt "${PROMPT_PATH}" \ + --camera "${CAMERA_PATH}" \ + --intrinsics "${INTRINSICS_PATH}" \ + --output_dir "${UPSTREAM_OUT}" \ + --name upstream \ + --num_frames "${NUM_FRAMES}" \ + --fps "${FPS}" \ + --step "${STEP}" \ + --cfg_scale "${CFG_SCALE}" \ + --seed "${SEED}" \ + --no_action_overlay \ + --stats_json "${UPSTREAM_OUT}/stats.json" \ + "${UPSTREAM_PRECISION_ARGS[@]}" \ + "${UPSTREAM_BACKEND_ARGS[@]}" \ + "${UPSTREAM_COMPILE_ARGS[@]}" \ + "${UPSTREAM_REFINER_ARGS[@]}" ) + + echo "[bench] FlashDreams run ${i}/${TOTAL_RUNS}" + ( cd "${SCRIPT_DIR}" && \ + uv run python "${SCRIPT_DIR}/run_native.py" \ + --image-path "${IMAGE_PATH}" \ + --prompt-path "${PROMPT_PATH}" \ + --camera-path "${CAMERA_PATH}" \ + --intrinsics-path "${INTRINSICS_PATH}" \ + --output-dir "${NATIVE_OUT}" \ + --name flashdreams \ + --num-frames "${NUM_FRAMES}" \ + --fps "${FPS}" \ + --step "${STEP}" \ + --cfg-scale "${CFG_SCALE}" \ + --seed "${SEED}" \ + --stats-json "${NATIVE_OUT}/stats.json" \ + "${NATIVE_PRECISION_ARGS[@]}" \ + "${NATIVE_BACKEND_ARGS[@]}" \ + "${NATIVE_COMPILE_ARGS[@]}" \ + "${NATIVE_REFINER_ARGS[@]}" ) +done + +SUMMARY_JSON="${OUTPUT_DIR}/bench.json" +SUMMARY_MD="${OUTPUT_DIR}/bench.md" +SUMMARY_CHART_MD="${OUTPUT_DIR}/perf.md" +SUMMARY_FLAGS=() +if _is_true "${NO_REFINER}"; then + SUMMARY_FLAGS+=(--no-refiner) +fi +if _is_true "${COMPILE_STAGE1}"; then + SUMMARY_FLAGS+=(--compile-stage1) +fi +if _is_true "${FORCE_CUDNN_SDPA}"; then + SUMMARY_FLAGS+=(--force-cudnn-sdpa) +fi +echo "[bench] summarising -> ${SUMMARY_MD}" +( cd "${SCRIPT_DIR}" && \ + uv run python "${SCRIPT_DIR}/bench_summary.py" \ + --upstream-dir "${UPSTREAM_ROOT}" \ + --flashdreams-dir "${NATIVE_ROOT}" \ + --warmup-runs "${WARMUP_RUNS}" \ + --image-path "${IMAGE_PATH}" \ + --prompt-path "${PROMPT_PATH}" \ + --camera-path "${CAMERA_PATH}" \ + --intrinsics-path "${INTRINSICS_PATH}" \ + --num-frames "${NUM_FRAMES}" \ + --seed "${SEED}" \ + --device-label "${DEVICE_LABEL}" \ + --chart-label "${CHART_LABEL}" \ + --stage1-precision "${STAGE1_PRECISION}" \ + --refiner-precision "${REFINER_PRECISION}" \ + --quant-backend "${QUANT_BACKEND}" \ + "${SUMMARY_FLAGS[@]}" \ + --output-json "${SUMMARY_JSON}" \ + --output-md "${SUMMARY_MD}" \ + --output-chart-md "${SUMMARY_CHART_MD}" ) + +echo "[bench] done." +echo " summary: ${SUMMARY_MD}" +echo " chart data: ${SUMMARY_CHART_MD}" diff --git a/integrations/sana/tests/parity_check/bench_summary.py b/integrations/sana/tests/parity_check/bench_summary.py new file mode 100644 index 000000000..c0d8bbf22 --- /dev/null +++ b/integrations/sana/tests/parity_check/bench_summary.py @@ -0,0 +1,327 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Summarize SANA-WM upstream and FlashDreams benchmark stats.""" + +from __future__ import annotations + +import argparse +import json +import statistics +from pathlib import Path +from typing import Any + + +def _load_stats(root: Path) -> list[dict[str, Any]]: + paths = sorted(root.glob("run_*/stats.json")) + if not paths and (root / "stats.json").exists(): + paths = [root / "stats.json"] + return [json.loads(path.read_text(encoding="utf-8")) | {"_path": str(path)} for path in paths] + + +def _median(values: list[float]) -> float | None: + return statistics.median(values) if values else None + + +def _p90(values: list[float]) -> float | None: + if not values: + return None + ordered = sorted(values) + index = max(0, min(len(ordered) - 1, round(0.9 * (len(ordered) - 1)))) + return ordered[index] + + +def _upstream_stage_ms(item: dict[str, Any], key: str) -> float | None: + timings = item.get("timings_s") + if not isinstance(timings, dict): + return None + for candidate in key.split("|"): + value = timings.get(candidate) + if isinstance(value, (int, float)): + return float(value) * 1000.0 + return None + + +def _native_stage_ms(item: dict[str, Any], key: str) -> float | None: + stats = item.get("stats_ms") + if not isinstance(stats, dict): + return None + value = stats.get(key) + return float(value) if isinstance(value, (int, float)) else None + + +def _upstream_mem_gib(item: dict[str, Any]) -> float | None: + value = item.get("mem_peak_gib") + return float(value) if isinstance(value, (int, float)) else None + + +def _native_mem_gib(item: dict[str, Any]) -> float | None: + stats = item.get("stats_ms") + if not isinstance(stats, dict): + return None + value = stats.get("mem_peak_gib") + return float(value) if isinstance(value, (int, float)) else None + + +def _collect( + items: list[dict[str, Any]], + warmup_runs: int, + stage_reader, + mem_reader, + stages: dict[str, str], +) -> dict[str, Any]: + kept = items[warmup_runs:] + rows: dict[str, Any] = { + "runs_total": len(items), + "warmup_runs": warmup_runs, + "runs_measured": len(kept), + "paths": [item.get("_path") for item in kept], + } + wall = [ + float(item["wall_s"]) + for item in kept + if isinstance(item.get("wall_s"), (int, float)) + ] + rows["wall_median_s"] = _median(wall) + rows["wall_p90_s"] = _p90(wall) + memory = [ + value + for item in kept + if (value := mem_reader(item)) is not None + ] + rows["mem_peak_median_gib"] = _median(memory) + rows["mem_peak_p90_gib"] = _p90(memory) + for label, key in stages.items(): + values = [ + value + for item in kept + if (value := stage_reader(item, key)) is not None + ] + rows[f"{label}_median_ms"] = _median(values) + rows[f"{label}_p90_ms"] = _p90(values) + return rows + + +def _fmt(value: Any, suffix: str = "") -> str: + if value is None: + return "n/a" + if isinstance(value, float): + return f"{value:.2f}{suffix}" + return f"{value}{suffix}" + + +def _ms_per_frame(wall_s: float | None, num_frames: int) -> float | None: + if wall_s is None or num_frames <= 0: + return None + return wall_s * 1000.0 / num_frames + + +def _metric_value(summary: dict[str, Any], side: str, key: str) -> float | None: + value = summary[side].get(key) + return float(value) if isinstance(value, (int, float)) else None + + +def _generation_ms_per_frame( + summary: dict[str, Any], + side: str, + *, + percentile: str = "median", +) -> float | None: + num_frames = int(summary["inputs"]["num_frames"]) + return _ms_per_frame( + _metric_value(summary, side, f"wall_{percentile}_s"), + num_frames, + ) + + +def _sum_optional(*values: float | None) -> float | None: + if any(value is None for value in values): + return None + return sum(float(value) for value in values) + + +def _render_markdown(summary: dict[str, Any]) -> str: + upstream = summary["upstream"] + native = summary["flashdreams"] + rows = [ + "# SANA-WM parity harness benchmark", + "", + "## Inputs", + "", + f"- image: `{summary['inputs']['image_path']}`", + f"- prompt: `{summary['inputs']['prompt_path']}`", + f"- camera: `{summary['inputs']['camera_path']}`", + f"- intrinsics: `{summary['inputs']['intrinsics_path']}`", + f"- num_frames: `{summary['inputs']['num_frames']}`", + f"- seed: `{summary['inputs']['seed']}`", + f"- no_refiner: `{summary['inputs']['no_refiner']}`", + f"- stage1_precision: `{summary['inputs']['stage1_precision']}`", + f"- refiner_precision: `{summary['inputs']['refiner_precision']}`", + f"- quant_backend: `{summary['inputs']['quant_backend']}`", + f"- compile_stage1: `{summary['inputs']['compile_stage1']}`", + f"- force_cudnn_sdpa: `{summary['inputs']['force_cudnn_sdpa']}`", + f"- warmup runs discarded: `{summary['inputs']['warmup_runs']}`", + "", + "## Benchmark metric", + "", + "The chart metric is post-load generation latency per generated frame.", + "Model construction, checkpoint loading, video writing, and frame dumps are outside this timing boundary.", + "With the default `NO_REFINER=1`, the timed work covers conditioning, Stage-1 DiT, and SANA VAE decode.", + "", + "| metric | upstream | FlashDreams |", + "| --- | ---: | ---: |", + f"| measured runs | {upstream['runs_measured']} | {native['runs_measured']} |", + f"| generation median / frame | {_fmt(_generation_ms_per_frame(summary, 'upstream'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams'), ' ms')} |", + f"| generation p90 / frame | {_fmt(_generation_ms_per_frame(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams', percentile='p90'), ' ms')} |", + f"| wall median | {_fmt(upstream['wall_median_s'], ' s')} | {_fmt(native['wall_median_s'], ' s')} |", + f"| wall p90 | {_fmt(upstream['wall_p90_s'], ' s')} | {_fmt(native['wall_p90_s'], ' s')} |", + "", + "## Timing breakdown", + "", + "| stage | upstream median | FlashDreams median |", + "| --- | ---: | ---: |", + f"| Stage-1 incl. conditioning median | {_fmt(upstream['stage1_total_median_ms'], ' ms')} | {_fmt(_sum_optional(native['encode_median_ms'], native['dit_median_ms']), ' ms')} |", + f"| Stage-1 DiT median | {_fmt(upstream['dit_median_ms'], ' ms')} | {_fmt(native['dit_median_ms'], ' ms')} |", + f"| conditioning/encode median | n/a | {_fmt(native['encode_median_ms'], ' ms')} |", + f"| VAE decode median | {_fmt(upstream['vae_decode_median_ms'], ' ms')} | {_fmt(native['vae_decode_median_ms'], ' ms')} |", + f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} | {_fmt(native['mem_peak_median_gib'], ' GiB')} |", + ] + if upstream.get("refiner_median_ms") is not None: + rows.extend( + [ + f"| refiner median | {_fmt(upstream['refiner_median_ms'], ' ms')} | n/a |", + ] + ) + rows.append("") + return "\n".join(rows) + + +def _render_chart_markdown(summary: dict[str, Any], device_label: str) -> str: + official = _generation_ms_per_frame(summary, "upstream") + flashdreams = _generation_ms_per_frame(summary, "flashdreams") + if official is None or flashdreams is None: + raise ValueError("cannot render chart data without wall-clock stats") + return "\n".join( + [ + "# SANA-WM Benchmark Data (ms/frame)", + "", + "| device | official | flashdreams |", + "| --- | ---: | ---: |", + f"| {device_label} | {official:.2f} | {flashdreams:.2f} |", + "", + ] + ) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--upstream-dir", type=Path, required=True) + parser.add_argument("--flashdreams-dir", type=Path, required=True) + parser.add_argument("--warmup-runs", type=int, default=1) + parser.add_argument("--image-path", type=Path, required=True) + parser.add_argument("--prompt-path", type=Path, required=True) + parser.add_argument("--camera-path", type=Path, required=True) + parser.add_argument("--intrinsics-path", type=Path, required=True) + parser.add_argument("--num-frames", type=int, required=True) + parser.add_argument("--seed", type=int, required=True) + parser.add_argument("--no-refiner", action="store_true") + parser.add_argument("--stage1-precision", choices=["bf16", "fp8", "fp4"], default="bf16") + parser.add_argument("--refiner-precision", choices=["bf16", "fp8", "fp4"], default="bf16") + parser.add_argument( + "--quant-backend", + choices=["auto", "torch", "torch-fp8", "torch-fp4"], + default="auto", + ) + parser.add_argument("--compile-stage1", action="store_true") + parser.add_argument("--force-cudnn-sdpa", action="store_true") + parser.add_argument("--device-label", default="GPU") + parser.add_argument("--chart-label", default=None) + parser.add_argument("--output-json", type=Path, required=True) + parser.add_argument("--output-md", type=Path, required=True) + parser.add_argument("--output-chart-md", type=Path, default=None) + args = parser.parse_args(argv) + + upstream_items = _load_stats(args.upstream_dir) + native_items = _load_stats(args.flashdreams_dir) + summary = { + "inputs": { + "image_path": str(args.image_path), + "prompt_path": str(args.prompt_path), + "camera_path": str(args.camera_path), + "intrinsics_path": str(args.intrinsics_path), + "num_frames": args.num_frames, + "seed": args.seed, + "no_refiner": args.no_refiner, + "stage1_precision": args.stage1_precision, + "refiner_precision": args.refiner_precision, + "quant_backend": args.quant_backend, + "compile_stage1": args.compile_stage1, + "force_cudnn_sdpa": args.force_cudnn_sdpa, + "warmup_runs": args.warmup_runs, + "device_label": args.device_label, + "chart_label": args.chart_label or args.device_label, + }, + "upstream": _collect( + upstream_items, + args.warmup_runs, + _upstream_stage_ms, + _upstream_mem_gib, + { + "dit": "stage1_dit_s|stage1_sample_s", + "stage1_total": "stage1_sample_s", + "refiner": "refiner_s", + "vae_decode": "vae_decode_s", + }, + ), + "flashdreams": _collect( + native_items, + args.warmup_runs, + _native_stage_ms, + _native_mem_gib, + { + "encode": "encode_ms", + "dit": "diffuse_ms", + "vae_decode": "decode_ms", + }, + ), + } + summary["benchmark"] = { + "metric": "generation_ms_per_frame", + "unit": "ms/frame", + "timing_boundary": ( + "pipeline.generate after model setup; excludes model construction, " + "checkpoint loading, video writing, and frame dumps" + ), + "device_label": args.device_label, + "chart_label": args.chart_label or args.device_label, + "official": _generation_ms_per_frame(summary, "upstream"), + "flashdreams": _generation_ms_per_frame(summary, "flashdreams"), + } + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + args.output_md.parent.mkdir(parents=True, exist_ok=True) + report = _render_markdown(summary) + args.output_md.write_text(report, encoding="utf-8") + if args.output_chart_md is not None: + args.output_chart_md.parent.mkdir(parents=True, exist_ok=True) + args.output_chart_md.write_text( + _render_chart_markdown(summary, args.chart_label or args.device_label), + encoding="utf-8", + ) + print(report) + + +if __name__ == "__main__": + main() diff --git a/integrations/sana/tests/parity_check/bench_sweep_summary.py b/integrations/sana/tests/parity_check/bench_sweep_summary.py new file mode 100644 index 000000000..c2343325f --- /dev/null +++ b/integrations/sana/tests/parity_check/bench_sweep_summary.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Aggregate SANA-WM precision benchmark summaries into chart data.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +def _load_item(raw: str) -> dict[str, Any]: + if ":" not in raw: + raise argparse.ArgumentTypeError( + f"expected LABEL:/path/to/bench.json, got {raw!r}" + ) + label, path_raw = raw.split(":", 1) + label = label.strip() + path = Path(path_raw) + if not label: + raise argparse.ArgumentTypeError(f"empty label in {raw!r}") + if not path.exists(): + raise argparse.ArgumentTypeError(f"bench summary does not exist: {path}") + summary = json.loads(path.read_text(encoding="utf-8")) + benchmark = summary.get("benchmark") + if not isinstance(benchmark, dict): + raise argparse.ArgumentTypeError(f"{path} has no benchmark object") + official = benchmark.get("official") + flashdreams = benchmark.get("flashdreams") + if not isinstance(official, (int, float)) or not isinstance( + flashdreams, (int, float) + ): + raise argparse.ArgumentTypeError( + f"{path} benchmark must contain numeric official and flashdreams values" + ) + return { + "label": label, + "path": str(path), + "official": float(official), + "flashdreams": float(flashdreams), + "inputs": summary.get("inputs", {}), + } + + +def _render_chart(rows: list[dict[str, Any]]) -> str: + lines = [ + "# SANA-WM Benchmark Data (ms/frame)", + "", + "| precision | official | flashdreams |", + "| --- | ---: | ---: |", + ] + for row in rows: + lines.append( + f"| {row['label']} | {row['official']:.2f} | {row['flashdreams']:.2f} |" + ) + lines.append("") + return "\n".join(lines) + + +def _render_report(rows: list[dict[str, Any]]) -> str: + lines = [ + "# SANA-WM precision benchmark sweep", + "", + "The chart metric is post-load generation latency per generated frame.", + "", + "| precision | official | FlashDreams | source |", + "| --- | ---: | ---: | --- |", + ] + for row in rows: + lines.append( + f"| {row['label']} | {row['official']:.2f} ms/frame | " + f"{row['flashdreams']:.2f} ms/frame | `{row['path']}` |" + ) + lines.append("") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--item", + action="append", + type=_load_item, + required=True, + help="Precision row as LABEL:/path/to/bench.json. Repeat once per row.", + ) + parser.add_argument("--output-json", type=Path, required=True) + parser.add_argument("--output-md", type=Path, required=True) + parser.add_argument("--output-chart-md", type=Path, required=True) + args = parser.parse_args(argv) + + payload = { + "benchmark": { + "metric": "generation_ms_per_frame", + "unit": "ms/frame", + "timing_boundary": ( + "pipeline.generate after model setup; excludes model construction, " + "checkpoint loading, video writing, and frame dumps" + ), + }, + "rows": args.item, + } + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + args.output_md.parent.mkdir(parents=True, exist_ok=True) + report = _render_report(args.item) + args.output_md.write_text(report, encoding="utf-8") + args.output_chart_md.parent.mkdir(parents=True, exist_ok=True) + args.output_chart_md.write_text(_render_chart(args.item), encoding="utf-8") + print(report) + + +if __name__ == "__main__": + main() diff --git a/integrations/sana/tests/parity_check/changes.patch b/integrations/sana/tests/parity_check/changes.patch new file mode 100644 index 000000000..10f349d4c --- /dev/null +++ b/integrations/sana/tests/parity_check/changes.patch @@ -0,0 +1,215 @@ +diff --git a/inference_video_scripts/wm/inference_sana_wm.py b/inference_video_scripts/wm/inference_sana_wm.py +index 5e07197..17db817 100644 +--- a/inference_video_scripts/wm/inference_sana_wm.py ++++ b/inference_video_scripts/wm/inference_sana_wm.py +@@ -937,6 +937,101 @@ def write_video(output_dir: Path, name: str, video_hwc: np.ndarray, fps: int, lo + return video_path + + ++def write_json(path: Path, payload: dict[str, object]) -> None: ++ path.parent.mkdir(parents=True, exist_ok=True) ++ path.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") ++ ++ ++def apply_backend_defaults() -> None: ++ torch.backends.cudnn.benchmark = True ++ torch.set_float32_matmul_precision("high") ++ if torch.cuda.is_available(): ++ torch.backends.cuda.enable_flash_sdp(False) ++ torch.backends.cuda.enable_math_sdp(True) ++ torch.backends.cuda.enable_mem_efficient_sdp(False) ++ torch.backends.cuda.enable_cudnn_sdp(True) ++ try: ++ import torch._inductor.config as inductor_config ++ ++ inductor_config.coordinate_descent_tuning = True ++ inductor_config.epilogue_fusion = True ++ except Exception: ++ pass ++ ++ ++def apply_precision_args(args: argparse.Namespace, logger: logging.Logger) -> None: ++ stage1_precision = args.stage1_precision ++ refiner_precision = "bf16" if args.no_refiner else args.refiner_precision ++ ++ if stage1_precision == "bf16": ++ os.environ.pop("SANA_WM_STAGE1_NVFP4", None) ++ os.environ.pop("SANA_WM_STAGE1_NVFP4_MODE", None) ++ os.environ.pop("SANA_WM_STAGE1_NVFP4_SCOPE", None) ++ os.environ.pop("SANA_WM_STAGE1_LINEARIZE_FFN", None) ++ os.environ.pop("SANA_WM_STAGE1_QUANT", None) ++ else: ++ os.environ["SANA_WM_STAGE1_NVFP4"] = "1" ++ os.environ["SANA_WM_STAGE1_NVFP4_MODE"] = "self_attn+cross+ffn" ++ os.environ["SANA_WM_STAGE1_NVFP4_SCOPE"] = "block" ++ os.environ["SANA_WM_STAGE1_LINEARIZE_FFN"] = "1" ++ os.environ["SANA_WM_STAGE1_QUANT"] = ( ++ "fp8block" if stage1_precision == "fp8" else "nvfp4" ++ ) ++ ++ if refiner_precision == "bf16": ++ os.environ.pop("SANA_WM_REFINER_NVFP4", None) ++ os.environ.pop("SANA_WM_REFINER_QUANT", None) ++ else: ++ os.environ["SANA_WM_REFINER_NVFP4"] = "1" ++ os.environ["SANA_WM_REFINER_QUANT"] = ( ++ "fp8block" if refiner_precision == "fp8" else "nvfp4" ++ ) ++ ++ if stage1_precision != "bf16" or refiner_precision != "bf16": ++ need = ( ++ "NVFP4BlockScaling" ++ if "fp4" in (stage1_precision, refiner_precision) ++ else "Float8BlockScaling" ++ ) ++ try: ++ import transformer_engine.common.recipe as te_recipe ++ import transformer_engine.pytorch # noqa: F401 ++ ++ if not hasattr(te_recipe, need): ++ raise ImportError(f"installed Transformer Engine lacks {need}") ++ except Exception as exc: ++ raise SystemExit( ++ "--stage1_precision/--refiner_precision fp8/fp4 require NVIDIA " ++ f"Transformer Engine >= 2.x for {need} ({exc})." ++ ) ++ ++ if "fp4" in (stage1_precision, refiner_precision) and torch.cuda.is_available(): ++ major, minor = torch.cuda.get_device_capability() ++ if major < 10: ++ logger.warning( ++ "fp4 (NVFP4) requires a Blackwell GPU (sm_100+); detected sm_%d%d.", ++ major, ++ minor, ++ ) ++ logger.info("[precision] stage1=%s refiner=%s", stage1_precision, refiner_precision) ++ ++ ++def time_bound_method(obj: object, name: str, timing_key: str, timings: dict[str, float]) -> None: ++ original = getattr(obj, name) ++ ++ def wrapped(*args, **kwargs): ++ if torch.cuda.is_available(): ++ torch.cuda.synchronize() ++ t0 = time.perf_counter() ++ result = original(*args, **kwargs) ++ if torch.cuda.is_available(): ++ torch.cuda.synchronize() ++ timings[timing_key] = time.perf_counter() - t0 ++ return result ++ ++ setattr(obj, name, wrapped) ++ ++ + # ============================================================================ + # Pipeline + # ============================================================================ +@@ -2132,6 +2227,30 @@ def _build_parser() -> argparse.ArgumentParser: + action="store_true", + help="Skip rendering the WASD + joystick overlay on the output video.", + ) ++ p.add_argument("--dump_frames", type=Path, default=None, help="Optional .npy path for decoded uint8 frames.") ++ p.add_argument("--stats_json", type=Path, default=None, help="Optional JSON path for timing and memory stats.") ++ p.add_argument( ++ "--force_cudnn_sdpa", ++ action="store_true", ++ help="Force PyTorch scaled_dot_product_attention to use the cuDNN backend.", ++ ) ++ p.add_argument( ++ "--compile_stage1", ++ action="store_true", ++ help="Wrap the Stage-1 DiT with torch.compile(mode='max-autotune-no-cudagraphs').", ++ ) ++ p.add_argument( ++ "--stage1_precision", ++ choices=["bf16", "fp8", "fp4"], ++ default="bf16", ++ help="Stage-1 DiT precision for benchmark parity.", ++ ) ++ p.add_argument( ++ "--refiner_precision", ++ choices=["bf16", "fp8", "fp4"], ++ default="bf16", ++ help="LTX-2 refiner precision for benchmark parity.", ++ ) + + # Weights and config. + p.add_argument( +@@ -2222,7 +2341,11 @@ def _snap_num_frames(n: int, stride: int = 8, *, upper_bound: int | None = None) + def main() -> None: + args = _build_parser().parse_args() + ++ if args.force_cudnn_sdpa: ++ apply_backend_defaults() ++ + logger = get_root_logger() ++ apply_precision_args(args, logger) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + image = Image.open(args.image).convert("RGB") +@@ -2276,6 +2399,15 @@ def main() -> None: + offload_refiner=args.offload_refiner, + logger=logger, + ) ++ if args.compile_stage1: ++ pipeline.model = torch.compile(pipeline.model, mode="max-autotune-no-cudagraphs") ++ ++ timings: dict[str, float] = {} ++ time_bound_method(pipeline, "_sample_stage1", "stage1_sample_s", timings) ++ time_bound_method(pipeline, "_dispatch_solver", "stage1_dit_s", timings) ++ time_bound_method(pipeline, "_decode_with_sana_vae", "vae_decode_s", timings) ++ if refiner is not None: ++ time_bound_method(pipeline, "_refine", "refiner_s", timings) + + denoising_step_list: list[int] | None = None + if args.denoising_step_list: +@@ -2308,14 +2440,48 @@ def main() -> None: + save_stage1=args.save_stage1, + ) + ++ if torch.cuda.is_available(): ++ torch.cuda.reset_peak_memory_stats(device) ++ torch.cuda.synchronize(device) ++ wall_start = time.perf_counter() + out = pipeline.generate(cropped, prompt, c2w, intrinsics_vec4, params) ++ if torch.cuda.is_available(): ++ torch.cuda.synchronize(device) ++ wall_s = time.perf_counter() - wall_start + video_hwc = out["video"] + + if not args.no_action_overlay: + logger.info("Compositing action overlay onto the output video.") + video_hwc = apply_overlay(video_hwc, out["c2w"]) + +- write_video(args.output_dir, args.name, video_hwc, params.fps, logger) ++ video_path = write_video(args.output_dir, args.name, video_hwc, params.fps, logger) ++ ++ if args.dump_frames is not None: ++ args.dump_frames.parent.mkdir(parents=True, exist_ok=True) ++ np.save(args.dump_frames, np.asarray(video_hwc, dtype=np.uint8)) ++ ++ if args.stats_json is not None: ++ stats: dict[str, object] = { ++ "backend": "upstream", ++ "entrypoint": "inference_video_scripts/wm/inference_sana_wm.py", ++ "video_path": str(video_path), ++ "video_shape": list(np.asarray(video_hwc).shape), ++ "timings_s": timings, ++ "num_frames": num_frames, ++ "seed": args.seed, ++ "fps": params.fps, ++ "step": params.step, ++ "cfg_scale": params.cfg_scale, ++ "no_refiner": args.no_refiner, ++ "stage1_precision": args.stage1_precision, ++ "refiner_precision": args.refiner_precision, ++ "compile_stage1": args.compile_stage1, ++ "force_cudnn_sdpa": args.force_cudnn_sdpa, ++ "wall_s": wall_s, ++ } ++ if torch.cuda.is_available(): ++ stats["mem_peak_gib"] = torch.cuda.max_memory_allocated(device) / (1024**3) ++ write_json(args.stats_json, stats) + + stage1_video = out.get("stage1_video") + if stage1_video is not None: diff --git a/integrations/sana/tests/parity_check/compat/mmcv/__init__.py b/integrations/sana/tests/parity_check/compat/mmcv/__init__.py new file mode 100644 index 000000000..9b2aebaae --- /dev/null +++ b/integrations/sana/tests/parity_check/compat/mmcv/__init__.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Minimal MMCV compatibility used by the SANA-WM parity harness. + +The upstream inference entrypoint only needs registry construction plus a few +utility functions at import time. This shim avoids installing old mmcv builds in +the isolated parity venv. +""" + +from __future__ import annotations + +import os +import pickle +from pathlib import Path +from typing import Any, Callable + + +class Registry: + """Small subset of ``mmcv.Registry`` used by SANA model registration.""" + + def __init__(self, name: str) -> None: + self.name = name + self.module_dict: dict[str, Any] = {} + + def __contains__(self, key: str) -> bool: + return key in self.module_dict + + def get(self, key: str) -> Any: + return self.module_dict.get(key) + + def register_module( + self, + module: Any | None = None, + *, + name: str | None = None, + force: bool = False, + ) -> Callable[[Any], Any] | Any: + """Register ``module`` or return a class decorator.""" + + def _register(obj: Any) -> Any: + key = name or obj.__name__ + if key in self.module_dict and not force: + raise KeyError(f"{key!r} is already registered in {self.name}.") + self.module_dict[key] = obj + return obj + + if module is not None: + return _register(module) + return _register + + def build( + self, + cfg: dict[str, Any], + *, + default_args: dict[str, Any] | None = None, + ) -> Any: + return build_from_cfg(cfg, self, default_args=default_args) + + +class Config(dict): + """Tiny dict-backed stand-in for import-time ``mmcv.Config`` references.""" + + @classmethod + def fromfile(cls, filename: str | os.PathLike[str]) -> "Config": + raise NotImplementedError( + "mmcv.Config.fromfile is not implemented in the parity harness shim." + ) + + +def build_from_cfg( + cfg: dict[str, Any], + registry: Registry, + *, + default_args: dict[str, Any] | None = None, +) -> Any: + """Instantiate a registered object from a config dict.""" + + if not isinstance(cfg, dict): + raise TypeError(f"cfg must be a dict, got {type(cfg).__name__}.") + if "type" not in cfg: + raise KeyError("cfg must contain key 'type'.") + args = dict(default_args or {}) + args.update({k: v for k, v in cfg.items() if k != "type"}) + obj_type = cfg["type"] + if isinstance(obj_type, str): + obj_cls = registry.get(obj_type) + if obj_cls is None: + raise KeyError(f"{obj_type!r} is not registered in {registry.name}.") + else: + obj_cls = obj_type + return obj_cls(**args) + + +def mkdir_or_exist(path: str | os.PathLike[str]) -> None: + Path(path).mkdir(parents=True, exist_ok=True) + + +def dump(obj: Any, file: str | os.PathLike[str]) -> None: + with open(file, "wb") as handle: + pickle.dump(obj, handle) + + +def load(file: str | os.PathLike[str]) -> Any: + with open(file, "rb") as handle: + return pickle.load(handle) + + +__all__ = [ + "Config", + "Registry", + "build_from_cfg", + "dump", + "load", + "mkdir_or_exist", +] diff --git a/integrations/sana/tests/parity_check/compat/mmcv/runner.py b/integrations/sana/tests/parity_check/compat/mmcv/runner.py new file mode 100644 index 000000000..3e82685a2 --- /dev/null +++ b/integrations/sana/tests/parity_check/compat/mmcv/runner.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Small ``mmcv.runner`` compatibility layer for SANA inference imports.""" + +from __future__ import annotations + +import torch.distributed as dist + + +def get_dist_info() -> tuple[int, int]: + if dist.is_available() and dist.is_initialized(): + return dist.get_rank(), dist.get_world_size() + return 0, 1 + + +class DefaultOptimizerConstructor: + pass + + +OPTIMIZERS = {} +OPTIMIZER_BUILDERS = {} + + +def build_optimizer(*_args, **_kwargs): + raise NotImplementedError("Optimizer construction is not available in the parity harness.") + + +__all__ = [ + "DefaultOptimizerConstructor", + "OPTIMIZER_BUILDERS", + "OPTIMIZERS", + "build_optimizer", + "get_dist_info", +] diff --git a/integrations/sana/tests/parity_check/compat/mmcv/utils/__init__.py b/integrations/sana/tests/parity_check/compat/mmcv/utils/__init__.py new file mode 100644 index 000000000..3acb82c06 --- /dev/null +++ b/integrations/sana/tests/parity_check/compat/mmcv/utils/__init__.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Minimal ``mmcv.utils`` namespace for SANA inference imports.""" + +from __future__ import annotations + +import torch.nn as nn + +_BatchNorm = nn.modules.batchnorm._BatchNorm +_InstanceNorm = nn.modules.instancenorm._InstanceNorm + +__all__ = ["_BatchNorm", "_InstanceNorm"] diff --git a/integrations/sana/tests/parity_check/compat/mmcv/utils/logging.py b/integrations/sana/tests/parity_check/compat/mmcv/utils/logging.py new file mode 100644 index 000000000..bb2396088 --- /dev/null +++ b/integrations/sana/tests/parity_check/compat/mmcv/utils/logging.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Logging globals expected by upstream SANA.""" + +logger_initialized: dict[str, bool] = {} + +__all__ = ["logger_initialized"] diff --git a/integrations/sana/tests/parity_check/diff_parity.py b/integrations/sana/tests/parity_check/diff_parity.py new file mode 100644 index 000000000..db9d395e2 --- /dev/null +++ b/integrations/sana/tests/parity_check/diff_parity.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compute SANA-WM frame parity as mean absolute uint8 delta.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import numpy as np + + +def _load_frames(path: Path) -> np.ndarray: + frames = np.load(path) + if frames.ndim != 4 or frames.shape[-1] != 3: + raise ValueError(f"{path} must contain [T,H,W,3] frames; got {frames.shape}.") + if frames.dtype != np.uint8: + frames = np.clip(frames, 0, 255).astype(np.uint8) + return frames + + +def _summary(upstream: np.ndarray, flashdreams: np.ndarray) -> dict[str, Any]: + if upstream.shape != flashdreams.shape: + raise ValueError( + "shape mismatch: " + f"upstream={tuple(upstream.shape)} flashdreams={tuple(flashdreams.shape)}" + ) + diff = np.abs(upstream.astype(np.int16) - flashdreams.astype(np.int16)) + per_frame = diff.reshape(diff.shape[0], -1).mean(axis=1) + return { + "shape": list(upstream.shape), + "mean_abs_delta": float(diff.mean()), + "mean_abs_delta_over_255": float(diff.mean() / 255.0), + "max_abs_delta": int(diff.max()), + "per_frame_mean_abs_delta": [float(v) for v in per_frame], + "per_frame_max_abs_delta": [ + int(v) for v in diff.reshape(diff.shape[0], -1).max(axis=1) + ], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--upstream", type=Path, required=True) + parser.add_argument("--flashdreams", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + summary = _summary(_load_frames(args.upstream), _load_frames(args.flashdreams)) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + + print( + "mean |Delta|: " + f"{summary['mean_abs_delta']:.4f} / 255 " + f"({summary['mean_abs_delta_over_255']:.6f})" + ) + print(f"max |Delta|: {summary['max_abs_delta']} / 255") + for idx, value in enumerate(summary["per_frame_mean_abs_delta"]): + print(f"frame {idx:04d}: mean |Delta| {value:.4f} / 255") + + +if __name__ == "__main__": + main() diff --git a/integrations/sana/tests/parity_check/pyproject.toml b/integrations/sana/tests/parity_check/pyproject.toml new file mode 100644 index 000000000..0f2fbdf09 --- /dev/null +++ b/integrations/sana/tests/parity_check/pyproject.toml @@ -0,0 +1,47 @@ +[project] +name = "sana-wm-parity-check" +version = "0.0.0" +description = "Isolated venv for the SANA-WM upstream parity and benchmark harness." +requires-python = ">=3.12,<3.13" +dependencies = [ + "flashdreams", + "flashdreams-sana-wm", + "accelerate>=1.3", + "diffusers>=0.37", + "einops>=0.7", + "flash-linear-attention>=0.4.2", + "ftfy>=6.0", + "huggingface-hub>=0.36", + "imageio[ffmpeg]>=2.31", + "numpy>=1.24,<2.5", + "omegaconf>=2.3", + "opencv-python-headless>=4.8", + "pillow>=10", + "protobuf>=7.35.0,<8", + "pyrallis>=0.3", + "pytz>=2024.0", + "pyyaml>=6.0", + "qwen-vl-utils>=0.0.8", + "safetensors>=0.5", + "scipy>=1.11", + "sentencepiece>=0.2", + "termcolor>=2.4", + "timm>=0.6.13", + "torch>=2.11", + "torchvision>=0.26", + "tqdm>=4.60", + "transformer-engine[pytorch,core-cu13]>=2.12; sys_platform != 'win32'", + "transformers>=5.0,<6", + "triton>=3.6; sys_platform == 'linux'", +] + +[tool.uv.sources] +flashdreams = { path = "../../../../flashdreams", editable = true } +flashdreams-sana-wm = { path = "../..", editable = true } + +[tool.uv] +managed = true +override-dependencies = [ + "nvidia-cublas>=13.4", +] +no-build-isolation-package = ["transformer-engine-torch"] diff --git a/integrations/sana/tests/parity_check/run.sh b/integrations/sana/tests/parity_check/run.sh new file mode 100644 index 000000000..a5460210d --- /dev/null +++ b/integrations/sana/tests/parity_check/run.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Patch pinned upstream SANA-WM with instrumentation, run it and FlashDreams on +# the same demo input, dump decoded uint8 frames, and compute mean |Delta| / 255. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" + +_abspath() { + case "$1" in + /*) printf '%s\n' "$1" ;; + *) printf '%s/%s\n' "${PWD}" "$1" ;; + esac +} + +SANA_REPO="$(_abspath "${SANA_REPO:-${HOME}/dev/Sana}")" +PATCH_FILE="${SCRIPT_DIR}/changes.patch" +REPO_URL="https://github.com/NVlabs/Sana.git" +PIN_COMMIT="6298508" + +OUTPUT_DIR="$(_abspath "${OUTPUT_DIR:-${SCRIPT_DIR}/outputs/parity}")" +UPSTREAM_OUT="${OUTPUT_DIR}/upstream" +NATIVE_OUT="${OUTPUT_DIR}/flashdreams" +IMAGE_PATH="$(_abspath "${IMAGE_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.png}")" +PROMPT_PATH="$(_abspath "${PROMPT_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.txt}")" +CAMERA_PATH="$(_abspath "${CAMERA_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_pose.npy}")" +INTRINSICS_PATH="$(_abspath "${INTRINSICS_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_intrinsics.npy}")" +NUM_FRAMES="${NUM_FRAMES:-121}" +FPS="${FPS:-16}" +STEP="${STEP:-60}" +CFG_SCALE="${CFG_SCALE:-5.0}" +SEED="${SEED:-42}" +NO_REFINER="${NO_REFINER:-1}" +FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" +COMPILE_STAGE1="${COMPILE_STAGE1:-0}" +STAGE1_PRECISION="${STAGE1_PRECISION:-bf16}" +REFINER_PRECISION="${REFINER_PRECISION:-${STAGE1_PRECISION}}" +QUANT_BACKEND="${QUANT_BACKEND:-auto}" + +_is_true() { + case "${1,,}" in + 1|true|yes|on) return 0 ;; + *) return 1 ;; + esac +} + +if [[ ! -d "${SANA_REPO}/.git" ]]; then + echo "[setup] cloning ${REPO_URL} -> ${SANA_REPO}" + git clone "${REPO_URL}" "${SANA_REPO}" +else + echo "[setup] repo already present at ${SANA_REPO}, skipping clone" +fi + +cd "${SANA_REPO}" +CURRENT_COMMIT="$(git rev-parse --short HEAD)" +if [[ "${CURRENT_COMMIT}" != "${PIN_COMMIT}" ]]; then + echo "[setup] checking out pinned commit ${PIN_COMMIT}" + git checkout "${PIN_COMMIT}" +else + echo "[setup] already at pinned commit ${PIN_COMMIT}, skipping checkout" +fi + +if git apply --reverse --check "${PATCH_FILE}" >/dev/null 2>&1; then + echo "[setup] patch already applied, skipping" +elif git apply --check "${PATCH_FILE}" >/dev/null 2>&1; then + echo "[setup] applying ${PATCH_FILE}" + git apply "${PATCH_FILE}" +else + echo "[setup] ERROR: ${PATCH_FILE} neither cleanly applies nor is already applied." >&2 + exit 1 +fi + +echo "[setup] ensuring Python deps via uv sync (isolated venv)" +( cd "${SCRIPT_DIR}" && uv sync ) +UPSTREAM_PYTHONPATH="${SCRIPT_DIR}/compat:${SANA_REPO}" +if [[ -n "${PYTHONPATH:-}" ]]; then + UPSTREAM_PYTHONPATH="${UPSTREAM_PYTHONPATH}:${PYTHONPATH}" +fi + +mkdir -p "${UPSTREAM_OUT}" "${NATIVE_OUT}" + +UPSTREAM_FRAMES="${UPSTREAM_OUT}/frames.npy" +NATIVE_FRAMES="${NATIVE_OUT}/frames.npy" +UPSTREAM_STATS="${UPSTREAM_OUT}/stats.json" +NATIVE_STATS="${NATIVE_OUT}/stats.json" + +UPSTREAM_REFINER_ARGS=() +NATIVE_REFINER_ARGS=() +if _is_true "${NO_REFINER}"; then + UPSTREAM_REFINER_ARGS+=(--no_refiner) + NATIVE_REFINER_ARGS+=(--no-refiner) +fi +UPSTREAM_BACKEND_ARGS=() +NATIVE_BACKEND_ARGS=() +if _is_true "${FORCE_CUDNN_SDPA}"; then + UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) + NATIVE_BACKEND_ARGS+=(--force-cudnn-sdpa) +fi +UPSTREAM_COMPILE_ARGS=() +NATIVE_COMPILE_ARGS=() +if _is_true "${COMPILE_STAGE1}"; then + UPSTREAM_COMPILE_ARGS+=(--compile_stage1) + NATIVE_COMPILE_ARGS+=(--compile-stage1) +fi +UPSTREAM_PRECISION_ARGS=( + --stage1_precision "${STAGE1_PRECISION}" + --refiner_precision "${REFINER_PRECISION}" +) +NATIVE_PRECISION_ARGS=( + --stage1-precision "${STAGE1_PRECISION}" + --refiner-precision "${REFINER_PRECISION}" + --quant-backend "${QUANT_BACKEND}" +) + +echo "[run] upstream SANA-WM -> ${UPSTREAM_OUT}" +( cd "${SCRIPT_DIR}" && \ + PYTHONPATH="${UPSTREAM_PYTHONPATH}" \ + uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm.py" \ + --image "${IMAGE_PATH}" \ + --prompt "${PROMPT_PATH}" \ + --camera "${CAMERA_PATH}" \ + --intrinsics "${INTRINSICS_PATH}" \ + --output_dir "${UPSTREAM_OUT}" \ + --name upstream \ + --num_frames "${NUM_FRAMES}" \ + --fps "${FPS}" \ + --step "${STEP}" \ + --cfg_scale "${CFG_SCALE}" \ + --seed "${SEED}" \ + --no_action_overlay \ + --dump_frames "${UPSTREAM_FRAMES}" \ + --stats_json "${UPSTREAM_STATS}" \ + "${UPSTREAM_PRECISION_ARGS[@]}" \ + "${UPSTREAM_BACKEND_ARGS[@]}" \ + "${UPSTREAM_COMPILE_ARGS[@]}" \ + "${UPSTREAM_REFINER_ARGS[@]}" ) + +echo "[run] FlashDreams SANA-WM -> ${NATIVE_OUT}" +( cd "${SCRIPT_DIR}" && \ + uv run python "${SCRIPT_DIR}/run_native.py" \ + --image-path "${IMAGE_PATH}" \ + --prompt-path "${PROMPT_PATH}" \ + --camera-path "${CAMERA_PATH}" \ + --intrinsics-path "${INTRINSICS_PATH}" \ + --output-dir "${NATIVE_OUT}" \ + --name flashdreams \ + --num-frames "${NUM_FRAMES}" \ + --fps "${FPS}" \ + --step "${STEP}" \ + --cfg-scale "${CFG_SCALE}" \ + --seed "${SEED}" \ + --dump-frames "${NATIVE_FRAMES}" \ + --stats-json "${NATIVE_STATS}" \ + "${NATIVE_PRECISION_ARGS[@]}" \ + "${NATIVE_BACKEND_ARGS[@]}" \ + "${NATIVE_COMPILE_ARGS[@]}" \ + "${NATIVE_REFINER_ARGS[@]}" ) + +echo "[diff] summarising parity -> ${OUTPUT_DIR}/parity.json" +( cd "${SCRIPT_DIR}" && \ + uv run python "${SCRIPT_DIR}/diff_parity.py" \ + --upstream "${UPSTREAM_FRAMES}" \ + --flashdreams "${NATIVE_FRAMES}" \ + --output "${OUTPUT_DIR}/parity.json" ) + +echo "[run] done." +echo " upstream frames : ${UPSTREAM_FRAMES}" +echo " flashdreams frames: ${NATIVE_FRAMES}" +echo " parity JSON : ${OUTPUT_DIR}/parity.json" diff --git a/integrations/sana/tests/parity_check/run_native.py b/integrations/sana/tests/parity_check/run_native.py new file mode 100644 index 000000000..c93903fb9 --- /dev/null +++ b/integrations/sana/tests/parity_check/run_native.py @@ -0,0 +1,319 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run the FlashDreams SANA-WM pipeline with frame dumps and profiler JSON.""" + +from __future__ import annotations + +import argparse +from contextlib import nullcontext +import json +import os +import time +from pathlib import Path +from typing import Any + +import imageio.v3 as iio +import numpy as np +import torch +from PIL import Image + +from flashdreams.infra.config import derive_config +from sana_wm.camera import ( + default_intrinsics_vec4, + load_intrinsics, + resize_center_crop_geometry, + snap_num_frames, + transform_intrinsics_for_crop, +) +from sana_wm.conditioning import SanaWMI2VConditioningRequest +from sana_wm.config import RUNNER_SANA_WM_BIDIRECTIONAL +from sana_wm.constants import DEFAULT_VIDEO_HEIGHT, DEFAULT_VIDEO_WIDTH +from sana_wm.decoder import SanaWMDecodedVideo +from sana_wm.runner import ( + _active_quantized_precisions, + _pipeline_config, + _resolve_quant_backend, + _validate_precision_request, +) + + +def _apply_backend_defaults() -> None: + torch.backends.cudnn.benchmark = True + torch.set_float32_matmul_precision("high") + if torch.cuda.is_available(): + torch.backends.cuda.enable_flash_sdp(False) + torch.backends.cuda.enable_math_sdp(True) + torch.backends.cuda.enable_mem_efficient_sdp(False) + torch.backends.cuda.enable_cudnn_sdp(True) + try: + import torch._inductor.config as inductor_config + + inductor_config.coordinate_descent_tuning = True + inductor_config.epilogue_fusion = True + except Exception: + pass + + +def _resolve_device(device: str) -> torch.device: + if device == "auto": + if torch.cuda.is_available(): + return torch.device(f"cuda:{int(os.environ.get('LOCAL_RANK', '0'))}") + return torch.device("cpu") + if device == "cuda" and torch.cuda.is_available(): + return torch.device(f"cuda:{int(os.environ.get('LOCAL_RANK', '0'))}") + return torch.device(device) + + +def _prepare_inputs( + *, + image_path: Path, + camera_path: Path, + intrinsics_path: Path | None, + intrinsics_hfov_deg: float, + num_frames_requested: int, +) -> tuple[Image.Image, np.ndarray, np.ndarray, int]: + image = Image.open(image_path).convert("RGB") + c2w_full = np.load(camera_path).astype(np.float32) + if c2w_full.ndim != 3 or c2w_full.shape[1:] != (4, 4): + raise ValueError(f"--camera-path must be [F,4,4]; got {c2w_full.shape}.") + + num_frames = min(num_frames_requested, c2w_full.shape[0]) + num_frames = snap_num_frames(num_frames, stride=8, upper_bound=c2w_full.shape[0]) + c2w = c2w_full[:num_frames] + + resized_size, crop_offset = resize_center_crop_geometry( + image.size, + target_h=DEFAULT_VIDEO_HEIGHT, + target_w=DEFAULT_VIDEO_WIDTH, + ) + resized = image.resize(resized_size, Image.LANCZOS) + left, top = crop_offset + cropped = resized.crop( + (left, top, left + DEFAULT_VIDEO_WIDTH, top + DEFAULT_VIDEO_HEIGHT) + ) + if intrinsics_path is None: + intrinsics_src = default_intrinsics_vec4( + image.size, + num_frames, + hfov_deg=intrinsics_hfov_deg, + ) + else: + intrinsics_src = load_intrinsics(intrinsics_path, num_frames) + intrinsics_vec4 = transform_intrinsics_for_crop( + intrinsics_src, + image.size, + resized_size, + crop_offset, + ) + return cropped, c2w, intrinsics_vec4, num_frames + + +def _install_stage1_compile_hook(pipeline: Any) -> None: + transformer = pipeline.diffusion_model.transformer + original_ensure_model = transformer._ensure_model + compiled = {"done": False} + + def _ensure_model_with_compile() -> None: + original_ensure_model() + if compiled["done"]: + return + transformer.model = torch.compile( + transformer.model, + mode="max-autotune-no-cudagraphs", + ) + compiled["done"] = True + + transformer._ensure_model = _ensure_model_with_compile + + +def _write_video(output_dir: Path, name: str, frames: np.ndarray, fps: int) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / f"{name}_generated.mp4" + iio.imwrite(path, frames, fps=fps) + return path + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--image-path", type=Path, required=True) + parser.add_argument("--prompt-path", type=Path, required=True) + parser.add_argument("--camera-path", type=Path, required=True) + parser.add_argument("--intrinsics-path", type=Path, default=None) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--name", default="flashdreams") + parser.add_argument("--dump-frames", type=Path, default=None) + parser.add_argument("--stats-json", type=Path, default=None) + parser.add_argument("--num-frames", type=int, default=161) + parser.add_argument("--fps", type=int, default=16) + parser.add_argument("--step", type=int, default=60) + parser.add_argument("--cfg-scale", type=float, default=5.0) + parser.add_argument("--flow-shift", type=float, default=None) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--negative-prompt", default="") + parser.add_argument("--no-refiner", action="store_true") + parser.add_argument("--save-stage1", action="store_true") + parser.add_argument("--device", default="auto") + parser.add_argument("--intrinsics-hfov-deg", type=float, default=90.0) + parser.add_argument("--stage1-precision", choices=["bf16", "fp8", "fp4"], default="bf16") + parser.add_argument("--refiner-precision", choices=["bf16", "fp8", "fp4"], default="bf16") + parser.add_argument( + "--quant-backend", + choices=["auto", "torch", "torch-fp8", "torch-fp4"], + default="auto", + ) + parser.add_argument("--force-cudnn-sdpa", action="store_true") + parser.add_argument("--compile-stage1", action="store_true") + args = parser.parse_args() + + if args.force_cudnn_sdpa: + _apply_backend_defaults() + + prompt = args.prompt_path.read_text(encoding="utf-8", errors="replace").strip() + if not prompt: + raise ValueError(f"Prompt file is empty: {args.prompt_path}") + + device = _resolve_device(args.device) + quantized = _active_quantized_precisions( + stage1_precision=args.stage1_precision, + refiner_precision=args.refiner_precision, + refiner_enabled=not args.no_refiner, + ) + quant_backend = _resolve_quant_backend(args.quant_backend, quantized) + _validate_precision_request( + device=device, + stage1_precision=args.stage1_precision, + refiner_precision=args.refiner_precision, + refiner_enabled=not args.no_refiner, + quant_backend=args.quant_backend, + ) + with nullcontext(): + image, c2w, intrinsics_vec4, num_frames = _prepare_inputs( + image_path=args.image_path, + camera_path=args.camera_path, + intrinsics_path=args.intrinsics_path, + intrinsics_hfov_deg=args.intrinsics_hfov_deg, + num_frames_requested=args.num_frames, + ) + runner_cfg = derive_config( + RUNNER_SANA_WM_BIDIRECTIONAL, + output_dir=args.output_dir, + runner_name=args.name, + num_frames=num_frames, + fps=args.fps, + step=args.step, + cfg_scale=args.cfg_scale, + flow_shift=args.flow_shift, + seed=args.seed, + negative_prompt=args.negative_prompt, + no_refiner=args.no_refiner, + save_stage1=args.save_stage1, + refiner_seed=args.seed, + stage1_precision=args.stage1_precision, + refiner_precision=args.refiner_precision, + quant_backend=args.quant_backend, + ) + pipeline_cfg = _pipeline_config(runner_cfg, quant_backend=quant_backend) + pipeline_cfg = derive_config(pipeline_cfg, enable_sync_and_profile=True) + pipeline = pipeline_cfg.setup().to(device).eval() + if args.compile_stage1: + _install_stage1_compile_hook(pipeline) + encoder = getattr(pipeline, "encoder", None) + if encoder is not None: + text_encoder = getattr(encoder, "text_encoder", None) + if text_encoder is not None: + text_encoder._ensure_text_encoder() + first_frame_encoder = getattr(encoder, "first_frame_encoder", None) + if first_frame_encoder is not None: + first_frame_encoder._ensure_vae() + pipeline.diffusion_model.transformer._ensure_model() + decoder = getattr(pipeline, "decoder", None) + vae_decoder = getattr(decoder, "vae_decoder", None) + if vae_decoder is not None: + vae_decoder._ensure_vae() + + cache = pipeline.initialize_cache( + decoder_context={ + "prompt": prompt, + "fps": args.fps, + "save_stage1": args.save_stage1, + "refiner_seed": args.seed, + "sink_size": runner_cfg.sink_size, + } + ) + if torch.cuda.is_available(): + torch.cuda.reset_peak_memory_stats(device) + torch.cuda.synchronize(device) + generation_start = time.perf_counter() + decoded = pipeline.generate( + 0, + cache, + input=SanaWMI2VConditioningRequest( + image=image, + prompt=prompt, + poses_c2w=c2w, + intrinsics_vec4=intrinsics_vec4, + num_frames=num_frames, + fps=args.fps, + steps=args.step, + cfg_scale=args.cfg_scale, + flow_shift=args.flow_shift, + seed=args.seed, + negative_prompt=args.negative_prompt, + ), + ) + if torch.cuda.is_available(): + torch.cuda.synchronize(device) + wall_s = time.perf_counter() - generation_start + stats = pipeline.finalize(0, cache) or {} + + if not isinstance(decoded, SanaWMDecodedVideo): + raise TypeError(f"expected SanaWMDecodedVideo, got {type(decoded).__name__}") + + frames = np.asarray(decoded.video_hwc, dtype=np.uint8) + video_path = _write_video(args.output_dir, args.name, frames, args.fps) + if args.dump_frames is not None: + args.dump_frames.parent.mkdir(parents=True, exist_ok=True) + np.save(args.dump_frames, frames) + if args.stats_json is not None: + _write_json( + args.stats_json, + { + "backend": "flashdreams", + "runner": "sana-wm-bidirectional", + "video_path": str(video_path), + "video_shape": list(frames.shape), + "num_frames": num_frames, + "seed": args.seed, + "fps": args.fps, + "step": args.step, + "cfg_scale": args.cfg_scale, + "no_refiner": args.no_refiner, + "compile_stage1": args.compile_stage1, + "force_cudnn_sdpa": args.force_cudnn_sdpa, + "wall_s": wall_s, + "stats_ms": stats, + }, + ) + print(f"[native] wrote {video_path}") + + +if __name__ == "__main__": + main() diff --git a/integrations/sana/tests/parity_check/uv.lock b/integrations/sana/tests/parity_check/uv.lock new file mode 100644 index 000000000..baa519ed3 --- /dev/null +++ b/integrations/sana/tests/parity_check/uv.lock @@ -0,0 +1,1676 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" +resolution-markers = [ + "sys_platform == 'win32'", + "sys_platform != 'win32'", +] + +[manifest] +overrides = [{ name = "nvidia-cublas", specifier = ">=13.4" }] + +[[package]] +name = "accelerate" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8d/75/94cd5d389649578aca399e5aa822637eec18319a1dadc400ffe2f9a7493f/accelerate-1.14.0.tar.gz", hash = "sha256:41b9c4377a54e0b460a959b0defa1b736e4ca0a2373252d9a539964c2afe3c8d", size = 412167, upload-time = "2026-06-11T13:45:52.326Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/db/253133d7e7cb40d3af384bb2f5c0b4a2b7fdcffbc95c688cc67a20a3c103/accelerate-1.14.0-py3-none-any.whl", hash = "sha256:e94390c2863b873be18f623f9df48a0d8fe5eff13ea7f1a00092b0a7904888c6", size = 389246, upload-time = "2026-06-11T13:45:50.477Z" }, +] + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "antlr4-python3-runtime" +version = "4.9.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b", size = 117034, upload-time = "2021-11-06T17:52:23.524Z" } + +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "av" +version = "18.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/a4/570a5a35c8638aba01e739925846c35fdd6b0756a15526766d0a4dd3b7df/av-18.0.0.tar.gz", hash = "sha256:4ef7e72c3d3a872584a1215173b16e0226811037f40dcdbf75992631098df1ba", size = 4340222, upload-time = "2026-07-02T06:37:58.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/4a/9e3463df030e063d757fa12f0f39be6541b45b06b5bad48c2ce361b924bf/av-18.0.0-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:149289d40e732a6e49c9530bc245b49d9964cfd1c8c9e06778703b7d5bba6b25", size = 22499354, upload-time = "2026-07-02T06:36:58.751Z" }, + { url = "https://files.pythonhosted.org/packages/77/b3/2576a44b4f39c7462ced4c17fec04c756f7b0f3c5cb940d124173e417d6a/av-18.0.0-cp311-abi3-macosx_14_0_arm64.whl", hash = "sha256:35274c20d2ad3b4774fe632bcef2e34af79858ddf899352339cc3babbc13a484", size = 18175248, upload-time = "2026-07-02T06:37:01.741Z" }, + { url = "https://files.pythonhosted.org/packages/84/74/6732f17b96dc23fd23b876b2805435855abdc8a3b397142be4e581165de8/av-18.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4d683b7747a0ba9222b8a5f81e41db5f796e7f64473454ec4fe2548e083c2fa0", size = 33387843, upload-time = "2026-07-02T06:37:05.097Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b9/7708c43fed7ae28b4a1bad060b4221e3334cd827cec24f7165902a6ac1f4/av-18.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ae56b40b6f8b067a8ad2dac664fbfbabac7f7a55b9a7bb031eb99289252bc017", size = 35536910, upload-time = "2026-07-02T06:37:08.806Z" }, + { url = "https://files.pythonhosted.org/packages/5a/94/eba99691d184f6a395a242d54dc370e2fd2265e95bbc98e2963a0fdbdd6c/av-18.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ea2e8ebbce521f21b55df9400e00d721623c9020ef158f5a188a96130be0743f", size = 38984619, upload-time = "2026-07-02T06:37:11.861Z" }, + { url = "https://files.pythonhosted.org/packages/c9/cf/0d7aee07fe16aa9ffdf96043c14bed5485a52c0dea4259de87aa306ecab4/av-18.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef96dabb3e50dac249913145dff5424b302b257fd95dcb64be3c7b7a8aef16d1", size = 34451176, upload-time = "2026-07-02T06:37:15.154Z" }, + { url = "https://files.pythonhosted.org/packages/76/92/810da80b12680d4c4fe235bd1b4003289be9213ac7f114b77b8ecf0e3b3e/av-18.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0f65518a184613e41536f29e8758c8e3d8293e46bf5bef108f04f925bbfa3f44", size = 36619869, upload-time = "2026-07-02T06:37:18.495Z" }, + { url = "https://files.pythonhosted.org/packages/11/85/0f121ff43dc5a70696676c98a8f1674e2fa787614c2abaacb15fa1a9bc99/av-18.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:aaf4d354d2beaa6651e4f92e54409a578bde64f79c0beef9a30b388d06f7c629", size = 27556236, upload-time = "2026-07-02T06:37:21.388Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f6/2509754d4d2356abc6fc0ea3d57c12ade29bac23a1fb7fc215a53ca518fb/av-18.0.0-cp311-abi3-win_arm64.whl", hash = "sha256:adac2b3833b6cb9bd6cb52664a522b94db453615b3675b1dbb26e13fe1c80da6", size = 20221133, upload-time = "2026-07-02T06:37:23.88Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.53" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/da/0e90eab875f2eb4b8708fef2c198f2559ed1e451a1016f1cd4fcdcfbfbe3/boto3-1.43.53.tar.gz", hash = "sha256:c80425acab314d7af09609562053f565139e1fe49108eacfcc1601ebfaee235b", size = 112678, upload-time = "2026-07-21T19:28:53.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/27/7e72d25fdde77668b7bd4fa47381192dd2aa64fb77265e4bab786fd9fe2a/boto3-1.43.53-py3-none-any.whl", hash = "sha256:5383e705d8a976a14f23bb8c113c07a396931a019db98fce4cdc68650ec6e4d8", size = 140025, upload-time = "2026-07-21T19:28:50.957Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.53" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/6b/ebcefacc4de3cd4f1c449540d86877f76c2f5e586a620831012decbb2b2c/botocore-1.43.53.tar.gz", hash = "sha256:36d93dd8db68ee75f6b61ca9f775161b8168844e4601698701530e6efdded141", size = 15720336, upload-time = "2026-07-21T19:28:41.547Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/e5/1b60e394f0fff97ee70dd16913382b7dffda85b98e639c0c9e8ff56cfaa7/botocore-1.43.53-py3-none-any.whl", hash = "sha256:b7ee9a70d187e5348883c820990ccd9436ab14e2bd6622741fc96fe561e816b8", size = 15404628, upload-time = "2026-07-21T19:28:37.475Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, + { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "cuda-bindings" +version = "13.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/67/5e7dba1ba576dd73da5dee894ca076ca5e959450dfff66d6d510a255d1f7/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49", size = 6025351, upload-time = "2026-05-29T23:11:49.685Z" }, + { url = "https://files.pythonhosted.org/packages/39/2a/6d2e9047d1fb243dbaa364b01e0297534b9ed7fd27dba1c9f361519cf69b/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a", size = 6657965, upload-time = "2026-05-29T23:11:52.227Z" }, +] + +[[package]] +name = "cuda-pathfinder" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/b4/d088047afe39827556df21118cac9ffd20cc3f968c99a7681494d1eb333c/cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51", size = 54591, upload-time = "2026-07-21T15:03:56.224Z" }, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.3.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cudart = [ + { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cufft = [ + { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cufile = [ + { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cupti = [ + { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +curand = [ + { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cusolver = [ + { name = "nvidia-cublas" }, + { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cusparse = [ + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +nvtx = [ + { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] + +[[package]] +name = "diffusers" +version = "0.39.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "importlib-metadata" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "regex" }, + { name = "requests" }, + { name = "safetensors" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/81/6095237b86a3116c4789f28c4435d5296c00c0fc74ffde99008fd6b3a36c/diffusers-0.39.0.tar.gz", hash = "sha256:14bb1d98c85a0e463d734c99aaa73b480a7bc9bad22af30fbf730ef8f09c1d67", size = 4651240, upload-time = "2026-07-03T08:48:47.904Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/3f/7469c46e9d22307ea686bab687d70e6bf328722952f9d10339f5e913e608/diffusers-0.39.0-py3-none-any.whl", hash = "sha256:912aca51b5787365110806e984d5555735bf8a461073bb8459029d0bca7870ef", size = 5631176, upload-time = "2026-07-03T08:48:45.337Z" }, +] + +[[package]] +name = "docstring-parser" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, +] + +[[package]] +name = "einops" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/77/850bef8d72ffb9219f0b1aac23fbc1bf7d038ee6ea666f331fa273031aa2/einops-0.8.2.tar.gz", hash = "sha256:609da665570e5e265e27283aab09e7f279ade90c4f01bcfca111f3d3e13f2827", size = 56261, upload-time = "2026-01-26T04:13:17.638Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl", hash = "sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193", size = 65638, upload-time = "2026-01-26T04:13:18.546Z" }, +] + +[[package]] +name = "filelock" +version = "3.32.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/80/8232b582c4b318b817cf1274ba74976b07b34d35ef439b3eb948f98645a1/filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402", size = 213757, upload-time = "2026-07-21T13:17:42.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, +] + +[[package]] +name = "fla-core" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "einops" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4e/62/99e149f19a447ce809d7f4fa64ae61c073da337505b9db7f389502470820/fla_core-0.5.1.tar.gz", hash = "sha256:7f3cf56edfbaa9115f4937d1181372e5c7b11809ad8eb2e411fffc3caf729f48", size = 498800, upload-time = "2026-06-18T18:17:15.377Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/78/a55ee7a62515dcb9220770dd99dfe59ac6599da8af84ad1d20f9f407df4d/fla_core-0.5.1-py3-none-any.whl", hash = "sha256:02150d34aa1e37f6b8ed9b2feec5d29af93680573e5077ed981238179c11fb06", size = 702955, upload-time = "2026-06-18T18:17:12.229Z" }, +] + +[[package]] +name = "flash-linear-attention" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fla-core" }, + { name = "transformers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/e8/8f115be585a046795e4a1f7ade727889219bb66b8d96cc668b8c9e437c0e/flash_linear_attention-0.5.1.tar.gz", hash = "sha256:8840fd4c37de8b0612dc8fd493867f3d330672ba2f17c024a2ce37239634e247", size = 221733, upload-time = "2026-06-18T18:17:16.661Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/3c/5819fb19dc071302ca818616a4e64d4454e1f1193929eb8365c8d38e9052/flash_linear_attention-0.5.1-py3-none-any.whl", hash = "sha256:9022862f0a238752372c81290694b8b2ed1cb2d13fc40d340ffeeb554d6bee5c", size = 403096, upload-time = "2026-06-18T18:17:14.025Z" }, +] + +[[package]] +name = "flashdreams" +source = { editable = "../../../../flashdreams" } +dependencies = [ + { name = "boto3" }, + { name = "botocore" }, + { name = "einops" }, + { name = "filelock" }, + { name = "ftfy" }, + { name = "huggingface-hub" }, + { name = "loguru" }, + { name = "numpy" }, + { name = "nvidia-ml-py" }, + { name = "safetensors" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "sys_platform == 'win32'" }, + { name = "tqdm" }, + { name = "transformers" }, + { name = "triton-windows", marker = "sys_platform == 'win32'" }, + { name = "tyro" }, + { name = "urllib3" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiohttp", marker = "extra == 'serving'", specifier = ">=3.9" }, + { name = "aiortc", marker = "extra == 'serving'", specifier = ">=1.9" }, + { name = "boto3", specifier = ">=1.35" }, + { name = "botocore", specifier = ">=1.35" }, + { name = "einops", specifier = ">=0.7" }, + { name = "filelock", specifier = ">=3" }, + { name = "ftfy", specifier = ">=6.0" }, + { name = "huggingface-hub", specifier = ">=0.33" }, + { name = "loguru", specifier = ">=0.7" }, + { name = "mediapy", marker = "extra == 'dev'", specifier = ">=1.1" }, + { name = "mediapy", marker = "extra == 'examples'", specifier = ">=1.1" }, + { name = "mediapy", marker = "extra == 'runners'", specifier = ">=1.1" }, + { name = "numpy", specifier = ">=1.24,<2.5" }, + { name = "nvidia-ml-py", specifier = ">=12.0" }, + { name = "opencv-python-headless", marker = "extra == 'examples'", specifier = ">=4.5" }, + { name = "opencv-python-headless", marker = "extra == 'runners'", specifier = ">=4.5" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, + { name = "pytest-manual-marker", marker = "extra == 'dev'", specifier = ">=2.0" }, + { name = "safetensors", specifier = ">=0.4" }, + { name = "scipy", marker = "extra == 'examples'", specifier = ">=1.11" }, + { name = "scipy", marker = "extra == 'runners'", specifier = ">=1.11" }, + { name = "tomli", marker = "extra == 'dev'", specifier = ">=2.0" }, + { name = "torch", marker = "sys_platform != 'win32'", specifier = ">=2.9" }, + { name = "torch", marker = "sys_platform == 'win32'", specifier = ">=2.9", index = "https://download.pytorch.org/whl/cu130" }, + { name = "tqdm", specifier = ">=4.60" }, + { name = "transformer-engine", extras = ["core-cu13", "pytorch"], marker = "sys_platform != 'win32' and extra == 'dev'", specifier = ">=2.12" }, + { name = "transformers", specifier = ">=5.0,<6" }, + { name = "triton-windows", marker = "sys_platform == 'win32'", specifier = ">=3.5.0" }, + { name = "tyro", specifier = ">=1.0" }, + { name = "urllib3", specifier = ">=2.7.0" }, +] +provides-extras = ["dev", "examples", "runners", "serving"] + +[package.metadata.requires-dev] +cuda12 = [ + { name = "torch", marker = "sys_platform != 'win32'", specifier = ">=2.9", index = "https://download.pytorch.org/whl/cu128", conflict = { package = "flashdreams", group = "cuda12" } }, + { name = "torch", marker = "sys_platform == 'win32'", specifier = ">=2.9", index = "https://download.pytorch.org/whl/cu130" }, + { name = "torchvision", marker = "sys_platform != 'win32'", specifier = ">=0.24", index = "https://download.pytorch.org/whl/cu128", conflict = { package = "flashdreams", group = "cuda12" } }, + { name = "torchvision", marker = "sys_platform == 'win32'", specifier = ">=0.24", index = "https://download.pytorch.org/whl/cu130" }, +] +cuda13 = [ + { name = "torch", marker = "sys_platform != 'win32'", specifier = ">=2.9" }, + { name = "torch", marker = "sys_platform == 'win32'", specifier = ">=2.9", index = "https://download.pytorch.org/whl/cu130" }, + { name = "torchvision", marker = "sys_platform != 'win32'", specifier = ">=0.24" }, + { name = "torchvision", marker = "sys_platform == 'win32'", specifier = ">=0.24", index = "https://download.pytorch.org/whl/cu130" }, +] + +[[package]] +name = "flashdreams-sana-wm" +version = "0.1.0" +source = { editable = "../../" } +dependencies = [ + { name = "accelerate" }, + { name = "diffusers" }, + { name = "flashdreams" }, + { name = "imageio", extra = ["ffmpeg"] }, + { name = "pillow" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torchvision" }, + { name = "transformers" }, +] + +[package.metadata] +requires-dist = [ + { name = "accelerate", specifier = ">=1.0" }, + { name = "diffusers", specifier = ">=0.36" }, + { name = "flashdreams", editable = "../../../../flashdreams" }, + { name = "imageio", extras = ["ffmpeg"], specifier = ">=2.31" }, + { name = "pillow", specifier = ">=10" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "safetensors", specifier = ">=0.5" }, + { name = "torchvision", specifier = ">=0.26" }, + { name = "transformers", specifier = ">=5.0,<6" }, +] +provides-extras = ["dev"] + +[[package]] +name = "fsspec" +version = "2026.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, +] + +[[package]] +name = "ftfy" +version = "6.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a5/d3/8650919bc3c7c6e90ee3fa7fd618bf373cbbe55dff043bd67353dbb20cd8/ftfy-6.3.1.tar.gz", hash = "sha256:9b3c3d90f84fb267fe64d375a07b7f8912d817cf86009ae134aa03e1819506ec", size = 308927, upload-time = "2024-10-26T00:50:35.149Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/6e/81d47999aebc1b155f81eca4477a616a70f238a2549848c38983f3c22a82/ftfy-6.3.1-py3-none-any.whl", hash = "sha256:7c70eb532015cd2f9adb53f101fb6c7945988d023a085d127d1573dc49dd0083", size = 44821, upload-time = "2024-10-26T00:50:33.425Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "hf-xet" +version = "1.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/63/39/67be8d71f900d9a55761b6022821d6679fb56c64f1b6063d5af2c2606727/hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47", size = 903674, upload-time = "2026-07-16T17:29:56.833Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/ba/2b70603c7552db82baeb2623e2336898304a17328845151be4fe1f48d420/hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799", size = 4033760, upload-time = "2026-07-16T17:29:43.884Z" }, + { url = "https://files.pythonhosted.org/packages/60/ac/b097a86a1e4a6098f3a79382643ab09d5733d87ccc864877ad1e12b49b70/hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed", size = 3841438, upload-time = "2026-07-16T17:29:45.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/35/db860aa3a0780660324a506ad4b3d322ddc6ecbba4b9340aed0942cbf21c/hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c", size = 4428006, upload-time = "2026-07-16T17:29:46.996Z" }, + { url = "https://files.pythonhosted.org/packages/af/6b/832dd980af4b0c3ae0660e309285f2ffcdff2faa38129390dbb47aa4a3f9/hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025", size = 4221099, upload-time = "2026-07-16T17:29:48.525Z" }, + { url = "https://files.pythonhosted.org/packages/9e/05/ae50f0d34e3254e6c3e208beb2519f6b8673016fc4b3643badaf6450d186/hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f", size = 4420766, upload-time = "2026-07-16T17:29:50.092Z" }, + { url = "https://files.pythonhosted.org/packages/07/a9/c050bc2743a2bcd68928bfee157b08681667a164a24ec95fbfcfcd717e08/hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097", size = 4636716, upload-time = "2026-07-16T17:29:51.62Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f8/68b01c5c2edb56ac9a67b3d076ffddcb90867abaee923923eb34e7a14e76/hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65", size = 3988373, upload-time = "2026-07-16T17:29:53.395Z" }, + { url = "https://files.pythonhosted.org/packages/39/c6/988383e9dc17294d536fcbcd6fd16eed882e411ad16c954984a53e47b09c/hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d", size = 3816957, upload-time = "2026-07-16T17:29:54.991Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "huggingface-hub" +version = "1.24.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/9b/d3bb4e7d792835daf34dd7091bbc7d7b4e0437d9388f1ea7239cce49f478/huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5", size = 921848, upload-time = "2026-07-17T09:54:01.022Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/c3/aeaaf3911d2529614be18d1c8b5496afc185560e76568063d517283318af/huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59", size = 771904, upload-time = "2026-07-17T09:53:59.106Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "imageio" +version = "2.37.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "pillow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/62/aa770a9307508d2a2a2c62d536a49347bffe9e55322db27838d3c93d0b07/imageio-2.37.4.tar.gz", hash = "sha256:e45cbc5e83502047fb138f7f585f7f105a136a57eea5f4b3cfc6ce1b52720bd3", size = 390173, upload-time = "2026-07-20T05:26:11.369Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/2d/ca050652104bab2cf55e569db2a178b1b61cb041fef28307f2db383f6d9f/imageio-2.37.4-py3-none-any.whl", hash = "sha256:1ab2e22c8debf700f24c3ac43e8f95f3b3a8110c83b93411e97b4b0b2cd1c7e6", size = 318000, upload-time = "2026-07-20T05:26:09.874Z" }, +] + +[package.optional-dependencies] +ffmpeg = [ + { name = "imageio-ffmpeg" }, + { name = "psutil" }, +] + +[[package]] +name = "imageio-ffmpeg" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/bd/c3343c721f2a1b0c9fc71c1aebf1966a3b7f08c2eea8ed5437a2865611d6/imageio_ffmpeg-0.6.0.tar.gz", hash = "sha256:e2556bed8e005564a9f925bb7afa4002d82770d6b08825078b7697ab88ba1755", size = 25210, upload-time = "2025-01-16T21:34:32.747Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/58/87ef68ac83f4c7690961bce288fd8e382bc5f1513860fc7f90a9c1c1c6bf/imageio_ffmpeg-0.6.0-py3-none-macosx_10_9_intel.macosx_10_9_x86_64.whl", hash = "sha256:9d2baaf867088508d4a3458e61eeb30e945c4ad8016025545f66c4b5aaef0a61", size = 24932969, upload-time = "2025-01-16T21:34:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/40/5c/f3d8a657d362cc93b81aab8feda487317da5b5d31c0e1fdfd5e986e55d17/imageio_ffmpeg-0.6.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b1ae3173414b5fc5f538a726c4e48ea97edc0d2cdc11f103afee655c463fa742", size = 21113891, upload-time = "2025-01-16T21:34:00.277Z" }, + { url = "https://files.pythonhosted.org/packages/33/e7/1925bfbc563c39c1d2e82501d8372734a5c725e53ac3b31b4c2d081e895b/imageio_ffmpeg-0.6.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1d47bebd83d2c5fc770720d211855f208af8a596c82d17730aa51e815cdee6dc", size = 25632706, upload-time = "2025-01-16T21:33:53.475Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2d/43c8522a2038e9d0e7dbdf3a61195ecc31ca576fb1527a528c877e87d973/imageio_ffmpeg-0.6.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:c7e46fcec401dd990405049d2e2f475e2b397779df2519b544b8aab515195282", size = 29498237, upload-time = "2025-01-16T21:34:13.726Z" }, + { url = "https://files.pythonhosted.org/packages/a0/13/59da54728351883c3c1d9fca1710ab8eee82c7beba585df8f25ca925f08f/imageio_ffmpeg-0.6.0-py3-none-win32.whl", hash = "sha256:196faa79366b4a82f95c0f4053191d2013f4714a715780f0ad2a68ff37483cc2", size = 19652251, upload-time = "2025-01-16T21:34:06.812Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c6/fa760e12a2483469e2bf5058c5faff664acf66cadb4df2ad6205b016a73d/imageio_ffmpeg-0.6.0-py3-none-win_amd64.whl", hash = "sha256:02fa47c83703c37df6bfe4896aab339013f62bf02c5ebf2dce6da56af04ffc0a", size = 31246824, upload-time = "2025-01-16T21:34:28.6Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "loguru" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "win32-setctime", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "ml-dtypes" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314, upload-time = "2025-11-17T22:32:31.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/b8/3c70881695e056f8a32f8b941126cf78775d9a4d7feba8abcb52cb7b04f2/ml_dtypes-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac", size = 676927, upload-time = "2025-11-17T22:31:48.182Z" }, + { url = "https://files.pythonhosted.org/packages/54/0f/428ef6881782e5ebb7eca459689448c0394fa0a80bea3aa9262cba5445ea/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900", size = 5028464, upload-time = "2025-11-17T22:31:50.135Z" }, + { url = "https://files.pythonhosted.org/packages/3a/cb/28ce52eb94390dda42599c98ea0204d74799e4d8047a0eb559b6fd648056/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff", size = 5009002, upload-time = "2025-11-17T22:31:52.001Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, + { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, + { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, + { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, +] + +[[package]] +name = "nvdlfw-inspect" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/86/94188e03e5d4dd7b73c390b0cddcde5618b3799c18e327b2bf15763f6137/nvdlfw_inspect-0.2.2-py3-none-any.whl", hash = "sha256:8a4dc2814c5a4cd19ae304170b9bfa514538ef3c3eb243a45a82404ec3cb279d", size = 30964, upload-time = "2025-12-03T10:52:01.933Z" }, +] + +[[package]] +name = "nvidia-cublas" +version = "13.6.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cuda-nvrtc" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/22/7c08d8e93f6a2e4879ac83c12696aecaf17a0fc2c9e8d204caceaa3b8426/nvidia_cublas-13.6.0.2-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:946f6a252b1cc72d8de912c75975fd6d8ba44f67d4e5044fe764ddb909f4a688", size = 518599300, upload-time = "2026-06-29T16:54:56.859Z" }, + { url = "https://files.pythonhosted.org/packages/fd/6c/173c7a3db77a6592210f73f194f0f8ed5e51b6ec61cfed7b1eee06ac5fd3/nvidia_cublas-13.6.0.2-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b82c80c886cea6da6e149a5c3bdba274f12b7e4ec4b00a050b916b0446fb4153", size = 410473152, upload-time = "2026-06-29T16:55:54.297Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, + { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.20.0.48" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, + { url = "https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304", size = 366173588, upload-time = "2026-03-09T19:29:34.474Z" }, +] + +[[package]] +name = "nvidia-cufft" +version = "12.0.0.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, +] + +[[package]] +name = "nvidia-cufile" +version = "1.15.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, +] + +[[package]] +name = "nvidia-curand" +version = "10.4.0.35" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, +] + +[[package]] +name = "nvidia-cusolver" +version = "12.0.4.66" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, + { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, +] + +[[package]] +name = "nvidia-cusparse" +version = "12.6.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, + { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu13" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/e1/cdc1797eadf82d3a9a575a19b33fdc871a97edbec42c00b5b5e914f4aff4/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f", size = 221051344, upload-time = "2025-09-05T18:49:51.289Z" }, + { url = "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0", size = 170148586, upload-time = "2025-09-05T18:50:50.248Z" }, +] + +[[package]] +name = "nvidia-ml-py" +version = "13.610.43" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/b5/a8fbc356f768fa5c9cfd646668fd7d34bf55bdd1c6e20754642a64d930d4/nvidia_ml_py-13.610.43.tar.gz", hash = "sha256:65437eb73d68d0c62c931ca4d45038472faff03bd0b8729abba4b899f70d60f2", size = 52109, upload-time = "2026-06-01T18:54:08.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/45/caa600acfab94560807a20a64b5830d2cd3c3202b7f1328644d70b7d6bd8/nvidia_ml_py-13.610.43-py3-none-any.whl", hash = "sha256:f13c72698edef492f985cc225f14faafe68ae065a2e407f45bdf6f4b9b43fde8", size = 53163, upload-time = "2026-06-01T18:54:07.704Z" }, +] + +[[package]] +name = "nvidia-nccl-cu13" +version = "2.29.7" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/0d/daf50d44177ee0cbc7ff0a0c91eb5ff676c82be42f9a970bc7597f440c3a/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5", size = 206014712, upload-time = "2026-03-03T05:34:20.843Z" }, + { url = "https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d", size = 205976000, upload-time = "2026-03-03T05:36:24.472Z" }, +] + +[[package]] +name = "nvidia-nvjitlink" +version = "13.3.33" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:26a6de7fb4c8fdaa7703d3dad720d6d427ddfea5c48a528fd97c11733ad830e5", size = 40742423, upload-time = "2026-05-26T16:54:51.613Z" }, + { url = "https://files.pythonhosted.org/packages/69/30/45414e35ff2eee7db3da037e5707037ccf9d2b5218ffbdb055ea4d5aa98a/nvidia_nvjitlink-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ce48b37dfeb3cb1eae4cf85adacb47d7a6539ea2272870c9a3628ce275c2037e", size = 39168635, upload-time = "2026-05-26T16:54:13.906Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu13" +version = "3.4.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, +] + +[[package]] +name = "nvidia-nvtx" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, +] + +[[package]] +name = "omegaconf" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "antlr4-python3-runtime" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/3d/e4b57b8d9008c6ebe0d5eff901f91d5700cf7bdb8c8863df817463a7fd5e/omegaconf-2.3.1.tar.gz", hash = "sha256:e5e7de64aeebeddaf8e6d3f7a783b32ac2a01c0fbd9c878012caecb891a1f42a", size = 3298472, upload-time = "2026-06-11T05:05:12.885Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/0e/152509871bf30df6fc38569f52a2db9b55dd41aae957adae50a053ac7778/omegaconf-2.3.1-py3-none-any.whl", hash = "sha256:3d701d14e9a8828f1edd28bb70b725908b34277cdd72cf7d6a83f94dadc6b6a0", size = 79502, upload-time = "2026-06-11T05:05:09.954Z" }, +] + +[[package]] +name = "onnx" +version = "1.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "protobuf" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/19/8ea73a64b368b75fe339771a20a02bc61ea1f551484c9e3d9d0bfbd0450f/onnx-1.22.0.tar.gz", hash = "sha256:ef40c0aaf0b643857ea9306fc7eddce17eaf9fb0407e4801f1fc5758443a38e0", size = 12024721, upload-time = "2026-06-15T12:50:05.354Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/6a/481561f1093834376ed493e4ca42a73e5be0d50031f2969c86593bdc7c96/onnx-1.22.0-cp312-abi3-macosx_12_0_universal2.whl", hash = "sha256:596fbf0490947533c1c1045ba860851dc9fb77471023dac9a71ba5b42ceab103", size = 20167081, upload-time = "2026-06-15T12:49:32.078Z" }, + { url = "https://files.pythonhosted.org/packages/84/55/b34fc2aa30aa54b4a775402d24c4082242c720283a274fe976ac8eb94480/onnx-1.22.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae5a563f281cd9d2845622cecf6c092a57e4ee1b138f66fdbbdd4200567a5e16", size = 18889249, upload-time = "2026-06-15T12:49:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/09/a6/bd32357e6cc1ecb473afd78193d7231724f284435d2db25696ecfaaa1503/onnx-1.22.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:955e02e1f6d385b53d52f9cd7b9cdf5caf417c300bcfe3c64c6d542be763845b", size = 19106514, upload-time = "2026-06-15T12:49:37.424Z" }, + { url = "https://files.pythonhosted.org/packages/5a/9d/3af461ac6c714b8b369cb71499659932f4f12cfb066250b62f7567c3d530/onnx-1.22.0-cp312-abi3-pyemscripten_2025_0_wasm32.whl", hash = "sha256:82e9f27fc1223cb06d68a56bed6f9d3caf3d0dad1b61bce45006d529b15bd94c", size = 16966387, upload-time = "2026-06-15T12:49:40.918Z" }, +] + +[[package]] +name = "onnx-ir" +version = "0.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "onnx" }, + { name = "sympy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/35/e6/672fefb2f108d077f58181a7babf4c0f8d1182a30353ffc9c79c63afc5ee/onnx_ir-0.2.1.tar.gz", hash = "sha256:8b8b10a93f43e65962104de6070c43c5dacb0e3cdfefc7c8059dd83c9db64f35", size = 144279, upload-time = "2026-04-20T20:21:47.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/aa/f7a53321c60b9ad9ee184b6018292ed6b5389947592a2c8c09c736bb7f9e/onnx_ir-0.2.1-py3-none-any.whl", hash = "sha256:c7285da889312f91882de2092e298a9eeeefbfc1d1951c49d983992967eb09a7", size = 166792, upload-time = "2026-04-20T20:21:46.357Z" }, +] + +[[package]] +name = "onnxscript" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "onnx" }, + { name = "onnx-ir" }, + { name = "packaging" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/3a/4d79bce3f460e0df7fed54a92ce80827f25da66511da368bb00783ad8d20/onnxscript-0.7.1.tar.gz", hash = "sha256:309fb86484b11fa4ded90dba580e0d63f1a0827588e521cecaf2eeddb46d6e86", size = 618160, upload-time = "2026-06-29T23:33:21.526Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/bd/a0c8e737b6afda10e42a597787d53d5b66e00268df6f59184701eeae37d9/onnxscript-0.7.1-py3-none-any.whl", hash = "sha256:544763b7fdef49940cdd9412ff5135cbae96d59ac6bc1921457f21280f40f4b7", size = 721970, upload-time = "2026-06-29T23:33:23.298Z" }, +] + +[[package]] +name = "opencv-python-headless" +version = "5.0.0.93" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/99/76b7c80252aa83c1af16393454aafd125a0287101afe8deb0a6821af0e30/opencv_python_headless-5.0.0.93.tar.gz", hash = "sha256:b82f9831daab90b725c7c1ee1b36cb5732c367096ac76d119e64e14eb70d5f3c", size = 81817738, upload-time = "2026-07-02T07:01:06.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/7c/8c8097891c509d98cd128493835c95631c80be6a8f37ed9d25716c2e16f1/opencv_python_headless-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:030ca5e0837a2963ab36ef896baa9767eb8d2b83353fb28af5a521e40dd8756f", size = 48322581, upload-time = "2026-07-02T05:50:34.207Z" }, + { url = "https://files.pythonhosted.org/packages/90/8c/eab2ad388c3cbab2a350c10c2ef19ce6bd099240afc31789032c996bab52/opencv_python_headless-5.0.0.93-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:1e55af3abfb462eeeabe5c775f12bdb36216d8a93a3583d69e6bd6e1d6ba7d00", size = 34782894, upload-time = "2026-07-02T05:51:39.856Z" }, + { url = "https://files.pythonhosted.org/packages/ec/78/afca939f40ffe2b2380bfa86f812b2f7d4acc5a27b27dc41b49cad7ce7b4/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10818d91510e05c04568ae12b5cd120779c70c01bf897b001a6221fe430df80f", size = 36521085, upload-time = "2026-07-02T06:55:24.429Z" }, + { url = "https://files.pythonhosted.org/packages/2b/97/8170e9819764c47e436c130d3ff6cfb73b58f923eae9d3a03d8982b04aec/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09a872a157c1376ab922a69bbf22f9a95bcc7b658a9d8b436a60212b02b2eeb4", size = 56563598, upload-time = "2026-07-02T06:55:47.355Z" }, + { url = "https://files.pythonhosted.org/packages/3a/98/1a28a7101e31801042b3098871a74b76c61581d328ef40774ff4edb53a56/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:840bd717c21e5c11cadadc022a823315ea417f961213d06b4df010e019eb16f4", size = 39648433, upload-time = "2026-07-02T06:56:04.255Z" }, + { url = "https://files.pythonhosted.org/packages/9b/21/f6ef335f6e65724aa78b8d792b48d40a48c381715f1e62f5a5049e09d07e/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ed709fdf9aa0bd1f2ed8549e71d19449b03a675bb581eb292285f6861953be37", size = 61204038, upload-time = "2026-07-02T06:56:41.823Z" }, + { url = "https://files.pythonhosted.org/packages/d0/8f/b8756467ea991449a293797f6b3fa80fcfdd29598a0a60d1cd5715b96e61/opencv_python_headless-5.0.0.93-cp37-abi3-win32.whl", hash = "sha256:c6bcd96b185975ea240d22cfdb15a1f6d080cc95264cfbe2621f21bb144d89b9", size = 35411237, upload-time = "2026-07-02T05:50:12.901Z" }, + { url = "https://files.pythonhosted.org/packages/b8/88/763b967f7efd7226b82c9fae16d560cba049b1f0c036647e65c610fd636e/opencv_python_headless-5.0.0.93-cp37-abi3-win_amd64.whl", hash = "sha256:829717b6a95554f273e49e357cee3b3a2a26b6f4842fbc1bed2b45bdd8f87e0e", size = 43825962, upload-time = "2026-07-02T05:50:09.627Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pillow" +version = "12.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, + { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, + { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, + { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, + { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, + { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, + { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, +] + +[[package]] +name = "protobuf" +version = "7.35.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" }, + { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" }, + { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, +] + +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pyrallis" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "typing-inspect" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/ff/4d865cd0166b3e0b0b09b1f5399114fe27c836b74f45f580d61efecf28dd/pyrallis-0.3.1.tar.gz", hash = "sha256:ab7298f31c633d4858ec3b045a30e6cc9b9f7ab50f21ea93aa3fb28532ac4b6e", size = 33799, upload-time = "2022-03-14T08:23:25.805Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/21/896ce8acd58566f38ffa179b833fd52b4c285fd4e90842ad6b8362372639/pyrallis-0.3.1-py3-none-any.whl", hash = "sha256:632370b563486495f5f9e7caf86cddffab8351214a3bdc60ae0d23b261ebdb59", size = 33419, upload-time = "2022-03-14T08:23:24.193Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytz" +version = "2026.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "qwen-vl-utils" +version = "0.0.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "av" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/b1/ad4fc2260a3badd278b38d642f3b987412f1f6682f0ef2b31b0572d5caa8/qwen_vl_utils-0.0.14.tar.gz", hash = "sha256:9c7cad5ae803b3a10f8bb7194deb12aeacdd032f92f4224e880c73587a7346ad", size = 8453, upload-time = "2025-09-23T09:38:57.532Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/43/80f67e0336cb2fc725f8e06f7fe35c1d0fe946f4d2b8b2175e797e07349e/qwen_vl_utils-0.0.14-py3-none-any.whl", hash = "sha256:5e28657bfd031e56bd447c5901b58ddfc3835285ed100f4c56580e0ade054e96", size = 8120, upload-time = "2025-09-23T09:38:56.297Z" }, +] + +[[package]] +name = "regex" +version = "2026.7.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/b9/d11d7e501ac8fd7d617684423ebb9561e0b998481c1e4cbc0cb212c5d74a/regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d", size = 496778, upload-time = "2026-07-19T00:17:05.677Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a9/a5ab6f312f24318019170dc485d5421fe4f89e43a98640da50d95a8a7041/regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd", size = 297122, upload-time = "2026-07-19T00:17:07.59Z" }, + { url = "https://files.pythonhosted.org/packages/b3/63/4cab4d7f2d384a144d420b763d97674cb70619c878ea6fcd7640d0e62143/regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6", size = 292009, upload-time = "2026-07-19T00:17:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/22/85/102a81b218298957d4ea7d2f084fae537a71add9d6ff93c8e67284c5f45e/regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797", size = 796708, upload-time = "2026-07-19T00:17:11.542Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/dc136af5629938a037cd2b304c12240e132ec92f38be8ff9cc89af2a1f2d/regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18", size = 865651, upload-time = "2026-07-19T00:17:13.312Z" }, + { url = "https://files.pythonhosted.org/packages/e0/75/67402ae3cd9c8c988a4c805d15ee3eef015e7ca4cb112cf3e640fc1f4153/regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511", size = 911756, upload-time = "2026-07-19T00:17:15.015Z" }, + { url = "https://files.pythonhosted.org/packages/2a/8e/096d00c7c480ef2ff4265349b14e2261d4ab787ba1f74e2e80d1c58079c3/regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68", size = 801798, upload-time = "2026-07-19T00:17:17.208Z" }, + { url = "https://files.pythonhosted.org/packages/f0/41/e7ecac6edb5722417f85cc67eaf386322fbe8acf6918ec2fdc37c20dd9d0/regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11", size = 776933, upload-time = "2026-07-19T00:17:19.347Z" }, + { url = "https://files.pythonhosted.org/packages/6f/69/03c9b3f058d66403e0ca2c938696e81d51cd4c6d47ec5265f02f96948d9a/regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986", size = 784338, upload-time = "2026-07-19T00:17:21.057Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f7/b38ab3d43f284afbb618fcd15d0e77eb786ae461ce1f6bc7494619ddc0f2/regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b", size = 860452, upload-time = "2026-07-19T00:17:23.119Z" }, + { url = "https://files.pythonhosted.org/packages/15/5c/ff60ef0571121714f3cf9920bc183071e384a10b556d042e0fdb06cc07a5/regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb", size = 765958, upload-time = "2026-07-19T00:17:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0f/bd34021162c0ab47f9a315bd56cd5642e920c8e5668a75ef6c6a6fca590d/regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035", size = 851765, upload-time = "2026-07-19T00:17:26.993Z" }, + { url = "https://files.pythonhosted.org/packages/2a/20/a2ca43edade0595cccfdc98636739f536d9e26898e7dbddc2b9e98898953/regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a", size = 789714, upload-time = "2026-07-19T00:17:28.699Z" }, + { url = "https://files.pythonhosted.org/packages/5d/47/e02db4015d424fc83c00ea0ac8c5e5ec14397943de9abf909d5ce3a25931/regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5", size = 267157, upload-time = "2026-07-19T00:17:31.051Z" }, + { url = "https://files.pythonhosted.org/packages/08/8e/c780c131f79b42ed22d1bd7da4096c2c35f813e835acd02ef0f018bd892c/regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312", size = 277777, upload-time = "2026-07-19T00:17:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4c/e4d7e086449bdf379d89774bf1f89dc4a41943f3c5a6125a03905b34b5fb/regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d", size = 277136, upload-time = "2026-07-19T00:17:34.803Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/65/da/4bef7ce7bb989b222aa4785a413896dbec53306dfc59c6ce7d16a7ffbd6a/s3transfer-0.19.1.tar.gz", hash = "sha256:d3d6371dc3f1e5c5427b2b457bcf13bcf87bec334c95aed18642eae61f6926f3", size = 165354, upload-time = "2026-07-10T19:32:04.849Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/23/e84c64ad0e8bc59cd1b2ef98def848deff0ef3456c542afe74d51e9e8c85/s3transfer-0.19.1-py3-none-any.whl", hash = "sha256:d5fd7005ee39307455ad5f310b5ea67f4b1960d7fed5b3671ee50c249de675de", size = 90072, upload-time = "2026-07-10T19:32:03.673Z" }, +] + +[[package]] +name = "safetensors" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, +] + +[[package]] +name = "sana-wm-parity-check" +version = "0.0.0" +source = { virtual = "." } +dependencies = [ + { name = "accelerate" }, + { name = "diffusers" }, + { name = "einops" }, + { name = "flash-linear-attention" }, + { name = "flashdreams" }, + { name = "flashdreams-sana-wm" }, + { name = "ftfy" }, + { name = "huggingface-hub" }, + { name = "imageio", extra = ["ffmpeg"] }, + { name = "numpy" }, + { name = "omegaconf" }, + { name = "opencv-python-headless" }, + { name = "pillow" }, + { name = "protobuf" }, + { name = "pyrallis" }, + { name = "pytz" }, + { name = "pyyaml" }, + { name = "qwen-vl-utils" }, + { name = "safetensors" }, + { name = "scipy" }, + { name = "sentencepiece" }, + { name = "termcolor" }, + { name = "timm" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "sys_platform == 'win32'" }, + { name = "torchvision" }, + { name = "tqdm" }, + { name = "transformer-engine", extra = ["core-cu13", "pytorch"], marker = "sys_platform != 'win32'" }, + { name = "transformers" }, + { name = "triton", marker = "sys_platform == 'linux'" }, +] + +[package.metadata] +requires-dist = [ + { name = "accelerate", specifier = ">=1.3" }, + { name = "diffusers", specifier = ">=0.37" }, + { name = "einops", specifier = ">=0.7" }, + { name = "flash-linear-attention", specifier = ">=0.4.2" }, + { name = "flashdreams", editable = "../../../../flashdreams" }, + { name = "flashdreams-sana-wm", editable = "../../" }, + { name = "ftfy", specifier = ">=6.0" }, + { name = "huggingface-hub", specifier = ">=0.36" }, + { name = "imageio", extras = ["ffmpeg"], specifier = ">=2.31" }, + { name = "numpy", specifier = ">=1.24,<2.5" }, + { name = "omegaconf", specifier = ">=2.3" }, + { name = "opencv-python-headless", specifier = ">=4.8" }, + { name = "pillow", specifier = ">=10" }, + { name = "protobuf", specifier = ">=7.35.0,<8" }, + { name = "pyrallis", specifier = ">=0.3" }, + { name = "pytz", specifier = ">=2024.0" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "qwen-vl-utils", specifier = ">=0.0.8" }, + { name = "safetensors", specifier = ">=0.5" }, + { name = "scipy", specifier = ">=1.11" }, + { name = "sentencepiece", specifier = ">=0.2" }, + { name = "termcolor", specifier = ">=2.4" }, + { name = "timm", specifier = ">=0.6.13" }, + { name = "torch", specifier = ">=2.11" }, + { name = "torchvision", specifier = ">=0.26" }, + { name = "tqdm", specifier = ">=4.60" }, + { name = "transformer-engine", extras = ["pytorch", "core-cu13"], marker = "sys_platform != 'win32'", specifier = ">=2.12" }, + { name = "transformers", specifier = ">=5.0,<6" }, + { name = "triton", marker = "sys_platform == 'linux'", specifier = ">=3.6" }, +] + +[[package]] +name = "scipy" +version = "1.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/19/ca10ead60b0acc80b2b833c2c4a4f2ff753d0f58b811f70d911c7e94a25c/scipy-1.18.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7bd21faaf5a1a3b2eff922d02db5f191b99a6518db9078a8fb23169f6d22259a", size = 31056519, upload-time = "2026-06-19T14:59:45.203Z" }, + { url = "https://files.pythonhosted.org/packages/96/72/1e6442a00cd2924d361aa1b642ab6373ec35c6fabf311a760be9f76e0f13/scipy-1.18.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:265915e79107de9f946b855e50d7470d5893ec3f54b342e1aa6201cbdcd8bb6b", size = 28681889, upload-time = "2026-06-19T14:59:48.103Z" }, + { url = "https://files.pythonhosted.org/packages/9b/2d/11dd93d21e147a73ba22bd75c0b9208d3a2e0ec76d53170ce7d9029b1015/scipy-1.18.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9ab7b758be6940954a713ee466e2043e9f6e2ed965c1fce5c91039f4be3d90a9", size = 20423580, upload-time = "2026-06-19T14:59:50.665Z" }, + { url = "https://files.pythonhosted.org/packages/9c/01/93552f75e0d2a7dd115a45e59209c51e8d514daff02fc887d2623be06fe1/scipy-1.18.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:97b6cddaaee0a779ef6b5ca83c9604b27cc16b2b8fc22c142652df8793319fb8", size = 23054441, upload-time = "2026-06-19T14:59:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/3c/23/21f5e703643d66f21faa6b4c73195bfcad70c55efcb4f1ab327cd7c4101a/scipy-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:52a96e21517c7292375c0e27dd796a811f03fcea5fd4d108fdfea8145dcf17ab", size = 33968720, upload-time = "2026-06-19T14:59:56.415Z" }, + { url = "https://files.pythonhosted.org/packages/dd/aa/1b939f6c67ed68635bb538e6752d3dacc02f66535182e939a89581a44e9c/scipy-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f55797419e16e7f30cf88ffb3113ce0467f00cfe3f70d5c281730b21769bfc2", size = 35287115, upload-time = "2026-06-19T14:59:59.411Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ff/eec46be7e9234208f801062b53e1983085eddebd693f6c9bfb03b459830d/scipy-1.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ad033410e2e0672ffdc1042110cef20e1c46f8fd0616cee1d44d8d58fad8fc11", size = 35577989, upload-time = "2026-06-19T15:00:02.235Z" }, + { url = "https://files.pythonhosted.org/packages/84/ca/210d4759c7210bb7d269437421959b39a33434e2776b60c5cb8a763bb30a/scipy-1.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a55985d54c769c872e64b7f4c8a81cc30ef700cc04296abbbf3705439c126de", size = 37421717, upload-time = "2026-06-19T15:00:05.102Z" }, + { url = "https://files.pythonhosted.org/packages/2b/54/9a9edb45345bd6744da5ddfb6628e5d5185920494c6a67ec45b6381004cb/scipy-1.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:71ccc8faa2dd16ac310233203474a8b5cb67f10dedd54a3116d34943f4b19132", size = 36597428, upload-time = "2026-06-19T15:00:08.112Z" }, + { url = "https://files.pythonhosted.org/packages/99/0e/33f32a2a58987e26aec0f7df252cbbad1e90ae77bdbc76f40dd4ed0cf0ea/scipy-1.18.0-cp312-cp312-win_arm64.whl", hash = "sha256:d88363fd9d8fbd3511bd273f1a49efb2a540773ddf92a91d57498ce7dd7f3e76", size = 24351481, upload-time = "2026-06-19T15:00:11.103Z" }, +] + +[[package]] +name = "sentencepiece" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/33/ea3cb3839607eb175da835244a798f797f478c5ddf0e8ecdf57ea85a4c70/sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6", size = 8218435, upload-time = "2026-07-12T08:39:34.701Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/13/7a562289c8d5b49ebdf3f9c1e8ab67cf14a8743b1d90c8f406bfdec36b72/sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b", size = 2188384, upload-time = "2026-07-12T08:38:28.486Z" }, + { url = "https://files.pythonhosted.org/packages/85/d1/912f14fd5eae168aba726ffb6a9a2dc1c71fe7676c53da6f5c442b886d4a/sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820", size = 1441553, upload-time = "2026-07-12T08:38:30.552Z" }, + { url = "https://files.pythonhosted.org/packages/bd/44/caa9cab5f261a019e2808bc5046152775dc57352ba9cbae7525e9e7a1ed4/sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838", size = 1347176, upload-time = "2026-07-12T08:38:32.211Z" }, + { url = "https://files.pythonhosted.org/packages/19/90/cd798935668cff71d309d8ff10385844ecf216b1fe454f1993ed8bf2cb91/sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a", size = 1325200, upload-time = "2026-07-12T08:38:33.689Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2d/37e3da037318a70066ded0d51bc2a7f35491ae6338dd993d5eb1503fc3b5/sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da", size = 1397736, upload-time = "2026-07-12T08:38:35.771Z" }, + { url = "https://files.pythonhosted.org/packages/8d/11/753fca2e6b109be3ab7867abf357dfe48677fe726ae5a5363d0b54ca9450/sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383", size = 1248030, upload-time = "2026-07-12T08:38:37.604Z" }, + { url = "https://files.pythonhosted.org/packages/e2/0a/70efbe861ca182d7d4b6e1a20f58e043400848fa9f2915229f082e221648/sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d", size = 1187325, upload-time = "2026-07-12T08:38:39.348Z" }, +] + +[[package]] +name = "setuptools" +version = "83.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "termcolor" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/79/cf31d7a93a8fdc6aa0fbb665be84426a8c5a557d9240b6239e9e11e35fc5/termcolor-3.3.0.tar.gz", hash = "sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5", size = 14434, upload-time = "2025-12-29T12:55:21.882Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5", size = 7734, upload-time = "2025-12-29T12:55:20.718Z" }, +] + +[[package]] +name = "timm" +version = "1.0.28" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "sys_platform == 'win32'" }, + { name = "torchvision" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/35/03/e41389ac641747bfec48d016fde8be1eade1901e6f2c1aedcb0c8cb4b5d9/timm-1.0.28.tar.gz", hash = "sha256:3789d313fdd5541a327b60180d70dbb4bdec73db8ff0655e413db3c3d134a9a4", size = 2451413, upload-time = "2026-07-11T17:24:32.615Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/76/de1bfac17d183c49c6d0887903d3064ced51cf1d9ba7a8d611c1a8808c4f/timm-1.0.28-py3-none-any.whl", hash = "sha256:e577b88da96b3a722ea5e2f042455ce6f715d398304d8e63b17d126ed7d89968", size = 2597944, upload-time = "2026-07-11T17:24:30.869Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.22.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, + { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, + { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, + { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, + { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, + { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, + { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, + { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, + { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, +] + +[[package]] +name = "torch" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'win32'", +] +dependencies = [ + { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "triton", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/3a/ed0f4d4d1dcde03bced7aac9a28e800abcdc0cbd06b6775044c9fbd877b7/torch-2.13.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2fe228aba290d14b9f31b049be550dbd469c3fd3013d7a19705b30454da97027", size = 111213045, upload-time = "2026-07-08T16:05:22.997Z" }, + { url = "https://files.pythonhosted.org/packages/df/a9/f6a2a4d763ff1df02e9a64c477029db614295bc9367f4131223791ccc243/torch-2.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:572df8be8ffb4599c88cbd6a0726f1f854f4da65d2e3c09f0e2c2283333cd6d4", size = 427210998, upload-time = "2026-07-08T16:04:37.708Z" }, + { url = "https://files.pythonhosted.org/packages/f3/82/fea946351658e6534db52d2cc12bc53087cbf87f9440c5f180f367c1950b/torch-2.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:796633c4cdf0fe2cdced72d8f88f22e73dbcfce83132763162f6d4bff13b820b", size = 526605292, upload-time = "2026-07-08T16:04:22.81Z" }, +] + +[[package]] +name = "torch" +version = "2.13.0+cu130" +source = { registry = "https://download.pytorch.org/whl/cu130" } +resolution-markers = [ + "sys_platform == 'win32'", +] +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cu130/torch-2.13.0%2Bcu130-cp312-cp312-win_amd64.whl", hash = "sha256:2efab1e83604ca628c6d85b9e188c153690980498d1297081a9dad704919303c", upload-time = "2026-07-08T20:26:27Z" }, +] + +[[package]] +name = "torchvision" +version = "0.28.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "pillow" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "sys_platform == 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/49/c1cab1ecbb3ff1a380a3f99283db1dee61b8afe354f6352c643b65937130/torchvision-0.28.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:e9f54c30cd52e3ef7fd034cc69b7bb7e0964e1c8f8743e018ab92e95b40f9eee", size = 1856020, upload-time = "2026-07-08T16:07:52.182Z" }, + { url = "https://files.pythonhosted.org/packages/f0/4c/95233776e2def960e5abb7a07931230a545f43717a56a1e1140162033598/torchvision-0.28.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5cf78ebc401ce64ae19b8c55de866bb836797d559a4de9c25ccbe74cfa642d3a", size = 7842127, upload-time = "2026-07-08T16:07:53.446Z" }, + { url = "https://files.pythonhosted.org/packages/93/e4/e9b2495d0d57b9f60d63c57d0a910410a81b4b073bf70917bef815291119/torchvision-0.28.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:028a3d481b37d785605620d7cdad897064c5a55bae2aa1f2658766333e291940", size = 7675040, upload-time = "2026-07-08T16:07:58.017Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9c/55ed9cb6dfe3ee9c837df5cd0e758372e5829aa38b8dd71343aa632cc4e2/torchvision-0.28.0-cp312-cp312-win_amd64.whl", hash = "sha256:87dc16b2df427c1318ad335f1e2be2b3b15b2cf20f7934c83b0505a48425ee5d", size = 4085785, upload-time = "2026-07-08T16:07:50.928Z" }, +] + +[[package]] +name = "tqdm" +version = "4.69.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8c/69/40407dfc835517f058b603dbf37a6df094d8582b015a51eddc988febbcb7/tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b", size = 792569, upload-time = "2026-07-17T18:09:06.2Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/21/99a0cdaf54eb35e77623c41b5a2c9472ee4404bba687052791fe2aba6773/tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622", size = 676680, upload-time = "2026-07-17T18:09:04.172Z" }, +] + +[[package]] +name = "transformer-engine" +version = "2.17.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/26/e501a474a3f36c2d439561fa4f2094a3c0344df439500dda55937a8fccf7/transformer_engine-2.17.0-py3-none-any.whl", hash = "sha256:c274fd74ea2e4caa7132921e1ecfa3847931abbed9f6b8cab61346cdb833bc76", size = 1001710, upload-time = "2026-07-09T00:36:15.453Z" }, +] + +[package.optional-dependencies] +core-cu13 = [ + { name = "transformer-engine-cu13" }, +] +pytorch = [ + { name = "transformer-engine-torch" }, +] + +[[package]] +name = "transformer-engine-cu13" +version = "2.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "packaging" }, + { name = "pydantic" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/21/5e0ec562539798ffd375c218b8bd7612b387455ccc79ae14576f411c9609/transformer_engine_cu13-2.17.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:1c8e6627cf02358201f513ab902c7b4f82f179214bd775edc97ce0f2001d82da", size = 244570183, upload-time = "2026-07-09T00:36:35.507Z" }, +] + +[[package]] +name = "transformer-engine-torch" +version = "2.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "einops" }, + { name = "nvdlfw-inspect" }, + { name = "onnx" }, + { name = "onnxscript" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, + { name = "transformer-engine-cu13" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/f1/ad613157261ac71262f5f260acf07fe22ae5ee0bb0675622c4f19a3e8afb/transformer_engine_torch-2.17.0.tar.gz", hash = "sha256:52ca109cdbc987ca02f87735fa375da8a02e117acb87e25dcb7e27dca37aa87f", size = 369712, upload-time = "2026-07-09T00:36:28.065Z" } + +[[package]] +name = "transformers" +version = "5.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/fb/2a2ba88f325e68a921d8b69ff63b477830b2e73ade9a3c8c8cab2f06d741/transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173", size = 9295927, upload-time = "2026-07-16T09:41:57.773Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/67/8d85ca2323233ae3c0365a659c4e52ee1f587b440e4bc577e7d8e4416d0f/transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725", size = 11625234, upload-time = "2026-07-16T09:41:54.143Z" }, +] + +[[package]] +name = "triton" +version = "3.7.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/fa/f856e24deb462d5f18bd4b5a746957862ab9b6ee5834bda60605ec348366/triton-3.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9497f2e696ee368862a181a90b2dcc03ca978cc4f602abd67c7d81022a6988e1", size = 184692359, upload-time = "2026-06-17T20:03:48.288Z" }, + { url = "https://files.pythonhosted.org/packages/c4/6f/fb96d15db6f36d6eae4cafb998c2e0353bf59d7c4ea1662d7497f269134a/triton-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e40869937a68206ec70d7f25bb7ec6433cb083f9135e1f36dbd318dc449a728", size = 197719725, upload-time = "2026-06-17T19:53:20.419Z" }, +] + +[[package]] +name = "triton-windows" +version = "3.7.1.post27" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/30/325b420efd0047e119679c646a9a410db216069800ec009fae3da26c69a3/triton_windows-3.7.1.post27-cp312-cp312-win_amd64.whl", hash = "sha256:f5406230d7dbf6965bc4051fcad27b81c39ba4a4bfde06f494dd7ff4eb325a9e", size = 49683004, upload-time = "2026-06-21T16:48:14.02Z" }, +] + +[[package]] +name = "typeguard" +version = "4.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/67/1c/dfba5c4633cafc4c701f237d2ba63b416805047fd6d96aab4cfc40969f98/typeguard-4.5.2.tar.gz", hash = "sha256:5a16dcac23502039299c97c8941651bc33d7ea8cc4b2f7d6bbb1b528f6eea423", size = 80240, upload-time = "2026-05-14T12:59:40.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/29/74eeb4d3f3ae61ca096b018ad486b3b3c74b17bec09ab4edab721cbefec3/typeguard-4.5.2-py3-none-any.whl", hash = "sha256:fcf9de18bd945cdb4c7b996e12b4c51ce83f92f191314a6d7cf1739586ec98cf", size = 36748, upload-time = "2026-05-14T12:59:39.473Z" }, +] + +[[package]] +name = "typer" +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430, upload-time = "2026-07-15T19:21:07.007Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716, upload-time = "2026-07-15T19:21:05.553Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspect" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825, upload-time = "2023-05-24T20:25:47.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827, upload-time = "2023-05-24T20:25:45.287Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "tyro" +version = "1.0.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docstring-parser" }, + { name = "typeguard" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/78/a5749a6c1ee9abc2999e294f339f8f72476d1a60bb95fc0e86156aafed3b/tyro-1.0.15.tar.gz", hash = "sha256:3f1d60887723eecb9c489f195d11f079c4a1f33df74b723552ad31ec57c667bb", size = 593822, upload-time = "2026-06-20T08:48:28.364Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/28/d607636187cf6c18eb72efb5d65c1d1b9e451db03d84676f835dd488fdbd/tyro-1.0.15-py3-none-any.whl", hash = "sha256:982da1d566005f1b2a6b56f6be6c6929c96f0e5fab9d61bc6097573ddfaf8d13", size = 215045, upload-time = "2026-06-20T08:48:27.104Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/74/c6428f875774288bec1396f5bfcbc2d925700a4dad61727fd5f2b12f249d/wcwidth-0.8.2.tar.gz", hash = "sha256:91fbef97204b96a3d4d421609b80340b760cf33e26da123ff243d76b1fda8dda", size = 1466253, upload-time = "2026-06-29T18:11:11.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl", hash = "sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85", size = 323166, upload-time = "2026-06-29T18:11:09.888Z" }, +] + +[[package]] +name = "win32-setctime" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867, upload-time = "2024-12-07T15:28:28.314Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, +] + +[[package]] +name = "zipp" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, +] From 69c4daad2342842f305481f27d059e129e324b3d Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 23 Jul 2026 17:07:47 -0700 Subject: [PATCH 23/64] Remove local docs --- .../parity_check/BENCHMARK_FINALIZATION.md | 61 ----------------- .../sana/tests/parity_check/PERF_HANDOFF.md | 68 ------------------- 2 files changed, 129 deletions(-) delete mode 100644 integrations/sana/tests/parity_check/BENCHMARK_FINALIZATION.md delete mode 100644 integrations/sana/tests/parity_check/PERF_HANDOFF.md diff --git a/integrations/sana/tests/parity_check/BENCHMARK_FINALIZATION.md b/integrations/sana/tests/parity_check/BENCHMARK_FINALIZATION.md deleted file mode 100644 index d6c6f2654..000000000 --- a/integrations/sana/tests/parity_check/BENCHMARK_FINALIZATION.md +++ /dev/null @@ -1,61 +0,0 @@ - - -# SANA-WM Benchmark Finalization - -Use this after the perf optimization pass is done and the final BF16, FP8, and -FP4 numbers are ready for the model card. - -1. Run the precision sweep: - - ```bash - cd integrations/sana/tests/parity_check - DEVICE_LABEL="" BENCH_PRECISIONS=bf16,fp8,fp4 bash bench.sh - ``` - -2. Copy `outputs/bench/perf.md` to - `docs/source/_static/performance/sana_wm/perf-.md`. - -3. Add a `Profiling benchmark` section to `docs/source/models/sana_wm.rst` - between `What to expect` and `Citation`. - -4. The section should describe post-load generation latency per generated frame - for FlashDreams SANA-WM versus the official `NVlabs/Sana` implementation - under matched BF16, FP8, and FP4 settings. - -5. Use the standard benchmark chart block: - - ```rst - Profiling benchmark - ------------------- - - Here is the profiling benchmark on post-load generation latency per generated - frame for FlashDreams SANA-WM compared to the - `official SANA-WM implementation `_ under - matched BF16, FP8, and FP4 settings. - - .. raw:: html - -
-
-
-

- This chart shows post-load generation latency per generated frame in milliseconds on a single GPU. - For the official SANA-WM implementation, see - this instruction. -

-
-
- - ``` - -Keep the chart metric unchanged: post-load generation latency per generated -frame, in ms/frame. diff --git a/integrations/sana/tests/parity_check/PERF_HANDOFF.md b/integrations/sana/tests/parity_check/PERF_HANDOFF.md deleted file mode 100644 index aaed1d654..000000000 --- a/integrations/sana/tests/parity_check/PERF_HANDOFF.md +++ /dev/null @@ -1,68 +0,0 @@ - - -# SANA-WM Perf Handoff - -This note is only for the next SANA-WM performance investigation. It should not -be copied into the model card or public README. - -## Observed Delta - -The best current BF16, no-refiner, 121-frame, 60-step run still has -FlashDreams slower than upstream: - -| artifact | official | FlashDreams | gap | -| --- | ---: | ---: | ---: | -| `outputs/bf16_step60_bidirectional_default/bench.md` | 640.89 ms/frame | 663.60 ms/frame | +22.71 ms/frame | - -Component medians from that run: - -| component | official | FlashDreams | -| --- | ---: | ---: | -| wall | 77.55 s | 80.30 s | -| Stage-1 DiT | 75,565.73 ms | 78,237.04 ms | -| VAE decode row | 1,340.43 ms | 1,584.82 ms | -| peak memory | 14.37 GiB | 17.98 GiB | - -The main target is the 60-step Stage-1 DiT path. The decode row also still has -a smaller gap. - -## Perf Theories Tested - -| theory | result | evidence | -| --- | --- | --- | -| FlashDreams was slower because VAE tiling used low-memory tiles. | Partly right. | Early probes showed FlashDreams VAE decode around 5.2 s. Switching to upstream-style tiles dropped it to about 1.5 s. This fixed a large initial problem, but not the remaining 60-step gap. | -| Remaining VAE gap is mostly raw VAE forward kernels. | Probably wrong. | Later internal VAE timing was close to upstream, while the harness decode row remained slower. The remaining cost is likely after the VAE forward: clamp, permute, CPU transfer, numpy materialization, or `torch.cuda.empty_cache()`. | -| Matching upstream's bidirectional chunk default would close the gap. | Correct for config parity, insufficient for speed. | `SanaWMStage1Spec.chunk_size=None` with `chunk_split_strategy="first_chunk_plus_one"` matches upstream. The 8-step probe became slightly favorable to FlashDreams, but the 60-step run stayed slower. | -| Softmax camera Q/K/V transforms staying in higher precision were slowing blocks. | Right. | After casting transformed softmax-camera Q/K/V back to BF16, targeted one-step block timings dropped from roughly 106-107 ms to roughly 49-52 ms for those blocks. This change is kept. | -| Forcing cuDNN SDPA would make the default comparison faster. | Wrong. | The forced-cuDNN probe was much slower and used about 80 GiB. Keep `FORCE_CUDNN_SDPA=1` only as a backend-isolation probe. | -| `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` helps this path. | Unproven. | No run showed a speed or memory benefit for SANA-WM. The harness and docs do not set allocator overrides. | -| Direct SDPA on head dim 112 is faster than padding to 128. | Not supported by clean evidence. | The direct-D112 probe had suspicious backend and memory behavior. Current code keeps padding to 128. | -| `torch.inference_mode()` could reduce framework overhead versus `no_grad`. | Untested. | Upstream wraps generation with `@torch.inference_mode()`. FlashDreams framework generation uses `@torch.no_grad()`. A harness-only test was started, then reverted before measurement. | - -## Next Probes - -1. Reproduce the 60-step BF16 no-refiner benchmark once before changing code, - using `bench.sh`, to confirm the current gap on the current branch. -2. Profile Stage-1 DiT without using profiler-perturbed timings as headline - numbers. Focus on block-level attention, cross-attention, FFN, camera QKV, - and Python/framework overhead. -3. Test `torch.inference_mode()` around the FlashDreams generation path and - compare against the same command with `no_grad`. -4. Split FlashDreams decode timing into VAE forward, clamp, permute, CPU copy, - numpy conversion, and `empty_cache`. -5. Test disabling or configuring `torch.cuda.empty_cache()` after decode. The - optimization goal allows spending more memory for speed. - -## Local Provenance - -These paths are under ignored `outputs/` directories and may not exist in a -fresh checkout: - -- `outputs/bf16_step60_bidirectional_default/bench.md` -- `outputs/bf16_step8_bidirectional_default/bench.md` -- `outputs/bf16_step60_softmax_bf16/bench.md` -- `outputs/bench_121_noalloc/bench.md` -- `outputs/fd_profile_step1_softmax_bf16/stats.json` From 8a3ef240a70f63fb9f18a90cded0f65c52001a6e Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 23 Jul 2026 17:15:55 -0700 Subject: [PATCH 24/64] More benchmarking scripts, missing from prev commits --- .../tests/test_parity_benchmark_summary.py | 242 ++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 integrations/sana/tests/test_parity_benchmark_summary.py diff --git a/integrations/sana/tests/test_parity_benchmark_summary.py b/integrations/sana/tests/test_parity_benchmark_summary.py new file mode 100644 index 000000000..cebdc2d5f --- /dev/null +++ b/integrations/sana/tests/test_parity_benchmark_summary.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CPU-safe tests for the SANA-WM parity benchmark summary.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType + +import pytest + +pytestmark = pytest.mark.ci_cpu + + +def _load_bench_summary() -> ModuleType: + path = Path("integrations/sana/tests/parity_check/bench_summary.py") + spec = importlib.util.spec_from_file_location("sana_wm_bench_summary", path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load_bench_sweep_summary() -> ModuleType: + path = Path("integrations/sana/tests/parity_check/bench_sweep_summary.py") + spec = importlib.util.spec_from_file_location("sana_wm_bench_sweep_summary", path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_benchmark_summary_uses_generation_ms_per_frame_for_chart() -> None: + module = _load_bench_summary() + summary = { + "inputs": { + "image_path": "image.png", + "prompt_path": "prompt.txt", + "camera_path": "pose.npy", + "intrinsics_path": "intrinsics.npy", + "num_frames": 20, + "seed": 42, + "no_refiner": True, + "stage1_precision": "fp8", + "refiner_precision": "fp8", + "quant_backend": "torch-fp8", + "compile_stage1": False, + "force_cudnn_sdpa": True, + "warmup_runs": 1, + }, + "upstream": { + "runs_measured": 3, + "wall_median_s": 2.0, + "wall_p90_s": 2.2, + "stage1_total_median_ms": 1600.0, + "dit_median_ms": 1500.0, + "vae_decode_median_ms": 300.0, + "mem_peak_median_gib": 10.0, + }, + "flashdreams": { + "runs_measured": 3, + "wall_median_s": 1.5, + "wall_p90_s": 1.8, + "encode_median_ms": 100.0, + "dit_median_ms": 1100.0, + "vae_decode_median_ms": 250.0, + "mem_peak_median_gib": 12.0, + }, + } + + report = module._render_markdown(summary) + chart = module._render_chart_markdown(summary, "Test GPU") + + assert "## Benchmark metric" in report + assert "- stage1_precision: `fp8`" in report + assert "generation median / frame | 100.00 ms | 75.00 ms" in report + assert "## Timing breakdown" in report + assert "| Stage-1 DiT median | 1500.00 ms | 1100.00 ms |" in report + assert "Stage-1 DiT metric" not in report + assert "Generation after model load" not in report + assert chart == ( + "# SANA-WM Benchmark Data (ms/frame)\n" + "\n" + "| device | official | flashdreams |\n" + "| --- | ---: | ---: |\n" + "| Test GPU | 100.00 | 75.00 |\n" + ) + + +def test_benchmark_summary_writes_chart_data(tmp_path: Path) -> None: + module = _load_bench_summary() + upstream = tmp_path / "upstream" / "run_0" + flashdreams = tmp_path / "flashdreams" / "run_0" + upstream.mkdir(parents=True) + flashdreams.mkdir(parents=True) + (upstream / "stats.json").write_text( + json.dumps( + { + "wall_s": 4.0, + "mem_peak_gib": 10.0, + "timings_s": { + "stage1_sample_s": 3.5, + "stage1_dit_s": 3.0, + "vae_decode_s": 0.5, + }, + } + ), + encoding="utf-8", + ) + (flashdreams / "stats.json").write_text( + json.dumps( + { + "wall_s": 2.0, + "stats_ms": { + "encode_ms": 100.0, + "diffuse_ms": 1500.0, + "decode_ms": 400.0, + "mem_peak_gib": 12.0, + }, + } + ), + encoding="utf-8", + ) + out_json = tmp_path / "bench.json" + out_md = tmp_path / "bench.md" + out_chart = tmp_path / "perf.md" + + module.main( + [ + "--upstream-dir", + str(tmp_path / "upstream"), + "--flashdreams-dir", + str(tmp_path / "flashdreams"), + "--warmup-runs", + "0", + "--image-path", + "image.png", + "--prompt-path", + "prompt.txt", + "--camera-path", + "pose.npy", + "--intrinsics-path", + "intrinsics.npy", + "--num-frames", + "40", + "--seed", + "42", + "--device-label", + "Test GPU", + "--chart-label", + "BF16", + "--stage1-precision", + "bf16", + "--refiner-precision", + "bf16", + "--output-json", + str(out_json), + "--output-md", + str(out_md), + "--output-chart-md", + str(out_chart), + ] + ) + + assert "| BF16 | 100.00 | 50.00 |" in out_chart.read_text(encoding="utf-8") + assert "generation median / frame | 100.00 ms | 50.00 ms" in out_md.read_text( + encoding="utf-8" + ) + summary = json.loads(out_json.read_text(encoding="utf-8")) + assert summary["benchmark"] == { + "metric": "generation_ms_per_frame", + "unit": "ms/frame", + "timing_boundary": ( + "pipeline.generate after model setup; excludes model construction, " + "checkpoint loading, video writing, and frame dumps" + ), + "device_label": "Test GPU", + "chart_label": "BF16", + "official": 100.0, + "flashdreams": 50.0, + } + + +def test_benchmark_sweep_summary_writes_precision_chart_data(tmp_path: Path) -> None: + module = _load_bench_sweep_summary() + bf16_json = tmp_path / "bf16.json" + fp8_json = tmp_path / "fp8.json" + bf16_json.write_text( + json.dumps({"benchmark": {"official": 100.0, "flashdreams": 90.0}}), + encoding="utf-8", + ) + fp8_json.write_text( + json.dumps({"benchmark": {"official": 80.0, "flashdreams": 70.0}}), + encoding="utf-8", + ) + out_json = tmp_path / "bench.json" + out_md = tmp_path / "bench.md" + out_chart = tmp_path / "perf.md" + + module.main( + [ + "--item", + f"BF16:{bf16_json}", + "--item", + f"FP8:{fp8_json}", + "--output-json", + str(out_json), + "--output-md", + str(out_md), + "--output-chart-md", + str(out_chart), + ] + ) + + assert out_chart.read_text(encoding="utf-8") == ( + "# SANA-WM Benchmark Data (ms/frame)\n" + "\n" + "| precision | official | flashdreams |\n" + "| --- | ---: | ---: |\n" + "| BF16 | 100.00 | 90.00 |\n" + "| FP8 | 80.00 | 70.00 |\n" + ) + payload = json.loads(out_json.read_text(encoding="utf-8")) + assert payload["benchmark"]["metric"] == "generation_ms_per_frame" + assert [row["label"] for row in payload["rows"]] == ["BF16", "FP8"] From 9b48f4171893f11f095275655a23366d029b1b3c Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 24 Jul 2026 13:44:24 -0700 Subject: [PATCH 25/64] Optmizations to catch up to upstream --- integrations/sana/sana_wm/decoder.py | 21 +- integrations/sana/sana_wm/ops/__init__.py | 4 + .../sana/sana_wm/ops/fused_cam_gdn.py | 2448 +++++++++++++++++ integrations/sana/sana_wm/ops/fused_gdn.py | 2249 +++++++++++++++ .../sana/sana_wm/ops/fused_gdn_chunkwise.py | 2269 +++++++++++++++ integrations/sana/sana_wm/refiner.py | 98 +- integrations/sana/sana_wm/runner.py | 35 +- integrations/sana/sana_wm/stage1_model.py | 630 ++++- integrations/sana/sana_wm/transformer.py | 10 +- 9 files changed, 7652 insertions(+), 112 deletions(-) create mode 100644 integrations/sana/sana_wm/ops/__init__.py create mode 100644 integrations/sana/sana_wm/ops/fused_cam_gdn.py create mode 100644 integrations/sana/sana_wm/ops/fused_gdn.py create mode 100644 integrations/sana/sana_wm/ops/fused_gdn_chunkwise.py diff --git a/integrations/sana/sana_wm/decoder.py b/integrations/sana/sana_wm/decoder.py index 34c53b8df..38fcf5794 100644 --- a/integrations/sana/sana_wm/decoder.py +++ b/integrations/sana/sana_wm/decoder.py @@ -91,12 +91,12 @@ class SanaWMLTX2VAEDecoderConfig(InstantiateConfig): offload_vae: bool = False """Move the VAE to CPU between decode calls.""" - vae_tile_sample_min_width: int = 256 - vae_tile_sample_stride_width: int = 224 - vae_tile_sample_min_height: int = 256 - vae_tile_sample_stride_height: int = 192 - vae_tile_sample_min_num_frames: int = 24 - vae_tile_sample_stride_num_frames: int = 8 + vae_tile_sample_min_width: int = 512 + vae_tile_sample_stride_width: int = 448 + vae_tile_sample_min_height: int = 512 + vae_tile_sample_stride_height: int = 448 + vae_tile_sample_min_num_frames: int = 96 + vae_tile_sample_stride_num_frames: int = 64 vae_oom_retry_tile_sample_min_width: int = 128 vae_oom_retry_tile_sample_stride_width: int = 64 @@ -185,8 +185,10 @@ def decode_latents(self, latents: Tensor) -> np.ndarray: decoded = torch.stack(decoded, dim=0) video = ( torch.clamp(127.5 * decoded + 127.5, 0, 255) + .to(torch.uint8) .permute(0, 2, 3, 4, 1) - .to("cpu", dtype=torch.uint8) + .contiguous() + .cpu() .numpy()[0] ) if self.config.offload_vae: @@ -314,6 +316,8 @@ class SanaWMLTX2LatentRefinerConfig(InstantiateConfig): refiner_precision: Precision = "bf16" quant_backend: QuantBackend = "torch" offload_refiner: bool = False + cache_text_encoder: bool = False + """Keep Gemma cached on CPU after prompt encoding for repeated pipeline use.""" class SanaWMLTX2LatentRefiner(nn.Module): @@ -383,6 +387,7 @@ def _ensure_refiner(self) -> None: device=self.device, precision=self.config.refiner_precision, quant_backend=self.config.quant_backend, + cache_text_encoder=self.config.cache_text_encoder, ) self._refiner_built = True logger.info( @@ -457,6 +462,8 @@ def forward( ) video_hwc = self.vae_decoder.decode_latents(output_latent) + if self.refiner is not None: + video_hwc = video_hwc[1:] stage1_video_hwc = None if cache.save_stage1 and self.refiner is not None: stage1_video_hwc = self.vae_decoder.decode_latents(stage1_latent) diff --git a/integrations/sana/sana_wm/ops/__init__.py b/integrations/sana/sana_wm/ops/__init__.py new file mode 100644 index 000000000..07bb2847d --- /dev/null +++ b/integrations/sana/sana_wm/ops/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SANA-WM integration-local Triton kernels.""" diff --git a/integrations/sana/sana_wm/ops/fused_cam_gdn.py b/integrations/sana/sana_wm/ops/fused_cam_gdn.py new file mode 100644 index 000000000..6f6229637 --- /dev/null +++ b/integrations/sana/sana_wm/ops/fused_cam_gdn.py @@ -0,0 +1,2448 @@ +"""Triton-fused camera-branch UCPE single-path delta rule. + +Companion to :mod:`diffusion.model.ops.fused_gdn` (main GDN +branch). This module fuses the *camera* branch of +:class:`ChunkCausalGDNUCPESinglePathLiteLA` through two Triton kernels: + +1. ``_cam_prep_kernel`` — fuses, per ``(batch, token, head)``, the RMSNorm + over the full ``C`` channels, ReLU on Q/K, K-scale on K, the UCPE 4x4 + block-diagonal projection matrix on the first ``D/2`` dims, and the + interleaved-pair complex RoPE on the second ``D/2`` dims. Q, K, V are + processed in one pass. The kernel also emits per-token pre-UCPE and + post-UCPE ``||k||^2`` so the caller can compute the inflation-squared + factor used for Dynamic Beta Discounting. + +2. ``_cam_scan_kernel`` — fuses the numerator-only single-path delta-rule + scan per ``(batch, head)``. ``REVERSE=1`` implements the + ``flip_and_shift`` backward pass semantics directly, avoiding the + torch-side flips in the per-chunk backward loop. + +The prep and scan kernels are runtime paths. Torch implementations below are +kept only for fallback backward paths and focused validation. + +Notes: + - V skips RMSNorm / ReLU / K-scale but receives the same UCPE 4x4 + + RoPE transforms as K (apply_fn_kv in reference). + - The short convolution on K and the inverse UCPE output transform + (``apply_fn_o``) stay in PyTorch — they are single lightweight ops + not on the critical path. +""" + +# ruff: noqa: E501 + +from __future__ import annotations + +import os + +import torch +import triton +import triton.language as tl + +from .fused_gdn_chunkwise import cam_scan_chunkwise + +# ============================================================================= +# Scalar helpers +# ============================================================================= + + +def _invert_SE3(transforms: torch.Tensor) -> torch.Tensor: + """Invert a 4x4 SE(3) matrix batch (closed-form). + + Mirrors the reference ``_invert_SE3`` in ``sana_camctrl_blocks.py``; + inlined to keep this module dependency-light. + """ + assert transforms.shape[-2:] == (4, 4) + Rinv = transforms[..., :3, :3].transpose(-1, -2) + out = torch.zeros_like(transforms) + out[..., :3, :3] = Rinv + out[..., :3, 3] = -torch.einsum("...ij,...j->...i", Rinv, transforms[..., :3, 3]) + out[..., 3, 3] = 1.0 + return out + + +def _process_camera_conditions_raymats_only( + camera_conditions: torch.Tensor, + B: int, + HW: tuple[int, int, int], + patch_size: tuple[int, int, int], +) -> torch.Tensor: + """Lightweight variant of ``_process_camera_conditions_ucpe`` — raymats only. + + Computes *only* the per-ray ``world -> ray_local`` SE(3) transforms used + by UCPE single-path. Skips the ``compute_up_lat_map`` path (absmap) that + the cam branch never consumes — that saves ~1 ms per block on H100. + + Args: + camera_conditions: ``(B, F, 20)`` — ``[c2w_16 | fx | fy | cx | cy]``. + B: Batch size (redundant with ``camera_conditions.shape[0]``; kept + for parity with the reference signature). + HW: ``(T_latent, H_latent, W_latent)`` from the caller. + patch_size: ``(pt, ph, pw)`` patch embedding stride. + + Returns: + ``raymats`` of shape ``(B, F, H_latent, W_latent, 4, 4)``. + """ + F_dim = camera_conditions.shape[1] + c2w_flat = camera_conditions[..., :16] + C_to_W = c2w_flat.view(B, F_dim, 4, 4) + + fx = camera_conditions[..., 16] + fy = camera_conditions[..., 17] + cx = camera_conditions[..., 18] + cy = camera_conditions[..., 19] + H_dim, W_dim = HW[1], HW[2] + image_width = W_dim * patch_size[2] + image_height = H_dim * patch_size[1] + + xi = torch.zeros( + (B, F_dim), + device=camera_conditions.device, + dtype=camera_conditions.dtype, + ) + x_fov = compute_fov_from_fx_xi( + fx, + xi, + image_width, + device=camera_conditions.device, + dtype=camera_conditions.dtype, + ).view(B, F_dim) + y_fov = compute_fov_from_fx_xi( + fy, + xi, + image_height, + device=camera_conditions.device, + dtype=camera_conditions.dtype, + ).view(B, F_dim) + + d_cam = ucm_unproject_grid_fov( + x_fov, + y_fov, + xi, + H_dim, + W_dim, + cx / patch_size[2], + cy / patch_size[1], + device=camera_conditions.device, + dtype=camera_conditions.dtype, + ) + if d_cam.ndim == 4 and d_cam.shape[0] == B * F_dim: + d_cam = d_cam.view(B, F_dim, H_dim, W_dim, 3) + + return world_to_ray_mats(d_cam, C_to_W) # (B, F, H, W, 4, 4) + + +def _precompute_cam_inv_rms(raw: torch.Tensor, eps: float) -> torch.Tensor: + """Compute ``1/RMS`` per ``(b, n)`` over full-``C`` channels. + + Args: + raw: ``(B, N, H, D)`` raw QKV projection output (typically fp32). + eps: RMSNorm epsilon. + + Returns: + ``inv_rms`` of shape ``(B, N)`` in fp32, contiguous. + """ + B, N, H, D = raw.shape + C = H * D + sq_sum = (raw.float() * raw.float()).sum(dim=(-1, -2)) # (B, N) + return torch.rsqrt(sq_sum / C + eps).contiguous() + + +def _prepare_ucpe_rope_tables( + rotary_emb_cam: torch.Tensor, + N: int, + D_half: int, + device: torch.device, +) -> tuple[torch.Tensor, torch.Tensor]: + """Convert complex RoPE ``(1, 1, N, D_half//2)`` to interleaved ``(N, D_half)`` cos/sin. + + Uses the interleaved-pair convention: + y[2i] = x[2i]*cos[i] - x[2i+1]*sin[i] + y[2i+1] = x[2i]*sin[i] + x[2i+1]*cos[i] + encoded as ``y[d] = x[d]*cos_exp[d] + x[d^1]*sin_exp[d]`` with + sin_exp[2i] = -sin[i], sin_exp[2i+1] = +sin[i]. + """ + del device # all outputs inherit device from freqs + freqs = rotary_emb_cam.squeeze(0).squeeze(0) # (N, D_half//2) complex + cos_half = freqs.real.float() + sin_half = freqs.imag.float() + rope_cos = cos_half.repeat_interleave(2, dim=-1).contiguous() + rope_sin = torch.stack([-sin_half, sin_half], dim=-1).reshape(N, D_half).contiguous() + return rope_cos, rope_sin + + +# ============================================================================= +# Triton kernels +# ============================================================================= + + +_DEFAULT_BLOCK_S = 64 + + +@triton.jit +def _cam_prep_kernel( + q_raw_ptr, # (B, N, H, D) contiguous, any fp dtype + k_raw_ptr, # (B, N, H, D) contiguous (post short-conv on K) + v_raw_ptr, # (B, N, H, D) contiguous + q_inv_rms_ptr, # (B, N) float32 — precomputed over full C channels + k_inv_rms_ptr, # (B, N) float32 + q_norm_w_ptr, # (C,) = (H*D,) float32 + k_norm_w_ptr, # (C,) float32 + proj_q_ptr, # (B, N, 4, 4) — applied to Q first D/2 dims (P_T) + proj_kv_ptr, # (B, N, 4, 4) — applied to K,V first D/2 dims (P_inv) + rope_cos_ptr, # (N, D_rope) float32, D_rope = D//2 + rope_sin_ptr, # (N, D_rope) float32 + # --- outputs in (B, H, D, N) layout, same strides pattern --- + q_out_ptr, + k_out_ptr, + v_out_ptr, + k_pre_norm_sq_ptr, # (B, H, N) float32 — ||k_pre_ucpe||^2 + k_post_norm_sq_ptr, # (B, H, N) float32 — ||k_post_ucpe||^2 + # --- dims --- + H: tl.constexpr, + N: tl.constexpr, + D: tl.constexpr, # head dim + D_HALF: tl.constexpr, # D // 2 + N_GROUPS: tl.constexpr, # D_HALF // 4 + K_SCALE, + # --- tile sizes --- + BLOCK_D_ROPE: tl.constexpr, # next pow2 of D_HALF (rope block) + BLOCK_GROUPS: tl.constexpr, # next pow2 of N_GROUPS +): + """One program per (b, n, h) — processes a single (Q, K, V) head slice. + + Loads the first D_HALF dims as a (N_GROUPS, 4) tile (for the UCPE + block-diagonal 4x4 projmat), and the second D_HALF dims as a + (D_HALF,) vector (for RoPE). No redundant loads. + """ + pid = tl.program_id(0) + h_idx = pid % H + bn_idx = pid // H + b_idx = bn_idx // N + n_idx = bn_idx % N + + # layout (B, N, H, D) contiguous + row_base = b_idx * (N * H * D) + n_idx * (H * D) + h_idx * D + nw_off = h_idx * D + + # ---- load inv-RMS (scalar, shared across heads for this token) ---- + q_inv_rms = tl.load(q_inv_rms_ptr + bn_idx).to(tl.float32) + k_inv_rms = tl.load(k_inv_rms_ptr + bn_idx).to(tl.float32) + + # ---- load per-token P matrices (4,4) shared across heads ---- + proj_base = (b_idx * N + n_idx) * 16 + offs_i = tl.arange(0, 4) + offs_j = tl.arange(0, 4) + P_q = tl.load(proj_q_ptr + proj_base + offs_i[:, None] * 4 + offs_j[None, :]).to(tl.float32) + P_kv = tl.load(proj_kv_ptr + proj_base + offs_i[:, None] * 4 + offs_j[None, :]).to(tl.float32) + + # ================================================================== + # Pass 1 — UCPE block-diagonal projmat on first D_HALF dims + # ================================================================== + offs_g = tl.arange(0, BLOCK_GROUPS) + mask_g = offs_g < N_GROUPS + offs_gj = offs_g[:, None] * 4 + offs_j[None, :] # (BLOCK_GROUPS, 4) + mask_gj = mask_g[:, None] + + q_half = tl.load(q_raw_ptr + row_base + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) + k_half = tl.load(k_raw_ptr + row_base + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) + v_half = tl.load(v_raw_ptr + row_base + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) + + q_nw_half = tl.load(q_norm_w_ptr + nw_off + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) + k_nw_half = tl.load(k_norm_w_ptr + nw_off + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) + + q_half = q_half * q_inv_rms * q_nw_half + q_half = tl.where(q_half > 0, q_half, 0.0) + + k_half = k_half * k_inv_rms * k_nw_half + k_half = tl.where(k_half > 0, k_half, 0.0) * K_SCALE + + # Pre-UCPE ||k||^2 contribution from first half + k_half_masked = tl.where(mask_gj, k_half, 0.0) + k_pre_half_sq = tl.sum(k_half_masked * k_half_masked) + + # Apply 4x4 projmat: out[g, i] = sum_j P[i, j] * in[g, j] + # (BLOCK_GROUPS, 1, 4) * (1, 4, 4) -> (BLOCK_GROUPS, 4, 4), sum axis=-1 + q_half_out = tl.sum(q_half[:, None, :] * P_q[None, :, :], axis=-1) + k_half_out = tl.sum(k_half[:, None, :] * P_kv[None, :, :], axis=-1) + v_half_out = tl.sum(v_half[:, None, :] * P_kv[None, :, :], axis=-1) + + # Post-UCPE ||k||^2 contribution from first half + k_half_out_masked = tl.where(mask_gj, k_half_out, 0.0) + k_post_half_sq = tl.sum(k_half_out_masked * k_half_out_masked) + + # ================================================================== + # Pass 2 — RoPE on second D_HALF dims + # ================================================================== + offs_r = tl.arange(0, BLOCK_D_ROPE) + mask_r = offs_r < D_HALF + offs_r_pair = offs_r ^ 1 + mask_r_pair = offs_r_pair < D_HALF + + rope_row = n_idx * D_HALF + cos_v = tl.load(rope_cos_ptr + rope_row + offs_r, mask=mask_r, other=1.0).to(tl.float32) + sin_v = tl.load(rope_sin_ptr + rope_row + offs_r, mask=mask_r, other=0.0).to(tl.float32) + + # Load second-half raw values and their pair partners + rope_base = row_base + D_HALF + q_r = tl.load(q_raw_ptr + rope_base + offs_r, mask=mask_r, other=0.0).to(tl.float32) + k_r = tl.load(k_raw_ptr + rope_base + offs_r, mask=mask_r, other=0.0).to(tl.float32) + v_r = tl.load(v_raw_ptr + rope_base + offs_r, mask=mask_r, other=0.0).to(tl.float32) + q_r_pair = tl.load(q_raw_ptr + rope_base + offs_r_pair, mask=mask_r_pair, other=0.0).to(tl.float32) + k_r_pair = tl.load(k_raw_ptr + rope_base + offs_r_pair, mask=mask_r_pair, other=0.0).to(tl.float32) + v_r_pair = tl.load(v_raw_ptr + rope_base + offs_r_pair, mask=mask_r_pair, other=0.0).to(tl.float32) + + q_nw_r = tl.load(q_norm_w_ptr + nw_off + D_HALF + offs_r, mask=mask_r, other=0.0).to(tl.float32) + k_nw_r = tl.load(k_norm_w_ptr + nw_off + D_HALF + offs_r, mask=mask_r, other=0.0).to(tl.float32) + q_nw_r_pair = tl.load(q_norm_w_ptr + nw_off + D_HALF + offs_r_pair, mask=mask_r_pair, other=0.0).to(tl.float32) + k_nw_r_pair = tl.load(k_norm_w_ptr + nw_off + D_HALF + offs_r_pair, mask=mask_r_pair, other=0.0).to(tl.float32) + + q_r_n = q_r * q_inv_rms * q_nw_r + q_r_n = tl.where(q_r_n > 0, q_r_n, 0.0) + q_r_pair_n = q_r_pair * q_inv_rms * q_nw_r_pair + q_r_pair_n = tl.where(q_r_pair_n > 0, q_r_pair_n, 0.0) + + k_r_n = k_r * k_inv_rms * k_nw_r + k_r_n = tl.where(k_r_n > 0, k_r_n, 0.0) * K_SCALE + k_r_pair_n = k_r_pair * k_inv_rms * k_nw_r_pair + k_r_pair_n = tl.where(k_r_pair_n > 0, k_r_pair_n, 0.0) * K_SCALE + + # Pre-UCPE ||k||^2 contribution from second half (using post-ReLU/scale k_r_n) + k_r_n_masked = tl.where(mask_r, k_r_n, 0.0) + k_pre_rope_sq = tl.sum(k_r_n_masked * k_r_n_masked) + + q_rope_out = q_r_n * cos_v + q_r_pair_n * sin_v + k_rope_out = k_r_n * cos_v + k_r_pair_n * sin_v + v_rope_out = v_r * cos_v + v_r_pair * sin_v + + # Post-UCPE ||k||^2 contribution from second half + k_rope_masked = tl.where(mask_r, k_rope_out, 0.0) + k_post_rope_sq = tl.sum(k_rope_masked * k_rope_masked) + + # Store scalar per-token norm squares + norm_out_idx = (b_idx * H + h_idx) * N + n_idx + tl.store(k_pre_norm_sq_ptr + norm_out_idx, k_pre_half_sq + k_pre_rope_sq) + tl.store(k_post_norm_sq_ptr + norm_out_idx, k_post_half_sq + k_post_rope_sq) + + # ================================================================== + # Store outputs in (B, H, D, N) layout: ptr[b, h, d, n] = base_bh + d*N + n + # ================================================================== + out_base = b_idx * (H * D * N) + h_idx * (D * N) + n_idx + + # First half: d = g*4 + i, write at out_base + d*N (strided by N). + offs_d_half = offs_g[:, None] * 4 + offs_i[None, :] # (BLOCK_GROUPS, 4) + mask_d_half = mask_g[:, None] + tl.store(q_out_ptr + out_base + offs_d_half * N, q_half_out, mask=mask_d_half) + tl.store(k_out_ptr + out_base + offs_d_half * N, k_half_out, mask=mask_d_half) + tl.store(v_out_ptr + out_base + offs_d_half * N, v_half_out, mask=mask_d_half) + + # Second half (RoPE region): d = D_HALF + r + offs_d_r = D_HALF + offs_r # (BLOCK_D_ROPE,) + tl.store(q_out_ptr + out_base + offs_d_r * N, q_rope_out, mask=mask_r) + tl.store(k_out_ptr + out_base + offs_d_r * N, k_rope_out, mask=mask_r) + tl.store(v_out_ptr + out_base + offs_d_r * N, v_rope_out, mask=mask_r) + + +@triton.jit +def _cam_prep_bwd_kernel( + # --- forward inputs (replayed for ReLU mask + k_post_kscale recompute) --- + q_raw_ptr, # (B, N, H, D) contiguous, any fp dtype + k_raw_ptr, # (B, N, H, D) contiguous (post short-conv on K) + q_norm_w_ptr, # (C,) = (H*D,) float32 + k_norm_w_ptr, # (C,) float32 + q_inv_rms_ptr, # (B, N) float32 — saved from forward + k_inv_rms_ptr, # (B, N) float32 + proj_q_ptr, # (B, N, 4, 4) — applied to Q first D/2 dims (P_T) + proj_kv_ptr, # (B, N, 4, 4) — applied to K,V first D/2 dims (P_inv) + rope_cos_ptr, # (N, D_rope) float32, D_rope = D//2 + rope_sin_ptr, # (N, D_rope) float32 + # --- upstream gradients (B, H, D, N) layout matching forward outputs --- + d_q_out_ptr, # grad of q_out (any dtype, cast to fp32 on load) + eff_d_k_out_ptr, # grad of k_out + inflation_sq contribution through k_out (fp32) + d_v_out_ptr, # grad of v_out + # --- inflation_sq direct grad to k_post_kscale^2 sum (B, H, N) fp32 --- + d_pre_k_sq_ptr, + # --- outputs --- + d_q_post_norm_ptr, # (B, N, H, D) fp32 — grad after RoPE^T+UCPE^T+Kscale+ReLU; consumed by torch RMSNorm bwd + d_k_post_norm_ptr, # (B, N, H, D) fp32 — same for K + dv_raw_ptr, # (B, N, H, D) fp32 — final dv_raw (V skips norm/ReLU/Kscale) + # --- dims --- + H: tl.constexpr, + N: tl.constexpr, + D: tl.constexpr, + D_HALF: tl.constexpr, + N_GROUPS: tl.constexpr, + K_SCALE, + # --- tile sizes --- + BLOCK_D_ROPE: tl.constexpr, # next pow2 of D_HALF (rope block) + BLOCK_GROUPS: tl.constexpr, # next pow2 of N_GROUPS +): + """One program per (b, n, h) — matches the forward kernel's parallelism. + + Implements the bwd of the fused fwd: + forward order: RMSNorm -> ReLU -> [K-scale on K] -> UCPE (first D/2) + -> RoPE (second D/2) -> output (B, H, D, N). + backward order: RoPE^T (second D/2) -> UCPE^T (first D/2) + -> K-scale (only K) -> ReLU mask -> emit d_post_norm + intermediates for the cross-head RMSNorm bwd handled + outside (in :func:`_cam_prep_bwd_dispatch`). + + The full-channel RMSNorm bwd's outer-product term and per-channel weight + grad both require a sum over ``H*D`` (the full ``C``) per token, which + couples heads. We deliberately leave that step in PyTorch (a couple of + fused element-wise + reduction ops) — see :func:`_cam_prep_bwd_dispatch`. + + Inflation handling: the kernel takes an *effective* ``dO_k`` that already + includes the contribution from ``grad_inflation_sq`` flowing through + ``k_out`` (i.e. ``eff_dO_k = grad_k + 2 * k_out * d_post_k_sq``), plus a + per-(b, h, n) scalar ``d_pre_k_sq`` that is the chain-rule contribution + of ``grad_inflation_sq`` into the pre-UCPE ``||k_post_kscale||^2`` sum. + Inside the kernel we recompute ``k_post_kscale`` (= post-norm * ReLU * + K_SCALE) and add ``2 * k_post_kscale[d] * d_pre_k_sq`` as a direct + contribution to ``d_k_post_kscale``. + """ + pid = tl.program_id(0) + h_idx = pid % H + bn_idx = pid // H + b_idx = bn_idx // N + n_idx = bn_idx % N + + # ---- load saved scalars: inv-RMS (per b, n) and d_pre_k_sq (per b, h, n) ---- + q_inv_rms = tl.load(q_inv_rms_ptr + bn_idx).to(tl.float32) + k_inv_rms = tl.load(k_inv_rms_ptr + bn_idx).to(tl.float32) + bhn_idx = (b_idx * H + h_idx) * N + n_idx + d_pre_k_sq = tl.load(d_pre_k_sq_ptr + bhn_idx).to(tl.float32) + + # ---- load per-token P matrices (shared across heads) ---- + proj_base = (b_idx * N + n_idx) * 16 + offs_i = tl.arange(0, 4) + offs_j = tl.arange(0, 4) + P_q = tl.load(proj_q_ptr + proj_base + offs_i[:, None] * 4 + offs_j[None, :]).to(tl.float32) + P_kv = tl.load(proj_kv_ptr + proj_base + offs_i[:, None] * 4 + offs_j[None, :]).to(tl.float32) + + # ---- layout offsets ---- + row_base = b_idx * (N * H * D) + n_idx * (H * D) + h_idx * D # (B, N, H, D) + nw_off = h_idx * D + out_base_BHDN = b_idx * (H * D * N) + h_idx * (D * N) + n_idx # (B, H, D, N) for dO_* + norm_base = row_base # d_*_post_norm and dv_raw share the (B, N, H, D) layout + + # ============================================================ + # First half — UCPE region + # ============================================================ + offs_g = tl.arange(0, BLOCK_GROUPS) + mask_g = offs_g < N_GROUPS + offs_gj = offs_g[:, None] * 4 + offs_j[None, :] # (BLOCK_GROUPS, 4) + mask_gj = mask_g[:, None] + + # Recompute post-norm (pre-ReLU) Q/K for the ReLU mask + k_post_kscale. + q_half_raw = tl.load(q_raw_ptr + row_base + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) + k_half_raw = tl.load(k_raw_ptr + row_base + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) + q_nw_half = tl.load(q_norm_w_ptr + nw_off + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) + k_nw_half = tl.load(k_norm_w_ptr + nw_off + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) + + q_post_norm_half = q_half_raw * q_inv_rms * q_nw_half + k_post_norm_half = k_half_raw * k_inv_rms * k_nw_half + q_relu_mask_half = q_post_norm_half > 0 + k_relu_mask_half = k_post_norm_half > 0 + + # k_post_kscale = relu(k_post_norm) * K_SCALE (used for direct inflation contribution) + k_post_relu_half = tl.where(k_relu_mask_half, k_post_norm_half, 0.0) + k_post_kscale_half = k_post_relu_half * K_SCALE + + # Load upstream gradients for first half. + # In (B, H, D, N): ptr[b, h, d, n] = out_base_BHDN + d * N. d = g*4 + i. + offs_d_half = offs_g[:, None] * 4 + offs_i[None, :] # (BLOCK_GROUPS, 4) + mask_d_half = mask_g[:, None] + dO_q_half = tl.load(d_q_out_ptr + out_base_BHDN + offs_d_half * N, mask=mask_d_half, other=0.0).to(tl.float32) + dO_k_eff_half = tl.load(eff_d_k_out_ptr + out_base_BHDN + offs_d_half * N, mask=mask_d_half, other=0.0).to( + tl.float32 + ) + dO_v_half = tl.load(d_v_out_ptr + out_base_BHDN + offs_d_half * N, mask=mask_d_half, other=0.0).to(tl.float32) + + # UCPE^T: din[g, j] = sum_i P[i, j] * dout[g, i] + # forward: out[g, i] = sum_j q[g, j] * P[i, j] -- so bwd sums over i + d_q_post_relu_half = tl.sum(dO_q_half[:, :, None] * P_q[None, :, :], axis=1) + d_k_post_kscale_via_ucpe_half = tl.sum(dO_k_eff_half[:, :, None] * P_kv[None, :, :], axis=1) + d_v_first_half = tl.sum(dO_v_half[:, :, None] * P_kv[None, :, :], axis=1) + + # K direct inflation contribution: 2 * k_post_kscale * d_pre_k_sq + d_k_post_kscale_half = d_k_post_kscale_via_ucpe_half + 2.0 * k_post_kscale_half * d_pre_k_sq + + # K-scale bwd (multiply by K_SCALE) + d_k_post_relu_half = d_k_post_kscale_half * K_SCALE + + # ReLU mask + d_q_post_norm_half = tl.where(q_relu_mask_half, d_q_post_relu_half, 0.0) + d_k_post_norm_half = tl.where(k_relu_mask_half, d_k_post_relu_half, 0.0) + + # Mask out-of-bounds groups to 0 explicitly (for safety on uneven N_GROUPS). + d_q_post_norm_half = tl.where(mask_d_half, d_q_post_norm_half, 0.0) + d_k_post_norm_half = tl.where(mask_d_half, d_k_post_norm_half, 0.0) + d_v_first_half = tl.where(mask_d_half, d_v_first_half, 0.0) + + # Store at (B, N, H, D), d = g*4+i + tl.store(d_q_post_norm_ptr + norm_base + offs_d_half, d_q_post_norm_half, mask=mask_d_half) + tl.store(d_k_post_norm_ptr + norm_base + offs_d_half, d_k_post_norm_half, mask=mask_d_half) + tl.store(dv_raw_ptr + norm_base + offs_d_half, d_v_first_half, mask=mask_d_half) + + # ============================================================ + # Second half — RoPE region + # ============================================================ + offs_r = tl.arange(0, BLOCK_D_ROPE) + mask_r = offs_r < D_HALF + offs_r_pair = offs_r ^ 1 + mask_r_pair = offs_r_pair < D_HALF + + rope_row = n_idx * D_HALF + cos_v = tl.load(rope_cos_ptr + rope_row + offs_r, mask=mask_r, other=1.0).to(tl.float32) + sin_v_pair = tl.load(rope_sin_ptr + rope_row + offs_r_pair, mask=mask_r_pair, other=0.0).to(tl.float32) + + # Recompute post-norm (pre-ReLU) Q/K for the ReLU mask. + rope_base = row_base + D_HALF + q_r_raw = tl.load(q_raw_ptr + rope_base + offs_r, mask=mask_r, other=0.0).to(tl.float32) + k_r_raw = tl.load(k_raw_ptr + rope_base + offs_r, mask=mask_r, other=0.0).to(tl.float32) + q_nw_r = tl.load(q_norm_w_ptr + nw_off + D_HALF + offs_r, mask=mask_r, other=0.0).to(tl.float32) + k_nw_r = tl.load(k_norm_w_ptr + nw_off + D_HALF + offs_r, mask=mask_r, other=0.0).to(tl.float32) + + q_post_norm_r = q_r_raw * q_inv_rms * q_nw_r + k_post_norm_r = k_r_raw * k_inv_rms * k_nw_r + q_relu_mask_r = q_post_norm_r > 0 + k_relu_mask_r = k_post_norm_r > 0 + + k_post_relu_r = tl.where(k_relu_mask_r, k_post_norm_r, 0.0) + k_post_kscale_r = k_post_relu_r * K_SCALE + + # Load upstream gradients (second half) — direct + pair. + offs_d_r = D_HALF + offs_r + offs_d_r_pair = D_HALF + offs_r_pair + dO_q_r = tl.load(d_q_out_ptr + out_base_BHDN + offs_d_r * N, mask=mask_r, other=0.0).to(tl.float32) + dO_k_eff_r = tl.load(eff_d_k_out_ptr + out_base_BHDN + offs_d_r * N, mask=mask_r, other=0.0).to(tl.float32) + dO_v_r = tl.load(d_v_out_ptr + out_base_BHDN + offs_d_r * N, mask=mask_r, other=0.0).to(tl.float32) + dO_q_r_pair = tl.load(d_q_out_ptr + out_base_BHDN + offs_d_r_pair * N, mask=mask_r_pair, other=0.0).to(tl.float32) + dO_k_eff_r_pair = tl.load(eff_d_k_out_ptr + out_base_BHDN + offs_d_r_pair * N, mask=mask_r_pair, other=0.0).to( + tl.float32 + ) + dO_v_r_pair = tl.load(d_v_out_ptr + out_base_BHDN + offs_d_r_pair * N, mask=mask_r_pair, other=0.0).to(tl.float32) + + # RoPE^T: forward y[r] = x[r]*cos[r] + x[r^1]*sin[r] + # bwd dx[r] = dy[r]*cos[r] + dy[r^1]*sin[r^1] + d_q_post_relu_r = dO_q_r * cos_v + dO_q_r_pair * sin_v_pair + d_k_post_kscale_via_rope_r = dO_k_eff_r * cos_v + dO_k_eff_r_pair * sin_v_pair + d_v_second_r = dO_v_r * cos_v + dO_v_r_pair * sin_v_pair + + # K direct inflation contribution + d_k_post_kscale_r = d_k_post_kscale_via_rope_r + 2.0 * k_post_kscale_r * d_pre_k_sq + + # K-scale bwd + d_k_post_relu_r = d_k_post_kscale_r * K_SCALE + + # ReLU mask + d_q_post_norm_r = tl.where(q_relu_mask_r, d_q_post_relu_r, 0.0) + d_k_post_norm_r = tl.where(k_relu_mask_r, d_k_post_relu_r, 0.0) + + # Out-of-bound mask + d_q_post_norm_r = tl.where(mask_r, d_q_post_norm_r, 0.0) + d_k_post_norm_r = tl.where(mask_r, d_k_post_norm_r, 0.0) + d_v_second_r = tl.where(mask_r, d_v_second_r, 0.0) + + norm_offs_r = D_HALF + offs_r + tl.store(d_q_post_norm_ptr + norm_base + norm_offs_r, d_q_post_norm_r, mask=mask_r) + tl.store(d_k_post_norm_ptr + norm_base + norm_offs_r, d_k_post_norm_r, mask=mask_r) + tl.store(dv_raw_ptr + norm_base + norm_offs_r, d_v_second_r, mask=mask_r) + + +@triton.jit +def _cam_scan_kernel( + # --- inputs (B, H, D, N) contiguous, fp32 --- + q_ptr, + k_ptr, + v_ptr, + # --- gates --- + beta_ptr, # (B, H, F, S) contiguous + decay_ptr, # (B, H, F) contiguous + # --- output (B, H, D, N) fp32 --- + out_ptr, + # --- saved state snapshots (used when SAVE_STATES=1) --- + state_pre_ptr, # (B, H, F, BLOCK_D, BLOCK_D) fp32 — state after decay, before update + state_post_ptr, # (B, H, F, BLOCK_D, BLOCK_D) fp32 — state after update + # --- forward-direction cache state (used when LOAD_INIT_STATE / SAVE_FINAL_STATE) --- + init_state_ptr, # (B*H, BLOCK_D, BLOCK_D) fp32 — state at end of prefix + final_state_ptr, # (B*H, BLOCK_D, BLOCK_D) fp32 — state after last frame's update + # --- dims --- + H: tl.constexpr, + F: tl.constexpr, + S: tl.constexpr, + D: tl.constexpr, + N: tl.constexpr, # F * S + REVERSE: tl.constexpr, + SAVE_STATES: tl.constexpr, + LOAD_INIT_STATE: tl.constexpr, + SAVE_FINAL_STATE: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """One program per (b, h) — runs the full numerator-only delta-rule scan. + + When ``SAVE_STATES=1`` the kernel additionally writes per-frame snapshots + of ``state_prev`` (state after applying ``decay``, before the K@delta + update) to ``state_pre_ptr`` and ``state_curr`` (state after the update) + to ``state_post_ptr``, both indexed by ``q_frame``. The bwd kernel + (``_cam_scan_bwd_kernel``) consumes these snapshots for both + ``REVERSE=0`` and ``REVERSE=1``. In ``REVERSE=1`` the slot at + ``q_frame=F-1`` always holds the all-zero state (skip-update, decay=1 + on the zero initial state), and the bwd kernel reads exactly that — + no special-case load is needed. + + When ``LOAD_INIT_STATE=1`` (forward direction only — wrapper enforces + ``REVERSE=0``) the per-program ``state_curr`` is initialized from + ``init_state_ptr`` instead of zero. The convention is: the loaded value + is the state AT THE END of a prefix sequence (i.e., AFTER the prefix's + last update, BEFORE any further decay applied here). On the very first + frame, the kernel's own ``state_curr *= g`` then applies ``decay[0]`` + to this loaded state — which is exactly the decay that the global + sequence's f=K-th frame would have applied. This keeps split/resume + state trajectories identical from frame K onwards. + + When ``SAVE_FINAL_STATE=1`` (forward direction only) the final + ``state_curr`` (after the last frame's update) is written to + ``final_state_ptr``. This is the state to be loaded with + ``LOAD_INIT_STATE`` for a downstream segment. + """ + pid = tl.program_id(0) + pid_b = pid // H + pid_h = pid % H + bh = pid_b * H + pid_h + + offs_d = tl.arange(0, BLOCK_D) + mask_d = offs_d < D + offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] + mask_dd = mask_d[:, None] & mask_d[None, :] + + base_bh = pid_b * (H * D * N) + pid_h * (D * N) + q_bh = q_ptr + base_bh + k_bh = k_ptr + base_bh + v_bh = v_ptr + base_bh + out_bh = out_ptr + base_bh + beta_bh = beta_ptr + bh * F * S + decay_bh = decay_ptr + bh * F + if SAVE_STATES: + spre_bh = state_pre_ptr + bh * F * BLOCK_D * BLOCK_D + spost_bh = state_post_ptr + bh * F * BLOCK_D * BLOCK_D + + # State: (D_k, D_v) in the upstream convention. Here we call rows "k-dim" + # (input dim of state) and cols "v-dim" (output dim of state). + if LOAD_INIT_STATE: + init_bh = init_state_ptr + bh * BLOCK_D * BLOCK_D + state_curr = tl.load(init_bh + offs_dd, mask=mask_dd, other=0.0).to(tl.float32) + else: + state_curr = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) + + for f_iter in range(F): + if REVERSE: + q_frame = F - 1 - f_iter + kv_frame = F - f_iter if f_iter > 0 else 0 + skip_update = f_iter == 0 + else: + q_frame = f_iter + kv_frame = f_iter + skip_update = False + + if REVERSE and f_iter == 0: + g = 1.0 + else: + g = tl.load(decay_bh + kv_frame).to(tl.float32) + state_curr = state_curr * g + state_prev = state_curr # fp32 snapshot (same tensor, kept for clarity) + + if SAVE_STATES: + tl.store( + spre_bh + q_frame * BLOCK_D * BLOCK_D + offs_dd, + state_prev, + mask=mask_dd, + ) + + if skip_update == 0: + kv_n_base = kv_frame * S + f_beta = beta_bh + kv_frame * S + + for s0 in range(0, S, BLOCK_S): + offs_s = s0 + tl.arange(0, BLOCK_S) + mask_s = offs_s < S + mask_sd = mask_s[:, None] & mask_d[None, :] + n_idx = kv_n_base + offs_s + + # Load K, V tiles (BLOCK_S, BLOCK_D) from (B, H, D, N) layout: + # ptr[s, d] = base_bh + offs_d[d] * N + n_idx[s] + k_ptrs = k_bh + offs_d[None, :] * N + n_idx[:, None] + v_ptrs = v_bh + offs_d[None, :] * N + n_idx[:, None] + K = tl.load(k_ptrs, mask=mask_sd, other=0.0) + V = tl.load(v_ptrs, mask=mask_sd, other=0.0) + + bt = tl.load(f_beta + offs_s, mask=mask_s, other=0.0).to(tl.float32) + + # V_pred = K @ state_prev : (BLOCK_S, BLOCK_D) + V_pred = tl.dot( + K, + state_prev, + out_dtype=tl.float32, + input_precision="tf32", + ) + dv = (V - V_pred) * bt[:, None] + state_curr += tl.dot( + tl.trans(K), + dv, + out_dtype=tl.float32, + input_precision="tf32", + ) + + if SAVE_STATES: + tl.store( + spost_bh + q_frame * BLOCK_D * BLOCK_D + offs_dd, + state_curr, + mask=mask_dd, + ) + + # --- Pass 2: output --- + state_out = state_curr + q_n_base = q_frame * S + for s0 in range(0, S, BLOCK_S): + offs_s = s0 + tl.arange(0, BLOCK_S) + mask_s = offs_s < S + mask_sd = mask_s[:, None] & mask_d[None, :] + n_idx = q_n_base + offs_s + + q_ptrs = q_bh + offs_d[None, :] * N + n_idx[:, None] + Q = tl.load(q_ptrs, mask=mask_sd, other=0.0) + + # num = Q @ state_out : (BLOCK_S, BLOCK_D), rows=S, cols=D_v + num = tl.dot( + Q, + state_out, + out_dtype=tl.float32, + input_precision="tf32", + ) + + # Store transposed into (B, H, D, N): + # ptr[d, s] = out_bh + offs_d[d] * N + n_idx[s] + out_ptrs = out_bh + offs_d[:, None] * N + n_idx[None, :] + mask_ds = mask_d[:, None] & mask_s[None, :] + tl.store(out_ptrs, tl.trans(num), mask=mask_ds) + + if SAVE_FINAL_STATE: + final_bh = final_state_ptr + bh * BLOCK_D * BLOCK_D + tl.store(final_bh + offs_dd, state_curr, mask=mask_dd) + + +# ============================================================================= +# Backward Triton kernel +# ============================================================================= + + +@triton.jit +def _cam_scan_bwd_kernel( + # --- forward inputs (B, H, D, N) fp32 contiguous --- + q_ptr, + k_ptr, + v_ptr, + # --- gates --- + beta_ptr, # (B, H, F, S) fp32 + decay_ptr, # (B, H, F) fp32 + # --- saved state snapshots (B*H, F, BLOCK_D, BLOCK_D) fp32, indexed by q_frame --- + state_pre_ptr, # state after decay, before update + state_post_ptr, # state after update + # --- upstream gradient (B, H, D, N) fp32 --- + grad_out_ptr, + # --- output gradients --- + dq_ptr, # (B, H, D, N) fp32 + dk_ptr, + dv_ptr, + dbeta_ptr, # (B, H, F, S) fp32 + ddecay_ptr, # (B, H, F) fp32 + # --- dims --- + H: tl.constexpr, + F: tl.constexpr, + S: tl.constexpr, + D: tl.constexpr, + N: tl.constexpr, # F * S + REVERSE: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """Reverse-time backward of the numerator-only delta-rule scan. + + One program per ``(b, h)``. Walks the same ``f_iter`` index space as the + forward kernel — but in reverse time order — replaying the recurrence + using the per-``q_frame`` state snapshots saved by the forward pass with + ``SAVE_STATES=1``. Accumulates gradients into ``dq``, ``dk``, ``dv``, + ``dbeta``, ``ddecay``. + + Forward indexing (matched here exactly):: + + REVERSE=False: q_frame = kv_frame = f_iter, skip_update = False + REVERSE=True : q_frame = F-1-f_iter, + kv_frame = F-f_iter (skip when f_iter==0) + g = decay[kv_frame] (1.0 when f_iter==0) + + Per-iteration backward derivation (when ``not skip_update``): + + ds_post += Q.T @ d_out # accumulate output grad + dQ[q] = d_out @ s_post.T + + ddelta = K @ ds_post # via K.T @ delta term + dV[kv] = ddelta * beta[kv,:] + dbeta[kv,:] = sum_d (ddelta * (V - K @ s_pre)) + dV_pred = -ddelta * beta[kv,:] + dK[kv] = delta @ ds_post.T + dV_pred @ s_pre.T + ds_pre = ds_post + K.T @ dV_pred # direct + V_pred path + + ddecay[kv] = sum(ds_pre * s_pre[q]) / g (= 0 when state_in is 0) + ds_post[next-bwd-iter] = ds_pre * g # propagate through decay + + For the ``skip_update`` branch (``REVERSE=True`` and ``f_iter==0``) the + update / decay path is bypassed: ``state_post == state_pre == 0`` so + ``dQ == 0``, ``ds_post`` is left unchanged across the iter, and + no writes to ``dK``, ``dV``, ``dbeta`` or ``ddecay`` happen at + ``kv_frame == 0``. Caller MUST pre-zero those output buffers (the + dispatch wrapper uses ``torch.zeros_like``). + + The ``ddecay`` formula uses ``state_pre / g``; for ``REVERSE=False`` at + fwd frame 0 we hardcode ``ddecay[0] = 0`` (state_in is exactly 0, but + the division would amplify any rounding noise). For very small ``g`` + the existing ``+ 1e-12`` epsilon is matched verbatim from + ``_fused_gdn_bwd_kernel``. + """ + pid = tl.program_id(0) + pid_b = pid // H + pid_h = pid % H + bh = pid_b * H + pid_h + + base_bh = pid_b * (H * D * N) + pid_h * (D * N) + q_bh = q_ptr + base_bh + k_bh = k_ptr + base_bh + v_bh = v_ptr + base_bh + do_bh = grad_out_ptr + base_bh + dq_bh = dq_ptr + base_bh + dk_bh = dk_ptr + base_bh + dv_bh = dv_ptr + base_bh + + beta_bh = beta_ptr + bh * F * S + decay_bh = decay_ptr + bh * F + dbeta_bh = dbeta_ptr + bh * F * S + ddecay_bh = ddecay_ptr + bh * F + + spre_bh = state_pre_ptr + bh * F * BLOCK_D * BLOCK_D + spost_bh = state_post_ptr + bh * F * BLOCK_D * BLOCK_D + + offs_d = tl.arange(0, BLOCK_D) + mask_d = offs_d < D + offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] + mask_dd = mask_d[:, None] & mask_d[None, :] + + # bf16 operands for tl.dot keep shared-memory pressure manageable for large + # BLOCK_D (e.g., 128 in reference). fp32 accumulators preserve precision. + grad_dtype = tl.bfloat16 + grad_ip: tl.constexpr = "tf32" + + # Reverse-time accumulator: gradient w.r.t. ``state_post`` for the iter + # currently being processed. + ds_post = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) + + for f_rev in range(F): + # Walk fwd iters in reverse: F-1, F-2, ..., 0. + f_iter = F - 1 - f_rev + + if REVERSE: + q_frame = F - 1 - f_iter + kv_frame = F - f_iter if f_iter > 0 else 0 + skip_update = f_iter == 0 + else: + q_frame = f_iter + kv_frame = f_iter + skip_update = 0 + + if REVERSE and f_iter == 0: + g = 1.0 + else: + g = tl.load(decay_bh + kv_frame).to(tl.float32) + + # ---- Pass 2: dQ + ds_post += Q.T @ d_out ---------------------- + # Load state_post[q_frame] (zero in the REVERSE skip-update slot — + # the fwd save still writes the all-zero state at that index). + state = tl.load( + spost_bh + q_frame * BLOCK_D * BLOCK_D + offs_dd, + mask=mask_dd, + other=0.0, + ) + + q_n_base = q_frame * S + for s0 in range(0, S, BLOCK_S): + offs_s = s0 + tl.arange(0, BLOCK_S) + mask_s = offs_s < S + mask_sd = mask_s[:, None] & mask_d[None, :] + n_idx = q_n_base + offs_s + + q_ptrs = q_bh + offs_d[None, :] * N + n_idx[:, None] + Q = tl.load(q_ptrs, mask=mask_sd, other=0.0) + + do_ptrs = do_bh + offs_d[None, :] * N + n_idx[:, None] + dO = tl.load(do_ptrs, mask=mask_sd, other=0.0) + + ds_post += tl.dot( + tl.trans(Q.to(grad_dtype)), + dO.to(grad_dtype), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + + dQ = tl.dot( + dO.to(grad_dtype), + tl.trans(state.to(grad_dtype)), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + + dq_ptrs = dq_bh + offs_d[:, None] * N + n_idx[None, :] + mask_ds = mask_d[:, None] & mask_s[None, :] + tl.store(dq_ptrs, tl.trans(dQ), mask=mask_ds) + + if skip_update == 0: + # ---- Reload state with state_pre[q_frame] for Pass 1 ---- + state = tl.load( + spre_bh + q_frame * BLOCK_D * BLOCK_D + offs_dd, + mask=mask_dd, + other=0.0, + ) + + # ds_pre starts equal to ds_post (direct pass-through term). + ds_pre = ds_post + + kv_n_base = kv_frame * S + for s0 in range(0, S, BLOCK_S): + offs_s = s0 + tl.arange(0, BLOCK_S) + mask_s = offs_s < S + mask_sd = mask_s[:, None] & mask_d[None, :] + n_idx = kv_n_base + offs_s + + k_ptrs = k_bh + offs_d[None, :] * N + n_idx[:, None] + v_ptrs = v_bh + offs_d[None, :] * N + n_idx[:, None] + K = tl.load(k_ptrs, mask=mask_sd, other=0.0) + V = tl.load(v_ptrs, mask=mask_sd, other=0.0) + + bt = tl.load(beta_bh + kv_frame * S + offs_s, mask=mask_s, other=0.0).to(tl.float32) + + V_pred = tl.dot( + K.to(grad_dtype), + state.to(grad_dtype), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + r = V - V_pred + delta = r * bt[:, None] + + ddelta = tl.dot( + K.to(grad_dtype), + ds_post.to(grad_dtype), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + + dV = ddelta * bt[:, None] + dv_ptrs = dv_bh + offs_d[:, None] * N + n_idx[None, :] + mask_ds = mask_d[:, None] & mask_s[None, :] + tl.store(dv_ptrs, tl.trans(dV), mask=mask_ds) + + dbeta_st = tl.sum(ddelta * r, axis=1) + tl.store(dbeta_bh + kv_frame * S + offs_s, dbeta_st, mask=mask_s) + + dV_pred = -ddelta * bt[:, None] + + dK_part1 = tl.dot( + delta.to(grad_dtype), + tl.trans(ds_post.to(grad_dtype)), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + dK_part2 = tl.dot( + dV_pred.to(grad_dtype), + tl.trans(state.to(grad_dtype)), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + dK = dK_part1 + dK_part2 + dk_ptrs = dk_bh + offs_d[:, None] * N + n_idx[None, :] + tl.store(dk_ptrs, tl.trans(dK), mask=mask_ds) + + ds_pre += tl.dot( + tl.trans(K.to(grad_dtype)), + dV_pred.to(grad_dtype), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + + # ---- ddecay[kv_frame] ---- + # state_pre = state_in * g, so state_in = state_pre / g. + # ddecay = sum(ds_pre * state_in) = sum(ds_pre * state_pre) / g. + # For REVERSE=False fwd frame 0, state_in is exactly 0 — hardcode + # 0 to avoid amplifying rounding noise via 1/g. + if (REVERSE == 0) and (f_iter == 0): + ddecay_f = 0.0 + else: + inv_g = 1.0 / (g + 1e-12) + ddecay_f = tl.sum(ds_pre * state) * inv_g + tl.store(ddecay_bh + kv_frame, ddecay_f) + + # Propagate to next bwd iter (which is fwd's previous iter). + ds_post = ds_pre * g + # else (skip_update branch): state_post == state_pre == 0, no + # ddecay write, no kv-side writes; ds_post passes through unchanged + # since ∂state_post/∂state_in = I when the update is skipped and g=1. + # In REVERSE=True this is the LAST bwd iter (f_iter=0) so the + # carried-over ds_post is discarded. + + +# ============================================================================= +# Python wrappers +# ============================================================================= + + +def cam_prep_func( + q_raw: torch.Tensor, + k_raw: torch.Tensor, + v_raw: torch.Tensor, + *, + q_norm_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + proj_q: torch.Tensor, # (B, N, 4, 4) + proj_kv: torch.Tensor, # (B, N, 4, 4) + rope_cos: torch.Tensor, # (N, D//2) + rope_sin: torch.Tensor, # (N, D//2) + k_scale: float, + norm_eps: float, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Fused RMSNorm + ReLU + (K-scale on K) + UCPE 4x4 + RoPE for the cam branch. + + Args: + q_raw, k_raw, v_raw: ``(B, N, H, D)`` contiguous (any fp dtype). + ``K`` must already have the short convolution applied. + q_norm_weight, k_norm_weight: ``(C,) = (H*D,)`` fp32. + proj_q, proj_kv: ``(B, N, 4, 4)`` fp32 (``P_T`` and ``P_inv`` in UCPE). + rope_cos, rope_sin: ``(N, D//2)`` fp32 interleaved-pair tables. + k_scale: ``(D^-0.5) * (S^-0.5)``. + norm_eps: RMSNorm epsilon. + + Returns: + q_trans, k_trans, v_trans: ``(B, H, D, N)`` same dtype as ``q_raw``. + inflation_sq: ``(B, H, N)`` fp32, ratio + ``(||k_post_ucpe|| / ||k_pre_ucpe||)^2`` per token/head. + """ + B, N, H, D = q_raw.shape + assert k_raw.shape == q_raw.shape and v_raw.shape == q_raw.shape + assert D % 2 == 0 and (D // 2) % 4 == 0, f"D={D} must be 2x and (D/2) % 4 == 0" + D_half = D // 2 + N_groups = D_half // 4 + + assert q_raw.is_contiguous() and k_raw.is_contiguous() and v_raw.is_contiguous() + assert proj_q.shape == (B, N, 4, 4) and proj_q.is_contiguous() + assert proj_kv.shape == (B, N, 4, 4) and proj_kv.is_contiguous() + assert rope_cos.shape == (N, D_half) and rope_cos.is_contiguous() + assert rope_sin.shape == (N, D_half) and rope_sin.is_contiguous() + assert q_norm_weight.numel() == H * D and q_norm_weight.dtype == torch.float32 + assert k_norm_weight.numel() == H * D and k_norm_weight.dtype == torch.float32 + + # Precompute inv-RMS over full C channels (shared across heads per token). + q_inv_rms = _precompute_cam_inv_rms(q_raw, norm_eps) + k_inv_rms = _precompute_cam_inv_rms(k_raw, norm_eps) + + out_dtype = q_raw.dtype + q_out = torch.empty(B, H, D, N, dtype=out_dtype, device=q_raw.device) + k_out = torch.empty(B, H, D, N, dtype=out_dtype, device=q_raw.device) + v_out = torch.empty(B, H, D, N, dtype=out_dtype, device=q_raw.device) + k_pre_sq = torch.empty(B, H, N, dtype=torch.float32, device=q_raw.device) + k_post_sq = torch.empty(B, H, N, dtype=torch.float32, device=q_raw.device) + + BLOCK_D_ROPE = triton.next_power_of_2(D_half) + BLOCK_GROUPS = triton.next_power_of_2(N_groups) + + grid = (B * N * H,) + _cam_prep_kernel[grid]( + q_raw, + k_raw, + v_raw, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + proj_q, + proj_kv, + rope_cos, + rope_sin, + q_out, + k_out, + v_out, + k_pre_sq, + k_post_sq, + H=H, + N=N, + D=D, + D_HALF=D_half, + N_GROUPS=N_groups, + K_SCALE=k_scale, + BLOCK_D_ROPE=BLOCK_D_ROPE, + BLOCK_GROUPS=BLOCK_GROUPS, + num_warps=1, + ) + # inflation_sq = (clamp(sqrt(post), 1e-6) / clamp(sqrt(pre), 1e-6))^2 + # = clamp(post, 1e-12) / clamp(pre, 1e-12) (equivalent). + inflation_sq = k_post_sq.clamp_min(1e-12) / k_pre_sq.clamp_min(1e-12) + return q_out, k_out, v_out, inflation_sq + + +def _run_cam_prep_fwd_save( + q_raw: torch.Tensor, + k_raw: torch.Tensor, + v_raw: torch.Tensor, + *, + q_norm_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + proj_q: torch.Tensor, + proj_kv: torch.Tensor, + rope_cos: torch.Tensor, + rope_sin: torch.Tensor, + k_scale: float, + norm_eps: float, +) -> tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, +]: + """Run :func:`cam_prep_func` while exposing intermediates needed by the bwd kernel. + + Mirrors :func:`cam_prep_func` exactly (same kernel launch, same outputs) + but additionally returns the per-token ``inv_rms`` for both Q and K, plus + the raw ``||k_pre_ucpe||^2`` and ``||k_post_ucpe||^2`` per ``(B, H, N)``. + These are required by :func:`_cam_prep_bwd_dispatch` to (a) replay the + ReLU/RMSNorm chain in fp32 without re-summing over the full ``C`` + channels and (b) chain ``grad_inflation_sq`` back through ``k_out`` and + the pre-UCPE ``||k||^2`` term with the correct ``clamp_min`` indicators. + + Returns: + ``(q_out, k_out, v_out, inflation_sq, q_inv_rms, k_inv_rms, + k_pre_sq, k_post_sq)``. The first four match :func:`cam_prep_func`; + the rest are fp32 contiguous saved-state tensors for the backward. + """ + B, N, H, D = q_raw.shape + assert k_raw.shape == q_raw.shape and v_raw.shape == q_raw.shape + assert D % 2 == 0 and (D // 2) % 4 == 0, f"D={D} must be 2x and (D/2) % 4 == 0" + D_half = D // 2 + N_groups = D_half // 4 + + assert q_raw.is_contiguous() and k_raw.is_contiguous() and v_raw.is_contiguous() + assert proj_q.shape == (B, N, 4, 4) and proj_q.is_contiguous() + assert proj_kv.shape == (B, N, 4, 4) and proj_kv.is_contiguous() + assert rope_cos.shape == (N, D_half) and rope_cos.is_contiguous() + assert rope_sin.shape == (N, D_half) and rope_sin.is_contiguous() + assert q_norm_weight.numel() == H * D and q_norm_weight.dtype == torch.float32 + assert k_norm_weight.numel() == H * D and k_norm_weight.dtype == torch.float32 + + q_inv_rms = _precompute_cam_inv_rms(q_raw, norm_eps) + k_inv_rms = _precompute_cam_inv_rms(k_raw, norm_eps) + + out_dtype = q_raw.dtype + q_out = torch.empty(B, H, D, N, dtype=out_dtype, device=q_raw.device) + k_out = torch.empty(B, H, D, N, dtype=out_dtype, device=q_raw.device) + v_out = torch.empty(B, H, D, N, dtype=out_dtype, device=q_raw.device) + k_pre_sq = torch.empty(B, H, N, dtype=torch.float32, device=q_raw.device) + k_post_sq = torch.empty(B, H, N, dtype=torch.float32, device=q_raw.device) + + BLOCK_D_ROPE = triton.next_power_of_2(D_half) + BLOCK_GROUPS = triton.next_power_of_2(N_groups) + + _cam_prep_kernel[(B * N * H,)]( + q_raw, + k_raw, + v_raw, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + proj_q, + proj_kv, + rope_cos, + rope_sin, + q_out, + k_out, + v_out, + k_pre_sq, + k_post_sq, + H=H, + N=N, + D=D, + D_HALF=D_half, + N_GROUPS=N_groups, + K_SCALE=k_scale, + BLOCK_D_ROPE=BLOCK_D_ROPE, + BLOCK_GROUPS=BLOCK_GROUPS, + num_warps=1, + ) + inflation_sq = k_post_sq.clamp_min(1e-12) / k_pre_sq.clamp_min(1e-12) + return q_out, k_out, v_out, inflation_sq, q_inv_rms, k_inv_rms, k_pre_sq, k_post_sq + + +def _cam_prep_bwd_dispatch( + q_raw: torch.Tensor, + k_raw: torch.Tensor, + q_norm_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + proj_q: torch.Tensor, + proj_kv: torch.Tensor, + rope_cos: torch.Tensor, + rope_sin: torch.Tensor, + q_inv_rms: torch.Tensor, + k_inv_rms: torch.Tensor, + k_pre_sq: torch.Tensor, + k_post_sq: torch.Tensor, + k_out: torch.Tensor, + *, + grad_q: torch.Tensor | None, + grad_k: torch.Tensor | None, + grad_v: torch.Tensor | None, + grad_inflation_sq: torch.Tensor | None, + k_scale: float, +) -> tuple[ + torch.Tensor | None, + torch.Tensor | None, + torch.Tensor | None, + torch.Tensor | None, + torch.Tensor | None, +]: + """Hybrid Triton + torch backward for :func:`cam_prep_func`. + + Pipeline: + + 1. **(torch)** Chain ``grad_inflation_sq`` through ``k_post_sq`` and + ``k_pre_sq`` to produce + (a) ``eff_d_k_out = grad_k + 2 * k_out * d_post_k_sq`` and + (b) ``d_pre_k_sq`` — a per-(B, H, N) scalar fed into the kernel as a + direct contribution to ``d_k_post_kscale``. Both ``clamp_min(1e-12)`` + indicators are honored so the gradient is exactly 0 in the (rare) + saturating regime, matching :func:`_torch_cam_prep_reference`. + + 2. **(Triton)** Launch :func:`_cam_prep_bwd_kernel` per ``(b, n, h)`` + to apply RoPE^T, UCPE^T, K-scale^T and the ReLU mask. Emits + ``d_q_post_norm`` / ``d_k_post_norm`` ``(B, N, H, D)`` fp32 + intermediates (the post-norm pre-RMSNorm grad slots) and writes + ``dv_raw`` directly (V skips RMSNorm/ReLU/K-scale). + + 3. **(torch)** Apply the full-channel RMSNorm bwd. The cross-head + coupling means ``S_q[b, n] = sum_{h,d} d_q_post_norm * q_raw * + q_norm_w`` is reduced over ``H*D``; we then form + ``dq_raw = d_q_post_norm * inv_rms_q * q_norm_w + - inv_rms_q^3 / C * q_raw * S_q`` + elementwise, and ``dq_norm_weight[c] = sum_{b,n} + d_q_post_norm[b,n,c] * q_raw[b,n,c] * inv_rms_q[b,n]``. + + Returns: + ``(dq_raw, dk_raw, dv_raw, dq_norm_weight, dk_norm_weight)``. Each + slot is ``None`` if upstream did not request that grad — handled + by the caller via ``ctx.needs_input_grad``. ``dq_raw`` / ``dk_raw`` + / ``dv_raw`` are returned in the same dtype as ``q_raw``; + norm-weight grads are fp32 (matching the input dtype). + """ + B, N, H, D = q_raw.shape + assert k_raw.shape == q_raw.shape + assert q_raw.is_contiguous() and k_raw.is_contiguous() + assert q_inv_rms.shape == (B, N) and k_inv_rms.shape == (B, N) + assert k_pre_sq.shape == (B, H, N) and k_post_sq.shape == (B, H, N) + D_half = D // 2 + N_groups = D_half // 4 + C = H * D + + # ---- prepare inflation_sq grad chain (in fp32) ---- + eps_floor = 1e-12 + pre_clamped = k_pre_sq.clamp_min(eps_floor) + if grad_inflation_sq is not None: + gis = grad_inflation_sq.to(torch.float32) + post_clamped = k_post_sq.clamp_min(eps_floor) + pre_indicator = (k_pre_sq >= eps_floor).to(torch.float32) + post_indicator = (k_post_sq >= eps_floor).to(torch.float32) + # d(inflation_sq)/d(post_k_sq) = (1 / pre_clamped) * post_indicator + # d(inflation_sq)/d(pre_k_sq) = -post_clamped / pre_clamped^2 * pre_indicator + d_post_k_sq = (post_indicator / pre_clamped) * gis # (B, H, N) + d_pre_k_sq = (-post_clamped / (pre_clamped * pre_clamped) * pre_indicator) * gis # (B, H, N) + else: + d_post_k_sq = torch.zeros_like(k_pre_sq) + d_pre_k_sq = torch.zeros_like(k_pre_sq) + + # eff_d_k_out: (B, H, D, N) fp32, contiguous + if grad_k is None: + grad_k_f32 = torch.zeros((B, H, D, N), dtype=torch.float32, device=q_raw.device) + else: + grad_k_f32 = grad_k.to(torch.float32) + if grad_inflation_sq is not None: + # k_out: (B, H, D, N), d_post_k_sq: (B, H, N) → broadcast over D dim. + eff_d_k_out = (grad_k_f32 + 2.0 * k_out.to(torch.float32) * d_post_k_sq.unsqueeze(2)).contiguous() + else: + eff_d_k_out = grad_k_f32.contiguous() + d_pre_k_sq = d_pre_k_sq.contiguous() + + # grad_q / grad_v as fp32 (B, H, D, N) contiguous (zero-fill if absent) + if grad_q is None: + grad_q_f32 = torch.zeros((B, H, D, N), dtype=torch.float32, device=q_raw.device) + else: + grad_q_f32 = grad_q.to(torch.float32).contiguous() + if grad_v is None: + grad_v_f32 = torch.zeros((B, H, D, N), dtype=torch.float32, device=q_raw.device) + else: + grad_v_f32 = grad_v.to(torch.float32).contiguous() + + # ---- allocate outputs / intermediates ---- + d_q_post_norm = torch.empty((B, N, H, D), dtype=torch.float32, device=q_raw.device) + d_k_post_norm = torch.empty((B, N, H, D), dtype=torch.float32, device=q_raw.device) + dv_raw_f32 = torch.empty((B, N, H, D), dtype=torch.float32, device=q_raw.device) + + BLOCK_D_ROPE = triton.next_power_of_2(D_half) + BLOCK_GROUPS = triton.next_power_of_2(N_groups) + + _cam_prep_bwd_kernel[(B * N * H,)]( + q_raw, + k_raw, + q_norm_weight, + k_norm_weight, + q_inv_rms, + k_inv_rms, + proj_q, + proj_kv, + rope_cos, + rope_sin, + grad_q_f32, + eff_d_k_out, + grad_v_f32, + d_pre_k_sq, + d_q_post_norm, + d_k_post_norm, + dv_raw_f32, + H=H, + N=N, + D=D, + D_HALF=D_half, + N_GROUPS=N_groups, + K_SCALE=k_scale, + BLOCK_D_ROPE=BLOCK_D_ROPE, + BLOCK_GROUPS=BLOCK_GROUPS, + num_warps=1, + ) + + # ---- torch RMSNorm bwd over the saved post-norm grads ---- + # Cast raw inputs to fp32 for the cross-head reduction (matches kernel + # numerics — the kernel uses fp32 internally as well). + q_raw_f32 = q_raw.to(torch.float32) + k_raw_f32 = k_raw.to(torch.float32) + q_inv_rms_view = q_inv_rms.view(B, N, 1, 1) + k_inv_rms_view = k_inv_rms.view(B, N, 1, 1) + q_nw_view = q_norm_weight.view(1, 1, H, D) + k_nw_view = k_norm_weight.view(1, 1, H, D) + + # S_q[b, n] = sum_{h, d} d_q_post_norm[b, n, h, d] * q_raw[b, n, h, d] * q_norm_w[h, d] + weighted_q = d_q_post_norm * q_raw_f32 # reused for dq_norm_weight reduction + weighted_k = d_k_post_norm * k_raw_f32 + S_q = (weighted_q * q_nw_view).sum(dim=(2, 3)) # (B, N) + S_k = (weighted_k * k_nw_view).sum(dim=(2, 3)) + + # dq_raw[b, n, h, d] = d_q_post_norm * inv_rms_q * q_norm_w + # - inv_rms_q^3 / C * q_raw * S_q + inv_q3 = (q_inv_rms**3).view(B, N, 1, 1) + inv_k3 = (k_inv_rms**3).view(B, N, 1, 1) + inv_C = 1.0 / float(C) + dq_raw_f32 = d_q_post_norm * q_inv_rms_view * q_nw_view - inv_q3 * inv_C * q_raw_f32 * S_q.view(B, N, 1, 1) + dk_raw_f32 = d_k_post_norm * k_inv_rms_view * k_nw_view - inv_k3 * inv_C * k_raw_f32 * S_k.view(B, N, 1, 1) + + # dq_norm_weight[h, d] = sum_{b, n} d_q_post_norm[b, n, h, d] * q_raw[b, n, h, d] * inv_rms_q[b, n] + dq_norm_weight = (weighted_q * q_inv_rms_view).sum(dim=(0, 1)).reshape(-1).contiguous() + dk_norm_weight = (weighted_k * k_inv_rms_view).sum(dim=(0, 1)).reshape(-1).contiguous() + + # Cast Q/K/V grads back to input dtype to match torch.autograd convention. + dq_raw = dq_raw_f32.to(q_raw.dtype) + dk_raw = dk_raw_f32.to(q_raw.dtype) + dv_raw = dv_raw_f32.to(q_raw.dtype) + + return dq_raw, dk_raw, dv_raw, dq_norm_weight, dk_norm_weight + + +def cam_scan_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + *, + reverse: bool = False, + init_state: torch.Tensor | None = None, + save_final_state: bool = False, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """Fused numerator-only single-path delta-rule scan for the cam branch. + + Args: + q, k, v: ``(B, H, D, N)`` fp32 contiguous tensors. + beta: ``(B, H, F, S)`` fp32 contiguous. + decay: ``(B, H, F)`` fp32 contiguous. + reverse: If ``True``, run the scan as the backward pass (equivalent + to ``flip_and_shift``-ing the inputs along the frame axis and + running forward). + init_state: optional ``(B*H, BLOCK_D, BLOCK_D)`` fp32 contiguous + tensor holding the forward-scan KV state at the END of a prefix + sequence (i.e., AFTER the prefix's last update, BEFORE any + further decay applied by this call). When provided, the kernel + resumes the scan from this state instead of zero. ``BLOCK_D = + next_pow2(D)`` and only the top-left ``D x D`` submatrix is + read. Forward direction only — raises ``NotImplementedError`` + if combined with ``reverse=True``. + save_final_state: when True, allocate a fresh fp32 zero buffer for + the final KV state (after the last frame's update) and pass it + to the kernel for write-out. Returned as the second tuple slot. + Forward direction only. + + Returns: + ``out`` of shape ``(B, H, D, N)`` fp32 matching + ``torch_chunk_cam_single_path_delta_rule`` with ``chunk_size >= T``. + + When ``save_final_state=True``, returns ``(out, final_state)`` where + ``final_state`` is fp32 ``(B*H, BLOCK_D, BLOCK_D)``. + + Raises: + NotImplementedError: if ``reverse=True`` is combined with state + passing. The cam branch's anti-causal scan resets per chunk in + the reference block, so there is no global cross-prefix state + to cache for the reverse direction. + """ + # Chunkwise integration (2026-05-06): dispatch all paths (fwd + reverse) + # to `cam_scan_chunkwise`. Reverse uses chunkwise's existing direction=2 + # mode in phase_b_triton, which has the same flip-and-shift semantics as + # cam's REVERSE=1 path. Bypass via FUSED_GDN_FORCE_LEGACY=1. + if os.environ.get("FUSED_GDN_FORCE_LEGACY", "0") != "1": + return cam_scan_chunkwise( + q, + k, + v, + beta, + decay, + reverse=reverse, + init_state=init_state, + save_final_state=save_final_state, + ) + + assert q.shape == k.shape == v.shape + B, H, D, N = q.shape + assert beta.shape[0] == B and beta.shape[1] == H + F_frames = beta.shape[2] + assert N % F_frames == 0 + S = N // F_frames + assert beta.shape == (B, H, F_frames, S), f"beta shape {beta.shape}" + assert decay.shape == (B, H, F_frames), f"decay shape {decay.shape}" + assert q.is_contiguous() and k.is_contiguous() and v.is_contiguous() + assert beta.is_contiguous() and decay.is_contiguous() + assert q.dtype == torch.float32 + + BLOCK_D = triton.next_power_of_2(D) + BLOCK_S = _DEFAULT_BLOCK_S + num_warps = 4 + num_stages = 1 + + if reverse and (init_state is not None or save_final_state): + raise NotImplementedError( + "cam_scan_func: state passing (init_state / save_final_state) is " + "only supported for the forward direction (reverse=False). The " + "cam branch's anti-causal pass resets per chunk; there is no " + "global cross-prefix state to cache for the reverse direction." + ) + + if init_state is not None: + expected_shape = (B * H, BLOCK_D, BLOCK_D) + if tuple(init_state.shape) != expected_shape: + raise ValueError( + f"cam_scan_func: init_state shape {tuple(init_state.shape)} " + f"does not match expected {expected_shape} (BLOCK_D=next_pow2(D)={BLOCK_D})." + ) + if init_state.dtype != torch.float32: + raise ValueError(f"cam_scan_func: init_state must be fp32 (got {init_state.dtype}).") + if not init_state.is_contiguous(): + raise ValueError("cam_scan_func: init_state must be contiguous.") + if init_state.device != q.device: + raise ValueError("cam_scan_func: init_state must be on the same device as q.") + load_init = 1 + else: + load_init = 0 + + if save_final_state: + final_state = torch.zeros(B * H, BLOCK_D, BLOCK_D, device=q.device, dtype=torch.float32) + save_final = 1 + else: + final_state = None + save_final = 0 + + out = torch.empty_like(q) + + dummy_state = torch.empty(1, device=q.device, dtype=torch.float32) + init_state_ptr = init_state if load_init else dummy_state + final_state_ptr = final_state if save_final else dummy_state + + _cam_scan_kernel[(B * H,)]( + q, + k, + v, + beta, + decay, + out, + dummy_state, + dummy_state, + init_state_ptr, + final_state_ptr, + H=H, + F=F_frames, + S=S, + D=D, + N=N, + REVERSE=1 if reverse else 0, + SAVE_STATES=0, + LOAD_INIT_STATE=load_init, + SAVE_FINAL_STATE=save_final, + BLOCK_D=BLOCK_D, + BLOCK_S=BLOCK_S, + num_warps=num_warps, + num_stages=num_stages, + ) + if save_final_state: + return out, final_state + return out + + +def _run_cam_scan_fwd_save( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + *, + reverse: bool = False, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Run the forward scan with per-frame state snapshots saved. + + Used by :class:`CamScanFunction` to preserve the ``(state_pre, state_post)`` + snapshots that the Triton bwd kernel consumes. Snapshots are indexed by + ``q_frame`` (matching the existing fwd-kernel save logic), so the bwd + kernel can load them with the same ``q_frame`` derived in its + ``REVERSE``-aware iteration. + + Returns: + (out, state_pre, state_post). ``state_pre`` and ``state_post`` are + ``(B, H, F, BLOCK_D, BLOCK_D)`` fp32 with ``BLOCK_D = next_pow2(D)``. + Padding columns/rows past ``D`` are zero-masked on store. + """ + assert q.shape == k.shape == v.shape + B, H, D, N = q.shape + F_frames = beta.shape[2] + S = N // F_frames + assert beta.shape == (B, H, F_frames, S) + assert decay.shape == (B, H, F_frames) + assert q.is_contiguous() and k.is_contiguous() and v.is_contiguous() + assert beta.is_contiguous() and decay.is_contiguous() + assert q.dtype == torch.float32 + + BLOCK_D = triton.next_power_of_2(D) + BLOCK_S = _DEFAULT_BLOCK_S + num_warps = 4 + num_stages = 1 + + out = torch.empty_like(q) + state_pre = torch.zeros(B * H, F_frames, BLOCK_D, BLOCK_D, device=q.device, dtype=torch.float32) + state_post = torch.zeros(B * H, F_frames, BLOCK_D, BLOCK_D, device=q.device, dtype=torch.float32) + dummy_state = torch.empty(1, device=q.device, dtype=torch.float32) + + _cam_scan_kernel[(B * H,)]( + q, + k, + v, + beta, + decay, + out, + state_pre, + state_post, + dummy_state, + dummy_state, + H=H, + F=F_frames, + S=S, + D=D, + N=N, + REVERSE=1 if reverse else 0, + SAVE_STATES=1, + LOAD_INIT_STATE=0, + SAVE_FINAL_STATE=0, + BLOCK_D=BLOCK_D, + BLOCK_S=BLOCK_S, + num_warps=num_warps, + num_stages=num_stages, + ) + return out, state_pre, state_post + + +def _cam_scan_bwd_dispatch( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + state_pre: torch.Tensor, + state_post: torch.Tensor, + grad_out: torch.Tensor, + *, + reverse: bool = False, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Launch ``_cam_scan_bwd_kernel`` and return ``(dq, dk, dv, dbeta, ddecay)``. + + All gradient outputs are fp32 contiguous, matching the dtype of the + forward inputs. ``grad_out`` is cast to fp32 before launching. + + When ``reverse=True``, ``kv_frame=0`` is never visited (only used in the + skipped first iter), so ``dk[..., 0, :]``, ``dv[..., 0, :]``, + ``dbeta[..., 0, :]`` and ``ddecay[..., 0]`` must remain zero. We pre-zero + every output buffer here so the kernel only needs to write the live slots. + """ + assert q.shape == k.shape == v.shape + B, H, D, N = q.shape + F_frames = beta.shape[2] + S = N // F_frames + + grad_out_f32 = grad_out.to(torch.float32).contiguous() + dq = torch.zeros_like(q) + dk = torch.zeros_like(k) + dv = torch.zeros_like(v) + dbeta = torch.zeros_like(beta) + ddecay = torch.zeros_like(decay) + + BLOCK_D = triton.next_power_of_2(D) + # For small S, ``next_pow2(S) < _DEFAULT_BLOCK_S`` — using the smaller value + # avoids zero-padding huge unused tiles into shared memory. + BLOCK_S = min(_DEFAULT_BLOCK_S, max(triton.next_power_of_2(S), 16)) + num_stages = 1 + REVERSE = 1 if reverse else 0 + + last_err: Exception | None = None + for num_warps in (4, 2, 1): + try: + _cam_scan_bwd_kernel[(B * H,)]( + q, + k, + v, + beta, + decay, + state_pre, + state_post, + grad_out_f32, + dq, + dk, + dv, + dbeta, + ddecay, + H=H, + F=F_frames, + S=S, + D=D, + N=N, + REVERSE=REVERSE, + BLOCK_D=BLOCK_D, + BLOCK_S=BLOCK_S, + num_warps=num_warps, + num_stages=num_stages, + ) + return dq, dk, dv, dbeta, ddecay + except triton.runtime.errors.OutOfResources as exc: + last_err = exc + continue + raise RuntimeError("_cam_scan_bwd_kernel exhausted all num_warps choices: " + str(last_err)) + + +# ============================================================================= +# Section: Torch reference implementations used by fallback backward paths +# ============================================================================= +# These references replicate the Triton-kernel math (full-channel RMSNorm + +# ReLU + K-scale + 4x4 UCPE projmat + interleaved-pair real-valued RoPE, then +# numerator-only single-path delta-rule scan). They run in fp32 internally and +# cast outputs back to the input dtype, matching the kernels. + + +def _flip_and_shift(x: torch.Tensor, dim: int, shift_val: float) -> torch.Tensor: + """Flip ``x`` along ``dim`` and right-shift by one (pad with ``shift_val``). + + Matches the reference ``sana_gdn_blocks.flip_and_shift`` semantics. + """ + x_flip = torch.flip(x, dims=[dim]) + x_shifted = x_flip.narrow(dim, 0, x.shape[dim] - 1) + pad_shape = list(x.shape) + pad_shape[dim] = 1 + padding = torch.full(pad_shape, shift_val, device=x.device, dtype=x.dtype) + return torch.cat([padding, x_shifted], dim=dim) + + +def _torch_cam_scan_single_chunk( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + init_state: torch.Tensor | None = None, + return_final_state: bool = False, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """Pure-torch single-chunk delta-rule scan (numerator-only). + + Algebraically equivalent to ``torch_chunk_cam_single_path_delta_rule`` with + ``chunk_size >= T``; matches the Triton ``_cam_scan_kernel`` math exactly + (which also runs as a single chunk over all F frames). + + Args: + q, k, v: ``(B, H, D, N)`` fp32 contiguous. + beta: ``(B, H, F, S)`` or ``(B, H, F)`` fp32 contiguous. + decay: ``(B, H, F)`` fp32 contiguous. + + Returns: + out: ``(B, H, D, N)`` fp32 contiguous, ``N = F * S``. + """ + B, H, D, N = q.shape + if beta.ndim == 4: + T = beta.shape[2] + elif beta.ndim == 3: + T = beta.shape[2] + else: + raise ValueError(f"beta must be (B,H,F[,S]); got ndim={beta.ndim}") + if N % T != 0: + raise ValueError(f"N ({N}) must be divisible by T ({T}).") + S = N // T + + def to_frame_seq(x: torch.Tensor) -> torch.Tensor: + return x.view(B, H, D, T, S).permute(0, 1, 3, 2, 4) # (B, H, T, D, S) + + q_t = to_frame_seq(q) + k_t = to_frame_seq(k) + v_t = to_frame_seq(v) + + if beta.ndim == 4: + beta_view = beta.unsqueeze(3) # (B, H, T, 1, S) + else: + beta_view = beta.view(B, H, T, 1, 1) + decay_view = decay.view(B, H, T, 1, 1) + + eye = torch.eye(D, device=q.device, dtype=q.dtype).view(1, 1, 1, D, D) + + k_beta = k_t * beta_view + W = decay_view * (eye - torch.matmul(k_beta, k_t.transpose(-1, -2))) + U = torch.matmul(v_t * beta_view, k_t.transpose(-1, -2)) + + state = ( + torch.zeros(B, H, D, D, device=q.device, dtype=q.dtype) + if init_state is None + else init_state.to(device=q.device, dtype=q.dtype) + ) + s_kv_list: list[torch.Tensor] = [] + for t in range(T): + state = torch.matmul(state, W[:, :, t]) + U[:, :, t] + s_kv_list.append(state) + s_all = torch.stack(s_kv_list, dim=2) # (B, H, T, D, D) + + out_t = torch.matmul(s_all, q_t) # (B, H, T, D, S) + out = out_t.permute(0, 1, 3, 2, 4).reshape(B, H, D, N) + return (out, state) if return_final_state else out + + +def _torch_cam_scan_reference( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + *, + reverse: bool = False, +) -> torch.Tensor: + """Pure-torch reference for ``cam_scan_func`` supporting ``reverse=True``. + + For ``reverse=False`` this is the standard forward delta-rule scan. + + For ``reverse=True`` we emulate the Triton kernel's per-chunk + ``flip_and_shift`` semantics (q is flipped only; k/v/beta are + flip-and-shifted with pad value 0; decay is flip-and-shifted with pad + value 1; output is then flipped back along the time axis). + """ + if not reverse: + return _torch_cam_scan_single_chunk(q, k, v, beta, decay) + + B, H, D, N = q.shape + if beta.ndim == 4: + T = beta.shape[2] + elif beta.ndim == 3: + T = beta.shape[2] + else: + raise ValueError(f"beta must be (B,H,F[,S]); got ndim={beta.ndim}") + S = N // T + + def to_frame(x: torch.Tensor) -> torch.Tensor: + return x.view(B, H, D, T, S).permute(0, 1, 3, 2, 4) # (B, H, T, D, S) + + def from_frame(x: torch.Tensor) -> torch.Tensor: + return x.permute(0, 1, 3, 2, 4).reshape(B, H, D, N) + + q_bwd = torch.flip(to_frame(q), dims=[2]) + k_bwd = _flip_and_shift(to_frame(k), dim=2, shift_val=0.0) + v_bwd = _flip_and_shift(to_frame(v), dim=2, shift_val=0.0) + beta_bwd = _flip_and_shift(beta, dim=2, shift_val=0.0) + decay_bwd = _flip_and_shift(decay, dim=2, shift_val=1.0) + + out_bwd = _torch_cam_scan_single_chunk( + from_frame(q_bwd), + from_frame(k_bwd), + from_frame(v_bwd), + beta_bwd, + decay_bwd, + ) + out_bwd_t = out_bwd.view(B, H, D, T, S) # already in (B, H, D, T, S) + return torch.flip(out_bwd_t, dims=[3]).reshape(B, H, D, N) + + +def _torch_cam_prep_reference( + q_raw: torch.Tensor, + k_raw: torch.Tensor, + v_raw: torch.Tensor, + *, + q_norm_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + proj_q: torch.Tensor, + proj_kv: torch.Tensor, + rope_cos: torch.Tensor, + rope_sin: torch.Tensor, + k_scale: float, + norm_eps: float, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Pure-torch reference for ``cam_prep_func`` matching ``_cam_prep_kernel``. + + Replicates exactly: + - Full-channel (over ``H*D``) RMSNorm + per-channel weight on Q, K. + - ReLU on Q, K. + - K-scale on K. + - 4x4 UCPE projmat on first ``D/2`` dims (Q via ``proj_q``, + K and V via ``proj_kv``). + - Interleaved-pair real-valued RoPE on second ``D/2`` dims using + ``rope_cos`` / ``rope_sin`` (the same tables passed to the kernel). + - ``inflation_sq = ||k_post_ucpe||^2 / ||k_pre_ucpe||^2`` per (B, H, N), + with the same ``clamp_min(1e-12)`` floor as ``cam_prep_func``. + + All math runs in fp32 internally; outputs ``(q, k, v)`` are cast back to + ``q_raw.dtype`` and ``inflation_sq`` is fp32. + """ + B, N, H, D = q_raw.shape + if D % 2 != 0: + raise ValueError(f"D ({D}) must be even.") + if (D // 2) % 4 != 0: + raise ValueError(f"D/2 ({D // 2}) must be divisible by 4 (UCPE projmat).") + C = H * D + D_half = D // 2 + n_groups = D_half // 4 + + q32 = q_raw.float() + k32 = k_raw.float() + v32 = v_raw.float() + + # ---- Full-channel RMSNorm + per-channel weight (Q, K only) ---- + q_inv_rms = torch.rsqrt((q32 * q32).sum(dim=(-1, -2)) / C + norm_eps) # (B, N) + k_inv_rms = torch.rsqrt((k32 * k32).sum(dim=(-1, -2)) / C + norm_eps) + q_nw = q_norm_weight.float().view(1, 1, H, D) + k_nw = k_norm_weight.float().view(1, 1, H, D) + q_normed = q32 * q_inv_rms.view(B, N, 1, 1) * q_nw + k_normed = k32 * k_inv_rms.view(B, N, 1, 1) * k_nw + + # ---- ReLU + K-scale ---- + q_normed = torch.relu(q_normed) + k_normed = torch.relu(k_normed) * k_scale + + # ---- Pre-UCPE ||k||^2 over the full D dim ---- + pre_k_sq_BNH = (k_normed * k_normed).sum(dim=-1) # (B, N, H) + + # ---- UCPE 4x4 projmat on first half ---- + q_first = q_normed[..., :D_half].reshape(B, N, H, n_groups, 4) + k_first = k_normed[..., :D_half].reshape(B, N, H, n_groups, 4) + v_first = v32[..., :D_half].reshape(B, N, H, n_groups, 4) + + # out[b,n,h,g,i] = sum_j P[b,n,i,j] * x[b,n,h,g,j] + # einsum: 'bnij,bnhgj->bnhgi' + proj_q_f = proj_q.float() + proj_kv_f = proj_kv.float() + q_first_proj = torch.einsum("bnij,bnhgj->bnhgi", proj_q_f, q_first).reshape(B, N, H, D_half) + k_first_proj = torch.einsum("bnij,bnhgj->bnhgi", proj_kv_f, k_first).reshape(B, N, H, D_half) + v_first_proj = torch.einsum("bnij,bnhgj->bnhgi", proj_kv_f, v_first).reshape(B, N, H, D_half) + + # ---- Interleaved-pair real-valued RoPE on second half ---- + # Kernel form: y[d] = x[d]*rope_cos[d] + x[d^1]*rope_sin[d] + # where rope_cos/rope_sin come from _prepare_ucpe_rope_tables. + q_second = q_normed[..., D_half:] + k_second = k_normed[..., D_half:] + v_second = v32[..., D_half:] + + def _pair_swap(x: torch.Tensor) -> torch.Tensor: + # Swap consecutive pairs along the last dim: (..., D_half) where D_half is even. + # x[..., 2i] <-> x[..., 2i+1]. + x_pairs = x.unflatten(-1, (D_half // 2, 2)) + x_swapped = x_pairs.flip(-1) + return x_swapped.flatten(-2) + + cos_b = rope_cos.float().view(1, N, 1, D_half) + sin_b = rope_sin.float().view(1, N, 1, D_half) + q_rope = q_second * cos_b + _pair_swap(q_second) * sin_b + k_rope = k_second * cos_b + _pair_swap(k_second) * sin_b + v_rope = v_second * cos_b + _pair_swap(v_second) * sin_b + + # ---- Reassemble (B, N, H, D) and post-UCPE k norm ---- + q_out_BNHD = torch.cat([q_first_proj, q_rope], dim=-1) + k_out_BNHD = torch.cat([k_first_proj, k_rope], dim=-1) + v_out_BNHD = torch.cat([v_first_proj, v_rope], dim=-1) + + post_k_sq_BNH = (k_out_BNHD * k_out_BNHD).sum(dim=-1) # (B, N, H) + + out_dtype = q_raw.dtype + q_out = q_out_BNHD.to(out_dtype).permute(0, 2, 3, 1).contiguous() + k_out = k_out_BNHD.to(out_dtype).permute(0, 2, 3, 1).contiguous() + v_out = v_out_BNHD.to(out_dtype).permute(0, 2, 3, 1).contiguous() + + pre_k_sq = pre_k_sq_BNH.permute(0, 2, 1).contiguous() # (B, H, N) + post_k_sq = post_k_sq_BNH.permute(0, 2, 1).contiguous() + inflation_sq = post_k_sq.clamp_min(1e-12) / pre_k_sq.clamp_min(1e-12) + return q_out, k_out, v_out, inflation_sq + + +# ============================================================================= +# Section: Autograd-enabled wrappers +# ============================================================================= + + +def _cam_scan_torch_fallback_backward( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + needs: tuple[bool, bool, bool, bool, bool], + grad_out: torch.Tensor, + reverse: bool, +) -> list[torch.Tensor | None]: + """Recompute the cam-branch scan via the torch reference and return grads. + + Used when ``CAM_SCAN_BWD_FALLBACK=1`` forces the torch-recompute backward + path. + """ + detached = [] + for tensor, need in zip((q, k, v, beta, decay), needs): + t = tensor.detach() + if need: + t = t.requires_grad_(True) + detached.append(t) + active = [t for t in detached if t.requires_grad] + + with torch.enable_grad(): + q_d, k_d, v_d, beta_d, decay_d = detached + ref_out = _torch_cam_scan_reference(q_d, k_d, v_d, beta_d, decay_d, reverse=reverse) + if active: + active_grads = torch.autograd.grad( + outputs=ref_out, + inputs=tuple(active), + grad_outputs=grad_out.to(ref_out.dtype), + allow_unused=True, + ) + else: + active_grads = [] + + grads: list[torch.Tensor | None] = [] + active_iter = iter(active_grads) + for tensor in detached: + grads.append(next(active_iter) if tensor.requires_grad else None) + return grads + + +class CamScanFunction(torch.autograd.Function): + """Autograd ``Function`` wrapping ``cam_scan_func``. + + Forward calls the Triton ``_cam_scan_kernel`` with ``SAVE_STATES=1`` so + per-frame state snapshots (``state_pre[q_frame]``, ``state_post[q_frame]``) + are kept for the backward pass. Backward runs the true Triton bwd + kernel (``_cam_scan_bwd_kernel``) for both ``reverse=False`` and + ``reverse=True``, replaying the recurrence in reverse time using the + saved snapshots. + + Set ``CAM_SCAN_BWD_FALLBACK=1`` to force the torch-recompute backward + validation path. + """ + + @staticmethod + def forward( + ctx, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + reverse: bool, + ) -> torch.Tensor: + ctx.set_materialize_grads(False) + ctx.reverse = bool(reverse) + + force_torch_fallback = os.environ.get("CAM_SCAN_BWD_FALLBACK", "0") == "1" + ctx.use_triton_bwd = not force_torch_fallback + + if ctx.use_triton_bwd: + out, state_pre, state_post = _run_cam_scan_fwd_save(q, k, v, beta, decay, reverse=ctx.reverse) + ctx.save_for_backward(q, k, v, beta, decay, state_pre, state_post) + return out + + # Torch-fallback backward path: don't bother saving state snapshots. + ctx.save_for_backward(q, k, v, beta, decay) + return cam_scan_func(q, k, v, beta, decay, reverse=reverse) + + @staticmethod + def backward(ctx, grad_out): # type: ignore[override] + if grad_out is None: + return (None, None, None, None, None, None) + + if ctx.use_triton_bwd: + q, k, v, beta, decay, state_pre, state_post = ctx.saved_tensors + needs = ctx.needs_input_grad[:5] # q, k, v, beta, decay + dq, dk, dv, dbeta, ddecay = _cam_scan_bwd_dispatch( + q, + k, + v, + beta, + decay, + state_pre, + state_post, + grad_out, + reverse=ctx.reverse, + ) + grads: list[torch.Tensor | None] = [ + dq if needs[0] else None, + dk if needs[1] else None, + dv if needs[2] else None, + dbeta if needs[3] else None, + ddecay if needs[4] else None, + ] + return (*grads, None) + + # Env-var torch-recompute backward. + q, k, v, beta, decay = ctx.saved_tensors + needs = ctx.needs_input_grad[:5] + grads = _cam_scan_torch_fallback_backward(q, k, v, beta, decay, needs, grad_out, ctx.reverse) + return (*grads, None) + + +def cam_scan_func_with_grad( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + *, + reverse: bool = False, +) -> torch.Tensor: + """Autograd-enabled wrapper around :func:`cam_scan_func`. + + Forward is identical to :func:`cam_scan_func`; backward is computed via + a torch reference (``_torch_cam_scan_reference``). Use this in training + paths where any of ``q, k, v, beta, decay`` may require gradients. + + Inference paths can keep calling :func:`cam_scan_func` directly to avoid + the small autograd bookkeeping overhead. + """ + return CamScanFunction.apply(q, k, v, beta, decay, reverse) + + +def _cam_prep_torch_fallback_backward( + q_raw: torch.Tensor, + k_raw: torch.Tensor, + v_raw: torch.Tensor, + q_norm_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + proj_q: torch.Tensor, + proj_kv: torch.Tensor, + rope_cos: torch.Tensor, + rope_sin: torch.Tensor, + needs: tuple[bool, ...], + grad_q: torch.Tensor | None, + grad_k: torch.Tensor | None, + grad_v: torch.Tensor | None, + grad_inflation_sq: torch.Tensor | None, + k_scale: float, + norm_eps: float, +) -> list[torch.Tensor | None]: + """Recompute the cam-branch prep via the torch reference and return grads. + + Used when any of ``proj_q / proj_kv / rope_cos / rope_sin`` requests a + gradient (the Triton kernel does not produce those grads), or when + ``CAM_PREP_BWD_FALLBACK=1`` forces the torch-recompute backward path. + + Args: + q_raw, k_raw, ..., rope_sin: the nine tensor inputs of + :func:`cam_prep_func` (in the same order as + :class:`CamPrepFunction.forward`'s arg list). + needs: ``ctx.needs_input_grad[:9]`` — boolean per-input flags. + grad_q, grad_k, grad_v, grad_inflation_sq: upstream gradients. + k_scale, norm_eps: scalar fwd args. + + Returns: + A 9-element list of ``torch.Tensor | None`` aligned with the + ``saved`` tuple. Entries that didn't request a gradient are ``None``. + """ + saved = ( + q_raw, + k_raw, + v_raw, + q_norm_weight, + k_norm_weight, + proj_q, + proj_kv, + rope_cos, + rope_sin, + ) + detached: list[torch.Tensor] = [] + for tensor, need in zip(saved, needs): + t = tensor.detach() + if need: + t = t.requires_grad_(True) + detached.append(t) + active = [t for t in detached if t.requires_grad] + + with torch.enable_grad(): + (q_d, k_d, v_d, qnw_d, knw_d, pq_d, pkv_d, rc_d, rs_d) = detached + ref_q, ref_k, ref_v, ref_inf = _torch_cam_prep_reference( + q_d, + k_d, + v_d, + q_norm_weight=qnw_d, + k_norm_weight=knw_d, + proj_q=pq_d, + proj_kv=pkv_d, + rope_cos=rc_d, + rope_sin=rs_d, + k_scale=k_scale, + norm_eps=norm_eps, + ) + + outputs = [] + grad_outputs = [] + if grad_q is not None: + outputs.append(ref_q) + grad_outputs.append(grad_q.to(ref_q.dtype)) + if grad_k is not None: + outputs.append(ref_k) + grad_outputs.append(grad_k.to(ref_k.dtype)) + if grad_v is not None: + outputs.append(ref_v) + grad_outputs.append(grad_v.to(ref_v.dtype)) + if grad_inflation_sq is not None: + outputs.append(ref_inf) + grad_outputs.append(grad_inflation_sq.to(ref_inf.dtype)) + + if active and outputs: + active_grads = torch.autograd.grad( + outputs=tuple(outputs), + inputs=tuple(active), + grad_outputs=tuple(grad_outputs), + allow_unused=True, + ) + else: + active_grads = [] + + grads: list[torch.Tensor | None] = [] + active_iter = iter(active_grads) + for tensor in detached: + grads.append(next(active_iter) if tensor.requires_grad else None) + return grads + + +class CamPrepFunction(torch.autograd.Function): + """Autograd ``Function`` wrapping ``cam_prep_func``. + + Forward calls the fused Triton ``_cam_prep_kernel`` via + :func:`_run_cam_prep_fwd_save` so the per-token ``inv_rms`` / + ``k_pre_sq`` / ``k_post_sq`` snapshots required by the bwd kernel are + preserved alongside the standard outputs. + + Backward runs the true Triton bwd kernel via + :func:`_cam_prep_bwd_dispatch` for the standard training path + (``q_raw``, ``k_raw``, ``v_raw``, ``q_norm_weight``, ``k_norm_weight`` + only request grads). The Triton path implements: + + * RoPE^T, UCPE^T, K-scale^T, and ReLU mask in a single fused kernel + (one program per ``(b, n, h)``); + * ``grad_inflation_sq`` chain through ``k_post_sq`` (added into + ``eff_dO_k``) and ``k_pre_sq`` (direct contribution to + ``d_k_post_kscale``), with ``clamp_min(1e-12)`` indicators honored; + * The full-channel RMSNorm bwd (per-token cross-head reduction) is + done in PyTorch on the kernel's ``d_q_post_norm`` / + ``d_k_post_norm`` intermediates — see + :func:`_cam_prep_bwd_dispatch` for details. + + The torch-recompute fallback (running :func:`_torch_cam_prep_reference` + under autograd) is selected when any of ``proj_q / proj_kv / rope_cos / + rope_sin`` requests a gradient (the Triton path emits ``None`` for those + slots) or when ``CAM_PREP_BWD_FALLBACK=1`` is set. + """ + + @staticmethod + def forward( + ctx, + q_raw: torch.Tensor, + k_raw: torch.Tensor, + v_raw: torch.Tensor, + q_norm_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + proj_q: torch.Tensor, + proj_kv: torch.Tensor, + rope_cos: torch.Tensor, + rope_sin: torch.Tensor, + k_scale: float, + norm_eps: float, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + ctx.set_materialize_grads(False) + ctx.k_scale = float(k_scale) + ctx.norm_eps = float(norm_eps) + + force_torch_fallback = os.environ.get("CAM_PREP_BWD_FALLBACK", "0") == "1" + ctx.use_triton_bwd = not force_torch_fallback + + if ctx.use_triton_bwd: + ( + q_out, + k_out, + v_out, + inflation_sq, + q_inv_rms, + k_inv_rms, + k_pre_sq, + k_post_sq, + ) = _run_cam_prep_fwd_save( + q_raw, + k_raw, + v_raw, + q_norm_weight=q_norm_weight, + k_norm_weight=k_norm_weight, + proj_q=proj_q, + proj_kv=proj_kv, + rope_cos=rope_cos, + rope_sin=rope_sin, + k_scale=k_scale, + norm_eps=norm_eps, + ) + ctx.save_for_backward( + q_raw, + k_raw, + v_raw, + q_norm_weight, + k_norm_weight, + proj_q, + proj_kv, + rope_cos, + rope_sin, + q_inv_rms, + k_inv_rms, + k_pre_sq, + k_post_sq, + k_out, + ) + else: + q_out, k_out, v_out, inflation_sq = cam_prep_func( + q_raw, + k_raw, + v_raw, + q_norm_weight=q_norm_weight, + k_norm_weight=k_norm_weight, + proj_q=proj_q, + proj_kv=proj_kv, + rope_cos=rope_cos, + rope_sin=rope_sin, + k_scale=k_scale, + norm_eps=norm_eps, + ) + ctx.save_for_backward( + q_raw, + k_raw, + v_raw, + q_norm_weight, + k_norm_weight, + proj_q, + proj_kv, + rope_cos, + rope_sin, + ) + return q_out, k_out, v_out, inflation_sq + + @staticmethod + def backward(ctx, grad_q, grad_k, grad_v, grad_inflation_sq): # type: ignore[override] + if grad_q is None and grad_k is None and grad_v is None and grad_inflation_sq is None: + return tuple([None] * 11) + + needs = ctx.needs_input_grad[:9] # nine tensor inputs + # If anyone outside (q_raw, k_raw, v_raw, q_norm_weight, k_norm_weight) + # requests a grad, the Triton bwd cannot handle it — fall back to the + # torch reference. + triton_bwd_supported_needs = needs[:5] + proj_or_rope_needs_grad = any(needs[5:]) + + if ctx.use_triton_bwd and not proj_or_rope_needs_grad: + ( + q_raw, + k_raw, + v_raw, + q_norm_weight, + k_norm_weight, + proj_q, + proj_kv, + rope_cos, + rope_sin, + q_inv_rms, + k_inv_rms, + k_pre_sq, + k_post_sq, + k_out, + ) = ctx.saved_tensors + + ( + dq_raw, + dk_raw, + dv_raw, + dq_norm_weight, + dk_norm_weight, + ) = _cam_prep_bwd_dispatch( + q_raw, + k_raw, + q_norm_weight, + k_norm_weight, + proj_q, + proj_kv, + rope_cos, + rope_sin, + q_inv_rms, + k_inv_rms, + k_pre_sq, + k_post_sq, + k_out, + grad_q=grad_q, + grad_k=grad_k, + grad_v=grad_v, + grad_inflation_sq=grad_inflation_sq, + k_scale=ctx.k_scale, + ) + grads: list[torch.Tensor | None] = [ + dq_raw if triton_bwd_supported_needs[0] else None, + dk_raw if triton_bwd_supported_needs[1] else None, + dv_raw if triton_bwd_supported_needs[2] else None, + dq_norm_weight if triton_bwd_supported_needs[3] else None, + dk_norm_weight if triton_bwd_supported_needs[4] else None, + None, # proj_q + None, # proj_kv + None, # rope_cos + None, # rope_sin + ] + return (*grads, None, None) + + # Torch fallback path. ``ctx.saved_tensors`` holds either 9 (legacy + # forward) or 14 (Triton fwd save) tensors — slice the leading nine. + saved = ctx.saved_tensors[:9] + ( + q_raw, + k_raw, + v_raw, + q_norm_weight, + k_norm_weight, + proj_q, + proj_kv, + rope_cos, + rope_sin, + ) = saved + grads = _cam_prep_torch_fallback_backward( + q_raw, + k_raw, + v_raw, + q_norm_weight, + k_norm_weight, + proj_q, + proj_kv, + rope_cos, + rope_sin, + needs=needs, + grad_q=grad_q, + grad_k=grad_k, + grad_v=grad_v, + grad_inflation_sq=grad_inflation_sq, + k_scale=ctx.k_scale, + norm_eps=ctx.norm_eps, + ) + # Two trailing None for non-tensor scalars (k_scale, norm_eps). + return (*grads, None, None) + + +def cam_prep_func_with_grad( + q_raw: torch.Tensor, + k_raw: torch.Tensor, + v_raw: torch.Tensor, + *, + q_norm_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + proj_q: torch.Tensor, + proj_kv: torch.Tensor, + rope_cos: torch.Tensor, + rope_sin: torch.Tensor, + k_scale: float, + norm_eps: float, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Autograd-enabled wrapper around :func:`cam_prep_func`. + + Forward is identical to :func:`cam_prep_func`; backward is computed via a + torch reference (``_torch_cam_prep_reference``). Use this in training + paths so gradients flow back through Q/K/V projection inputs and through + the RMSNorm weights. + + Inference paths can keep calling :func:`cam_prep_func` directly. + """ + return CamPrepFunction.apply( + q_raw, + k_raw, + v_raw, + q_norm_weight, + k_norm_weight, + proj_q, + proj_kv, + rope_cos, + rope_sin, + k_scale, + norm_eps, + ) + + +__all__ = [ + "CamPrepFunction", + "CamScanFunction", + "_cam_prep_bwd_dispatch", + "_cam_prep_bwd_kernel", + "_cam_prep_kernel", + "_cam_prep_torch_fallback_backward", + "_cam_scan_bwd_dispatch", + "_cam_scan_bwd_kernel", + "_cam_scan_kernel", + "_invert_SE3", + "_precompute_cam_inv_rms", + "_prepare_ucpe_rope_tables", + "_process_camera_conditions_raymats_only", + "_run_cam_prep_fwd_save", + "_run_cam_scan_fwd_save", + "_torch_cam_prep_reference", + "cam_prep_func", + "cam_prep_func_with_grad", + "cam_scan_func", + "cam_scan_func_with_grad", +] diff --git a/integrations/sana/sana_wm/ops/fused_gdn.py b/integrations/sana/sana_wm/ops/fused_gdn.py new file mode 100644 index 000000000..447e0f5ca --- /dev/null +++ b/integrations/sana/sana_wm/ops/fused_gdn.py @@ -0,0 +1,2249 @@ +"""Fused-BiGDN Triton kernels used by SANA-WM GDN attention blocks. + +Includes the unified forward kernel, backward kernels, RoPE/RMS helpers, and +autograd wrappers used by the Triton GDN attention blocks. + +Precision knob: env var ``FUSED_GDN_PRECISION`` or ``PRECISION_OVERRIDE``: + 0=IEEE fp32 dots, 1=TF32, 2=bf16 TC + fp32 state [default], 3=bf16 TC + bf16 state. +""" + +# ruff: noqa: E501 + +from __future__ import annotations + +import os + +import torch +import triton +import triton.language as tl + +# ===================================================================== +# GPU-adaptive kernel config +# ===================================================================== + + +def _get_kernel_config() -> dict: + """Return optimal kernel parameters for the current GPU. + + STATE_FP32: use fp32 state_prev when SRAM is large enough. + - bf16 state_prev: ~96KB total SRAM (fits GB10's 101KB). + - fp32 state_prev: ~128KB total SRAM (needs H100's 228KB+). + """ + if not torch.cuda.is_available(): + return {"BLOCK_S": 64, "num_stages": 1, "num_warps": 4, "STATE_FP32": False} + smem = torch.cuda.get_device_properties(0).shared_memory_per_multiprocessor + state_fp32 = smem >= 150 * 1024 # H100 (228KB) yes, GB10 (101KB) no + return {"BLOCK_S": 64, "num_stages": 1, "num_warps": 8, "STATE_FP32": state_fp32} + + +_KCFG = None + + +def _kcfg(): + global _KCFG + if _KCFG is None: + _KCFG = _get_kernel_config() + return _KCFG + + +# precision=0 → IEEE fp32 dots + fp32 state (DOT_PRECISION=2, STATE_FP32=1) +# precision=1 → TF32 dots + fp32 state (DOT_PRECISION=1, STATE_FP32=1) +# precision=2 → bf16 dots + fp32 state (DOT_PRECISION=0, STATE_FP32=1) [default] +# precision=3 → bf16 dots + bf16 state (DOT_PRECISION=0, STATE_FP32=0) +def _precision_params(precision: int) -> tuple: + if precision == 0: + return 2, True + elif precision == 1: + return 1, True + elif precision == 3: + return 0, False + else: # default + return 0, True + + +_env_prec = os.environ.get("FUSED_GDN_PRECISION", None) +PRECISION_OVERRIDE: int | None = int(_env_prec) if _env_prec is not None else None + + +def _resolve_launch_config() -> tuple: + """Returns (prec, dot_prec, state_fp32, num_warps). + + Uses ``PRECISION_OVERRIDE`` when set; otherwise falls back to ``_kcfg()`` + (which picks ``STATE_FP32`` based on per-GPU SRAM). ``num_warps`` is + clamped to 4 when dots run on fp32 operands (more registers needed). + """ + cfg = _kcfg() + prec = PRECISION_OVERRIDE if PRECISION_OVERRIDE is not None else 2 + dot_prec, state_fp32 = _precision_params(prec) + if PRECISION_OVERRIDE is None: + state_fp32 = cfg["STATE_FP32"] + nw = cfg["num_warps"] + if dot_prec >= 1: + nw = min(nw, 4) + return prec, dot_prec, state_fp32, nw + + +def _prepare_launch(D: int, beta: torch.Tensor, decay: torch.Tensor) -> tuple: + """Shared launcher preamble. + + Returns (BLOCK_D, BLOCK_S, dot_prec, state_fp32, nw, cfg, beta_c, decay_c). + ``beta_c`` / ``decay_c`` are the contiguous copies the kernel needs. + """ + BLOCK_D = triton.next_power_of_2(D) + cfg = _kcfg() + BLOCK_S = cfg["BLOCK_S"] + _, dot_prec, state_fp32, nw = _resolve_launch_config() + return BLOCK_D, BLOCK_S, dot_prec, state_fp32, nw, cfg, beta.contiguous(), decay.contiguous() + + +# ===================================================================== +# Unified forward Triton Mega-Kernel (inference-only variant) +# ===================================================================== +# Fuses: RMSNorm + ReLU + k_scale + RoPE + BiGDN recurrence. +# +# Inputs: +# qkv (B, N, 3, H, D) interleaved — strides passed explicitly. +# beta (B, H, F, S), decay (B, H, F) contiguous. +# q_norm_w, k_norm_w (H*D,) full-channel — only read when QK_NORM=1. +# rope_cos, rope_sin (N, D) contiguous. +# q_inv_rms, k_inv_rms (B, N) full-channel — only read when USE_PRECOMPUTED_RMS=1. +# +# Outputs: +# out (B, N, H, D) = num / (den + eps) — unused by BiGDN wrappers. +# num (B, N, H, D) = numerator before divide (summed across directions). +# den (B, H, N) = denominator before divide (summed across directions). +# +# NOTE (inference-only build): upstream also supports SAVE_STATE, +# LOAD_INIT_STATE, SAVE_FINAL_STATE for training backward / state caching. +# Those constexpr branches are preserved in the kernel so the source stays +# 1-for-1 with upstream (they compile away when launched with flags=0). + + +@triton.jit +def _fused_gdn_kernel( + # ---- interleaved QKV : (B, N, 3, H, D) ---- + qkv_ptr, + stride_b: tl.constexpr, + stride_n: tl.constexpr, + stride_3: tl.constexpr, + stride_h: tl.constexpr, + stride_d: tl.constexpr, + # ---- gates ---- + beta_ptr, + decay_ptr, + # ---- inv-RMS (B, N) — only read when USE_PRECOMPUTED_RMS=1 ---- + q_inv_rms_ptr, + k_inv_rms_ptr, + # ---- norm weights (H*D,) full-channel — only read when QK_NORM=1 ---- + q_norm_w_ptr, + k_norm_w_ptr, + # ---- RoPE tables (N, D) contiguous ---- + rope_cos_ptr, + rope_sin_ptr, + # ---- outputs ---- + out_ptr, # (B, N, H, D) + num_ptr, # (B, N, H, D) + den_ptr, # (B, H, N) + # ---- saved-state dummies (unused in this build but kept for signature parity) ---- + saved_state_ptr, + saved_z_ptr, + saved_state_curr_ptr, + saved_z_curr_ptr, + init_state_kv_ptr, + init_state_z_ptr, + final_state_kv_ptr, + final_state_z_ptr, + # ---- scalars / dims ---- + H: tl.constexpr, + F: tl.constexpr, + S: tl.constexpr, + D: tl.constexpr, + K_SCALE, + NORM_EPS: tl.constexpr, + EPS: tl.constexpr, + QK_NORM: tl.constexpr, + USE_PRECOMPUTED_RMS: tl.constexpr, + STATE_FP32: tl.constexpr, + DOT_PRECISION: tl.constexpr, + REVERSE: tl.constexpr, + SAVE_STATE: tl.constexpr, + LOAD_INIT_STATE: tl.constexpr, + SAVE_FINAL_STATE: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_S: tl.constexpr, +): + # ---- dot product precision / operand dtype ---- + if DOT_PRECISION >= 1: + dot_dtype = tl.float32 + else: + dot_dtype = tl.bfloat16 + dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" + + # ---- program → (batch, head) ---- + pid = tl.program_id(0) + pid_b = pid // H + pid_h = pid % H + N = F * S + bh = pid_b * H + pid_h + + # ---- base pointers ---- + qkv_bh = qkv_ptr + pid_b * stride_b + pid_h * stride_h + out_bh = out_ptr + pid_b * (N * H * D) + pid_h * D + num_bh = num_ptr + pid_b * (N * H * D) + pid_h * D + den_bh = den_ptr + bh * N + beta_bh = beta_ptr + bh * (F * S) + decay_bh = decay_ptr + bh * F + if SAVE_STATE: + st_bh = saved_state_ptr + bh * F * BLOCK_D * BLOCK_D + sz_bh = saved_z_ptr + bh * F * BLOCK_D + stc_bh = saved_state_curr_ptr + bh * F * BLOCK_D * BLOCK_D + szc_bh = saved_z_curr_ptr + bh * F * BLOCK_D + + # ---- D-index helpers ---- + offs_d = tl.arange(0, BLOCK_D) + mask_d = offs_d < D + offs_d_pair = offs_d ^ 1 + mask_d_pair = offs_d_pair < D + D_inv = 1.0 / D + + # ---- full-channel norm weights (only when QK_NORM=1) ---- + nw_offset = pid_h * D + if QK_NORM: + q_nw = tl.load(q_norm_w_ptr + nw_offset + offs_d, mask=mask_d, other=0.0).to(tl.float32) + k_nw = tl.load(k_norm_w_ptr + nw_offset + offs_d, mask=mask_d, other=0.0).to(tl.float32) + q_nw_pair = tl.load(q_norm_w_ptr + nw_offset + offs_d_pair, mask=mask_d_pair, other=0.0).to(tl.float32) + k_nw_pair = tl.load(k_norm_w_ptr + nw_offset + offs_d_pair, mask=mask_d_pair, other=0.0).to(tl.float32) + + k_scale = K_SCALE + offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] + mask_dd = mask_d[:, None] & mask_d[None, :] + + # ---- double-buffer state ---- + if LOAD_INIT_STATE: + init_kv_bh = init_state_kv_ptr + bh * BLOCK_D * BLOCK_D + state_curr = tl.load(init_kv_bh + offs_dd, mask=mask_dd, other=0.0).to(tl.float32) + init_z_bh = init_state_z_ptr + bh * BLOCK_D + state_z_curr = tl.load(init_z_bh + offs_d, mask=mask_d, other=0.0).to(tl.float32) + else: + state_curr = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) + state_z_curr = tl.zeros([BLOCK_D], dtype=tl.float32) + if STATE_FP32: + state_prev = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) + else: + state_prev = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.bfloat16) + state_z_prev = tl.zeros([BLOCK_D], dtype=tl.float32) + + # ======================================================== + # Temporal loop — serial over F + # ======================================================== + for f_iter in range(F): + if REVERSE: + q_frame = F - 1 - f_iter + kv_frame = F - f_iter if f_iter > 0 else 0 # unused at f=0 + skip_update = f_iter == 0 + else: + q_frame = f_iter + kv_frame = f_iter + skip_update = False + + # ---- decay + state snapshot ---- + if REVERSE and f_iter == 0: + g = 1.0 + else: + g = tl.load(decay_bh + kv_frame).to(tl.float32) + state_curr = state_curr * g + state_z_curr = state_z_curr * g + if STATE_FP32: + state_prev = state_curr + 0.0 + else: + state_prev = state_curr.to(tl.bfloat16) + state_z_prev = state_z_curr + + if SAVE_STATE: + st_f = st_bh + q_frame * BLOCK_D * BLOCK_D + tl.store(st_f + offs_dd, state_prev, mask=mask_dd) + tl.store(sz_bh + q_frame * BLOCK_D + offs_d, state_z_prev, mask=mask_d) + + # ------------------------------------------ + # Pass 1 — State Accumulation + # ------------------------------------------ + if skip_update == False: + kv_n_base = kv_frame * S + f_beta = beta_bh + kv_frame * S + + for s0 in range(0, S, BLOCK_S): + offs_s = s0 + tl.arange(0, BLOCK_S) + mask_s = offs_s < S + mask_sd = mask_s[:, None] & mask_d[None, :] + mask_sd_pair = mask_s[:, None] & mask_d_pair[None, :] + n_idx = kv_n_base + offs_s + + k_ptrs = qkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d[None, :] * stride_d + v_ptrs = qkv_bh + n_idx[:, None] * stride_n + 2 * stride_3 + offs_d[None, :] * stride_d + K_raw = tl.load(k_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + V_raw = tl.load(v_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + + if QK_NORM: + if USE_PRECOMPUTED_RMS: + k_inv_rms = tl.load(k_inv_rms_ptr + pid_b * N + n_idx, mask=mask_s, other=1.0).to(tl.float32) + else: + k_var = tl.sum(K_raw * K_raw, axis=1) * D_inv + k_inv_rms = 1.0 / tl.sqrt(k_var + NORM_EPS) + K_normed = K_raw * k_inv_rms[:, None] * k_nw[None, :] + else: + K_normed = K_raw + K = tl.where(K_normed > 0, K_normed, 0.0) * k_scale + + k_pair_ptrs = qkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d_pair[None, :] * stride_d + K_pair_raw = tl.load(k_pair_ptrs, mask=mask_sd_pair, other=0.0).to(tl.float32) + if QK_NORM: + K_pair_normed = K_pair_raw * k_inv_rms[:, None] * k_nw_pair[None, :] + else: + K_pair_normed = K_pair_raw + K_pair = tl.where(K_pair_normed > 0, K_pair_normed, 0.0) * k_scale + + rope_ptrs = n_idx[:, None] * D + offs_d[None, :] + Cos = tl.load(rope_cos_ptr + rope_ptrs, mask=mask_sd, other=1.0).to(tl.float32) + Sin = tl.load(rope_sin_ptr + rope_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + K_rot = K * Cos + K_pair * Sin + + bt = tl.load(f_beta + offs_s, mask=mask_s, other=0.0).to(tl.float32) + + K_rot_dc = K_rot.to(dot_dtype) + V_pred = tl.dot(K_rot_dc, state_prev.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) + dv = (V_raw - V_pred) * bt[:, None] + state_curr += tl.dot(tl.trans(K_rot), dv, out_dtype=tl.float32, input_precision="tf32") + + z_hat = tl.sum(K * state_z_prev[None, :], axis=1) + dz = (1.0 - z_hat) * bt + state_z_curr += tl.sum(K * dz[:, None], axis=0) + + if SAVE_STATE: + stc_f = stc_bh + q_frame * BLOCK_D * BLOCK_D + tl.store(stc_f + offs_dd, state_curr, mask=mask_dd) + tl.store(szc_bh + q_frame * BLOCK_D + offs_d, state_z_curr, mask=mask_d) + + # ------------------------------------------ + # Pass 2 — Output (reads state_curr, inclusive) + # ------------------------------------------ + state_out = state_curr.to(dot_dtype) + state_z_out = state_z_curr + q_n_base = q_frame * S + + for s0 in range(0, S, BLOCK_S): + offs_s = s0 + tl.arange(0, BLOCK_S) + mask_s = offs_s < S + mask_sd = mask_s[:, None] & mask_d[None, :] + mask_sd_pair = mask_s[:, None] & mask_d_pair[None, :] + n_idx = q_n_base + offs_s + + q_ptrs = qkv_bh + n_idx[:, None] * stride_n + 0 * stride_3 + offs_d[None, :] * stride_d + q_pair_ptrs = qkv_bh + n_idx[:, None] * stride_n + 0 * stride_3 + offs_d_pair[None, :] * stride_d + Q_raw = tl.load(q_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + Q_pair_raw = tl.load(q_pair_ptrs, mask=mask_sd_pair, other=0.0).to(tl.float32) + + if QK_NORM: + if USE_PRECOMPUTED_RMS: + q_inv_rms = tl.load(q_inv_rms_ptr + pid_b * N + n_idx, mask=mask_s, other=1.0).to(tl.float32) + else: + q_var = tl.sum(Q_raw * Q_raw, axis=1) * D_inv + q_inv_rms = 1.0 / tl.sqrt(q_var + NORM_EPS) + Q_normed = Q_raw * q_inv_rms[:, None] * q_nw[None, :] + Q_pair_normed = Q_pair_raw * q_inv_rms[:, None] * q_nw_pair[None, :] + else: + Q_normed = Q_raw + Q_pair_normed = Q_pair_raw + Q = tl.where(Q_normed > 0, Q_normed, 0.0) + Q_pair = tl.where(Q_pair_normed > 0, Q_pair_normed, 0.0) + + rope_ptrs = n_idx[:, None] * D + offs_d[None, :] + Cos = tl.load(rope_cos_ptr + rope_ptrs, mask=mask_sd, other=1.0).to(tl.float32) + Sin = tl.load(rope_sin_ptr + rope_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + Q_rot = Q * Cos + Q_pair * Sin + + num = tl.dot(Q_rot.to(dot_dtype), state_out, out_dtype=tl.float32, input_precision=dot_ip) + den = tl.sum(Q * state_z_out[None, :], axis=1) + + result = num / (den[:, None] + EPS) + out_ptrs = out_bh + n_idx[:, None] * (H * D) + offs_d[None, :] + num_ptrs = num_bh + n_idx[:, None] * (H * D) + offs_d[None, :] + tl.store(out_ptrs, result.to(tl.bfloat16), mask=mask_sd) + tl.store(num_ptrs, num.to(tl.bfloat16), mask=mask_sd) + tl.store(den_bh + n_idx, den.to(tl.bfloat16), mask=mask_s) + + if SAVE_FINAL_STATE: + final_kv_bh = final_state_kv_ptr + bh * BLOCK_D * BLOCK_D + tl.store(final_kv_bh + offs_dd, state_curr, mask=mask_dd) + final_z_bh = final_state_z_ptr + bh * BLOCK_D + tl.store(final_z_bh + offs_d, state_z_curr, mask=mask_d) + + +# ===================================================================== +# Python wrappers +# ===================================================================== + + +def prepare_rope_tables(rotary_emb, N: int, D: int, device) -> tuple[torch.Tensor, torch.Tensor]: + """Complex rotary_emb `(1, 1, N, D//2)` → expanded (N, D) cos/sin tables. + + Encodes the interleaved-pair rotation + y[2i] = x[2i]*cos[i] - x[2i+1]*sin[i] + y[2i+1] = x[2i]*sin[i] + x[2i+1]*cos[i] + as y[d] = x[d]*cos_exp[d] + x[d^1]*sin_exp[d] + where sin_exp[2i] = -sin[i], sin_exp[2i+1] = +sin[i]. + + Returns (cos_exp, sin_exp) both (N, D) float32, contiguous. + """ + if rotary_emb is None: + return ( + torch.ones(N, D, device=device, dtype=torch.float32), + torch.zeros(N, D, device=device, dtype=torch.float32), + ) + freqs = rotary_emb.squeeze(0).squeeze(0) # (N, D//2) complex + cos_half = freqs.real.float() + sin_half = freqs.imag.float() + rope_cos = cos_half.repeat_interleave(2, dim=-1) + rope_sin = torch.stack([-sin_half, sin_half], dim=-1).reshape(N, D) + return rope_cos.contiguous(), rope_sin.contiguous() + + +def _precompute_inv_rms(qkv: torch.Tensor, idx: int, C: int, eps: float = 1e-5) -> torch.Tensor: + """Compute 1/RMS for one component of QKV over the full C = H*D channel dim. + + Args: + qkv: (B, N, 3, H, D) + idx: 0 for Q, 1 for K, 2 for V + C: H*D (channel count) + eps: RMSNorm epsilon + + Returns: + inv_rms: (B, N) float32 + """ + raw = qkv[:, :, idx].float() # (B, N, H, D) + sq_sum = (raw * raw).sum(dim=(-2, -1)) # (B, N) + return torch.rsqrt(sq_sum / C + eps) + + +# ===================================================================== +# Fused single-pass Q+K inverse-RMS Triton kernel +# ===================================================================== +# Single Triton launch that reads each `(b, n)` row of `qkv` once and emits +# both `q_inv_rms[b, n]` and `k_inv_rms[b, n]`. Replaces two separate PyTorch +# scans (cast→square→sum→rsqrt) over `qkv[:, :, 0]` and `qkv[:, :, 1]`. +# +# Layout assumed: `qkv` is (B, N, 3, H, D) contiguous, so the C = H*D channels +# for a given (b, n, qkv_idx) live in a contiguous memory span. + + +@triton.jit +def _fused_qk_inv_rms_kernel( + qkv_ptr, # *T_in (B, N, 3, H, D), contiguous + q_inv_rms_ptr, # *float32 (B, N) + k_inv_rms_ptr, # *float32 (B, N) + N: tl.constexpr, + C: tl.constexpr, # H * D + eps, + BLOCK_C: tl.constexpr, +): + bn_id = tl.program_id(0) + qkv_row_stride = 3 * C + row_base = bn_id * qkv_row_stride + q_base = row_base + k_base = row_base + C + + offs = tl.arange(0, BLOCK_C) + mask = offs < C + + q_vals = tl.load(qkv_ptr + q_base + offs, mask=mask, other=0.0).to(tl.float32) + k_vals = tl.load(qkv_ptr + k_base + offs, mask=mask, other=0.0).to(tl.float32) + + q_sq = tl.sum(q_vals * q_vals, axis=0) + k_sq = tl.sum(k_vals * k_vals, axis=0) + + inv_c = 1.0 / C + q_inv = tl.rsqrt(q_sq * inv_c + eps) + k_inv = tl.rsqrt(k_sq * inv_c + eps) + + tl.store(q_inv_rms_ptr + bn_id, q_inv) + tl.store(k_inv_rms_ptr + bn_id, k_inv) + + +def fused_qk_inv_rms( + qkv: torch.Tensor, + eps: float = 1e-5, +) -> tuple[torch.Tensor, torch.Tensor]: + """Single-pass Triton fused Q+K inverse-RMS. + + Replaces ``(_precompute_inv_rms(qkv, 0, C, eps), _precompute_inv_rms(qkv, 1, C, eps))`` + with one launch that reads each ``(b, n)`` row of ``qkv`` exactly once. + + Args: + qkv: (B, N, 3, H, D) contiguous tensor, any fp dtype. + eps: RMSNorm epsilon. + + Returns: + (q_inv_rms, k_inv_rms), each (B, N) float32 contiguous. + """ + assert qkv.is_contiguous(), "qkv must be contiguous (B, N, 3, H, D)" + assert qkv.dim() == 5 and qkv.shape[2] == 3, f"expected (B, N, 3, H, D), got {tuple(qkv.shape)}" + B, N, _, H, D = qkv.shape + C = H * D + q_inv_rms = torch.empty((B, N), dtype=torch.float32, device=qkv.device) + k_inv_rms = torch.empty((B, N), dtype=torch.float32, device=qkv.device) + BLOCK_C = triton.next_power_of_2(C) + _fused_qk_inv_rms_kernel[(B * N,)]( + qkv, + q_inv_rms, + k_inv_rms, + N=N, + C=C, + eps=eps, + BLOCK_C=BLOCK_C, + ) + return q_inv_rms, k_inv_rms + + +@triton.jit +def _fused_bidi_merge_kernel( + num_fwd_ptr, + num_bwd_ptr, + den_fwd_ptr, + den_bwd_ptr, + gate_ptr, + out_ptr, + B, + N, + H, + D, + eps, + snum_b, + snum_n, + snum_h, + snum_d, + sden_b, + sden_h, + sden_n, + APPLY_GATE: tl.constexpr, + PRE_SUMMED: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid_bh = tl.program_id(0) + pid_n = tl.program_id(1) + b = pid_bh // H + h = pid_bh % H + + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + mask_n = offs_n < N + mask_d = offs_d < D + mask_nd = mask_n[:, None] & mask_d[None, :] + + num_base = b * snum_b + offs_n[:, None] * snum_n + h * snum_h + offs_d[None, :] * snum_d + nf = tl.load(num_fwd_ptr + num_base, mask=mask_nd, other=0.0).to(tl.float32) + den_base = b * sden_b + h * sden_h + offs_n * sden_n + df = tl.load(den_fwd_ptr + den_base, mask=mask_n, other=0.0).to(tl.float32) + + if PRE_SUMMED: + num_total = nf + den_total = df + eps + else: + nb = tl.load(num_bwd_ptr + num_base, mask=mask_nd, other=0.0).to(tl.float32) + db = tl.load(den_bwd_ptr + den_base, mask=mask_n, other=0.0).to(tl.float32) + num_total = nf + nb + den_total = df + db + eps + out_val = num_total / den_total[:, None] + + if APPLY_GATE: + g = tl.load(gate_ptr + num_base, mask=mask_nd, other=0.0).to(tl.float32) + silu_g = g * (1.0 / (1.0 + tl.exp(-g))) + out_val = out_val * silu_g + + tl.store(out_ptr + num_base, out_val.to(tl.bfloat16), mask=mask_nd) + + +def fused_bidi_merge( + num_fwd: torch.Tensor, + num_bwd: torch.Tensor | None, + den_fwd: torch.Tensor, + den_bwd: torch.Tensor | None, + eps: float, + gate: torch.Tensor | None = None, +) -> torch.Tensor: + pre_summed = num_bwd is None + assert (num_bwd is None) == (den_bwd is None), "num_bwd/den_bwd must both be None or both provided" + if not pre_summed: + assert num_fwd.shape == num_bwd.shape and den_fwd.shape == den_bwd.shape + assert num_fwd.dtype == num_bwd.dtype and den_fwd.dtype == den_bwd.dtype + B, N, H, D = num_fwd.shape + out = torch.empty( + B, N, H, D, device=num_fwd.device, dtype=(torch.float32 if num_fwd.dtype == torch.float32 else torch.bfloat16) + ) + BLOCK_D = triton.next_power_of_2(D) + BLOCK_N = 64 + grid = (B * H, triton.cdiv(N, BLOCK_N)) + if gate is not None: + assert gate.shape == (B, N, H, D), f"gate shape {gate.shape} != {(B, N, H, D)}" + gate_arg = gate + apply_gate = 1 + else: + gate_arg = num_fwd + apply_gate = 0 + num_bwd_arg = num_bwd if num_bwd is not None else num_fwd + den_bwd_arg = den_bwd if den_bwd is not None else den_fwd + _fused_bidi_merge_kernel[grid]( + num_fwd, + num_bwd_arg, + den_fwd, + den_bwd_arg, + gate_arg, + out, + B, + N, + H, + D, + float(eps), + num_fwd.stride(0), + num_fwd.stride(1), + num_fwd.stride(2), + num_fwd.stride(3), + den_fwd.stride(0), + den_fwd.stride(1), + den_fwd.stride(2), + APPLY_GATE=apply_gate, + PRE_SUMMED=1 if pre_summed else 0, + BLOCK_N=BLOCK_N, + BLOCK_D=BLOCK_D, + ) + return out + + +# ===================================================================== +# Single-direction GDN entry point (delegates to chunkwise) +# ===================================================================== + + +def fused_gdn_func( + qkv: torch.Tensor, # (B, N, 3, H, D) + q_inv_rms: torch.Tensor, # (B, N) float32 + k_inv_rms: torch.Tensor, # (B, N) float32 + q_norm_weight: torch.Tensor, # (C,) = (H*D,) float32 + k_norm_weight: torch.Tensor, # (C,) float32 + rope_cos: torch.Tensor, # (N, D) float32 + rope_sin: torch.Tensor, # (N, D) float32 + beta: torch.Tensor, # (B, H, F, S) + decay: torch.Tensor, # (B, H, F) + F: int, + S: int, + k_scale: float, + eps: float = 1e-6, + reverse: bool = False, + init_state_kv: torch.Tensor | None = None, + init_state_z: torch.Tensor | None = None, + save_final_state: bool = False, +) -> tuple[torch.Tensor, torch.Tensor] | tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """One direction of fused BiGDN via the unified kernel. + + Args: + qkv .. eps: see kernel signature. + reverse: forward (False) or anti-causal (True) scan. + init_state_kv: optional ``(B*H, BLOCK_D, BLOCK_D)`` fp32 contiguous + tensor holding the forward-scan KV state at the END of a prefix + sequence (i.e., AFTER the prefix's last update, BEFORE any further + decay applied by this call). When provided, the kernel resumes the + scan from this state instead of zero. ``BLOCK_D = next_pow2(D)``. + Only the top-left ``D x D`` submatrix of the tile is read. + init_state_z: optional ``(B*H, BLOCK_D)`` fp32 contiguous companion + for the Z denominator state. Must be provided iff ``init_state_kv`` + is provided. + save_final_state: when True, allocate fresh fp32 zero buffers for the + final KV / Z state (after the last frame's update) and pass them to + the kernel for write-out. Returns the buffers as additional outputs. + + Returns: + ``(num, den)`` — bf16 numerator ``(B, N, H, D)`` and denominator + ``(B, H, N)`` before divide. + + When ``save_final_state=True``, also returns + ``(final_state_kv, final_state_z)`` fp32 with shapes + ``(B*H, BLOCK_D, BLOCK_D)`` and ``(B*H, BLOCK_D)``. + + Raises: + NotImplementedError: if any state I/O argument is set together with + ``reverse=True``. The kernel supports state passing in both + directions, but state I/O is only defined for the forward direction + here to avoid silent misuse. + """ + # Dispatch both the stateless bidi case and the stateful forward path to + # chunkwise so split-equivalence uses one numeric implementation. + # Bypass via env: FUSED_GDN_FORCE_LEGACY=1. + if os.environ.get("FUSED_GDN_FORCE_LEGACY", "0") != "1": + from .fused_gdn_chunkwise import ( + fused_gdn_func_chunkwise, + fused_gdn_stateful_chunkwise, + ) + + # Validate state I/O args upfront — preserves the legacy fused_gdn_func's + # validation contract (callers depend on these specific ValueError / + # NotImplementedError signatures, e.g., test_state_validation). + if (init_state_kv is None) != (init_state_z is None): + raise ValueError( + "fused_gdn_func: init_state_kv and init_state_z must be provided together " + "(both None or both fp32 tensors)." + ) + if reverse and (init_state_kv is not None or save_final_state): + raise NotImplementedError( + "fused_gdn_func: state passing (init_state_kv / init_state_z / " + "save_final_state) is only supported for the forward direction " + "(reverse=False)." + ) + if init_state_kv is not None: + B_q, _N, _three, H_q, D_q = qkv.shape + BLOCK_D_q = triton.next_power_of_2(D_q) + expected_kv = (B_q * H_q, BLOCK_D_q, BLOCK_D_q) + expected_z = (B_q * H_q, BLOCK_D_q) + if tuple(init_state_kv.shape) != expected_kv: + raise ValueError( + f"fused_gdn_func: init_state_kv shape {tuple(init_state_kv.shape)} != " f"expected {expected_kv}." + ) + if tuple(init_state_z.shape) != expected_z: + raise ValueError( + f"fused_gdn_func: init_state_z shape {tuple(init_state_z.shape)} != " f"expected {expected_z}." + ) + if init_state_kv.dtype != torch.float32 or init_state_z.dtype != torch.float32: + raise ValueError( + f"fused_gdn_func: init_state_kv/init_state_z must be fp32 " + f"(got {init_state_kv.dtype}, {init_state_z.dtype})." + ) + if not init_state_kv.is_contiguous() or not init_state_z.is_contiguous(): + raise ValueError("fused_gdn_func: init_state_kv / init_state_z must be contiguous.") + + # Stateless path + if init_state_kv is None and init_state_z is None and not save_final_state: + return fused_gdn_func_chunkwise( + qkv, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + beta, + decay, + F=F, + S=S, + k_scale=k_scale, + eps=eps, + reverse=reverse, + ) + + # Stateful path: shape-adapt state I/O. + # state_kv: (B*H, BLOCK_D, BLOCK_D) row-major as M[K_feat, V_feat] + # chunkwise stateful: takes user-facing (B, H, D_in, D_out) and transposes + # internally to (B*H, D_out, D_in) for kernel storage. + # state_z: (B*H, BLOCK_D) + # chunkwise stateful: (B, H, D, 1) or (B, H, D) + B, N, _three, H, D = qkv.shape + BLOCK_D = triton.next_power_of_2(D) + + ck_init_kv = None + ck_init_z = None + if init_state_kv is not None: + # (B*H, BLOCK_D, BLOCK_D) → (B, H, BLOCK_D, BLOCK_D)[:, :, :D, :D] + # then transpose so chunkwise's internal `.transpose(-1, -2)` undoes it. + ck_init_kv = init_state_kv.view(B, H, BLOCK_D, BLOCK_D)[:, :, :D, :D].transpose(-1, -2).contiguous() + if init_state_z is not None: + # (B*H, BLOCK_D) → (B, H, BLOCK_D)[:, :, :D] → (B, H, D, 1) + ck_init_z = init_state_z.view(B, H, BLOCK_D)[:, :, :D].unsqueeze(-1).contiguous() + + result = fused_gdn_stateful_chunkwise( + qkv, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + beta, + decay, + F=F, + S=S, + k_scale=k_scale, + eps=eps, + reverse=reverse, + init_state_kv=ck_init_kv, + init_state_z=ck_init_z, + return_final_state=save_final_state, + ) + + if not save_final_state: + return result # (num, den) + + num, den, ck_state_kv, ck_state_z = result + # chunkwise returns state_kv as (B, H, D, D), [K_feat, V_feat] (post its + # internal back-transpose). Convert to stateful (B*H, BLOCK_D, BLOCK_D) + # by transposing back to internal storage and padding to BLOCK_D. + out_state_kv = torch.zeros(B * H, BLOCK_D, BLOCK_D, device=qkv.device, dtype=torch.float32) + out_state_kv[:, :D, :D] = ck_state_kv.transpose(-1, -2).reshape(B * H, D, D) + out_state_z = torch.zeros(B * H, BLOCK_D, device=qkv.device, dtype=torch.float32) + out_state_z[:, :D] = ck_state_z.squeeze(-1).reshape(B * H, D) + return num, den, out_state_kv, out_state_z + + B, N, three, H, D = qkv.shape + assert three == 3 + + BLOCK_D, BLOCK_S, dot_prec, state_fp32, nw, cfg, beta, decay = _prepare_launch(D, beta, decay) + + has_init_state = init_state_kv is not None or init_state_z is not None + if reverse and (has_init_state or save_final_state): + raise NotImplementedError( + "fused_gdn_func: state passing (init_state_kv / init_state_z / " + "save_final_state) is only supported for the forward direction " + "(reverse=False). The chunk-causal anti-causal pass resets state " + "per chunk and has no global cross-prefix state to cache." + ) + + if has_init_state: + if init_state_kv is None or init_state_z is None: + raise ValueError( + "fused_gdn_func: init_state_kv and init_state_z must be " + "provided together (got " + f"init_state_kv={'set' if init_state_kv is not None else 'None'}, " + f"init_state_z={'set' if init_state_z is not None else 'None'})." + ) + expected_kv_shape = (B * H, BLOCK_D, BLOCK_D) + expected_z_shape = (B * H, BLOCK_D) + if tuple(init_state_kv.shape) != expected_kv_shape: + raise ValueError( + f"fused_gdn_func: init_state_kv shape {tuple(init_state_kv.shape)} " + f"does not match expected {expected_kv_shape} (BLOCK_D=next_pow2(D)={BLOCK_D})." + ) + if tuple(init_state_z.shape) != expected_z_shape: + raise ValueError( + f"fused_gdn_func: init_state_z shape {tuple(init_state_z.shape)} " + f"does not match expected {expected_z_shape}." + ) + if init_state_kv.dtype != torch.float32 or init_state_z.dtype != torch.float32: + raise ValueError( + "fused_gdn_func: init_state_kv and init_state_z must be fp32 " + f"(got {init_state_kv.dtype}, {init_state_z.dtype})." + ) + if not init_state_kv.is_contiguous() or not init_state_z.is_contiguous(): + raise ValueError("fused_gdn_func: init_state_kv and init_state_z must be contiguous.") + if init_state_kv.device != qkv.device or init_state_z.device != qkv.device: + raise ValueError("fused_gdn_func: init_state_* must live on the same device as qkv.") + load_init = 1 + init_kv_arg = init_state_kv + init_z_arg = init_state_z + else: + load_init = 0 + init_kv_arg = None # placeholder set below + + if save_final_state: + final_state_kv = torch.zeros(B * H, BLOCK_D, BLOCK_D, device=qkv.device, dtype=torch.float32) + final_state_z = torch.zeros(B * H, BLOCK_D, device=qkv.device, dtype=torch.float32) + save_final = 1 + else: + final_state_kv = None + final_state_z = None + save_final = 0 + + num = torch.empty(B, N, H, D, device=qkv.device, dtype=qkv.dtype) + den = torch.empty(B, H, N, device=qkv.device, dtype=qkv.dtype) + dummy = torch.empty(1, device=qkv.device, dtype=torch.float32) + + # Resolve pointer args for the unused slots to a shared scratch tensor; + # the kernel compiles the corresponding load/store away when the + # constexpr flag is 0. + init_kv_ptr = init_kv_arg if load_init else dummy + init_z_ptr = init_z_arg if load_init else dummy + final_kv_ptr = final_state_kv if save_final else dummy + final_z_ptr = final_state_z if save_final else dummy + + _fused_gdn_kernel[(B * H,)]( + qkv, + qkv.stride(0), + qkv.stride(1), + qkv.stride(2), + qkv.stride(3), + qkv.stride(4), + beta, + decay, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + num, # `out_ptr` reuses `num` buffer (result immediately overwritten below) + num, + den, + dummy, + dummy, + dummy, + dummy, # saved-state dummies (SAVE_STATE=0) + init_kv_ptr, + init_z_ptr, + final_kv_ptr, + final_z_ptr, + H=H, + F=F, + S=S, + D=D, + K_SCALE=k_scale, + NORM_EPS=1e-5, # unused with USE_PRECOMPUTED_RMS=1 + EPS=eps, + QK_NORM=1, + USE_PRECOMPUTED_RMS=1, + STATE_FP32=1 if state_fp32 else 0, + DOT_PRECISION=dot_prec, + REVERSE=1 if reverse else 0, + SAVE_STATE=0, + LOAD_INIT_STATE=load_init, + SAVE_FINAL_STATE=save_final, + BLOCK_D=BLOCK_D, + BLOCK_S=BLOCK_S, + num_stages=cfg["num_stages"], + num_warps=nw, + ) + if save_final_state: + return num, den, final_state_kv, final_state_z + return num, den + + +def fused_bigdn_func( + qkv: torch.Tensor, # (B, N, 3, H, D) + q_inv_rms: torch.Tensor, # (B, N) — pre-computed via `_precompute_inv_rms` + k_inv_rms: torch.Tensor, # (B, N) + q_norm_weight: torch.Tensor, # (C,) float32 + k_norm_weight: torch.Tensor, # (C,) + rope_cos: torch.Tensor, # (N, D) + rope_sin: torch.Tensor, # (N, D) + beta: torch.Tensor, # (B, H, F, S) + decay: torch.Tensor, # (B, H, F) + F: int, + S: int, + k_scale: float, + eps: float = 1e-6, + # -- chunk-causal extensions (not in upstream; see adapter notes below) -- + qkv_bwd: torch.Tensor | None = None, + beta_bwd: torch.Tensor | None = None, + decay_bwd: torch.Tensor | None = None, + q_inv_rms_bwd: torch.Tensor | None = None, + k_inv_rms_bwd: torch.Tensor | None = None, +) -> torch.Tensor: + """Full bidirectional fused GDN. + + Returns: out (B, N, H, D) bf16 = (num_fwd + num_bwd) / (den_fwd + den_bwd + eps). + + Chunk-causal extensions (optional): + For chunk-causal GDN we need to zero state at chunk boundaries in the + BACKWARD direction only. Pass separately pre-processed backward tensors + (decay_bwd with zeros at boundary frames, and optionally qkv_bwd / + beta_bwd with K/V or beta zeroed at boundary frames). If any `*_bwd` + argument is None, the forward tensor is reused. + """ + if ( + os.environ.get("FUSED_GDN_FORCE_LEGACY", "0") != "1" + and qkv_bwd is None + and beta_bwd is None + and decay_bwd is None + and q_inv_rms_bwd is None + and k_inv_rms_bwd is None + ): + from .fused_gdn_chunkwise import fused_bigdn_bidi_chunkwise + + return fused_bigdn_bidi_chunkwise( + qkv, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + beta, + decay, + F=F, + S=S, + k_scale=k_scale, + eps=eps, + ) + + num_fwd, den_fwd = fused_gdn_func( + qkv, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + beta, + decay, + F=F, + S=S, + k_scale=k_scale, + eps=eps, + reverse=False, + ) + num_bwd, den_bwd = fused_gdn_func( + qkv if qkv_bwd is None else qkv_bwd, + q_inv_rms if q_inv_rms_bwd is None else q_inv_rms_bwd, + k_inv_rms if k_inv_rms_bwd is None else k_inv_rms_bwd, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + beta if beta_bwd is None else beta_bwd, + decay if decay_bwd is None else decay_bwd, + F=F, + S=S, + k_scale=k_scale, + eps=eps, + reverse=True, + ) + # num: (B, N, H, D), den: (B, H, N). Fuse then divide. + total_num = num_fwd + num_bwd + total_den = (den_fwd + den_bwd).permute(0, 2, 1).unsqueeze(-1) # (B, N, H, 1) + return total_num / (total_den + eps) + + +# ===================================================================== +# Backward / autograd Functions +# ===================================================================== +# Adds: +# 1. ``_fused_gdn_bwd_kernel`` -- Triton jit kernel that replays the +# forward recurrence in reverse time using per-frame state snapshots +# written by the forward kernel under ``SAVE_STATE=1``. +# 2. ``_run_fwd_save`` -- helper that runs the existing forward +# ``_fused_gdn_kernel`` with ``SAVE_STATE=1``. Adapted to pass our +# extra ``init_state_kv_ptr / init_state_z_ptr / final_state_kv_ptr / +# final_state_z_ptr`` pointers + ``LOAD_INIT_STATE / SAVE_FINAL_STATE`` +# constexpr flags (all unused on the autograd path -> dummy / 0). +# 3. ``FusedGDNFunction`` -- autograd Function for unidirectional GDN +# with ``QK_NORM=1`` (in-kernel per-head RMSNorm). +# 4. ``FusedBiGDNFunction`` -- autograd Function for bidirectional BiGDN. +# Pre-normalizes Q/K in PyTorch with full-channel RMSNorm, runs the +# forward kernel twice with ``QK_NORM=0 + SAVE_STATE=1``, fuses +# ``(num_fwd + num_bwd) / (den_fwd + den_bwd + eps)``. Backward +# computes ``dnum / dden`` from upstream ``dout`` and runs the bwd +# kernel twice with ``BIDI_MODE=1``. +# 5. Python wrappers ``fused_gdn_forward_with_grad`` / +# ``fused_bigdn_forward_with_grad`` -- drop-in autograd-enabled +# replacements for ``fused_gdn_func`` / ``fused_bigdn_func``. +# +# Chunk-causal autograd support: ``FusedBiGDNFunction`` (and the public +# wrapper ``fused_bigdn_forward_with_grad``) accepts optional +# ``beta_bwd`` / ``decay_bwd`` overrides for the reverse-direction +# kernel call -- exactly the same masking convention used by the +# inference path ``fused_bigdn_func``. When provided, the reverse +# direction's forward and backward kernels both run on these masked +# tensors, and the backward returns separate gradient tensors +# (``dbeta_bwd`` / ``ddecay_bwd``) so autograd can route them back +# through any ``clone() + index = 0`` masking the caller applied. + + +@triton.jit +def _fused_gdn_bwd_kernel( + # ---- original inputs ---- + qkv_ptr, + stride_b: tl.constexpr, + stride_n: tl.constexpr, + stride_3: tl.constexpr, + stride_h: tl.constexpr, + stride_d: tl.constexpr, + beta_ptr, + decay_ptr, + q_norm_w_ptr, + k_norm_w_ptr, + rope_cos_ptr, + rope_sin_ptr, + # ---- saved from forward ---- + saved_state_ptr, # (B*H, F, BLOCK_D, BLOCK_D) -- state_prev snapshots + saved_z_ptr, # (B*H, F, BLOCK_D) + saved_state_curr_ptr, # (B*H, F, BLOCK_D, BLOCK_D) -- state_curr (after update) + saved_z_curr_ptr, # (B*H, F, BLOCK_D) + # ---- upstream gradient / pre-computed dnum ---- + dout_ptr, # GDN mode: (B, N, H, D) upstream grad. BiDI mode: pre-computed dnum + # ---- BiDI mode: external dden ---- + dden_ext_ptr, # BiDI mode: (B, H, N) pre-computed dden. GDN mode: unused + # ---- output gradients ---- + dqkv_ptr, # (B, N, 3, H, D) -- same layout as qkv + dbeta_ptr, # (B, H, F, S) + ddecay_ptr, # (B, H, F) + # ---- dims ---- + H: tl.constexpr, + F: tl.constexpr, + S: tl.constexpr, + D: tl.constexpr, + K_SCALE, + NORM_EPS: tl.constexpr, + EPS: tl.constexpr, + QK_NORM: tl.constexpr, + STATE_FP32: tl.constexpr, + REVERSE_BWD: tl.constexpr, # 0=backward of forward GDN, 1=backward of reversed GDN + BIDI_MODE: tl.constexpr, # 0=GDN (compute dnum/dden), 1=BiGDN (use provided) + DOT_PRECISION: tl.constexpr, # 0=bf16 TC, 1=TF32 TC, 2=IEEE fp32 + BLOCK_D: tl.constexpr, + BLOCK_S: tl.constexpr, +): + pid = tl.program_id(0) + pid_b = pid // H + pid_h = pid % H + N: tl.constexpr = F * S + bh = pid_b * H + pid_h + + qkv_bh = qkv_ptr + pid_b * stride_b + pid_h * stride_h + dqkv_bh = dqkv_ptr + pid_b * stride_b + pid_h * stride_h + dout_bh = dout_ptr + pid_b * (N * H * D) + pid_h * D + beta_bh = beta_ptr + bh * (F * S) + decay_bh = decay_ptr + bh * F + dbeta_bh = dbeta_ptr + bh * (F * S) + ddecay_bh = ddecay_ptr + bh * F + st_bh = saved_state_ptr + bh * F * BLOCK_D * BLOCK_D + sz_bh = saved_z_ptr + bh * F * BLOCK_D + stc_bh = saved_state_curr_ptr + bh * F * BLOCK_D * BLOCK_D + szc_bh = saved_z_curr_ptr + bh * F * BLOCK_D + if BIDI_MODE: + dden_ext_bh = dden_ext_ptr + bh * N + + offs_d = tl.arange(0, BLOCK_D) + mask_d = offs_d < D + offs_d_pair = offs_d ^ 1 + mask_d_pair = offs_d_pair < D + + nw_offset = pid_h * D + if QK_NORM: + q_nw = tl.load(q_norm_w_ptr + nw_offset + offs_d, mask=mask_d, other=0.0).to(tl.float32) + k_nw = tl.load(k_norm_w_ptr + nw_offset + offs_d, mask=mask_d, other=0.0).to(tl.float32) + q_nw_pair = tl.load(q_norm_w_ptr + nw_offset + offs_d_pair, mask=mask_d_pair, other=0.0).to(tl.float32) + k_nw_pair = tl.load(k_norm_w_ptr + nw_offset + offs_d_pair, mask=mask_d_pair, other=0.0).to(tl.float32) + + D_inv = 1.0 / D + k_scale = K_SCALE + + # Dot precision: mirror forward kernel + if DOT_PRECISION >= 1: + dot_dtype = tl.float32 + else: + dot_dtype = tl.bfloat16 + dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" + + # Gradient matmuls: always use bf16 TC + TF32 input precision (matching PyTorch backward) + grad_dtype = tl.bfloat16 + grad_ip: tl.constexpr = "tf32" + + # ---- Gradient state accumulators (reverse time) ---- + dstate = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) + dstate_z = tl.zeros([BLOCK_D], dtype=tl.float32) + + for f_rev in range(F): + # Backward iterates in reverse of forward direction. + if REVERSE_BWD: + f = f_rev # backward of reversed GDN: iterate 0..F-1 + # In fwd_save REVERSE, q_frame=f had kv_frame=f+1 (or skip at f=F-1). + kv_frame_bwd = f + 1 if f < F - 1 else f + skip_bwd = f == F - 1 # f=F-1 was dummy step (f_iter=0 in fwd) + else: + f = F - 1 - f_rev # backward of forward GDN: iterate F-1..0 + kv_frame_bwd = f + skip_bwd = False + q_n_base = f * S + kv_n_base = kv_frame_bwd * S + f_beta = beta_bh + kv_frame_bwd * S + + # ---- Load state_curr for Pass 2 output (both directions use inclusive) ---- + st_f = st_bh + f * BLOCK_D * BLOCK_D + offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] + mask_dd = mask_d[:, None] & mask_d[None, :] + + stc_f = stc_bh + f * BLOCK_D * BLOCK_D + P_state = tl.load(stc_f + offs_dd, mask=mask_dd, other=0.0) + Pz_state = tl.load(szc_bh + f * BLOCK_D + offs_d, mask=mask_d, other=0.0) + if STATE_FP32 == 0: + P_state = P_state.to(tl.float32) + + # Decay: for REVERSE_BWD, use decay[kv_frame] matching fwd_save. + if REVERSE_BWD and skip_bwd: + g = 1.0 + elif REVERSE_BWD: + g = tl.load(decay_bh + kv_frame_bwd).to(tl.float32) + else: + g = tl.load(decay_bh + f).to(tl.float32) + + # ======================================================== + # Pass 2 backward: Output gradients -> dQ, dstate, dstate_z + # ======================================================== + for s0 in range(0, S, BLOCK_S): + offs_s = s0 + tl.arange(0, BLOCK_S) + mask_s = offs_s < S + mask_sd = mask_s[:, None] & mask_d[None, :] + mask_sd_pair = mask_s[:, None] & mask_d_pair[None, :] + n_idx = q_n_base + offs_s # Q data from q_frame + + # Load dout; recompute Q, Q_pair, Q_rot, num, den from saved P_f/Pz_f. + dout_ptrs = dout_bh + n_idx[:, None] * (H * D) + offs_d[None, :] + d_out = tl.load(dout_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + + # Recompute Q, Q_pair, Q_rot (same as forward). + q_ptrs = qkv_bh + n_idx[:, None] * stride_n + 0 * stride_3 + offs_d[None, :] * stride_d + Q_raw = tl.load(q_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + q_pair_ptrs = qkv_bh + n_idx[:, None] * stride_n + 0 * stride_3 + offs_d_pair[None, :] * stride_d + Q_pair_raw = tl.load(q_pair_ptrs, mask=mask_sd_pair, other=0.0).to(tl.float32) + + if QK_NORM: + q_var = tl.sum(Q_raw * Q_raw, axis=1) * D_inv + q_inv_rms = 1.0 / tl.sqrt(q_var + NORM_EPS) + Q_normed = Q_raw * q_inv_rms[:, None] * q_nw[None, :] + Q_pair_normed = Q_pair_raw * q_inv_rms[:, None] * q_nw_pair[None, :] + else: + Q_normed = Q_raw + Q_pair_normed = Q_pair_raw + Q = tl.where(Q_normed > 0, Q_normed, 0.0) + Q_pair = tl.where(Q_pair_normed > 0, Q_pair_normed, 0.0) + + rope_ptrs = n_idx[:, None] * D + offs_d[None, :] + Cos = tl.load(rope_cos_ptr + rope_ptrs, mask=mask_sd, other=1.0).to(tl.float32) + Sin = tl.load(rope_sin_ptr + rope_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + Q_rot = Q * Cos + Q_pair * Sin + + # Compute dnum and dden. + if BIDI_MODE: + # BiGDN: dnum and dden pre-computed externally from total num/den. + dnum = d_out # dout_ptr already contains pre-computed dnum + dden = tl.load(dden_ext_bh + n_idx, mask=mask_s, other=0.0).to(tl.float32) + else: + # GDN: recompute num/den using direction-appropriate state. + num_tile = tl.dot( + Q_rot.to(dot_dtype), P_state.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip + ) + den_tile = tl.sum(Q * Pz_state[None, :], axis=1) + inv_den = 1.0 / (den_tile + EPS) + dnum = d_out * inv_den[:, None] + dden = -tl.sum(d_out * num_tile, axis=1) * inv_den * inv_den + + # dstate += Q_rot^T @ dnum (state contribution from num = Q_rot @ P_state). + dstate = dstate + tl.dot( + tl.trans(Q_rot.to(grad_dtype)), + dnum.to(grad_dtype), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + + # dstate_z += sum(dden * Q, axis=0) (Pz contribution from den = Q . Pz). + dstate_z += tl.sum(dden[:, None] * Q, axis=0) + + # dQ_rot = dnum @ P_state^T (uses state that forward's output read). + dQ_rot = tl.dot( + dnum.to(grad_dtype), + tl.trans(P_state.to(grad_dtype)), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + + # dQ_from_den = dden * Pz_state. + dQ_from_den = dden[:, None] * Pz_state[None, :] + + # RoPE inverse for Q: store dQ_rot, reload at paired indices. + # Store dQ_rot temporarily to dqkv[Q] at normal d positions. + dq_ptrs = dqkv_bh + n_idx[:, None] * stride_n + 0 * stride_3 + offs_d[None, :] * stride_d + tl.store(dq_ptrs, dQ_rot.to(tl.bfloat16), mask=mask_sd) + # The XOR-paired channel can be owned by another warp. Synchronize + # both sides of the scratch roundtrip before dqkv is overwritten. + tl.debug_barrier() + + # Load dQ_rot at paired positions. + dq_pair_ptrs = dqkv_bh + n_idx[:, None] * stride_n + 0 * stride_3 + offs_d_pair[None, :] * stride_d + dQ_rot_pair = tl.load(dq_pair_ptrs, mask=mask_sd_pair, other=0.0).to(tl.float32) + tl.debug_barrier() + + # RoPE inverse: dQ = dQ_rot * Cos - dQ_rot_pair * Sin. + dQ = dQ_rot * Cos - dQ_rot_pair * Sin + dQ_from_den + + # ReLU backward. + relu_mask_q = (Q_normed > 0).to(tl.float32) + dQ_normed = dQ * relu_mask_q + + # Norm backward (QK_NORM) or direct (no norm). + if QK_NORM: + gw = dQ_normed * q_nw[None, :] + corr = tl.sum(gw * Q_raw, axis=1) * D_inv * q_inv_rms * q_inv_rms + dQ_raw = q_inv_rms[:, None] * (gw - Q_raw * corr[:, None]) + else: + dQ_raw = dQ_normed + + # Store final dQ_raw to dqkv[Q]. + tl.store(dq_ptrs, dQ_raw.to(tl.bfloat16), mask=mask_sd) + + # Both directions use inclusive output (state_curr), so capture dDelta AFTER Pass 2. + dDelta = dstate + dDelta_z = dstate_z + + # ======================================================== + # Reload state_prev for Pass 1 backward (reuse P_state variable) + # ======================================================== + P_state = tl.load(st_f + offs_dd, mask=mask_dd, other=0.0) + if STATE_FP32 == 0: + P_state = P_state.to(tl.float32) + Pz_state = tl.load(sz_bh + f * BLOCK_D + offs_d, mask=mask_d, other=0.0) + + # ======================================================== + # Pass 1 backward: State update gradients -> dK, dV, dbeta, dstate + # Skip for REVERSE_BWD dummy frame (skip_bwd=True) to avoid clobbering. + # ======================================================== + if skip_bwd == False: + for s0 in range(0, S, BLOCK_S): + offs_s = s0 + tl.arange(0, BLOCK_S) + mask_s = offs_s < S + mask_sd = mask_s[:, None] & mask_d[None, :] + mask_sd_pair = mask_s[:, None] & mask_d_pair[None, :] + n_idx = kv_n_base + offs_s # K/V from kv_frame + + # Recompute K, K_pair, K_rot, V. + k_ptrs = qkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d[None, :] * stride_d + v_ptrs = qkv_bh + n_idx[:, None] * stride_n + 2 * stride_3 + offs_d[None, :] * stride_d + K_raw = tl.load(k_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + V_raw = tl.load(v_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + + k_pair_ptrs = qkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d_pair[None, :] * stride_d + K_pair_raw = tl.load(k_pair_ptrs, mask=mask_sd_pair, other=0.0).to(tl.float32) + + if QK_NORM: + k_var = tl.sum(K_raw * K_raw, axis=1) * D_inv + k_inv_rms = 1.0 / tl.sqrt(k_var + NORM_EPS) + K_normed = K_raw * k_inv_rms[:, None] * k_nw[None, :] + K_pair_normed = K_pair_raw * k_inv_rms[:, None] * k_nw_pair[None, :] + else: + K_normed = K_raw + K_pair_normed = K_pair_raw + K = tl.where(K_normed > 0, K_normed, 0.0) * k_scale + K_pair = tl.where(K_pair_normed > 0, K_pair_normed, 0.0) * k_scale + + rope_ptrs = n_idx[:, None] * D + offs_d[None, :] + Cos = tl.load(rope_cos_ptr + rope_ptrs, mask=mask_sd, other=1.0).to(tl.float32) + Sin = tl.load(rope_sin_ptr + rope_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + K_rot = K * Cos + K_pair * Sin + + bt = tl.load(f_beta + offs_s, mask=mask_s, other=0.0).to(tl.float32) + + # Recompute V_pred and delta_v. + K_rot_dc = K_rot.to(dot_dtype) + V_pred = tl.dot(K_rot_dc, P_state.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) + delta_v = (V_raw - V_pred) * bt[:, None] + + # ---- KV stream backward ---- + ddelta_v = tl.dot( + K_rot.to(grad_dtype), + dDelta.to(grad_dtype), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + + dK_rot_from_delta = tl.dot( + delta_v.to(grad_dtype), + tl.trans(dDelta.to(grad_dtype)), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + + dV = ddelta_v * bt[:, None] + dbeta_kv = tl.sum(ddelta_v * (V_raw - V_pred), axis=1) + + dV_pred = -ddelta_v * bt[:, None] + dK_rot_from_vpred = tl.dot( + dV_pred.to(grad_dtype), + tl.trans(P_state.to(grad_dtype)), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + + dstate = dstate + tl.dot( + tl.trans(K_rot.to(grad_dtype)), + dV_pred.to(grad_dtype), + out_dtype=tl.float32, + input_precision=grad_ip, + ) + + dK_rot = dK_rot_from_delta + dK_rot_from_vpred + + # ---- Z stream backward ---- + z_hat = tl.sum(K * Pz_state[None, :], axis=1) + dz = (1.0 - z_hat) * bt + + ddz = tl.sum(K * dDelta_z[None, :], axis=1) + dz_hat = -ddz * bt + dK_z = dDelta_z[None, :] * dz[:, None] + dz_hat[:, None] * Pz_state[None, :] + dstate_z = dstate_z + tl.sum(dz_hat[:, None] * K, axis=0) + + dbeta_z = ddz * (1.0 - z_hat) + dbeta_total = dbeta_kv + dbeta_z + tl.store(dbeta_bh + kv_frame_bwd * S + offs_s, dbeta_total.to(tl.bfloat16), mask=mask_s) + + # ---- RoPE inverse for K ---- + dk_ptrs = dqkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d[None, :] * stride_d + tl.store(dk_ptrs, dK_rot.to(tl.bfloat16), mask=mask_sd) + # Match the dQ synchronization: all temporary values must be + # visible before paired loads and consumed before overwrites. + tl.debug_barrier() + dk_pair_ptrs = dqkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d_pair[None, :] * stride_d + dK_rot_pair = tl.load(dk_pair_ptrs, mask=mask_sd_pair, other=0.0).to(tl.float32) + tl.debug_barrier() + + dK_from_kv = dK_rot * Cos - dK_rot_pair * Sin + dK_total = dK_from_kv + dK_z + + relu_mask_k = (K_normed > 0).to(tl.float32) + dK_normed = dK_total * k_scale * relu_mask_k + + if QK_NORM: + gw_k = dK_normed * k_nw[None, :] + corr_k = tl.sum(gw_k * K_raw, axis=1) * D_inv * k_inv_rms * k_inv_rms + dK_raw = k_inv_rms[:, None] * (gw_k - K_raw * corr_k[:, None]) + else: + dK_raw = dK_normed + + tl.store(dk_ptrs, dK_raw.to(tl.bfloat16), mask=mask_sd) + dv_ptrs = dqkv_bh + n_idx[:, None] * stride_n + 2 * stride_3 + offs_d[None, :] * stride_d + tl.store(dv_ptrs, dV.to(tl.bfloat16), mask=mask_sd) + + # ======================================================== + # Decay backward (inside skip_bwd guard) + # ======================================================== + is_first_frame = f_rev == F - 1 + if is_first_frame: + ddecay_f = 0.0 + else: + inv_g = 1.0 / (g + 1e-12) + ddecay_kv = tl.sum(dstate * P_state) * inv_g + ddecay_z_val = tl.sum(dstate_z * Pz_state) * inv_g + ddecay_f = ddecay_kv + ddecay_z_val + tl.store(ddecay_bh + kv_frame_bwd, ddecay_f) + + # Propagate gradient through decay: dS_{f-1} = g[f] * dP_f. + dstate = dstate * g + dstate_z = dstate_z * g + + +# ===================================================================== +# Forward-with-state-save helper (for autograd Functions) +# ===================================================================== + + +def _run_fwd_save( + qkv, + beta, + decay, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + F: int, + S: int, + k_scale: float, + norm_eps: float, + eps: float, + qk_norm: bool, + reverse: bool, + cfg, +): + """Run forward kernel for one direction with ``SAVE_STATE=1``. + + Returns ``(num, den, saved_state, saved_z, saved_state_curr, saved_z_curr)``. + The forward kernel writes ``out = num/(den+eps)`` first and then overwrites + the same buffer with raw ``num``, so the returned ``num`` tensor holds raw + numerator values (matching the BiGDN combine-then-divide convention). + """ + B, N, three, H, D = qkv.shape + BLOCK_D, BLOCK_S, dot_prec, state_fp32, nw, _, beta, decay = _prepare_launch(D, beta, decay) + + num_out = torch.empty(B, N, H, D, device=qkv.device, dtype=qkv.dtype) + den_out = torch.empty(B, H, N, device=qkv.device, dtype=qkv.dtype) + state_dtype = torch.float32 if state_fp32 else torch.bfloat16 + saved_state = torch.empty(B * H, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=state_dtype) + saved_z = torch.empty(B * H, F, BLOCK_D, device=qkv.device, dtype=torch.float32) + saved_state_curr = torch.empty(B * H, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=torch.float32) + saved_z_curr = torch.empty(B * H, F, BLOCK_D, device=qkv.device, dtype=torch.float32) + # The kernel writes ``out = num/(den+eps)`` first then overwrites with raw num + # in the same buffer. Reuse num_out as the (discarded) ``out`` slot so the + # final contents end up being raw num. + out_discard = num_out + dummy_inv = torch.empty(1, device=qkv.device, dtype=torch.float32) + + _fused_gdn_kernel[(B * H,)]( + qkv, + qkv.stride(0), + qkv.stride(1), + qkv.stride(2), + qkv.stride(3), + qkv.stride(4), + beta, + decay, + dummy_inv, + dummy_inv, # unused inv_rms ptrs (USE_PRECOMPUTED_RMS=0) + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + out_discard, + num_out, + den_out, + saved_state, + saved_z, + saved_state_curr, + saved_z_curr, + dummy_inv, + dummy_inv, + dummy_inv, + dummy_inv, # init/final-state dummies + H=H, + F=F, + S=S, + D=D, + K_SCALE=k_scale, + NORM_EPS=norm_eps, + EPS=eps, + QK_NORM=1 if qk_norm else 0, + USE_PRECOMPUTED_RMS=0, + STATE_FP32=1 if state_fp32 else 0, + DOT_PRECISION=dot_prec, + REVERSE=1 if reverse else 0, + SAVE_STATE=1, + LOAD_INIT_STATE=0, + SAVE_FINAL_STATE=0, + BLOCK_D=BLOCK_D, + BLOCK_S=BLOCK_S, + num_stages=cfg["num_stages"], + num_warps=nw, + ) + return num_out, den_out, saved_state, saved_z, saved_state_curr, saved_z_curr + + +# ===================================================================== +# Unidirectional GDN autograd Function +# ===================================================================== + + +class FusedGDNFunction(torch.autograd.Function): + """Autograd Function for unidirectional fused GDN with in-kernel RMSNorm. + + Forward runs ``_fused_gdn_kernel`` with ``QK_NORM=1`` and ``SAVE_STATE=1``, + saving per-frame state snapshots for backward. Backward runs + ``_fused_gdn_bwd_kernel`` with ``BIDI_MODE=0`` (kernel computes + ``dnum``/``dden`` from upstream ``dout``). + """ + + @staticmethod + def forward( + ctx, + qkv, + beta, + decay, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + F: int, + S: int, + k_scale: float = 1.0, + norm_eps: float = 1e-6, + eps: float = 1e-6, + qk_norm: bool = True, + ): + B, N, three, H, D = qkv.shape + assert three == 3 and N == F * S + + BLOCK_D, BLOCK_S, dot_prec, state_fp32, nw, cfg, beta, decay = _prepare_launch(D, beta, decay) + + if q_norm_weight is None: + q_norm_weight = torch.ones(D, device=qkv.device, dtype=torch.float32) + if k_norm_weight is None: + k_norm_weight = torch.ones(D, device=qkv.device, dtype=torch.float32) + + out = torch.empty(B, N, H, D, device=qkv.device, dtype=qkv.dtype) + + # Saved states for backward. + state_dtype = torch.float32 if state_fp32 else torch.bfloat16 + saved_state = torch.empty(B * H, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=state_dtype) + saved_z = torch.empty(B * H, F, BLOCK_D, device=qkv.device, dtype=torch.float32) + saved_state_curr = torch.empty(B * H, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=torch.float32) + saved_z_curr = torch.empty(B * H, F, BLOCK_D, device=qkv.device, dtype=torch.float32) + + # Dummy num/den for forward kernel (still writes them but we discard). + num_out = torch.empty(B, N, H, D, device=qkv.device, dtype=qkv.dtype) + den_out = torch.empty(B, H, N, device=qkv.device, dtype=qkv.dtype) + dummy_inv = torch.empty(1, device=qkv.device, dtype=torch.float32) + + _fused_gdn_kernel[(B * H,)]( + qkv, + qkv.stride(0), + qkv.stride(1), + qkv.stride(2), + qkv.stride(3), + qkv.stride(4), + beta, + decay, + dummy_inv, + dummy_inv, # unused inv_rms ptrs (USE_PRECOMPUTED_RMS=0) + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + out, + num_out, + den_out, + saved_state, + saved_z, + saved_state_curr, + saved_z_curr, + dummy_inv, + dummy_inv, + dummy_inv, + dummy_inv, # init/final-state dummies + H=H, + F=F, + S=S, + D=D, + K_SCALE=k_scale, + NORM_EPS=norm_eps, + EPS=eps, + QK_NORM=1 if qk_norm else 0, + USE_PRECOMPUTED_RMS=0, + STATE_FP32=1 if state_fp32 else 0, + DOT_PRECISION=dot_prec, + REVERSE=0, + SAVE_STATE=1, + LOAD_INIT_STATE=0, + SAVE_FINAL_STATE=0, + BLOCK_D=BLOCK_D, + BLOCK_S=BLOCK_S, + num_stages=cfg["num_stages"], + num_warps=nw, + ) + del num_out, den_out + + ctx.save_for_backward( + qkv, + beta, + decay, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + saved_state, + saved_z, + saved_state_curr, + saved_z_curr, + ) + ctx.F = F + ctx.S = S + ctx.k_scale = k_scale + ctx.norm_eps = norm_eps + ctx.eps = eps + ctx.qk_norm = qk_norm + ctx.dot_prec = dot_prec + return out + + @staticmethod + def backward(ctx, dout): + ( + qkv, + beta, + decay, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + saved_state, + saved_z, + saved_state_curr, + saved_z_curr, + ) = ctx.saved_tensors + + B, N, three, H, D = qkv.shape + F_val = ctx.F + S = ctx.S + + BLOCK_D, BLOCK_S_BWD, _, _, _, cfg, beta, decay = _prepare_launch(D, beta, decay) + dqkv = torch.zeros_like(qkv) + dbeta = torch.zeros_like(beta) + ddecay = torch.zeros_like(decay) + + # Dummy dden_ext (unused in GDN mode). + dden_ext = torch.empty(1, device=qkv.device, dtype=torch.float32) + + # Progressive num_warps reduction on tmem overflow. + nw = cfg["num_warps"] + if ctx.dot_prec >= 1: + nw = min(nw, 4) + while nw >= 1: + try: + _fused_gdn_bwd_kernel[(B * H,)]( + qkv, + qkv.stride(0), + qkv.stride(1), + qkv.stride(2), + qkv.stride(3), + qkv.stride(4), + beta, + decay, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + saved_state, + saved_z, + saved_state_curr, + saved_z_curr, + dout.contiguous(), + dden_ext, + dqkv, + dbeta, + ddecay, + H=H, + F=F_val, + S=S, + D=D, + K_SCALE=ctx.k_scale, + NORM_EPS=ctx.norm_eps, + EPS=ctx.eps, + QK_NORM=1 if ctx.qk_norm else 0, + STATE_FP32=1 if cfg["STATE_FP32"] else 0, + REVERSE_BWD=0, + BIDI_MODE=0, + DOT_PRECISION=ctx.dot_prec, + BLOCK_D=BLOCK_D, + BLOCK_S=BLOCK_S_BWD, + num_stages=cfg["num_stages"], + num_warps=nw, + ) + break + except Exception as e: + if "OutOfResources" in str(type(e).__name__) or "out of resource" in str(e).lower(): + nw = nw // 2 + if nw < 1: + raise RuntimeError( + "FusedGDN backward: Triton kernel OutOfResources at all warp " + f"counts (8, 4, 2, 1). Most recent error: {e}" + ) from e + else: + raise + + return dqkv, dbeta, ddecay, None, None, None, None, None, None, None, None, None, None + + +def fused_gdn_forward_with_grad( + qkv: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + q_norm_weight: torch.Tensor | None, + k_norm_weight: torch.Tensor | None, + rope_cos: torch.Tensor, + rope_sin: torch.Tensor, + F: int, + S: int, + k_scale: float = 1.0, + norm_eps: float = 1e-6, + eps: float = 1e-6, + qk_norm: bool = True, +) -> torch.Tensor: + """Drop-in autograd-enabled replacement for the unidirectional GDN path. + + Unlike ``fused_gdn_func`` (which expects pre-computed ``q_inv_rms``/ + ``k_inv_rms``), this wrapper computes per-head RMSNorm inside the + Triton kernel (``QK_NORM=1`` / ``USE_PRECOMPUTED_RMS=0``) so the + backward kernel can reproduce the exact same normed Q/K when + replaying the recurrence. + """ + return FusedGDNFunction.apply( + qkv, + beta, + decay, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + F, + S, + k_scale, + norm_eps, + eps, + qk_norm, + ) + + +# ===================================================================== +# Bidirectional BiGDN autograd Function (full-channel RMSNorm in Python) +# ===================================================================== + + +class FusedBiGDNFunction(torch.autograd.Function): + """Autograd Function for bidirectional fused BiGDN. + + Full-channel RMSNorm is applied in Python (so the norm backward can + couple all heads correctly), then the kernel runs with ``QK_NORM=0`` + on the pre-normed QKV. Forward and reverse directions are run + separately with ``SAVE_STATE=1``, and combined as + ``out = (num_fwd + num_bwd) / (den_fwd + den_bwd + eps)``. + + Backward computes ``dnum`` and ``dden`` from upstream ``dout`` and + runs the bwd kernel twice (forward + reverse) with ``BIDI_MODE=1``. + Norm backward is computed in Python (full-channel RMSNorm couples + all heads). + + Chunk-causal masking (optional): + Pass ``beta_bwd`` and/or ``decay_bwd`` to override the beta/decay + tensors used by the **reverse-direction** kernel calls (forward + save + backward). The forward direction always uses the + unmasked ``beta`` / ``decay``. This mirrors the inference path + in :func:`fused_bigdn_func` and unlocks chunk-causal autograd + training: callers typically build ``beta_bwd`` / ``decay_bwd`` + as ``beta.clone()`` / ``decay.clone()`` with interior chunk + boundaries zeroed, so the anti-causal scan resets state at + every chunk boundary. + + When ``beta_bwd`` is ``None``, the kernel-emitted reverse- + direction beta gradient is summed into the forward-direction + gradient (returned via the ``beta`` slot) and the ``beta_bwd`` + gradient slot returns ``None``. When ``beta_bwd`` is provided, + the two gradient streams are kept separate: the forward- + direction gradient flows through the ``beta`` slot and the + reverse-direction gradient flows through the ``beta_bwd`` slot + so autograd can route them through any ``clone() + index = 0`` + masking applied by the caller. ``decay_bwd`` is handled + identically. + """ + + @staticmethod + def forward( + ctx, + qkv, + beta, + decay, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + F: int, + S: int, + k_scale: float = 1.0, + norm_eps: float = 1e-5, + eps: float = 1e-6, + beta_bwd: torch.Tensor | None = None, + decay_bwd: torch.Tensor | None = None, + ): + B, N, three, H, D = qkv.shape + C = H * D + assert three == 3 and N == F * S + cfg = _kcfg() + + if q_norm_weight is None: + q_norm_weight = torch.ones(C, device=qkv.device, dtype=torch.float32) + if k_norm_weight is None: + k_norm_weight = torch.ones(C, device=qkv.device, dtype=torch.float32) + + # Full-channel RMSNorm: inv_rms over all H*D dims. + q_raw = qkv[:, :, 0].float() # (B, N, H, D) + k_raw = qkv[:, :, 1].float() + q_inv_rms = torch.rsqrt((q_raw * q_raw).sum(dim=(-2, -1)) / C + norm_eps) # (B, N) + k_inv_rms = torch.rsqrt((k_raw * k_raw).sum(dim=(-2, -1)) / C + norm_eps) + + # Apply norm to Q and K: Q_normed = Q_raw * inv_rms * weight. + q_nw_hd = q_norm_weight.reshape(H, D) + k_nw_hd = k_norm_weight.reshape(H, D) + qkv_normed = qkv.clone() + qkv_normed[:, :, 0] = (q_raw * q_inv_rms[:, :, None, None] * q_nw_hd[None, None]).to(qkv.dtype) + qkv_normed[:, :, 1] = (k_raw * k_inv_rms[:, :, None, None] * k_nw_hd[None, None]).to(qkv.dtype) + + # Reverse-direction beta/decay overrides for chunk-causal masking. + # When the caller supplies ``beta_bwd`` / ``decay_bwd`` (typically + # ``beta.clone()`` / ``decay.clone()`` with interior chunk-boundary + # frames zeroed), the reverse-direction kernel reads them instead of + # the unmasked tensors so the anti-causal scan resets state at chunk + # boundaries. The forward (causal) direction always uses the + # unmasked ``beta`` / ``decay``. + beta_for_bwd_dir = beta_bwd if beta_bwd is not None else beta + decay_for_bwd_dir = decay_bwd if decay_bwd is not None else decay + + # Run forward-save with QK_NORM=0 on pre-normed data. + dummy_nw = torch.ones(D, device=qkv.device, dtype=torch.float32) + num_fwd, den_fwd, sv_fwd, sz_fwd, svc_fwd, szc_fwd = _run_fwd_save( + qkv_normed, + beta, + decay, + dummy_nw, + dummy_nw, + rope_cos, + rope_sin, + F, + S, + k_scale, + norm_eps, + eps, + False, + False, + cfg, + ) + num_bwd, den_bwd, sv_bwd, sz_bwd, svc_bwd, szc_bwd = _run_fwd_save( + qkv_normed, + beta_for_bwd_dir, + decay_for_bwd_dir, + dummy_nw, + dummy_nw, + rope_cos, + rope_sin, + F, + S, + k_scale, + norm_eps, + eps, + False, + True, + cfg, + ) + + # Combine: out = (num_fwd + num_bwd) / (den_fwd + den_bwd + eps). + total_num = num_fwd.float() + num_bwd.float() + total_den = den_fwd.float() + den_bwd.float() + total_den_exp = total_den.permute(0, 2, 1).unsqueeze(-1) # (B, N, H, 1) + out = (total_num / (total_den_exp + eps)).to(qkv.dtype) + + # Save ``beta_bwd`` / ``decay_bwd`` (possibly ``None``) so the + # backward pass can (a) replay the reverse-direction kernel against + # the same masked inputs, and (b) decide whether to keep the + # reverse-direction beta/decay gradients separate (caller-supplied + # override) or fold them into the forward-direction gradient + # (no override, legacy behaviour). + ctx.save_for_backward( + qkv, + beta, + decay, + q_norm_weight, + k_norm_weight, + q_inv_rms, + k_inv_rms, + rope_cos, + rope_sin, + sv_fwd, + sz_fwd, + svc_fwd, + szc_fwd, + sv_bwd, + sz_bwd, + svc_bwd, + szc_bwd, + out, + total_den.to(qkv.dtype), + beta_bwd, + decay_bwd, + ) + _, _dot_prec, _, _ = _resolve_launch_config() + ctx.dot_prec = _dot_prec + ctx.F = F + ctx.S = S + ctx.k_scale = k_scale + ctx.norm_eps = norm_eps + ctx.eps = eps + return out + + @staticmethod + def backward(ctx, dout): + ( + qkv, + beta, + decay, + q_norm_weight, + k_norm_weight, + q_inv_rms, + k_inv_rms, + rope_cos, + rope_sin, + sv_fwd, + sz_fwd, + svc_fwd, + szc_fwd, + sv_bwd, + sz_bwd, + svc_bwd, + szc_bwd, + out, + total_den_saved, + beta_bwd_saved, + decay_bwd_saved, + ) = ctx.saved_tensors + + # Track whether the caller supplied separate ``beta_bwd`` / + # ``decay_bwd`` overrides; this controls whether the reverse- + # direction kernel gradients are summed into the forward-direction + # slot (legacy behaviour) or routed back through dedicated grad + # slots so autograd can flow through the caller's masking ops + # (``clone() + index = 0``). + has_beta_bwd = beta_bwd_saved is not None + has_decay_bwd = decay_bwd_saved is not None + + B, N, three, H, D = qkv.shape + C = H * D + + # Recompute qkv_normed (avoid saving B*N*3*H*D extra tensor). + q_raw = qkv[:, :, 0].float() + k_raw = qkv[:, :, 1].float() + q_nw_hd = q_norm_weight.reshape(H, D) + k_nw_hd = k_norm_weight.reshape(H, D) + qkv_normed = qkv.clone() + qkv_normed[:, :, 0] = (q_raw * q_inv_rms[:, :, None, None] * q_nw_hd[None, None]).to(qkv.dtype) + qkv_normed[:, :, 1] = (k_raw * k_inv_rms[:, :, None, None] * k_nw_hd[None, None]).to(qkv.dtype) + F_val = ctx.F + S = ctx.S + eps = ctx.eps + BLOCK_D, _, _, _, _, cfg, beta, decay = _prepare_launch(D, beta, decay) + + # Reverse-direction beta/decay actually fed to the reverse kernel. + # When the caller supplied an override, we replay against the + # masked tensor; otherwise we reuse the unmasked beta/decay so the + # legacy summing path is bit-identical to the pre-extension + # behaviour. + if has_beta_bwd: + beta_for_bwd_dir = beta_bwd_saved.contiguous() + else: + beta_for_bwd_dir = beta + if has_decay_bwd: + decay_for_bwd_dir = decay_bwd_saved.contiguous() + else: + decay_for_bwd_dir = decay + + # ---- Pre-compute dnum and dden ---- + total_den_exp = total_den_saved.float().permute(0, 2, 1).unsqueeze(-1) + inv_total_den = 1.0 / (total_den_exp + eps) + dnum = (dout.float() * inv_total_den).to(qkv.dtype).contiguous() + dden = ( + (-(dout.float() * out.float()).sum(dim=-1) * inv_total_den.squeeze(-1)) + .permute(0, 2, 1) + .to(qkv.dtype) + .contiguous() + ) + del out, total_den_saved, total_den_exp, inv_total_den + + dummy_nw = torch.ones(D, device=qkv.device, dtype=torch.float32) + + # ---- Backward for forward direction (QK_NORM=0, operates on normed QKV) ---- + dqkv_fwd = torch.zeros_like(qkv) + dbeta_fwd = torch.zeros_like(beta) + ddecay_fwd = torch.zeros_like(decay) + + def _run_triton_bwd(sv, sz, svc, szc, dqkv_out, dbeta_out, ddecay_out, reverse_bwd, beta_kernel, decay_kernel): + """Try backward kernel with progressively fewer warps on tmem overflow. + + ``beta_kernel`` / ``decay_kernel`` are passed as explicit + arguments (instead of closing over the outer-scope ``beta`` / + ``decay``) so the reverse-direction call can replay against the + chunk-causal-masked tensors (``beta_bwd`` / ``decay_bwd``) when + present, while the forward-direction call always uses the + unmasked ``beta`` / ``decay``. + """ + nw = cfg["num_warps"] + if ctx.dot_prec >= 1: + nw = min(nw, 4) + while nw >= 1: + try: + _fused_gdn_bwd_kernel[(B * H,)]( + qkv_normed, + qkv_normed.stride(0), + qkv_normed.stride(1), + qkv_normed.stride(2), + qkv_normed.stride(3), + qkv_normed.stride(4), + beta_kernel, + decay_kernel, + dummy_nw, + dummy_nw, + rope_cos, + rope_sin, + sv, + sz, + svc, + szc, + dnum, + dden, + dqkv_out, + dbeta_out, + ddecay_out, + H=H, + F=F_val, + S=S, + D=D, + K_SCALE=ctx.k_scale, + NORM_EPS=ctx.norm_eps, + EPS=eps, + QK_NORM=0, + STATE_FP32=1 if cfg["STATE_FP32"] else 0, + REVERSE_BWD=reverse_bwd, + BIDI_MODE=1, + DOT_PRECISION=ctx.dot_prec, + BLOCK_D=BLOCK_D, + BLOCK_S=cfg["BLOCK_S"], + num_stages=cfg["num_stages"], + num_warps=nw, + ) + return # success + except Exception as e: + if "OutOfResources" in str(type(e).__name__) or "out of resource" in str(e).lower(): + nw = nw // 2 + if nw >= 1: + continue + raise RuntimeError( + "FusedBiGDN backward: Triton kernel OutOfResources at all warp counts " + f"(8, 4, 2, 1). Most recent error: {e}" + ) from e + else: + raise + + _run_triton_bwd(sv_fwd, sz_fwd, svc_fwd, szc_fwd, dqkv_fwd, dbeta_fwd, ddecay_fwd, 0, beta, decay) + del sv_fwd, sz_fwd, svc_fwd, szc_fwd + + # ---- Backward for reversed direction (replays against masked beta/decay if any) ---- + # Allocate kernel-output gradients with the exact shape the kernel + # writes — these always match the input ``beta_for_bwd_dir`` / + # ``decay_for_bwd_dir`` shapes (override or fall-back). + dqkv_bwd = torch.zeros_like(qkv) + dbeta_bwd_kernel = torch.zeros_like(beta_for_bwd_dir) + ddecay_bwd_kernel = torch.zeros_like(decay_for_bwd_dir) + + _run_triton_bwd( + sv_bwd, + sz_bwd, + svc_bwd, + szc_bwd, + dqkv_bwd, + dbeta_bwd_kernel, + ddecay_bwd_kernel, + 1, + beta_for_bwd_dir, + decay_for_bwd_dir, + ) + del sv_bwd, sz_bwd, svc_bwd, szc_bwd + del qkv_normed, dnum, dden + + # Q/K/V gradient is always summed: qkv is shared by both directions. + dqkv_fwd += dqkv_bwd + del dqkv_bwd + + # Beta gradient: route depends on whether the caller supplied an + # override. With override -> keep separate (so autograd routes the + # reverse-direction grad through the caller's clone+mask op). + # Without override -> sum into the forward-direction grad + # (legacy behaviour, bit-identical to pre-extension code). + if has_beta_bwd: + dbeta = dbeta_fwd + dbeta_bwd_out: torch.Tensor | None = dbeta_bwd_kernel + else: + dbeta_fwd += dbeta_bwd_kernel + dbeta = dbeta_fwd + dbeta_bwd_out = None + del dbeta_bwd_kernel + + # Decay gradient: same routing logic, independent of beta override. + if has_decay_bwd: + ddecay = ddecay_fwd + ddecay_bwd_out: torch.Tensor | None = ddecay_bwd_kernel + else: + ddecay_fwd += ddecay_bwd_kernel + ddecay = ddecay_fwd + ddecay_bwd_out = None + del ddecay_bwd_kernel + + dqkv_normed = dqkv_fwd + + # ---- Full-channel RMSNorm backward for Q and K ---- + # y = x * inv_rms * w -> dL/dx = inv_rms*w*dL/dy - inv_rms^3/C * x * sum(w*dL/dy*x) + # Process Q and K sequentially to reduce peak fp32 memory. + + # Q norm backward. + q_irms = q_inv_rms[:, :, None, None] + dq_normed = dqkv_normed[:, :, 0].float() + gw_q = dq_normed * q_nw_hd[None, None] + dq_nw = (dq_normed * q_raw * q_irms).sum(dim=(0, 1)).reshape(-1) + corr_q = (gw_q * q_raw).sum(dim=(-2, -1), keepdim=True) + dqkv_normed[:, :, 0] = (q_irms * gw_q - (q_irms**3) / C * q_raw * corr_q).to(qkv.dtype) + del dq_normed, gw_q, corr_q, q_raw, q_irms + + # K norm backward. + k_irms = k_inv_rms[:, :, None, None] + dk_normed = dqkv_normed[:, :, 1].float() + gw_k = dk_normed * k_nw_hd[None, None] + dk_nw = (dk_normed * k_raw * k_irms).sum(dim=(0, 1)).reshape(-1) + corr_k = (gw_k * k_raw).sum(dim=(-2, -1), keepdim=True) + dqkv_normed[:, :, 1] = (k_irms * gw_k - (k_irms**3) / C * k_raw * corr_k).to(qkv.dtype) + del dk_normed, gw_k, corr_k, k_raw, k_irms + + return ( + dqkv_normed, + dbeta, + ddecay, + dq_nw.to(q_norm_weight.dtype), + dk_nw.to(k_norm_weight.dtype), + None, # rope_cos + None, # rope_sin + None, # F + None, # S + None, # k_scale + None, # norm_eps + None, # eps + dbeta_bwd_out, # beta_bwd + ddecay_bwd_out, # decay_bwd + ) + + +def fused_bigdn_forward_with_grad( + qkv: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + q_norm_weight: torch.Tensor | None, + k_norm_weight: torch.Tensor | None, + rope_cos: torch.Tensor, + rope_sin: torch.Tensor, + F: int, + S: int, + k_scale: float = 1.0, + norm_eps: float = 1e-5, + eps: float = 1e-6, + beta_bwd: torch.Tensor | None = None, + decay_bwd: torch.Tensor | None = None, +) -> torch.Tensor: + """Bidirectional fused BiGDN with autograd support (full-channel RMSNorm). + + Unlike ``fused_bigdn_func`` (which expects pre-computed ``q_inv_rms`` / + ``k_inv_rms``), this wrapper computes the full-channel inv-RMS in Python + so the norm backward can flow through the autograd graph naturally. + + Chunk-causal masking (optional): + Pass ``beta_bwd`` and/or ``decay_bwd`` to override the beta/decay + tensors used by the **reverse-direction** kernel only. These are + typically built by the caller as ``beta.clone()`` / ``decay.clone()`` + with interior chunk-boundary frames zeroed, so the anti-causal scan + resets state at chunk boundaries while the causal scan keeps full + context. The reverse-direction beta/decay gradients are routed + back through the ``beta_bwd`` / ``decay_bwd`` slots (instead of + being summed into the forward-direction grad), which lets autograd + flow the reverse-direction gradient through the caller's + ``clone() + index = 0`` masking op. + + When ``beta_bwd`` / ``decay_bwd`` is ``None`` (default), behaviour + is bit-identical to the pre-extension full-sequence-bidirectional + path: the reverse-direction kernel uses the unmasked ``beta`` / + ``decay`` and its kernel-emitted gradient is summed into the + forward-direction gradient before being returned. + """ + return FusedBiGDNFunction.apply( + qkv, + beta, + decay, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + F, + S, + k_scale, + norm_eps, + eps, + beta_bwd, + decay_bwd, + ) diff --git a/integrations/sana/sana_wm/ops/fused_gdn_chunkwise.py b/integrations/sana/sana_wm/ops/fused_gdn_chunkwise.py new file mode 100644 index 000000000..715231860 --- /dev/null +++ b/integrations/sana/sana_wm/ops/fused_gdn_chunkwise.py @@ -0,0 +1,2269 @@ +# Copyright 2024 NVIDIA CORPORATION & AFFILIATES +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +""" +Fused GDN — Chunkwise-parallel forward (v2). + +V2 changes vs v1: + 1. Phase A is split into TWO kernels along the GDN data streams (KV and Z; + these are the two gating sub-paths within the GDN block, not CUDA + streams — both kernels are launched on the same CUDA stream): + _phase_a_kv_kernel: P_kv (with K_rot) + A (with K_rot, V) — uses RoPE + _phase_a_z_kernel : P_z (with K) + B (with K) — no RoPE + Z block is genuinely lighter (no V/Cos/Sin loads, no K_pair flip). + On H100 this enables 2 blocks/SM resident → multi-tenancy / latency hiding. + 2. Phase A stores (I - P_kv) and (I - P_z) instead of P_kv/P_z. Phase B then + uses these directly: M = g · (I-P_kv)·M + A_f. The MMA `(I-P_kv) @ M` folds + the identity-add into the matmul (no separate M-PM elementwise pass). + +BiGDN inference path: QK_NORM=1, USE_PRECOMPUTED_RMS=1, SAVE_STATE=0. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import torch +import triton +import triton.language as tl + +_CAM_IDENTITY_CACHE: dict[ + tuple[str, int | None, int, int, int], tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor] +] = {} + +# ════════════════════════════════════════════════════════════════ +# Per-architecture launch config (auto-selected via compute capability) +# ════════════════════════════════════════════════════════════════ +# +# Empirically tuned at production config (B=1..8, T=11, S=920, H=20, D=112) on +# A100 / H100 / GB200. Two effects matter: +# +# 1. **Precision sets BLOCK_S**: fp32 operand fragments are 2× the size of +# bf16. BLOCK_S=64 + fp32 → register spills (catastrophic, 40-100× slower). +# BLOCK_S=32 + fp32 → no spills. So fp32 mode forces BLOCK_S=32 everywhere. +# +# 2. **Arch sets BLOCK_S for bf16**: A100 (192 KB SRAM, fewer registers per +# block) prefers BLOCK_S=32 even at bf16. H100/GB200 (228 KB SRAM) tolerate +# BLOCK_S=64 cleanly at bf16. +# +# Each entry: (phase_a_warps, phase_a_BLOCK_S, +# phase_b_warps, phase_b_stages, +# phase_c_warps, phase_c_BLOCK_S, phase_c_stages) + +# ── Launch-config tuning table ───────────────────────────────────── +# +# We tune 8 knobs across 3 phases: +# Phase A : (nw, BS) streaming accumulator in registers +# Phase B : (nw, use_acc, ns) serial-F scan with persistent M in regs +# Phase C : (nw, BS, ns) streams Pass-2 output; loads fp32 M[128,128] +# +# Each arch × precision combination gets a named entry below. Values come from +# empirical sweeps (see commit log: T6 A100/H100 sweep 2026-04-19; Blackwell-DC +# 2026-04-20; Spark GB10 tuning notes in commits 5da52db6 / 3ad104d0) and from +# kernel-structure analysis (Phase B's persistent M[128,128] fp32 is 64 KB → nw +# controls register spread; Phase C's loaded M[128,128] is 64 KB → BS controls +# transient SMEM footprint). +# +# Adding a new arch: pick the closest existing bucket, then override individual +# fields in _CHUNKWISE_SHAPE_OVERRIDES once a targeted sweep lands. + + +@dataclass(frozen=True) +class _PhaseCfg: + nw: int # num_warps + BS: int = 0 # BLOCK_S (Phase A/C only; 0 = N/A for Phase B) + ns: int = 1 # num_stages + use_acc: bool = False # Phase B only: fold A_f via MMA accumulator + + +@dataclass(frozen=True) +class _ChunkwiseCfg: + A: _PhaseCfg + B: _PhaseCfg + C: _PhaseCfg + + def as_tuple(self) -> tuple: + """Flatten to the 8-tuple the legacy API returns.""" + return ( + self.A.nw, + self.A.BS, + self.B.nw, + self.B.ns, + self.B.use_acc, + self.C.nw, + self.C.BS, + self.C.ns, + ) + + +# ────────────────────────────────────────────────────────────────── +# Primary tuning table: (arch_key, prec_key) → _ChunkwiseCfg. +# Arch keys: +# "ampere" sm_80 A100 (164 KB SRAM, no WGMMA) +# "hopper" sm_90 H100 (228 KB SRAM, WGMMA) +# "blackwell_dc" sm_100 B200 / GB200 (228 KB SRAM, WGMMA v2) +# "blackwell_spark" sm_120+ with < 150 KB SRAM 5090 / GB10 (~102 KB SRAM) +# Prec keys: +# "bf16" dot_prec == 0 (bf16 TC, half-size operand fragments) +# "fp32" dot_prec >= 1 (TF32 TC or IEEE Markidis 3-pass; same launch shape) +# ────────────────────────────────────────────────────────────────── +_CHUNKWISE_TUNING: dict[tuple[str, str], _ChunkwiseCfg] = { + # A100: smaller SRAM than Hopper, no WGMMA → bigger CTAs hide MMA latency. + # Phase B fp32 needs nw=32 to spread persistent M across warps (no acc-fusion + # available pre-Hopper, so ns=2 fills the MMA pipeline slot instead). + ("ampere", "bf16"): _ChunkwiseCfg( + A=_PhaseCfg(nw=8, BS=32), + B=_PhaseCfg(nw=8, use_acc=False, ns=1), + C=_PhaseCfg(nw=4, BS=32, ns=1), # nw=4 bf16 C: 27% faster than nw=8 per T6 + ), + ("ampere", "fp32"): _ChunkwiseCfg( + # 2026-04-30 PM retune: Phase A nw=8 → 16 BS=32 yields 8-13× speedup + # across F ∈ {3, 5, 11, 14, 17, 20} (cos=1.0 verified). Old nw=8 was a + # legacy default never re-swept; sweep showed nw=16 dominates every F. + # Closes A100 sink/rolling chunkwise regression where Phase B was + # already optimal (sub-percent tuning gap) — Phase A was the bottleneck. + A=_PhaseCfg(nw=16, BS=32), + B=_PhaseCfg(nw=32, use_acc=False, ns=2), # ns=2 fills pipe (no acc-fusion) + C=_PhaseCfg(nw=16, BS=32, ns=1), # 2026-04-30 retune: nw=16 BS=32 is 2.8x faster (was nw=8 BS=16) + ), + # Hopper (H100): WGMMA + 228 KB SRAM → big tiles win at bf16. + # Phase B fp32 uses acc-fusion (MMA accumulator folds A_f in one op, +12%). + ("hopper", "bf16"): _ChunkwiseCfg( + A=_PhaseCfg(nw=8, BS=64), + B=_PhaseCfg(nw=4, use_acc=False, ns=1), # small CTAs pack better on WGMMA + C=_PhaseCfg(nw=8, BS=32, ns=1), + ), + ("hopper", "fp32"): _ChunkwiseCfg( + A=_PhaseCfg(nw=8, BS=32), # fp32 operand 2× bigger → half BS + B=_PhaseCfg( + nw=32, use_acc=False, ns=1 + ), # 2026-04-29 retune: acc_fusion=False is 3x faster post precision-gate fix + C=_PhaseCfg(nw=16, BS=32, ns=1), # 2026-04-30 retune: nw=16 BS=32 is 1.7x faster (was nw=8 BS=16) + ), + # Blackwell-DC (B200 / GB200): 228 KB SRAM + improved WGMMA codegen. + # bf16 likes small CTAs (nw=4); fp32 stays at nw=8 (nw=4 + BS=64 fp32 = 92× regression). + ("blackwell_dc", "bf16"): _ChunkwiseCfg( + A=_PhaseCfg(nw=4, BS=64), + B=_PhaseCfg(nw=4, use_acc=False, ns=1), + C=_PhaseCfg(nw=8, BS=64, ns=1), # 228 KB SRAM leaves room for BS=64 bf16 + ), + ("blackwell_dc", "fp32"): _ChunkwiseCfg( + A=_PhaseCfg( + nw=8, BS=128 + ), # 2026-04-30 retune: nw=8 BS=128 ~5% faster at production F=3-6 (sweep across F=3,5,6,11) + B=_PhaseCfg( + nw=32, use_acc=False, ns=3 + ), # 2026-04-29 retune: 14x faster (was nw=8 acc=True 17ms; now nw=32 ns=3 acc=False 1.23ms) + C=_PhaseCfg( + nw=4, BS=64, ns=1 + ), # 2026-04-30 retune: nw=4 BS=64 is 3-5x faster than old nw=8 BS=16 (sweep 2026-04-30) + ), + # Blackwell-Spark (5090 / GB10, ~102 KB SRAM): shares SRAM penalty of small + # chips but not Blackwell-DC's WGMMA-v2 register-spread benefit. Empirically + # behaves like Hopper at fp32 (Phase B wants nw=32 to spread persistent M + # across warps, not nw=8 like DC). BS shrunk one step vs DC; Phase A bf16 + # wants nw=8 (nw=4 tested 22× slower per 2026-04-20 sweep). + # Sweep 2026-04-24 (prod dim F=11 S=920): Phase B nw=32 gives 1.84×/2.65× + # (GB10/5090) at fp32 over prior nw=8 setting. + ("blackwell_spark", "bf16"): _ChunkwiseCfg( + A=_PhaseCfg(nw=8, BS=32), + B=_PhaseCfg(nw=8, use_acc=False, ns=1), # nw=8 (not 4) at bf16: ~5% across F=3,6,11 + # 2026-05-06 P1/P2 retune (5090, F=11 S=920): C.nw=4 BS=32 is ~3.5% + # faster than nw=8 (Phase C is bandwidth-bound, fewer warps schedules + # better on the small SRAM). BS=64 bf16 on Spark OOMs SRAM. + C=_PhaseCfg(nw=4, BS=32, ns=1), + ), + ("blackwell_spark", "fp32"): _ChunkwiseCfg( + A=_PhaseCfg(nw=8, BS=16), # fp32 operand 2× bigger → BS=16 (half of DC's 32) + # 2026-05-06 retune: nw=16 OOMs the 102 KB SRAM cap at TF32 on 5090 + # (131 KB needed). nw=8 fits and is within noise of the prior nw=16 + # benchmark. The Phase B D-tile path (auto-enabled on spark, see + # `_pick_phase_b_d_splits`) is ~2.6× faster than this baseline at TF32 + # and ~13% faster at IEEE — these baseline params only apply when + # PHASE_B_D_SPLITS=1 is forced. + B=_PhaseCfg(nw=8, use_acc=False, ns=1), + C=_PhaseCfg(nw=8, BS=16, ns=1), # binding constraint: M.fp32 64 KB + Q stage + ), +} + + +# ────────────────────────────────────────────────────────────────── +# Shape-aware override table: empty by default. Keyed by +# (arch_key, prec_key, shape_hint) +# where shape_hint is a free-form string (e.g. "small_BH", "large_F", +# "B>=8") chosen when populating. Lookup is exact-match; values are +# full `_ChunkwiseCfg` instances (no partial overrides — copy-paste +# from `_CHUNKWISE_TUNING` and edit the one phase you want to change). +# +# Leave empty unless a targeted sweep shows a particular shape regresses +# with the broad arch config. Adding here is strictly additive — base +# table remains the fallback. +# ────────────────────────────────────────────────────────────────── +_CHUNKWISE_SHAPE_OVERRIDES: dict[tuple[str, str, str], _ChunkwiseCfg] = {} + + +# Per-(cap, dot_prec) exact overrides (pins a specific GPU model if the arch +# bucket is wrong for it). Also empty by default. +_ARCH_OVERRIDES: dict = {} + + +def _arch_key(cap: tuple) -> str: + """Map compute capability → named arch bucket in `_CHUNKWISE_TUNING`. + + Blackwell (cap[0] >= 10) is split into "blackwell_dc" and "blackwell_spark" + by SRAM size (≥150 KB vs less). Without CUDA or for unknown archs we + default to the conservative "ampere" bucket. + """ + if cap[0] == 8: + return "ampere" + if cap[0] == 9: + return "hopper" + if cap[0] >= 10: + has_big_sram = True + if torch.cuda.is_available(): + props = torch.cuda.get_device_properties(0) + smem = getattr(props, "shared_memory_per_multiprocessor", 228 * 1024) + has_big_sram = smem >= 150 * 1024 + return "blackwell_dc" if has_big_sram else "blackwell_spark" + return "ampere" + + +def _prec_key(dot_prec: int) -> str: + return "fp32" if dot_prec >= 1 else "bf16" + + +def _auto_config(dot_prec: int, cap: tuple, shape_hint: str | None = None) -> tuple: + """Look up chunkwise kernel launch params from the tuning table. + + Resolution order: + 1. `_ARCH_OVERRIDES[(cap, dot_prec)]` — exact-capability pin, highest priority. + 2. `_CHUNKWISE_SHAPE_OVERRIDES[(arch, prec, shape_hint)]` — sweep-driven overrides. + 3. `_CHUNKWISE_TUNING[(arch, prec)]` — primary per-(arch, prec) table. + 4. Fallback to ("ampere", prec) if the arch is unrecognised. + + Returns the legacy 8-tuple `(a_nw, a_BS, b_nw, b_ns, b_use_acc, c_nw, c_BS, c_ns)` + for backward compatibility with `_get_arch_config` callers. + """ + arch = _arch_key(cap) + prec = _prec_key(dot_prec) + + if shape_hint is not None: + cfg = _CHUNKWISE_SHAPE_OVERRIDES.get((arch, prec, shape_hint)) + if cfg is not None: + return cfg.as_tuple() + + cfg = _CHUNKWISE_TUNING.get((arch, prec)) or _CHUNKWISE_TUNING[("ampere", prec)] + return cfg.as_tuple() + + +def _get_arch_config( + dot_precision: int = 0, + shape_hint: str | None = None, + device: torch.device | int | None = None, +): + """Returns (a_warps, a_BLOCK_S, b_warps, b_stages, b_use_acc_fusion, + c_warps, c_BLOCK_S, c_stages). + + dot_precision: 0=bf16 TC, 1=TF32 TC, 2=IEEE fp32. + shape_hint: optional string key for `_CHUNKWISE_SHAPE_OVERRIDES`. + device: device whose capability drives the lookup. Defaults to the + current CUDA device — pass ``qkv.device`` (or any input + tensor's device) when launching kernels in heterogeneous + or multi-GPU single-process setups so the right tuning + bucket is chosen. + """ + if not torch.cuda.is_available(): + cap = (9, 0) # assume modern when querying from CPU + else: + if device is None: + dev_idx = torch.cuda.current_device() + elif isinstance(device, int): + dev_idx = device + else: + dev_idx = device.index if device.index is not None else torch.cuda.current_device() + cap = torch.cuda.get_device_capability(dev_idx) + key = (cap, dot_precision) + if key in _ARCH_OVERRIDES: + return _ARCH_OVERRIDES[key] + return _auto_config(dot_precision, cap, shape_hint) + + +# ════════════════════════════════════════════════════════════════ +# Phase A — split into KV and Z kernels +# ════════════════════════════════════════════════════════════════ + + +@triton.jit +def _phase_a_kv_kernel( + qkv_ptr, + stride_b: tl.constexpr, + stride_n: tl.constexpr, + stride_3: tl.constexpr, + stride_h: tl.constexpr, + stride_d: tl.constexpr, + beta_ptr, + k_inv_rms_ptr, + k_norm_w_ptr, + rope_cos_ptr, + rope_sin_ptr, + I_minus_P_kv_ptr, # output: (I - K_rot^T diag(β) K_rot) + A_ptr, # output: K_rot^T diag(β) V + H: tl.constexpr, + F: tl.constexpr, + S: tl.constexpr, + D: tl.constexpr, + K_SCALE, + NORM_EPS: tl.constexpr, + DOT_PRECISION: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_S: tl.constexpr, + SKIP_RELU: tl.constexpr = False, +): + if DOT_PRECISION >= 1: + dot_dtype = tl.float32 + else: + dot_dtype = tl.bfloat16 + dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" + + pid = tl.program_id(0) + pid_b = pid // (H * F) + pid_hf = pid % (H * F) + pid_h = pid_hf // F + pid_f = pid_hf % F + bh = pid_b * H + pid_h + N: tl.constexpr = F * S + + qkv_bh = qkv_ptr + pid_b * stride_b + pid_h * stride_h + beta_bhf = beta_ptr + bh * (F * S) + pid_f * S + I_P_kv_bhf = I_minus_P_kv_ptr + bh * F * BLOCK_D * BLOCK_D + pid_f * BLOCK_D * BLOCK_D + A_bhf = A_ptr + bh * F * BLOCK_D * BLOCK_D + pid_f * BLOCK_D * BLOCK_D + + offs_d = tl.arange(0, BLOCK_D) + mask_d = offs_d < D + offs_d_pair = offs_d ^ 1 + mask_d_pair = offs_d_pair < D + + nw_offset = pid_h * D + k_nw = tl.load(k_norm_w_ptr + nw_offset + offs_d, mask=mask_d, other=0.0).to(tl.float32) + k_nw_pair = tl.load(k_norm_w_ptr + nw_offset + offs_d_pair, mask=mask_d_pair, other=0.0).to(tl.float32) + + # KV stream accumulators (in-loop fp32 to avoid bf16 round-off compounding) + P_kv_acc = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) + A_acc = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) + + k_scale = K_SCALE + n_base = pid_f * S + + for s0 in range(0, S, BLOCK_S): + offs_s = s0 + tl.arange(0, BLOCK_S) + mask_s = offs_s < S + mask_sd = mask_s[:, None] & mask_d[None, :] + n_idx = n_base + offs_s + + k_ptrs = qkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d[None, :] * stride_d + v_ptrs = qkv_bh + n_idx[:, None] * stride_n + 2 * stride_3 + offs_d[None, :] * stride_d + K_raw = tl.load(k_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + V_raw = tl.load(v_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + beta_t = tl.load(beta_bhf + offs_s, mask=mask_s, other=0.0).to(tl.float32) + + k_inv_rms = tl.load(k_inv_rms_ptr + pid_b * N + n_idx, mask=mask_s, other=1.0).to(tl.float32) + K_normed = K_raw * k_inv_rms[:, None] * k_nw[None, :] + if SKIP_RELU: + K = K_normed * k_scale + else: + K = tl.where(K_normed > 0, K_normed, 0.0) * k_scale + + K_pair_raw = tl.reshape( + tl.flip(tl.reshape(K_raw, (BLOCK_S, BLOCK_D // 2, 2)), dim=2), + (BLOCK_S, BLOCK_D), + ) + K_pair_normed = K_pair_raw * k_inv_rms[:, None] * k_nw_pair[None, :] + if SKIP_RELU: + K_pair = K_pair_normed * k_scale + else: + K_pair = tl.where(K_pair_normed > 0, K_pair_normed, 0.0) * k_scale + + rope_ptrs = n_idx[:, None] * D + offs_d[None, :] + Cos = tl.load(rope_cos_ptr + rope_ptrs, mask=mask_sd, other=1.0).to(tl.float32) + Sin = tl.load(rope_sin_ptr + rope_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + K_rot = K * Cos + K_pair * Sin + + beta_Krot = beta_t[:, None] * K_rot + beta_V = beta_t[:, None] * V_raw + + K_rot_T = tl.trans(K_rot) + P_kv_acc += tl.dot(K_rot_T.to(dot_dtype), beta_Krot.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) + A_acc += tl.dot(K_rot_T.to(dot_dtype), beta_V.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) + + # Store bf16 outputs. Padded positions are 0 by construction (K_rot is 0 outside D). + offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] + diag_in_range = (offs_d[:, None] == offs_d[None, :]) & mask_d[:, None] & mask_d[None, :] + I_minus_P_kv = tl.where(diag_in_range, 1.0 - P_kv_acc, -P_kv_acc) + if DOT_PRECISION >= 1: + tl.store(I_P_kv_bhf + offs_dd, I_minus_P_kv) + tl.store(A_bhf + offs_dd, A_acc) + else: + tl.store(I_P_kv_bhf + offs_dd, I_minus_P_kv.to(tl.bfloat16)) + tl.store(A_bhf + offs_dd, A_acc.to(tl.bfloat16)) + + +@triton.jit +def _phase_a_z_kernel( + qkv_ptr, + stride_b: tl.constexpr, + stride_n: tl.constexpr, + stride_3: tl.constexpr, + stride_h: tl.constexpr, + stride_d: tl.constexpr, + beta_ptr, + k_inv_rms_ptr, + k_norm_w_ptr, + I_minus_P_z_ptr, # output: (I - K^T diag(β) K) + B_ptr, # output: K^T β + H: tl.constexpr, + F: tl.constexpr, + S: tl.constexpr, + D: tl.constexpr, + K_SCALE, + NORM_EPS: tl.constexpr, + DOT_PRECISION: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """Z stream: uses K (no RoPE). Cheaper than KV — no V load, no RoPE compute, + no K_pair derivation.""" + if DOT_PRECISION >= 1: + dot_dtype = tl.float32 + else: + dot_dtype = tl.bfloat16 + dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" + + pid = tl.program_id(0) + pid_b = pid // (H * F) + pid_hf = pid % (H * F) + pid_h = pid_hf // F + pid_f = pid_hf % F + bh = pid_b * H + pid_h + N: tl.constexpr = F * S + + qkv_bh = qkv_ptr + pid_b * stride_b + pid_h * stride_h + beta_bhf = beta_ptr + bh * (F * S) + pid_f * S + I_P_z_bhf = I_minus_P_z_ptr + bh * F * BLOCK_D * BLOCK_D + pid_f * BLOCK_D * BLOCK_D + B_bhf = B_ptr + bh * F * BLOCK_D + pid_f * BLOCK_D + + offs_d = tl.arange(0, BLOCK_D) + mask_d = offs_d < D + + nw_offset = pid_h * D + k_nw = tl.load(k_norm_w_ptr + nw_offset + offs_d, mask=mask_d, other=0.0).to(tl.float32) + + P_z_acc = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) + B_acc = tl.zeros([BLOCK_D], dtype=tl.float32) + + k_scale = K_SCALE + n_base = pid_f * S + + for s0 in range(0, S, BLOCK_S): + offs_s = s0 + tl.arange(0, BLOCK_S) + mask_s = offs_s < S + mask_sd = mask_s[:, None] & mask_d[None, :] + n_idx = n_base + offs_s + + # Only K_raw needed (no V, no Cos/Sin) + k_ptrs = qkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d[None, :] * stride_d + K_raw = tl.load(k_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + beta_t = tl.load(beta_bhf + offs_s, mask=mask_s, other=0.0).to(tl.float32) + + k_inv_rms = tl.load(k_inv_rms_ptr + pid_b * N + n_idx, mask=mask_s, other=1.0).to(tl.float32) + K_normed = K_raw * k_inv_rms[:, None] * k_nw[None, :] + K = tl.where(K_normed > 0, K_normed, 0.0) * k_scale + + beta_K = beta_t[:, None] * K + + K_T = tl.trans(K) + P_z_acc += tl.dot(K_T.to(dot_dtype), beta_K.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) + B_acc += tl.sum(beta_K, axis=0) + + offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] + diag_in_range = (offs_d[:, None] == offs_d[None, :]) & mask_d[:, None] & mask_d[None, :] + I_minus_P_z = tl.where(diag_in_range, 1.0 - P_z_acc, -P_z_acc) + + if DOT_PRECISION >= 1: + tl.store(I_P_z_bhf + offs_dd, I_minus_P_z) + else: + tl.store(I_P_z_bhf + offs_dd, I_minus_P_z.to(tl.bfloat16)) + # B stays fp32 (vector, only 0.5 KB, negligible HBM cost) + tl.store(B_bhf + offs_d, B_acc) + + +def phase_a( + qkv: torch.Tensor, + beta: torch.Tensor, + q_inv_rms: torch.Tensor, + k_inv_rms: torch.Tensor, + q_norm_w: torch.Tensor, + k_norm_w: torch.Tensor, + rope_cos: torch.Tensor, + rope_sin: torch.Tensor, + F: int, + S: int, + k_scale: float = 1.0, + norm_eps: float = 1e-5, + num_warps: int | None = None, + num_stages: int = 1, + BLOCK_S: int | None = None, + dot_precision: int = 0, + skip_relu: bool = False, + skip_z: bool = False, +): + """Compute (I-P_kv), A, (I-P_z), B for all (B, H, F) via 2 kernels (KV + Z). + + `skip_relu=True` makes the K-stream prep a pure linear chain (no ReLU on + K_normed * k_scale). Used by the camera-branch chunkwise wrapper, where K + has already been ReLU'd by the cam_prep kernel and subsequently rotated + by UCPE+RoPE — re-applying ReLU on the rotated values would clobber + legitimate negatives. + + `skip_z=True` skips the Phase A Z kernel entirely and returns placeholder + tensors for I_P_z and B_z. Used by NUM_ONLY callers (camera branch) to + avoid wasted Z-stream prep when the denominator scan won't be used. + """ + # Auto-pick (num_warps, BLOCK_S) per arch+precision unless overridden + if num_warps is None or BLOCK_S is None: + a_w, a_bs, *_ = _get_arch_config(dot_precision, device=qkv.device) + if num_warps is None: + num_warps = a_w + if BLOCK_S is None: + BLOCK_S = a_bs + B, N, three, H, D = qkv.shape + assert three == 3 and N == F * S + BLOCK_D = triton.next_power_of_2(D) + BH = B * H + + # FAIR-COMPARE PATCH: keep fp32 inter-phase bridge at P0/P1 to match pytorch/fused + bridge_dtype = torch.float32 if dot_precision >= 1 else torch.bfloat16 + I_P_kv = torch.empty(BH, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=bridge_dtype) + A = torch.empty(BH, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=bridge_dtype) + + beta_c = beta.contiguous() + grid = (BH * F,) + + _phase_a_kv_kernel[grid]( + qkv, + qkv.stride(0), + qkv.stride(1), + qkv.stride(2), + qkv.stride(3), + qkv.stride(4), + beta_c, + k_inv_rms, + k_norm_w, + rope_cos, + rope_sin, + I_P_kv, + A, + H=H, + F=F, + S=S, + D=D, + K_SCALE=k_scale, + NORM_EPS=norm_eps, + DOT_PRECISION=dot_precision, + BLOCK_D=BLOCK_D, + BLOCK_S=BLOCK_S, + SKIP_RELU=skip_relu, + num_warps=num_warps, + num_stages=num_stages, + ) + + if skip_z: + # NUM_ONLY callers (camera branch) do not consume the Z scan. Return + # placeholders and let Phase B skip all Z loads/stores as well. + I_P_z = torch.empty(1, device=qkv.device, dtype=bridge_dtype) + B_z = torch.empty(1, device=qkv.device, dtype=torch.float32) + return I_P_kv, A, I_P_z, B_z + + I_P_z = torch.empty(BH, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=bridge_dtype) + # B stays fp32 — small vector (0.5 KB/frame), no benefit to downcast + B_z = torch.empty(BH, F, BLOCK_D, device=qkv.device, dtype=torch.float32) + + _phase_a_z_kernel[grid]( + qkv, + qkv.stride(0), + qkv.stride(1), + qkv.stride(2), + qkv.stride(3), + qkv.stride(4), + beta_c, + k_inv_rms, + k_norm_w, + I_P_z, + B_z, + H=H, + F=F, + S=S, + D=D, + K_SCALE=k_scale, + NORM_EPS=norm_eps, + DOT_PRECISION=dot_precision, + BLOCK_D=BLOCK_D, + BLOCK_S=BLOCK_S, + num_warps=num_warps, + num_stages=num_stages, + ) + return I_P_kv, A, I_P_z, B_z + + +# ════════════════════════════════════════════════════════════════ +# Phase B — serial scan, uses pre-stored (I - P) so MMA folds in M +# ════════════════════════════════════════════════════════════════ + + +@triton.jit +def _phase_b_kernel( + I_P_kv_ptr, + A_ptr, + I_P_z_ptr, + B_ptr, + decay_ptr, + M_fwd_ptr, + z_fwd_ptr, + M_rev_ptr, + z_rev_ptr, + init_state_kv_ptr, # (BH, BLOCK_D, BLOCK_D) — read when LOAD_INIT_STATE=1 + init_state_z_ptr, # (BH, BLOCK_D) + final_state_kv_ptr, # (BH, BLOCK_D, BLOCK_D) — written when SAVE_FINAL_STATE=1 + final_state_z_ptr, # (BH, BLOCK_D) + BH: tl.constexpr, + F: tl.constexpr, + BLOCK_D: tl.constexpr, + DOT_PRECISION: tl.constexpr, + USE_ACC_FUSION: tl.constexpr, + LOAD_INIT_STATE: tl.constexpr, # forward scan seeded with init state (vs zeros) + SAVE_FINAL_STATE: tl.constexpr, # write M_{F-1} of forward scan to final_state_* + DIRECTION: tl.constexpr, # 0=both, 1=fwd-only, 2=rev-only + COMBINED_HISTORY: tl.constexpr, # 1 → rev branch read-add-stores into M_fwd_ptr + # (M_hist[f] = M_fwd[f] + M_rev[f]); skips the F-1 zero-write so the fwd + # value at F-1 is preserved (rev contribution there is exactly zero anyway). + # Only meaningful when DIRECTION=0. Saves one Phase C launch + one M-shaped + # buffer downstream (Phase C runs once on M_hist instead of twice). + SKIP_Z: tl.constexpr, +): + if DOT_PRECISION >= 1: + dot_dtype = tl.float32 + else: + dot_dtype = tl.bfloat16 + dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" + + pid = tl.program_id(0) + bh = pid + + offs_d = tl.arange(0, BLOCK_D) + offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] + + # ── Forward scan (skip when DIRECTION=2 i.e. rev-only) ── + if DIRECTION != 2: + if LOAD_INIT_STATE: + M = tl.load(init_state_kv_ptr + bh * BLOCK_D * BLOCK_D + offs_dd).to(tl.float32) + if not SKIP_Z: + z = tl.load(init_state_z_ptr + bh * BLOCK_D + offs_d).to(tl.float32) + else: + M = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) + if not SKIP_Z: + z = tl.zeros([BLOCK_D], dtype=tl.float32) + for f in range(F): + I_P_kv_f = tl.load(I_P_kv_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd) + A_f = tl.load(A_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd) + g_f = tl.load(decay_ptr + bh * F + f).to(tl.float32) + + # M = g · (I - P_kv) M + A_f + if USE_ACC_FUSION: + # Pre-scale (I-P) by g, accumulate A_f directly via the MMA accumulator. + # Result: A_f + g·(I-P)·M in one MMA — no separate M_temp tensor. + I_P_scaled = I_P_kv_f.to(tl.float32) * g_f + M = tl.dot( + I_P_scaled.to(dot_dtype), + M.to(dot_dtype), + acc=A_f.to(tl.float32), + out_dtype=tl.float32, + input_precision=dot_ip, + ) + else: + M_temp = tl.dot(I_P_kv_f.to(dot_dtype), M.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) + M = g_f * M_temp + A_f + + tl.store(M_fwd_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd, M) + if not SKIP_Z: + I_P_z_f = tl.load(I_P_z_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd) + B_f = tl.load(B_ptr + bh * F * BLOCK_D + f * BLOCK_D + offs_d) + # z = g · (I - P_z) z + B_f + z_temp = tl.sum(I_P_z_f * z[None, :], axis=1) + z = g_f * z_temp + B_f + tl.store(z_fwd_ptr + bh * F * BLOCK_D + f * BLOCK_D + offs_d, z) + + # Save terminal forward state for state-cached inference (autoregressive sampling). + if SAVE_FINAL_STATE: + tl.store(final_state_kv_ptr + bh * BLOCK_D * BLOCK_D + offs_dd, M) + if not SKIP_Z: + tl.store(final_state_z_ptr + bh * BLOCK_D + offs_d, z) + + # ── Reverse scan (skip when DIRECTION=1 i.e. fwd-only) ── + if DIRECTION != 1: + M = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) + if not SKIP_Z: + z = tl.zeros([BLOCK_D], dtype=tl.float32) + # COMBINED_HISTORY mode: rev contributions get read-add-stored into the + # fwd buffer (which thereby becomes M_hist = M_fwd + M_rev). The F-1 + # zero-write is skipped so M_hist[F-1] keeps the fwd value (rev value + # there is zero by construction, so no add needed). + if not COMBINED_HISTORY: + tl.store(M_rev_ptr + bh * F * BLOCK_D * BLOCK_D + (F - 1) * BLOCK_D * BLOCK_D + offs_dd, M) + if not SKIP_Z: + tl.store(z_rev_ptr + bh * F * BLOCK_D + (F - 1) * BLOCK_D + offs_d, z) + for f_iter in range(F - 1): + f_src = F - 1 - f_iter + f_dst = f_src - 1 + I_P_kv_f = tl.load(I_P_kv_ptr + bh * F * BLOCK_D * BLOCK_D + f_src * BLOCK_D * BLOCK_D + offs_dd) + A_f = tl.load(A_ptr + bh * F * BLOCK_D * BLOCK_D + f_src * BLOCK_D * BLOCK_D + offs_dd) + g_f = tl.load(decay_ptr + bh * F + f_src).to(tl.float32) + + if USE_ACC_FUSION: + I_P_scaled = I_P_kv_f.to(tl.float32) * g_f + M = tl.dot( + I_P_scaled.to(dot_dtype), + M.to(dot_dtype), + acc=A_f.to(tl.float32), + out_dtype=tl.float32, + input_precision=dot_ip, + ) + else: + M_temp = tl.dot(I_P_kv_f.to(dot_dtype), M.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) + M = g_f * M_temp + A_f + + if not SKIP_Z: + I_P_z_f = tl.load(I_P_z_ptr + bh * F * BLOCK_D * BLOCK_D + f_src * BLOCK_D * BLOCK_D + offs_dd) + B_f = tl.load(B_ptr + bh * F * BLOCK_D + f_src * BLOCK_D + offs_d) + z_temp = tl.sum(I_P_z_f * z[None, :], axis=1) + z = g_f * z_temp + B_f + + if COMBINED_HISTORY: + # Read-add-store into the fwd buffer. The fwd loop has already + # written M_fwd[f_dst] to this slot; we add the rev contribution + # in place. Stays in L1/L2 since fwd just touched it. + M_addr = M_fwd_ptr + bh * F * BLOCK_D * BLOCK_D + f_dst * BLOCK_D * BLOCK_D + offs_dd + tl.store(M_addr, tl.load(M_addr) + M) + if not SKIP_Z: + z_addr = z_fwd_ptr + bh * F * BLOCK_D + f_dst * BLOCK_D + offs_d + tl.store(z_addr, tl.load(z_addr) + z) + else: + tl.store(M_rev_ptr + bh * F * BLOCK_D * BLOCK_D + f_dst * BLOCK_D * BLOCK_D + offs_dd, M) + if not SKIP_Z: + tl.store(z_rev_ptr + bh * F * BLOCK_D + f_dst * BLOCK_D + offs_d, z) + + +def phase_b_triton( + I_P_kv, + A, + I_P_z, + B, + decay, + F, + num_warps=None, + num_stages=None, + use_acc_fusion=None, + dot_precision=0, + init_state_kv=None, + init_state_z=None, + return_final_state=False, + direction=0, + combined_history=False, + skip_z=False, +): + """Phase B serial-F scan over (B*H,). + + Forward scan can be seeded with `init_state_kv`/`init_state_z` (autoregressive + sampling chunk > 0) and can write the terminal `M_{F-1}`/`z_{F-1}` to caller- + provided buffers when `return_final_state=True`. + + `direction`: 0=both (default), 1=forward-only, 2=reverse-only. Forward-only + skips reverse scan + reverse output buffers; reverse-only skips forward scan + + state load/save. Used by single-direction state-cached entry points. + + `combined_history` (only meaningful with direction=0): the rev branch + read-add-stores into the fwd buffer so its contents become + M_hist[f] = M_fwd[f] + M_rev[f] (and same for z). Lets the caller run + Phase C exactly once on the combined history, since Phase C is linear in + M and z (`Q @ (M_fwd + M_rev) = Q @ M_fwd + Q @ M_rev`). When set, + M_rev/z_rev outputs are placeholder dummies; only M_fwd/z_fwd carry data. + + `skip_z`: skip the denominator/Z recurrence entirely. Used by camera + numerator-only scans where Phase C runs with `num_only=True`. + + Returns (M_fwd, z_fwd, M_rev, z_rev) — and additionally (final_kv, final_z) + when return_final_state=True. Skipped-direction outputs are returned as a + 1-element placeholder tensor (kernel never touches them when DIRECTION + gates them off); callers should always discard the slot they didn't ask + for. Reverse scan is always seeded with zeros (per upstream's bidi + state-cache convention — only forward state is cached). + """ + BH = I_P_kv.shape[0] + _, _, BLOCK_D, _ = A.shape # A is always full [BH, F, BLOCK_D, BLOCK_D] + device, fdtype = I_P_kv.device, torch.float32 + + if num_warps is None or num_stages is None or use_acc_fusion is None: + _, _, b_w, b_s, b_acc, *_ = _get_arch_config(dot_precision, device=device) + if num_warps is None: + num_warps = b_w + if num_stages is None: + num_stages = b_s + if use_acc_fusion is None: + use_acc_fusion = b_acc + + if combined_history and direction != 0: + raise ValueError("combined_history=True requires direction=0 (bidi)") + + # Phase B kernel is DIRECTION-gated (constexpr); skipped-direction writes + # never happen, so we can hand it a 1-element placeholder for the inactive + # buffers and free ~4× M_fwd-shaped allocations per single-direction call. + decay_flat = decay.reshape(BH, F).contiguous().float() + + load_init = init_state_kv is not None + dummy = torch.empty(1, device=device, dtype=fdtype) + full_M = lambda: torch.empty(BH, F, BLOCK_D, BLOCK_D, device=device, dtype=fdtype) + full_z = lambda: torch.empty(BH, F, BLOCK_D, device=device, dtype=fdtype) + M_fwd = dummy if direction == 2 else full_M() + z_fwd = dummy if (direction == 2 or skip_z) else full_z() + # Combined-history mode reuses M_fwd/z_fwd as M_hist/z_hist; rev outputs + # become placeholders even though DIRECTION!=1. + M_rev = dummy if (direction == 1 or combined_history) else full_M() + z_rev = dummy if (direction == 1 or combined_history or skip_z) else full_z() + if load_init: + init_kv = init_state_kv.contiguous().view(BH, BLOCK_D, BLOCK_D) + init_z = dummy if skip_z else init_state_z.contiguous().view(BH, BLOCK_D) + else: + init_kv = dummy + init_z = dummy + + if return_final_state: + final_kv = torch.empty(BH, BLOCK_D, BLOCK_D, device=device, dtype=fdtype) + final_z = dummy if skip_z else torch.empty(BH, BLOCK_D, device=device, dtype=fdtype) + else: + final_kv = dummy + final_z = dummy + + d_splits, nw_override, ns_override, acc_override = _pick_phase_b_d_splits(BLOCK_D, dot_precision=dot_precision) + if d_splits > 1: + D_TILE = BLOCK_D // d_splits + # Use D-tile-specific tuning if available, else fall back to baseline tuning + nw_use = nw_override if nw_override is not None else num_warps + ns_use = ns_override if ns_override is not None else num_stages + acc_use = acc_override if acc_override is not None else use_acc_fusion + _phase_b_dtile_kernel[(BH, d_splits)]( + I_P_kv, + A, + I_P_z, + B, + decay_flat, + M_fwd, + z_fwd, + M_rev, + z_rev, + init_kv, + init_z, + final_kv, + final_z, + BH=BH, + F=F, + BLOCK_D=BLOCK_D, + D_TILE=D_TILE, + DOT_PRECISION=dot_precision, + USE_ACC_FUSION=acc_use, + LOAD_INIT_STATE=1 if load_init else 0, + SAVE_FINAL_STATE=1 if return_final_state else 0, + DIRECTION=direction, + COMBINED_HISTORY=1 if combined_history else 0, + SKIP_Z=1 if skip_z else 0, + num_warps=nw_use, + num_stages=ns_use, + ) + else: + _phase_b_kernel[(BH,)]( + I_P_kv, + A, + I_P_z, + B, + decay_flat, + M_fwd, + z_fwd, + M_rev, + z_rev, + init_kv, + init_z, + final_kv, + final_z, + BH=BH, + F=F, + BLOCK_D=BLOCK_D, + DOT_PRECISION=dot_precision, + USE_ACC_FUSION=use_acc_fusion, + LOAD_INIT_STATE=1 if load_init else 0, + SAVE_FINAL_STATE=1 if return_final_state else 0, + DIRECTION=direction, + COMBINED_HISTORY=1 if combined_history else 0, + SKIP_Z=1 if skip_z else 0, + num_warps=num_warps, + num_stages=num_stages, + ) + if return_final_state: + return M_fwd, z_fwd, M_rev, z_rev, final_kv, final_z + return M_fwd, z_fwd, M_rev, z_rev + + +# ════════════════════════════════════════════════════════════════ +# Phase B D-tile — j-axis split for grid parallelism (#118) +# ════════════════════════════════════════════════════════════════ +# Same recurrence as _phase_b_kernel but each program owns a D_TILE-wide +# slice of M's output column dim. Grid: (BH, d_splits). M_new[*, j_tile] +# only depends on M_prev[*, j_tile] and full (I-P_kv) — independent across +# j-tiles. z is unsplittable; only `pid_d == 0` updates/writes z. +@triton.jit +def _phase_b_dtile_kernel( + I_P_kv_ptr, + A_ptr, + I_P_z_ptr, + B_ptr, + decay_ptr, + M_fwd_ptr, + z_fwd_ptr, + M_rev_ptr, + z_rev_ptr, + init_state_kv_ptr, + init_state_z_ptr, + final_state_kv_ptr, + final_state_z_ptr, + BH: tl.constexpr, + F: tl.constexpr, + BLOCK_D: tl.constexpr, + D_TILE: tl.constexpr, + DOT_PRECISION: tl.constexpr, + USE_ACC_FUSION: tl.constexpr, + LOAD_INIT_STATE: tl.constexpr, + SAVE_FINAL_STATE: tl.constexpr, + DIRECTION: tl.constexpr, + COMBINED_HISTORY: tl.constexpr, + SKIP_Z: tl.constexpr, +): + if DOT_PRECISION >= 1: + dot_dtype = tl.float32 + else: + dot_dtype = tl.bfloat16 + dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" + + pid_bh = tl.program_id(0) + pid_d = tl.program_id(1) + bh = pid_bh + + offs_d_full = tl.arange(0, BLOCK_D) + offs_d_tile = pid_d * D_TILE + tl.arange(0, D_TILE) + offs_dd_full = offs_d_full[:, None] * BLOCK_D + offs_d_full[None, :] + offs_dd_tile = offs_d_full[:, None] * BLOCK_D + offs_d_tile[None, :] + + is_lead = pid_d == 0 + + if DIRECTION != 2: + if LOAD_INIT_STATE: + M = tl.load(init_state_kv_ptr + bh * BLOCK_D * BLOCK_D + offs_dd_tile).to(tl.float32) + else: + M = tl.zeros([BLOCK_D, D_TILE], dtype=tl.float32) + if not SKIP_Z: + z = tl.zeros([BLOCK_D], dtype=tl.float32) + if is_lead and LOAD_INIT_STATE: + z = tl.load(init_state_z_ptr + bh * BLOCK_D + offs_d_full).to(tl.float32) + + for f in range(F): + I_P_kv_f = tl.load(I_P_kv_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd_full) + A_f = tl.load(A_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd_tile) + g_f = tl.load(decay_ptr + bh * F + f).to(tl.float32) + + if USE_ACC_FUSION: + I_P_scaled = I_P_kv_f.to(tl.float32) * g_f + M = tl.dot( + I_P_scaled.to(dot_dtype), + M.to(dot_dtype), + acc=A_f.to(tl.float32), + out_dtype=tl.float32, + input_precision=dot_ip, + ) + else: + M_temp = tl.dot(I_P_kv_f.to(dot_dtype), M.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) + M = g_f * M_temp + A_f + + tl.store(M_fwd_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd_tile, M) + + if is_lead and not SKIP_Z: + I_P_z_f = tl.load(I_P_z_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd_full) + B_f = tl.load(B_ptr + bh * F * BLOCK_D + f * BLOCK_D + offs_d_full) + z_temp = tl.sum(I_P_z_f * z[None, :], axis=1) + z = g_f * z_temp + B_f + tl.store(z_fwd_ptr + bh * F * BLOCK_D + f * BLOCK_D + offs_d_full, z) + + if SAVE_FINAL_STATE: + tl.store(final_state_kv_ptr + bh * BLOCK_D * BLOCK_D + offs_dd_tile, M) + if is_lead and not SKIP_Z: + tl.store(final_state_z_ptr + bh * BLOCK_D + offs_d_full, z) + + if DIRECTION != 1: + M = tl.zeros([BLOCK_D, D_TILE], dtype=tl.float32) + if not SKIP_Z: + z = tl.zeros([BLOCK_D], dtype=tl.float32) + + if not COMBINED_HISTORY: + tl.store(M_rev_ptr + bh * F * BLOCK_D * BLOCK_D + (F - 1) * BLOCK_D * BLOCK_D + offs_dd_tile, M) + if is_lead and not SKIP_Z: + tl.store(z_rev_ptr + bh * F * BLOCK_D + (F - 1) * BLOCK_D + offs_d_full, z) + + for f_iter in range(F - 1): + f_src = F - 1 - f_iter + f_dst = f_src - 1 + I_P_kv_f = tl.load(I_P_kv_ptr + bh * F * BLOCK_D * BLOCK_D + f_src * BLOCK_D * BLOCK_D + offs_dd_full) + A_f = tl.load(A_ptr + bh * F * BLOCK_D * BLOCK_D + f_src * BLOCK_D * BLOCK_D + offs_dd_tile) + g_f = tl.load(decay_ptr + bh * F + f_src).to(tl.float32) + + if USE_ACC_FUSION: + I_P_scaled = I_P_kv_f.to(tl.float32) * g_f + M = tl.dot( + I_P_scaled.to(dot_dtype), + M.to(dot_dtype), + acc=A_f.to(tl.float32), + out_dtype=tl.float32, + input_precision=dot_ip, + ) + else: + M_temp = tl.dot(I_P_kv_f.to(dot_dtype), M.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) + M = g_f * M_temp + A_f + + if is_lead and not SKIP_Z: + I_P_z_f = tl.load(I_P_z_ptr + bh * F * BLOCK_D * BLOCK_D + f_src * BLOCK_D * BLOCK_D + offs_dd_full) + B_f = tl.load(B_ptr + bh * F * BLOCK_D + f_src * BLOCK_D + offs_d_full) + z_temp = tl.sum(I_P_z_f * z[None, :], axis=1) + z = g_f * z_temp + B_f + + if COMBINED_HISTORY: + M_addr = M_fwd_ptr + bh * F * BLOCK_D * BLOCK_D + f_dst * BLOCK_D * BLOCK_D + offs_dd_tile + tl.store(M_addr, tl.load(M_addr) + M) + if is_lead and not SKIP_Z: + z_addr = z_fwd_ptr + bh * F * BLOCK_D + f_dst * BLOCK_D + offs_d_full + tl.store(z_addr, tl.load(z_addr) + z) + else: + tl.store(M_rev_ptr + bh * F * BLOCK_D * BLOCK_D + f_dst * BLOCK_D * BLOCK_D + offs_dd_tile, M) + if is_lead and not SKIP_Z: + tl.store(z_rev_ptr + bh * F * BLOCK_D + f_dst * BLOCK_D + offs_d_full, z) + + +_PHASE_B_DTILE_ARCH_CACHE: dict = {} # (dev, dot_prec) -> (d_splits, nw, ns, acc) + + +# Per-arch D-tile optimum from 2026-04-29 sweep (T=11 B=1 P0 IEEE): +# WGMMA-server (A100 sm_80, H100 sm_90): (d=4, nw=32, ns=1, acc=True) +# Blackwell-family (GB200 sm_100, 5090 sm_120, GB10 sm_121, Ada sm_89): +# (d=8, nw=4, ns=1, acc=False) +# Both clusters were tested across 96 configs (4 ds × 4 nw × 3 ns × 2 acc). +def _pick_phase_b_d_splits(BLOCK_D: int, dot_precision: int = 0): + """Returns (d_splits, nw_override, ns_override, acc_override). + + `d_splits=1` → use baseline `_phase_b_kernel` with `_CHUNKWISE_TUNING` config. + `d_splits>1` → use `_phase_b_dtile_kernel` with overrides for nw/ns/acc. + Override via env: PHASE_B_D_SPLITS, PHASE_B_DTILE_NW, PHASE_B_DTILE_NS, + PHASE_B_DTILE_ACC (1=True / 0=False). + """ + import os + + env_d = os.environ.get("PHASE_B_D_SPLITS", None) + if env_d is not None: + d = int(env_d) + if d < 1 or BLOCK_D % d != 0: + return (1, None, None, None) + nw = int(os.environ.get("PHASE_B_DTILE_NW", "0")) or None + ns = int(os.environ.get("PHASE_B_DTILE_NS", "0")) or None + acc_env = os.environ.get("PHASE_B_DTILE_ACC", None) + acc = bool(int(acc_env)) if acc_env is not None else None + return (d, nw, ns, acc) + try: + import torch + + if not torch.cuda.is_available(): + return (1, None, None, None) + dev = torch.cuda.current_device() + cache_key = (dev, dot_precision) + if cache_key not in _PHASE_B_DTILE_ARCH_CACHE: + cap = torch.cuda.get_device_capability(dev) + major, minor = cap[0], cap[1] + if dot_precision == 2: + # IEEE fp32: D-tile dominates baseline on every arch (96-config sweep). + if major == 8 and minor == 0: + cfg = (4, 32, 1, True) # A100 + elif major == 9: + cfg = (4, 32, 1, True) # H100 (Hopper) + elif major == 8 and minor == 9: + cfg = (8, 4, 1, False) # Ada (assume Blackwell-like) + elif major >= 10: + cfg = (8, 4, 1, False) # GB200/B200, 5090, GB10 + else: + cfg = (1, None, None, None) # unknown — baseline + else: + # bf16/TF32: cap-specific dispatch. Multi-arch sweep 2026-05-06 + # (F=11 S=920) determined per-cap whether D-tile beats the + # baseline _phase_b_kernel: + # sm_80 A100: D-tile WIN 1.09× (P1) / 1.02× (P2) — (4,8,2,F). + # sm_90 H100: D-tile WIN ~10% — P1 (4,8,2,F); P2 (8,8,2,F). + # Use (4,8,2,F) for both (P2 within 0.4%). + # sm_100 GB200: D-tile WIN ~12% — (4,8,2,F) both precisions. + # sm_120 5090: D-tile WIN 2.6× (P1) / 1.13× (P2) — (8,8,1,F). + # TF32 baseline OOMs at 102 KB SRAM cap. + # sm_121 GB10: D-tile LOSS 4% — baseline wins. Despite same + # reported SRAM/SM as sm_120, the baseline + # kernel fits all configs up to nw=16 ns=2 on + # sm_121 (Triton/codegen difference between + # consumer-Blackwell variants), so baseline + # saturates the chip without needing D-tile. + if major == 8 and minor == 0: + cfg = (4, 8, 2, False) # A100 + elif major == 9: + cfg = (4, 8, 2, False) # H100 + elif major == 10: + cfg = (4, 8, 2, False) # GB200 / B200 + elif major == 12 and minor == 0: + cfg = (8, 8, 1, False) # 5090 + elif major == 12 and minor == 1: + cfg = (1, None, None, None) # GB10 — baseline wins + else: + cfg = (1, None, None, None) # Ada, unknown + _PHASE_B_DTILE_ARCH_CACHE[cache_key] = cfg + return _PHASE_B_DTILE_ARCH_CACHE[cache_key] + except Exception: + return (1, None, None, None) + + +# ════════════════════════════════════════════════════════════════ +# Phase C — Pass 2 output (per (B, H, F)). Same as v1. +# ════════════════════════════════════════════════════════════════ + + +@triton.jit +def _phase_c_kernel( + qkv_ptr, + stride_b: tl.constexpr, + stride_n: tl.constexpr, + stride_3: tl.constexpr, + stride_h: tl.constexpr, + stride_d: tl.constexpr, + q_inv_rms_ptr, + q_norm_w_ptr, + rope_cos_ptr, + rope_sin_ptr, + M_ptr, + z_ptr, + num_ptr, + den_ptr, + H: tl.constexpr, + F: tl.constexpr, + S: tl.constexpr, + D: tl.constexpr, + NORM_EPS: tl.constexpr, + DOT_PRECISION: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_S: tl.constexpr, + ACCUMULATE: tl.constexpr = False, + SKIP_LAST_F: tl.constexpr = False, + SKIP_RELU: tl.constexpr = False, + NUM_ONLY: tl.constexpr = False, +): + if DOT_PRECISION >= 1: + dot_dtype = tl.float32 + else: + dot_dtype = tl.bfloat16 + dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" + + pid = tl.program_id(0) + pid_b = pid // (H * F) + pid_hf = pid % (H * F) + pid_h = pid_hf // F + pid_f = pid_hf % F + bh = pid_b * H + pid_h + N: tl.constexpr = F * S + + # Reverse-accumulate callers pass SKIP_LAST_F=True: M_rev[F-1] / z_rev[F-1] + # are exactly zero (Phase B initializes the reverse scan with zeros and the + # write loop only fills f 0, Q_normed, 0.0) + Q_pair = tl.where(Q_pair_normed > 0, Q_pair_normed, 0.0) + + rope_ptrs = n_idx[:, None] * D + offs_d[None, :] + Cos = tl.load(rope_cos_ptr + rope_ptrs, mask=mask_sd, other=1.0).to(tl.float32) + Sin = tl.load(rope_sin_ptr + rope_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + Q_rot = Q * Cos + Q_pair * Sin + + num = tl.dot(Q_rot.to(dot_dtype), M_f.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) + if not NUM_ONLY: + den = tl.sum(Q * z_f[None, :], axis=1) + + num_ptrs = num_bh + n_idx[:, None] * (H * D) + offs_d[None, :] + if not NUM_ONLY: + den_ptrs = den_bh + n_idx + if ACCUMULATE: + # Used by reverse-direction Phase C: add this pass onto forward's + # already-written buffer instead of allocating a separate one. + prev_num = tl.load(num_ptrs, mask=mask_sd, other=0.0).to(tl.float32) + num = num + prev_num + if not NUM_ONLY: + prev_den = tl.load(den_ptrs, mask=mask_s, other=0.0).to(tl.float32) + den = den + prev_den + if DOT_PRECISION >= 1: + tl.store(num_ptrs, num, mask=mask_sd) + if not NUM_ONLY: + tl.store(den_ptrs, den, mask=mask_s) + else: + tl.store(num_ptrs, num.to(tl.bfloat16), mask=mask_sd) + if not NUM_ONLY: + tl.store(den_ptrs, den.to(tl.bfloat16), mask=mask_s) + + +def phase_c( + qkv, + q_inv_rms, + q_norm_w, + rope_cos, + rope_sin, + M, + z, + F, + S, + num_warps=None, + num_stages=None, + BLOCK_S=None, + dot_precision=0, + num_out=None, + den_out=None, + accumulate=False, + skip_last_frame=False, + skip_relu: bool = False, + num_only: bool = False, +): + """Phase C Pass-2 output. Optionally accumulates into caller-provided + ``num_out``/``den_out`` buffers (used to fuse reverse-direction output into + forward-direction buffer without allocating a separate one — saves ~45 MB + at B=1 bf16, ~180 MB at B=4). + + ``skip_last_frame=True`` early-returns the f=F-1 programs. Valid for the + reverse-accumulate call only, where M[F-1]/z[F-1] are guaranteed zero. + + ``skip_relu=True`` matches Phase A KV's flag — used by the camera-branch + chunkwise wrapper where Q has already been ReLU'd by cam_prep before + being rotated by UCPE+RoPE; re-applying ReLU on the rotated Q would + clobber legitimate negatives. + + ``num_only=True`` skips the denominator computation and store entirely + (kernel writes only ``num_out``; ``den_out`` is allowed to be None / + unallocated). Used by the camera-branch which has no Z scan. + """ + if num_warps is None or num_stages is None or BLOCK_S is None: + *_, c_w, c_bs, c_s = _get_arch_config(dot_precision, device=qkv.device) + if num_warps is None: + num_warps = c_w + if num_stages is None: + num_stages = c_s + if BLOCK_S is None: + BLOCK_S = c_bs + B, N, three, H, D = qkv.shape + BLOCK_D = triton.next_power_of_2(D) + if num_out is None: + num_out = torch.empty( + B, N, H, D, device=qkv.device, dtype=(torch.float32 if dot_precision >= 1 else torch.bfloat16) + ) + if den_out is None and not num_only: + den_out = torch.empty( + B, H, N, device=qkv.device, dtype=(torch.float32 if dot_precision >= 1 else torch.bfloat16) + ) + elif num_only and den_out is None: + # Pass a 1-element placeholder; kernel guards den loads/stores under NUM_ONLY. + den_out = torch.empty(1, device=qkv.device, dtype=(torch.float32 if dot_precision >= 1 else torch.bfloat16)) + + _phase_c_kernel[(B * H * F,)]( + qkv, + qkv.stride(0), + qkv.stride(1), + qkv.stride(2), + qkv.stride(3), + qkv.stride(4), + q_inv_rms, + q_norm_w, + rope_cos, + rope_sin, + M, + z, + num_out, + den_out, + H=H, + F=F, + S=S, + D=D, + NORM_EPS=1e-5, + DOT_PRECISION=dot_precision, + BLOCK_D=BLOCK_D, + BLOCK_S=BLOCK_S, + ACCUMULATE=1 if accumulate else 0, + SKIP_LAST_F=skip_last_frame, + SKIP_RELU=skip_relu, + NUM_ONLY=num_only, + num_warps=num_warps, + num_stages=num_stages, + ) + return num_out, den_out + + +def fused_bigdn_bidi_chunkwise( + qkv, + q_inv_rms, + k_inv_rms, + q_norm_w, + k_norm_w, + rope_cos, + rope_sin, + beta, + decay, + F, + S, + k_scale=1.0, + eps=1e-6, + norm_eps=1e-5, + dot_precision=0, + init_state_kv=None, + init_state_z=None, + return_final_state=False, +): + """Bidi chunkwise GDN forward, optionally with state-cache for autoregressive + sampling (chunk 0 = full bidi with state save; chunks > 0 seed forward scan + from saved state). Reverse always seeds from zero per upstream convention. + + Pipeline (2026-04-25 restructure): Phase A once → Phase B direction=0 with + combined_history=True (fwd seeded with init_state and saves final state; + rev zero-seeded; rev output summed into fwd buffer in-kernel via read- + add-store so on exit M_hist[f] = M_fwd[f] + M_rev[f]) → Phase C ONCE on + M_hist. Phase C linearity `Q @ (M_fwd + M_rev) = Q @ M_fwd + Q @ M_rev` + makes the in-kernel sum exact. + + Replaces the prior 2× Phase B + 2× Phase C pattern. Saves one Phase C + launch + one Q+RoPE HBM pass and one M-shape buffer per call. + """ + I_P_kv, A, I_P_z, B_z = phase_a( + qkv, + beta, + q_inv_rms, + k_inv_rms, + q_norm_w, + k_norm_w, + rope_cos, + rope_sin, + F=F, + S=S, + k_scale=k_scale, + norm_eps=norm_eps, + dot_precision=dot_precision, + ) + + if return_final_state: + M_hist, z_hist, _, _, final_kv, final_z = phase_b_triton( + I_P_kv, + A, + I_P_z, + B_z, + decay, + F=F, + dot_precision=dot_precision, + direction=0, + init_state_kv=init_state_kv, + init_state_z=init_state_z, + return_final_state=True, + combined_history=True, + ) + else: + M_hist, z_hist, _, _ = phase_b_triton( + I_P_kv, + A, + I_P_z, + B_z, + decay, + F=F, + dot_precision=dot_precision, + direction=0, + init_state_kv=init_state_kv, + init_state_z=init_state_z, + combined_history=True, + ) + num_out, den_out = phase_c( + qkv, + q_inv_rms, + q_norm_w, + rope_cos, + rope_sin, + M_hist, + z_hist, + F=F, + S=S, + dot_precision=dot_precision, + accumulate=False, + ) + del M_hist, z_hist, I_P_kv, A, I_P_z, B_z + + # ── Final divide ── + total_den = den_out.float().permute(0, 2, 1).unsqueeze(-1) # (B, N, H, 1) + out = (num_out.float() / (total_den + eps)).to(qkv.dtype) + del num_out, den_out, total_den + if return_final_state: + B = qkv.shape[0] + H = qkv.shape[3] + D = qkv.shape[4] + BLOCK_D = final_kv.shape[1] + state_kv = final_kv.view(B, H, BLOCK_D, BLOCK_D)[:, :, :D, :D].transpose(-1, -2).contiguous() + state_z = final_z.view(B, H, BLOCK_D)[:, :, :D].unsqueeze(-1).contiguous() + return out, state_kv, state_z + return out + + +def _default_dot_prec(): + """Pull dot_precision from `_resolve_launch_config` (honors PRECISION_OVERRIDE).""" + from .fused_gdn import _resolve_launch_config + + _, dot_prec, _, _ = _resolve_launch_config() + return dot_prec + + +def fused_gdn_func_chunkwise( + qkv, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + beta, + decay, + F, + S, + k_scale, + eps=1e-6, + reverse=False, + dot_precision=None, +): + """Single-direction chunkwise GDN — drop-in for `fused_gdn.fused_gdn_func`. + + Computes only one scan direction (Phase B + Phase C × 1) and returns + `(num, den)` shape-compatible with the upstream function. dot_precision + defaults to whatever `_resolve_launch_config` returns (honors module-level + `PRECISION_OVERRIDE`). + """ + if dot_precision is None: + dot_precision = _default_dot_prec() + direction = 2 if reverse else 1 + I_P_kv, A, I_P_z, B_z = phase_a( + qkv, + beta, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + F=F, + S=S, + k_scale=k_scale, + dot_precision=dot_precision, + ) + M_fwd, z_fwd, M_rev, z_rev = phase_b_triton( + I_P_kv, + A, + I_P_z, + B_z, + decay, + F=F, + dot_precision=dot_precision, + direction=direction, + ) + M_use = M_rev if reverse else M_fwd + z_use = z_rev if reverse else z_fwd + num, den = phase_c( + qkv, q_inv_rms, q_norm_weight, rope_cos, rope_sin, M_use, z_use, F=F, S=S, dot_precision=dot_precision + ) + return num, den + + +def fused_gdn_stateful_chunkwise( + qkv, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + beta, + decay, + F, + S, + k_scale, + eps=1e-6, + reverse=False, + init_state_kv=None, + init_state_z=None, + return_final_state=False, + dot_precision=None, +): + """Single-direction chunkwise GDN with optional state cache — drop-in for + `fused_gdn.fused_gdn_stateful`. Forward direction supports state load/save + (used for autoregressive sampling); reverse direction always runs fresh + (per upstream's bidi state-cache convention). + """ + if dot_precision is None: + dot_precision = _default_dot_prec() + direction = 2 if reverse else 1 + if reverse and (init_state_kv is not None or return_final_state): + raise ValueError( + "fused_gdn_stateful_chunkwise: state cache is forward-only (matching " + "upstream's bidi convention); pass reverse=False or omit state args." + ) + I_P_kv, A, I_P_z, B_z = phase_a( + qkv, + beta, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + F=F, + S=S, + k_scale=k_scale, + dot_precision=dot_precision, + ) + # Pad caller-supplied state from (B,H,D,D)/(B,H,D,1) to (BH, BLOCK_D, BLOCK_D)/(BH, BLOCK_D). + # Needed because the state returned by this function is unpadded (B,H,D,D), + # but phase_b_triton's kernel expects the padded layout. + init_kv_padded, init_z_padded = init_state_kv, init_state_z + if init_state_kv is not None: + B_, H_, D_in, D_out = init_state_kv.shape + BLOCK_D_ = I_P_kv.shape[-1] + if D_in != BLOCK_D_ or D_out != BLOCK_D_: + pad_in = BLOCK_D_ - D_in + pad_out = BLOCK_D_ - D_out + init_kv_padded = torch.nn.functional.pad( + init_state_kv.transpose(-1, -2).reshape(B_ * H_, D_out, D_in), (0, pad_in, 0, pad_out) + ).contiguous() + else: + init_kv_padded = init_state_kv.transpose(-1, -2).reshape(B_ * H_, BLOCK_D_, BLOCK_D_).contiguous() + # z: (B, H, D) or (B, H, D, 1) → (BH, BLOCK_D) + z_ = init_state_z.squeeze(-1) if init_state_z.dim() == 4 else init_state_z + Bz_, Hz_, Dz_ = z_.shape + if Dz_ != BLOCK_D_: + init_z_padded = torch.nn.functional.pad(z_.reshape(Bz_ * Hz_, Dz_), (0, BLOCK_D_ - Dz_)).contiguous() + else: + init_z_padded = z_.reshape(Bz_ * Hz_, Dz_).contiguous() + if return_final_state: + M_fwd, z_fwd, M_rev, z_rev, final_kv, final_z = phase_b_triton( + I_P_kv, + A, + I_P_z, + B_z, + decay, + F=F, + dot_precision=dot_precision, + direction=direction, + init_state_kv=init_kv_padded, + init_state_z=init_z_padded, + return_final_state=True, + ) + else: + M_fwd, z_fwd, M_rev, z_rev = phase_b_triton( + I_P_kv, + A, + I_P_z, + B_z, + decay, + F=F, + dot_precision=dot_precision, + direction=direction, + init_state_kv=init_kv_padded, + init_state_z=init_z_padded, + ) + M_use = M_rev if reverse else M_fwd + z_use = z_rev if reverse else z_fwd + num, den = phase_c( + qkv, q_inv_rms, q_norm_weight, rope_cos, rope_sin, M_use, z_use, F=F, S=S, dot_precision=dot_precision + ) + if return_final_state: + B = qkv.shape[0] + H = qkv.shape[3] + D = qkv.shape[4] + BLOCK_D = final_kv.shape[1] + state_kv = final_kv.view(B, H, BLOCK_D, BLOCK_D)[:, :, :D, :D].transpose(-1, -2).contiguous() + state_z = final_z.view(B, H, BLOCK_D)[:, :, :D].unsqueeze(-1).contiguous() + return num, den, state_kv, state_z + return num, den + + +def fused_bidi_stateful_chunkwise_shared_phase_a( + qkv, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + beta, + decay, + F, + S, + k_scale, + eps=1e-6, + init_state_kv=None, + init_state_z=None, + dot_precision=None, +): + """Bidi state-cached chunkwise GDN with shared Phase A and combined-history + Phase B. Default chunkwise path for ``_fused_statecached_forward``. + + Pipeline (per layer per step): + 1. Phase A once over qkv — K/V/RoPE pre-norm; was previously duplicated + across two streams. + 2. Phase B with direction=0 + combined_history=True — single program does + fwd then rev; fwd writes M_hist; rev read-add-stores into the same + buffer so on exit M_hist[f] = M_fwd[f] + M_rev[f] (same for z). + Forward branch loads init_state and saves final state. + 3. Phase C ONCE on M_hist/z_hist — Phase C is linear in M/z so + `Q @ (M_fwd + M_rev) = Q @ M_fwd + Q @ M_rev`. + + Returns ``(num_combined, den_combined, state_kv, state_z)`` — caller hands + the num/den pair to ``fused_bidi_merge(num, None, den, None, eps, gate)`` + in PRE_SUMMED mode. + + HBM-traffic delta vs the prior 2× Phase C version (per call, B=1 prod): + saved : 1× Phase C Q+RoPE pass (~90 MB) + saved : one (B,N,H,D) num and (B,H,N) den allocation + cost : Phase B rev does read-add of M_hist (~14 MB extra per layer) + net : ~76 MB saved + 1 fewer kernel launch + + Measured speed on GB10 (sm_121) at H=20, S=920, D=112, vs the prior + shared-Phase-A-with-2×-Phase-C path, across production F values: + P0 IEEE fp32 : 1.26-1.42× (F=3,6,11; B=1,2) + P2 bf16+fp32-st : 1.57-1.80× + P3 bf16+bf16-st : 1.63-1.96× + Correctness cos ≥ 0.999997 across all cells, state_kv exact. + """ + if dot_precision is None: + dot_precision = _default_dot_prec() + + I_P_kv, A, I_P_z, B_z = phase_a( + qkv, + beta, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + F=F, + S=S, + k_scale=k_scale, + dot_precision=dot_precision, + ) + + init_kv_padded, init_z_padded = init_state_kv, init_state_z + if init_state_kv is not None: + B_, H_, D_in, D_out = init_state_kv.shape + BLOCK_D_ = I_P_kv.shape[-1] + if D_in != BLOCK_D_ or D_out != BLOCK_D_: + pad_in = BLOCK_D_ - D_in + pad_out = BLOCK_D_ - D_out + init_kv_padded = torch.nn.functional.pad( + init_state_kv.transpose(-1, -2).reshape(B_ * H_, D_out, D_in), (0, pad_in, 0, pad_out) + ).contiguous() + else: + init_kv_padded = init_state_kv.transpose(-1, -2).reshape(B_ * H_, BLOCK_D_, BLOCK_D_).contiguous() + z_ = init_state_z.squeeze(-1) if init_state_z.dim() == 4 else init_state_z + Bz_, Hz_, Dz_ = z_.shape + if Dz_ != BLOCK_D_: + init_z_padded = torch.nn.functional.pad(z_.reshape(Bz_ * Hz_, Dz_), (0, BLOCK_D_ - Dz_)).contiguous() + else: + init_z_padded = z_.reshape(Bz_ * Hz_, Dz_).contiguous() + + # combined_history=True routes the rev contribution into the fwd buffer → + # M_hist[f] = M_fwd[f] + M_rev[f]. M_rev/z_rev outputs are placeholders. + M_hist, z_hist, _, _, final_kv, final_z = phase_b_triton( + I_P_kv, + A, + I_P_z, + B_z, + decay, + F=F, + dot_precision=dot_precision, + direction=0, + init_state_kv=init_kv_padded, + init_state_z=init_z_padded, + return_final_state=True, + combined_history=True, + ) + + num, den = phase_c( + qkv, q_inv_rms, q_norm_weight, rope_cos, rope_sin, M_hist, z_hist, F=F, S=S, dot_precision=dot_precision + ) + + B = qkv.shape[0] + H = qkv.shape[3] + D = qkv.shape[4] + BLOCK_D = final_kv.shape[1] + state_kv = final_kv.view(B, H, BLOCK_D, BLOCK_D)[:, :, :D, :D].transpose(-1, -2).contiguous() + state_z = final_z.view(B, H, BLOCK_D)[:, :, :D].unsqueeze(-1).contiguous() + return num, den, state_kv, state_z + + +def fused_bigdn_stateful_chunkwise( + qkv, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + beta, + decay, + F, + S, + k_scale, + eps=1e-6, + return_final_state=False, + dot_precision=None, +): + """Drop-in replacement for `fused_gdn.fused_bigdn_stateful` using the + chunkwise pipeline. Same signature, same return shape: + output (B, N, H, D), and if return_final_state: + (state_kv, state_z). + dot_precision defaults to whatever `_resolve_launch_config` returns. + """ + if dot_precision is None: + dot_precision = _default_dot_prec() + if return_final_state: + out, state_kv, state_z = fused_bigdn_bidi_chunkwise( + qkv, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + beta, + decay, + F=F, + S=S, + k_scale=k_scale, + eps=eps, + dot_precision=dot_precision, + return_final_state=True, + ) + return out, state_kv, state_z + out = fused_bigdn_bidi_chunkwise( + qkv, + q_inv_rms, + k_inv_rms, + q_norm_weight, + k_norm_weight, + rope_cos, + rope_sin, + beta, + decay, + F=F, + S=S, + k_scale=k_scale, + eps=eps, + dot_precision=dot_precision, + ) + return out + + +# ───────────────────────────────────────────────────────────────────────────── +# Camera-branch wrapper — numerator-only single-path delta-rule scan via +# chunkwise. Drop-in for `diffusion.model.ops.fused_cam_gdn.cam_scan_func`. +# +# Cam math expanded: +# state = state * g # apply decay +# state += K^T @ ((V - K @ state) * β) # delta-rule +# Equivalently: +# state_new = g (I - K^T β K) state_old + K^T β V +# = g (I - P_kv) state_old + A +# This is bit-identical to chunkwise's Phase B M update, so the scan kernel +# is reusable. The only differences from main GDN: +# 1. Q/K/V come pre-prepped (cam_prep_kernel did RMSNorm+ReLU+UCPE+RoPE). +# We disable chunkwise's prep with identity tables (k_inv_rms=1, k_nw=1, +# k_scale=1, rope_cos=1, rope_sin=0) AND skip_relu=True (because cam +# applied ReLU BEFORE UCPE; the post-UCPE values can have legitimate +# negatives that re-applying ReLU would clobber). +# 2. No Z denominator scan; output is num-only (out = Q @ M, no /Z). +# skip_z=True elides Phase A Z; num_only=True elides Phase C den compute. +# ───────────────────────────────────────────────────────────────────────────── +def _cam_identity_tables( + *, + B: int, + N: int, + H: int, + D: int, + device: torch.device, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Cached identity RMS/RoPE tables used by ``cam_scan_chunkwise``.""" + device_index = device.index if device.type == "cuda" else None + key = (device.type, device_index, B, N, H * D, D) + cached = _CAM_IDENTITY_CACHE.get(key) + if cached is not None: + return cached + + ones_inv_rms = torch.ones(B, N, device=device, dtype=torch.float32) + ones_nw = torch.ones(H * D, device=device, dtype=torch.float32) + ones_cos = torch.ones(N, D, device=device, dtype=torch.float32) + zeros_sin = torch.zeros(N, D, device=device, dtype=torch.float32) + cached = (ones_inv_rms, ones_nw, ones_cos, zeros_sin) + _CAM_IDENTITY_CACHE[key] = cached + return cached + + +def cam_scan_chunkwise( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + *, + reverse: bool = False, + init_state: torch.Tensor | None = None, + save_final_state: bool = False, + dot_precision: int | None = None, +): + """Drop-in chunkwise replacement for `cam_scan_func`. + + Args mirror `cam_scan_func` exactly: + q, k, v: ``(B, H, D, N)`` fp32 contiguous (cam-prep'd: RMSNorm+ReLU+UCPE+RoPE) + beta: ``(B, H, F, S)`` fp32 contiguous + decay: ``(B, H, F)`` fp32 contiguous + reverse: bwd flip-and-shift semantics (autograd path); not yet supported. + init_state: optional ``(B*H, BLOCK_D, BLOCK_D)`` fp32 — cross-chunk AR state. + save_final_state: when True, also returns ``(out, final_state)``. + + Returns ``out`` of shape ``(B, H, D, N)`` fp32, or + ``(out, final_state: (B*H, BLOCK_D, BLOCK_D))`` if save_final_state=True. + """ + assert q.shape == k.shape == v.shape, f"q/k/v shape mismatch: {q.shape} {k.shape} {v.shape}" + assert q.is_contiguous() and k.is_contiguous() and v.is_contiguous() + assert beta.is_contiguous() and decay.is_contiguous() + assert q.dtype == torch.float32, f"cam_scan_chunkwise requires fp32 q/k/v (got {q.dtype})" + + if reverse and (init_state is not None or save_final_state): + raise NotImplementedError( + "cam_scan_chunkwise: state passing (init_state / save_final_state) is " + "only supported for the forward direction (reverse=False). The cam " + "branch's anti-causal pass resets per chunk; there is no global " + "cross-prefix state to cache for the reverse direction." + ) + + B, H, D, N = q.shape + F = beta.shape[2] + assert N % F == 0 + S = N // F + assert beta.shape == (B, H, F, S) + assert decay.shape == (B, H, F) + + BLOCK_D = triton.next_power_of_2(D) + + if dot_precision is None: + dot_precision = _default_dot_prec() + + # Repack (B, H, D, N) → (B, N, 3, H, D) for chunkwise's qkv layout. + # Avoid ``stack(...).permute(...).contiguous()`` because that materializes + # two large tensors. Direct packing allocates the destination once. + qkv = torch.empty(B, N, 3, H, D, device=q.device, dtype=q.dtype) + qkv[:, :, 0].copy_(q.permute(0, 3, 1, 2)) + qkv[:, :, 1].copy_(k.permute(0, 3, 1, 2)) + qkv[:, :, 2].copy_(v.permute(0, 3, 1, 2)) + + # Identity prep tables — make chunkwise's RMSNorm + RoPE no-ops. + ones_inv_rms, ones_nw, ones_cos, zeros_sin = _cam_identity_tables(B=B, N=N, H=H, D=D, device=q.device) + + # Phase A (skip_relu=True for cam-prep'd K; skip_z=True since cam has no Z scan). + # k_scale=1.0 because cam_prep already applied K-scale. + I_P_kv, A_, I_P_z, B_z = phase_a( + qkv, + beta, + ones_inv_rms, + ones_inv_rms, + ones_nw, + ones_nw, + ones_cos, + zeros_sin, + F=F, + S=S, + k_scale=1.0, + norm_eps=1e-5, + dot_precision=dot_precision, + skip_relu=True, + skip_z=True, + ) + + # Phase B (forward direction only; cam supports init_state on fwd, save_final + # on fwd; no rev). Pads (B*H, D, D) ↔ (B*H, BLOCK_D, BLOCK_D) inline. + init_kv_padded = None + init_z_padded = None + if init_state is not None: + if init_state.shape != (B * H, BLOCK_D, BLOCK_D): + raise ValueError( + f"cam_scan_chunkwise: init_state shape {tuple(init_state.shape)} " + f"!= expected (B*H, BLOCK_D, BLOCK_D) = {(B * H, BLOCK_D, BLOCK_D)}" + ) + if init_state.dtype != torch.float32: + raise ValueError(f"cam_scan_chunkwise: init_state must be fp32 (got {init_state.dtype}).") + if not init_state.is_contiguous(): + raise ValueError("cam_scan_chunkwise: init_state must be contiguous.") + # Cam stores state as M[K_feat, V_feat]. Chunkwise's Phase B kernel reads + # state with offs_dd = i*BLOCK_D + j where i is the fwd loop's M row. + # Storage layout matches cam's (row-major (D_K, D_V)), so a direct cast + # to fp32 contiguous is enough — no transpose needed. + init_kv_padded = init_state.to(torch.float32).contiguous() + # No Z state in cam — pass zeros to satisfy phase_b_triton. + init_z_padded = torch.zeros(B * H, BLOCK_D, device=q.device, dtype=torch.float32) + + direction = 2 if reverse else 1 + if save_final_state: + M_fwd, z_fwd_out, M_rev, z_rev_out, final_kv, _final_z = phase_b_triton( + I_P_kv, + A_, + I_P_z, + B_z, + decay, + F=F, + dot_precision=dot_precision, + direction=direction, + init_state_kv=init_kv_padded, + init_state_z=init_z_padded, + return_final_state=True, + skip_z=True, + ) + else: + M_fwd, z_fwd_out, M_rev, z_rev_out = phase_b_triton( + I_P_kv, + A_, + I_P_z, + B_z, + decay, + F=F, + dot_precision=dot_precision, + direction=direction, + init_state_kv=init_kv_padded, + init_state_z=init_z_padded, + skip_z=True, + ) + + # For reverse (flip-and-shift bwd), Phase B's reverse mode produces M_rev + # such that M_rev[F-1] = 0 and M_rev[t] = state computed from K/V at frames + # {F-1, F-2, ..., t+1} — exactly cam's REVERSE=1 semantics. + M_use = M_rev if reverse else M_fwd + z_use = z_rev_out if reverse else z_fwd_out + + # Phase C — num-only (NUM_ONLY=True skips den compute + store). + # z is unused with NUM_ONLY but still required by the kernel signature. + num_out, _ = phase_c( + qkv, + ones_inv_rms, + ones_nw, + ones_cos, + zeros_sin, + M_use, + z_use, + F=F, + S=S, + dot_precision=dot_precision, + skip_relu=True, + num_only=True, + ) + + # Convert chunkwise output (B, N, H, D) → cam's (B, H, D, N) layout, fp32. + out = num_out.permute(0, 2, 3, 1).contiguous().to(torch.float32) + + if save_final_state: + return out, final_kv # final_kv already (B*H, BLOCK_D, BLOCK_D) fp32 + return out + + +def cam_scan_bidi_chunkwise( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + decay: torch.Tensor, + *, + dot_precision: int | None = None, +) -> torch.Tensor: + """Bidirectional camera scan using shared chunkwise phases. + + This is equivalent to ``cam_scan_chunkwise(..., reverse=False) + + cam_scan_chunkwise(..., reverse=True)`` for full bidirectional attention, + but it packs QKV once, runs Phase A once, combines forward/reverse histories + inside Phase B, and runs Phase C once on the summed state. + """ + assert q.shape == k.shape == v.shape, f"q/k/v shape mismatch: {q.shape} {k.shape} {v.shape}" + assert q.is_contiguous() and k.is_contiguous() and v.is_contiguous() + assert beta.is_contiguous() and decay.is_contiguous() + assert q.dtype == torch.float32, f"cam_scan_bidi_chunkwise requires fp32 q/k/v (got {q.dtype})" + + B, H, D, N = q.shape + F = beta.shape[2] + assert N % F == 0 + S = N // F + assert beta.shape == (B, H, F, S) + assert decay.shape == (B, H, F) + + if dot_precision is None: + dot_precision = _default_dot_prec() + + qkv = torch.empty(B, N, 3, H, D, device=q.device, dtype=q.dtype) + qkv[:, :, 0].copy_(q.permute(0, 3, 1, 2)) + qkv[:, :, 1].copy_(k.permute(0, 3, 1, 2)) + qkv[:, :, 2].copy_(v.permute(0, 3, 1, 2)) + + ones_inv_rms, ones_nw, ones_cos, zeros_sin = _cam_identity_tables(B=B, N=N, H=H, D=D, device=q.device) + I_P_kv, A_, I_P_z, B_z = phase_a( + qkv, + beta, + ones_inv_rms, + ones_inv_rms, + ones_nw, + ones_nw, + ones_cos, + zeros_sin, + F=F, + S=S, + k_scale=1.0, + norm_eps=1e-5, + dot_precision=dot_precision, + skip_relu=True, + skip_z=True, + ) + M_hist, z_hist, _, _ = phase_b_triton( + I_P_kv, + A_, + I_P_z, + B_z, + decay, + F=F, + dot_precision=dot_precision, + direction=0, + combined_history=True, + skip_z=True, + ) + num_out, _ = phase_c( + qkv, + ones_inv_rms, + ones_nw, + ones_cos, + zeros_sin, + M_hist, + z_hist, + F=F, + S=S, + dot_precision=dot_precision, + skip_relu=True, + num_only=True, + ) + return num_out.permute(0, 2, 3, 1).contiguous().to(torch.float32) + + +def cam_scan_pair_chunkwise( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + beta_fwd: torch.Tensor, + decay_fwd: torch.Tensor, + beta_rev: torch.Tensor, + decay_rev: torch.Tensor, + *, + dot_precision: int | None = None, +) -> torch.Tensor: + """Sum a forward camera scan and a separately-gated reverse scan. + + Chunk-causal camera attention needs the reverse branch to use boundary-masked + gates while the forward branch uses the original gates. This wrapper keeps + that exact behavior but shares QKV packing, identity tables, and the final + output layout conversion across the two scans. + """ + assert q.shape == k.shape == v.shape, f"q/k/v shape mismatch: {q.shape} {k.shape} {v.shape}" + assert q.is_contiguous() and k.is_contiguous() and v.is_contiguous() + assert beta_fwd.is_contiguous() and decay_fwd.is_contiguous() + assert beta_rev.is_contiguous() and decay_rev.is_contiguous() + assert q.dtype == torch.float32, f"cam_scan_pair_chunkwise requires fp32 q/k/v (got {q.dtype})" + + B, H, D, N = q.shape + F = beta_fwd.shape[2] + assert N % F == 0 + S = N // F + assert beta_fwd.shape == beta_rev.shape == (B, H, F, S) + assert decay_fwd.shape == decay_rev.shape == (B, H, F) + + if dot_precision is None: + dot_precision = _default_dot_prec() + + qkv = torch.empty(B, N, 3, H, D, device=q.device, dtype=q.dtype) + qkv[:, :, 0].copy_(q.permute(0, 3, 1, 2)) + qkv[:, :, 1].copy_(k.permute(0, 3, 1, 2)) + qkv[:, :, 2].copy_(v.permute(0, 3, 1, 2)) + + ones_inv_rms, ones_nw, ones_cos, zeros_sin = _cam_identity_tables(B=B, N=N, H=H, D=D, device=q.device) + + I_P_kv, A_, I_P_z, B_z = phase_a( + qkv, + beta_fwd, + ones_inv_rms, + ones_inv_rms, + ones_nw, + ones_nw, + ones_cos, + zeros_sin, + F=F, + S=S, + k_scale=1.0, + norm_eps=1e-5, + dot_precision=dot_precision, + skip_relu=True, + skip_z=True, + ) + M_fwd, z_fwd, _, _ = phase_b_triton( + I_P_kv, + A_, + I_P_z, + B_z, + decay_fwd, + F=F, + dot_precision=dot_precision, + direction=1, + skip_z=True, + ) + num_out, _ = phase_c( + qkv, + ones_inv_rms, + ones_nw, + ones_cos, + zeros_sin, + M_fwd, + z_fwd, + F=F, + S=S, + dot_precision=dot_precision, + skip_relu=True, + num_only=True, + ) + del I_P_kv, A_, I_P_z, B_z, M_fwd, z_fwd + + I_P_kv, A_, I_P_z, B_z = phase_a( + qkv, + beta_rev, + ones_inv_rms, + ones_inv_rms, + ones_nw, + ones_nw, + ones_cos, + zeros_sin, + F=F, + S=S, + k_scale=1.0, + norm_eps=1e-5, + dot_precision=dot_precision, + skip_relu=True, + skip_z=True, + ) + _, _, M_rev, z_rev = phase_b_triton( + I_P_kv, + A_, + I_P_z, + B_z, + decay_rev, + F=F, + dot_precision=dot_precision, + direction=2, + skip_z=True, + ) + phase_c( + qkv, + ones_inv_rms, + ones_nw, + ones_cos, + zeros_sin, + M_rev, + z_rev, + F=F, + S=S, + dot_precision=dot_precision, + num_out=num_out, + accumulate=True, + skip_relu=True, + num_only=True, + ) + return num_out.permute(0, 2, 3, 1).contiguous().to(torch.float32) diff --git a/integrations/sana/sana_wm/refiner.py b/integrations/sana/sana_wm/refiner.py index 0d7c61107..8a1246281 100644 --- a/integrations/sana/sana_wm/refiner.py +++ b/integrations/sana/sana_wm/refiner.py @@ -48,6 +48,70 @@ ) +def _move_tensor_attr(module: nn.Module, name: str, device: torch.device | str) -> None: + tensor = getattr(module, name, None) + if isinstance(tensor, nn.Parameter): + setattr(module, name, nn.Parameter(tensor.to(device), requires_grad=tensor.requires_grad)) + elif isinstance(tensor, Tensor): + setattr(module, name, tensor.to(device)) + + +def _offload_video_unused_audio_modules( + transformer: nn.Module, + device: torch.device | str = "cpu", +) -> None: + for name in ( + "audio_proj_in", + "audio_caption_projection", + "audio_time_embed", + "av_cross_attn_video_scale_shift", + "av_cross_attn_audio_scale_shift", + "av_cross_attn_video_a2v_gate", + "av_cross_attn_audio_v2a_gate", + "audio_rope", + "cross_attn_rope", + "cross_attn_audio_rope", + "audio_norm_out", + "audio_proj_out", + ): + child = getattr(transformer, name, None) + if isinstance(child, nn.Module): + child.to(device) + for block in getattr(transformer, "transformer_blocks", ()): + for name in ( + "audio_norm1", + "audio_attn1", + "audio_norm2", + "audio_attn2", + "audio_to_video_norm", + "audio_to_video_attn", + "video_to_audio_norm", + "video_to_audio_attn", + "audio_norm3", + "audio_ff", + ): + child = getattr(block, name, None) + if isinstance(child, nn.Module): + child.to(device) + + +def _move_ltx2_video_modules_to( + transformer: nn.Module, + device: torch.device | str, +) -> None: + for name in ("proj_in", "caption_projection", "time_embed", "rope", "norm_out", "proj_out"): + child = getattr(transformer, name, None) + if isinstance(child, nn.Module): + child.to(device) + _move_tensor_attr(transformer, "scale_shift_table", device) + for block in getattr(transformer, "transformer_blocks", ()): + _move_tensor_attr(block, "scale_shift_table", device) + for name in ("norm1", "attn1", "norm2", "attn2", "norm3", "ff"): + child = getattr(block, name, None) + if isinstance(child, nn.Module): + child.to(device) + + class SanaWMLTX2Refiner(nn.Module): """Run the SANA-WM LTX-2 latent refiner without importing a Sana checkout.""" @@ -61,6 +125,7 @@ def __init__( precision: Precision = "bf16", quant_backend: QuantBackend = "torch", text_max_sequence_length: int = 1024, + cache_text_encoder: bool = False, ) -> None: super().__init__() self.refiner_root = Path(refiner_root) @@ -70,6 +135,7 @@ def __init__( self.precision = precision self.quant_backend = quant_backend self.text_max_sequence_length = int(text_max_sequence_length) + self.cache_text_encoder = bool(cache_text_encoder) self._quantized = False self._text_encoder_built = False self.transformer, self.connectors = self._load_diffusers_components() @@ -94,7 +160,9 @@ def refine_latents( ) prompt_embeds, prompt_attention_mask = self._encode_prompt(prompt) - self.transformer.to(self.device) + + _move_ltx2_video_modules_to(self.transformer, self.device) + _offload_video_unused_audio_modules(self.transformer, "cpu") self.transformer.eval() self._prepare_quantization() @@ -205,14 +273,11 @@ def _prepare_quantization(self) -> None: def _ensure_text_encoder(self) -> None: """Load the Gemma tokenizer + encoder once and cache them in CPU RAM. - The encoder is ~20 GB; reloading it from disk on every refine call - dominated repeated-use latency, so the built module is kept cached and - a reused refiner pays the load once. It stays on the CPU between calls - and is moved to the GPU only for the encode forward (see - :meth:`_encode_prompt`) so it never inflates peak memory during the - denoise/decode phases. When ``offload_refiner`` is set the whole - refiner (and this encoder) is released between runs by - ``release_runtime``. + The encoder is ~20 GB. By default it is released after the one-shot + prompt encode to match upstream's single-generation path and avoid a + large GPU-to-CPU copy. Set ``cache_text_encoder=True`` for repeated + pipeline reuse, where paying the copy once can beat reloading Gemma on + every call. """ if self._text_encoder_built: return @@ -255,6 +320,7 @@ def _encode_prompt(self, prompt: str) -> tuple[Tensor, Tensor]: # Pull the cached encoder onto the GPU only for the forward, then park # it back on the CPU so it is absent during denoise/decode. self.text_encoder.to(self.device) + text_backbone: nn.Module | None = None try: text_backbone = getattr(self.text_encoder, "model", self.text_encoder) outputs = text_backbone( @@ -266,9 +332,17 @@ def _encode_prompt(self, prompt: str) -> tuple[Tensor, Tensor]: sequence_lengths = attention_mask.sum(dim=-1) del outputs finally: - self.text_encoder.to("cpu") - if torch.cuda.is_available(): - torch.cuda.empty_cache() + release_text_encoder = not self.cache_text_encoder + if release_text_encoder: + del text_backbone + del self.text_encoder + self._text_encoder_built = False + if torch.cuda.is_available(): + torch.cuda.empty_cache() + else: + self.text_encoder.to("cpu") + if torch.cuda.is_available(): + torch.cuda.empty_cache() prompt_embeds = _pack_text_embeds( hidden_states, diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index bce53ea96..ec7c73161 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -294,23 +294,24 @@ def run(self) -> None: "sink_size": cfg.sink_size, } ) - decoded = pipeline.generate( - 0, - cache, - input=SanaWMI2VConditioningRequest( - image=image, - prompt=prompt, - poses_c2w=c2w, - intrinsics_vec4=intrinsics_vec4, - num_frames=num_frames, - fps=cfg.fps, - steps=cfg.step, - cfg_scale=cfg.cfg_scale, - flow_shift=cfg.flow_shift, - seed=cfg.seed, - negative_prompt=cfg.negative_prompt, - ), - ) + with torch.inference_mode(): + decoded = pipeline.generate( + 0, + cache, + input=SanaWMI2VConditioningRequest( + image=image, + prompt=prompt, + poses_c2w=c2w, + intrinsics_vec4=intrinsics_vec4, + num_frames=num_frames, + fps=cfg.fps, + steps=cfg.step, + cfg_scale=cfg.cfg_scale, + flow_shift=cfg.flow_shift, + seed=cfg.seed, + negative_prompt=cfg.negative_prompt, + ), + ) pipeline.finalize(0, cache) if not isinstance(decoded, SanaWMDecodedVideo): raise TypeError( diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index 522b2b44c..c376a14c5 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -24,12 +24,33 @@ from collections.abc import Callable from dataclasses import dataclass import math +import os import torch import torch.nn as nn import torch.nn.functional as F from torch import Tensor +try: + from .ops.fused_cam_gdn import ( + _prepare_ucpe_rope_tables as _fused_prepare_ucpe_rope_tables, + cam_prep_func as _fused_cam_prep_func, + ) + from .ops.fused_gdn import ( + fused_bigdn_func as _fused_bigdn_func, + fused_qk_inv_rms as _fused_qk_inv_rms, + prepare_rope_tables as _fused_prepare_rope_tables, + ) + from .ops.fused_gdn_chunkwise import ( + cam_scan_bidi_chunkwise as _fused_cam_scan_bidi_chunkwise, + ) +except ImportError: + _fused_bigdn_func = None + _fused_cam_prep_func = None + _fused_cam_scan_bidi_chunkwise = None + _fused_prepare_rope_tables = None + _fused_prepare_ucpe_rope_tables = None + @dataclass(frozen=True) class SanaWMStage1Spec: @@ -50,6 +71,8 @@ class SanaWMStage1Spec: plucker_channels: int = 48 raymap_channels: int = 3 softmax_every_n: int = 4 + chunk_size: int | None = None + chunk_split_strategy: str = "first_chunk_plus_one" @property def mlp_inner_size(self) -> int: @@ -70,6 +93,17 @@ def block_uses_gdn(self, index: int) -> bool: """Architecture spec for the public SANA-WM bidirectional Stage-1 checkpoint.""" +@dataclass(frozen=True) +class _CameraProjectionCache: + raymats: Tensor + proj: Tensor + proj_q: Tensor + proj_kv: Tensor + rope_cam: Tensor | None + rope_cos: Tensor + rope_sin: Tensor + + class RMSNorm(nn.Module): """RMSNorm parameter container matching the SANA checkpoint schema.""" @@ -242,10 +276,11 @@ def forward(self, x: Tensor, *, frames: int, height: int, width: int) -> Tensor: """Run spatial GLU plus temporal aggregation.""" batch, tokens, channels = x.shape x_2d = x.reshape(batch * frames, height, width, channels).permute(0, 3, 1, 2) - x_2d = F.silu(self.inverted_conv(x_2d)) + x_2d = F.silu(self.inverted_conv(x_2d), inplace=True) x_2d = self.depth_conv(x_2d) value, gate = x_2d.chunk(2, dim=1) - x_2d = self.point_conv(value * F.silu(gate)) + gate = F.silu(gate, inplace=not torch.is_grad_enabled()) + x_2d = self.point_conv(value * gate) x_time = x_2d.view(batch, frames, channels, height * width).permute(0, 2, 1, 3) x_time = x_time + self.t_conv(x_time) @@ -360,6 +395,7 @@ def forward( HW: tuple[int, int, int] | None = None, rotary_emb: Tensor | None = None, camera_conditions: Tensor | None = None, + camera_cache: _CameraProjectionCache | None = None, apply_output_gate: bool = True, **kwargs: object, ) -> Tensor: @@ -372,7 +408,9 @@ def forward( f"channels={channels} != heads*dim={self.heads * self.dim}" ) - precomputed_gates = self._compute_frame_gates(x, HW) + precomputed_gates = ( + self._compute_frame_gates(x, HW) if self.use_gdn_convs else None + ) if self.use_gdn_convs: main_raw = self._forward_gdn_main( x, @@ -385,6 +423,9 @@ def forward( x, HW=HW, rotary_emb=rotary_emb, + chunk_size=kwargs.get("chunk_size"), + chunk_split_strategy=str(kwargs.get("chunk_split_strategy", "uniform")), + chunk_index=kwargs.get("chunk_index"), ) cam_contrib: Tensor | int = 0 @@ -395,6 +436,7 @@ def forward( HW=HW, rotary_emb=rotary_emb, camera_conditions=camera_conditions, + camera_cache=camera_cache, precomputed_gates=precomputed_gates, ) else: @@ -403,12 +445,21 @@ def forward( HW=HW, rotary_emb=rotary_emb, camera_conditions=camera_conditions, + camera_cache=camera_cache, + chunk_size=kwargs.get("chunk_size"), + chunk_split_strategy=str(kwargs.get("chunk_split_strategy", "uniform")), + chunk_index=kwargs.get("chunk_index"), ) cam_contrib = self.out_proj_cam(cam_raw) combined = main_raw + cam_contrib if apply_output_gate: - combined = _apply_output_gate(combined, x, self.output_gate) + combined = _apply_output_gate( + combined, + x, + self.output_gate.weight, + self.output_gate.bias, + ) combined = self.proj(combined.to(dtype=self.proj.weight.dtype)) del kwargs return combined @@ -421,6 +472,14 @@ def _forward_gdn_main( rotary_emb: Tensor | None, precomputed_gates: tuple[Tensor, Tensor], ) -> Tensor: + if _use_fused_gdn(x): + return self._forward_fused_gdn_main( + x, + HW=HW, + rotary_emb=rotary_emb, + precomputed_gates=precomputed_gates, + ) + batch, tokens, channels = x.shape qkv = self.qkv(x).reshape(batch, tokens, 3, self.heads, self.dim) if hasattr(self, "conv_k"): @@ -460,14 +519,67 @@ def _forward_gdn_main( ) return out.reshape(batch, tokens, channels).to(dtype=x.dtype) + def _forward_fused_gdn_main( + self, + x: Tensor, + *, + HW: tuple[int, int, int], + rotary_emb: Tensor | None, + precomputed_gates: tuple[Tensor, Tensor], + ) -> Tensor: + if ( + _fused_bigdn_func is None + or _fused_qk_inv_rms is None + or _fused_prepare_rope_tables is None + ): + raise RuntimeError("fused SANA-WM GDN kernels are not available.") + + batch, tokens, channels = x.shape + frames, height, width = HW + spatial = height * width + if tokens != frames * spatial: + raise ValueError(f"tokens={tokens} != T*H*W={frames * spatial}") + qkv = self.qkv(x).reshape(batch, tokens, 3, self.heads, self.dim).contiguous() + if hasattr(self, "conv_k"): + k_raw = qkv[:, :, 1].contiguous().reshape(batch, tokens, channels) + k_conv = _apply_bidirectional_temporal_conv(k_raw, self.conv_k, HW) + qkv[:, :, 1].copy_(k_conv.reshape(batch, tokens, self.heads, self.dim)) + + beta, decay = precomputed_gates + q_inv_rms, k_inv_rms = _fused_qk_inv_rms(qkv, eps=self.q_norm.eps) + rope_cos, rope_sin = _fused_prepare_rope_tables( + rotary_emb, + tokens, + self.dim, + x.device, + ) + out = _fused_bigdn_func( + qkv, + q_inv_rms, + k_inv_rms, + self.q_norm.weight.float().contiguous(), + self.k_norm.weight.float().contiguous(), + rope_cos, + rope_sin, + beta.contiguous(), + decay.contiguous(), + F=frames, + S=spatial, + k_scale=_gdn_key_scale(self.dim, HW), + eps=self.eps, + ) + return out.reshape(batch, tokens, channels).to(dtype=x.dtype) + def _forward_softmax_main( self, x: Tensor, *, HW: tuple[int, int, int], rotary_emb: Tensor | None, + chunk_size: int | None, + chunk_split_strategy: str, + chunk_index: list[int] | None, ) -> Tensor: - del HW batch, tokens, channels = x.shape qkv = self.qkv(x).reshape(batch, tokens, 3, self.heads, self.dim) q, k, v = qkv.unbind(dim=2) @@ -485,12 +597,14 @@ def _forward_softmax_main( ) q = _apply_complex_rope(q, rotary_emb) k = _apply_complex_rope(k, rotary_emb) - out = F.scaled_dot_product_attention( + out = _scaled_dot_product_attention( q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), - dropout_p=0.0, - is_causal=False, + HW=HW, + chunk_size=chunk_size, + chunk_split_strategy=chunk_split_strategy, + chunk_index=chunk_index, ) return out.transpose(1, 2).reshape(batch, tokens, channels).to(dtype=x.dtype) @@ -501,8 +615,19 @@ def _forward_gdn_camera( HW: tuple[int, int, int], rotary_emb: Tensor | None, camera_conditions: Tensor, + camera_cache: _CameraProjectionCache | None, precomputed_gates: tuple[Tensor, Tensor], ) -> Tensor: + if _use_fused_gdn(x): + return self._forward_fused_gdn_camera( + x, + HW=HW, + rotary_emb=rotary_emb, + camera_conditions=camera_conditions, + camera_cache=camera_cache, + precomputed_gates=precomputed_gates, + ) + q_cam, k_cam, v_cam = _camera_qkv(self, x, HW) q_trans, k_trans, v_trans, inflation_sq, output_projector = _prepare_ucpe_qkv( q_cam, @@ -511,6 +636,7 @@ def _forward_gdn_camera( camera_conditions=camera_conditions, HW=HW, rotary_emb=rotary_emb, + camera_cache=camera_cache, q_norm_weight=self.q_norm_cam.weight, k_norm_weight=self.k_norm_cam.weight, norm_eps=self.q_norm_cam.eps, @@ -534,6 +660,76 @@ def _forward_gdn_camera( out = output_projector(out) return out.reshape(x.shape[0], x.shape[1], -1).to(dtype=x.dtype) + def _forward_fused_gdn_camera( + self, + x: Tensor, + *, + HW: tuple[int, int, int], + rotary_emb: Tensor | None, + camera_conditions: Tensor, + camera_cache: _CameraProjectionCache | None, + precomputed_gates: tuple[Tensor, Tensor], + ) -> Tensor: + if ( + _fused_cam_prep_func is None + or _fused_cam_scan_bidi_chunkwise is None + or _fused_prepare_ucpe_rope_tables is None + ): + raise RuntimeError("fused SANA-WM camera GDN kernels are not available.") + + q_raw, k_raw, v_raw = _camera_qkv(self, x, HW) + batch, tokens, heads, dim = q_raw.shape + frames, height, width = HW + spatial = height * width + if camera_cache is None: + camera_cache = _prepare_camera_projection_cache( + camera_conditions, + HW=HW, + rotary_emb=rotary_emb, + head_dim=dim, + ) + if camera_cache.rope_cam is None: + rope_cos = camera_cache.rope_cos + rope_sin = camera_cache.rope_sin + else: + rope_cos = camera_cache.rope_cos + rope_sin = camera_cache.rope_sin + + q_trans, k_trans, v_trans, inflation_sq = _fused_cam_prep_func( + q_raw.contiguous(), + k_raw.contiguous(), + v_raw.contiguous(), + q_norm_weight=self.q_norm_cam.weight.float().contiguous(), + k_norm_weight=self.k_norm_cam.weight.float().contiguous(), + proj_q=camera_cache.proj_q, + proj_kv=camera_cache.proj_kv, + rope_cos=rope_cos, + rope_sin=rope_sin, + k_scale=_gdn_key_scale(dim, HW), + norm_eps=self.q_norm_cam.eps, + ) + + beta, decay = precomputed_gates + frame_inflation = inflation_sq.reshape(batch, heads, frames, spatial).mean( + dim=-1, + ) + beta = beta / frame_inflation.unsqueeze(-1).clamp_min(1.0) + out = _fused_cam_scan_bidi_chunkwise( + q_trans.float().contiguous(), + k_trans.float().contiguous(), + v_trans.float().contiguous(), + beta.float().contiguous(), + decay.float().contiguous(), + ) + out = out.permute(0, 3, 1, 2).contiguous() + out = _ucpe_transform_apply( + out.to(dtype=x.dtype), + camera_cache.proj, + camera_cache.rope_cam, + inverse_rope=True, + ) + return out.reshape(batch, tokens, heads * dim).to(dtype=x.dtype) + def _forward_softmax_camera( self, x: Tensor, @@ -541,6 +737,10 @@ def _forward_softmax_camera( HW: tuple[int, int, int], rotary_emb: Tensor | None, camera_conditions: Tensor, + camera_cache: _CameraProjectionCache | None, + chunk_size: int | None, + chunk_split_strategy: str, + chunk_index: list[int] | None, ) -> Tensor: q_cam, k_cam, v_cam = _camera_qkv(self, x, HW) q_trans, k_trans, v_trans, output_projector = _prepare_ucpe_qkv_softmax( @@ -550,16 +750,19 @@ def _forward_softmax_camera( camera_conditions=camera_conditions, HW=HW, rotary_emb=rotary_emb, + camera_cache=camera_cache, q_norm_weight=self.q_norm_cam.weight, k_norm_weight=self.k_norm_cam.weight, norm_eps=self.q_norm_cam.eps, ) - out = F.scaled_dot_product_attention( + out = _scaled_dot_product_attention( q_trans.transpose(1, 2), k_trans.transpose(1, 2), v_trans.transpose(1, 2), - dropout_p=0.0, - is_causal=False, + HW=HW, + chunk_size=chunk_size, + chunk_split_strategy=chunk_split_strategy, + chunk_index=chunk_index, ) return output_projector(out.transpose(1, 2)).reshape( x.shape[0], @@ -577,13 +780,18 @@ def _compute_frame_gates( spatial = height * width if tokens != frames * spatial: raise ValueError(f"tokens={tokens} != T*H*W={frames * spatial}") - beta = self.beta_proj(x).sigmoid() - beta = beta.reshape(batch, frames, spatial, self.heads).permute(0, 3, 1, 2) - x_frame = x.reshape(batch, frames, spatial, channels).mean(dim=2) - gate = self.gate_proj(x_frame).float() - dt = self.dt_bias.float().view(1, 1, -1) - a_val = self.A_log.float().exp().view(1, 1, -1) - decay = (-a_val * F.softplus(gate + dt)).exp().transpose(1, 2) + beta, decay = _compute_frame_gates( + x, + frames, + spatial, + self.heads, + self.beta_proj.weight, + self.beta_proj.bias, + self.gate_proj.weight, + self.gate_proj.bias, + self.dt_bias, + self.A_log, + ) return beta.float(), decay.float() @@ -627,7 +835,8 @@ def forward( attn_mask = None if mask is not None: attn_mask = (1 - mask.to(dtype=q.dtype)) * -10000.0 - attn_mask = attn_mask[:, None, None, :] + if attn_mask.ndim == 2: + attn_mask = attn_mask[:, None, None].repeat(1, self.num_heads, 1, 1) out = F.scaled_dot_product_attention( q, k, @@ -748,6 +957,21 @@ def forward( x.device, ) camera_conditions = kwargs.get("camera_conditions") + camera_conditions = ( + camera_conditions.to(device=x.device, dtype=x.dtype) + if isinstance(camera_conditions, Tensor) + else None + ) + camera_cache = ( + _prepare_camera_projection_cache( + camera_conditions, + HW=(frames, height, width), + rotary_emb=rotary_emb, + head_dim=self.spec.head_dim, + ) + if camera_conditions is not None + else None + ) block_kwargs = { key: value for key, value in kwargs.items() if key != "camera_conditions" } @@ -759,7 +983,12 @@ def forward( timestep_embed = self.t_embedder(timestep.flatten()) timestep_embed = timestep_embed.unflatten(0, timestep.shape) - block_t = self.t_block(timestep_embed).reshape(batch, 1, frames, 6 * self.spec.hidden_size) + block_t = self.t_block(timestep_embed).reshape( + batch, + 1, + frames, + 6 * self.spec.hidden_size, + ) for block in self.blocks: x = block( @@ -773,11 +1002,10 @@ def forward( plucker_emb=plucker_emb, HW=(frames, height, width), rotary_emb=rotary_emb, - camera_conditions=( - camera_conditions.to(device=x.device, dtype=x.dtype) - if isinstance(camera_conditions, Tensor) - else None - ), + camera_conditions=camera_conditions, + camera_cache=camera_cache, + chunk_size=self.spec.chunk_size, + chunk_split_strategy=self.spec.chunk_split_strategy, **block_kwargs, ) @@ -807,11 +1035,224 @@ def _gdn_key_scale(head_dim: int, HW: tuple[int, int, int]) -> float: return (head_dim**-0.5) * ((HW[1] * HW[2]) ** -0.5) -def _apply_output_gate(out: Tensor, gate_x: Tensor, gate: nn.Module) -> Tensor: - gate_values = F.silu(gate(gate_x).float()) +def _sdpa_needs_head_pad(head_dim: int) -> bool: + return head_dim not in (32, 64, 128, 256) and head_dim < 256 + + +def _scaled_dot_product_attention( + xq: Tensor, + xk: Tensor, + xv: Tensor, + *, + HW: tuple[int, int, int] | None = None, + chunk_size: int | None = None, + chunk_split_strategy: str = "uniform", + chunk_index: list[int] | None = None, +) -> Tensor: + if HW is not None and chunk_size is not None and chunk_size < HW[0]: + frames, height, width = HW + spatial = height * width + boundaries = _normalize_chunk_index( + chunk_index, + frames, + chunk_size, + chunk_split_strategy, + ) + out_chunks = [] + for start, end in zip(boundaries[:-1], boundaries[1:]): + q_chunk = xq[:, :, start * spatial : end * spatial] + out_chunks.append( + _scaled_dot_product_attention_full( + q_chunk, + xk[:, :, : end * spatial], + xv[:, :, : end * spatial], + ), + ) + return torch.cat(out_chunks, dim=2) + return _scaled_dot_product_attention_full(xq, xk, xv) + + +def _scaled_dot_product_attention_full(xq: Tensor, xk: Tensor, xv: Tensor) -> Tensor: + head_dim = xq.shape[-1] + if not _sdpa_needs_head_pad(head_dim): + return F.scaled_dot_product_attention( + xq, + xk, + xv, + dropout_p=0.0, + is_causal=False, + ) + pad_to = 128 if head_dim <= 128 else 256 + pad_size = pad_to - head_dim + xq_padded = F.pad(xq, (0, pad_size)) + xk_padded = F.pad(xk, (0, pad_size)) + xv_padded = F.pad(xv, (0, pad_size)) + out = F.scaled_dot_product_attention( + xq_padded, + xk_padded, + xv_padded, + dropout_p=0.0, + is_causal=False, + scale=head_dim**-0.5, + ) + return out[..., :head_dim] + + +def _normalize_chunk_index( + chunk_index: list[int] | None, + frames: int, + chunk_size: int, + chunk_split_strategy: str, +) -> list[int]: + if chunk_index is None: + chunk_index = _chunk_index_from_chunk_size( + frames, + chunk_size, + chunk_split_strategy, + ) + else: + chunk_index = list(chunk_index) + if not chunk_index or chunk_index[0] != 0: + chunk_index = [0] + [idx for idx in chunk_index if idx > 0] + chunk_index = [idx for idx in chunk_index if idx < frames] + if not chunk_index: + chunk_index = [0] + if chunk_index[-1] != frames: + chunk_index.append(frames) + return chunk_index + + +def _chunk_index_from_chunk_size( + frames: int, + chunk_size: int, + chunk_split_strategy: str, +) -> list[int]: + if chunk_size <= 0: + raise ValueError(f"chunk_size must be > 0, got {chunk_size}.") + if frames <= 0: + raise ValueError(f"frames must be > 0, got {frames}.") + strategy = "uniform" if chunk_split_strategy is None else str(chunk_split_strategy) + strategy = strategy.lower() + if strategy in ("uniform", "default"): + indices = list(range(0, frames, chunk_size)) + if len(indices) > 1 and (frames - indices[-1]) < chunk_size: + indices.pop() + return indices + if strategy in ("first_frame", "first_frame_alone", "first_frame_only"): + if frames <= 1: + return [0] + indices = [0] + list(range(1, frames, chunk_size)) + if len(indices) > 2 and (frames - indices[-1]) < chunk_size: + indices.pop() + return indices + if strategy in ("first_plus_one", "first_chunk_plus_one"): + if frames <= chunk_size + 1: + return [0] + indices = [0] + list(range(chunk_size + 1, frames, chunk_size)) + if len(indices) > 1 and (frames - indices[-1]) < chunk_size: + indices.pop() + return indices + raise ValueError( + "Unknown chunk_split_strategy " + f"{chunk_split_strategy!r}; expected uniform, first_frame, or first_plus_one." + ) + + +def _use_fused_gdn(x: Tensor) -> bool: + return ( + x.is_cuda + and _fused_bigdn_func is not None + and os.environ.get("SANA_WM_DISABLE_FUSED_GDN", "0") != "1" + ) + + +def _prepare_camera_projection_cache( + camera_conditions: Tensor, + *, + HW: tuple[int, int, int], + rotary_emb: Tensor | None, + head_dim: int, +) -> _CameraProjectionCache: + batch, frames = camera_conditions.shape[:2] + latent_frames, height, width = HW + if frames != latent_frames: + raise ValueError( + f"camera_conditions has {frames} frames but latent grid has {latent_frames}." + ) + tokens = frames * height * width + raymats = _camera_ray_mats(camera_conditions.float(), HW).to( + dtype=camera_conditions.dtype, + ) + proj = raymats.reshape(batch, tokens, 4, 4) + proj_q = proj.transpose(-1, -2).contiguous() + proj_kv = _invert_se3(proj).contiguous() + rope_cam = _slice_rope_for_camera(rotary_emb, head_dim) + if rope_cam is not None and _fused_prepare_ucpe_rope_tables is not None: + rope_cos, rope_sin = _fused_prepare_ucpe_rope_tables( + rope_cam, + tokens, + head_dim // 2, + camera_conditions.device, + ) + else: + rope_cos = torch.ones( + tokens, + head_dim // 2, + device=camera_conditions.device, + dtype=torch.float32, + ) + rope_sin = torch.zeros( + tokens, + head_dim // 2, + device=camera_conditions.device, + dtype=torch.float32, + ) + return _CameraProjectionCache( + raymats=raymats, + proj=proj, + proj_q=proj_q, + proj_kv=proj_kv, + rope_cam=rope_cam, + rope_cos=rope_cos, + rope_sin=rope_sin, + ) + + +@torch.compile +def _apply_output_gate( + out: Tensor, + gate_x: Tensor, + gate_weight: Tensor, + gate_bias: Tensor, +) -> Tensor: + gate_values = F.silu(F.linear(gate_x, gate_weight, gate_bias).float()) return out * gate_values.to(dtype=out.dtype) +@torch.compile +def _compute_frame_gates( + x: Tensor, + frames: int, + spatial: int, + heads: int, + beta_weight: Tensor, + beta_bias: Tensor, + gate_weight: Tensor, + gate_bias: Tensor, + dt_bias: Tensor, + A_log: Tensor, +) -> tuple[Tensor, Tensor]: + batch, tokens, channels = x.shape + beta = F.linear(x, beta_weight, beta_bias).sigmoid() + beta = beta.reshape(batch, frames, spatial, heads).permute(0, 3, 1, 2) + x_frame = x.reshape(batch, frames, spatial, channels).mean(dim=2) + gate = F.linear(x_frame, gate_weight, gate_bias).float() + dt = dt_bias.float().view(1, 1, -1) + a_val = A_log.float().exp().view(1, 1, -1) + decay = (-a_val * F.softplus(gate + dt)).exp().transpose(1, 2) + return beta, decay + + def _apply_bidirectional_temporal_conv( x: Tensor, conv: nn.Conv1d, @@ -1088,12 +1529,29 @@ def _camera_qkv( HW: tuple[int, int, int], ) -> tuple[Tensor, Tensor, Tensor]: batch, tokens, _channels = x.shape - q_raw = module.q_proj_cam(x).reshape(batch, tokens, module.heads, module.dim) - k_raw_flat = module.k_proj_cam(x) + qkv_weight = torch.cat( + [ + module.q_proj_cam.weight, + module.k_proj_cam.weight, + module.v_proj_cam.weight, + ], + ) + qkv_bias = torch.cat( + [ + module.q_proj_cam.bias, + module.k_proj_cam.bias, + module.v_proj_cam.bias, + ], + ) + q_raw_flat, k_raw_flat, v_raw_flat = F.linear(x, qkv_weight, qkv_bias).chunk( + 3, + dim=-1, + ) if hasattr(module, "conv_k_cam"): k_raw_flat = _apply_bidirectional_temporal_conv(k_raw_flat, module.conv_k_cam, HW) + q_raw = q_raw_flat.reshape(batch, tokens, module.heads, module.dim) k_raw = k_raw_flat.reshape(batch, tokens, module.heads, module.dim) - v_raw = module.v_proj_cam(x).reshape(batch, tokens, module.heads, module.dim) + v_raw = v_raw_flat.reshape(batch, tokens, module.heads, module.dim) return q_raw, k_raw, v_raw @@ -1105,6 +1563,7 @@ def _prepare_ucpe_qkv( camera_conditions: Tensor, HW: tuple[int, int, int], rotary_emb: Tensor | None, + camera_cache: _CameraProjectionCache | None, q_norm_weight: Tensor, k_norm_weight: Tensor, norm_eps: float, @@ -1118,36 +1577,39 @@ def _prepare_ucpe_qkv( k_norm = F.relu(k_raw.float() * k_inv[:, :, None, None] * k_weight[None, None]) k_norm = k_norm * _gdn_key_scale(dim, HW) v_float = v_raw.float() - raymats = _camera_ray_mats(camera_conditions.float(), HW) - matrix_q = raymats.reshape(batch, tokens, 4, 4).transpose(-1, -2).contiguous() - matrix_kv = _invert_se3(raymats.reshape(batch, tokens, 4, 4)).contiguous() - rope_cam = _slice_rope_for_camera(rotary_emb, dim) + if camera_cache is None: + camera_cache = _prepare_camera_projection_cache( + camera_conditions, + HW=HW, + rotary_emb=rotary_emb, + head_dim=dim, + ) - q_trans, _q_pre, _q_post = _ucpe_transform( + q_trans = _ucpe_transform_apply( q_norm, - matrix_q, - rope_cam, + camera_cache.proj_q, + camera_cache.rope_cam, inverse_rope=False, ) k_trans, k_pre_sq, k_post_sq = _ucpe_transform( k_norm, - matrix_kv, - rope_cam, + camera_cache.proj_kv, + camera_cache.rope_cam, inverse_rope=False, ) - v_trans, _v_pre, _v_post = _ucpe_transform( + v_trans = _ucpe_transform_apply( v_float, - matrix_kv, - rope_cam, + camera_cache.proj_kv, + camera_cache.rope_cam, inverse_rope=False, ) inflation_sq = k_post_sq.clamp_min(1e-12) / k_pre_sq.clamp_min(1e-12) def output_projector(out: Tensor) -> Tensor: - projected, _pre, _post = _ucpe_transform( + projected = _ucpe_transform_apply( out.float(), - raymats.reshape(batch, tokens, 4, 4), - rope_cam, + camera_cache.proj, + camera_cache.rope_cam, inverse_rope=True, ) return projected.to(dtype=out.dtype) @@ -1163,6 +1625,7 @@ def _prepare_ucpe_qkv_softmax( camera_conditions: Tensor, HW: tuple[int, int, int], rotary_emb: Tensor | None, + camera_cache: _CameraProjectionCache | None, q_norm_weight: Tensor, k_norm_weight: Tensor, norm_eps: float, @@ -1172,37 +1635,38 @@ def _prepare_ucpe_qkv_softmax( k_inv = _inv_rms(k_raw, norm_eps) q_weight = q_norm_weight.float().view(heads, dim) k_weight = k_norm_weight.float().view(heads, dim) - q_norm = q_raw.float() * q_inv[:, :, None, None] * q_weight[None, None] - k_norm = k_raw.float() * k_inv[:, :, None, None] * k_weight[None, None] - v_float = v_raw.float() - raymats = _camera_ray_mats(camera_conditions.float(), HW) - matrix_q = raymats.reshape(batch, tokens, 4, 4).transpose(-1, -2).contiguous() - matrix_kv = _invert_se3(raymats.reshape(batch, tokens, 4, 4)).contiguous() - rope_cam = _slice_rope_for_camera(rotary_emb, dim) - q_trans, _q_pre, _q_post = _ucpe_transform( + q_norm = ( + q_raw.float() * q_inv[:, :, None, None] * q_weight[None, None] + ).to(dtype=q_raw.dtype) + k_norm = ( + k_raw.float() * k_inv[:, :, None, None] * k_weight[None, None] + ).to(dtype=k_raw.dtype) + if camera_cache is None: + camera_cache = _prepare_camera_projection_cache( + camera_conditions, + HW=HW, + rotary_emb=rotary_emb, + head_dim=dim, + ) + q_trans = _ucpe_transform_apply( q_norm, - matrix_q, - rope_cam, - inverse_rope=False, - ) - k_trans, _k_pre, _k_post = _ucpe_transform( - k_norm, - matrix_kv, - rope_cam, + camera_cache.proj_q, + camera_cache.rope_cam, inverse_rope=False, - ) - v_trans, _v_pre, _v_post = _ucpe_transform( - v_float, - matrix_kv, - rope_cam, + ).to(dtype=q_raw.dtype) + kv_trans = _ucpe_transform_apply( + torch.cat([k_norm, v_raw], dim=2), + camera_cache.proj_kv, + camera_cache.rope_cam, inverse_rope=False, - ) + ).to(dtype=q_raw.dtype) + k_trans, v_trans = kv_trans.chunk(2, dim=2) def output_projector(out: Tensor) -> Tensor: - projected, _pre, _post = _ucpe_transform( - out.float(), - raymats.reshape(batch, tokens, 4, 4), - rope_cam, + projected = _ucpe_transform_apply( + out.to(dtype=q_raw.dtype), + camera_cache.proj, + camera_cache.rope_cam, inverse_rope=True, ) return projected.to(dtype=out.dtype) @@ -1228,7 +1692,7 @@ def _ucpe_transform( if half % 4 != 0: raise ValueError(f"UCPE requires head_dim/2 divisible by 4, got {half}.") first = x[..., :half].reshape(batch, tokens, heads, half // 4, 4) - first_out = torch.einsum("bnij,bnhgj->bnhgi", matrix.float(), first.float()) + first_out = torch.einsum("bnij,bnhgj->bnhgi", matrix, first) first_out = first_out.reshape(batch, tokens, heads, half) second = x[..., half:] if inverse_rope and rotary_emb is not None: @@ -1241,6 +1705,28 @@ def _ucpe_transform( return out, pre_sq, post_sq +def _ucpe_transform_apply( + x: Tensor, + matrix: Tensor, + rotary_emb: Tensor | None, + *, + inverse_rope: bool, +) -> Tensor: + batch, tokens, heads, dim = x.shape + half = dim // 2 + if half % 4 != 0: + raise ValueError(f"UCPE requires head_dim/2 divisible by 4, got {half}.") + first = x[..., :half].reshape(batch, tokens, heads, half // 4, 4) + first_out = torch.einsum("bnij,bnhgj->bnhgi", matrix.float(), first.float()) + first_out = first_out.reshape(batch, tokens, heads, half) + second = x[..., half:] + if inverse_rope and rotary_emb is not None: + second_out = _apply_complex_rope(second, rotary_emb.conj()) + else: + second_out = _apply_complex_rope(second, rotary_emb) + return torch.cat([first_out, second_out], dim=-1) + + def _camera_ray_mats( camera_conditions: Tensor, HW: tuple[int, int, int], diff --git a/integrations/sana/sana_wm/transformer.py b/integrations/sana/sana_wm/transformer.py index 0b07b2810..dd744772c 100644 --- a/integrations/sana/sana_wm/transformer.py +++ b/integrations/sana/sana_wm/transformer.py @@ -337,10 +337,7 @@ def _predict_conditioned( torch.cat([conditioning.uncondition, conditioning.condition], dim=0), _batched_cfg_model_kwargs(conditioning.model_kwargs), ) - noise_pred_uncond, noise_pred_text = noise_pred.chunk(2, dim=0) - flow = noise_pred_uncond + conditioning.cfg_scale * ( - noise_pred_text - noise_pred_uncond - ) + flow = _cfg_guidance(noise_pred, conditioning.cfg_scale) return _zero_conditioned_frame_flow(flow, conditioning) def _predict_with_prompt( @@ -491,6 +488,11 @@ def _batched_cfg_model_kwargs(model_kwargs: dict[str, object]) -> dict[str, obje return kwargs +def _cfg_guidance(noise_pred: Tensor, cfg_scale: float) -> Tensor: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2, dim=0) + return noise_pred_uncond + cfg_scale * (noise_pred_text - noise_pred_uncond) + + def _conditioned_frame_timestep( *, noisy_latent: Tensor, From a9f56a42a35831002c9d0d11d687cdcf83007aa0 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 24 Jul 2026 13:44:52 -0700 Subject: [PATCH 26/64] Split benchmarks by GPU, chnage metrics, update model card --- .../performance/sana_wm/perf-bf16-0724.md | 5 ++ docs/source/models/sana_wm.rst | 31 +++++++++ integrations/sana/README.md | 14 ++-- .../sana/tests/parity_check/README.md | 31 ++++++--- integrations/sana/tests/parity_check/bench.sh | 11 ++-- .../sana/tests/parity_check/bench_summary.py | 38 ++++++++--- .../tests/parity_check/bench_sweep_summary.py | 15 +++-- .../sana/tests/parity_check/run_native.py | 37 ++++++----- .../tests/test_parity_benchmark_summary.py | 64 +++++++++++++------ integrations/sana/tests/test_smoke.py | 30 +++++---- 10 files changed, 190 insertions(+), 86 deletions(-) create mode 100644 docs/source/_static/performance/sana_wm/perf-bf16-0724.md diff --git a/docs/source/_static/performance/sana_wm/perf-bf16-0724.md b/docs/source/_static/performance/sana_wm/perf-bf16-0724.md new file mode 100644 index 000000000..95ada7025 --- /dev/null +++ b/docs/source/_static/performance/sana_wm/perf-bf16-0724.md @@ -0,0 +1,5 @@ +# SANA-WM BF16 Benchmark Data (ms) + +| device | official | flashdreams | +| --- | ---: | ---: | +| GB202 | 77826.74 | 76938.72 | diff --git a/docs/source/models/sana_wm.rst b/docs/source/models/sana_wm.rst index b58ccb795..735fa64a1 100644 --- a/docs/source/models/sana_wm.rst +++ b/docs/source/models/sana_wm.rst @@ -124,6 +124,37 @@ What to expect See :doc:`/developer_guides/inference_pipeline_overview` for what one pass does end-to-end. +Profiling benchmark +------------------- + +Here is the BF16 profiling benchmark on post-load generation latency per +generated clip for FlashDreams SANA-WM compared to the +`official SANA-WM implementation `_ under +matched settings. + +.. raw:: html + +
+
+
+

+ This chart shows post-load generation latency per generated clip in milliseconds for a 121-frame + Stage-1-only BF16 run. The measured GB202 row used an NVIDIA RTX PRO 6000 Blackwell + Workstation Edition. + Model construction, checkpoint loading, video writing, and frame dumps are outside the timing boundary. + For the official SANA-WM implementation, see + this instruction. +

+
+
+ + Citation -------- diff --git a/integrations/sana/README.md b/integrations/sana/README.md index 16f64236b..2690a0e4e 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -146,10 +146,16 @@ DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh ``` `bench.sh` writes `outputs/bench/bench.md` for review and `outputs/bench/perf.md` -for model-card chart data. Both use the same benchmark metric: post-load -generation latency per generated frame. Stage-1 DiT, conditioning/encode, VAE -decode, optional refiner, and memory rows are reported as the breakdown of that -metric. +for chart-ready data. Both use the same benchmark metric: post-load generation +latency per generated clip. SANA-WM renders each requested bidirectional clip in +one generation pass, rather than as independently timed frames. Stage-1 DiT, +conditioning/encode, VAE decode, optional refiner, memory, and frame-normalized +diagnostic rows are reported as breakdowns of that metric. + +For precision sweeps, copy the per-precision chart data from +`outputs/bench//perf.md` into the docs. Each model-card chart should +cover one precision and use GPU/device as the differentiating row, matching the +other FlashDreams model-card benchmark charts. ## Tests diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index d0119a828..5171f7376 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -65,18 +65,33 @@ Outputs are under `outputs/bench/`: - `bench.json` - machine-readable inputs, medians, p90s, memory, and stage timings. - `bench.md` - human-readable report. -- `perf.md` - chart-ready model-card data using the same benchmark metric as - `bench.md`. +- `perf.md` - chart-ready data using the same benchmark metric as `bench.md`. -The benchmark metric is post-load generation latency per generated frame. Model +The benchmark metric is post-load generation latency per generated clip. Model construction, checkpoint loading, video writing, and frame dumps are outside the -timing boundary. With the default `NO_REFINER=1`, the timed work covers -conditioning, Stage-1 DiT, and SANA VAE decode. Set `NO_REFINER=0` to benchmark -the full Stage-1 + LTX-2 refiner path with the same timing boundary. +timing boundary. SANA-WM renders each requested bidirectional clip in one +generation pass, rather than as independently timed frames. With the default +`NO_REFINER=1`, the timed work covers conditioning, Stage-1 DiT, and SANA VAE +decode. Set `NO_REFINER=0` to benchmark the full Stage-1 + LTX-2 refiner path +with the same timing boundary. `bench.md` also reports Stage-1 DiT, conditioning/encode, VAE decode, optional -refiner, and memory breakdowns. Those rows explain the benchmark result; they -are not a second benchmark metric. +refiner, memory, and frame-normalized diagnostic breakdowns. Those rows explain +the benchmark result; they are not a second benchmark metric. + +The model card should use one chart per precision. Each chart file should have +GPU/device as the first column and implementation as the series columns, for +example: + +```markdown +| device | official | flashdreams | +| --- | ---: | ---: | +| GB202 | 77826.74 | 76938.72 | +``` + +In precision-sweep mode, the chart-ready files are +`outputs/bench//perf.md`. The top-level `outputs/bench/perf.md` is a +precision summary, not the model-card chart data. Set `FORCE_CUDNN_SDPA=1` only for backend-isolation probes. It is not the default SANA-WM benchmark setting. diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index 2769d7e59..c7ec2c296 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -15,7 +15,7 @@ # limitations under the License. # Stack-matched SANA-WM upstream vs FlashDreams benchmark. The chart metric is -# post-load generation latency per generated frame. Defaults to Stage-1-only so +# post-load generation latency per generated clip. Defaults to Stage-1-only so # conditioning, Stage-1 DiT, and SANA VAE decode are measured without the LTX-2 # refiner. @@ -90,7 +90,7 @@ if [[ -n "${BENCH_PRECISIONS}" ]]; then STAGE1_PRECISION="${PRECISION}" \ REFINER_PRECISION="${PRECISION}" \ QUANT_BACKEND="${QUANT_BACKEND}" \ - CHART_LABEL="${PRECISION_LABEL}" \ + CHART_LABEL="${CHART_LABEL}" \ bash "${SCRIPT_DIR}/bench.sh" SWEEP_ITEMS+=(--item "${PRECISION_LABEL}:${PRECISION_OUTPUT_DIR}/bench.json") done @@ -98,7 +98,7 @@ if [[ -n "${BENCH_PRECISIONS}" ]]; then echo "[bench] ERROR: BENCH_PRECISIONS produced no precision rows." >&2 exit 1 fi - echo "[bench] aggregating precision sweep -> ${OUTPUT_DIR}/perf.md" + echo "[bench] aggregating precision sweep -> ${OUTPUT_DIR}/bench.md" ( cd "${SCRIPT_DIR}" && \ uv run python "${SCRIPT_DIR}/bench_sweep_summary.py" \ "${SWEEP_ITEMS[@]}" \ @@ -106,8 +106,9 @@ if [[ -n "${BENCH_PRECISIONS}" ]]; then --output-md "${OUTPUT_DIR}/bench.md" \ --output-chart-md "${OUTPUT_DIR}/perf.md" ) echo "[bench] done." - echo " summary: ${OUTPUT_DIR}/bench.md" - echo " chart data: ${OUTPUT_DIR}/perf.md" + echo " precision summary: ${OUTPUT_DIR}/bench.md" + echo " precision data: ${OUTPUT_DIR}/perf.md" + echo " chart data: ${OUTPUT_DIR}/{bf16,fp8,fp4}/perf.md" exit 0 fi diff --git a/integrations/sana/tests/parity_check/bench_summary.py b/integrations/sana/tests/parity_check/bench_summary.py index c0d8bbf22..52b40d8a9 100644 --- a/integrations/sana/tests/parity_check/bench_summary.py +++ b/integrations/sana/tests/parity_check/bench_summary.py @@ -128,11 +128,26 @@ def _ms_per_frame(wall_s: float | None, num_frames: int) -> float | None: return wall_s * 1000.0 / num_frames +def _ms_per_clip(wall_s: float | None) -> float | None: + if wall_s is None: + return None + return wall_s * 1000.0 + + def _metric_value(summary: dict[str, Any], side: str, key: str) -> float | None: value = summary[side].get(key) return float(value) if isinstance(value, (int, float)) else None +def _generation_ms_per_clip( + summary: dict[str, Any], + side: str, + *, + percentile: str = "median", +) -> float | None: + return _ms_per_clip(_metric_value(summary, side, f"wall_{percentile}_s")) + + def _generation_ms_per_frame( summary: dict[str, Any], side: str, @@ -176,15 +191,18 @@ def _render_markdown(summary: dict[str, Any]) -> str: "", "## Benchmark metric", "", - "The chart metric is post-load generation latency per generated frame.", + "The chart metric is post-load generation latency per generated clip.", "Model construction, checkpoint loading, video writing, and frame dumps are outside this timing boundary.", + "SANA-WM renders each requested bidirectional clip in one generation pass, not as independently timed frames.", "With the default `NO_REFINER=1`, the timed work covers conditioning, Stage-1 DiT, and SANA VAE decode.", "", "| metric | upstream | FlashDreams |", "| --- | ---: | ---: |", f"| measured runs | {upstream['runs_measured']} | {native['runs_measured']} |", - f"| generation median / frame | {_fmt(_generation_ms_per_frame(summary, 'upstream'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams'), ' ms')} |", - f"| generation p90 / frame | {_fmt(_generation_ms_per_frame(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams', percentile='p90'), ' ms')} |", + f"| generation median / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream'), ' ms')} | {_fmt(_generation_ms_per_clip(summary, 'flashdreams'), ' ms')} |", + f"| generation p90 / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_generation_ms_per_clip(summary, 'flashdreams', percentile='p90'), ' ms')} |", + f"| generation median / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'upstream'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams'), ' ms')} |", + f"| generation p90 / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams', percentile='p90'), ' ms')} |", f"| wall median | {_fmt(upstream['wall_median_s'], ' s')} | {_fmt(native['wall_median_s'], ' s')} |", f"| wall p90 | {_fmt(upstream['wall_p90_s'], ' s')} | {_fmt(native['wall_p90_s'], ' s')} |", "", @@ -209,13 +227,13 @@ def _render_markdown(summary: dict[str, Any]) -> str: def _render_chart_markdown(summary: dict[str, Any], device_label: str) -> str: - official = _generation_ms_per_frame(summary, "upstream") - flashdreams = _generation_ms_per_frame(summary, "flashdreams") + official = _generation_ms_per_clip(summary, "upstream") + flashdreams = _generation_ms_per_clip(summary, "flashdreams") if official is None or flashdreams is None: raise ValueError("cannot render chart data without wall-clock stats") return "\n".join( [ - "# SANA-WM Benchmark Data (ms/frame)", + "# SANA-WM Benchmark Data (ms)", "", "| device | official | flashdreams |", "| --- | ---: | ---: |", @@ -298,16 +316,16 @@ def main(argv: list[str] | None = None) -> None: ), } summary["benchmark"] = { - "metric": "generation_ms_per_frame", - "unit": "ms/frame", + "metric": "generation_ms_per_clip", + "unit": "ms", "timing_boundary": ( "pipeline.generate after model setup; excludes model construction, " "checkpoint loading, video writing, and frame dumps" ), "device_label": args.device_label, "chart_label": args.chart_label or args.device_label, - "official": _generation_ms_per_frame(summary, "upstream"), - "flashdreams": _generation_ms_per_frame(summary, "flashdreams"), + "official": _generation_ms_per_clip(summary, "upstream"), + "flashdreams": _generation_ms_per_clip(summary, "flashdreams"), } args.output_json.parent.mkdir(parents=True, exist_ok=True) args.output_json.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") diff --git a/integrations/sana/tests/parity_check/bench_sweep_summary.py b/integrations/sana/tests/parity_check/bench_sweep_summary.py index c2343325f..258b752ff 100644 --- a/integrations/sana/tests/parity_check/bench_sweep_summary.py +++ b/integrations/sana/tests/parity_check/bench_sweep_summary.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Aggregate SANA-WM precision benchmark summaries into chart data.""" +"""Aggregate SANA-WM precision benchmark summaries.""" from __future__ import annotations @@ -58,7 +58,7 @@ def _load_item(raw: str) -> dict[str, Any]: def _render_chart(rows: list[dict[str, Any]]) -> str: lines = [ - "# SANA-WM Benchmark Data (ms/frame)", + "# SANA-WM Precision Sweep Summary (ms)", "", "| precision | official | flashdreams |", "| --- | ---: | ---: |", @@ -75,15 +75,16 @@ def _render_report(rows: list[dict[str, Any]]) -> str: lines = [ "# SANA-WM precision benchmark sweep", "", - "The chart metric is post-load generation latency per generated frame.", + "The chart metric is post-load generation latency per generated clip.", + "Model-card chart data is grouped by GPU/device in each precision subdirectory's `perf.md`.", "", "| precision | official | FlashDreams | source |", "| --- | ---: | ---: | --- |", ] for row in rows: lines.append( - f"| {row['label']} | {row['official']:.2f} ms/frame | " - f"{row['flashdreams']:.2f} ms/frame | `{row['path']}` |" + f"| {row['label']} | {row['official']:.2f} ms | " + f"{row['flashdreams']:.2f} ms | `{row['path']}` |" ) lines.append("") return "\n".join(lines) @@ -105,8 +106,8 @@ def main(argv: list[str] | None = None) -> None: payload = { "benchmark": { - "metric": "generation_ms_per_frame", - "unit": "ms/frame", + "metric": "generation_ms_per_clip", + "unit": "ms", "timing_boundary": ( "pipeline.generate after model setup; excludes model construction, " "checkpoint loading, video writing, and frame dumps" diff --git a/integrations/sana/tests/parity_check/run_native.py b/integrations/sana/tests/parity_check/run_native.py index c93903fb9..d49601ab7 100644 --- a/integrations/sana/tests/parity_check/run_native.py +++ b/integrations/sana/tests/parity_check/run_native.py @@ -249,6 +249,20 @@ def main() -> None: if vae_decoder is not None: vae_decoder._ensure_vae() + request = SanaWMI2VConditioningRequest( + image=image, + prompt=prompt, + poses_c2w=c2w, + intrinsics_vec4=intrinsics_vec4, + num_frames=num_frames, + fps=args.fps, + steps=args.step, + cfg_scale=args.cfg_scale, + flow_shift=args.flow_shift, + seed=args.seed, + negative_prompt=args.negative_prompt, + ) + cache = pipeline.initialize_cache( decoder_context={ "prompt": prompt, @@ -262,23 +276,12 @@ def main() -> None: torch.cuda.reset_peak_memory_stats(device) torch.cuda.synchronize(device) generation_start = time.perf_counter() - decoded = pipeline.generate( - 0, - cache, - input=SanaWMI2VConditioningRequest( - image=image, - prompt=prompt, - poses_c2w=c2w, - intrinsics_vec4=intrinsics_vec4, - num_frames=num_frames, - fps=args.fps, - steps=args.step, - cfg_scale=args.cfg_scale, - flow_shift=args.flow_shift, - seed=args.seed, - negative_prompt=args.negative_prompt, - ), - ) + with torch.inference_mode(): + decoded = pipeline.generate( + 0, + cache, + input=request, + ) if torch.cuda.is_available(): torch.cuda.synchronize(device) wall_s = time.perf_counter() - generation_start diff --git a/integrations/sana/tests/test_parity_benchmark_summary.py b/integrations/sana/tests/test_parity_benchmark_summary.py index cebdc2d5f..2a10e18b6 100644 --- a/integrations/sana/tests/test_parity_benchmark_summary.py +++ b/integrations/sana/tests/test_parity_benchmark_summary.py @@ -47,7 +47,7 @@ def _load_bench_sweep_summary() -> ModuleType: return module -def test_benchmark_summary_uses_generation_ms_per_frame_for_chart() -> None: +def test_benchmark_summary_uses_generation_ms_per_clip_for_chart() -> None: module = _load_bench_summary() summary = { "inputs": { @@ -90,17 +90,18 @@ def test_benchmark_summary_uses_generation_ms_per_frame_for_chart() -> None: assert "## Benchmark metric" in report assert "- stage1_precision: `fp8`" in report - assert "generation median / frame | 100.00 ms | 75.00 ms" in report + assert "generation median / clip | 2000.00 ms | 1500.00 ms" in report + assert "generation median / frame, diagnostic | 100.00 ms | 75.00 ms" in report assert "## Timing breakdown" in report assert "| Stage-1 DiT median | 1500.00 ms | 1100.00 ms |" in report assert "Stage-1 DiT metric" not in report assert "Generation after model load" not in report assert chart == ( - "# SANA-WM Benchmark Data (ms/frame)\n" + "# SANA-WM Benchmark Data (ms)\n" "\n" "| device | official | flashdreams |\n" "| --- | ---: | ---: |\n" - "| Test GPU | 100.00 | 75.00 |\n" + "| Test GPU | 2000.00 | 1500.00 |\n" ) @@ -164,8 +165,6 @@ def test_benchmark_summary_writes_chart_data(tmp_path: Path) -> None: "42", "--device-label", "Test GPU", - "--chart-label", - "BF16", "--stage1-precision", "bf16", "--refiner-precision", @@ -179,35 +178,57 @@ def test_benchmark_summary_writes_chart_data(tmp_path: Path) -> None: ] ) - assert "| BF16 | 100.00 | 50.00 |" in out_chart.read_text(encoding="utf-8") - assert "generation median / frame | 100.00 ms | 50.00 ms" in out_md.read_text( - encoding="utf-8" - ) + assert "| Test GPU | 4000.00 | 2000.00 |" in out_chart.read_text(encoding="utf-8") + report = out_md.read_text(encoding="utf-8") + assert "generation median / clip | 4000.00 ms | 2000.00 ms" in report + assert "generation median / frame, diagnostic | 100.00 ms | 50.00 ms" in report + assert "generation median / frame |" not in report + assert "per generated clip" in report + assert "not as independently timed frames" in report + assert "per generated frame" not in report summary = json.loads(out_json.read_text(encoding="utf-8")) assert summary["benchmark"] == { - "metric": "generation_ms_per_frame", - "unit": "ms/frame", + "metric": "generation_ms_per_clip", + "unit": "ms", "timing_boundary": ( "pipeline.generate after model setup; excludes model construction, " "checkpoint loading, video writing, and frame dumps" ), "device_label": "Test GPU", - "chart_label": "BF16", - "official": 100.0, - "flashdreams": 50.0, + "chart_label": "Test GPU", + "official": 4000.0, + "flashdreams": 2000.0, } +def test_benchmark_summary_keeps_frame_normalized_diagnostics() -> None: + module = _load_bench_summary() + assert module._generation_ms_per_frame( + { + "inputs": {"num_frames": 40}, + "upstream": {"wall_median_s": 4.0}, + }, + "upstream", + ) == 100.0 + assert module._generation_ms_per_clip( + { + "inputs": {"num_frames": 40}, + "upstream": {"wall_median_s": 4.0}, + }, + "upstream", + ) == 4000.0 + + def test_benchmark_sweep_summary_writes_precision_chart_data(tmp_path: Path) -> None: module = _load_bench_sweep_summary() bf16_json = tmp_path / "bf16.json" fp8_json = tmp_path / "fp8.json" bf16_json.write_text( - json.dumps({"benchmark": {"official": 100.0, "flashdreams": 90.0}}), + json.dumps({"benchmark": {"official": 1000.0, "flashdreams": 900.0}}), encoding="utf-8", ) fp8_json.write_text( - json.dumps({"benchmark": {"official": 80.0, "flashdreams": 70.0}}), + json.dumps({"benchmark": {"official": 800.0, "flashdreams": 700.0}}), encoding="utf-8", ) out_json = tmp_path / "bench.json" @@ -230,13 +251,14 @@ def test_benchmark_sweep_summary_writes_precision_chart_data(tmp_path: Path) -> ) assert out_chart.read_text(encoding="utf-8") == ( - "# SANA-WM Benchmark Data (ms/frame)\n" + "# SANA-WM Precision Sweep Summary (ms)\n" "\n" "| precision | official | flashdreams |\n" "| --- | ---: | ---: |\n" - "| BF16 | 100.00 | 90.00 |\n" - "| FP8 | 80.00 | 70.00 |\n" + "| BF16 | 1000.00 | 900.00 |\n" + "| FP8 | 800.00 | 700.00 |\n" ) payload = json.loads(out_json.read_text(encoding="utf-8")) - assert payload["benchmark"]["metric"] == "generation_ms_per_frame" + assert payload["benchmark"]["metric"] == "generation_ms_per_clip" + assert payload["benchmark"]["unit"] == "ms" assert [row["label"] for row in payload["rows"]] == ["BF16", "FP8"] diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index e079e56ed..38c08f97d 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -671,6 +671,8 @@ def test_stage1_model_matches_checkpoint_schema() -> None: """Pin the Stage-1 module to the public checkpoint schema.""" state = SanaWMStage1Model().state_dict() + assert SANA_WM_STAGE1_SPEC.chunk_size is None + assert SANA_WM_STAGE1_SPEC.chunk_split_strategy == "first_chunk_plus_one" assert len(state) == 872 assert tuple(state["x_embedder.proj.weight"].shape) == (2240, 128, 1, 1, 1) assert tuple(state["raymap_embedder.proj.weight"].shape) == (2240, 3, 1, 1, 1) @@ -843,8 +845,8 @@ def test_postprocess_releases_stage1_when_offloading() -> None: assert cache.conditioning is None -def test_vae_tiling_uses_low_memory_tiles() -> None: - """Keep VAE decode on the configured low-memory tiles.""" +def test_vae_tiling_defaults_match_upstream_fast_path() -> None: + """Keep VAE decode on the upstream-style fast tiles by default.""" class DummyVAE: def __init__(self) -> None: @@ -869,20 +871,20 @@ def enable_tiling(self, **kwargs: int) -> None: assert decoder.vae.calls == [ { - "tile_sample_min_height": 256, - "tile_sample_stride_height": 192, - "tile_sample_min_width": 256, - "tile_sample_stride_width": 224, - "tile_sample_min_num_frames": 24, - "tile_sample_stride_num_frames": 8, + "tile_sample_min_height": 512, + "tile_sample_stride_height": 448, + "tile_sample_min_width": 512, + "tile_sample_stride_width": 448, + "tile_sample_min_num_frames": 96, + "tile_sample_stride_num_frames": 64, } ] - assert decoder.vae.tile_sample_min_height == 256 - assert decoder.vae.tile_sample_stride_height == 192 - assert decoder.vae.tile_sample_min_width == 256 - assert decoder.vae.tile_sample_stride_width == 224 - assert decoder.vae.tile_sample_min_num_frames == 24 - assert decoder.vae.tile_sample_stride_num_frames == 8 + assert decoder.vae.tile_sample_min_height == 512 + assert decoder.vae.tile_sample_stride_height == 448 + assert decoder.vae.tile_sample_min_width == 512 + assert decoder.vae.tile_sample_stride_width == 448 + assert decoder.vae.tile_sample_min_num_frames == 96 + assert decoder.vae.tile_sample_stride_num_frames == 64 assert decoder.vae.use_framewise_encoding is True assert decoder.vae.use_framewise_decoding is True From 9372087096d068df2eeff2331028effbd5e7aac3 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 24 Jul 2026 15:45:51 -0700 Subject: [PATCH 27/64] Remove uupstream TE dep --- integrations/sana/tests/parity_check/pyproject.toml | 10 +++++++++- integrations/sana/tests/parity_check/uv.lock | 9 +++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/integrations/sana/tests/parity_check/pyproject.toml b/integrations/sana/tests/parity_check/pyproject.toml index 0f2fbdf09..f92ac2738 100644 --- a/integrations/sana/tests/parity_check/pyproject.toml +++ b/integrations/sana/tests/parity_check/pyproject.toml @@ -30,11 +30,19 @@ dependencies = [ "torch>=2.11", "torchvision>=0.26", "tqdm>=4.60", - "transformer-engine[pytorch,core-cu13]>=2.12; sys_platform != 'win32'", "transformers>=5.0,<6", "triton>=3.6; sys_platform == 'linux'", ] +[project.optional-dependencies] +# Transformer Engine is only needed for the FP8/FP4 quant backends. It is gated +# behind an opt-in extra so the default `uv sync` (BF16 benchmarking) does not +# try to build/import it. Install with `uv sync --extra quant` once a +# transformer-engine-torch build matching the local torch/CUDA ABI is available. +quant = [ + "transformer-engine[pytorch,core-cu13]>=2.12; sys_platform != 'win32'", +] + [tool.uv.sources] flashdreams = { path = "../../../../flashdreams", editable = true } flashdreams-sana-wm = { path = "../..", editable = true } diff --git a/integrations/sana/tests/parity_check/uv.lock b/integrations/sana/tests/parity_check/uv.lock index baa519ed3..81c9f57a0 100644 --- a/integrations/sana/tests/parity_check/uv.lock +++ b/integrations/sana/tests/parity_check/uv.lock @@ -1239,11 +1239,15 @@ dependencies = [ { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "sys_platform == 'win32'" }, { name = "torchvision" }, { name = "tqdm" }, - { name = "transformer-engine", extra = ["core-cu13", "pytorch"], marker = "sys_platform != 'win32'" }, { name = "transformers" }, { name = "triton", marker = "sys_platform == 'linux'" }, ] +[package.optional-dependencies] +quant = [ + { name = "transformer-engine", extra = ["core-cu13", "pytorch"], marker = "sys_platform != 'win32'" }, +] + [package.metadata] requires-dist = [ { name = "accelerate", specifier = ">=1.3" }, @@ -1272,10 +1276,11 @@ requires-dist = [ { name = "torch", specifier = ">=2.11" }, { name = "torchvision", specifier = ">=0.26" }, { name = "tqdm", specifier = ">=4.60" }, - { name = "transformer-engine", extras = ["pytorch", "core-cu13"], marker = "sys_platform != 'win32'", specifier = ">=2.12" }, + { name = "transformer-engine", extras = ["pytorch", "core-cu13"], marker = "sys_platform != 'win32' and extra == 'quant'", specifier = ">=2.12" }, { name = "transformers", specifier = ">=5.0,<6" }, { name = "triton", marker = "sys_platform == 'linux'", specifier = ">=3.6" }, ] +provides-extras = ["quant"] [[package]] name = "scipy" From 1435fc034b2790612f60287b891522284cf61d98 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Mon, 27 Jul 2026 10:23:45 -0700 Subject: [PATCH 28/64] Update data after Claude review --- .../sana_wm/{perf-bf16-0724.md => perf-bf16-0727.md} | 2 +- docs/source/models/sana_wm.rst | 5 +++-- integrations/sana/sana_wm/stage1_model.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) rename docs/source/_static/performance/sana_wm/{perf-bf16-0724.md => perf-bf16-0727.md} (74%) diff --git a/docs/source/_static/performance/sana_wm/perf-bf16-0724.md b/docs/source/_static/performance/sana_wm/perf-bf16-0727.md similarity index 74% rename from docs/source/_static/performance/sana_wm/perf-bf16-0724.md rename to docs/source/_static/performance/sana_wm/perf-bf16-0727.md index 95ada7025..9e9004f8e 100644 --- a/docs/source/_static/performance/sana_wm/perf-bf16-0724.md +++ b/docs/source/_static/performance/sana_wm/perf-bf16-0727.md @@ -2,4 +2,4 @@ | device | official | flashdreams | | --- | ---: | ---: | -| GB202 | 77826.74 | 76938.72 | +| GB202 | 76503.01 | 77109.93 | diff --git a/docs/source/models/sana_wm.rst b/docs/source/models/sana_wm.rst index 735fa64a1..ac297d626 100644 --- a/docs/source/models/sana_wm.rst +++ b/docs/source/models/sana_wm.rst @@ -130,7 +130,8 @@ Profiling benchmark Here is the BF16 profiling benchmark on post-load generation latency per generated clip for FlashDreams SANA-WM compared to the `official SANA-WM implementation `_ under -matched settings. +matched settings. On this metric the two implementations are at parity: the +measured difference is within run-to-run variance (under 1%). .. raw:: html @@ -138,7 +139,7 @@ matched settings.
diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index c376a14c5..4d44eab5f 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -1692,7 +1692,7 @@ def _ucpe_transform( if half % 4 != 0: raise ValueError(f"UCPE requires head_dim/2 divisible by 4, got {half}.") first = x[..., :half].reshape(batch, tokens, heads, half // 4, 4) - first_out = torch.einsum("bnij,bnhgj->bnhgi", matrix, first) + first_out = torch.einsum("bnij,bnhgj->bnhgi", matrix.float(), first.float()) first_out = first_out.reshape(batch, tokens, heads, half) second = x[..., half:] if inverse_rope and rotary_emb is not None: From edf91668864cd9abc7cb6733c2179825725e6f79 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Mon, 27 Jul 2026 10:53:18 -0700 Subject: [PATCH 29/64] Uaw GB300 numbers for model card --- docs/source/_static/performance/sana_wm/perf-bf16-0727.md | 2 +- docs/source/models/sana_wm.rst | 7 +++---- integrations/sana/tests/parity_check/bench.sh | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/source/_static/performance/sana_wm/perf-bf16-0727.md b/docs/source/_static/performance/sana_wm/perf-bf16-0727.md index 9e9004f8e..8ceb9ae16 100644 --- a/docs/source/_static/performance/sana_wm/perf-bf16-0727.md +++ b/docs/source/_static/performance/sana_wm/perf-bf16-0727.md @@ -2,4 +2,4 @@ | device | official | flashdreams | | --- | ---: | ---: | -| GB202 | 76503.01 | 77109.93 | +| GB300 | 33051.56 | 31640.88 | diff --git a/docs/source/models/sana_wm.rst b/docs/source/models/sana_wm.rst index ac297d626..8df606d75 100644 --- a/docs/source/models/sana_wm.rst +++ b/docs/source/models/sana_wm.rst @@ -130,8 +130,8 @@ Profiling benchmark Here is the BF16 profiling benchmark on post-load generation latency per generated clip for FlashDreams SANA-WM compared to the `official SANA-WM implementation `_ under -matched settings. On this metric the two implementations are at parity: the -measured difference is within run-to-run variance (under 1%). +matched settings. On GB300, FlashDreams is about 4% faster than the official +implementation at median clip latency. .. raw:: html @@ -146,8 +146,7 @@ measured difference is within run-to-run variance (under 1%).

This chart shows post-load generation latency per generated clip in milliseconds for a 121-frame - Stage-1-only BF16 run. The measured GB202 row used an NVIDIA RTX PRO 6000 Blackwell - Workstation Edition. + Stage-1-only BF16 run. The measured row used an NVIDIA GB300. Model construction, checkpoint loading, video writing, and frame dumps are outside the timing boundary. For the official SANA-WM implementation, see this instruction. diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index c7ec2c296..10df4260c 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -30,7 +30,7 @@ _abspath() { esac } -SANA_REPO="$(_abspath "${SANA_REPO:-${HOME}/dev/Sana}")" +SANA_REPO="$(_abspath "${SANA_REPO:-${SCRIPT_DIR}/Sana}")" PATCH_FILE="${SCRIPT_DIR}/changes.patch" REPO_URL="https://github.com/NVlabs/Sana.git" PIN_COMMIT="6298508" From fd6faa6787223bc2f83296e6cfed7083709f391c Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Mon, 27 Jul 2026 11:20:53 -0700 Subject: [PATCH 30/64] Fix bench labels --- .../sana/tests/parity_check/bench_summary.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/integrations/sana/tests/parity_check/bench_summary.py b/integrations/sana/tests/parity_check/bench_summary.py index 52b40d8a9..fa76ed17f 100644 --- a/integrations/sana/tests/parity_check/bench_summary.py +++ b/integrations/sana/tests/parity_check/bench_summary.py @@ -213,15 +213,25 @@ def _render_markdown(summary: dict[str, Any]) -> str: f"| Stage-1 incl. conditioning median | {_fmt(upstream['stage1_total_median_ms'], ' ms')} | {_fmt(_sum_optional(native['encode_median_ms'], native['dit_median_ms']), ' ms')} |", f"| Stage-1 DiT median | {_fmt(upstream['dit_median_ms'], ' ms')} | {_fmt(native['dit_median_ms'], ' ms')} |", f"| conditioning/encode median | n/a | {_fmt(native['encode_median_ms'], ' ms')} |", - f"| VAE decode median | {_fmt(upstream['vae_decode_median_ms'], ' ms')} | {_fmt(native['vae_decode_median_ms'], ' ms')} |", - f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} | {_fmt(native['mem_peak_median_gib'], ' GiB')} |", ] - if upstream.get("refiner_median_ms") is not None: - rows.extend( - [ - f"| refiner median | {_fmt(upstream['refiner_median_ms'], ' ms')} | n/a |", - ] + if summary["inputs"]["no_refiner"]: + # No refiner: FlashDreams `decode_ms` and upstream `vae_decode_s` are both + # the pure SANA VAE decode, so they compare directly. + rows.append( + f"| VAE decode median | {_fmt(upstream['vae_decode_median_ms'], ' ms')} | {_fmt(native['vae_decode_median_ms'], ' ms')} |" ) + else: + # Refiner enabled: both sides bundle the refiner denoise together with its + # VAE decode into a single measurement (upstream `refiner_s`, FlashDreams + # `decode_ms`). They are only apples-to-apples as one combined row; the + # standalone upstream `vae_decode_s` is the Stage-1 decode and is NOT + # comparable to FlashDreams `decode_ms`. + rows.append( + f"| refiner + VAE decode median | {_fmt(upstream.get('refiner_median_ms'), ' ms')} | {_fmt(native['vae_decode_median_ms'], ' ms')} |" + ) + rows.append( + f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} | {_fmt(native['mem_peak_median_gib'], ' GiB')} |" + ) rows.append("") return "\n".join(rows) From 67ed4093ceaf102ac7fe4b810c03a1fa5ab8b52a Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Mon, 27 Jul 2026 12:39:17 -0700 Subject: [PATCH 31/64] Update result to use refiner --- docs/source/_static/performance/sana_wm/perf-bf16-0727.md | 2 +- docs/source/models/sana_wm.rst | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/source/_static/performance/sana_wm/perf-bf16-0727.md b/docs/source/_static/performance/sana_wm/perf-bf16-0727.md index 8ceb9ae16..64ba5dca3 100644 --- a/docs/source/_static/performance/sana_wm/perf-bf16-0727.md +++ b/docs/source/_static/performance/sana_wm/perf-bf16-0727.md @@ -2,4 +2,4 @@ | device | official | flashdreams | | --- | ---: | ---: | -| GB300 | 33051.56 | 31640.88 | +| GB300 | 45823.92 | 42399.31 | diff --git a/docs/source/models/sana_wm.rst b/docs/source/models/sana_wm.rst index 8df606d75..a25299bda 100644 --- a/docs/source/models/sana_wm.rst +++ b/docs/source/models/sana_wm.rst @@ -130,8 +130,9 @@ Profiling benchmark Here is the BF16 profiling benchmark on post-load generation latency per generated clip for FlashDreams SANA-WM compared to the `official SANA-WM implementation `_ under -matched settings. On GB300, FlashDreams is about 4% faster than the official -implementation at median clip latency. +matched settings. On GB300, FlashDreams is about 7% faster than the official +implementation at median clip latency (full pipeline: Stage-1 DiT + LTX-2 +refiner, 10 measured runs). .. raw:: html @@ -146,7 +147,7 @@ implementation at median clip latency.

This chart shows post-load generation latency per generated clip in milliseconds for a 121-frame - Stage-1-only BF16 run. The measured row used an NVIDIA GB300. + full-pipeline BF16 run (Stage-1 DiT + LTX-2 refiner). The measured row used an NVIDIA GB300. Model construction, checkpoint loading, video writing, and frame dumps are outside the timing boundary. For the official SANA-WM implementation, see this instruction. From 3af585f1f48e45360d0d296c5cf5d53873a0856d Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Mon, 27 Jul 2026 13:27:00 -0700 Subject: [PATCH 32/64] Add some logging for debug --- integrations/sana/sana_wm/stage1_model.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index 4d44eab5f..3ad510d00 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -1158,12 +1158,22 @@ def _chunk_index_from_chunk_size( ) +_GDN_PATH_LOGGED = False + + def _use_fused_gdn(x: Tensor) -> bool: - return ( - x.is_cuda - and _fused_bigdn_func is not None - and os.environ.get("SANA_WM_DISABLE_FUSED_GDN", "0") != "1" - ) + available = _fused_bigdn_func is not None + disabled_by_env = os.environ.get("SANA_WM_DISABLE_FUSED_GDN", "0") == "1" + use = x.is_cuda and available and not disabled_by_env + global _GDN_PATH_LOGGED + if not _GDN_PATH_LOGGED and x.is_cuda: + _GDN_PATH_LOGGED = True + print( + f"[sana-wm] GDN path={'fused' if use else 'eager'} " + f"(fused_available={available}, disabled_by_env={disabled_by_env})", + flush=True, + ) + return use def _prepare_camera_projection_cache( From bbfec5c114ba3374735f1f1b0e8b8bd29adb9d38 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Mon, 27 Jul 2026 13:49:31 -0700 Subject: [PATCH 33/64] Clean old bench runs --- integrations/sana/tests/parity_check/bench.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index 10df4260c..c6f1a105f 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -147,6 +147,11 @@ fi mkdir -p "${UPSTREAM_ROOT}" "${NATIVE_ROOT}" +# Remove stale run_* dirs from prior (possibly longer) benchmarks. bench_summary +# reads every run_* dir, so leftovers from an earlier run with more MEASURED_RUNS +# would silently pollute this invocation's aggregate. +rm -rf "${UPSTREAM_ROOT:?}"/run_* "${NATIVE_ROOT:?}"/run_* + UPSTREAM_REFINER_ARGS=() NATIVE_REFINER_ARGS=() if _is_true "${NO_REFINER}"; then From 51767eada49ddec5d6aac0b7a4cc80e3c517ebbb Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Mon, 27 Jul 2026 16:42:36 -0700 Subject: [PATCH 34/64] Strip out upstream fused kernels --- .../sana/sana_wm/ops/fused_cam_gdn.py | 2448 ----------------- integrations/sana/sana_wm/ops/fused_gdn.py | 2249 --------------- .../sana/sana_wm/ops/fused_gdn_chunkwise.py | 2269 --------------- integrations/sana/sana_wm/stage1_model.py | 312 +-- 4 files changed, 39 insertions(+), 7239 deletions(-) delete mode 100644 integrations/sana/sana_wm/ops/fused_cam_gdn.py delete mode 100644 integrations/sana/sana_wm/ops/fused_gdn.py delete mode 100644 integrations/sana/sana_wm/ops/fused_gdn_chunkwise.py diff --git a/integrations/sana/sana_wm/ops/fused_cam_gdn.py b/integrations/sana/sana_wm/ops/fused_cam_gdn.py deleted file mode 100644 index 6f6229637..000000000 --- a/integrations/sana/sana_wm/ops/fused_cam_gdn.py +++ /dev/null @@ -1,2448 +0,0 @@ -"""Triton-fused camera-branch UCPE single-path delta rule. - -Companion to :mod:`diffusion.model.ops.fused_gdn` (main GDN -branch). This module fuses the *camera* branch of -:class:`ChunkCausalGDNUCPESinglePathLiteLA` through two Triton kernels: - -1. ``_cam_prep_kernel`` — fuses, per ``(batch, token, head)``, the RMSNorm - over the full ``C`` channels, ReLU on Q/K, K-scale on K, the UCPE 4x4 - block-diagonal projection matrix on the first ``D/2`` dims, and the - interleaved-pair complex RoPE on the second ``D/2`` dims. Q, K, V are - processed in one pass. The kernel also emits per-token pre-UCPE and - post-UCPE ``||k||^2`` so the caller can compute the inflation-squared - factor used for Dynamic Beta Discounting. - -2. ``_cam_scan_kernel`` — fuses the numerator-only single-path delta-rule - scan per ``(batch, head)``. ``REVERSE=1`` implements the - ``flip_and_shift`` backward pass semantics directly, avoiding the - torch-side flips in the per-chunk backward loop. - -The prep and scan kernels are runtime paths. Torch implementations below are -kept only for fallback backward paths and focused validation. - -Notes: - - V skips RMSNorm / ReLU / K-scale but receives the same UCPE 4x4 + - RoPE transforms as K (apply_fn_kv in reference). - - The short convolution on K and the inverse UCPE output transform - (``apply_fn_o``) stay in PyTorch — they are single lightweight ops - not on the critical path. -""" - -# ruff: noqa: E501 - -from __future__ import annotations - -import os - -import torch -import triton -import triton.language as tl - -from .fused_gdn_chunkwise import cam_scan_chunkwise - -# ============================================================================= -# Scalar helpers -# ============================================================================= - - -def _invert_SE3(transforms: torch.Tensor) -> torch.Tensor: - """Invert a 4x4 SE(3) matrix batch (closed-form). - - Mirrors the reference ``_invert_SE3`` in ``sana_camctrl_blocks.py``; - inlined to keep this module dependency-light. - """ - assert transforms.shape[-2:] == (4, 4) - Rinv = transforms[..., :3, :3].transpose(-1, -2) - out = torch.zeros_like(transforms) - out[..., :3, :3] = Rinv - out[..., :3, 3] = -torch.einsum("...ij,...j->...i", Rinv, transforms[..., :3, 3]) - out[..., 3, 3] = 1.0 - return out - - -def _process_camera_conditions_raymats_only( - camera_conditions: torch.Tensor, - B: int, - HW: tuple[int, int, int], - patch_size: tuple[int, int, int], -) -> torch.Tensor: - """Lightweight variant of ``_process_camera_conditions_ucpe`` — raymats only. - - Computes *only* the per-ray ``world -> ray_local`` SE(3) transforms used - by UCPE single-path. Skips the ``compute_up_lat_map`` path (absmap) that - the cam branch never consumes — that saves ~1 ms per block on H100. - - Args: - camera_conditions: ``(B, F, 20)`` — ``[c2w_16 | fx | fy | cx | cy]``. - B: Batch size (redundant with ``camera_conditions.shape[0]``; kept - for parity with the reference signature). - HW: ``(T_latent, H_latent, W_latent)`` from the caller. - patch_size: ``(pt, ph, pw)`` patch embedding stride. - - Returns: - ``raymats`` of shape ``(B, F, H_latent, W_latent, 4, 4)``. - """ - F_dim = camera_conditions.shape[1] - c2w_flat = camera_conditions[..., :16] - C_to_W = c2w_flat.view(B, F_dim, 4, 4) - - fx = camera_conditions[..., 16] - fy = camera_conditions[..., 17] - cx = camera_conditions[..., 18] - cy = camera_conditions[..., 19] - H_dim, W_dim = HW[1], HW[2] - image_width = W_dim * patch_size[2] - image_height = H_dim * patch_size[1] - - xi = torch.zeros( - (B, F_dim), - device=camera_conditions.device, - dtype=camera_conditions.dtype, - ) - x_fov = compute_fov_from_fx_xi( - fx, - xi, - image_width, - device=camera_conditions.device, - dtype=camera_conditions.dtype, - ).view(B, F_dim) - y_fov = compute_fov_from_fx_xi( - fy, - xi, - image_height, - device=camera_conditions.device, - dtype=camera_conditions.dtype, - ).view(B, F_dim) - - d_cam = ucm_unproject_grid_fov( - x_fov, - y_fov, - xi, - H_dim, - W_dim, - cx / patch_size[2], - cy / patch_size[1], - device=camera_conditions.device, - dtype=camera_conditions.dtype, - ) - if d_cam.ndim == 4 and d_cam.shape[0] == B * F_dim: - d_cam = d_cam.view(B, F_dim, H_dim, W_dim, 3) - - return world_to_ray_mats(d_cam, C_to_W) # (B, F, H, W, 4, 4) - - -def _precompute_cam_inv_rms(raw: torch.Tensor, eps: float) -> torch.Tensor: - """Compute ``1/RMS`` per ``(b, n)`` over full-``C`` channels. - - Args: - raw: ``(B, N, H, D)`` raw QKV projection output (typically fp32). - eps: RMSNorm epsilon. - - Returns: - ``inv_rms`` of shape ``(B, N)`` in fp32, contiguous. - """ - B, N, H, D = raw.shape - C = H * D - sq_sum = (raw.float() * raw.float()).sum(dim=(-1, -2)) # (B, N) - return torch.rsqrt(sq_sum / C + eps).contiguous() - - -def _prepare_ucpe_rope_tables( - rotary_emb_cam: torch.Tensor, - N: int, - D_half: int, - device: torch.device, -) -> tuple[torch.Tensor, torch.Tensor]: - """Convert complex RoPE ``(1, 1, N, D_half//2)`` to interleaved ``(N, D_half)`` cos/sin. - - Uses the interleaved-pair convention: - y[2i] = x[2i]*cos[i] - x[2i+1]*sin[i] - y[2i+1] = x[2i]*sin[i] + x[2i+1]*cos[i] - encoded as ``y[d] = x[d]*cos_exp[d] + x[d^1]*sin_exp[d]`` with - sin_exp[2i] = -sin[i], sin_exp[2i+1] = +sin[i]. - """ - del device # all outputs inherit device from freqs - freqs = rotary_emb_cam.squeeze(0).squeeze(0) # (N, D_half//2) complex - cos_half = freqs.real.float() - sin_half = freqs.imag.float() - rope_cos = cos_half.repeat_interleave(2, dim=-1).contiguous() - rope_sin = torch.stack([-sin_half, sin_half], dim=-1).reshape(N, D_half).contiguous() - return rope_cos, rope_sin - - -# ============================================================================= -# Triton kernels -# ============================================================================= - - -_DEFAULT_BLOCK_S = 64 - - -@triton.jit -def _cam_prep_kernel( - q_raw_ptr, # (B, N, H, D) contiguous, any fp dtype - k_raw_ptr, # (B, N, H, D) contiguous (post short-conv on K) - v_raw_ptr, # (B, N, H, D) contiguous - q_inv_rms_ptr, # (B, N) float32 — precomputed over full C channels - k_inv_rms_ptr, # (B, N) float32 - q_norm_w_ptr, # (C,) = (H*D,) float32 - k_norm_w_ptr, # (C,) float32 - proj_q_ptr, # (B, N, 4, 4) — applied to Q first D/2 dims (P_T) - proj_kv_ptr, # (B, N, 4, 4) — applied to K,V first D/2 dims (P_inv) - rope_cos_ptr, # (N, D_rope) float32, D_rope = D//2 - rope_sin_ptr, # (N, D_rope) float32 - # --- outputs in (B, H, D, N) layout, same strides pattern --- - q_out_ptr, - k_out_ptr, - v_out_ptr, - k_pre_norm_sq_ptr, # (B, H, N) float32 — ||k_pre_ucpe||^2 - k_post_norm_sq_ptr, # (B, H, N) float32 — ||k_post_ucpe||^2 - # --- dims --- - H: tl.constexpr, - N: tl.constexpr, - D: tl.constexpr, # head dim - D_HALF: tl.constexpr, # D // 2 - N_GROUPS: tl.constexpr, # D_HALF // 4 - K_SCALE, - # --- tile sizes --- - BLOCK_D_ROPE: tl.constexpr, # next pow2 of D_HALF (rope block) - BLOCK_GROUPS: tl.constexpr, # next pow2 of N_GROUPS -): - """One program per (b, n, h) — processes a single (Q, K, V) head slice. - - Loads the first D_HALF dims as a (N_GROUPS, 4) tile (for the UCPE - block-diagonal 4x4 projmat), and the second D_HALF dims as a - (D_HALF,) vector (for RoPE). No redundant loads. - """ - pid = tl.program_id(0) - h_idx = pid % H - bn_idx = pid // H - b_idx = bn_idx // N - n_idx = bn_idx % N - - # layout (B, N, H, D) contiguous - row_base = b_idx * (N * H * D) + n_idx * (H * D) + h_idx * D - nw_off = h_idx * D - - # ---- load inv-RMS (scalar, shared across heads for this token) ---- - q_inv_rms = tl.load(q_inv_rms_ptr + bn_idx).to(tl.float32) - k_inv_rms = tl.load(k_inv_rms_ptr + bn_idx).to(tl.float32) - - # ---- load per-token P matrices (4,4) shared across heads ---- - proj_base = (b_idx * N + n_idx) * 16 - offs_i = tl.arange(0, 4) - offs_j = tl.arange(0, 4) - P_q = tl.load(proj_q_ptr + proj_base + offs_i[:, None] * 4 + offs_j[None, :]).to(tl.float32) - P_kv = tl.load(proj_kv_ptr + proj_base + offs_i[:, None] * 4 + offs_j[None, :]).to(tl.float32) - - # ================================================================== - # Pass 1 — UCPE block-diagonal projmat on first D_HALF dims - # ================================================================== - offs_g = tl.arange(0, BLOCK_GROUPS) - mask_g = offs_g < N_GROUPS - offs_gj = offs_g[:, None] * 4 + offs_j[None, :] # (BLOCK_GROUPS, 4) - mask_gj = mask_g[:, None] - - q_half = tl.load(q_raw_ptr + row_base + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) - k_half = tl.load(k_raw_ptr + row_base + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) - v_half = tl.load(v_raw_ptr + row_base + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) - - q_nw_half = tl.load(q_norm_w_ptr + nw_off + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) - k_nw_half = tl.load(k_norm_w_ptr + nw_off + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) - - q_half = q_half * q_inv_rms * q_nw_half - q_half = tl.where(q_half > 0, q_half, 0.0) - - k_half = k_half * k_inv_rms * k_nw_half - k_half = tl.where(k_half > 0, k_half, 0.0) * K_SCALE - - # Pre-UCPE ||k||^2 contribution from first half - k_half_masked = tl.where(mask_gj, k_half, 0.0) - k_pre_half_sq = tl.sum(k_half_masked * k_half_masked) - - # Apply 4x4 projmat: out[g, i] = sum_j P[i, j] * in[g, j] - # (BLOCK_GROUPS, 1, 4) * (1, 4, 4) -> (BLOCK_GROUPS, 4, 4), sum axis=-1 - q_half_out = tl.sum(q_half[:, None, :] * P_q[None, :, :], axis=-1) - k_half_out = tl.sum(k_half[:, None, :] * P_kv[None, :, :], axis=-1) - v_half_out = tl.sum(v_half[:, None, :] * P_kv[None, :, :], axis=-1) - - # Post-UCPE ||k||^2 contribution from first half - k_half_out_masked = tl.where(mask_gj, k_half_out, 0.0) - k_post_half_sq = tl.sum(k_half_out_masked * k_half_out_masked) - - # ================================================================== - # Pass 2 — RoPE on second D_HALF dims - # ================================================================== - offs_r = tl.arange(0, BLOCK_D_ROPE) - mask_r = offs_r < D_HALF - offs_r_pair = offs_r ^ 1 - mask_r_pair = offs_r_pair < D_HALF - - rope_row = n_idx * D_HALF - cos_v = tl.load(rope_cos_ptr + rope_row + offs_r, mask=mask_r, other=1.0).to(tl.float32) - sin_v = tl.load(rope_sin_ptr + rope_row + offs_r, mask=mask_r, other=0.0).to(tl.float32) - - # Load second-half raw values and their pair partners - rope_base = row_base + D_HALF - q_r = tl.load(q_raw_ptr + rope_base + offs_r, mask=mask_r, other=0.0).to(tl.float32) - k_r = tl.load(k_raw_ptr + rope_base + offs_r, mask=mask_r, other=0.0).to(tl.float32) - v_r = tl.load(v_raw_ptr + rope_base + offs_r, mask=mask_r, other=0.0).to(tl.float32) - q_r_pair = tl.load(q_raw_ptr + rope_base + offs_r_pair, mask=mask_r_pair, other=0.0).to(tl.float32) - k_r_pair = tl.load(k_raw_ptr + rope_base + offs_r_pair, mask=mask_r_pair, other=0.0).to(tl.float32) - v_r_pair = tl.load(v_raw_ptr + rope_base + offs_r_pair, mask=mask_r_pair, other=0.0).to(tl.float32) - - q_nw_r = tl.load(q_norm_w_ptr + nw_off + D_HALF + offs_r, mask=mask_r, other=0.0).to(tl.float32) - k_nw_r = tl.load(k_norm_w_ptr + nw_off + D_HALF + offs_r, mask=mask_r, other=0.0).to(tl.float32) - q_nw_r_pair = tl.load(q_norm_w_ptr + nw_off + D_HALF + offs_r_pair, mask=mask_r_pair, other=0.0).to(tl.float32) - k_nw_r_pair = tl.load(k_norm_w_ptr + nw_off + D_HALF + offs_r_pair, mask=mask_r_pair, other=0.0).to(tl.float32) - - q_r_n = q_r * q_inv_rms * q_nw_r - q_r_n = tl.where(q_r_n > 0, q_r_n, 0.0) - q_r_pair_n = q_r_pair * q_inv_rms * q_nw_r_pair - q_r_pair_n = tl.where(q_r_pair_n > 0, q_r_pair_n, 0.0) - - k_r_n = k_r * k_inv_rms * k_nw_r - k_r_n = tl.where(k_r_n > 0, k_r_n, 0.0) * K_SCALE - k_r_pair_n = k_r_pair * k_inv_rms * k_nw_r_pair - k_r_pair_n = tl.where(k_r_pair_n > 0, k_r_pair_n, 0.0) * K_SCALE - - # Pre-UCPE ||k||^2 contribution from second half (using post-ReLU/scale k_r_n) - k_r_n_masked = tl.where(mask_r, k_r_n, 0.0) - k_pre_rope_sq = tl.sum(k_r_n_masked * k_r_n_masked) - - q_rope_out = q_r_n * cos_v + q_r_pair_n * sin_v - k_rope_out = k_r_n * cos_v + k_r_pair_n * sin_v - v_rope_out = v_r * cos_v + v_r_pair * sin_v - - # Post-UCPE ||k||^2 contribution from second half - k_rope_masked = tl.where(mask_r, k_rope_out, 0.0) - k_post_rope_sq = tl.sum(k_rope_masked * k_rope_masked) - - # Store scalar per-token norm squares - norm_out_idx = (b_idx * H + h_idx) * N + n_idx - tl.store(k_pre_norm_sq_ptr + norm_out_idx, k_pre_half_sq + k_pre_rope_sq) - tl.store(k_post_norm_sq_ptr + norm_out_idx, k_post_half_sq + k_post_rope_sq) - - # ================================================================== - # Store outputs in (B, H, D, N) layout: ptr[b, h, d, n] = base_bh + d*N + n - # ================================================================== - out_base = b_idx * (H * D * N) + h_idx * (D * N) + n_idx - - # First half: d = g*4 + i, write at out_base + d*N (strided by N). - offs_d_half = offs_g[:, None] * 4 + offs_i[None, :] # (BLOCK_GROUPS, 4) - mask_d_half = mask_g[:, None] - tl.store(q_out_ptr + out_base + offs_d_half * N, q_half_out, mask=mask_d_half) - tl.store(k_out_ptr + out_base + offs_d_half * N, k_half_out, mask=mask_d_half) - tl.store(v_out_ptr + out_base + offs_d_half * N, v_half_out, mask=mask_d_half) - - # Second half (RoPE region): d = D_HALF + r - offs_d_r = D_HALF + offs_r # (BLOCK_D_ROPE,) - tl.store(q_out_ptr + out_base + offs_d_r * N, q_rope_out, mask=mask_r) - tl.store(k_out_ptr + out_base + offs_d_r * N, k_rope_out, mask=mask_r) - tl.store(v_out_ptr + out_base + offs_d_r * N, v_rope_out, mask=mask_r) - - -@triton.jit -def _cam_prep_bwd_kernel( - # --- forward inputs (replayed for ReLU mask + k_post_kscale recompute) --- - q_raw_ptr, # (B, N, H, D) contiguous, any fp dtype - k_raw_ptr, # (B, N, H, D) contiguous (post short-conv on K) - q_norm_w_ptr, # (C,) = (H*D,) float32 - k_norm_w_ptr, # (C,) float32 - q_inv_rms_ptr, # (B, N) float32 — saved from forward - k_inv_rms_ptr, # (B, N) float32 - proj_q_ptr, # (B, N, 4, 4) — applied to Q first D/2 dims (P_T) - proj_kv_ptr, # (B, N, 4, 4) — applied to K,V first D/2 dims (P_inv) - rope_cos_ptr, # (N, D_rope) float32, D_rope = D//2 - rope_sin_ptr, # (N, D_rope) float32 - # --- upstream gradients (B, H, D, N) layout matching forward outputs --- - d_q_out_ptr, # grad of q_out (any dtype, cast to fp32 on load) - eff_d_k_out_ptr, # grad of k_out + inflation_sq contribution through k_out (fp32) - d_v_out_ptr, # grad of v_out - # --- inflation_sq direct grad to k_post_kscale^2 sum (B, H, N) fp32 --- - d_pre_k_sq_ptr, - # --- outputs --- - d_q_post_norm_ptr, # (B, N, H, D) fp32 — grad after RoPE^T+UCPE^T+Kscale+ReLU; consumed by torch RMSNorm bwd - d_k_post_norm_ptr, # (B, N, H, D) fp32 — same for K - dv_raw_ptr, # (B, N, H, D) fp32 — final dv_raw (V skips norm/ReLU/Kscale) - # --- dims --- - H: tl.constexpr, - N: tl.constexpr, - D: tl.constexpr, - D_HALF: tl.constexpr, - N_GROUPS: tl.constexpr, - K_SCALE, - # --- tile sizes --- - BLOCK_D_ROPE: tl.constexpr, # next pow2 of D_HALF (rope block) - BLOCK_GROUPS: tl.constexpr, # next pow2 of N_GROUPS -): - """One program per (b, n, h) — matches the forward kernel's parallelism. - - Implements the bwd of the fused fwd: - forward order: RMSNorm -> ReLU -> [K-scale on K] -> UCPE (first D/2) - -> RoPE (second D/2) -> output (B, H, D, N). - backward order: RoPE^T (second D/2) -> UCPE^T (first D/2) - -> K-scale (only K) -> ReLU mask -> emit d_post_norm - intermediates for the cross-head RMSNorm bwd handled - outside (in :func:`_cam_prep_bwd_dispatch`). - - The full-channel RMSNorm bwd's outer-product term and per-channel weight - grad both require a sum over ``H*D`` (the full ``C``) per token, which - couples heads. We deliberately leave that step in PyTorch (a couple of - fused element-wise + reduction ops) — see :func:`_cam_prep_bwd_dispatch`. - - Inflation handling: the kernel takes an *effective* ``dO_k`` that already - includes the contribution from ``grad_inflation_sq`` flowing through - ``k_out`` (i.e. ``eff_dO_k = grad_k + 2 * k_out * d_post_k_sq``), plus a - per-(b, h, n) scalar ``d_pre_k_sq`` that is the chain-rule contribution - of ``grad_inflation_sq`` into the pre-UCPE ``||k_post_kscale||^2`` sum. - Inside the kernel we recompute ``k_post_kscale`` (= post-norm * ReLU * - K_SCALE) and add ``2 * k_post_kscale[d] * d_pre_k_sq`` as a direct - contribution to ``d_k_post_kscale``. - """ - pid = tl.program_id(0) - h_idx = pid % H - bn_idx = pid // H - b_idx = bn_idx // N - n_idx = bn_idx % N - - # ---- load saved scalars: inv-RMS (per b, n) and d_pre_k_sq (per b, h, n) ---- - q_inv_rms = tl.load(q_inv_rms_ptr + bn_idx).to(tl.float32) - k_inv_rms = tl.load(k_inv_rms_ptr + bn_idx).to(tl.float32) - bhn_idx = (b_idx * H + h_idx) * N + n_idx - d_pre_k_sq = tl.load(d_pre_k_sq_ptr + bhn_idx).to(tl.float32) - - # ---- load per-token P matrices (shared across heads) ---- - proj_base = (b_idx * N + n_idx) * 16 - offs_i = tl.arange(0, 4) - offs_j = tl.arange(0, 4) - P_q = tl.load(proj_q_ptr + proj_base + offs_i[:, None] * 4 + offs_j[None, :]).to(tl.float32) - P_kv = tl.load(proj_kv_ptr + proj_base + offs_i[:, None] * 4 + offs_j[None, :]).to(tl.float32) - - # ---- layout offsets ---- - row_base = b_idx * (N * H * D) + n_idx * (H * D) + h_idx * D # (B, N, H, D) - nw_off = h_idx * D - out_base_BHDN = b_idx * (H * D * N) + h_idx * (D * N) + n_idx # (B, H, D, N) for dO_* - norm_base = row_base # d_*_post_norm and dv_raw share the (B, N, H, D) layout - - # ============================================================ - # First half — UCPE region - # ============================================================ - offs_g = tl.arange(0, BLOCK_GROUPS) - mask_g = offs_g < N_GROUPS - offs_gj = offs_g[:, None] * 4 + offs_j[None, :] # (BLOCK_GROUPS, 4) - mask_gj = mask_g[:, None] - - # Recompute post-norm (pre-ReLU) Q/K for the ReLU mask + k_post_kscale. - q_half_raw = tl.load(q_raw_ptr + row_base + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) - k_half_raw = tl.load(k_raw_ptr + row_base + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) - q_nw_half = tl.load(q_norm_w_ptr + nw_off + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) - k_nw_half = tl.load(k_norm_w_ptr + nw_off + offs_gj, mask=mask_gj, other=0.0).to(tl.float32) - - q_post_norm_half = q_half_raw * q_inv_rms * q_nw_half - k_post_norm_half = k_half_raw * k_inv_rms * k_nw_half - q_relu_mask_half = q_post_norm_half > 0 - k_relu_mask_half = k_post_norm_half > 0 - - # k_post_kscale = relu(k_post_norm) * K_SCALE (used for direct inflation contribution) - k_post_relu_half = tl.where(k_relu_mask_half, k_post_norm_half, 0.0) - k_post_kscale_half = k_post_relu_half * K_SCALE - - # Load upstream gradients for first half. - # In (B, H, D, N): ptr[b, h, d, n] = out_base_BHDN + d * N. d = g*4 + i. - offs_d_half = offs_g[:, None] * 4 + offs_i[None, :] # (BLOCK_GROUPS, 4) - mask_d_half = mask_g[:, None] - dO_q_half = tl.load(d_q_out_ptr + out_base_BHDN + offs_d_half * N, mask=mask_d_half, other=0.0).to(tl.float32) - dO_k_eff_half = tl.load(eff_d_k_out_ptr + out_base_BHDN + offs_d_half * N, mask=mask_d_half, other=0.0).to( - tl.float32 - ) - dO_v_half = tl.load(d_v_out_ptr + out_base_BHDN + offs_d_half * N, mask=mask_d_half, other=0.0).to(tl.float32) - - # UCPE^T: din[g, j] = sum_i P[i, j] * dout[g, i] - # forward: out[g, i] = sum_j q[g, j] * P[i, j] -- so bwd sums over i - d_q_post_relu_half = tl.sum(dO_q_half[:, :, None] * P_q[None, :, :], axis=1) - d_k_post_kscale_via_ucpe_half = tl.sum(dO_k_eff_half[:, :, None] * P_kv[None, :, :], axis=1) - d_v_first_half = tl.sum(dO_v_half[:, :, None] * P_kv[None, :, :], axis=1) - - # K direct inflation contribution: 2 * k_post_kscale * d_pre_k_sq - d_k_post_kscale_half = d_k_post_kscale_via_ucpe_half + 2.0 * k_post_kscale_half * d_pre_k_sq - - # K-scale bwd (multiply by K_SCALE) - d_k_post_relu_half = d_k_post_kscale_half * K_SCALE - - # ReLU mask - d_q_post_norm_half = tl.where(q_relu_mask_half, d_q_post_relu_half, 0.0) - d_k_post_norm_half = tl.where(k_relu_mask_half, d_k_post_relu_half, 0.0) - - # Mask out-of-bounds groups to 0 explicitly (for safety on uneven N_GROUPS). - d_q_post_norm_half = tl.where(mask_d_half, d_q_post_norm_half, 0.0) - d_k_post_norm_half = tl.where(mask_d_half, d_k_post_norm_half, 0.0) - d_v_first_half = tl.where(mask_d_half, d_v_first_half, 0.0) - - # Store at (B, N, H, D), d = g*4+i - tl.store(d_q_post_norm_ptr + norm_base + offs_d_half, d_q_post_norm_half, mask=mask_d_half) - tl.store(d_k_post_norm_ptr + norm_base + offs_d_half, d_k_post_norm_half, mask=mask_d_half) - tl.store(dv_raw_ptr + norm_base + offs_d_half, d_v_first_half, mask=mask_d_half) - - # ============================================================ - # Second half — RoPE region - # ============================================================ - offs_r = tl.arange(0, BLOCK_D_ROPE) - mask_r = offs_r < D_HALF - offs_r_pair = offs_r ^ 1 - mask_r_pair = offs_r_pair < D_HALF - - rope_row = n_idx * D_HALF - cos_v = tl.load(rope_cos_ptr + rope_row + offs_r, mask=mask_r, other=1.0).to(tl.float32) - sin_v_pair = tl.load(rope_sin_ptr + rope_row + offs_r_pair, mask=mask_r_pair, other=0.0).to(tl.float32) - - # Recompute post-norm (pre-ReLU) Q/K for the ReLU mask. - rope_base = row_base + D_HALF - q_r_raw = tl.load(q_raw_ptr + rope_base + offs_r, mask=mask_r, other=0.0).to(tl.float32) - k_r_raw = tl.load(k_raw_ptr + rope_base + offs_r, mask=mask_r, other=0.0).to(tl.float32) - q_nw_r = tl.load(q_norm_w_ptr + nw_off + D_HALF + offs_r, mask=mask_r, other=0.0).to(tl.float32) - k_nw_r = tl.load(k_norm_w_ptr + nw_off + D_HALF + offs_r, mask=mask_r, other=0.0).to(tl.float32) - - q_post_norm_r = q_r_raw * q_inv_rms * q_nw_r - k_post_norm_r = k_r_raw * k_inv_rms * k_nw_r - q_relu_mask_r = q_post_norm_r > 0 - k_relu_mask_r = k_post_norm_r > 0 - - k_post_relu_r = tl.where(k_relu_mask_r, k_post_norm_r, 0.0) - k_post_kscale_r = k_post_relu_r * K_SCALE - - # Load upstream gradients (second half) — direct + pair. - offs_d_r = D_HALF + offs_r - offs_d_r_pair = D_HALF + offs_r_pair - dO_q_r = tl.load(d_q_out_ptr + out_base_BHDN + offs_d_r * N, mask=mask_r, other=0.0).to(tl.float32) - dO_k_eff_r = tl.load(eff_d_k_out_ptr + out_base_BHDN + offs_d_r * N, mask=mask_r, other=0.0).to(tl.float32) - dO_v_r = tl.load(d_v_out_ptr + out_base_BHDN + offs_d_r * N, mask=mask_r, other=0.0).to(tl.float32) - dO_q_r_pair = tl.load(d_q_out_ptr + out_base_BHDN + offs_d_r_pair * N, mask=mask_r_pair, other=0.0).to(tl.float32) - dO_k_eff_r_pair = tl.load(eff_d_k_out_ptr + out_base_BHDN + offs_d_r_pair * N, mask=mask_r_pair, other=0.0).to( - tl.float32 - ) - dO_v_r_pair = tl.load(d_v_out_ptr + out_base_BHDN + offs_d_r_pair * N, mask=mask_r_pair, other=0.0).to(tl.float32) - - # RoPE^T: forward y[r] = x[r]*cos[r] + x[r^1]*sin[r] - # bwd dx[r] = dy[r]*cos[r] + dy[r^1]*sin[r^1] - d_q_post_relu_r = dO_q_r * cos_v + dO_q_r_pair * sin_v_pair - d_k_post_kscale_via_rope_r = dO_k_eff_r * cos_v + dO_k_eff_r_pair * sin_v_pair - d_v_second_r = dO_v_r * cos_v + dO_v_r_pair * sin_v_pair - - # K direct inflation contribution - d_k_post_kscale_r = d_k_post_kscale_via_rope_r + 2.0 * k_post_kscale_r * d_pre_k_sq - - # K-scale bwd - d_k_post_relu_r = d_k_post_kscale_r * K_SCALE - - # ReLU mask - d_q_post_norm_r = tl.where(q_relu_mask_r, d_q_post_relu_r, 0.0) - d_k_post_norm_r = tl.where(k_relu_mask_r, d_k_post_relu_r, 0.0) - - # Out-of-bound mask - d_q_post_norm_r = tl.where(mask_r, d_q_post_norm_r, 0.0) - d_k_post_norm_r = tl.where(mask_r, d_k_post_norm_r, 0.0) - d_v_second_r = tl.where(mask_r, d_v_second_r, 0.0) - - norm_offs_r = D_HALF + offs_r - tl.store(d_q_post_norm_ptr + norm_base + norm_offs_r, d_q_post_norm_r, mask=mask_r) - tl.store(d_k_post_norm_ptr + norm_base + norm_offs_r, d_k_post_norm_r, mask=mask_r) - tl.store(dv_raw_ptr + norm_base + norm_offs_r, d_v_second_r, mask=mask_r) - - -@triton.jit -def _cam_scan_kernel( - # --- inputs (B, H, D, N) contiguous, fp32 --- - q_ptr, - k_ptr, - v_ptr, - # --- gates --- - beta_ptr, # (B, H, F, S) contiguous - decay_ptr, # (B, H, F) contiguous - # --- output (B, H, D, N) fp32 --- - out_ptr, - # --- saved state snapshots (used when SAVE_STATES=1) --- - state_pre_ptr, # (B, H, F, BLOCK_D, BLOCK_D) fp32 — state after decay, before update - state_post_ptr, # (B, H, F, BLOCK_D, BLOCK_D) fp32 — state after update - # --- forward-direction cache state (used when LOAD_INIT_STATE / SAVE_FINAL_STATE) --- - init_state_ptr, # (B*H, BLOCK_D, BLOCK_D) fp32 — state at end of prefix - final_state_ptr, # (B*H, BLOCK_D, BLOCK_D) fp32 — state after last frame's update - # --- dims --- - H: tl.constexpr, - F: tl.constexpr, - S: tl.constexpr, - D: tl.constexpr, - N: tl.constexpr, # F * S - REVERSE: tl.constexpr, - SAVE_STATES: tl.constexpr, - LOAD_INIT_STATE: tl.constexpr, - SAVE_FINAL_STATE: tl.constexpr, - BLOCK_D: tl.constexpr, - BLOCK_S: tl.constexpr, -): - """One program per (b, h) — runs the full numerator-only delta-rule scan. - - When ``SAVE_STATES=1`` the kernel additionally writes per-frame snapshots - of ``state_prev`` (state after applying ``decay``, before the K@delta - update) to ``state_pre_ptr`` and ``state_curr`` (state after the update) - to ``state_post_ptr``, both indexed by ``q_frame``. The bwd kernel - (``_cam_scan_bwd_kernel``) consumes these snapshots for both - ``REVERSE=0`` and ``REVERSE=1``. In ``REVERSE=1`` the slot at - ``q_frame=F-1`` always holds the all-zero state (skip-update, decay=1 - on the zero initial state), and the bwd kernel reads exactly that — - no special-case load is needed. - - When ``LOAD_INIT_STATE=1`` (forward direction only — wrapper enforces - ``REVERSE=0``) the per-program ``state_curr`` is initialized from - ``init_state_ptr`` instead of zero. The convention is: the loaded value - is the state AT THE END of a prefix sequence (i.e., AFTER the prefix's - last update, BEFORE any further decay applied here). On the very first - frame, the kernel's own ``state_curr *= g`` then applies ``decay[0]`` - to this loaded state — which is exactly the decay that the global - sequence's f=K-th frame would have applied. This keeps split/resume - state trajectories identical from frame K onwards. - - When ``SAVE_FINAL_STATE=1`` (forward direction only) the final - ``state_curr`` (after the last frame's update) is written to - ``final_state_ptr``. This is the state to be loaded with - ``LOAD_INIT_STATE`` for a downstream segment. - """ - pid = tl.program_id(0) - pid_b = pid // H - pid_h = pid % H - bh = pid_b * H + pid_h - - offs_d = tl.arange(0, BLOCK_D) - mask_d = offs_d < D - offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] - mask_dd = mask_d[:, None] & mask_d[None, :] - - base_bh = pid_b * (H * D * N) + pid_h * (D * N) - q_bh = q_ptr + base_bh - k_bh = k_ptr + base_bh - v_bh = v_ptr + base_bh - out_bh = out_ptr + base_bh - beta_bh = beta_ptr + bh * F * S - decay_bh = decay_ptr + bh * F - if SAVE_STATES: - spre_bh = state_pre_ptr + bh * F * BLOCK_D * BLOCK_D - spost_bh = state_post_ptr + bh * F * BLOCK_D * BLOCK_D - - # State: (D_k, D_v) in the upstream convention. Here we call rows "k-dim" - # (input dim of state) and cols "v-dim" (output dim of state). - if LOAD_INIT_STATE: - init_bh = init_state_ptr + bh * BLOCK_D * BLOCK_D - state_curr = tl.load(init_bh + offs_dd, mask=mask_dd, other=0.0).to(tl.float32) - else: - state_curr = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) - - for f_iter in range(F): - if REVERSE: - q_frame = F - 1 - f_iter - kv_frame = F - f_iter if f_iter > 0 else 0 - skip_update = f_iter == 0 - else: - q_frame = f_iter - kv_frame = f_iter - skip_update = False - - if REVERSE and f_iter == 0: - g = 1.0 - else: - g = tl.load(decay_bh + kv_frame).to(tl.float32) - state_curr = state_curr * g - state_prev = state_curr # fp32 snapshot (same tensor, kept for clarity) - - if SAVE_STATES: - tl.store( - spre_bh + q_frame * BLOCK_D * BLOCK_D + offs_dd, - state_prev, - mask=mask_dd, - ) - - if skip_update == 0: - kv_n_base = kv_frame * S - f_beta = beta_bh + kv_frame * S - - for s0 in range(0, S, BLOCK_S): - offs_s = s0 + tl.arange(0, BLOCK_S) - mask_s = offs_s < S - mask_sd = mask_s[:, None] & mask_d[None, :] - n_idx = kv_n_base + offs_s - - # Load K, V tiles (BLOCK_S, BLOCK_D) from (B, H, D, N) layout: - # ptr[s, d] = base_bh + offs_d[d] * N + n_idx[s] - k_ptrs = k_bh + offs_d[None, :] * N + n_idx[:, None] - v_ptrs = v_bh + offs_d[None, :] * N + n_idx[:, None] - K = tl.load(k_ptrs, mask=mask_sd, other=0.0) - V = tl.load(v_ptrs, mask=mask_sd, other=0.0) - - bt = tl.load(f_beta + offs_s, mask=mask_s, other=0.0).to(tl.float32) - - # V_pred = K @ state_prev : (BLOCK_S, BLOCK_D) - V_pred = tl.dot( - K, - state_prev, - out_dtype=tl.float32, - input_precision="tf32", - ) - dv = (V - V_pred) * bt[:, None] - state_curr += tl.dot( - tl.trans(K), - dv, - out_dtype=tl.float32, - input_precision="tf32", - ) - - if SAVE_STATES: - tl.store( - spost_bh + q_frame * BLOCK_D * BLOCK_D + offs_dd, - state_curr, - mask=mask_dd, - ) - - # --- Pass 2: output --- - state_out = state_curr - q_n_base = q_frame * S - for s0 in range(0, S, BLOCK_S): - offs_s = s0 + tl.arange(0, BLOCK_S) - mask_s = offs_s < S - mask_sd = mask_s[:, None] & mask_d[None, :] - n_idx = q_n_base + offs_s - - q_ptrs = q_bh + offs_d[None, :] * N + n_idx[:, None] - Q = tl.load(q_ptrs, mask=mask_sd, other=0.0) - - # num = Q @ state_out : (BLOCK_S, BLOCK_D), rows=S, cols=D_v - num = tl.dot( - Q, - state_out, - out_dtype=tl.float32, - input_precision="tf32", - ) - - # Store transposed into (B, H, D, N): - # ptr[d, s] = out_bh + offs_d[d] * N + n_idx[s] - out_ptrs = out_bh + offs_d[:, None] * N + n_idx[None, :] - mask_ds = mask_d[:, None] & mask_s[None, :] - tl.store(out_ptrs, tl.trans(num), mask=mask_ds) - - if SAVE_FINAL_STATE: - final_bh = final_state_ptr + bh * BLOCK_D * BLOCK_D - tl.store(final_bh + offs_dd, state_curr, mask=mask_dd) - - -# ============================================================================= -# Backward Triton kernel -# ============================================================================= - - -@triton.jit -def _cam_scan_bwd_kernel( - # --- forward inputs (B, H, D, N) fp32 contiguous --- - q_ptr, - k_ptr, - v_ptr, - # --- gates --- - beta_ptr, # (B, H, F, S) fp32 - decay_ptr, # (B, H, F) fp32 - # --- saved state snapshots (B*H, F, BLOCK_D, BLOCK_D) fp32, indexed by q_frame --- - state_pre_ptr, # state after decay, before update - state_post_ptr, # state after update - # --- upstream gradient (B, H, D, N) fp32 --- - grad_out_ptr, - # --- output gradients --- - dq_ptr, # (B, H, D, N) fp32 - dk_ptr, - dv_ptr, - dbeta_ptr, # (B, H, F, S) fp32 - ddecay_ptr, # (B, H, F) fp32 - # --- dims --- - H: tl.constexpr, - F: tl.constexpr, - S: tl.constexpr, - D: tl.constexpr, - N: tl.constexpr, # F * S - REVERSE: tl.constexpr, - BLOCK_D: tl.constexpr, - BLOCK_S: tl.constexpr, -): - """Reverse-time backward of the numerator-only delta-rule scan. - - One program per ``(b, h)``. Walks the same ``f_iter`` index space as the - forward kernel — but in reverse time order — replaying the recurrence - using the per-``q_frame`` state snapshots saved by the forward pass with - ``SAVE_STATES=1``. Accumulates gradients into ``dq``, ``dk``, ``dv``, - ``dbeta``, ``ddecay``. - - Forward indexing (matched here exactly):: - - REVERSE=False: q_frame = kv_frame = f_iter, skip_update = False - REVERSE=True : q_frame = F-1-f_iter, - kv_frame = F-f_iter (skip when f_iter==0) - g = decay[kv_frame] (1.0 when f_iter==0) - - Per-iteration backward derivation (when ``not skip_update``): - - ds_post += Q.T @ d_out # accumulate output grad - dQ[q] = d_out @ s_post.T - - ddelta = K @ ds_post # via K.T @ delta term - dV[kv] = ddelta * beta[kv,:] - dbeta[kv,:] = sum_d (ddelta * (V - K @ s_pre)) - dV_pred = -ddelta * beta[kv,:] - dK[kv] = delta @ ds_post.T + dV_pred @ s_pre.T - ds_pre = ds_post + K.T @ dV_pred # direct + V_pred path - - ddecay[kv] = sum(ds_pre * s_pre[q]) / g (= 0 when state_in is 0) - ds_post[next-bwd-iter] = ds_pre * g # propagate through decay - - For the ``skip_update`` branch (``REVERSE=True`` and ``f_iter==0``) the - update / decay path is bypassed: ``state_post == state_pre == 0`` so - ``dQ == 0``, ``ds_post`` is left unchanged across the iter, and - no writes to ``dK``, ``dV``, ``dbeta`` or ``ddecay`` happen at - ``kv_frame == 0``. Caller MUST pre-zero those output buffers (the - dispatch wrapper uses ``torch.zeros_like``). - - The ``ddecay`` formula uses ``state_pre / g``; for ``REVERSE=False`` at - fwd frame 0 we hardcode ``ddecay[0] = 0`` (state_in is exactly 0, but - the division would amplify any rounding noise). For very small ``g`` - the existing ``+ 1e-12`` epsilon is matched verbatim from - ``_fused_gdn_bwd_kernel``. - """ - pid = tl.program_id(0) - pid_b = pid // H - pid_h = pid % H - bh = pid_b * H + pid_h - - base_bh = pid_b * (H * D * N) + pid_h * (D * N) - q_bh = q_ptr + base_bh - k_bh = k_ptr + base_bh - v_bh = v_ptr + base_bh - do_bh = grad_out_ptr + base_bh - dq_bh = dq_ptr + base_bh - dk_bh = dk_ptr + base_bh - dv_bh = dv_ptr + base_bh - - beta_bh = beta_ptr + bh * F * S - decay_bh = decay_ptr + bh * F - dbeta_bh = dbeta_ptr + bh * F * S - ddecay_bh = ddecay_ptr + bh * F - - spre_bh = state_pre_ptr + bh * F * BLOCK_D * BLOCK_D - spost_bh = state_post_ptr + bh * F * BLOCK_D * BLOCK_D - - offs_d = tl.arange(0, BLOCK_D) - mask_d = offs_d < D - offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] - mask_dd = mask_d[:, None] & mask_d[None, :] - - # bf16 operands for tl.dot keep shared-memory pressure manageable for large - # BLOCK_D (e.g., 128 in reference). fp32 accumulators preserve precision. - grad_dtype = tl.bfloat16 - grad_ip: tl.constexpr = "tf32" - - # Reverse-time accumulator: gradient w.r.t. ``state_post`` for the iter - # currently being processed. - ds_post = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) - - for f_rev in range(F): - # Walk fwd iters in reverse: F-1, F-2, ..., 0. - f_iter = F - 1 - f_rev - - if REVERSE: - q_frame = F - 1 - f_iter - kv_frame = F - f_iter if f_iter > 0 else 0 - skip_update = f_iter == 0 - else: - q_frame = f_iter - kv_frame = f_iter - skip_update = 0 - - if REVERSE and f_iter == 0: - g = 1.0 - else: - g = tl.load(decay_bh + kv_frame).to(tl.float32) - - # ---- Pass 2: dQ + ds_post += Q.T @ d_out ---------------------- - # Load state_post[q_frame] (zero in the REVERSE skip-update slot — - # the fwd save still writes the all-zero state at that index). - state = tl.load( - spost_bh + q_frame * BLOCK_D * BLOCK_D + offs_dd, - mask=mask_dd, - other=0.0, - ) - - q_n_base = q_frame * S - for s0 in range(0, S, BLOCK_S): - offs_s = s0 + tl.arange(0, BLOCK_S) - mask_s = offs_s < S - mask_sd = mask_s[:, None] & mask_d[None, :] - n_idx = q_n_base + offs_s - - q_ptrs = q_bh + offs_d[None, :] * N + n_idx[:, None] - Q = tl.load(q_ptrs, mask=mask_sd, other=0.0) - - do_ptrs = do_bh + offs_d[None, :] * N + n_idx[:, None] - dO = tl.load(do_ptrs, mask=mask_sd, other=0.0) - - ds_post += tl.dot( - tl.trans(Q.to(grad_dtype)), - dO.to(grad_dtype), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - - dQ = tl.dot( - dO.to(grad_dtype), - tl.trans(state.to(grad_dtype)), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - - dq_ptrs = dq_bh + offs_d[:, None] * N + n_idx[None, :] - mask_ds = mask_d[:, None] & mask_s[None, :] - tl.store(dq_ptrs, tl.trans(dQ), mask=mask_ds) - - if skip_update == 0: - # ---- Reload state with state_pre[q_frame] for Pass 1 ---- - state = tl.load( - spre_bh + q_frame * BLOCK_D * BLOCK_D + offs_dd, - mask=mask_dd, - other=0.0, - ) - - # ds_pre starts equal to ds_post (direct pass-through term). - ds_pre = ds_post - - kv_n_base = kv_frame * S - for s0 in range(0, S, BLOCK_S): - offs_s = s0 + tl.arange(0, BLOCK_S) - mask_s = offs_s < S - mask_sd = mask_s[:, None] & mask_d[None, :] - n_idx = kv_n_base + offs_s - - k_ptrs = k_bh + offs_d[None, :] * N + n_idx[:, None] - v_ptrs = v_bh + offs_d[None, :] * N + n_idx[:, None] - K = tl.load(k_ptrs, mask=mask_sd, other=0.0) - V = tl.load(v_ptrs, mask=mask_sd, other=0.0) - - bt = tl.load(beta_bh + kv_frame * S + offs_s, mask=mask_s, other=0.0).to(tl.float32) - - V_pred = tl.dot( - K.to(grad_dtype), - state.to(grad_dtype), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - r = V - V_pred - delta = r * bt[:, None] - - ddelta = tl.dot( - K.to(grad_dtype), - ds_post.to(grad_dtype), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - - dV = ddelta * bt[:, None] - dv_ptrs = dv_bh + offs_d[:, None] * N + n_idx[None, :] - mask_ds = mask_d[:, None] & mask_s[None, :] - tl.store(dv_ptrs, tl.trans(dV), mask=mask_ds) - - dbeta_st = tl.sum(ddelta * r, axis=1) - tl.store(dbeta_bh + kv_frame * S + offs_s, dbeta_st, mask=mask_s) - - dV_pred = -ddelta * bt[:, None] - - dK_part1 = tl.dot( - delta.to(grad_dtype), - tl.trans(ds_post.to(grad_dtype)), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - dK_part2 = tl.dot( - dV_pred.to(grad_dtype), - tl.trans(state.to(grad_dtype)), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - dK = dK_part1 + dK_part2 - dk_ptrs = dk_bh + offs_d[:, None] * N + n_idx[None, :] - tl.store(dk_ptrs, tl.trans(dK), mask=mask_ds) - - ds_pre += tl.dot( - tl.trans(K.to(grad_dtype)), - dV_pred.to(grad_dtype), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - - # ---- ddecay[kv_frame] ---- - # state_pre = state_in * g, so state_in = state_pre / g. - # ddecay = sum(ds_pre * state_in) = sum(ds_pre * state_pre) / g. - # For REVERSE=False fwd frame 0, state_in is exactly 0 — hardcode - # 0 to avoid amplifying rounding noise via 1/g. - if (REVERSE == 0) and (f_iter == 0): - ddecay_f = 0.0 - else: - inv_g = 1.0 / (g + 1e-12) - ddecay_f = tl.sum(ds_pre * state) * inv_g - tl.store(ddecay_bh + kv_frame, ddecay_f) - - # Propagate to next bwd iter (which is fwd's previous iter). - ds_post = ds_pre * g - # else (skip_update branch): state_post == state_pre == 0, no - # ddecay write, no kv-side writes; ds_post passes through unchanged - # since ∂state_post/∂state_in = I when the update is skipped and g=1. - # In REVERSE=True this is the LAST bwd iter (f_iter=0) so the - # carried-over ds_post is discarded. - - -# ============================================================================= -# Python wrappers -# ============================================================================= - - -def cam_prep_func( - q_raw: torch.Tensor, - k_raw: torch.Tensor, - v_raw: torch.Tensor, - *, - q_norm_weight: torch.Tensor, - k_norm_weight: torch.Tensor, - proj_q: torch.Tensor, # (B, N, 4, 4) - proj_kv: torch.Tensor, # (B, N, 4, 4) - rope_cos: torch.Tensor, # (N, D//2) - rope_sin: torch.Tensor, # (N, D//2) - k_scale: float, - norm_eps: float, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Fused RMSNorm + ReLU + (K-scale on K) + UCPE 4x4 + RoPE for the cam branch. - - Args: - q_raw, k_raw, v_raw: ``(B, N, H, D)`` contiguous (any fp dtype). - ``K`` must already have the short convolution applied. - q_norm_weight, k_norm_weight: ``(C,) = (H*D,)`` fp32. - proj_q, proj_kv: ``(B, N, 4, 4)`` fp32 (``P_T`` and ``P_inv`` in UCPE). - rope_cos, rope_sin: ``(N, D//2)`` fp32 interleaved-pair tables. - k_scale: ``(D^-0.5) * (S^-0.5)``. - norm_eps: RMSNorm epsilon. - - Returns: - q_trans, k_trans, v_trans: ``(B, H, D, N)`` same dtype as ``q_raw``. - inflation_sq: ``(B, H, N)`` fp32, ratio - ``(||k_post_ucpe|| / ||k_pre_ucpe||)^2`` per token/head. - """ - B, N, H, D = q_raw.shape - assert k_raw.shape == q_raw.shape and v_raw.shape == q_raw.shape - assert D % 2 == 0 and (D // 2) % 4 == 0, f"D={D} must be 2x and (D/2) % 4 == 0" - D_half = D // 2 - N_groups = D_half // 4 - - assert q_raw.is_contiguous() and k_raw.is_contiguous() and v_raw.is_contiguous() - assert proj_q.shape == (B, N, 4, 4) and proj_q.is_contiguous() - assert proj_kv.shape == (B, N, 4, 4) and proj_kv.is_contiguous() - assert rope_cos.shape == (N, D_half) and rope_cos.is_contiguous() - assert rope_sin.shape == (N, D_half) and rope_sin.is_contiguous() - assert q_norm_weight.numel() == H * D and q_norm_weight.dtype == torch.float32 - assert k_norm_weight.numel() == H * D and k_norm_weight.dtype == torch.float32 - - # Precompute inv-RMS over full C channels (shared across heads per token). - q_inv_rms = _precompute_cam_inv_rms(q_raw, norm_eps) - k_inv_rms = _precompute_cam_inv_rms(k_raw, norm_eps) - - out_dtype = q_raw.dtype - q_out = torch.empty(B, H, D, N, dtype=out_dtype, device=q_raw.device) - k_out = torch.empty(B, H, D, N, dtype=out_dtype, device=q_raw.device) - v_out = torch.empty(B, H, D, N, dtype=out_dtype, device=q_raw.device) - k_pre_sq = torch.empty(B, H, N, dtype=torch.float32, device=q_raw.device) - k_post_sq = torch.empty(B, H, N, dtype=torch.float32, device=q_raw.device) - - BLOCK_D_ROPE = triton.next_power_of_2(D_half) - BLOCK_GROUPS = triton.next_power_of_2(N_groups) - - grid = (B * N * H,) - _cam_prep_kernel[grid]( - q_raw, - k_raw, - v_raw, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - proj_q, - proj_kv, - rope_cos, - rope_sin, - q_out, - k_out, - v_out, - k_pre_sq, - k_post_sq, - H=H, - N=N, - D=D, - D_HALF=D_half, - N_GROUPS=N_groups, - K_SCALE=k_scale, - BLOCK_D_ROPE=BLOCK_D_ROPE, - BLOCK_GROUPS=BLOCK_GROUPS, - num_warps=1, - ) - # inflation_sq = (clamp(sqrt(post), 1e-6) / clamp(sqrt(pre), 1e-6))^2 - # = clamp(post, 1e-12) / clamp(pre, 1e-12) (equivalent). - inflation_sq = k_post_sq.clamp_min(1e-12) / k_pre_sq.clamp_min(1e-12) - return q_out, k_out, v_out, inflation_sq - - -def _run_cam_prep_fwd_save( - q_raw: torch.Tensor, - k_raw: torch.Tensor, - v_raw: torch.Tensor, - *, - q_norm_weight: torch.Tensor, - k_norm_weight: torch.Tensor, - proj_q: torch.Tensor, - proj_kv: torch.Tensor, - rope_cos: torch.Tensor, - rope_sin: torch.Tensor, - k_scale: float, - norm_eps: float, -) -> tuple[ - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, -]: - """Run :func:`cam_prep_func` while exposing intermediates needed by the bwd kernel. - - Mirrors :func:`cam_prep_func` exactly (same kernel launch, same outputs) - but additionally returns the per-token ``inv_rms`` for both Q and K, plus - the raw ``||k_pre_ucpe||^2`` and ``||k_post_ucpe||^2`` per ``(B, H, N)``. - These are required by :func:`_cam_prep_bwd_dispatch` to (a) replay the - ReLU/RMSNorm chain in fp32 without re-summing over the full ``C`` - channels and (b) chain ``grad_inflation_sq`` back through ``k_out`` and - the pre-UCPE ``||k||^2`` term with the correct ``clamp_min`` indicators. - - Returns: - ``(q_out, k_out, v_out, inflation_sq, q_inv_rms, k_inv_rms, - k_pre_sq, k_post_sq)``. The first four match :func:`cam_prep_func`; - the rest are fp32 contiguous saved-state tensors for the backward. - """ - B, N, H, D = q_raw.shape - assert k_raw.shape == q_raw.shape and v_raw.shape == q_raw.shape - assert D % 2 == 0 and (D // 2) % 4 == 0, f"D={D} must be 2x and (D/2) % 4 == 0" - D_half = D // 2 - N_groups = D_half // 4 - - assert q_raw.is_contiguous() and k_raw.is_contiguous() and v_raw.is_contiguous() - assert proj_q.shape == (B, N, 4, 4) and proj_q.is_contiguous() - assert proj_kv.shape == (B, N, 4, 4) and proj_kv.is_contiguous() - assert rope_cos.shape == (N, D_half) and rope_cos.is_contiguous() - assert rope_sin.shape == (N, D_half) and rope_sin.is_contiguous() - assert q_norm_weight.numel() == H * D and q_norm_weight.dtype == torch.float32 - assert k_norm_weight.numel() == H * D and k_norm_weight.dtype == torch.float32 - - q_inv_rms = _precompute_cam_inv_rms(q_raw, norm_eps) - k_inv_rms = _precompute_cam_inv_rms(k_raw, norm_eps) - - out_dtype = q_raw.dtype - q_out = torch.empty(B, H, D, N, dtype=out_dtype, device=q_raw.device) - k_out = torch.empty(B, H, D, N, dtype=out_dtype, device=q_raw.device) - v_out = torch.empty(B, H, D, N, dtype=out_dtype, device=q_raw.device) - k_pre_sq = torch.empty(B, H, N, dtype=torch.float32, device=q_raw.device) - k_post_sq = torch.empty(B, H, N, dtype=torch.float32, device=q_raw.device) - - BLOCK_D_ROPE = triton.next_power_of_2(D_half) - BLOCK_GROUPS = triton.next_power_of_2(N_groups) - - _cam_prep_kernel[(B * N * H,)]( - q_raw, - k_raw, - v_raw, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - proj_q, - proj_kv, - rope_cos, - rope_sin, - q_out, - k_out, - v_out, - k_pre_sq, - k_post_sq, - H=H, - N=N, - D=D, - D_HALF=D_half, - N_GROUPS=N_groups, - K_SCALE=k_scale, - BLOCK_D_ROPE=BLOCK_D_ROPE, - BLOCK_GROUPS=BLOCK_GROUPS, - num_warps=1, - ) - inflation_sq = k_post_sq.clamp_min(1e-12) / k_pre_sq.clamp_min(1e-12) - return q_out, k_out, v_out, inflation_sq, q_inv_rms, k_inv_rms, k_pre_sq, k_post_sq - - -def _cam_prep_bwd_dispatch( - q_raw: torch.Tensor, - k_raw: torch.Tensor, - q_norm_weight: torch.Tensor, - k_norm_weight: torch.Tensor, - proj_q: torch.Tensor, - proj_kv: torch.Tensor, - rope_cos: torch.Tensor, - rope_sin: torch.Tensor, - q_inv_rms: torch.Tensor, - k_inv_rms: torch.Tensor, - k_pre_sq: torch.Tensor, - k_post_sq: torch.Tensor, - k_out: torch.Tensor, - *, - grad_q: torch.Tensor | None, - grad_k: torch.Tensor | None, - grad_v: torch.Tensor | None, - grad_inflation_sq: torch.Tensor | None, - k_scale: float, -) -> tuple[ - torch.Tensor | None, - torch.Tensor | None, - torch.Tensor | None, - torch.Tensor | None, - torch.Tensor | None, -]: - """Hybrid Triton + torch backward for :func:`cam_prep_func`. - - Pipeline: - - 1. **(torch)** Chain ``grad_inflation_sq`` through ``k_post_sq`` and - ``k_pre_sq`` to produce - (a) ``eff_d_k_out = grad_k + 2 * k_out * d_post_k_sq`` and - (b) ``d_pre_k_sq`` — a per-(B, H, N) scalar fed into the kernel as a - direct contribution to ``d_k_post_kscale``. Both ``clamp_min(1e-12)`` - indicators are honored so the gradient is exactly 0 in the (rare) - saturating regime, matching :func:`_torch_cam_prep_reference`. - - 2. **(Triton)** Launch :func:`_cam_prep_bwd_kernel` per ``(b, n, h)`` - to apply RoPE^T, UCPE^T, K-scale^T and the ReLU mask. Emits - ``d_q_post_norm`` / ``d_k_post_norm`` ``(B, N, H, D)`` fp32 - intermediates (the post-norm pre-RMSNorm grad slots) and writes - ``dv_raw`` directly (V skips RMSNorm/ReLU/K-scale). - - 3. **(torch)** Apply the full-channel RMSNorm bwd. The cross-head - coupling means ``S_q[b, n] = sum_{h,d} d_q_post_norm * q_raw * - q_norm_w`` is reduced over ``H*D``; we then form - ``dq_raw = d_q_post_norm * inv_rms_q * q_norm_w - - inv_rms_q^3 / C * q_raw * S_q`` - elementwise, and ``dq_norm_weight[c] = sum_{b,n} - d_q_post_norm[b,n,c] * q_raw[b,n,c] * inv_rms_q[b,n]``. - - Returns: - ``(dq_raw, dk_raw, dv_raw, dq_norm_weight, dk_norm_weight)``. Each - slot is ``None`` if upstream did not request that grad — handled - by the caller via ``ctx.needs_input_grad``. ``dq_raw`` / ``dk_raw`` - / ``dv_raw`` are returned in the same dtype as ``q_raw``; - norm-weight grads are fp32 (matching the input dtype). - """ - B, N, H, D = q_raw.shape - assert k_raw.shape == q_raw.shape - assert q_raw.is_contiguous() and k_raw.is_contiguous() - assert q_inv_rms.shape == (B, N) and k_inv_rms.shape == (B, N) - assert k_pre_sq.shape == (B, H, N) and k_post_sq.shape == (B, H, N) - D_half = D // 2 - N_groups = D_half // 4 - C = H * D - - # ---- prepare inflation_sq grad chain (in fp32) ---- - eps_floor = 1e-12 - pre_clamped = k_pre_sq.clamp_min(eps_floor) - if grad_inflation_sq is not None: - gis = grad_inflation_sq.to(torch.float32) - post_clamped = k_post_sq.clamp_min(eps_floor) - pre_indicator = (k_pre_sq >= eps_floor).to(torch.float32) - post_indicator = (k_post_sq >= eps_floor).to(torch.float32) - # d(inflation_sq)/d(post_k_sq) = (1 / pre_clamped) * post_indicator - # d(inflation_sq)/d(pre_k_sq) = -post_clamped / pre_clamped^2 * pre_indicator - d_post_k_sq = (post_indicator / pre_clamped) * gis # (B, H, N) - d_pre_k_sq = (-post_clamped / (pre_clamped * pre_clamped) * pre_indicator) * gis # (B, H, N) - else: - d_post_k_sq = torch.zeros_like(k_pre_sq) - d_pre_k_sq = torch.zeros_like(k_pre_sq) - - # eff_d_k_out: (B, H, D, N) fp32, contiguous - if grad_k is None: - grad_k_f32 = torch.zeros((B, H, D, N), dtype=torch.float32, device=q_raw.device) - else: - grad_k_f32 = grad_k.to(torch.float32) - if grad_inflation_sq is not None: - # k_out: (B, H, D, N), d_post_k_sq: (B, H, N) → broadcast over D dim. - eff_d_k_out = (grad_k_f32 + 2.0 * k_out.to(torch.float32) * d_post_k_sq.unsqueeze(2)).contiguous() - else: - eff_d_k_out = grad_k_f32.contiguous() - d_pre_k_sq = d_pre_k_sq.contiguous() - - # grad_q / grad_v as fp32 (B, H, D, N) contiguous (zero-fill if absent) - if grad_q is None: - grad_q_f32 = torch.zeros((B, H, D, N), dtype=torch.float32, device=q_raw.device) - else: - grad_q_f32 = grad_q.to(torch.float32).contiguous() - if grad_v is None: - grad_v_f32 = torch.zeros((B, H, D, N), dtype=torch.float32, device=q_raw.device) - else: - grad_v_f32 = grad_v.to(torch.float32).contiguous() - - # ---- allocate outputs / intermediates ---- - d_q_post_norm = torch.empty((B, N, H, D), dtype=torch.float32, device=q_raw.device) - d_k_post_norm = torch.empty((B, N, H, D), dtype=torch.float32, device=q_raw.device) - dv_raw_f32 = torch.empty((B, N, H, D), dtype=torch.float32, device=q_raw.device) - - BLOCK_D_ROPE = triton.next_power_of_2(D_half) - BLOCK_GROUPS = triton.next_power_of_2(N_groups) - - _cam_prep_bwd_kernel[(B * N * H,)]( - q_raw, - k_raw, - q_norm_weight, - k_norm_weight, - q_inv_rms, - k_inv_rms, - proj_q, - proj_kv, - rope_cos, - rope_sin, - grad_q_f32, - eff_d_k_out, - grad_v_f32, - d_pre_k_sq, - d_q_post_norm, - d_k_post_norm, - dv_raw_f32, - H=H, - N=N, - D=D, - D_HALF=D_half, - N_GROUPS=N_groups, - K_SCALE=k_scale, - BLOCK_D_ROPE=BLOCK_D_ROPE, - BLOCK_GROUPS=BLOCK_GROUPS, - num_warps=1, - ) - - # ---- torch RMSNorm bwd over the saved post-norm grads ---- - # Cast raw inputs to fp32 for the cross-head reduction (matches kernel - # numerics — the kernel uses fp32 internally as well). - q_raw_f32 = q_raw.to(torch.float32) - k_raw_f32 = k_raw.to(torch.float32) - q_inv_rms_view = q_inv_rms.view(B, N, 1, 1) - k_inv_rms_view = k_inv_rms.view(B, N, 1, 1) - q_nw_view = q_norm_weight.view(1, 1, H, D) - k_nw_view = k_norm_weight.view(1, 1, H, D) - - # S_q[b, n] = sum_{h, d} d_q_post_norm[b, n, h, d] * q_raw[b, n, h, d] * q_norm_w[h, d] - weighted_q = d_q_post_norm * q_raw_f32 # reused for dq_norm_weight reduction - weighted_k = d_k_post_norm * k_raw_f32 - S_q = (weighted_q * q_nw_view).sum(dim=(2, 3)) # (B, N) - S_k = (weighted_k * k_nw_view).sum(dim=(2, 3)) - - # dq_raw[b, n, h, d] = d_q_post_norm * inv_rms_q * q_norm_w - # - inv_rms_q^3 / C * q_raw * S_q - inv_q3 = (q_inv_rms**3).view(B, N, 1, 1) - inv_k3 = (k_inv_rms**3).view(B, N, 1, 1) - inv_C = 1.0 / float(C) - dq_raw_f32 = d_q_post_norm * q_inv_rms_view * q_nw_view - inv_q3 * inv_C * q_raw_f32 * S_q.view(B, N, 1, 1) - dk_raw_f32 = d_k_post_norm * k_inv_rms_view * k_nw_view - inv_k3 * inv_C * k_raw_f32 * S_k.view(B, N, 1, 1) - - # dq_norm_weight[h, d] = sum_{b, n} d_q_post_norm[b, n, h, d] * q_raw[b, n, h, d] * inv_rms_q[b, n] - dq_norm_weight = (weighted_q * q_inv_rms_view).sum(dim=(0, 1)).reshape(-1).contiguous() - dk_norm_weight = (weighted_k * k_inv_rms_view).sum(dim=(0, 1)).reshape(-1).contiguous() - - # Cast Q/K/V grads back to input dtype to match torch.autograd convention. - dq_raw = dq_raw_f32.to(q_raw.dtype) - dk_raw = dk_raw_f32.to(q_raw.dtype) - dv_raw = dv_raw_f32.to(q_raw.dtype) - - return dq_raw, dk_raw, dv_raw, dq_norm_weight, dk_norm_weight - - -def cam_scan_func( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - beta: torch.Tensor, - decay: torch.Tensor, - *, - reverse: bool = False, - init_state: torch.Tensor | None = None, - save_final_state: bool = False, -) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - """Fused numerator-only single-path delta-rule scan for the cam branch. - - Args: - q, k, v: ``(B, H, D, N)`` fp32 contiguous tensors. - beta: ``(B, H, F, S)`` fp32 contiguous. - decay: ``(B, H, F)`` fp32 contiguous. - reverse: If ``True``, run the scan as the backward pass (equivalent - to ``flip_and_shift``-ing the inputs along the frame axis and - running forward). - init_state: optional ``(B*H, BLOCK_D, BLOCK_D)`` fp32 contiguous - tensor holding the forward-scan KV state at the END of a prefix - sequence (i.e., AFTER the prefix's last update, BEFORE any - further decay applied by this call). When provided, the kernel - resumes the scan from this state instead of zero. ``BLOCK_D = - next_pow2(D)`` and only the top-left ``D x D`` submatrix is - read. Forward direction only — raises ``NotImplementedError`` - if combined with ``reverse=True``. - save_final_state: when True, allocate a fresh fp32 zero buffer for - the final KV state (after the last frame's update) and pass it - to the kernel for write-out. Returned as the second tuple slot. - Forward direction only. - - Returns: - ``out`` of shape ``(B, H, D, N)`` fp32 matching - ``torch_chunk_cam_single_path_delta_rule`` with ``chunk_size >= T``. - - When ``save_final_state=True``, returns ``(out, final_state)`` where - ``final_state`` is fp32 ``(B*H, BLOCK_D, BLOCK_D)``. - - Raises: - NotImplementedError: if ``reverse=True`` is combined with state - passing. The cam branch's anti-causal scan resets per chunk in - the reference block, so there is no global cross-prefix state - to cache for the reverse direction. - """ - # Chunkwise integration (2026-05-06): dispatch all paths (fwd + reverse) - # to `cam_scan_chunkwise`. Reverse uses chunkwise's existing direction=2 - # mode in phase_b_triton, which has the same flip-and-shift semantics as - # cam's REVERSE=1 path. Bypass via FUSED_GDN_FORCE_LEGACY=1. - if os.environ.get("FUSED_GDN_FORCE_LEGACY", "0") != "1": - return cam_scan_chunkwise( - q, - k, - v, - beta, - decay, - reverse=reverse, - init_state=init_state, - save_final_state=save_final_state, - ) - - assert q.shape == k.shape == v.shape - B, H, D, N = q.shape - assert beta.shape[0] == B and beta.shape[1] == H - F_frames = beta.shape[2] - assert N % F_frames == 0 - S = N // F_frames - assert beta.shape == (B, H, F_frames, S), f"beta shape {beta.shape}" - assert decay.shape == (B, H, F_frames), f"decay shape {decay.shape}" - assert q.is_contiguous() and k.is_contiguous() and v.is_contiguous() - assert beta.is_contiguous() and decay.is_contiguous() - assert q.dtype == torch.float32 - - BLOCK_D = triton.next_power_of_2(D) - BLOCK_S = _DEFAULT_BLOCK_S - num_warps = 4 - num_stages = 1 - - if reverse and (init_state is not None or save_final_state): - raise NotImplementedError( - "cam_scan_func: state passing (init_state / save_final_state) is " - "only supported for the forward direction (reverse=False). The " - "cam branch's anti-causal pass resets per chunk; there is no " - "global cross-prefix state to cache for the reverse direction." - ) - - if init_state is not None: - expected_shape = (B * H, BLOCK_D, BLOCK_D) - if tuple(init_state.shape) != expected_shape: - raise ValueError( - f"cam_scan_func: init_state shape {tuple(init_state.shape)} " - f"does not match expected {expected_shape} (BLOCK_D=next_pow2(D)={BLOCK_D})." - ) - if init_state.dtype != torch.float32: - raise ValueError(f"cam_scan_func: init_state must be fp32 (got {init_state.dtype}).") - if not init_state.is_contiguous(): - raise ValueError("cam_scan_func: init_state must be contiguous.") - if init_state.device != q.device: - raise ValueError("cam_scan_func: init_state must be on the same device as q.") - load_init = 1 - else: - load_init = 0 - - if save_final_state: - final_state = torch.zeros(B * H, BLOCK_D, BLOCK_D, device=q.device, dtype=torch.float32) - save_final = 1 - else: - final_state = None - save_final = 0 - - out = torch.empty_like(q) - - dummy_state = torch.empty(1, device=q.device, dtype=torch.float32) - init_state_ptr = init_state if load_init else dummy_state - final_state_ptr = final_state if save_final else dummy_state - - _cam_scan_kernel[(B * H,)]( - q, - k, - v, - beta, - decay, - out, - dummy_state, - dummy_state, - init_state_ptr, - final_state_ptr, - H=H, - F=F_frames, - S=S, - D=D, - N=N, - REVERSE=1 if reverse else 0, - SAVE_STATES=0, - LOAD_INIT_STATE=load_init, - SAVE_FINAL_STATE=save_final, - BLOCK_D=BLOCK_D, - BLOCK_S=BLOCK_S, - num_warps=num_warps, - num_stages=num_stages, - ) - if save_final_state: - return out, final_state - return out - - -def _run_cam_scan_fwd_save( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - beta: torch.Tensor, - decay: torch.Tensor, - *, - reverse: bool = False, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Run the forward scan with per-frame state snapshots saved. - - Used by :class:`CamScanFunction` to preserve the ``(state_pre, state_post)`` - snapshots that the Triton bwd kernel consumes. Snapshots are indexed by - ``q_frame`` (matching the existing fwd-kernel save logic), so the bwd - kernel can load them with the same ``q_frame`` derived in its - ``REVERSE``-aware iteration. - - Returns: - (out, state_pre, state_post). ``state_pre`` and ``state_post`` are - ``(B, H, F, BLOCK_D, BLOCK_D)`` fp32 with ``BLOCK_D = next_pow2(D)``. - Padding columns/rows past ``D`` are zero-masked on store. - """ - assert q.shape == k.shape == v.shape - B, H, D, N = q.shape - F_frames = beta.shape[2] - S = N // F_frames - assert beta.shape == (B, H, F_frames, S) - assert decay.shape == (B, H, F_frames) - assert q.is_contiguous() and k.is_contiguous() and v.is_contiguous() - assert beta.is_contiguous() and decay.is_contiguous() - assert q.dtype == torch.float32 - - BLOCK_D = triton.next_power_of_2(D) - BLOCK_S = _DEFAULT_BLOCK_S - num_warps = 4 - num_stages = 1 - - out = torch.empty_like(q) - state_pre = torch.zeros(B * H, F_frames, BLOCK_D, BLOCK_D, device=q.device, dtype=torch.float32) - state_post = torch.zeros(B * H, F_frames, BLOCK_D, BLOCK_D, device=q.device, dtype=torch.float32) - dummy_state = torch.empty(1, device=q.device, dtype=torch.float32) - - _cam_scan_kernel[(B * H,)]( - q, - k, - v, - beta, - decay, - out, - state_pre, - state_post, - dummy_state, - dummy_state, - H=H, - F=F_frames, - S=S, - D=D, - N=N, - REVERSE=1 if reverse else 0, - SAVE_STATES=1, - LOAD_INIT_STATE=0, - SAVE_FINAL_STATE=0, - BLOCK_D=BLOCK_D, - BLOCK_S=BLOCK_S, - num_warps=num_warps, - num_stages=num_stages, - ) - return out, state_pre, state_post - - -def _cam_scan_bwd_dispatch( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - beta: torch.Tensor, - decay: torch.Tensor, - state_pre: torch.Tensor, - state_post: torch.Tensor, - grad_out: torch.Tensor, - *, - reverse: bool = False, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Launch ``_cam_scan_bwd_kernel`` and return ``(dq, dk, dv, dbeta, ddecay)``. - - All gradient outputs are fp32 contiguous, matching the dtype of the - forward inputs. ``grad_out`` is cast to fp32 before launching. - - When ``reverse=True``, ``kv_frame=0`` is never visited (only used in the - skipped first iter), so ``dk[..., 0, :]``, ``dv[..., 0, :]``, - ``dbeta[..., 0, :]`` and ``ddecay[..., 0]`` must remain zero. We pre-zero - every output buffer here so the kernel only needs to write the live slots. - """ - assert q.shape == k.shape == v.shape - B, H, D, N = q.shape - F_frames = beta.shape[2] - S = N // F_frames - - grad_out_f32 = grad_out.to(torch.float32).contiguous() - dq = torch.zeros_like(q) - dk = torch.zeros_like(k) - dv = torch.zeros_like(v) - dbeta = torch.zeros_like(beta) - ddecay = torch.zeros_like(decay) - - BLOCK_D = triton.next_power_of_2(D) - # For small S, ``next_pow2(S) < _DEFAULT_BLOCK_S`` — using the smaller value - # avoids zero-padding huge unused tiles into shared memory. - BLOCK_S = min(_DEFAULT_BLOCK_S, max(triton.next_power_of_2(S), 16)) - num_stages = 1 - REVERSE = 1 if reverse else 0 - - last_err: Exception | None = None - for num_warps in (4, 2, 1): - try: - _cam_scan_bwd_kernel[(B * H,)]( - q, - k, - v, - beta, - decay, - state_pre, - state_post, - grad_out_f32, - dq, - dk, - dv, - dbeta, - ddecay, - H=H, - F=F_frames, - S=S, - D=D, - N=N, - REVERSE=REVERSE, - BLOCK_D=BLOCK_D, - BLOCK_S=BLOCK_S, - num_warps=num_warps, - num_stages=num_stages, - ) - return dq, dk, dv, dbeta, ddecay - except triton.runtime.errors.OutOfResources as exc: - last_err = exc - continue - raise RuntimeError("_cam_scan_bwd_kernel exhausted all num_warps choices: " + str(last_err)) - - -# ============================================================================= -# Section: Torch reference implementations used by fallback backward paths -# ============================================================================= -# These references replicate the Triton-kernel math (full-channel RMSNorm + -# ReLU + K-scale + 4x4 UCPE projmat + interleaved-pair real-valued RoPE, then -# numerator-only single-path delta-rule scan). They run in fp32 internally and -# cast outputs back to the input dtype, matching the kernels. - - -def _flip_and_shift(x: torch.Tensor, dim: int, shift_val: float) -> torch.Tensor: - """Flip ``x`` along ``dim`` and right-shift by one (pad with ``shift_val``). - - Matches the reference ``sana_gdn_blocks.flip_and_shift`` semantics. - """ - x_flip = torch.flip(x, dims=[dim]) - x_shifted = x_flip.narrow(dim, 0, x.shape[dim] - 1) - pad_shape = list(x.shape) - pad_shape[dim] = 1 - padding = torch.full(pad_shape, shift_val, device=x.device, dtype=x.dtype) - return torch.cat([padding, x_shifted], dim=dim) - - -def _torch_cam_scan_single_chunk( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - beta: torch.Tensor, - decay: torch.Tensor, - init_state: torch.Tensor | None = None, - return_final_state: bool = False, -) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - """Pure-torch single-chunk delta-rule scan (numerator-only). - - Algebraically equivalent to ``torch_chunk_cam_single_path_delta_rule`` with - ``chunk_size >= T``; matches the Triton ``_cam_scan_kernel`` math exactly - (which also runs as a single chunk over all F frames). - - Args: - q, k, v: ``(B, H, D, N)`` fp32 contiguous. - beta: ``(B, H, F, S)`` or ``(B, H, F)`` fp32 contiguous. - decay: ``(B, H, F)`` fp32 contiguous. - - Returns: - out: ``(B, H, D, N)`` fp32 contiguous, ``N = F * S``. - """ - B, H, D, N = q.shape - if beta.ndim == 4: - T = beta.shape[2] - elif beta.ndim == 3: - T = beta.shape[2] - else: - raise ValueError(f"beta must be (B,H,F[,S]); got ndim={beta.ndim}") - if N % T != 0: - raise ValueError(f"N ({N}) must be divisible by T ({T}).") - S = N // T - - def to_frame_seq(x: torch.Tensor) -> torch.Tensor: - return x.view(B, H, D, T, S).permute(0, 1, 3, 2, 4) # (B, H, T, D, S) - - q_t = to_frame_seq(q) - k_t = to_frame_seq(k) - v_t = to_frame_seq(v) - - if beta.ndim == 4: - beta_view = beta.unsqueeze(3) # (B, H, T, 1, S) - else: - beta_view = beta.view(B, H, T, 1, 1) - decay_view = decay.view(B, H, T, 1, 1) - - eye = torch.eye(D, device=q.device, dtype=q.dtype).view(1, 1, 1, D, D) - - k_beta = k_t * beta_view - W = decay_view * (eye - torch.matmul(k_beta, k_t.transpose(-1, -2))) - U = torch.matmul(v_t * beta_view, k_t.transpose(-1, -2)) - - state = ( - torch.zeros(B, H, D, D, device=q.device, dtype=q.dtype) - if init_state is None - else init_state.to(device=q.device, dtype=q.dtype) - ) - s_kv_list: list[torch.Tensor] = [] - for t in range(T): - state = torch.matmul(state, W[:, :, t]) + U[:, :, t] - s_kv_list.append(state) - s_all = torch.stack(s_kv_list, dim=2) # (B, H, T, D, D) - - out_t = torch.matmul(s_all, q_t) # (B, H, T, D, S) - out = out_t.permute(0, 1, 3, 2, 4).reshape(B, H, D, N) - return (out, state) if return_final_state else out - - -def _torch_cam_scan_reference( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - beta: torch.Tensor, - decay: torch.Tensor, - *, - reverse: bool = False, -) -> torch.Tensor: - """Pure-torch reference for ``cam_scan_func`` supporting ``reverse=True``. - - For ``reverse=False`` this is the standard forward delta-rule scan. - - For ``reverse=True`` we emulate the Triton kernel's per-chunk - ``flip_and_shift`` semantics (q is flipped only; k/v/beta are - flip-and-shifted with pad value 0; decay is flip-and-shifted with pad - value 1; output is then flipped back along the time axis). - """ - if not reverse: - return _torch_cam_scan_single_chunk(q, k, v, beta, decay) - - B, H, D, N = q.shape - if beta.ndim == 4: - T = beta.shape[2] - elif beta.ndim == 3: - T = beta.shape[2] - else: - raise ValueError(f"beta must be (B,H,F[,S]); got ndim={beta.ndim}") - S = N // T - - def to_frame(x: torch.Tensor) -> torch.Tensor: - return x.view(B, H, D, T, S).permute(0, 1, 3, 2, 4) # (B, H, T, D, S) - - def from_frame(x: torch.Tensor) -> torch.Tensor: - return x.permute(0, 1, 3, 2, 4).reshape(B, H, D, N) - - q_bwd = torch.flip(to_frame(q), dims=[2]) - k_bwd = _flip_and_shift(to_frame(k), dim=2, shift_val=0.0) - v_bwd = _flip_and_shift(to_frame(v), dim=2, shift_val=0.0) - beta_bwd = _flip_and_shift(beta, dim=2, shift_val=0.0) - decay_bwd = _flip_and_shift(decay, dim=2, shift_val=1.0) - - out_bwd = _torch_cam_scan_single_chunk( - from_frame(q_bwd), - from_frame(k_bwd), - from_frame(v_bwd), - beta_bwd, - decay_bwd, - ) - out_bwd_t = out_bwd.view(B, H, D, T, S) # already in (B, H, D, T, S) - return torch.flip(out_bwd_t, dims=[3]).reshape(B, H, D, N) - - -def _torch_cam_prep_reference( - q_raw: torch.Tensor, - k_raw: torch.Tensor, - v_raw: torch.Tensor, - *, - q_norm_weight: torch.Tensor, - k_norm_weight: torch.Tensor, - proj_q: torch.Tensor, - proj_kv: torch.Tensor, - rope_cos: torch.Tensor, - rope_sin: torch.Tensor, - k_scale: float, - norm_eps: float, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Pure-torch reference for ``cam_prep_func`` matching ``_cam_prep_kernel``. - - Replicates exactly: - - Full-channel (over ``H*D``) RMSNorm + per-channel weight on Q, K. - - ReLU on Q, K. - - K-scale on K. - - 4x4 UCPE projmat on first ``D/2`` dims (Q via ``proj_q``, - K and V via ``proj_kv``). - - Interleaved-pair real-valued RoPE on second ``D/2`` dims using - ``rope_cos`` / ``rope_sin`` (the same tables passed to the kernel). - - ``inflation_sq = ||k_post_ucpe||^2 / ||k_pre_ucpe||^2`` per (B, H, N), - with the same ``clamp_min(1e-12)`` floor as ``cam_prep_func``. - - All math runs in fp32 internally; outputs ``(q, k, v)`` are cast back to - ``q_raw.dtype`` and ``inflation_sq`` is fp32. - """ - B, N, H, D = q_raw.shape - if D % 2 != 0: - raise ValueError(f"D ({D}) must be even.") - if (D // 2) % 4 != 0: - raise ValueError(f"D/2 ({D // 2}) must be divisible by 4 (UCPE projmat).") - C = H * D - D_half = D // 2 - n_groups = D_half // 4 - - q32 = q_raw.float() - k32 = k_raw.float() - v32 = v_raw.float() - - # ---- Full-channel RMSNorm + per-channel weight (Q, K only) ---- - q_inv_rms = torch.rsqrt((q32 * q32).sum(dim=(-1, -2)) / C + norm_eps) # (B, N) - k_inv_rms = torch.rsqrt((k32 * k32).sum(dim=(-1, -2)) / C + norm_eps) - q_nw = q_norm_weight.float().view(1, 1, H, D) - k_nw = k_norm_weight.float().view(1, 1, H, D) - q_normed = q32 * q_inv_rms.view(B, N, 1, 1) * q_nw - k_normed = k32 * k_inv_rms.view(B, N, 1, 1) * k_nw - - # ---- ReLU + K-scale ---- - q_normed = torch.relu(q_normed) - k_normed = torch.relu(k_normed) * k_scale - - # ---- Pre-UCPE ||k||^2 over the full D dim ---- - pre_k_sq_BNH = (k_normed * k_normed).sum(dim=-1) # (B, N, H) - - # ---- UCPE 4x4 projmat on first half ---- - q_first = q_normed[..., :D_half].reshape(B, N, H, n_groups, 4) - k_first = k_normed[..., :D_half].reshape(B, N, H, n_groups, 4) - v_first = v32[..., :D_half].reshape(B, N, H, n_groups, 4) - - # out[b,n,h,g,i] = sum_j P[b,n,i,j] * x[b,n,h,g,j] - # einsum: 'bnij,bnhgj->bnhgi' - proj_q_f = proj_q.float() - proj_kv_f = proj_kv.float() - q_first_proj = torch.einsum("bnij,bnhgj->bnhgi", proj_q_f, q_first).reshape(B, N, H, D_half) - k_first_proj = torch.einsum("bnij,bnhgj->bnhgi", proj_kv_f, k_first).reshape(B, N, H, D_half) - v_first_proj = torch.einsum("bnij,bnhgj->bnhgi", proj_kv_f, v_first).reshape(B, N, H, D_half) - - # ---- Interleaved-pair real-valued RoPE on second half ---- - # Kernel form: y[d] = x[d]*rope_cos[d] + x[d^1]*rope_sin[d] - # where rope_cos/rope_sin come from _prepare_ucpe_rope_tables. - q_second = q_normed[..., D_half:] - k_second = k_normed[..., D_half:] - v_second = v32[..., D_half:] - - def _pair_swap(x: torch.Tensor) -> torch.Tensor: - # Swap consecutive pairs along the last dim: (..., D_half) where D_half is even. - # x[..., 2i] <-> x[..., 2i+1]. - x_pairs = x.unflatten(-1, (D_half // 2, 2)) - x_swapped = x_pairs.flip(-1) - return x_swapped.flatten(-2) - - cos_b = rope_cos.float().view(1, N, 1, D_half) - sin_b = rope_sin.float().view(1, N, 1, D_half) - q_rope = q_second * cos_b + _pair_swap(q_second) * sin_b - k_rope = k_second * cos_b + _pair_swap(k_second) * sin_b - v_rope = v_second * cos_b + _pair_swap(v_second) * sin_b - - # ---- Reassemble (B, N, H, D) and post-UCPE k norm ---- - q_out_BNHD = torch.cat([q_first_proj, q_rope], dim=-1) - k_out_BNHD = torch.cat([k_first_proj, k_rope], dim=-1) - v_out_BNHD = torch.cat([v_first_proj, v_rope], dim=-1) - - post_k_sq_BNH = (k_out_BNHD * k_out_BNHD).sum(dim=-1) # (B, N, H) - - out_dtype = q_raw.dtype - q_out = q_out_BNHD.to(out_dtype).permute(0, 2, 3, 1).contiguous() - k_out = k_out_BNHD.to(out_dtype).permute(0, 2, 3, 1).contiguous() - v_out = v_out_BNHD.to(out_dtype).permute(0, 2, 3, 1).contiguous() - - pre_k_sq = pre_k_sq_BNH.permute(0, 2, 1).contiguous() # (B, H, N) - post_k_sq = post_k_sq_BNH.permute(0, 2, 1).contiguous() - inflation_sq = post_k_sq.clamp_min(1e-12) / pre_k_sq.clamp_min(1e-12) - return q_out, k_out, v_out, inflation_sq - - -# ============================================================================= -# Section: Autograd-enabled wrappers -# ============================================================================= - - -def _cam_scan_torch_fallback_backward( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - beta: torch.Tensor, - decay: torch.Tensor, - needs: tuple[bool, bool, bool, bool, bool], - grad_out: torch.Tensor, - reverse: bool, -) -> list[torch.Tensor | None]: - """Recompute the cam-branch scan via the torch reference and return grads. - - Used when ``CAM_SCAN_BWD_FALLBACK=1`` forces the torch-recompute backward - path. - """ - detached = [] - for tensor, need in zip((q, k, v, beta, decay), needs): - t = tensor.detach() - if need: - t = t.requires_grad_(True) - detached.append(t) - active = [t for t in detached if t.requires_grad] - - with torch.enable_grad(): - q_d, k_d, v_d, beta_d, decay_d = detached - ref_out = _torch_cam_scan_reference(q_d, k_d, v_d, beta_d, decay_d, reverse=reverse) - if active: - active_grads = torch.autograd.grad( - outputs=ref_out, - inputs=tuple(active), - grad_outputs=grad_out.to(ref_out.dtype), - allow_unused=True, - ) - else: - active_grads = [] - - grads: list[torch.Tensor | None] = [] - active_iter = iter(active_grads) - for tensor in detached: - grads.append(next(active_iter) if tensor.requires_grad else None) - return grads - - -class CamScanFunction(torch.autograd.Function): - """Autograd ``Function`` wrapping ``cam_scan_func``. - - Forward calls the Triton ``_cam_scan_kernel`` with ``SAVE_STATES=1`` so - per-frame state snapshots (``state_pre[q_frame]``, ``state_post[q_frame]``) - are kept for the backward pass. Backward runs the true Triton bwd - kernel (``_cam_scan_bwd_kernel``) for both ``reverse=False`` and - ``reverse=True``, replaying the recurrence in reverse time using the - saved snapshots. - - Set ``CAM_SCAN_BWD_FALLBACK=1`` to force the torch-recompute backward - validation path. - """ - - @staticmethod - def forward( - ctx, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - beta: torch.Tensor, - decay: torch.Tensor, - reverse: bool, - ) -> torch.Tensor: - ctx.set_materialize_grads(False) - ctx.reverse = bool(reverse) - - force_torch_fallback = os.environ.get("CAM_SCAN_BWD_FALLBACK", "0") == "1" - ctx.use_triton_bwd = not force_torch_fallback - - if ctx.use_triton_bwd: - out, state_pre, state_post = _run_cam_scan_fwd_save(q, k, v, beta, decay, reverse=ctx.reverse) - ctx.save_for_backward(q, k, v, beta, decay, state_pre, state_post) - return out - - # Torch-fallback backward path: don't bother saving state snapshots. - ctx.save_for_backward(q, k, v, beta, decay) - return cam_scan_func(q, k, v, beta, decay, reverse=reverse) - - @staticmethod - def backward(ctx, grad_out): # type: ignore[override] - if grad_out is None: - return (None, None, None, None, None, None) - - if ctx.use_triton_bwd: - q, k, v, beta, decay, state_pre, state_post = ctx.saved_tensors - needs = ctx.needs_input_grad[:5] # q, k, v, beta, decay - dq, dk, dv, dbeta, ddecay = _cam_scan_bwd_dispatch( - q, - k, - v, - beta, - decay, - state_pre, - state_post, - grad_out, - reverse=ctx.reverse, - ) - grads: list[torch.Tensor | None] = [ - dq if needs[0] else None, - dk if needs[1] else None, - dv if needs[2] else None, - dbeta if needs[3] else None, - ddecay if needs[4] else None, - ] - return (*grads, None) - - # Env-var torch-recompute backward. - q, k, v, beta, decay = ctx.saved_tensors - needs = ctx.needs_input_grad[:5] - grads = _cam_scan_torch_fallback_backward(q, k, v, beta, decay, needs, grad_out, ctx.reverse) - return (*grads, None) - - -def cam_scan_func_with_grad( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - beta: torch.Tensor, - decay: torch.Tensor, - *, - reverse: bool = False, -) -> torch.Tensor: - """Autograd-enabled wrapper around :func:`cam_scan_func`. - - Forward is identical to :func:`cam_scan_func`; backward is computed via - a torch reference (``_torch_cam_scan_reference``). Use this in training - paths where any of ``q, k, v, beta, decay`` may require gradients. - - Inference paths can keep calling :func:`cam_scan_func` directly to avoid - the small autograd bookkeeping overhead. - """ - return CamScanFunction.apply(q, k, v, beta, decay, reverse) - - -def _cam_prep_torch_fallback_backward( - q_raw: torch.Tensor, - k_raw: torch.Tensor, - v_raw: torch.Tensor, - q_norm_weight: torch.Tensor, - k_norm_weight: torch.Tensor, - proj_q: torch.Tensor, - proj_kv: torch.Tensor, - rope_cos: torch.Tensor, - rope_sin: torch.Tensor, - needs: tuple[bool, ...], - grad_q: torch.Tensor | None, - grad_k: torch.Tensor | None, - grad_v: torch.Tensor | None, - grad_inflation_sq: torch.Tensor | None, - k_scale: float, - norm_eps: float, -) -> list[torch.Tensor | None]: - """Recompute the cam-branch prep via the torch reference and return grads. - - Used when any of ``proj_q / proj_kv / rope_cos / rope_sin`` requests a - gradient (the Triton kernel does not produce those grads), or when - ``CAM_PREP_BWD_FALLBACK=1`` forces the torch-recompute backward path. - - Args: - q_raw, k_raw, ..., rope_sin: the nine tensor inputs of - :func:`cam_prep_func` (in the same order as - :class:`CamPrepFunction.forward`'s arg list). - needs: ``ctx.needs_input_grad[:9]`` — boolean per-input flags. - grad_q, grad_k, grad_v, grad_inflation_sq: upstream gradients. - k_scale, norm_eps: scalar fwd args. - - Returns: - A 9-element list of ``torch.Tensor | None`` aligned with the - ``saved`` tuple. Entries that didn't request a gradient are ``None``. - """ - saved = ( - q_raw, - k_raw, - v_raw, - q_norm_weight, - k_norm_weight, - proj_q, - proj_kv, - rope_cos, - rope_sin, - ) - detached: list[torch.Tensor] = [] - for tensor, need in zip(saved, needs): - t = tensor.detach() - if need: - t = t.requires_grad_(True) - detached.append(t) - active = [t for t in detached if t.requires_grad] - - with torch.enable_grad(): - (q_d, k_d, v_d, qnw_d, knw_d, pq_d, pkv_d, rc_d, rs_d) = detached - ref_q, ref_k, ref_v, ref_inf = _torch_cam_prep_reference( - q_d, - k_d, - v_d, - q_norm_weight=qnw_d, - k_norm_weight=knw_d, - proj_q=pq_d, - proj_kv=pkv_d, - rope_cos=rc_d, - rope_sin=rs_d, - k_scale=k_scale, - norm_eps=norm_eps, - ) - - outputs = [] - grad_outputs = [] - if grad_q is not None: - outputs.append(ref_q) - grad_outputs.append(grad_q.to(ref_q.dtype)) - if grad_k is not None: - outputs.append(ref_k) - grad_outputs.append(grad_k.to(ref_k.dtype)) - if grad_v is not None: - outputs.append(ref_v) - grad_outputs.append(grad_v.to(ref_v.dtype)) - if grad_inflation_sq is not None: - outputs.append(ref_inf) - grad_outputs.append(grad_inflation_sq.to(ref_inf.dtype)) - - if active and outputs: - active_grads = torch.autograd.grad( - outputs=tuple(outputs), - inputs=tuple(active), - grad_outputs=tuple(grad_outputs), - allow_unused=True, - ) - else: - active_grads = [] - - grads: list[torch.Tensor | None] = [] - active_iter = iter(active_grads) - for tensor in detached: - grads.append(next(active_iter) if tensor.requires_grad else None) - return grads - - -class CamPrepFunction(torch.autograd.Function): - """Autograd ``Function`` wrapping ``cam_prep_func``. - - Forward calls the fused Triton ``_cam_prep_kernel`` via - :func:`_run_cam_prep_fwd_save` so the per-token ``inv_rms`` / - ``k_pre_sq`` / ``k_post_sq`` snapshots required by the bwd kernel are - preserved alongside the standard outputs. - - Backward runs the true Triton bwd kernel via - :func:`_cam_prep_bwd_dispatch` for the standard training path - (``q_raw``, ``k_raw``, ``v_raw``, ``q_norm_weight``, ``k_norm_weight`` - only request grads). The Triton path implements: - - * RoPE^T, UCPE^T, K-scale^T, and ReLU mask in a single fused kernel - (one program per ``(b, n, h)``); - * ``grad_inflation_sq`` chain through ``k_post_sq`` (added into - ``eff_dO_k``) and ``k_pre_sq`` (direct contribution to - ``d_k_post_kscale``), with ``clamp_min(1e-12)`` indicators honored; - * The full-channel RMSNorm bwd (per-token cross-head reduction) is - done in PyTorch on the kernel's ``d_q_post_norm`` / - ``d_k_post_norm`` intermediates — see - :func:`_cam_prep_bwd_dispatch` for details. - - The torch-recompute fallback (running :func:`_torch_cam_prep_reference` - under autograd) is selected when any of ``proj_q / proj_kv / rope_cos / - rope_sin`` requests a gradient (the Triton path emits ``None`` for those - slots) or when ``CAM_PREP_BWD_FALLBACK=1`` is set. - """ - - @staticmethod - def forward( - ctx, - q_raw: torch.Tensor, - k_raw: torch.Tensor, - v_raw: torch.Tensor, - q_norm_weight: torch.Tensor, - k_norm_weight: torch.Tensor, - proj_q: torch.Tensor, - proj_kv: torch.Tensor, - rope_cos: torch.Tensor, - rope_sin: torch.Tensor, - k_scale: float, - norm_eps: float, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - ctx.set_materialize_grads(False) - ctx.k_scale = float(k_scale) - ctx.norm_eps = float(norm_eps) - - force_torch_fallback = os.environ.get("CAM_PREP_BWD_FALLBACK", "0") == "1" - ctx.use_triton_bwd = not force_torch_fallback - - if ctx.use_triton_bwd: - ( - q_out, - k_out, - v_out, - inflation_sq, - q_inv_rms, - k_inv_rms, - k_pre_sq, - k_post_sq, - ) = _run_cam_prep_fwd_save( - q_raw, - k_raw, - v_raw, - q_norm_weight=q_norm_weight, - k_norm_weight=k_norm_weight, - proj_q=proj_q, - proj_kv=proj_kv, - rope_cos=rope_cos, - rope_sin=rope_sin, - k_scale=k_scale, - norm_eps=norm_eps, - ) - ctx.save_for_backward( - q_raw, - k_raw, - v_raw, - q_norm_weight, - k_norm_weight, - proj_q, - proj_kv, - rope_cos, - rope_sin, - q_inv_rms, - k_inv_rms, - k_pre_sq, - k_post_sq, - k_out, - ) - else: - q_out, k_out, v_out, inflation_sq = cam_prep_func( - q_raw, - k_raw, - v_raw, - q_norm_weight=q_norm_weight, - k_norm_weight=k_norm_weight, - proj_q=proj_q, - proj_kv=proj_kv, - rope_cos=rope_cos, - rope_sin=rope_sin, - k_scale=k_scale, - norm_eps=norm_eps, - ) - ctx.save_for_backward( - q_raw, - k_raw, - v_raw, - q_norm_weight, - k_norm_weight, - proj_q, - proj_kv, - rope_cos, - rope_sin, - ) - return q_out, k_out, v_out, inflation_sq - - @staticmethod - def backward(ctx, grad_q, grad_k, grad_v, grad_inflation_sq): # type: ignore[override] - if grad_q is None and grad_k is None and grad_v is None and grad_inflation_sq is None: - return tuple([None] * 11) - - needs = ctx.needs_input_grad[:9] # nine tensor inputs - # If anyone outside (q_raw, k_raw, v_raw, q_norm_weight, k_norm_weight) - # requests a grad, the Triton bwd cannot handle it — fall back to the - # torch reference. - triton_bwd_supported_needs = needs[:5] - proj_or_rope_needs_grad = any(needs[5:]) - - if ctx.use_triton_bwd and not proj_or_rope_needs_grad: - ( - q_raw, - k_raw, - v_raw, - q_norm_weight, - k_norm_weight, - proj_q, - proj_kv, - rope_cos, - rope_sin, - q_inv_rms, - k_inv_rms, - k_pre_sq, - k_post_sq, - k_out, - ) = ctx.saved_tensors - - ( - dq_raw, - dk_raw, - dv_raw, - dq_norm_weight, - dk_norm_weight, - ) = _cam_prep_bwd_dispatch( - q_raw, - k_raw, - q_norm_weight, - k_norm_weight, - proj_q, - proj_kv, - rope_cos, - rope_sin, - q_inv_rms, - k_inv_rms, - k_pre_sq, - k_post_sq, - k_out, - grad_q=grad_q, - grad_k=grad_k, - grad_v=grad_v, - grad_inflation_sq=grad_inflation_sq, - k_scale=ctx.k_scale, - ) - grads: list[torch.Tensor | None] = [ - dq_raw if triton_bwd_supported_needs[0] else None, - dk_raw if triton_bwd_supported_needs[1] else None, - dv_raw if triton_bwd_supported_needs[2] else None, - dq_norm_weight if triton_bwd_supported_needs[3] else None, - dk_norm_weight if triton_bwd_supported_needs[4] else None, - None, # proj_q - None, # proj_kv - None, # rope_cos - None, # rope_sin - ] - return (*grads, None, None) - - # Torch fallback path. ``ctx.saved_tensors`` holds either 9 (legacy - # forward) or 14 (Triton fwd save) tensors — slice the leading nine. - saved = ctx.saved_tensors[:9] - ( - q_raw, - k_raw, - v_raw, - q_norm_weight, - k_norm_weight, - proj_q, - proj_kv, - rope_cos, - rope_sin, - ) = saved - grads = _cam_prep_torch_fallback_backward( - q_raw, - k_raw, - v_raw, - q_norm_weight, - k_norm_weight, - proj_q, - proj_kv, - rope_cos, - rope_sin, - needs=needs, - grad_q=grad_q, - grad_k=grad_k, - grad_v=grad_v, - grad_inflation_sq=grad_inflation_sq, - k_scale=ctx.k_scale, - norm_eps=ctx.norm_eps, - ) - # Two trailing None for non-tensor scalars (k_scale, norm_eps). - return (*grads, None, None) - - -def cam_prep_func_with_grad( - q_raw: torch.Tensor, - k_raw: torch.Tensor, - v_raw: torch.Tensor, - *, - q_norm_weight: torch.Tensor, - k_norm_weight: torch.Tensor, - proj_q: torch.Tensor, - proj_kv: torch.Tensor, - rope_cos: torch.Tensor, - rope_sin: torch.Tensor, - k_scale: float, - norm_eps: float, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Autograd-enabled wrapper around :func:`cam_prep_func`. - - Forward is identical to :func:`cam_prep_func`; backward is computed via a - torch reference (``_torch_cam_prep_reference``). Use this in training - paths so gradients flow back through Q/K/V projection inputs and through - the RMSNorm weights. - - Inference paths can keep calling :func:`cam_prep_func` directly. - """ - return CamPrepFunction.apply( - q_raw, - k_raw, - v_raw, - q_norm_weight, - k_norm_weight, - proj_q, - proj_kv, - rope_cos, - rope_sin, - k_scale, - norm_eps, - ) - - -__all__ = [ - "CamPrepFunction", - "CamScanFunction", - "_cam_prep_bwd_dispatch", - "_cam_prep_bwd_kernel", - "_cam_prep_kernel", - "_cam_prep_torch_fallback_backward", - "_cam_scan_bwd_dispatch", - "_cam_scan_bwd_kernel", - "_cam_scan_kernel", - "_invert_SE3", - "_precompute_cam_inv_rms", - "_prepare_ucpe_rope_tables", - "_process_camera_conditions_raymats_only", - "_run_cam_prep_fwd_save", - "_run_cam_scan_fwd_save", - "_torch_cam_prep_reference", - "cam_prep_func", - "cam_prep_func_with_grad", - "cam_scan_func", - "cam_scan_func_with_grad", -] diff --git a/integrations/sana/sana_wm/ops/fused_gdn.py b/integrations/sana/sana_wm/ops/fused_gdn.py deleted file mode 100644 index 447e0f5ca..000000000 --- a/integrations/sana/sana_wm/ops/fused_gdn.py +++ /dev/null @@ -1,2249 +0,0 @@ -"""Fused-BiGDN Triton kernels used by SANA-WM GDN attention blocks. - -Includes the unified forward kernel, backward kernels, RoPE/RMS helpers, and -autograd wrappers used by the Triton GDN attention blocks. - -Precision knob: env var ``FUSED_GDN_PRECISION`` or ``PRECISION_OVERRIDE``: - 0=IEEE fp32 dots, 1=TF32, 2=bf16 TC + fp32 state [default], 3=bf16 TC + bf16 state. -""" - -# ruff: noqa: E501 - -from __future__ import annotations - -import os - -import torch -import triton -import triton.language as tl - -# ===================================================================== -# GPU-adaptive kernel config -# ===================================================================== - - -def _get_kernel_config() -> dict: - """Return optimal kernel parameters for the current GPU. - - STATE_FP32: use fp32 state_prev when SRAM is large enough. - - bf16 state_prev: ~96KB total SRAM (fits GB10's 101KB). - - fp32 state_prev: ~128KB total SRAM (needs H100's 228KB+). - """ - if not torch.cuda.is_available(): - return {"BLOCK_S": 64, "num_stages": 1, "num_warps": 4, "STATE_FP32": False} - smem = torch.cuda.get_device_properties(0).shared_memory_per_multiprocessor - state_fp32 = smem >= 150 * 1024 # H100 (228KB) yes, GB10 (101KB) no - return {"BLOCK_S": 64, "num_stages": 1, "num_warps": 8, "STATE_FP32": state_fp32} - - -_KCFG = None - - -def _kcfg(): - global _KCFG - if _KCFG is None: - _KCFG = _get_kernel_config() - return _KCFG - - -# precision=0 → IEEE fp32 dots + fp32 state (DOT_PRECISION=2, STATE_FP32=1) -# precision=1 → TF32 dots + fp32 state (DOT_PRECISION=1, STATE_FP32=1) -# precision=2 → bf16 dots + fp32 state (DOT_PRECISION=0, STATE_FP32=1) [default] -# precision=3 → bf16 dots + bf16 state (DOT_PRECISION=0, STATE_FP32=0) -def _precision_params(precision: int) -> tuple: - if precision == 0: - return 2, True - elif precision == 1: - return 1, True - elif precision == 3: - return 0, False - else: # default - return 0, True - - -_env_prec = os.environ.get("FUSED_GDN_PRECISION", None) -PRECISION_OVERRIDE: int | None = int(_env_prec) if _env_prec is not None else None - - -def _resolve_launch_config() -> tuple: - """Returns (prec, dot_prec, state_fp32, num_warps). - - Uses ``PRECISION_OVERRIDE`` when set; otherwise falls back to ``_kcfg()`` - (which picks ``STATE_FP32`` based on per-GPU SRAM). ``num_warps`` is - clamped to 4 when dots run on fp32 operands (more registers needed). - """ - cfg = _kcfg() - prec = PRECISION_OVERRIDE if PRECISION_OVERRIDE is not None else 2 - dot_prec, state_fp32 = _precision_params(prec) - if PRECISION_OVERRIDE is None: - state_fp32 = cfg["STATE_FP32"] - nw = cfg["num_warps"] - if dot_prec >= 1: - nw = min(nw, 4) - return prec, dot_prec, state_fp32, nw - - -def _prepare_launch(D: int, beta: torch.Tensor, decay: torch.Tensor) -> tuple: - """Shared launcher preamble. - - Returns (BLOCK_D, BLOCK_S, dot_prec, state_fp32, nw, cfg, beta_c, decay_c). - ``beta_c`` / ``decay_c`` are the contiguous copies the kernel needs. - """ - BLOCK_D = triton.next_power_of_2(D) - cfg = _kcfg() - BLOCK_S = cfg["BLOCK_S"] - _, dot_prec, state_fp32, nw = _resolve_launch_config() - return BLOCK_D, BLOCK_S, dot_prec, state_fp32, nw, cfg, beta.contiguous(), decay.contiguous() - - -# ===================================================================== -# Unified forward Triton Mega-Kernel (inference-only variant) -# ===================================================================== -# Fuses: RMSNorm + ReLU + k_scale + RoPE + BiGDN recurrence. -# -# Inputs: -# qkv (B, N, 3, H, D) interleaved — strides passed explicitly. -# beta (B, H, F, S), decay (B, H, F) contiguous. -# q_norm_w, k_norm_w (H*D,) full-channel — only read when QK_NORM=1. -# rope_cos, rope_sin (N, D) contiguous. -# q_inv_rms, k_inv_rms (B, N) full-channel — only read when USE_PRECOMPUTED_RMS=1. -# -# Outputs: -# out (B, N, H, D) = num / (den + eps) — unused by BiGDN wrappers. -# num (B, N, H, D) = numerator before divide (summed across directions). -# den (B, H, N) = denominator before divide (summed across directions). -# -# NOTE (inference-only build): upstream also supports SAVE_STATE, -# LOAD_INIT_STATE, SAVE_FINAL_STATE for training backward / state caching. -# Those constexpr branches are preserved in the kernel so the source stays -# 1-for-1 with upstream (they compile away when launched with flags=0). - - -@triton.jit -def _fused_gdn_kernel( - # ---- interleaved QKV : (B, N, 3, H, D) ---- - qkv_ptr, - stride_b: tl.constexpr, - stride_n: tl.constexpr, - stride_3: tl.constexpr, - stride_h: tl.constexpr, - stride_d: tl.constexpr, - # ---- gates ---- - beta_ptr, - decay_ptr, - # ---- inv-RMS (B, N) — only read when USE_PRECOMPUTED_RMS=1 ---- - q_inv_rms_ptr, - k_inv_rms_ptr, - # ---- norm weights (H*D,) full-channel — only read when QK_NORM=1 ---- - q_norm_w_ptr, - k_norm_w_ptr, - # ---- RoPE tables (N, D) contiguous ---- - rope_cos_ptr, - rope_sin_ptr, - # ---- outputs ---- - out_ptr, # (B, N, H, D) - num_ptr, # (B, N, H, D) - den_ptr, # (B, H, N) - # ---- saved-state dummies (unused in this build but kept for signature parity) ---- - saved_state_ptr, - saved_z_ptr, - saved_state_curr_ptr, - saved_z_curr_ptr, - init_state_kv_ptr, - init_state_z_ptr, - final_state_kv_ptr, - final_state_z_ptr, - # ---- scalars / dims ---- - H: tl.constexpr, - F: tl.constexpr, - S: tl.constexpr, - D: tl.constexpr, - K_SCALE, - NORM_EPS: tl.constexpr, - EPS: tl.constexpr, - QK_NORM: tl.constexpr, - USE_PRECOMPUTED_RMS: tl.constexpr, - STATE_FP32: tl.constexpr, - DOT_PRECISION: tl.constexpr, - REVERSE: tl.constexpr, - SAVE_STATE: tl.constexpr, - LOAD_INIT_STATE: tl.constexpr, - SAVE_FINAL_STATE: tl.constexpr, - BLOCK_D: tl.constexpr, - BLOCK_S: tl.constexpr, -): - # ---- dot product precision / operand dtype ---- - if DOT_PRECISION >= 1: - dot_dtype = tl.float32 - else: - dot_dtype = tl.bfloat16 - dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" - - # ---- program → (batch, head) ---- - pid = tl.program_id(0) - pid_b = pid // H - pid_h = pid % H - N = F * S - bh = pid_b * H + pid_h - - # ---- base pointers ---- - qkv_bh = qkv_ptr + pid_b * stride_b + pid_h * stride_h - out_bh = out_ptr + pid_b * (N * H * D) + pid_h * D - num_bh = num_ptr + pid_b * (N * H * D) + pid_h * D - den_bh = den_ptr + bh * N - beta_bh = beta_ptr + bh * (F * S) - decay_bh = decay_ptr + bh * F - if SAVE_STATE: - st_bh = saved_state_ptr + bh * F * BLOCK_D * BLOCK_D - sz_bh = saved_z_ptr + bh * F * BLOCK_D - stc_bh = saved_state_curr_ptr + bh * F * BLOCK_D * BLOCK_D - szc_bh = saved_z_curr_ptr + bh * F * BLOCK_D - - # ---- D-index helpers ---- - offs_d = tl.arange(0, BLOCK_D) - mask_d = offs_d < D - offs_d_pair = offs_d ^ 1 - mask_d_pair = offs_d_pair < D - D_inv = 1.0 / D - - # ---- full-channel norm weights (only when QK_NORM=1) ---- - nw_offset = pid_h * D - if QK_NORM: - q_nw = tl.load(q_norm_w_ptr + nw_offset + offs_d, mask=mask_d, other=0.0).to(tl.float32) - k_nw = tl.load(k_norm_w_ptr + nw_offset + offs_d, mask=mask_d, other=0.0).to(tl.float32) - q_nw_pair = tl.load(q_norm_w_ptr + nw_offset + offs_d_pair, mask=mask_d_pair, other=0.0).to(tl.float32) - k_nw_pair = tl.load(k_norm_w_ptr + nw_offset + offs_d_pair, mask=mask_d_pair, other=0.0).to(tl.float32) - - k_scale = K_SCALE - offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] - mask_dd = mask_d[:, None] & mask_d[None, :] - - # ---- double-buffer state ---- - if LOAD_INIT_STATE: - init_kv_bh = init_state_kv_ptr + bh * BLOCK_D * BLOCK_D - state_curr = tl.load(init_kv_bh + offs_dd, mask=mask_dd, other=0.0).to(tl.float32) - init_z_bh = init_state_z_ptr + bh * BLOCK_D - state_z_curr = tl.load(init_z_bh + offs_d, mask=mask_d, other=0.0).to(tl.float32) - else: - state_curr = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) - state_z_curr = tl.zeros([BLOCK_D], dtype=tl.float32) - if STATE_FP32: - state_prev = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) - else: - state_prev = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.bfloat16) - state_z_prev = tl.zeros([BLOCK_D], dtype=tl.float32) - - # ======================================================== - # Temporal loop — serial over F - # ======================================================== - for f_iter in range(F): - if REVERSE: - q_frame = F - 1 - f_iter - kv_frame = F - f_iter if f_iter > 0 else 0 # unused at f=0 - skip_update = f_iter == 0 - else: - q_frame = f_iter - kv_frame = f_iter - skip_update = False - - # ---- decay + state snapshot ---- - if REVERSE and f_iter == 0: - g = 1.0 - else: - g = tl.load(decay_bh + kv_frame).to(tl.float32) - state_curr = state_curr * g - state_z_curr = state_z_curr * g - if STATE_FP32: - state_prev = state_curr + 0.0 - else: - state_prev = state_curr.to(tl.bfloat16) - state_z_prev = state_z_curr - - if SAVE_STATE: - st_f = st_bh + q_frame * BLOCK_D * BLOCK_D - tl.store(st_f + offs_dd, state_prev, mask=mask_dd) - tl.store(sz_bh + q_frame * BLOCK_D + offs_d, state_z_prev, mask=mask_d) - - # ------------------------------------------ - # Pass 1 — State Accumulation - # ------------------------------------------ - if skip_update == False: - kv_n_base = kv_frame * S - f_beta = beta_bh + kv_frame * S - - for s0 in range(0, S, BLOCK_S): - offs_s = s0 + tl.arange(0, BLOCK_S) - mask_s = offs_s < S - mask_sd = mask_s[:, None] & mask_d[None, :] - mask_sd_pair = mask_s[:, None] & mask_d_pair[None, :] - n_idx = kv_n_base + offs_s - - k_ptrs = qkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d[None, :] * stride_d - v_ptrs = qkv_bh + n_idx[:, None] * stride_n + 2 * stride_3 + offs_d[None, :] * stride_d - K_raw = tl.load(k_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - V_raw = tl.load(v_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - - if QK_NORM: - if USE_PRECOMPUTED_RMS: - k_inv_rms = tl.load(k_inv_rms_ptr + pid_b * N + n_idx, mask=mask_s, other=1.0).to(tl.float32) - else: - k_var = tl.sum(K_raw * K_raw, axis=1) * D_inv - k_inv_rms = 1.0 / tl.sqrt(k_var + NORM_EPS) - K_normed = K_raw * k_inv_rms[:, None] * k_nw[None, :] - else: - K_normed = K_raw - K = tl.where(K_normed > 0, K_normed, 0.0) * k_scale - - k_pair_ptrs = qkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d_pair[None, :] * stride_d - K_pair_raw = tl.load(k_pair_ptrs, mask=mask_sd_pair, other=0.0).to(tl.float32) - if QK_NORM: - K_pair_normed = K_pair_raw * k_inv_rms[:, None] * k_nw_pair[None, :] - else: - K_pair_normed = K_pair_raw - K_pair = tl.where(K_pair_normed > 0, K_pair_normed, 0.0) * k_scale - - rope_ptrs = n_idx[:, None] * D + offs_d[None, :] - Cos = tl.load(rope_cos_ptr + rope_ptrs, mask=mask_sd, other=1.0).to(tl.float32) - Sin = tl.load(rope_sin_ptr + rope_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - K_rot = K * Cos + K_pair * Sin - - bt = tl.load(f_beta + offs_s, mask=mask_s, other=0.0).to(tl.float32) - - K_rot_dc = K_rot.to(dot_dtype) - V_pred = tl.dot(K_rot_dc, state_prev.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) - dv = (V_raw - V_pred) * bt[:, None] - state_curr += tl.dot(tl.trans(K_rot), dv, out_dtype=tl.float32, input_precision="tf32") - - z_hat = tl.sum(K * state_z_prev[None, :], axis=1) - dz = (1.0 - z_hat) * bt - state_z_curr += tl.sum(K * dz[:, None], axis=0) - - if SAVE_STATE: - stc_f = stc_bh + q_frame * BLOCK_D * BLOCK_D - tl.store(stc_f + offs_dd, state_curr, mask=mask_dd) - tl.store(szc_bh + q_frame * BLOCK_D + offs_d, state_z_curr, mask=mask_d) - - # ------------------------------------------ - # Pass 2 — Output (reads state_curr, inclusive) - # ------------------------------------------ - state_out = state_curr.to(dot_dtype) - state_z_out = state_z_curr - q_n_base = q_frame * S - - for s0 in range(0, S, BLOCK_S): - offs_s = s0 + tl.arange(0, BLOCK_S) - mask_s = offs_s < S - mask_sd = mask_s[:, None] & mask_d[None, :] - mask_sd_pair = mask_s[:, None] & mask_d_pair[None, :] - n_idx = q_n_base + offs_s - - q_ptrs = qkv_bh + n_idx[:, None] * stride_n + 0 * stride_3 + offs_d[None, :] * stride_d - q_pair_ptrs = qkv_bh + n_idx[:, None] * stride_n + 0 * stride_3 + offs_d_pair[None, :] * stride_d - Q_raw = tl.load(q_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - Q_pair_raw = tl.load(q_pair_ptrs, mask=mask_sd_pair, other=0.0).to(tl.float32) - - if QK_NORM: - if USE_PRECOMPUTED_RMS: - q_inv_rms = tl.load(q_inv_rms_ptr + pid_b * N + n_idx, mask=mask_s, other=1.0).to(tl.float32) - else: - q_var = tl.sum(Q_raw * Q_raw, axis=1) * D_inv - q_inv_rms = 1.0 / tl.sqrt(q_var + NORM_EPS) - Q_normed = Q_raw * q_inv_rms[:, None] * q_nw[None, :] - Q_pair_normed = Q_pair_raw * q_inv_rms[:, None] * q_nw_pair[None, :] - else: - Q_normed = Q_raw - Q_pair_normed = Q_pair_raw - Q = tl.where(Q_normed > 0, Q_normed, 0.0) - Q_pair = tl.where(Q_pair_normed > 0, Q_pair_normed, 0.0) - - rope_ptrs = n_idx[:, None] * D + offs_d[None, :] - Cos = tl.load(rope_cos_ptr + rope_ptrs, mask=mask_sd, other=1.0).to(tl.float32) - Sin = tl.load(rope_sin_ptr + rope_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - Q_rot = Q * Cos + Q_pair * Sin - - num = tl.dot(Q_rot.to(dot_dtype), state_out, out_dtype=tl.float32, input_precision=dot_ip) - den = tl.sum(Q * state_z_out[None, :], axis=1) - - result = num / (den[:, None] + EPS) - out_ptrs = out_bh + n_idx[:, None] * (H * D) + offs_d[None, :] - num_ptrs = num_bh + n_idx[:, None] * (H * D) + offs_d[None, :] - tl.store(out_ptrs, result.to(tl.bfloat16), mask=mask_sd) - tl.store(num_ptrs, num.to(tl.bfloat16), mask=mask_sd) - tl.store(den_bh + n_idx, den.to(tl.bfloat16), mask=mask_s) - - if SAVE_FINAL_STATE: - final_kv_bh = final_state_kv_ptr + bh * BLOCK_D * BLOCK_D - tl.store(final_kv_bh + offs_dd, state_curr, mask=mask_dd) - final_z_bh = final_state_z_ptr + bh * BLOCK_D - tl.store(final_z_bh + offs_d, state_z_curr, mask=mask_d) - - -# ===================================================================== -# Python wrappers -# ===================================================================== - - -def prepare_rope_tables(rotary_emb, N: int, D: int, device) -> tuple[torch.Tensor, torch.Tensor]: - """Complex rotary_emb `(1, 1, N, D//2)` → expanded (N, D) cos/sin tables. - - Encodes the interleaved-pair rotation - y[2i] = x[2i]*cos[i] - x[2i+1]*sin[i] - y[2i+1] = x[2i]*sin[i] + x[2i+1]*cos[i] - as y[d] = x[d]*cos_exp[d] + x[d^1]*sin_exp[d] - where sin_exp[2i] = -sin[i], sin_exp[2i+1] = +sin[i]. - - Returns (cos_exp, sin_exp) both (N, D) float32, contiguous. - """ - if rotary_emb is None: - return ( - torch.ones(N, D, device=device, dtype=torch.float32), - torch.zeros(N, D, device=device, dtype=torch.float32), - ) - freqs = rotary_emb.squeeze(0).squeeze(0) # (N, D//2) complex - cos_half = freqs.real.float() - sin_half = freqs.imag.float() - rope_cos = cos_half.repeat_interleave(2, dim=-1) - rope_sin = torch.stack([-sin_half, sin_half], dim=-1).reshape(N, D) - return rope_cos.contiguous(), rope_sin.contiguous() - - -def _precompute_inv_rms(qkv: torch.Tensor, idx: int, C: int, eps: float = 1e-5) -> torch.Tensor: - """Compute 1/RMS for one component of QKV over the full C = H*D channel dim. - - Args: - qkv: (B, N, 3, H, D) - idx: 0 for Q, 1 for K, 2 for V - C: H*D (channel count) - eps: RMSNorm epsilon - - Returns: - inv_rms: (B, N) float32 - """ - raw = qkv[:, :, idx].float() # (B, N, H, D) - sq_sum = (raw * raw).sum(dim=(-2, -1)) # (B, N) - return torch.rsqrt(sq_sum / C + eps) - - -# ===================================================================== -# Fused single-pass Q+K inverse-RMS Triton kernel -# ===================================================================== -# Single Triton launch that reads each `(b, n)` row of `qkv` once and emits -# both `q_inv_rms[b, n]` and `k_inv_rms[b, n]`. Replaces two separate PyTorch -# scans (cast→square→sum→rsqrt) over `qkv[:, :, 0]` and `qkv[:, :, 1]`. -# -# Layout assumed: `qkv` is (B, N, 3, H, D) contiguous, so the C = H*D channels -# for a given (b, n, qkv_idx) live in a contiguous memory span. - - -@triton.jit -def _fused_qk_inv_rms_kernel( - qkv_ptr, # *T_in (B, N, 3, H, D), contiguous - q_inv_rms_ptr, # *float32 (B, N) - k_inv_rms_ptr, # *float32 (B, N) - N: tl.constexpr, - C: tl.constexpr, # H * D - eps, - BLOCK_C: tl.constexpr, -): - bn_id = tl.program_id(0) - qkv_row_stride = 3 * C - row_base = bn_id * qkv_row_stride - q_base = row_base - k_base = row_base + C - - offs = tl.arange(0, BLOCK_C) - mask = offs < C - - q_vals = tl.load(qkv_ptr + q_base + offs, mask=mask, other=0.0).to(tl.float32) - k_vals = tl.load(qkv_ptr + k_base + offs, mask=mask, other=0.0).to(tl.float32) - - q_sq = tl.sum(q_vals * q_vals, axis=0) - k_sq = tl.sum(k_vals * k_vals, axis=0) - - inv_c = 1.0 / C - q_inv = tl.rsqrt(q_sq * inv_c + eps) - k_inv = tl.rsqrt(k_sq * inv_c + eps) - - tl.store(q_inv_rms_ptr + bn_id, q_inv) - tl.store(k_inv_rms_ptr + bn_id, k_inv) - - -def fused_qk_inv_rms( - qkv: torch.Tensor, - eps: float = 1e-5, -) -> tuple[torch.Tensor, torch.Tensor]: - """Single-pass Triton fused Q+K inverse-RMS. - - Replaces ``(_precompute_inv_rms(qkv, 0, C, eps), _precompute_inv_rms(qkv, 1, C, eps))`` - with one launch that reads each ``(b, n)`` row of ``qkv`` exactly once. - - Args: - qkv: (B, N, 3, H, D) contiguous tensor, any fp dtype. - eps: RMSNorm epsilon. - - Returns: - (q_inv_rms, k_inv_rms), each (B, N) float32 contiguous. - """ - assert qkv.is_contiguous(), "qkv must be contiguous (B, N, 3, H, D)" - assert qkv.dim() == 5 and qkv.shape[2] == 3, f"expected (B, N, 3, H, D), got {tuple(qkv.shape)}" - B, N, _, H, D = qkv.shape - C = H * D - q_inv_rms = torch.empty((B, N), dtype=torch.float32, device=qkv.device) - k_inv_rms = torch.empty((B, N), dtype=torch.float32, device=qkv.device) - BLOCK_C = triton.next_power_of_2(C) - _fused_qk_inv_rms_kernel[(B * N,)]( - qkv, - q_inv_rms, - k_inv_rms, - N=N, - C=C, - eps=eps, - BLOCK_C=BLOCK_C, - ) - return q_inv_rms, k_inv_rms - - -@triton.jit -def _fused_bidi_merge_kernel( - num_fwd_ptr, - num_bwd_ptr, - den_fwd_ptr, - den_bwd_ptr, - gate_ptr, - out_ptr, - B, - N, - H, - D, - eps, - snum_b, - snum_n, - snum_h, - snum_d, - sden_b, - sden_h, - sden_n, - APPLY_GATE: tl.constexpr, - PRE_SUMMED: tl.constexpr, - BLOCK_N: tl.constexpr, - BLOCK_D: tl.constexpr, -): - pid_bh = tl.program_id(0) - pid_n = tl.program_id(1) - b = pid_bh // H - h = pid_bh % H - - offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) - offs_d = tl.arange(0, BLOCK_D) - mask_n = offs_n < N - mask_d = offs_d < D - mask_nd = mask_n[:, None] & mask_d[None, :] - - num_base = b * snum_b + offs_n[:, None] * snum_n + h * snum_h + offs_d[None, :] * snum_d - nf = tl.load(num_fwd_ptr + num_base, mask=mask_nd, other=0.0).to(tl.float32) - den_base = b * sden_b + h * sden_h + offs_n * sden_n - df = tl.load(den_fwd_ptr + den_base, mask=mask_n, other=0.0).to(tl.float32) - - if PRE_SUMMED: - num_total = nf - den_total = df + eps - else: - nb = tl.load(num_bwd_ptr + num_base, mask=mask_nd, other=0.0).to(tl.float32) - db = tl.load(den_bwd_ptr + den_base, mask=mask_n, other=0.0).to(tl.float32) - num_total = nf + nb - den_total = df + db + eps - out_val = num_total / den_total[:, None] - - if APPLY_GATE: - g = tl.load(gate_ptr + num_base, mask=mask_nd, other=0.0).to(tl.float32) - silu_g = g * (1.0 / (1.0 + tl.exp(-g))) - out_val = out_val * silu_g - - tl.store(out_ptr + num_base, out_val.to(tl.bfloat16), mask=mask_nd) - - -def fused_bidi_merge( - num_fwd: torch.Tensor, - num_bwd: torch.Tensor | None, - den_fwd: torch.Tensor, - den_bwd: torch.Tensor | None, - eps: float, - gate: torch.Tensor | None = None, -) -> torch.Tensor: - pre_summed = num_bwd is None - assert (num_bwd is None) == (den_bwd is None), "num_bwd/den_bwd must both be None or both provided" - if not pre_summed: - assert num_fwd.shape == num_bwd.shape and den_fwd.shape == den_bwd.shape - assert num_fwd.dtype == num_bwd.dtype and den_fwd.dtype == den_bwd.dtype - B, N, H, D = num_fwd.shape - out = torch.empty( - B, N, H, D, device=num_fwd.device, dtype=(torch.float32 if num_fwd.dtype == torch.float32 else torch.bfloat16) - ) - BLOCK_D = triton.next_power_of_2(D) - BLOCK_N = 64 - grid = (B * H, triton.cdiv(N, BLOCK_N)) - if gate is not None: - assert gate.shape == (B, N, H, D), f"gate shape {gate.shape} != {(B, N, H, D)}" - gate_arg = gate - apply_gate = 1 - else: - gate_arg = num_fwd - apply_gate = 0 - num_bwd_arg = num_bwd if num_bwd is not None else num_fwd - den_bwd_arg = den_bwd if den_bwd is not None else den_fwd - _fused_bidi_merge_kernel[grid]( - num_fwd, - num_bwd_arg, - den_fwd, - den_bwd_arg, - gate_arg, - out, - B, - N, - H, - D, - float(eps), - num_fwd.stride(0), - num_fwd.stride(1), - num_fwd.stride(2), - num_fwd.stride(3), - den_fwd.stride(0), - den_fwd.stride(1), - den_fwd.stride(2), - APPLY_GATE=apply_gate, - PRE_SUMMED=1 if pre_summed else 0, - BLOCK_N=BLOCK_N, - BLOCK_D=BLOCK_D, - ) - return out - - -# ===================================================================== -# Single-direction GDN entry point (delegates to chunkwise) -# ===================================================================== - - -def fused_gdn_func( - qkv: torch.Tensor, # (B, N, 3, H, D) - q_inv_rms: torch.Tensor, # (B, N) float32 - k_inv_rms: torch.Tensor, # (B, N) float32 - q_norm_weight: torch.Tensor, # (C,) = (H*D,) float32 - k_norm_weight: torch.Tensor, # (C,) float32 - rope_cos: torch.Tensor, # (N, D) float32 - rope_sin: torch.Tensor, # (N, D) float32 - beta: torch.Tensor, # (B, H, F, S) - decay: torch.Tensor, # (B, H, F) - F: int, - S: int, - k_scale: float, - eps: float = 1e-6, - reverse: bool = False, - init_state_kv: torch.Tensor | None = None, - init_state_z: torch.Tensor | None = None, - save_final_state: bool = False, -) -> tuple[torch.Tensor, torch.Tensor] | tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """One direction of fused BiGDN via the unified kernel. - - Args: - qkv .. eps: see kernel signature. - reverse: forward (False) or anti-causal (True) scan. - init_state_kv: optional ``(B*H, BLOCK_D, BLOCK_D)`` fp32 contiguous - tensor holding the forward-scan KV state at the END of a prefix - sequence (i.e., AFTER the prefix's last update, BEFORE any further - decay applied by this call). When provided, the kernel resumes the - scan from this state instead of zero. ``BLOCK_D = next_pow2(D)``. - Only the top-left ``D x D`` submatrix of the tile is read. - init_state_z: optional ``(B*H, BLOCK_D)`` fp32 contiguous companion - for the Z denominator state. Must be provided iff ``init_state_kv`` - is provided. - save_final_state: when True, allocate fresh fp32 zero buffers for the - final KV / Z state (after the last frame's update) and pass them to - the kernel for write-out. Returns the buffers as additional outputs. - - Returns: - ``(num, den)`` — bf16 numerator ``(B, N, H, D)`` and denominator - ``(B, H, N)`` before divide. - - When ``save_final_state=True``, also returns - ``(final_state_kv, final_state_z)`` fp32 with shapes - ``(B*H, BLOCK_D, BLOCK_D)`` and ``(B*H, BLOCK_D)``. - - Raises: - NotImplementedError: if any state I/O argument is set together with - ``reverse=True``. The kernel supports state passing in both - directions, but state I/O is only defined for the forward direction - here to avoid silent misuse. - """ - # Dispatch both the stateless bidi case and the stateful forward path to - # chunkwise so split-equivalence uses one numeric implementation. - # Bypass via env: FUSED_GDN_FORCE_LEGACY=1. - if os.environ.get("FUSED_GDN_FORCE_LEGACY", "0") != "1": - from .fused_gdn_chunkwise import ( - fused_gdn_func_chunkwise, - fused_gdn_stateful_chunkwise, - ) - - # Validate state I/O args upfront — preserves the legacy fused_gdn_func's - # validation contract (callers depend on these specific ValueError / - # NotImplementedError signatures, e.g., test_state_validation). - if (init_state_kv is None) != (init_state_z is None): - raise ValueError( - "fused_gdn_func: init_state_kv and init_state_z must be provided together " - "(both None or both fp32 tensors)." - ) - if reverse and (init_state_kv is not None or save_final_state): - raise NotImplementedError( - "fused_gdn_func: state passing (init_state_kv / init_state_z / " - "save_final_state) is only supported for the forward direction " - "(reverse=False)." - ) - if init_state_kv is not None: - B_q, _N, _three, H_q, D_q = qkv.shape - BLOCK_D_q = triton.next_power_of_2(D_q) - expected_kv = (B_q * H_q, BLOCK_D_q, BLOCK_D_q) - expected_z = (B_q * H_q, BLOCK_D_q) - if tuple(init_state_kv.shape) != expected_kv: - raise ValueError( - f"fused_gdn_func: init_state_kv shape {tuple(init_state_kv.shape)} != " f"expected {expected_kv}." - ) - if tuple(init_state_z.shape) != expected_z: - raise ValueError( - f"fused_gdn_func: init_state_z shape {tuple(init_state_z.shape)} != " f"expected {expected_z}." - ) - if init_state_kv.dtype != torch.float32 or init_state_z.dtype != torch.float32: - raise ValueError( - f"fused_gdn_func: init_state_kv/init_state_z must be fp32 " - f"(got {init_state_kv.dtype}, {init_state_z.dtype})." - ) - if not init_state_kv.is_contiguous() or not init_state_z.is_contiguous(): - raise ValueError("fused_gdn_func: init_state_kv / init_state_z must be contiguous.") - - # Stateless path - if init_state_kv is None and init_state_z is None and not save_final_state: - return fused_gdn_func_chunkwise( - qkv, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - beta, - decay, - F=F, - S=S, - k_scale=k_scale, - eps=eps, - reverse=reverse, - ) - - # Stateful path: shape-adapt state I/O. - # state_kv: (B*H, BLOCK_D, BLOCK_D) row-major as M[K_feat, V_feat] - # chunkwise stateful: takes user-facing (B, H, D_in, D_out) and transposes - # internally to (B*H, D_out, D_in) for kernel storage. - # state_z: (B*H, BLOCK_D) - # chunkwise stateful: (B, H, D, 1) or (B, H, D) - B, N, _three, H, D = qkv.shape - BLOCK_D = triton.next_power_of_2(D) - - ck_init_kv = None - ck_init_z = None - if init_state_kv is not None: - # (B*H, BLOCK_D, BLOCK_D) → (B, H, BLOCK_D, BLOCK_D)[:, :, :D, :D] - # then transpose so chunkwise's internal `.transpose(-1, -2)` undoes it. - ck_init_kv = init_state_kv.view(B, H, BLOCK_D, BLOCK_D)[:, :, :D, :D].transpose(-1, -2).contiguous() - if init_state_z is not None: - # (B*H, BLOCK_D) → (B, H, BLOCK_D)[:, :, :D] → (B, H, D, 1) - ck_init_z = init_state_z.view(B, H, BLOCK_D)[:, :, :D].unsqueeze(-1).contiguous() - - result = fused_gdn_stateful_chunkwise( - qkv, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - beta, - decay, - F=F, - S=S, - k_scale=k_scale, - eps=eps, - reverse=reverse, - init_state_kv=ck_init_kv, - init_state_z=ck_init_z, - return_final_state=save_final_state, - ) - - if not save_final_state: - return result # (num, den) - - num, den, ck_state_kv, ck_state_z = result - # chunkwise returns state_kv as (B, H, D, D), [K_feat, V_feat] (post its - # internal back-transpose). Convert to stateful (B*H, BLOCK_D, BLOCK_D) - # by transposing back to internal storage and padding to BLOCK_D. - out_state_kv = torch.zeros(B * H, BLOCK_D, BLOCK_D, device=qkv.device, dtype=torch.float32) - out_state_kv[:, :D, :D] = ck_state_kv.transpose(-1, -2).reshape(B * H, D, D) - out_state_z = torch.zeros(B * H, BLOCK_D, device=qkv.device, dtype=torch.float32) - out_state_z[:, :D] = ck_state_z.squeeze(-1).reshape(B * H, D) - return num, den, out_state_kv, out_state_z - - B, N, three, H, D = qkv.shape - assert three == 3 - - BLOCK_D, BLOCK_S, dot_prec, state_fp32, nw, cfg, beta, decay = _prepare_launch(D, beta, decay) - - has_init_state = init_state_kv is not None or init_state_z is not None - if reverse and (has_init_state or save_final_state): - raise NotImplementedError( - "fused_gdn_func: state passing (init_state_kv / init_state_z / " - "save_final_state) is only supported for the forward direction " - "(reverse=False). The chunk-causal anti-causal pass resets state " - "per chunk and has no global cross-prefix state to cache." - ) - - if has_init_state: - if init_state_kv is None or init_state_z is None: - raise ValueError( - "fused_gdn_func: init_state_kv and init_state_z must be " - "provided together (got " - f"init_state_kv={'set' if init_state_kv is not None else 'None'}, " - f"init_state_z={'set' if init_state_z is not None else 'None'})." - ) - expected_kv_shape = (B * H, BLOCK_D, BLOCK_D) - expected_z_shape = (B * H, BLOCK_D) - if tuple(init_state_kv.shape) != expected_kv_shape: - raise ValueError( - f"fused_gdn_func: init_state_kv shape {tuple(init_state_kv.shape)} " - f"does not match expected {expected_kv_shape} (BLOCK_D=next_pow2(D)={BLOCK_D})." - ) - if tuple(init_state_z.shape) != expected_z_shape: - raise ValueError( - f"fused_gdn_func: init_state_z shape {tuple(init_state_z.shape)} " - f"does not match expected {expected_z_shape}." - ) - if init_state_kv.dtype != torch.float32 or init_state_z.dtype != torch.float32: - raise ValueError( - "fused_gdn_func: init_state_kv and init_state_z must be fp32 " - f"(got {init_state_kv.dtype}, {init_state_z.dtype})." - ) - if not init_state_kv.is_contiguous() or not init_state_z.is_contiguous(): - raise ValueError("fused_gdn_func: init_state_kv and init_state_z must be contiguous.") - if init_state_kv.device != qkv.device or init_state_z.device != qkv.device: - raise ValueError("fused_gdn_func: init_state_* must live on the same device as qkv.") - load_init = 1 - init_kv_arg = init_state_kv - init_z_arg = init_state_z - else: - load_init = 0 - init_kv_arg = None # placeholder set below - - if save_final_state: - final_state_kv = torch.zeros(B * H, BLOCK_D, BLOCK_D, device=qkv.device, dtype=torch.float32) - final_state_z = torch.zeros(B * H, BLOCK_D, device=qkv.device, dtype=torch.float32) - save_final = 1 - else: - final_state_kv = None - final_state_z = None - save_final = 0 - - num = torch.empty(B, N, H, D, device=qkv.device, dtype=qkv.dtype) - den = torch.empty(B, H, N, device=qkv.device, dtype=qkv.dtype) - dummy = torch.empty(1, device=qkv.device, dtype=torch.float32) - - # Resolve pointer args for the unused slots to a shared scratch tensor; - # the kernel compiles the corresponding load/store away when the - # constexpr flag is 0. - init_kv_ptr = init_kv_arg if load_init else dummy - init_z_ptr = init_z_arg if load_init else dummy - final_kv_ptr = final_state_kv if save_final else dummy - final_z_ptr = final_state_z if save_final else dummy - - _fused_gdn_kernel[(B * H,)]( - qkv, - qkv.stride(0), - qkv.stride(1), - qkv.stride(2), - qkv.stride(3), - qkv.stride(4), - beta, - decay, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - num, # `out_ptr` reuses `num` buffer (result immediately overwritten below) - num, - den, - dummy, - dummy, - dummy, - dummy, # saved-state dummies (SAVE_STATE=0) - init_kv_ptr, - init_z_ptr, - final_kv_ptr, - final_z_ptr, - H=H, - F=F, - S=S, - D=D, - K_SCALE=k_scale, - NORM_EPS=1e-5, # unused with USE_PRECOMPUTED_RMS=1 - EPS=eps, - QK_NORM=1, - USE_PRECOMPUTED_RMS=1, - STATE_FP32=1 if state_fp32 else 0, - DOT_PRECISION=dot_prec, - REVERSE=1 if reverse else 0, - SAVE_STATE=0, - LOAD_INIT_STATE=load_init, - SAVE_FINAL_STATE=save_final, - BLOCK_D=BLOCK_D, - BLOCK_S=BLOCK_S, - num_stages=cfg["num_stages"], - num_warps=nw, - ) - if save_final_state: - return num, den, final_state_kv, final_state_z - return num, den - - -def fused_bigdn_func( - qkv: torch.Tensor, # (B, N, 3, H, D) - q_inv_rms: torch.Tensor, # (B, N) — pre-computed via `_precompute_inv_rms` - k_inv_rms: torch.Tensor, # (B, N) - q_norm_weight: torch.Tensor, # (C,) float32 - k_norm_weight: torch.Tensor, # (C,) - rope_cos: torch.Tensor, # (N, D) - rope_sin: torch.Tensor, # (N, D) - beta: torch.Tensor, # (B, H, F, S) - decay: torch.Tensor, # (B, H, F) - F: int, - S: int, - k_scale: float, - eps: float = 1e-6, - # -- chunk-causal extensions (not in upstream; see adapter notes below) -- - qkv_bwd: torch.Tensor | None = None, - beta_bwd: torch.Tensor | None = None, - decay_bwd: torch.Tensor | None = None, - q_inv_rms_bwd: torch.Tensor | None = None, - k_inv_rms_bwd: torch.Tensor | None = None, -) -> torch.Tensor: - """Full bidirectional fused GDN. - - Returns: out (B, N, H, D) bf16 = (num_fwd + num_bwd) / (den_fwd + den_bwd + eps). - - Chunk-causal extensions (optional): - For chunk-causal GDN we need to zero state at chunk boundaries in the - BACKWARD direction only. Pass separately pre-processed backward tensors - (decay_bwd with zeros at boundary frames, and optionally qkv_bwd / - beta_bwd with K/V or beta zeroed at boundary frames). If any `*_bwd` - argument is None, the forward tensor is reused. - """ - if ( - os.environ.get("FUSED_GDN_FORCE_LEGACY", "0") != "1" - and qkv_bwd is None - and beta_bwd is None - and decay_bwd is None - and q_inv_rms_bwd is None - and k_inv_rms_bwd is None - ): - from .fused_gdn_chunkwise import fused_bigdn_bidi_chunkwise - - return fused_bigdn_bidi_chunkwise( - qkv, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - beta, - decay, - F=F, - S=S, - k_scale=k_scale, - eps=eps, - ) - - num_fwd, den_fwd = fused_gdn_func( - qkv, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - beta, - decay, - F=F, - S=S, - k_scale=k_scale, - eps=eps, - reverse=False, - ) - num_bwd, den_bwd = fused_gdn_func( - qkv if qkv_bwd is None else qkv_bwd, - q_inv_rms if q_inv_rms_bwd is None else q_inv_rms_bwd, - k_inv_rms if k_inv_rms_bwd is None else k_inv_rms_bwd, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - beta if beta_bwd is None else beta_bwd, - decay if decay_bwd is None else decay_bwd, - F=F, - S=S, - k_scale=k_scale, - eps=eps, - reverse=True, - ) - # num: (B, N, H, D), den: (B, H, N). Fuse then divide. - total_num = num_fwd + num_bwd - total_den = (den_fwd + den_bwd).permute(0, 2, 1).unsqueeze(-1) # (B, N, H, 1) - return total_num / (total_den + eps) - - -# ===================================================================== -# Backward / autograd Functions -# ===================================================================== -# Adds: -# 1. ``_fused_gdn_bwd_kernel`` -- Triton jit kernel that replays the -# forward recurrence in reverse time using per-frame state snapshots -# written by the forward kernel under ``SAVE_STATE=1``. -# 2. ``_run_fwd_save`` -- helper that runs the existing forward -# ``_fused_gdn_kernel`` with ``SAVE_STATE=1``. Adapted to pass our -# extra ``init_state_kv_ptr / init_state_z_ptr / final_state_kv_ptr / -# final_state_z_ptr`` pointers + ``LOAD_INIT_STATE / SAVE_FINAL_STATE`` -# constexpr flags (all unused on the autograd path -> dummy / 0). -# 3. ``FusedGDNFunction`` -- autograd Function for unidirectional GDN -# with ``QK_NORM=1`` (in-kernel per-head RMSNorm). -# 4. ``FusedBiGDNFunction`` -- autograd Function for bidirectional BiGDN. -# Pre-normalizes Q/K in PyTorch with full-channel RMSNorm, runs the -# forward kernel twice with ``QK_NORM=0 + SAVE_STATE=1``, fuses -# ``(num_fwd + num_bwd) / (den_fwd + den_bwd + eps)``. Backward -# computes ``dnum / dden`` from upstream ``dout`` and runs the bwd -# kernel twice with ``BIDI_MODE=1``. -# 5. Python wrappers ``fused_gdn_forward_with_grad`` / -# ``fused_bigdn_forward_with_grad`` -- drop-in autograd-enabled -# replacements for ``fused_gdn_func`` / ``fused_bigdn_func``. -# -# Chunk-causal autograd support: ``FusedBiGDNFunction`` (and the public -# wrapper ``fused_bigdn_forward_with_grad``) accepts optional -# ``beta_bwd`` / ``decay_bwd`` overrides for the reverse-direction -# kernel call -- exactly the same masking convention used by the -# inference path ``fused_bigdn_func``. When provided, the reverse -# direction's forward and backward kernels both run on these masked -# tensors, and the backward returns separate gradient tensors -# (``dbeta_bwd`` / ``ddecay_bwd``) so autograd can route them back -# through any ``clone() + index = 0`` masking the caller applied. - - -@triton.jit -def _fused_gdn_bwd_kernel( - # ---- original inputs ---- - qkv_ptr, - stride_b: tl.constexpr, - stride_n: tl.constexpr, - stride_3: tl.constexpr, - stride_h: tl.constexpr, - stride_d: tl.constexpr, - beta_ptr, - decay_ptr, - q_norm_w_ptr, - k_norm_w_ptr, - rope_cos_ptr, - rope_sin_ptr, - # ---- saved from forward ---- - saved_state_ptr, # (B*H, F, BLOCK_D, BLOCK_D) -- state_prev snapshots - saved_z_ptr, # (B*H, F, BLOCK_D) - saved_state_curr_ptr, # (B*H, F, BLOCK_D, BLOCK_D) -- state_curr (after update) - saved_z_curr_ptr, # (B*H, F, BLOCK_D) - # ---- upstream gradient / pre-computed dnum ---- - dout_ptr, # GDN mode: (B, N, H, D) upstream grad. BiDI mode: pre-computed dnum - # ---- BiDI mode: external dden ---- - dden_ext_ptr, # BiDI mode: (B, H, N) pre-computed dden. GDN mode: unused - # ---- output gradients ---- - dqkv_ptr, # (B, N, 3, H, D) -- same layout as qkv - dbeta_ptr, # (B, H, F, S) - ddecay_ptr, # (B, H, F) - # ---- dims ---- - H: tl.constexpr, - F: tl.constexpr, - S: tl.constexpr, - D: tl.constexpr, - K_SCALE, - NORM_EPS: tl.constexpr, - EPS: tl.constexpr, - QK_NORM: tl.constexpr, - STATE_FP32: tl.constexpr, - REVERSE_BWD: tl.constexpr, # 0=backward of forward GDN, 1=backward of reversed GDN - BIDI_MODE: tl.constexpr, # 0=GDN (compute dnum/dden), 1=BiGDN (use provided) - DOT_PRECISION: tl.constexpr, # 0=bf16 TC, 1=TF32 TC, 2=IEEE fp32 - BLOCK_D: tl.constexpr, - BLOCK_S: tl.constexpr, -): - pid = tl.program_id(0) - pid_b = pid // H - pid_h = pid % H - N: tl.constexpr = F * S - bh = pid_b * H + pid_h - - qkv_bh = qkv_ptr + pid_b * stride_b + pid_h * stride_h - dqkv_bh = dqkv_ptr + pid_b * stride_b + pid_h * stride_h - dout_bh = dout_ptr + pid_b * (N * H * D) + pid_h * D - beta_bh = beta_ptr + bh * (F * S) - decay_bh = decay_ptr + bh * F - dbeta_bh = dbeta_ptr + bh * (F * S) - ddecay_bh = ddecay_ptr + bh * F - st_bh = saved_state_ptr + bh * F * BLOCK_D * BLOCK_D - sz_bh = saved_z_ptr + bh * F * BLOCK_D - stc_bh = saved_state_curr_ptr + bh * F * BLOCK_D * BLOCK_D - szc_bh = saved_z_curr_ptr + bh * F * BLOCK_D - if BIDI_MODE: - dden_ext_bh = dden_ext_ptr + bh * N - - offs_d = tl.arange(0, BLOCK_D) - mask_d = offs_d < D - offs_d_pair = offs_d ^ 1 - mask_d_pair = offs_d_pair < D - - nw_offset = pid_h * D - if QK_NORM: - q_nw = tl.load(q_norm_w_ptr + nw_offset + offs_d, mask=mask_d, other=0.0).to(tl.float32) - k_nw = tl.load(k_norm_w_ptr + nw_offset + offs_d, mask=mask_d, other=0.0).to(tl.float32) - q_nw_pair = tl.load(q_norm_w_ptr + nw_offset + offs_d_pair, mask=mask_d_pair, other=0.0).to(tl.float32) - k_nw_pair = tl.load(k_norm_w_ptr + nw_offset + offs_d_pair, mask=mask_d_pair, other=0.0).to(tl.float32) - - D_inv = 1.0 / D - k_scale = K_SCALE - - # Dot precision: mirror forward kernel - if DOT_PRECISION >= 1: - dot_dtype = tl.float32 - else: - dot_dtype = tl.bfloat16 - dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" - - # Gradient matmuls: always use bf16 TC + TF32 input precision (matching PyTorch backward) - grad_dtype = tl.bfloat16 - grad_ip: tl.constexpr = "tf32" - - # ---- Gradient state accumulators (reverse time) ---- - dstate = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) - dstate_z = tl.zeros([BLOCK_D], dtype=tl.float32) - - for f_rev in range(F): - # Backward iterates in reverse of forward direction. - if REVERSE_BWD: - f = f_rev # backward of reversed GDN: iterate 0..F-1 - # In fwd_save REVERSE, q_frame=f had kv_frame=f+1 (or skip at f=F-1). - kv_frame_bwd = f + 1 if f < F - 1 else f - skip_bwd = f == F - 1 # f=F-1 was dummy step (f_iter=0 in fwd) - else: - f = F - 1 - f_rev # backward of forward GDN: iterate F-1..0 - kv_frame_bwd = f - skip_bwd = False - q_n_base = f * S - kv_n_base = kv_frame_bwd * S - f_beta = beta_bh + kv_frame_bwd * S - - # ---- Load state_curr for Pass 2 output (both directions use inclusive) ---- - st_f = st_bh + f * BLOCK_D * BLOCK_D - offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] - mask_dd = mask_d[:, None] & mask_d[None, :] - - stc_f = stc_bh + f * BLOCK_D * BLOCK_D - P_state = tl.load(stc_f + offs_dd, mask=mask_dd, other=0.0) - Pz_state = tl.load(szc_bh + f * BLOCK_D + offs_d, mask=mask_d, other=0.0) - if STATE_FP32 == 0: - P_state = P_state.to(tl.float32) - - # Decay: for REVERSE_BWD, use decay[kv_frame] matching fwd_save. - if REVERSE_BWD and skip_bwd: - g = 1.0 - elif REVERSE_BWD: - g = tl.load(decay_bh + kv_frame_bwd).to(tl.float32) - else: - g = tl.load(decay_bh + f).to(tl.float32) - - # ======================================================== - # Pass 2 backward: Output gradients -> dQ, dstate, dstate_z - # ======================================================== - for s0 in range(0, S, BLOCK_S): - offs_s = s0 + tl.arange(0, BLOCK_S) - mask_s = offs_s < S - mask_sd = mask_s[:, None] & mask_d[None, :] - mask_sd_pair = mask_s[:, None] & mask_d_pair[None, :] - n_idx = q_n_base + offs_s # Q data from q_frame - - # Load dout; recompute Q, Q_pair, Q_rot, num, den from saved P_f/Pz_f. - dout_ptrs = dout_bh + n_idx[:, None] * (H * D) + offs_d[None, :] - d_out = tl.load(dout_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - - # Recompute Q, Q_pair, Q_rot (same as forward). - q_ptrs = qkv_bh + n_idx[:, None] * stride_n + 0 * stride_3 + offs_d[None, :] * stride_d - Q_raw = tl.load(q_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - q_pair_ptrs = qkv_bh + n_idx[:, None] * stride_n + 0 * stride_3 + offs_d_pair[None, :] * stride_d - Q_pair_raw = tl.load(q_pair_ptrs, mask=mask_sd_pair, other=0.0).to(tl.float32) - - if QK_NORM: - q_var = tl.sum(Q_raw * Q_raw, axis=1) * D_inv - q_inv_rms = 1.0 / tl.sqrt(q_var + NORM_EPS) - Q_normed = Q_raw * q_inv_rms[:, None] * q_nw[None, :] - Q_pair_normed = Q_pair_raw * q_inv_rms[:, None] * q_nw_pair[None, :] - else: - Q_normed = Q_raw - Q_pair_normed = Q_pair_raw - Q = tl.where(Q_normed > 0, Q_normed, 0.0) - Q_pair = tl.where(Q_pair_normed > 0, Q_pair_normed, 0.0) - - rope_ptrs = n_idx[:, None] * D + offs_d[None, :] - Cos = tl.load(rope_cos_ptr + rope_ptrs, mask=mask_sd, other=1.0).to(tl.float32) - Sin = tl.load(rope_sin_ptr + rope_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - Q_rot = Q * Cos + Q_pair * Sin - - # Compute dnum and dden. - if BIDI_MODE: - # BiGDN: dnum and dden pre-computed externally from total num/den. - dnum = d_out # dout_ptr already contains pre-computed dnum - dden = tl.load(dden_ext_bh + n_idx, mask=mask_s, other=0.0).to(tl.float32) - else: - # GDN: recompute num/den using direction-appropriate state. - num_tile = tl.dot( - Q_rot.to(dot_dtype), P_state.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip - ) - den_tile = tl.sum(Q * Pz_state[None, :], axis=1) - inv_den = 1.0 / (den_tile + EPS) - dnum = d_out * inv_den[:, None] - dden = -tl.sum(d_out * num_tile, axis=1) * inv_den * inv_den - - # dstate += Q_rot^T @ dnum (state contribution from num = Q_rot @ P_state). - dstate = dstate + tl.dot( - tl.trans(Q_rot.to(grad_dtype)), - dnum.to(grad_dtype), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - - # dstate_z += sum(dden * Q, axis=0) (Pz contribution from den = Q . Pz). - dstate_z += tl.sum(dden[:, None] * Q, axis=0) - - # dQ_rot = dnum @ P_state^T (uses state that forward's output read). - dQ_rot = tl.dot( - dnum.to(grad_dtype), - tl.trans(P_state.to(grad_dtype)), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - - # dQ_from_den = dden * Pz_state. - dQ_from_den = dden[:, None] * Pz_state[None, :] - - # RoPE inverse for Q: store dQ_rot, reload at paired indices. - # Store dQ_rot temporarily to dqkv[Q] at normal d positions. - dq_ptrs = dqkv_bh + n_idx[:, None] * stride_n + 0 * stride_3 + offs_d[None, :] * stride_d - tl.store(dq_ptrs, dQ_rot.to(tl.bfloat16), mask=mask_sd) - # The XOR-paired channel can be owned by another warp. Synchronize - # both sides of the scratch roundtrip before dqkv is overwritten. - tl.debug_barrier() - - # Load dQ_rot at paired positions. - dq_pair_ptrs = dqkv_bh + n_idx[:, None] * stride_n + 0 * stride_3 + offs_d_pair[None, :] * stride_d - dQ_rot_pair = tl.load(dq_pair_ptrs, mask=mask_sd_pair, other=0.0).to(tl.float32) - tl.debug_barrier() - - # RoPE inverse: dQ = dQ_rot * Cos - dQ_rot_pair * Sin. - dQ = dQ_rot * Cos - dQ_rot_pair * Sin + dQ_from_den - - # ReLU backward. - relu_mask_q = (Q_normed > 0).to(tl.float32) - dQ_normed = dQ * relu_mask_q - - # Norm backward (QK_NORM) or direct (no norm). - if QK_NORM: - gw = dQ_normed * q_nw[None, :] - corr = tl.sum(gw * Q_raw, axis=1) * D_inv * q_inv_rms * q_inv_rms - dQ_raw = q_inv_rms[:, None] * (gw - Q_raw * corr[:, None]) - else: - dQ_raw = dQ_normed - - # Store final dQ_raw to dqkv[Q]. - tl.store(dq_ptrs, dQ_raw.to(tl.bfloat16), mask=mask_sd) - - # Both directions use inclusive output (state_curr), so capture dDelta AFTER Pass 2. - dDelta = dstate - dDelta_z = dstate_z - - # ======================================================== - # Reload state_prev for Pass 1 backward (reuse P_state variable) - # ======================================================== - P_state = tl.load(st_f + offs_dd, mask=mask_dd, other=0.0) - if STATE_FP32 == 0: - P_state = P_state.to(tl.float32) - Pz_state = tl.load(sz_bh + f * BLOCK_D + offs_d, mask=mask_d, other=0.0) - - # ======================================================== - # Pass 1 backward: State update gradients -> dK, dV, dbeta, dstate - # Skip for REVERSE_BWD dummy frame (skip_bwd=True) to avoid clobbering. - # ======================================================== - if skip_bwd == False: - for s0 in range(0, S, BLOCK_S): - offs_s = s0 + tl.arange(0, BLOCK_S) - mask_s = offs_s < S - mask_sd = mask_s[:, None] & mask_d[None, :] - mask_sd_pair = mask_s[:, None] & mask_d_pair[None, :] - n_idx = kv_n_base + offs_s # K/V from kv_frame - - # Recompute K, K_pair, K_rot, V. - k_ptrs = qkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d[None, :] * stride_d - v_ptrs = qkv_bh + n_idx[:, None] * stride_n + 2 * stride_3 + offs_d[None, :] * stride_d - K_raw = tl.load(k_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - V_raw = tl.load(v_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - - k_pair_ptrs = qkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d_pair[None, :] * stride_d - K_pair_raw = tl.load(k_pair_ptrs, mask=mask_sd_pair, other=0.0).to(tl.float32) - - if QK_NORM: - k_var = tl.sum(K_raw * K_raw, axis=1) * D_inv - k_inv_rms = 1.0 / tl.sqrt(k_var + NORM_EPS) - K_normed = K_raw * k_inv_rms[:, None] * k_nw[None, :] - K_pair_normed = K_pair_raw * k_inv_rms[:, None] * k_nw_pair[None, :] - else: - K_normed = K_raw - K_pair_normed = K_pair_raw - K = tl.where(K_normed > 0, K_normed, 0.0) * k_scale - K_pair = tl.where(K_pair_normed > 0, K_pair_normed, 0.0) * k_scale - - rope_ptrs = n_idx[:, None] * D + offs_d[None, :] - Cos = tl.load(rope_cos_ptr + rope_ptrs, mask=mask_sd, other=1.0).to(tl.float32) - Sin = tl.load(rope_sin_ptr + rope_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - K_rot = K * Cos + K_pair * Sin - - bt = tl.load(f_beta + offs_s, mask=mask_s, other=0.0).to(tl.float32) - - # Recompute V_pred and delta_v. - K_rot_dc = K_rot.to(dot_dtype) - V_pred = tl.dot(K_rot_dc, P_state.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) - delta_v = (V_raw - V_pred) * bt[:, None] - - # ---- KV stream backward ---- - ddelta_v = tl.dot( - K_rot.to(grad_dtype), - dDelta.to(grad_dtype), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - - dK_rot_from_delta = tl.dot( - delta_v.to(grad_dtype), - tl.trans(dDelta.to(grad_dtype)), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - - dV = ddelta_v * bt[:, None] - dbeta_kv = tl.sum(ddelta_v * (V_raw - V_pred), axis=1) - - dV_pred = -ddelta_v * bt[:, None] - dK_rot_from_vpred = tl.dot( - dV_pred.to(grad_dtype), - tl.trans(P_state.to(grad_dtype)), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - - dstate = dstate + tl.dot( - tl.trans(K_rot.to(grad_dtype)), - dV_pred.to(grad_dtype), - out_dtype=tl.float32, - input_precision=grad_ip, - ) - - dK_rot = dK_rot_from_delta + dK_rot_from_vpred - - # ---- Z stream backward ---- - z_hat = tl.sum(K * Pz_state[None, :], axis=1) - dz = (1.0 - z_hat) * bt - - ddz = tl.sum(K * dDelta_z[None, :], axis=1) - dz_hat = -ddz * bt - dK_z = dDelta_z[None, :] * dz[:, None] + dz_hat[:, None] * Pz_state[None, :] - dstate_z = dstate_z + tl.sum(dz_hat[:, None] * K, axis=0) - - dbeta_z = ddz * (1.0 - z_hat) - dbeta_total = dbeta_kv + dbeta_z - tl.store(dbeta_bh + kv_frame_bwd * S + offs_s, dbeta_total.to(tl.bfloat16), mask=mask_s) - - # ---- RoPE inverse for K ---- - dk_ptrs = dqkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d[None, :] * stride_d - tl.store(dk_ptrs, dK_rot.to(tl.bfloat16), mask=mask_sd) - # Match the dQ synchronization: all temporary values must be - # visible before paired loads and consumed before overwrites. - tl.debug_barrier() - dk_pair_ptrs = dqkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d_pair[None, :] * stride_d - dK_rot_pair = tl.load(dk_pair_ptrs, mask=mask_sd_pair, other=0.0).to(tl.float32) - tl.debug_barrier() - - dK_from_kv = dK_rot * Cos - dK_rot_pair * Sin - dK_total = dK_from_kv + dK_z - - relu_mask_k = (K_normed > 0).to(tl.float32) - dK_normed = dK_total * k_scale * relu_mask_k - - if QK_NORM: - gw_k = dK_normed * k_nw[None, :] - corr_k = tl.sum(gw_k * K_raw, axis=1) * D_inv * k_inv_rms * k_inv_rms - dK_raw = k_inv_rms[:, None] * (gw_k - K_raw * corr_k[:, None]) - else: - dK_raw = dK_normed - - tl.store(dk_ptrs, dK_raw.to(tl.bfloat16), mask=mask_sd) - dv_ptrs = dqkv_bh + n_idx[:, None] * stride_n + 2 * stride_3 + offs_d[None, :] * stride_d - tl.store(dv_ptrs, dV.to(tl.bfloat16), mask=mask_sd) - - # ======================================================== - # Decay backward (inside skip_bwd guard) - # ======================================================== - is_first_frame = f_rev == F - 1 - if is_first_frame: - ddecay_f = 0.0 - else: - inv_g = 1.0 / (g + 1e-12) - ddecay_kv = tl.sum(dstate * P_state) * inv_g - ddecay_z_val = tl.sum(dstate_z * Pz_state) * inv_g - ddecay_f = ddecay_kv + ddecay_z_val - tl.store(ddecay_bh + kv_frame_bwd, ddecay_f) - - # Propagate gradient through decay: dS_{f-1} = g[f] * dP_f. - dstate = dstate * g - dstate_z = dstate_z * g - - -# ===================================================================== -# Forward-with-state-save helper (for autograd Functions) -# ===================================================================== - - -def _run_fwd_save( - qkv, - beta, - decay, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - F: int, - S: int, - k_scale: float, - norm_eps: float, - eps: float, - qk_norm: bool, - reverse: bool, - cfg, -): - """Run forward kernel for one direction with ``SAVE_STATE=1``. - - Returns ``(num, den, saved_state, saved_z, saved_state_curr, saved_z_curr)``. - The forward kernel writes ``out = num/(den+eps)`` first and then overwrites - the same buffer with raw ``num``, so the returned ``num`` tensor holds raw - numerator values (matching the BiGDN combine-then-divide convention). - """ - B, N, three, H, D = qkv.shape - BLOCK_D, BLOCK_S, dot_prec, state_fp32, nw, _, beta, decay = _prepare_launch(D, beta, decay) - - num_out = torch.empty(B, N, H, D, device=qkv.device, dtype=qkv.dtype) - den_out = torch.empty(B, H, N, device=qkv.device, dtype=qkv.dtype) - state_dtype = torch.float32 if state_fp32 else torch.bfloat16 - saved_state = torch.empty(B * H, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=state_dtype) - saved_z = torch.empty(B * H, F, BLOCK_D, device=qkv.device, dtype=torch.float32) - saved_state_curr = torch.empty(B * H, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=torch.float32) - saved_z_curr = torch.empty(B * H, F, BLOCK_D, device=qkv.device, dtype=torch.float32) - # The kernel writes ``out = num/(den+eps)`` first then overwrites with raw num - # in the same buffer. Reuse num_out as the (discarded) ``out`` slot so the - # final contents end up being raw num. - out_discard = num_out - dummy_inv = torch.empty(1, device=qkv.device, dtype=torch.float32) - - _fused_gdn_kernel[(B * H,)]( - qkv, - qkv.stride(0), - qkv.stride(1), - qkv.stride(2), - qkv.stride(3), - qkv.stride(4), - beta, - decay, - dummy_inv, - dummy_inv, # unused inv_rms ptrs (USE_PRECOMPUTED_RMS=0) - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - out_discard, - num_out, - den_out, - saved_state, - saved_z, - saved_state_curr, - saved_z_curr, - dummy_inv, - dummy_inv, - dummy_inv, - dummy_inv, # init/final-state dummies - H=H, - F=F, - S=S, - D=D, - K_SCALE=k_scale, - NORM_EPS=norm_eps, - EPS=eps, - QK_NORM=1 if qk_norm else 0, - USE_PRECOMPUTED_RMS=0, - STATE_FP32=1 if state_fp32 else 0, - DOT_PRECISION=dot_prec, - REVERSE=1 if reverse else 0, - SAVE_STATE=1, - LOAD_INIT_STATE=0, - SAVE_FINAL_STATE=0, - BLOCK_D=BLOCK_D, - BLOCK_S=BLOCK_S, - num_stages=cfg["num_stages"], - num_warps=nw, - ) - return num_out, den_out, saved_state, saved_z, saved_state_curr, saved_z_curr - - -# ===================================================================== -# Unidirectional GDN autograd Function -# ===================================================================== - - -class FusedGDNFunction(torch.autograd.Function): - """Autograd Function for unidirectional fused GDN with in-kernel RMSNorm. - - Forward runs ``_fused_gdn_kernel`` with ``QK_NORM=1`` and ``SAVE_STATE=1``, - saving per-frame state snapshots for backward. Backward runs - ``_fused_gdn_bwd_kernel`` with ``BIDI_MODE=0`` (kernel computes - ``dnum``/``dden`` from upstream ``dout``). - """ - - @staticmethod - def forward( - ctx, - qkv, - beta, - decay, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - F: int, - S: int, - k_scale: float = 1.0, - norm_eps: float = 1e-6, - eps: float = 1e-6, - qk_norm: bool = True, - ): - B, N, three, H, D = qkv.shape - assert three == 3 and N == F * S - - BLOCK_D, BLOCK_S, dot_prec, state_fp32, nw, cfg, beta, decay = _prepare_launch(D, beta, decay) - - if q_norm_weight is None: - q_norm_weight = torch.ones(D, device=qkv.device, dtype=torch.float32) - if k_norm_weight is None: - k_norm_weight = torch.ones(D, device=qkv.device, dtype=torch.float32) - - out = torch.empty(B, N, H, D, device=qkv.device, dtype=qkv.dtype) - - # Saved states for backward. - state_dtype = torch.float32 if state_fp32 else torch.bfloat16 - saved_state = torch.empty(B * H, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=state_dtype) - saved_z = torch.empty(B * H, F, BLOCK_D, device=qkv.device, dtype=torch.float32) - saved_state_curr = torch.empty(B * H, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=torch.float32) - saved_z_curr = torch.empty(B * H, F, BLOCK_D, device=qkv.device, dtype=torch.float32) - - # Dummy num/den for forward kernel (still writes them but we discard). - num_out = torch.empty(B, N, H, D, device=qkv.device, dtype=qkv.dtype) - den_out = torch.empty(B, H, N, device=qkv.device, dtype=qkv.dtype) - dummy_inv = torch.empty(1, device=qkv.device, dtype=torch.float32) - - _fused_gdn_kernel[(B * H,)]( - qkv, - qkv.stride(0), - qkv.stride(1), - qkv.stride(2), - qkv.stride(3), - qkv.stride(4), - beta, - decay, - dummy_inv, - dummy_inv, # unused inv_rms ptrs (USE_PRECOMPUTED_RMS=0) - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - out, - num_out, - den_out, - saved_state, - saved_z, - saved_state_curr, - saved_z_curr, - dummy_inv, - dummy_inv, - dummy_inv, - dummy_inv, # init/final-state dummies - H=H, - F=F, - S=S, - D=D, - K_SCALE=k_scale, - NORM_EPS=norm_eps, - EPS=eps, - QK_NORM=1 if qk_norm else 0, - USE_PRECOMPUTED_RMS=0, - STATE_FP32=1 if state_fp32 else 0, - DOT_PRECISION=dot_prec, - REVERSE=0, - SAVE_STATE=1, - LOAD_INIT_STATE=0, - SAVE_FINAL_STATE=0, - BLOCK_D=BLOCK_D, - BLOCK_S=BLOCK_S, - num_stages=cfg["num_stages"], - num_warps=nw, - ) - del num_out, den_out - - ctx.save_for_backward( - qkv, - beta, - decay, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - saved_state, - saved_z, - saved_state_curr, - saved_z_curr, - ) - ctx.F = F - ctx.S = S - ctx.k_scale = k_scale - ctx.norm_eps = norm_eps - ctx.eps = eps - ctx.qk_norm = qk_norm - ctx.dot_prec = dot_prec - return out - - @staticmethod - def backward(ctx, dout): - ( - qkv, - beta, - decay, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - saved_state, - saved_z, - saved_state_curr, - saved_z_curr, - ) = ctx.saved_tensors - - B, N, three, H, D = qkv.shape - F_val = ctx.F - S = ctx.S - - BLOCK_D, BLOCK_S_BWD, _, _, _, cfg, beta, decay = _prepare_launch(D, beta, decay) - dqkv = torch.zeros_like(qkv) - dbeta = torch.zeros_like(beta) - ddecay = torch.zeros_like(decay) - - # Dummy dden_ext (unused in GDN mode). - dden_ext = torch.empty(1, device=qkv.device, dtype=torch.float32) - - # Progressive num_warps reduction on tmem overflow. - nw = cfg["num_warps"] - if ctx.dot_prec >= 1: - nw = min(nw, 4) - while nw >= 1: - try: - _fused_gdn_bwd_kernel[(B * H,)]( - qkv, - qkv.stride(0), - qkv.stride(1), - qkv.stride(2), - qkv.stride(3), - qkv.stride(4), - beta, - decay, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - saved_state, - saved_z, - saved_state_curr, - saved_z_curr, - dout.contiguous(), - dden_ext, - dqkv, - dbeta, - ddecay, - H=H, - F=F_val, - S=S, - D=D, - K_SCALE=ctx.k_scale, - NORM_EPS=ctx.norm_eps, - EPS=ctx.eps, - QK_NORM=1 if ctx.qk_norm else 0, - STATE_FP32=1 if cfg["STATE_FP32"] else 0, - REVERSE_BWD=0, - BIDI_MODE=0, - DOT_PRECISION=ctx.dot_prec, - BLOCK_D=BLOCK_D, - BLOCK_S=BLOCK_S_BWD, - num_stages=cfg["num_stages"], - num_warps=nw, - ) - break - except Exception as e: - if "OutOfResources" in str(type(e).__name__) or "out of resource" in str(e).lower(): - nw = nw // 2 - if nw < 1: - raise RuntimeError( - "FusedGDN backward: Triton kernel OutOfResources at all warp " - f"counts (8, 4, 2, 1). Most recent error: {e}" - ) from e - else: - raise - - return dqkv, dbeta, ddecay, None, None, None, None, None, None, None, None, None, None - - -def fused_gdn_forward_with_grad( - qkv: torch.Tensor, - beta: torch.Tensor, - decay: torch.Tensor, - q_norm_weight: torch.Tensor | None, - k_norm_weight: torch.Tensor | None, - rope_cos: torch.Tensor, - rope_sin: torch.Tensor, - F: int, - S: int, - k_scale: float = 1.0, - norm_eps: float = 1e-6, - eps: float = 1e-6, - qk_norm: bool = True, -) -> torch.Tensor: - """Drop-in autograd-enabled replacement for the unidirectional GDN path. - - Unlike ``fused_gdn_func`` (which expects pre-computed ``q_inv_rms``/ - ``k_inv_rms``), this wrapper computes per-head RMSNorm inside the - Triton kernel (``QK_NORM=1`` / ``USE_PRECOMPUTED_RMS=0``) so the - backward kernel can reproduce the exact same normed Q/K when - replaying the recurrence. - """ - return FusedGDNFunction.apply( - qkv, - beta, - decay, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - F, - S, - k_scale, - norm_eps, - eps, - qk_norm, - ) - - -# ===================================================================== -# Bidirectional BiGDN autograd Function (full-channel RMSNorm in Python) -# ===================================================================== - - -class FusedBiGDNFunction(torch.autograd.Function): - """Autograd Function for bidirectional fused BiGDN. - - Full-channel RMSNorm is applied in Python (so the norm backward can - couple all heads correctly), then the kernel runs with ``QK_NORM=0`` - on the pre-normed QKV. Forward and reverse directions are run - separately with ``SAVE_STATE=1``, and combined as - ``out = (num_fwd + num_bwd) / (den_fwd + den_bwd + eps)``. - - Backward computes ``dnum`` and ``dden`` from upstream ``dout`` and - runs the bwd kernel twice (forward + reverse) with ``BIDI_MODE=1``. - Norm backward is computed in Python (full-channel RMSNorm couples - all heads). - - Chunk-causal masking (optional): - Pass ``beta_bwd`` and/or ``decay_bwd`` to override the beta/decay - tensors used by the **reverse-direction** kernel calls (forward - save + backward). The forward direction always uses the - unmasked ``beta`` / ``decay``. This mirrors the inference path - in :func:`fused_bigdn_func` and unlocks chunk-causal autograd - training: callers typically build ``beta_bwd`` / ``decay_bwd`` - as ``beta.clone()`` / ``decay.clone()`` with interior chunk - boundaries zeroed, so the anti-causal scan resets state at - every chunk boundary. - - When ``beta_bwd`` is ``None``, the kernel-emitted reverse- - direction beta gradient is summed into the forward-direction - gradient (returned via the ``beta`` slot) and the ``beta_bwd`` - gradient slot returns ``None``. When ``beta_bwd`` is provided, - the two gradient streams are kept separate: the forward- - direction gradient flows through the ``beta`` slot and the - reverse-direction gradient flows through the ``beta_bwd`` slot - so autograd can route them through any ``clone() + index = 0`` - masking applied by the caller. ``decay_bwd`` is handled - identically. - """ - - @staticmethod - def forward( - ctx, - qkv, - beta, - decay, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - F: int, - S: int, - k_scale: float = 1.0, - norm_eps: float = 1e-5, - eps: float = 1e-6, - beta_bwd: torch.Tensor | None = None, - decay_bwd: torch.Tensor | None = None, - ): - B, N, three, H, D = qkv.shape - C = H * D - assert three == 3 and N == F * S - cfg = _kcfg() - - if q_norm_weight is None: - q_norm_weight = torch.ones(C, device=qkv.device, dtype=torch.float32) - if k_norm_weight is None: - k_norm_weight = torch.ones(C, device=qkv.device, dtype=torch.float32) - - # Full-channel RMSNorm: inv_rms over all H*D dims. - q_raw = qkv[:, :, 0].float() # (B, N, H, D) - k_raw = qkv[:, :, 1].float() - q_inv_rms = torch.rsqrt((q_raw * q_raw).sum(dim=(-2, -1)) / C + norm_eps) # (B, N) - k_inv_rms = torch.rsqrt((k_raw * k_raw).sum(dim=(-2, -1)) / C + norm_eps) - - # Apply norm to Q and K: Q_normed = Q_raw * inv_rms * weight. - q_nw_hd = q_norm_weight.reshape(H, D) - k_nw_hd = k_norm_weight.reshape(H, D) - qkv_normed = qkv.clone() - qkv_normed[:, :, 0] = (q_raw * q_inv_rms[:, :, None, None] * q_nw_hd[None, None]).to(qkv.dtype) - qkv_normed[:, :, 1] = (k_raw * k_inv_rms[:, :, None, None] * k_nw_hd[None, None]).to(qkv.dtype) - - # Reverse-direction beta/decay overrides for chunk-causal masking. - # When the caller supplies ``beta_bwd`` / ``decay_bwd`` (typically - # ``beta.clone()`` / ``decay.clone()`` with interior chunk-boundary - # frames zeroed), the reverse-direction kernel reads them instead of - # the unmasked tensors so the anti-causal scan resets state at chunk - # boundaries. The forward (causal) direction always uses the - # unmasked ``beta`` / ``decay``. - beta_for_bwd_dir = beta_bwd if beta_bwd is not None else beta - decay_for_bwd_dir = decay_bwd if decay_bwd is not None else decay - - # Run forward-save with QK_NORM=0 on pre-normed data. - dummy_nw = torch.ones(D, device=qkv.device, dtype=torch.float32) - num_fwd, den_fwd, sv_fwd, sz_fwd, svc_fwd, szc_fwd = _run_fwd_save( - qkv_normed, - beta, - decay, - dummy_nw, - dummy_nw, - rope_cos, - rope_sin, - F, - S, - k_scale, - norm_eps, - eps, - False, - False, - cfg, - ) - num_bwd, den_bwd, sv_bwd, sz_bwd, svc_bwd, szc_bwd = _run_fwd_save( - qkv_normed, - beta_for_bwd_dir, - decay_for_bwd_dir, - dummy_nw, - dummy_nw, - rope_cos, - rope_sin, - F, - S, - k_scale, - norm_eps, - eps, - False, - True, - cfg, - ) - - # Combine: out = (num_fwd + num_bwd) / (den_fwd + den_bwd + eps). - total_num = num_fwd.float() + num_bwd.float() - total_den = den_fwd.float() + den_bwd.float() - total_den_exp = total_den.permute(0, 2, 1).unsqueeze(-1) # (B, N, H, 1) - out = (total_num / (total_den_exp + eps)).to(qkv.dtype) - - # Save ``beta_bwd`` / ``decay_bwd`` (possibly ``None``) so the - # backward pass can (a) replay the reverse-direction kernel against - # the same masked inputs, and (b) decide whether to keep the - # reverse-direction beta/decay gradients separate (caller-supplied - # override) or fold them into the forward-direction gradient - # (no override, legacy behaviour). - ctx.save_for_backward( - qkv, - beta, - decay, - q_norm_weight, - k_norm_weight, - q_inv_rms, - k_inv_rms, - rope_cos, - rope_sin, - sv_fwd, - sz_fwd, - svc_fwd, - szc_fwd, - sv_bwd, - sz_bwd, - svc_bwd, - szc_bwd, - out, - total_den.to(qkv.dtype), - beta_bwd, - decay_bwd, - ) - _, _dot_prec, _, _ = _resolve_launch_config() - ctx.dot_prec = _dot_prec - ctx.F = F - ctx.S = S - ctx.k_scale = k_scale - ctx.norm_eps = norm_eps - ctx.eps = eps - return out - - @staticmethod - def backward(ctx, dout): - ( - qkv, - beta, - decay, - q_norm_weight, - k_norm_weight, - q_inv_rms, - k_inv_rms, - rope_cos, - rope_sin, - sv_fwd, - sz_fwd, - svc_fwd, - szc_fwd, - sv_bwd, - sz_bwd, - svc_bwd, - szc_bwd, - out, - total_den_saved, - beta_bwd_saved, - decay_bwd_saved, - ) = ctx.saved_tensors - - # Track whether the caller supplied separate ``beta_bwd`` / - # ``decay_bwd`` overrides; this controls whether the reverse- - # direction kernel gradients are summed into the forward-direction - # slot (legacy behaviour) or routed back through dedicated grad - # slots so autograd can flow through the caller's masking ops - # (``clone() + index = 0``). - has_beta_bwd = beta_bwd_saved is not None - has_decay_bwd = decay_bwd_saved is not None - - B, N, three, H, D = qkv.shape - C = H * D - - # Recompute qkv_normed (avoid saving B*N*3*H*D extra tensor). - q_raw = qkv[:, :, 0].float() - k_raw = qkv[:, :, 1].float() - q_nw_hd = q_norm_weight.reshape(H, D) - k_nw_hd = k_norm_weight.reshape(H, D) - qkv_normed = qkv.clone() - qkv_normed[:, :, 0] = (q_raw * q_inv_rms[:, :, None, None] * q_nw_hd[None, None]).to(qkv.dtype) - qkv_normed[:, :, 1] = (k_raw * k_inv_rms[:, :, None, None] * k_nw_hd[None, None]).to(qkv.dtype) - F_val = ctx.F - S = ctx.S - eps = ctx.eps - BLOCK_D, _, _, _, _, cfg, beta, decay = _prepare_launch(D, beta, decay) - - # Reverse-direction beta/decay actually fed to the reverse kernel. - # When the caller supplied an override, we replay against the - # masked tensor; otherwise we reuse the unmasked beta/decay so the - # legacy summing path is bit-identical to the pre-extension - # behaviour. - if has_beta_bwd: - beta_for_bwd_dir = beta_bwd_saved.contiguous() - else: - beta_for_bwd_dir = beta - if has_decay_bwd: - decay_for_bwd_dir = decay_bwd_saved.contiguous() - else: - decay_for_bwd_dir = decay - - # ---- Pre-compute dnum and dden ---- - total_den_exp = total_den_saved.float().permute(0, 2, 1).unsqueeze(-1) - inv_total_den = 1.0 / (total_den_exp + eps) - dnum = (dout.float() * inv_total_den).to(qkv.dtype).contiguous() - dden = ( - (-(dout.float() * out.float()).sum(dim=-1) * inv_total_den.squeeze(-1)) - .permute(0, 2, 1) - .to(qkv.dtype) - .contiguous() - ) - del out, total_den_saved, total_den_exp, inv_total_den - - dummy_nw = torch.ones(D, device=qkv.device, dtype=torch.float32) - - # ---- Backward for forward direction (QK_NORM=0, operates on normed QKV) ---- - dqkv_fwd = torch.zeros_like(qkv) - dbeta_fwd = torch.zeros_like(beta) - ddecay_fwd = torch.zeros_like(decay) - - def _run_triton_bwd(sv, sz, svc, szc, dqkv_out, dbeta_out, ddecay_out, reverse_bwd, beta_kernel, decay_kernel): - """Try backward kernel with progressively fewer warps on tmem overflow. - - ``beta_kernel`` / ``decay_kernel`` are passed as explicit - arguments (instead of closing over the outer-scope ``beta`` / - ``decay``) so the reverse-direction call can replay against the - chunk-causal-masked tensors (``beta_bwd`` / ``decay_bwd``) when - present, while the forward-direction call always uses the - unmasked ``beta`` / ``decay``. - """ - nw = cfg["num_warps"] - if ctx.dot_prec >= 1: - nw = min(nw, 4) - while nw >= 1: - try: - _fused_gdn_bwd_kernel[(B * H,)]( - qkv_normed, - qkv_normed.stride(0), - qkv_normed.stride(1), - qkv_normed.stride(2), - qkv_normed.stride(3), - qkv_normed.stride(4), - beta_kernel, - decay_kernel, - dummy_nw, - dummy_nw, - rope_cos, - rope_sin, - sv, - sz, - svc, - szc, - dnum, - dden, - dqkv_out, - dbeta_out, - ddecay_out, - H=H, - F=F_val, - S=S, - D=D, - K_SCALE=ctx.k_scale, - NORM_EPS=ctx.norm_eps, - EPS=eps, - QK_NORM=0, - STATE_FP32=1 if cfg["STATE_FP32"] else 0, - REVERSE_BWD=reverse_bwd, - BIDI_MODE=1, - DOT_PRECISION=ctx.dot_prec, - BLOCK_D=BLOCK_D, - BLOCK_S=cfg["BLOCK_S"], - num_stages=cfg["num_stages"], - num_warps=nw, - ) - return # success - except Exception as e: - if "OutOfResources" in str(type(e).__name__) or "out of resource" in str(e).lower(): - nw = nw // 2 - if nw >= 1: - continue - raise RuntimeError( - "FusedBiGDN backward: Triton kernel OutOfResources at all warp counts " - f"(8, 4, 2, 1). Most recent error: {e}" - ) from e - else: - raise - - _run_triton_bwd(sv_fwd, sz_fwd, svc_fwd, szc_fwd, dqkv_fwd, dbeta_fwd, ddecay_fwd, 0, beta, decay) - del sv_fwd, sz_fwd, svc_fwd, szc_fwd - - # ---- Backward for reversed direction (replays against masked beta/decay if any) ---- - # Allocate kernel-output gradients with the exact shape the kernel - # writes — these always match the input ``beta_for_bwd_dir`` / - # ``decay_for_bwd_dir`` shapes (override or fall-back). - dqkv_bwd = torch.zeros_like(qkv) - dbeta_bwd_kernel = torch.zeros_like(beta_for_bwd_dir) - ddecay_bwd_kernel = torch.zeros_like(decay_for_bwd_dir) - - _run_triton_bwd( - sv_bwd, - sz_bwd, - svc_bwd, - szc_bwd, - dqkv_bwd, - dbeta_bwd_kernel, - ddecay_bwd_kernel, - 1, - beta_for_bwd_dir, - decay_for_bwd_dir, - ) - del sv_bwd, sz_bwd, svc_bwd, szc_bwd - del qkv_normed, dnum, dden - - # Q/K/V gradient is always summed: qkv is shared by both directions. - dqkv_fwd += dqkv_bwd - del dqkv_bwd - - # Beta gradient: route depends on whether the caller supplied an - # override. With override -> keep separate (so autograd routes the - # reverse-direction grad through the caller's clone+mask op). - # Without override -> sum into the forward-direction grad - # (legacy behaviour, bit-identical to pre-extension code). - if has_beta_bwd: - dbeta = dbeta_fwd - dbeta_bwd_out: torch.Tensor | None = dbeta_bwd_kernel - else: - dbeta_fwd += dbeta_bwd_kernel - dbeta = dbeta_fwd - dbeta_bwd_out = None - del dbeta_bwd_kernel - - # Decay gradient: same routing logic, independent of beta override. - if has_decay_bwd: - ddecay = ddecay_fwd - ddecay_bwd_out: torch.Tensor | None = ddecay_bwd_kernel - else: - ddecay_fwd += ddecay_bwd_kernel - ddecay = ddecay_fwd - ddecay_bwd_out = None - del ddecay_bwd_kernel - - dqkv_normed = dqkv_fwd - - # ---- Full-channel RMSNorm backward for Q and K ---- - # y = x * inv_rms * w -> dL/dx = inv_rms*w*dL/dy - inv_rms^3/C * x * sum(w*dL/dy*x) - # Process Q and K sequentially to reduce peak fp32 memory. - - # Q norm backward. - q_irms = q_inv_rms[:, :, None, None] - dq_normed = dqkv_normed[:, :, 0].float() - gw_q = dq_normed * q_nw_hd[None, None] - dq_nw = (dq_normed * q_raw * q_irms).sum(dim=(0, 1)).reshape(-1) - corr_q = (gw_q * q_raw).sum(dim=(-2, -1), keepdim=True) - dqkv_normed[:, :, 0] = (q_irms * gw_q - (q_irms**3) / C * q_raw * corr_q).to(qkv.dtype) - del dq_normed, gw_q, corr_q, q_raw, q_irms - - # K norm backward. - k_irms = k_inv_rms[:, :, None, None] - dk_normed = dqkv_normed[:, :, 1].float() - gw_k = dk_normed * k_nw_hd[None, None] - dk_nw = (dk_normed * k_raw * k_irms).sum(dim=(0, 1)).reshape(-1) - corr_k = (gw_k * k_raw).sum(dim=(-2, -1), keepdim=True) - dqkv_normed[:, :, 1] = (k_irms * gw_k - (k_irms**3) / C * k_raw * corr_k).to(qkv.dtype) - del dk_normed, gw_k, corr_k, k_raw, k_irms - - return ( - dqkv_normed, - dbeta, - ddecay, - dq_nw.to(q_norm_weight.dtype), - dk_nw.to(k_norm_weight.dtype), - None, # rope_cos - None, # rope_sin - None, # F - None, # S - None, # k_scale - None, # norm_eps - None, # eps - dbeta_bwd_out, # beta_bwd - ddecay_bwd_out, # decay_bwd - ) - - -def fused_bigdn_forward_with_grad( - qkv: torch.Tensor, - beta: torch.Tensor, - decay: torch.Tensor, - q_norm_weight: torch.Tensor | None, - k_norm_weight: torch.Tensor | None, - rope_cos: torch.Tensor, - rope_sin: torch.Tensor, - F: int, - S: int, - k_scale: float = 1.0, - norm_eps: float = 1e-5, - eps: float = 1e-6, - beta_bwd: torch.Tensor | None = None, - decay_bwd: torch.Tensor | None = None, -) -> torch.Tensor: - """Bidirectional fused BiGDN with autograd support (full-channel RMSNorm). - - Unlike ``fused_bigdn_func`` (which expects pre-computed ``q_inv_rms`` / - ``k_inv_rms``), this wrapper computes the full-channel inv-RMS in Python - so the norm backward can flow through the autograd graph naturally. - - Chunk-causal masking (optional): - Pass ``beta_bwd`` and/or ``decay_bwd`` to override the beta/decay - tensors used by the **reverse-direction** kernel only. These are - typically built by the caller as ``beta.clone()`` / ``decay.clone()`` - with interior chunk-boundary frames zeroed, so the anti-causal scan - resets state at chunk boundaries while the causal scan keeps full - context. The reverse-direction beta/decay gradients are routed - back through the ``beta_bwd`` / ``decay_bwd`` slots (instead of - being summed into the forward-direction grad), which lets autograd - flow the reverse-direction gradient through the caller's - ``clone() + index = 0`` masking op. - - When ``beta_bwd`` / ``decay_bwd`` is ``None`` (default), behaviour - is bit-identical to the pre-extension full-sequence-bidirectional - path: the reverse-direction kernel uses the unmasked ``beta`` / - ``decay`` and its kernel-emitted gradient is summed into the - forward-direction gradient before being returned. - """ - return FusedBiGDNFunction.apply( - qkv, - beta, - decay, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - F, - S, - k_scale, - norm_eps, - eps, - beta_bwd, - decay_bwd, - ) diff --git a/integrations/sana/sana_wm/ops/fused_gdn_chunkwise.py b/integrations/sana/sana_wm/ops/fused_gdn_chunkwise.py deleted file mode 100644 index 715231860..000000000 --- a/integrations/sana/sana_wm/ops/fused_gdn_chunkwise.py +++ /dev/null @@ -1,2269 +0,0 @@ -# Copyright 2024 NVIDIA CORPORATION & AFFILIATES -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -""" -Fused GDN — Chunkwise-parallel forward (v2). - -V2 changes vs v1: - 1. Phase A is split into TWO kernels along the GDN data streams (KV and Z; - these are the two gating sub-paths within the GDN block, not CUDA - streams — both kernels are launched on the same CUDA stream): - _phase_a_kv_kernel: P_kv (with K_rot) + A (with K_rot, V) — uses RoPE - _phase_a_z_kernel : P_z (with K) + B (with K) — no RoPE - Z block is genuinely lighter (no V/Cos/Sin loads, no K_pair flip). - On H100 this enables 2 blocks/SM resident → multi-tenancy / latency hiding. - 2. Phase A stores (I - P_kv) and (I - P_z) instead of P_kv/P_z. Phase B then - uses these directly: M = g · (I-P_kv)·M + A_f. The MMA `(I-P_kv) @ M` folds - the identity-add into the matmul (no separate M-PM elementwise pass). - -BiGDN inference path: QK_NORM=1, USE_PRECOMPUTED_RMS=1, SAVE_STATE=0. -""" - -from __future__ import annotations - -from dataclasses import dataclass - -import torch -import triton -import triton.language as tl - -_CAM_IDENTITY_CACHE: dict[ - tuple[str, int | None, int, int, int], tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor] -] = {} - -# ════════════════════════════════════════════════════════════════ -# Per-architecture launch config (auto-selected via compute capability) -# ════════════════════════════════════════════════════════════════ -# -# Empirically tuned at production config (B=1..8, T=11, S=920, H=20, D=112) on -# A100 / H100 / GB200. Two effects matter: -# -# 1. **Precision sets BLOCK_S**: fp32 operand fragments are 2× the size of -# bf16. BLOCK_S=64 + fp32 → register spills (catastrophic, 40-100× slower). -# BLOCK_S=32 + fp32 → no spills. So fp32 mode forces BLOCK_S=32 everywhere. -# -# 2. **Arch sets BLOCK_S for bf16**: A100 (192 KB SRAM, fewer registers per -# block) prefers BLOCK_S=32 even at bf16. H100/GB200 (228 KB SRAM) tolerate -# BLOCK_S=64 cleanly at bf16. -# -# Each entry: (phase_a_warps, phase_a_BLOCK_S, -# phase_b_warps, phase_b_stages, -# phase_c_warps, phase_c_BLOCK_S, phase_c_stages) - -# ── Launch-config tuning table ───────────────────────────────────── -# -# We tune 8 knobs across 3 phases: -# Phase A : (nw, BS) streaming accumulator in registers -# Phase B : (nw, use_acc, ns) serial-F scan with persistent M in regs -# Phase C : (nw, BS, ns) streams Pass-2 output; loads fp32 M[128,128] -# -# Each arch × precision combination gets a named entry below. Values come from -# empirical sweeps (see commit log: T6 A100/H100 sweep 2026-04-19; Blackwell-DC -# 2026-04-20; Spark GB10 tuning notes in commits 5da52db6 / 3ad104d0) and from -# kernel-structure analysis (Phase B's persistent M[128,128] fp32 is 64 KB → nw -# controls register spread; Phase C's loaded M[128,128] is 64 KB → BS controls -# transient SMEM footprint). -# -# Adding a new arch: pick the closest existing bucket, then override individual -# fields in _CHUNKWISE_SHAPE_OVERRIDES once a targeted sweep lands. - - -@dataclass(frozen=True) -class _PhaseCfg: - nw: int # num_warps - BS: int = 0 # BLOCK_S (Phase A/C only; 0 = N/A for Phase B) - ns: int = 1 # num_stages - use_acc: bool = False # Phase B only: fold A_f via MMA accumulator - - -@dataclass(frozen=True) -class _ChunkwiseCfg: - A: _PhaseCfg - B: _PhaseCfg - C: _PhaseCfg - - def as_tuple(self) -> tuple: - """Flatten to the 8-tuple the legacy API returns.""" - return ( - self.A.nw, - self.A.BS, - self.B.nw, - self.B.ns, - self.B.use_acc, - self.C.nw, - self.C.BS, - self.C.ns, - ) - - -# ────────────────────────────────────────────────────────────────── -# Primary tuning table: (arch_key, prec_key) → _ChunkwiseCfg. -# Arch keys: -# "ampere" sm_80 A100 (164 KB SRAM, no WGMMA) -# "hopper" sm_90 H100 (228 KB SRAM, WGMMA) -# "blackwell_dc" sm_100 B200 / GB200 (228 KB SRAM, WGMMA v2) -# "blackwell_spark" sm_120+ with < 150 KB SRAM 5090 / GB10 (~102 KB SRAM) -# Prec keys: -# "bf16" dot_prec == 0 (bf16 TC, half-size operand fragments) -# "fp32" dot_prec >= 1 (TF32 TC or IEEE Markidis 3-pass; same launch shape) -# ────────────────────────────────────────────────────────────────── -_CHUNKWISE_TUNING: dict[tuple[str, str], _ChunkwiseCfg] = { - # A100: smaller SRAM than Hopper, no WGMMA → bigger CTAs hide MMA latency. - # Phase B fp32 needs nw=32 to spread persistent M across warps (no acc-fusion - # available pre-Hopper, so ns=2 fills the MMA pipeline slot instead). - ("ampere", "bf16"): _ChunkwiseCfg( - A=_PhaseCfg(nw=8, BS=32), - B=_PhaseCfg(nw=8, use_acc=False, ns=1), - C=_PhaseCfg(nw=4, BS=32, ns=1), # nw=4 bf16 C: 27% faster than nw=8 per T6 - ), - ("ampere", "fp32"): _ChunkwiseCfg( - # 2026-04-30 PM retune: Phase A nw=8 → 16 BS=32 yields 8-13× speedup - # across F ∈ {3, 5, 11, 14, 17, 20} (cos=1.0 verified). Old nw=8 was a - # legacy default never re-swept; sweep showed nw=16 dominates every F. - # Closes A100 sink/rolling chunkwise regression where Phase B was - # already optimal (sub-percent tuning gap) — Phase A was the bottleneck. - A=_PhaseCfg(nw=16, BS=32), - B=_PhaseCfg(nw=32, use_acc=False, ns=2), # ns=2 fills pipe (no acc-fusion) - C=_PhaseCfg(nw=16, BS=32, ns=1), # 2026-04-30 retune: nw=16 BS=32 is 2.8x faster (was nw=8 BS=16) - ), - # Hopper (H100): WGMMA + 228 KB SRAM → big tiles win at bf16. - # Phase B fp32 uses acc-fusion (MMA accumulator folds A_f in one op, +12%). - ("hopper", "bf16"): _ChunkwiseCfg( - A=_PhaseCfg(nw=8, BS=64), - B=_PhaseCfg(nw=4, use_acc=False, ns=1), # small CTAs pack better on WGMMA - C=_PhaseCfg(nw=8, BS=32, ns=1), - ), - ("hopper", "fp32"): _ChunkwiseCfg( - A=_PhaseCfg(nw=8, BS=32), # fp32 operand 2× bigger → half BS - B=_PhaseCfg( - nw=32, use_acc=False, ns=1 - ), # 2026-04-29 retune: acc_fusion=False is 3x faster post precision-gate fix - C=_PhaseCfg(nw=16, BS=32, ns=1), # 2026-04-30 retune: nw=16 BS=32 is 1.7x faster (was nw=8 BS=16) - ), - # Blackwell-DC (B200 / GB200): 228 KB SRAM + improved WGMMA codegen. - # bf16 likes small CTAs (nw=4); fp32 stays at nw=8 (nw=4 + BS=64 fp32 = 92× regression). - ("blackwell_dc", "bf16"): _ChunkwiseCfg( - A=_PhaseCfg(nw=4, BS=64), - B=_PhaseCfg(nw=4, use_acc=False, ns=1), - C=_PhaseCfg(nw=8, BS=64, ns=1), # 228 KB SRAM leaves room for BS=64 bf16 - ), - ("blackwell_dc", "fp32"): _ChunkwiseCfg( - A=_PhaseCfg( - nw=8, BS=128 - ), # 2026-04-30 retune: nw=8 BS=128 ~5% faster at production F=3-6 (sweep across F=3,5,6,11) - B=_PhaseCfg( - nw=32, use_acc=False, ns=3 - ), # 2026-04-29 retune: 14x faster (was nw=8 acc=True 17ms; now nw=32 ns=3 acc=False 1.23ms) - C=_PhaseCfg( - nw=4, BS=64, ns=1 - ), # 2026-04-30 retune: nw=4 BS=64 is 3-5x faster than old nw=8 BS=16 (sweep 2026-04-30) - ), - # Blackwell-Spark (5090 / GB10, ~102 KB SRAM): shares SRAM penalty of small - # chips but not Blackwell-DC's WGMMA-v2 register-spread benefit. Empirically - # behaves like Hopper at fp32 (Phase B wants nw=32 to spread persistent M - # across warps, not nw=8 like DC). BS shrunk one step vs DC; Phase A bf16 - # wants nw=8 (nw=4 tested 22× slower per 2026-04-20 sweep). - # Sweep 2026-04-24 (prod dim F=11 S=920): Phase B nw=32 gives 1.84×/2.65× - # (GB10/5090) at fp32 over prior nw=8 setting. - ("blackwell_spark", "bf16"): _ChunkwiseCfg( - A=_PhaseCfg(nw=8, BS=32), - B=_PhaseCfg(nw=8, use_acc=False, ns=1), # nw=8 (not 4) at bf16: ~5% across F=3,6,11 - # 2026-05-06 P1/P2 retune (5090, F=11 S=920): C.nw=4 BS=32 is ~3.5% - # faster than nw=8 (Phase C is bandwidth-bound, fewer warps schedules - # better on the small SRAM). BS=64 bf16 on Spark OOMs SRAM. - C=_PhaseCfg(nw=4, BS=32, ns=1), - ), - ("blackwell_spark", "fp32"): _ChunkwiseCfg( - A=_PhaseCfg(nw=8, BS=16), # fp32 operand 2× bigger → BS=16 (half of DC's 32) - # 2026-05-06 retune: nw=16 OOMs the 102 KB SRAM cap at TF32 on 5090 - # (131 KB needed). nw=8 fits and is within noise of the prior nw=16 - # benchmark. The Phase B D-tile path (auto-enabled on spark, see - # `_pick_phase_b_d_splits`) is ~2.6× faster than this baseline at TF32 - # and ~13% faster at IEEE — these baseline params only apply when - # PHASE_B_D_SPLITS=1 is forced. - B=_PhaseCfg(nw=8, use_acc=False, ns=1), - C=_PhaseCfg(nw=8, BS=16, ns=1), # binding constraint: M.fp32 64 KB + Q stage - ), -} - - -# ────────────────────────────────────────────────────────────────── -# Shape-aware override table: empty by default. Keyed by -# (arch_key, prec_key, shape_hint) -# where shape_hint is a free-form string (e.g. "small_BH", "large_F", -# "B>=8") chosen when populating. Lookup is exact-match; values are -# full `_ChunkwiseCfg` instances (no partial overrides — copy-paste -# from `_CHUNKWISE_TUNING` and edit the one phase you want to change). -# -# Leave empty unless a targeted sweep shows a particular shape regresses -# with the broad arch config. Adding here is strictly additive — base -# table remains the fallback. -# ────────────────────────────────────────────────────────────────── -_CHUNKWISE_SHAPE_OVERRIDES: dict[tuple[str, str, str], _ChunkwiseCfg] = {} - - -# Per-(cap, dot_prec) exact overrides (pins a specific GPU model if the arch -# bucket is wrong for it). Also empty by default. -_ARCH_OVERRIDES: dict = {} - - -def _arch_key(cap: tuple) -> str: - """Map compute capability → named arch bucket in `_CHUNKWISE_TUNING`. - - Blackwell (cap[0] >= 10) is split into "blackwell_dc" and "blackwell_spark" - by SRAM size (≥150 KB vs less). Without CUDA or for unknown archs we - default to the conservative "ampere" bucket. - """ - if cap[0] == 8: - return "ampere" - if cap[0] == 9: - return "hopper" - if cap[0] >= 10: - has_big_sram = True - if torch.cuda.is_available(): - props = torch.cuda.get_device_properties(0) - smem = getattr(props, "shared_memory_per_multiprocessor", 228 * 1024) - has_big_sram = smem >= 150 * 1024 - return "blackwell_dc" if has_big_sram else "blackwell_spark" - return "ampere" - - -def _prec_key(dot_prec: int) -> str: - return "fp32" if dot_prec >= 1 else "bf16" - - -def _auto_config(dot_prec: int, cap: tuple, shape_hint: str | None = None) -> tuple: - """Look up chunkwise kernel launch params from the tuning table. - - Resolution order: - 1. `_ARCH_OVERRIDES[(cap, dot_prec)]` — exact-capability pin, highest priority. - 2. `_CHUNKWISE_SHAPE_OVERRIDES[(arch, prec, shape_hint)]` — sweep-driven overrides. - 3. `_CHUNKWISE_TUNING[(arch, prec)]` — primary per-(arch, prec) table. - 4. Fallback to ("ampere", prec) if the arch is unrecognised. - - Returns the legacy 8-tuple `(a_nw, a_BS, b_nw, b_ns, b_use_acc, c_nw, c_BS, c_ns)` - for backward compatibility with `_get_arch_config` callers. - """ - arch = _arch_key(cap) - prec = _prec_key(dot_prec) - - if shape_hint is not None: - cfg = _CHUNKWISE_SHAPE_OVERRIDES.get((arch, prec, shape_hint)) - if cfg is not None: - return cfg.as_tuple() - - cfg = _CHUNKWISE_TUNING.get((arch, prec)) or _CHUNKWISE_TUNING[("ampere", prec)] - return cfg.as_tuple() - - -def _get_arch_config( - dot_precision: int = 0, - shape_hint: str | None = None, - device: torch.device | int | None = None, -): - """Returns (a_warps, a_BLOCK_S, b_warps, b_stages, b_use_acc_fusion, - c_warps, c_BLOCK_S, c_stages). - - dot_precision: 0=bf16 TC, 1=TF32 TC, 2=IEEE fp32. - shape_hint: optional string key for `_CHUNKWISE_SHAPE_OVERRIDES`. - device: device whose capability drives the lookup. Defaults to the - current CUDA device — pass ``qkv.device`` (or any input - tensor's device) when launching kernels in heterogeneous - or multi-GPU single-process setups so the right tuning - bucket is chosen. - """ - if not torch.cuda.is_available(): - cap = (9, 0) # assume modern when querying from CPU - else: - if device is None: - dev_idx = torch.cuda.current_device() - elif isinstance(device, int): - dev_idx = device - else: - dev_idx = device.index if device.index is not None else torch.cuda.current_device() - cap = torch.cuda.get_device_capability(dev_idx) - key = (cap, dot_precision) - if key in _ARCH_OVERRIDES: - return _ARCH_OVERRIDES[key] - return _auto_config(dot_precision, cap, shape_hint) - - -# ════════════════════════════════════════════════════════════════ -# Phase A — split into KV and Z kernels -# ════════════════════════════════════════════════════════════════ - - -@triton.jit -def _phase_a_kv_kernel( - qkv_ptr, - stride_b: tl.constexpr, - stride_n: tl.constexpr, - stride_3: tl.constexpr, - stride_h: tl.constexpr, - stride_d: tl.constexpr, - beta_ptr, - k_inv_rms_ptr, - k_norm_w_ptr, - rope_cos_ptr, - rope_sin_ptr, - I_minus_P_kv_ptr, # output: (I - K_rot^T diag(β) K_rot) - A_ptr, # output: K_rot^T diag(β) V - H: tl.constexpr, - F: tl.constexpr, - S: tl.constexpr, - D: tl.constexpr, - K_SCALE, - NORM_EPS: tl.constexpr, - DOT_PRECISION: tl.constexpr, - BLOCK_D: tl.constexpr, - BLOCK_S: tl.constexpr, - SKIP_RELU: tl.constexpr = False, -): - if DOT_PRECISION >= 1: - dot_dtype = tl.float32 - else: - dot_dtype = tl.bfloat16 - dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" - - pid = tl.program_id(0) - pid_b = pid // (H * F) - pid_hf = pid % (H * F) - pid_h = pid_hf // F - pid_f = pid_hf % F - bh = pid_b * H + pid_h - N: tl.constexpr = F * S - - qkv_bh = qkv_ptr + pid_b * stride_b + pid_h * stride_h - beta_bhf = beta_ptr + bh * (F * S) + pid_f * S - I_P_kv_bhf = I_minus_P_kv_ptr + bh * F * BLOCK_D * BLOCK_D + pid_f * BLOCK_D * BLOCK_D - A_bhf = A_ptr + bh * F * BLOCK_D * BLOCK_D + pid_f * BLOCK_D * BLOCK_D - - offs_d = tl.arange(0, BLOCK_D) - mask_d = offs_d < D - offs_d_pair = offs_d ^ 1 - mask_d_pair = offs_d_pair < D - - nw_offset = pid_h * D - k_nw = tl.load(k_norm_w_ptr + nw_offset + offs_d, mask=mask_d, other=0.0).to(tl.float32) - k_nw_pair = tl.load(k_norm_w_ptr + nw_offset + offs_d_pair, mask=mask_d_pair, other=0.0).to(tl.float32) - - # KV stream accumulators (in-loop fp32 to avoid bf16 round-off compounding) - P_kv_acc = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) - A_acc = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) - - k_scale = K_SCALE - n_base = pid_f * S - - for s0 in range(0, S, BLOCK_S): - offs_s = s0 + tl.arange(0, BLOCK_S) - mask_s = offs_s < S - mask_sd = mask_s[:, None] & mask_d[None, :] - n_idx = n_base + offs_s - - k_ptrs = qkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d[None, :] * stride_d - v_ptrs = qkv_bh + n_idx[:, None] * stride_n + 2 * stride_3 + offs_d[None, :] * stride_d - K_raw = tl.load(k_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - V_raw = tl.load(v_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - beta_t = tl.load(beta_bhf + offs_s, mask=mask_s, other=0.0).to(tl.float32) - - k_inv_rms = tl.load(k_inv_rms_ptr + pid_b * N + n_idx, mask=mask_s, other=1.0).to(tl.float32) - K_normed = K_raw * k_inv_rms[:, None] * k_nw[None, :] - if SKIP_RELU: - K = K_normed * k_scale - else: - K = tl.where(K_normed > 0, K_normed, 0.0) * k_scale - - K_pair_raw = tl.reshape( - tl.flip(tl.reshape(K_raw, (BLOCK_S, BLOCK_D // 2, 2)), dim=2), - (BLOCK_S, BLOCK_D), - ) - K_pair_normed = K_pair_raw * k_inv_rms[:, None] * k_nw_pair[None, :] - if SKIP_RELU: - K_pair = K_pair_normed * k_scale - else: - K_pair = tl.where(K_pair_normed > 0, K_pair_normed, 0.0) * k_scale - - rope_ptrs = n_idx[:, None] * D + offs_d[None, :] - Cos = tl.load(rope_cos_ptr + rope_ptrs, mask=mask_sd, other=1.0).to(tl.float32) - Sin = tl.load(rope_sin_ptr + rope_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - K_rot = K * Cos + K_pair * Sin - - beta_Krot = beta_t[:, None] * K_rot - beta_V = beta_t[:, None] * V_raw - - K_rot_T = tl.trans(K_rot) - P_kv_acc += tl.dot(K_rot_T.to(dot_dtype), beta_Krot.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) - A_acc += tl.dot(K_rot_T.to(dot_dtype), beta_V.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) - - # Store bf16 outputs. Padded positions are 0 by construction (K_rot is 0 outside D). - offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] - diag_in_range = (offs_d[:, None] == offs_d[None, :]) & mask_d[:, None] & mask_d[None, :] - I_minus_P_kv = tl.where(diag_in_range, 1.0 - P_kv_acc, -P_kv_acc) - if DOT_PRECISION >= 1: - tl.store(I_P_kv_bhf + offs_dd, I_minus_P_kv) - tl.store(A_bhf + offs_dd, A_acc) - else: - tl.store(I_P_kv_bhf + offs_dd, I_minus_P_kv.to(tl.bfloat16)) - tl.store(A_bhf + offs_dd, A_acc.to(tl.bfloat16)) - - -@triton.jit -def _phase_a_z_kernel( - qkv_ptr, - stride_b: tl.constexpr, - stride_n: tl.constexpr, - stride_3: tl.constexpr, - stride_h: tl.constexpr, - stride_d: tl.constexpr, - beta_ptr, - k_inv_rms_ptr, - k_norm_w_ptr, - I_minus_P_z_ptr, # output: (I - K^T diag(β) K) - B_ptr, # output: K^T β - H: tl.constexpr, - F: tl.constexpr, - S: tl.constexpr, - D: tl.constexpr, - K_SCALE, - NORM_EPS: tl.constexpr, - DOT_PRECISION: tl.constexpr, - BLOCK_D: tl.constexpr, - BLOCK_S: tl.constexpr, -): - """Z stream: uses K (no RoPE). Cheaper than KV — no V load, no RoPE compute, - no K_pair derivation.""" - if DOT_PRECISION >= 1: - dot_dtype = tl.float32 - else: - dot_dtype = tl.bfloat16 - dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" - - pid = tl.program_id(0) - pid_b = pid // (H * F) - pid_hf = pid % (H * F) - pid_h = pid_hf // F - pid_f = pid_hf % F - bh = pid_b * H + pid_h - N: tl.constexpr = F * S - - qkv_bh = qkv_ptr + pid_b * stride_b + pid_h * stride_h - beta_bhf = beta_ptr + bh * (F * S) + pid_f * S - I_P_z_bhf = I_minus_P_z_ptr + bh * F * BLOCK_D * BLOCK_D + pid_f * BLOCK_D * BLOCK_D - B_bhf = B_ptr + bh * F * BLOCK_D + pid_f * BLOCK_D - - offs_d = tl.arange(0, BLOCK_D) - mask_d = offs_d < D - - nw_offset = pid_h * D - k_nw = tl.load(k_norm_w_ptr + nw_offset + offs_d, mask=mask_d, other=0.0).to(tl.float32) - - P_z_acc = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) - B_acc = tl.zeros([BLOCK_D], dtype=tl.float32) - - k_scale = K_SCALE - n_base = pid_f * S - - for s0 in range(0, S, BLOCK_S): - offs_s = s0 + tl.arange(0, BLOCK_S) - mask_s = offs_s < S - mask_sd = mask_s[:, None] & mask_d[None, :] - n_idx = n_base + offs_s - - # Only K_raw needed (no V, no Cos/Sin) - k_ptrs = qkv_bh + n_idx[:, None] * stride_n + 1 * stride_3 + offs_d[None, :] * stride_d - K_raw = tl.load(k_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - beta_t = tl.load(beta_bhf + offs_s, mask=mask_s, other=0.0).to(tl.float32) - - k_inv_rms = tl.load(k_inv_rms_ptr + pid_b * N + n_idx, mask=mask_s, other=1.0).to(tl.float32) - K_normed = K_raw * k_inv_rms[:, None] * k_nw[None, :] - K = tl.where(K_normed > 0, K_normed, 0.0) * k_scale - - beta_K = beta_t[:, None] * K - - K_T = tl.trans(K) - P_z_acc += tl.dot(K_T.to(dot_dtype), beta_K.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) - B_acc += tl.sum(beta_K, axis=0) - - offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] - diag_in_range = (offs_d[:, None] == offs_d[None, :]) & mask_d[:, None] & mask_d[None, :] - I_minus_P_z = tl.where(diag_in_range, 1.0 - P_z_acc, -P_z_acc) - - if DOT_PRECISION >= 1: - tl.store(I_P_z_bhf + offs_dd, I_minus_P_z) - else: - tl.store(I_P_z_bhf + offs_dd, I_minus_P_z.to(tl.bfloat16)) - # B stays fp32 (vector, only 0.5 KB, negligible HBM cost) - tl.store(B_bhf + offs_d, B_acc) - - -def phase_a( - qkv: torch.Tensor, - beta: torch.Tensor, - q_inv_rms: torch.Tensor, - k_inv_rms: torch.Tensor, - q_norm_w: torch.Tensor, - k_norm_w: torch.Tensor, - rope_cos: torch.Tensor, - rope_sin: torch.Tensor, - F: int, - S: int, - k_scale: float = 1.0, - norm_eps: float = 1e-5, - num_warps: int | None = None, - num_stages: int = 1, - BLOCK_S: int | None = None, - dot_precision: int = 0, - skip_relu: bool = False, - skip_z: bool = False, -): - """Compute (I-P_kv), A, (I-P_z), B for all (B, H, F) via 2 kernels (KV + Z). - - `skip_relu=True` makes the K-stream prep a pure linear chain (no ReLU on - K_normed * k_scale). Used by the camera-branch chunkwise wrapper, where K - has already been ReLU'd by the cam_prep kernel and subsequently rotated - by UCPE+RoPE — re-applying ReLU on the rotated values would clobber - legitimate negatives. - - `skip_z=True` skips the Phase A Z kernel entirely and returns placeholder - tensors for I_P_z and B_z. Used by NUM_ONLY callers (camera branch) to - avoid wasted Z-stream prep when the denominator scan won't be used. - """ - # Auto-pick (num_warps, BLOCK_S) per arch+precision unless overridden - if num_warps is None or BLOCK_S is None: - a_w, a_bs, *_ = _get_arch_config(dot_precision, device=qkv.device) - if num_warps is None: - num_warps = a_w - if BLOCK_S is None: - BLOCK_S = a_bs - B, N, three, H, D = qkv.shape - assert three == 3 and N == F * S - BLOCK_D = triton.next_power_of_2(D) - BH = B * H - - # FAIR-COMPARE PATCH: keep fp32 inter-phase bridge at P0/P1 to match pytorch/fused - bridge_dtype = torch.float32 if dot_precision >= 1 else torch.bfloat16 - I_P_kv = torch.empty(BH, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=bridge_dtype) - A = torch.empty(BH, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=bridge_dtype) - - beta_c = beta.contiguous() - grid = (BH * F,) - - _phase_a_kv_kernel[grid]( - qkv, - qkv.stride(0), - qkv.stride(1), - qkv.stride(2), - qkv.stride(3), - qkv.stride(4), - beta_c, - k_inv_rms, - k_norm_w, - rope_cos, - rope_sin, - I_P_kv, - A, - H=H, - F=F, - S=S, - D=D, - K_SCALE=k_scale, - NORM_EPS=norm_eps, - DOT_PRECISION=dot_precision, - BLOCK_D=BLOCK_D, - BLOCK_S=BLOCK_S, - SKIP_RELU=skip_relu, - num_warps=num_warps, - num_stages=num_stages, - ) - - if skip_z: - # NUM_ONLY callers (camera branch) do not consume the Z scan. Return - # placeholders and let Phase B skip all Z loads/stores as well. - I_P_z = torch.empty(1, device=qkv.device, dtype=bridge_dtype) - B_z = torch.empty(1, device=qkv.device, dtype=torch.float32) - return I_P_kv, A, I_P_z, B_z - - I_P_z = torch.empty(BH, F, BLOCK_D, BLOCK_D, device=qkv.device, dtype=bridge_dtype) - # B stays fp32 — small vector (0.5 KB/frame), no benefit to downcast - B_z = torch.empty(BH, F, BLOCK_D, device=qkv.device, dtype=torch.float32) - - _phase_a_z_kernel[grid]( - qkv, - qkv.stride(0), - qkv.stride(1), - qkv.stride(2), - qkv.stride(3), - qkv.stride(4), - beta_c, - k_inv_rms, - k_norm_w, - I_P_z, - B_z, - H=H, - F=F, - S=S, - D=D, - K_SCALE=k_scale, - NORM_EPS=norm_eps, - DOT_PRECISION=dot_precision, - BLOCK_D=BLOCK_D, - BLOCK_S=BLOCK_S, - num_warps=num_warps, - num_stages=num_stages, - ) - return I_P_kv, A, I_P_z, B_z - - -# ════════════════════════════════════════════════════════════════ -# Phase B — serial scan, uses pre-stored (I - P) so MMA folds in M -# ════════════════════════════════════════════════════════════════ - - -@triton.jit -def _phase_b_kernel( - I_P_kv_ptr, - A_ptr, - I_P_z_ptr, - B_ptr, - decay_ptr, - M_fwd_ptr, - z_fwd_ptr, - M_rev_ptr, - z_rev_ptr, - init_state_kv_ptr, # (BH, BLOCK_D, BLOCK_D) — read when LOAD_INIT_STATE=1 - init_state_z_ptr, # (BH, BLOCK_D) - final_state_kv_ptr, # (BH, BLOCK_D, BLOCK_D) — written when SAVE_FINAL_STATE=1 - final_state_z_ptr, # (BH, BLOCK_D) - BH: tl.constexpr, - F: tl.constexpr, - BLOCK_D: tl.constexpr, - DOT_PRECISION: tl.constexpr, - USE_ACC_FUSION: tl.constexpr, - LOAD_INIT_STATE: tl.constexpr, # forward scan seeded with init state (vs zeros) - SAVE_FINAL_STATE: tl.constexpr, # write M_{F-1} of forward scan to final_state_* - DIRECTION: tl.constexpr, # 0=both, 1=fwd-only, 2=rev-only - COMBINED_HISTORY: tl.constexpr, # 1 → rev branch read-add-stores into M_fwd_ptr - # (M_hist[f] = M_fwd[f] + M_rev[f]); skips the F-1 zero-write so the fwd - # value at F-1 is preserved (rev contribution there is exactly zero anyway). - # Only meaningful when DIRECTION=0. Saves one Phase C launch + one M-shaped - # buffer downstream (Phase C runs once on M_hist instead of twice). - SKIP_Z: tl.constexpr, -): - if DOT_PRECISION >= 1: - dot_dtype = tl.float32 - else: - dot_dtype = tl.bfloat16 - dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" - - pid = tl.program_id(0) - bh = pid - - offs_d = tl.arange(0, BLOCK_D) - offs_dd = offs_d[:, None] * BLOCK_D + offs_d[None, :] - - # ── Forward scan (skip when DIRECTION=2 i.e. rev-only) ── - if DIRECTION != 2: - if LOAD_INIT_STATE: - M = tl.load(init_state_kv_ptr + bh * BLOCK_D * BLOCK_D + offs_dd).to(tl.float32) - if not SKIP_Z: - z = tl.load(init_state_z_ptr + bh * BLOCK_D + offs_d).to(tl.float32) - else: - M = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) - if not SKIP_Z: - z = tl.zeros([BLOCK_D], dtype=tl.float32) - for f in range(F): - I_P_kv_f = tl.load(I_P_kv_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd) - A_f = tl.load(A_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd) - g_f = tl.load(decay_ptr + bh * F + f).to(tl.float32) - - # M = g · (I - P_kv) M + A_f - if USE_ACC_FUSION: - # Pre-scale (I-P) by g, accumulate A_f directly via the MMA accumulator. - # Result: A_f + g·(I-P)·M in one MMA — no separate M_temp tensor. - I_P_scaled = I_P_kv_f.to(tl.float32) * g_f - M = tl.dot( - I_P_scaled.to(dot_dtype), - M.to(dot_dtype), - acc=A_f.to(tl.float32), - out_dtype=tl.float32, - input_precision=dot_ip, - ) - else: - M_temp = tl.dot(I_P_kv_f.to(dot_dtype), M.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) - M = g_f * M_temp + A_f - - tl.store(M_fwd_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd, M) - if not SKIP_Z: - I_P_z_f = tl.load(I_P_z_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd) - B_f = tl.load(B_ptr + bh * F * BLOCK_D + f * BLOCK_D + offs_d) - # z = g · (I - P_z) z + B_f - z_temp = tl.sum(I_P_z_f * z[None, :], axis=1) - z = g_f * z_temp + B_f - tl.store(z_fwd_ptr + bh * F * BLOCK_D + f * BLOCK_D + offs_d, z) - - # Save terminal forward state for state-cached inference (autoregressive sampling). - if SAVE_FINAL_STATE: - tl.store(final_state_kv_ptr + bh * BLOCK_D * BLOCK_D + offs_dd, M) - if not SKIP_Z: - tl.store(final_state_z_ptr + bh * BLOCK_D + offs_d, z) - - # ── Reverse scan (skip when DIRECTION=1 i.e. fwd-only) ── - if DIRECTION != 1: - M = tl.zeros([BLOCK_D, BLOCK_D], dtype=tl.float32) - if not SKIP_Z: - z = tl.zeros([BLOCK_D], dtype=tl.float32) - # COMBINED_HISTORY mode: rev contributions get read-add-stored into the - # fwd buffer (which thereby becomes M_hist = M_fwd + M_rev). The F-1 - # zero-write is skipped so M_hist[F-1] keeps the fwd value (rev value - # there is zero by construction, so no add needed). - if not COMBINED_HISTORY: - tl.store(M_rev_ptr + bh * F * BLOCK_D * BLOCK_D + (F - 1) * BLOCK_D * BLOCK_D + offs_dd, M) - if not SKIP_Z: - tl.store(z_rev_ptr + bh * F * BLOCK_D + (F - 1) * BLOCK_D + offs_d, z) - for f_iter in range(F - 1): - f_src = F - 1 - f_iter - f_dst = f_src - 1 - I_P_kv_f = tl.load(I_P_kv_ptr + bh * F * BLOCK_D * BLOCK_D + f_src * BLOCK_D * BLOCK_D + offs_dd) - A_f = tl.load(A_ptr + bh * F * BLOCK_D * BLOCK_D + f_src * BLOCK_D * BLOCK_D + offs_dd) - g_f = tl.load(decay_ptr + bh * F + f_src).to(tl.float32) - - if USE_ACC_FUSION: - I_P_scaled = I_P_kv_f.to(tl.float32) * g_f - M = tl.dot( - I_P_scaled.to(dot_dtype), - M.to(dot_dtype), - acc=A_f.to(tl.float32), - out_dtype=tl.float32, - input_precision=dot_ip, - ) - else: - M_temp = tl.dot(I_P_kv_f.to(dot_dtype), M.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) - M = g_f * M_temp + A_f - - if not SKIP_Z: - I_P_z_f = tl.load(I_P_z_ptr + bh * F * BLOCK_D * BLOCK_D + f_src * BLOCK_D * BLOCK_D + offs_dd) - B_f = tl.load(B_ptr + bh * F * BLOCK_D + f_src * BLOCK_D + offs_d) - z_temp = tl.sum(I_P_z_f * z[None, :], axis=1) - z = g_f * z_temp + B_f - - if COMBINED_HISTORY: - # Read-add-store into the fwd buffer. The fwd loop has already - # written M_fwd[f_dst] to this slot; we add the rev contribution - # in place. Stays in L1/L2 since fwd just touched it. - M_addr = M_fwd_ptr + bh * F * BLOCK_D * BLOCK_D + f_dst * BLOCK_D * BLOCK_D + offs_dd - tl.store(M_addr, tl.load(M_addr) + M) - if not SKIP_Z: - z_addr = z_fwd_ptr + bh * F * BLOCK_D + f_dst * BLOCK_D + offs_d - tl.store(z_addr, tl.load(z_addr) + z) - else: - tl.store(M_rev_ptr + bh * F * BLOCK_D * BLOCK_D + f_dst * BLOCK_D * BLOCK_D + offs_dd, M) - if not SKIP_Z: - tl.store(z_rev_ptr + bh * F * BLOCK_D + f_dst * BLOCK_D + offs_d, z) - - -def phase_b_triton( - I_P_kv, - A, - I_P_z, - B, - decay, - F, - num_warps=None, - num_stages=None, - use_acc_fusion=None, - dot_precision=0, - init_state_kv=None, - init_state_z=None, - return_final_state=False, - direction=0, - combined_history=False, - skip_z=False, -): - """Phase B serial-F scan over (B*H,). - - Forward scan can be seeded with `init_state_kv`/`init_state_z` (autoregressive - sampling chunk > 0) and can write the terminal `M_{F-1}`/`z_{F-1}` to caller- - provided buffers when `return_final_state=True`. - - `direction`: 0=both (default), 1=forward-only, 2=reverse-only. Forward-only - skips reverse scan + reverse output buffers; reverse-only skips forward scan - + state load/save. Used by single-direction state-cached entry points. - - `combined_history` (only meaningful with direction=0): the rev branch - read-add-stores into the fwd buffer so its contents become - M_hist[f] = M_fwd[f] + M_rev[f] (and same for z). Lets the caller run - Phase C exactly once on the combined history, since Phase C is linear in - M and z (`Q @ (M_fwd + M_rev) = Q @ M_fwd + Q @ M_rev`). When set, - M_rev/z_rev outputs are placeholder dummies; only M_fwd/z_fwd carry data. - - `skip_z`: skip the denominator/Z recurrence entirely. Used by camera - numerator-only scans where Phase C runs with `num_only=True`. - - Returns (M_fwd, z_fwd, M_rev, z_rev) — and additionally (final_kv, final_z) - when return_final_state=True. Skipped-direction outputs are returned as a - 1-element placeholder tensor (kernel never touches them when DIRECTION - gates them off); callers should always discard the slot they didn't ask - for. Reverse scan is always seeded with zeros (per upstream's bidi - state-cache convention — only forward state is cached). - """ - BH = I_P_kv.shape[0] - _, _, BLOCK_D, _ = A.shape # A is always full [BH, F, BLOCK_D, BLOCK_D] - device, fdtype = I_P_kv.device, torch.float32 - - if num_warps is None or num_stages is None or use_acc_fusion is None: - _, _, b_w, b_s, b_acc, *_ = _get_arch_config(dot_precision, device=device) - if num_warps is None: - num_warps = b_w - if num_stages is None: - num_stages = b_s - if use_acc_fusion is None: - use_acc_fusion = b_acc - - if combined_history and direction != 0: - raise ValueError("combined_history=True requires direction=0 (bidi)") - - # Phase B kernel is DIRECTION-gated (constexpr); skipped-direction writes - # never happen, so we can hand it a 1-element placeholder for the inactive - # buffers and free ~4× M_fwd-shaped allocations per single-direction call. - decay_flat = decay.reshape(BH, F).contiguous().float() - - load_init = init_state_kv is not None - dummy = torch.empty(1, device=device, dtype=fdtype) - full_M = lambda: torch.empty(BH, F, BLOCK_D, BLOCK_D, device=device, dtype=fdtype) - full_z = lambda: torch.empty(BH, F, BLOCK_D, device=device, dtype=fdtype) - M_fwd = dummy if direction == 2 else full_M() - z_fwd = dummy if (direction == 2 or skip_z) else full_z() - # Combined-history mode reuses M_fwd/z_fwd as M_hist/z_hist; rev outputs - # become placeholders even though DIRECTION!=1. - M_rev = dummy if (direction == 1 or combined_history) else full_M() - z_rev = dummy if (direction == 1 or combined_history or skip_z) else full_z() - if load_init: - init_kv = init_state_kv.contiguous().view(BH, BLOCK_D, BLOCK_D) - init_z = dummy if skip_z else init_state_z.contiguous().view(BH, BLOCK_D) - else: - init_kv = dummy - init_z = dummy - - if return_final_state: - final_kv = torch.empty(BH, BLOCK_D, BLOCK_D, device=device, dtype=fdtype) - final_z = dummy if skip_z else torch.empty(BH, BLOCK_D, device=device, dtype=fdtype) - else: - final_kv = dummy - final_z = dummy - - d_splits, nw_override, ns_override, acc_override = _pick_phase_b_d_splits(BLOCK_D, dot_precision=dot_precision) - if d_splits > 1: - D_TILE = BLOCK_D // d_splits - # Use D-tile-specific tuning if available, else fall back to baseline tuning - nw_use = nw_override if nw_override is not None else num_warps - ns_use = ns_override if ns_override is not None else num_stages - acc_use = acc_override if acc_override is not None else use_acc_fusion - _phase_b_dtile_kernel[(BH, d_splits)]( - I_P_kv, - A, - I_P_z, - B, - decay_flat, - M_fwd, - z_fwd, - M_rev, - z_rev, - init_kv, - init_z, - final_kv, - final_z, - BH=BH, - F=F, - BLOCK_D=BLOCK_D, - D_TILE=D_TILE, - DOT_PRECISION=dot_precision, - USE_ACC_FUSION=acc_use, - LOAD_INIT_STATE=1 if load_init else 0, - SAVE_FINAL_STATE=1 if return_final_state else 0, - DIRECTION=direction, - COMBINED_HISTORY=1 if combined_history else 0, - SKIP_Z=1 if skip_z else 0, - num_warps=nw_use, - num_stages=ns_use, - ) - else: - _phase_b_kernel[(BH,)]( - I_P_kv, - A, - I_P_z, - B, - decay_flat, - M_fwd, - z_fwd, - M_rev, - z_rev, - init_kv, - init_z, - final_kv, - final_z, - BH=BH, - F=F, - BLOCK_D=BLOCK_D, - DOT_PRECISION=dot_precision, - USE_ACC_FUSION=use_acc_fusion, - LOAD_INIT_STATE=1 if load_init else 0, - SAVE_FINAL_STATE=1 if return_final_state else 0, - DIRECTION=direction, - COMBINED_HISTORY=1 if combined_history else 0, - SKIP_Z=1 if skip_z else 0, - num_warps=num_warps, - num_stages=num_stages, - ) - if return_final_state: - return M_fwd, z_fwd, M_rev, z_rev, final_kv, final_z - return M_fwd, z_fwd, M_rev, z_rev - - -# ════════════════════════════════════════════════════════════════ -# Phase B D-tile — j-axis split for grid parallelism (#118) -# ════════════════════════════════════════════════════════════════ -# Same recurrence as _phase_b_kernel but each program owns a D_TILE-wide -# slice of M's output column dim. Grid: (BH, d_splits). M_new[*, j_tile] -# only depends on M_prev[*, j_tile] and full (I-P_kv) — independent across -# j-tiles. z is unsplittable; only `pid_d == 0` updates/writes z. -@triton.jit -def _phase_b_dtile_kernel( - I_P_kv_ptr, - A_ptr, - I_P_z_ptr, - B_ptr, - decay_ptr, - M_fwd_ptr, - z_fwd_ptr, - M_rev_ptr, - z_rev_ptr, - init_state_kv_ptr, - init_state_z_ptr, - final_state_kv_ptr, - final_state_z_ptr, - BH: tl.constexpr, - F: tl.constexpr, - BLOCK_D: tl.constexpr, - D_TILE: tl.constexpr, - DOT_PRECISION: tl.constexpr, - USE_ACC_FUSION: tl.constexpr, - LOAD_INIT_STATE: tl.constexpr, - SAVE_FINAL_STATE: tl.constexpr, - DIRECTION: tl.constexpr, - COMBINED_HISTORY: tl.constexpr, - SKIP_Z: tl.constexpr, -): - if DOT_PRECISION >= 1: - dot_dtype = tl.float32 - else: - dot_dtype = tl.bfloat16 - dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" - - pid_bh = tl.program_id(0) - pid_d = tl.program_id(1) - bh = pid_bh - - offs_d_full = tl.arange(0, BLOCK_D) - offs_d_tile = pid_d * D_TILE + tl.arange(0, D_TILE) - offs_dd_full = offs_d_full[:, None] * BLOCK_D + offs_d_full[None, :] - offs_dd_tile = offs_d_full[:, None] * BLOCK_D + offs_d_tile[None, :] - - is_lead = pid_d == 0 - - if DIRECTION != 2: - if LOAD_INIT_STATE: - M = tl.load(init_state_kv_ptr + bh * BLOCK_D * BLOCK_D + offs_dd_tile).to(tl.float32) - else: - M = tl.zeros([BLOCK_D, D_TILE], dtype=tl.float32) - if not SKIP_Z: - z = tl.zeros([BLOCK_D], dtype=tl.float32) - if is_lead and LOAD_INIT_STATE: - z = tl.load(init_state_z_ptr + bh * BLOCK_D + offs_d_full).to(tl.float32) - - for f in range(F): - I_P_kv_f = tl.load(I_P_kv_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd_full) - A_f = tl.load(A_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd_tile) - g_f = tl.load(decay_ptr + bh * F + f).to(tl.float32) - - if USE_ACC_FUSION: - I_P_scaled = I_P_kv_f.to(tl.float32) * g_f - M = tl.dot( - I_P_scaled.to(dot_dtype), - M.to(dot_dtype), - acc=A_f.to(tl.float32), - out_dtype=tl.float32, - input_precision=dot_ip, - ) - else: - M_temp = tl.dot(I_P_kv_f.to(dot_dtype), M.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) - M = g_f * M_temp + A_f - - tl.store(M_fwd_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd_tile, M) - - if is_lead and not SKIP_Z: - I_P_z_f = tl.load(I_P_z_ptr + bh * F * BLOCK_D * BLOCK_D + f * BLOCK_D * BLOCK_D + offs_dd_full) - B_f = tl.load(B_ptr + bh * F * BLOCK_D + f * BLOCK_D + offs_d_full) - z_temp = tl.sum(I_P_z_f * z[None, :], axis=1) - z = g_f * z_temp + B_f - tl.store(z_fwd_ptr + bh * F * BLOCK_D + f * BLOCK_D + offs_d_full, z) - - if SAVE_FINAL_STATE: - tl.store(final_state_kv_ptr + bh * BLOCK_D * BLOCK_D + offs_dd_tile, M) - if is_lead and not SKIP_Z: - tl.store(final_state_z_ptr + bh * BLOCK_D + offs_d_full, z) - - if DIRECTION != 1: - M = tl.zeros([BLOCK_D, D_TILE], dtype=tl.float32) - if not SKIP_Z: - z = tl.zeros([BLOCK_D], dtype=tl.float32) - - if not COMBINED_HISTORY: - tl.store(M_rev_ptr + bh * F * BLOCK_D * BLOCK_D + (F - 1) * BLOCK_D * BLOCK_D + offs_dd_tile, M) - if is_lead and not SKIP_Z: - tl.store(z_rev_ptr + bh * F * BLOCK_D + (F - 1) * BLOCK_D + offs_d_full, z) - - for f_iter in range(F - 1): - f_src = F - 1 - f_iter - f_dst = f_src - 1 - I_P_kv_f = tl.load(I_P_kv_ptr + bh * F * BLOCK_D * BLOCK_D + f_src * BLOCK_D * BLOCK_D + offs_dd_full) - A_f = tl.load(A_ptr + bh * F * BLOCK_D * BLOCK_D + f_src * BLOCK_D * BLOCK_D + offs_dd_tile) - g_f = tl.load(decay_ptr + bh * F + f_src).to(tl.float32) - - if USE_ACC_FUSION: - I_P_scaled = I_P_kv_f.to(tl.float32) * g_f - M = tl.dot( - I_P_scaled.to(dot_dtype), - M.to(dot_dtype), - acc=A_f.to(tl.float32), - out_dtype=tl.float32, - input_precision=dot_ip, - ) - else: - M_temp = tl.dot(I_P_kv_f.to(dot_dtype), M.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) - M = g_f * M_temp + A_f - - if is_lead and not SKIP_Z: - I_P_z_f = tl.load(I_P_z_ptr + bh * F * BLOCK_D * BLOCK_D + f_src * BLOCK_D * BLOCK_D + offs_dd_full) - B_f = tl.load(B_ptr + bh * F * BLOCK_D + f_src * BLOCK_D + offs_d_full) - z_temp = tl.sum(I_P_z_f * z[None, :], axis=1) - z = g_f * z_temp + B_f - - if COMBINED_HISTORY: - M_addr = M_fwd_ptr + bh * F * BLOCK_D * BLOCK_D + f_dst * BLOCK_D * BLOCK_D + offs_dd_tile - tl.store(M_addr, tl.load(M_addr) + M) - if is_lead and not SKIP_Z: - z_addr = z_fwd_ptr + bh * F * BLOCK_D + f_dst * BLOCK_D + offs_d_full - tl.store(z_addr, tl.load(z_addr) + z) - else: - tl.store(M_rev_ptr + bh * F * BLOCK_D * BLOCK_D + f_dst * BLOCK_D * BLOCK_D + offs_dd_tile, M) - if is_lead and not SKIP_Z: - tl.store(z_rev_ptr + bh * F * BLOCK_D + f_dst * BLOCK_D + offs_d_full, z) - - -_PHASE_B_DTILE_ARCH_CACHE: dict = {} # (dev, dot_prec) -> (d_splits, nw, ns, acc) - - -# Per-arch D-tile optimum from 2026-04-29 sweep (T=11 B=1 P0 IEEE): -# WGMMA-server (A100 sm_80, H100 sm_90): (d=4, nw=32, ns=1, acc=True) -# Blackwell-family (GB200 sm_100, 5090 sm_120, GB10 sm_121, Ada sm_89): -# (d=8, nw=4, ns=1, acc=False) -# Both clusters were tested across 96 configs (4 ds × 4 nw × 3 ns × 2 acc). -def _pick_phase_b_d_splits(BLOCK_D: int, dot_precision: int = 0): - """Returns (d_splits, nw_override, ns_override, acc_override). - - `d_splits=1` → use baseline `_phase_b_kernel` with `_CHUNKWISE_TUNING` config. - `d_splits>1` → use `_phase_b_dtile_kernel` with overrides for nw/ns/acc. - Override via env: PHASE_B_D_SPLITS, PHASE_B_DTILE_NW, PHASE_B_DTILE_NS, - PHASE_B_DTILE_ACC (1=True / 0=False). - """ - import os - - env_d = os.environ.get("PHASE_B_D_SPLITS", None) - if env_d is not None: - d = int(env_d) - if d < 1 or BLOCK_D % d != 0: - return (1, None, None, None) - nw = int(os.environ.get("PHASE_B_DTILE_NW", "0")) or None - ns = int(os.environ.get("PHASE_B_DTILE_NS", "0")) or None - acc_env = os.environ.get("PHASE_B_DTILE_ACC", None) - acc = bool(int(acc_env)) if acc_env is not None else None - return (d, nw, ns, acc) - try: - import torch - - if not torch.cuda.is_available(): - return (1, None, None, None) - dev = torch.cuda.current_device() - cache_key = (dev, dot_precision) - if cache_key not in _PHASE_B_DTILE_ARCH_CACHE: - cap = torch.cuda.get_device_capability(dev) - major, minor = cap[0], cap[1] - if dot_precision == 2: - # IEEE fp32: D-tile dominates baseline on every arch (96-config sweep). - if major == 8 and minor == 0: - cfg = (4, 32, 1, True) # A100 - elif major == 9: - cfg = (4, 32, 1, True) # H100 (Hopper) - elif major == 8 and minor == 9: - cfg = (8, 4, 1, False) # Ada (assume Blackwell-like) - elif major >= 10: - cfg = (8, 4, 1, False) # GB200/B200, 5090, GB10 - else: - cfg = (1, None, None, None) # unknown — baseline - else: - # bf16/TF32: cap-specific dispatch. Multi-arch sweep 2026-05-06 - # (F=11 S=920) determined per-cap whether D-tile beats the - # baseline _phase_b_kernel: - # sm_80 A100: D-tile WIN 1.09× (P1) / 1.02× (P2) — (4,8,2,F). - # sm_90 H100: D-tile WIN ~10% — P1 (4,8,2,F); P2 (8,8,2,F). - # Use (4,8,2,F) for both (P2 within 0.4%). - # sm_100 GB200: D-tile WIN ~12% — (4,8,2,F) both precisions. - # sm_120 5090: D-tile WIN 2.6× (P1) / 1.13× (P2) — (8,8,1,F). - # TF32 baseline OOMs at 102 KB SRAM cap. - # sm_121 GB10: D-tile LOSS 4% — baseline wins. Despite same - # reported SRAM/SM as sm_120, the baseline - # kernel fits all configs up to nw=16 ns=2 on - # sm_121 (Triton/codegen difference between - # consumer-Blackwell variants), so baseline - # saturates the chip without needing D-tile. - if major == 8 and minor == 0: - cfg = (4, 8, 2, False) # A100 - elif major == 9: - cfg = (4, 8, 2, False) # H100 - elif major == 10: - cfg = (4, 8, 2, False) # GB200 / B200 - elif major == 12 and minor == 0: - cfg = (8, 8, 1, False) # 5090 - elif major == 12 and minor == 1: - cfg = (1, None, None, None) # GB10 — baseline wins - else: - cfg = (1, None, None, None) # Ada, unknown - _PHASE_B_DTILE_ARCH_CACHE[cache_key] = cfg - return _PHASE_B_DTILE_ARCH_CACHE[cache_key] - except Exception: - return (1, None, None, None) - - -# ════════════════════════════════════════════════════════════════ -# Phase C — Pass 2 output (per (B, H, F)). Same as v1. -# ════════════════════════════════════════════════════════════════ - - -@triton.jit -def _phase_c_kernel( - qkv_ptr, - stride_b: tl.constexpr, - stride_n: tl.constexpr, - stride_3: tl.constexpr, - stride_h: tl.constexpr, - stride_d: tl.constexpr, - q_inv_rms_ptr, - q_norm_w_ptr, - rope_cos_ptr, - rope_sin_ptr, - M_ptr, - z_ptr, - num_ptr, - den_ptr, - H: tl.constexpr, - F: tl.constexpr, - S: tl.constexpr, - D: tl.constexpr, - NORM_EPS: tl.constexpr, - DOT_PRECISION: tl.constexpr, - BLOCK_D: tl.constexpr, - BLOCK_S: tl.constexpr, - ACCUMULATE: tl.constexpr = False, - SKIP_LAST_F: tl.constexpr = False, - SKIP_RELU: tl.constexpr = False, - NUM_ONLY: tl.constexpr = False, -): - if DOT_PRECISION >= 1: - dot_dtype = tl.float32 - else: - dot_dtype = tl.bfloat16 - dot_ip: tl.constexpr = "ieee" if DOT_PRECISION == 2 else "tf32" - - pid = tl.program_id(0) - pid_b = pid // (H * F) - pid_hf = pid % (H * F) - pid_h = pid_hf // F - pid_f = pid_hf % F - bh = pid_b * H + pid_h - N: tl.constexpr = F * S - - # Reverse-accumulate callers pass SKIP_LAST_F=True: M_rev[F-1] / z_rev[F-1] - # are exactly zero (Phase B initializes the reverse scan with zeros and the - # write loop only fills f 0, Q_normed, 0.0) - Q_pair = tl.where(Q_pair_normed > 0, Q_pair_normed, 0.0) - - rope_ptrs = n_idx[:, None] * D + offs_d[None, :] - Cos = tl.load(rope_cos_ptr + rope_ptrs, mask=mask_sd, other=1.0).to(tl.float32) - Sin = tl.load(rope_sin_ptr + rope_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - Q_rot = Q * Cos + Q_pair * Sin - - num = tl.dot(Q_rot.to(dot_dtype), M_f.to(dot_dtype), out_dtype=tl.float32, input_precision=dot_ip) - if not NUM_ONLY: - den = tl.sum(Q * z_f[None, :], axis=1) - - num_ptrs = num_bh + n_idx[:, None] * (H * D) + offs_d[None, :] - if not NUM_ONLY: - den_ptrs = den_bh + n_idx - if ACCUMULATE: - # Used by reverse-direction Phase C: add this pass onto forward's - # already-written buffer instead of allocating a separate one. - prev_num = tl.load(num_ptrs, mask=mask_sd, other=0.0).to(tl.float32) - num = num + prev_num - if not NUM_ONLY: - prev_den = tl.load(den_ptrs, mask=mask_s, other=0.0).to(tl.float32) - den = den + prev_den - if DOT_PRECISION >= 1: - tl.store(num_ptrs, num, mask=mask_sd) - if not NUM_ONLY: - tl.store(den_ptrs, den, mask=mask_s) - else: - tl.store(num_ptrs, num.to(tl.bfloat16), mask=mask_sd) - if not NUM_ONLY: - tl.store(den_ptrs, den.to(tl.bfloat16), mask=mask_s) - - -def phase_c( - qkv, - q_inv_rms, - q_norm_w, - rope_cos, - rope_sin, - M, - z, - F, - S, - num_warps=None, - num_stages=None, - BLOCK_S=None, - dot_precision=0, - num_out=None, - den_out=None, - accumulate=False, - skip_last_frame=False, - skip_relu: bool = False, - num_only: bool = False, -): - """Phase C Pass-2 output. Optionally accumulates into caller-provided - ``num_out``/``den_out`` buffers (used to fuse reverse-direction output into - forward-direction buffer without allocating a separate one — saves ~45 MB - at B=1 bf16, ~180 MB at B=4). - - ``skip_last_frame=True`` early-returns the f=F-1 programs. Valid for the - reverse-accumulate call only, where M[F-1]/z[F-1] are guaranteed zero. - - ``skip_relu=True`` matches Phase A KV's flag — used by the camera-branch - chunkwise wrapper where Q has already been ReLU'd by cam_prep before - being rotated by UCPE+RoPE; re-applying ReLU on the rotated Q would - clobber legitimate negatives. - - ``num_only=True`` skips the denominator computation and store entirely - (kernel writes only ``num_out``; ``den_out`` is allowed to be None / - unallocated). Used by the camera-branch which has no Z scan. - """ - if num_warps is None or num_stages is None or BLOCK_S is None: - *_, c_w, c_bs, c_s = _get_arch_config(dot_precision, device=qkv.device) - if num_warps is None: - num_warps = c_w - if num_stages is None: - num_stages = c_s - if BLOCK_S is None: - BLOCK_S = c_bs - B, N, three, H, D = qkv.shape - BLOCK_D = triton.next_power_of_2(D) - if num_out is None: - num_out = torch.empty( - B, N, H, D, device=qkv.device, dtype=(torch.float32 if dot_precision >= 1 else torch.bfloat16) - ) - if den_out is None and not num_only: - den_out = torch.empty( - B, H, N, device=qkv.device, dtype=(torch.float32 if dot_precision >= 1 else torch.bfloat16) - ) - elif num_only and den_out is None: - # Pass a 1-element placeholder; kernel guards den loads/stores under NUM_ONLY. - den_out = torch.empty(1, device=qkv.device, dtype=(torch.float32 if dot_precision >= 1 else torch.bfloat16)) - - _phase_c_kernel[(B * H * F,)]( - qkv, - qkv.stride(0), - qkv.stride(1), - qkv.stride(2), - qkv.stride(3), - qkv.stride(4), - q_inv_rms, - q_norm_w, - rope_cos, - rope_sin, - M, - z, - num_out, - den_out, - H=H, - F=F, - S=S, - D=D, - NORM_EPS=1e-5, - DOT_PRECISION=dot_precision, - BLOCK_D=BLOCK_D, - BLOCK_S=BLOCK_S, - ACCUMULATE=1 if accumulate else 0, - SKIP_LAST_F=skip_last_frame, - SKIP_RELU=skip_relu, - NUM_ONLY=num_only, - num_warps=num_warps, - num_stages=num_stages, - ) - return num_out, den_out - - -def fused_bigdn_bidi_chunkwise( - qkv, - q_inv_rms, - k_inv_rms, - q_norm_w, - k_norm_w, - rope_cos, - rope_sin, - beta, - decay, - F, - S, - k_scale=1.0, - eps=1e-6, - norm_eps=1e-5, - dot_precision=0, - init_state_kv=None, - init_state_z=None, - return_final_state=False, -): - """Bidi chunkwise GDN forward, optionally with state-cache for autoregressive - sampling (chunk 0 = full bidi with state save; chunks > 0 seed forward scan - from saved state). Reverse always seeds from zero per upstream convention. - - Pipeline (2026-04-25 restructure): Phase A once → Phase B direction=0 with - combined_history=True (fwd seeded with init_state and saves final state; - rev zero-seeded; rev output summed into fwd buffer in-kernel via read- - add-store so on exit M_hist[f] = M_fwd[f] + M_rev[f]) → Phase C ONCE on - M_hist. Phase C linearity `Q @ (M_fwd + M_rev) = Q @ M_fwd + Q @ M_rev` - makes the in-kernel sum exact. - - Replaces the prior 2× Phase B + 2× Phase C pattern. Saves one Phase C - launch + one Q+RoPE HBM pass and one M-shape buffer per call. - """ - I_P_kv, A, I_P_z, B_z = phase_a( - qkv, - beta, - q_inv_rms, - k_inv_rms, - q_norm_w, - k_norm_w, - rope_cos, - rope_sin, - F=F, - S=S, - k_scale=k_scale, - norm_eps=norm_eps, - dot_precision=dot_precision, - ) - - if return_final_state: - M_hist, z_hist, _, _, final_kv, final_z = phase_b_triton( - I_P_kv, - A, - I_P_z, - B_z, - decay, - F=F, - dot_precision=dot_precision, - direction=0, - init_state_kv=init_state_kv, - init_state_z=init_state_z, - return_final_state=True, - combined_history=True, - ) - else: - M_hist, z_hist, _, _ = phase_b_triton( - I_P_kv, - A, - I_P_z, - B_z, - decay, - F=F, - dot_precision=dot_precision, - direction=0, - init_state_kv=init_state_kv, - init_state_z=init_state_z, - combined_history=True, - ) - num_out, den_out = phase_c( - qkv, - q_inv_rms, - q_norm_w, - rope_cos, - rope_sin, - M_hist, - z_hist, - F=F, - S=S, - dot_precision=dot_precision, - accumulate=False, - ) - del M_hist, z_hist, I_P_kv, A, I_P_z, B_z - - # ── Final divide ── - total_den = den_out.float().permute(0, 2, 1).unsqueeze(-1) # (B, N, H, 1) - out = (num_out.float() / (total_den + eps)).to(qkv.dtype) - del num_out, den_out, total_den - if return_final_state: - B = qkv.shape[0] - H = qkv.shape[3] - D = qkv.shape[4] - BLOCK_D = final_kv.shape[1] - state_kv = final_kv.view(B, H, BLOCK_D, BLOCK_D)[:, :, :D, :D].transpose(-1, -2).contiguous() - state_z = final_z.view(B, H, BLOCK_D)[:, :, :D].unsqueeze(-1).contiguous() - return out, state_kv, state_z - return out - - -def _default_dot_prec(): - """Pull dot_precision from `_resolve_launch_config` (honors PRECISION_OVERRIDE).""" - from .fused_gdn import _resolve_launch_config - - _, dot_prec, _, _ = _resolve_launch_config() - return dot_prec - - -def fused_gdn_func_chunkwise( - qkv, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - beta, - decay, - F, - S, - k_scale, - eps=1e-6, - reverse=False, - dot_precision=None, -): - """Single-direction chunkwise GDN — drop-in for `fused_gdn.fused_gdn_func`. - - Computes only one scan direction (Phase B + Phase C × 1) and returns - `(num, den)` shape-compatible with the upstream function. dot_precision - defaults to whatever `_resolve_launch_config` returns (honors module-level - `PRECISION_OVERRIDE`). - """ - if dot_precision is None: - dot_precision = _default_dot_prec() - direction = 2 if reverse else 1 - I_P_kv, A, I_P_z, B_z = phase_a( - qkv, - beta, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - F=F, - S=S, - k_scale=k_scale, - dot_precision=dot_precision, - ) - M_fwd, z_fwd, M_rev, z_rev = phase_b_triton( - I_P_kv, - A, - I_P_z, - B_z, - decay, - F=F, - dot_precision=dot_precision, - direction=direction, - ) - M_use = M_rev if reverse else M_fwd - z_use = z_rev if reverse else z_fwd - num, den = phase_c( - qkv, q_inv_rms, q_norm_weight, rope_cos, rope_sin, M_use, z_use, F=F, S=S, dot_precision=dot_precision - ) - return num, den - - -def fused_gdn_stateful_chunkwise( - qkv, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - beta, - decay, - F, - S, - k_scale, - eps=1e-6, - reverse=False, - init_state_kv=None, - init_state_z=None, - return_final_state=False, - dot_precision=None, -): - """Single-direction chunkwise GDN with optional state cache — drop-in for - `fused_gdn.fused_gdn_stateful`. Forward direction supports state load/save - (used for autoregressive sampling); reverse direction always runs fresh - (per upstream's bidi state-cache convention). - """ - if dot_precision is None: - dot_precision = _default_dot_prec() - direction = 2 if reverse else 1 - if reverse and (init_state_kv is not None or return_final_state): - raise ValueError( - "fused_gdn_stateful_chunkwise: state cache is forward-only (matching " - "upstream's bidi convention); pass reverse=False or omit state args." - ) - I_P_kv, A, I_P_z, B_z = phase_a( - qkv, - beta, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - F=F, - S=S, - k_scale=k_scale, - dot_precision=dot_precision, - ) - # Pad caller-supplied state from (B,H,D,D)/(B,H,D,1) to (BH, BLOCK_D, BLOCK_D)/(BH, BLOCK_D). - # Needed because the state returned by this function is unpadded (B,H,D,D), - # but phase_b_triton's kernel expects the padded layout. - init_kv_padded, init_z_padded = init_state_kv, init_state_z - if init_state_kv is not None: - B_, H_, D_in, D_out = init_state_kv.shape - BLOCK_D_ = I_P_kv.shape[-1] - if D_in != BLOCK_D_ or D_out != BLOCK_D_: - pad_in = BLOCK_D_ - D_in - pad_out = BLOCK_D_ - D_out - init_kv_padded = torch.nn.functional.pad( - init_state_kv.transpose(-1, -2).reshape(B_ * H_, D_out, D_in), (0, pad_in, 0, pad_out) - ).contiguous() - else: - init_kv_padded = init_state_kv.transpose(-1, -2).reshape(B_ * H_, BLOCK_D_, BLOCK_D_).contiguous() - # z: (B, H, D) or (B, H, D, 1) → (BH, BLOCK_D) - z_ = init_state_z.squeeze(-1) if init_state_z.dim() == 4 else init_state_z - Bz_, Hz_, Dz_ = z_.shape - if Dz_ != BLOCK_D_: - init_z_padded = torch.nn.functional.pad(z_.reshape(Bz_ * Hz_, Dz_), (0, BLOCK_D_ - Dz_)).contiguous() - else: - init_z_padded = z_.reshape(Bz_ * Hz_, Dz_).contiguous() - if return_final_state: - M_fwd, z_fwd, M_rev, z_rev, final_kv, final_z = phase_b_triton( - I_P_kv, - A, - I_P_z, - B_z, - decay, - F=F, - dot_precision=dot_precision, - direction=direction, - init_state_kv=init_kv_padded, - init_state_z=init_z_padded, - return_final_state=True, - ) - else: - M_fwd, z_fwd, M_rev, z_rev = phase_b_triton( - I_P_kv, - A, - I_P_z, - B_z, - decay, - F=F, - dot_precision=dot_precision, - direction=direction, - init_state_kv=init_kv_padded, - init_state_z=init_z_padded, - ) - M_use = M_rev if reverse else M_fwd - z_use = z_rev if reverse else z_fwd - num, den = phase_c( - qkv, q_inv_rms, q_norm_weight, rope_cos, rope_sin, M_use, z_use, F=F, S=S, dot_precision=dot_precision - ) - if return_final_state: - B = qkv.shape[0] - H = qkv.shape[3] - D = qkv.shape[4] - BLOCK_D = final_kv.shape[1] - state_kv = final_kv.view(B, H, BLOCK_D, BLOCK_D)[:, :, :D, :D].transpose(-1, -2).contiguous() - state_z = final_z.view(B, H, BLOCK_D)[:, :, :D].unsqueeze(-1).contiguous() - return num, den, state_kv, state_z - return num, den - - -def fused_bidi_stateful_chunkwise_shared_phase_a( - qkv, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - beta, - decay, - F, - S, - k_scale, - eps=1e-6, - init_state_kv=None, - init_state_z=None, - dot_precision=None, -): - """Bidi state-cached chunkwise GDN with shared Phase A and combined-history - Phase B. Default chunkwise path for ``_fused_statecached_forward``. - - Pipeline (per layer per step): - 1. Phase A once over qkv — K/V/RoPE pre-norm; was previously duplicated - across two streams. - 2. Phase B with direction=0 + combined_history=True — single program does - fwd then rev; fwd writes M_hist; rev read-add-stores into the same - buffer so on exit M_hist[f] = M_fwd[f] + M_rev[f] (same for z). - Forward branch loads init_state and saves final state. - 3. Phase C ONCE on M_hist/z_hist — Phase C is linear in M/z so - `Q @ (M_fwd + M_rev) = Q @ M_fwd + Q @ M_rev`. - - Returns ``(num_combined, den_combined, state_kv, state_z)`` — caller hands - the num/den pair to ``fused_bidi_merge(num, None, den, None, eps, gate)`` - in PRE_SUMMED mode. - - HBM-traffic delta vs the prior 2× Phase C version (per call, B=1 prod): - saved : 1× Phase C Q+RoPE pass (~90 MB) - saved : one (B,N,H,D) num and (B,H,N) den allocation - cost : Phase B rev does read-add of M_hist (~14 MB extra per layer) - net : ~76 MB saved + 1 fewer kernel launch - - Measured speed on GB10 (sm_121) at H=20, S=920, D=112, vs the prior - shared-Phase-A-with-2×-Phase-C path, across production F values: - P0 IEEE fp32 : 1.26-1.42× (F=3,6,11; B=1,2) - P2 bf16+fp32-st : 1.57-1.80× - P3 bf16+bf16-st : 1.63-1.96× - Correctness cos ≥ 0.999997 across all cells, state_kv exact. - """ - if dot_precision is None: - dot_precision = _default_dot_prec() - - I_P_kv, A, I_P_z, B_z = phase_a( - qkv, - beta, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - F=F, - S=S, - k_scale=k_scale, - dot_precision=dot_precision, - ) - - init_kv_padded, init_z_padded = init_state_kv, init_state_z - if init_state_kv is not None: - B_, H_, D_in, D_out = init_state_kv.shape - BLOCK_D_ = I_P_kv.shape[-1] - if D_in != BLOCK_D_ or D_out != BLOCK_D_: - pad_in = BLOCK_D_ - D_in - pad_out = BLOCK_D_ - D_out - init_kv_padded = torch.nn.functional.pad( - init_state_kv.transpose(-1, -2).reshape(B_ * H_, D_out, D_in), (0, pad_in, 0, pad_out) - ).contiguous() - else: - init_kv_padded = init_state_kv.transpose(-1, -2).reshape(B_ * H_, BLOCK_D_, BLOCK_D_).contiguous() - z_ = init_state_z.squeeze(-1) if init_state_z.dim() == 4 else init_state_z - Bz_, Hz_, Dz_ = z_.shape - if Dz_ != BLOCK_D_: - init_z_padded = torch.nn.functional.pad(z_.reshape(Bz_ * Hz_, Dz_), (0, BLOCK_D_ - Dz_)).contiguous() - else: - init_z_padded = z_.reshape(Bz_ * Hz_, Dz_).contiguous() - - # combined_history=True routes the rev contribution into the fwd buffer → - # M_hist[f] = M_fwd[f] + M_rev[f]. M_rev/z_rev outputs are placeholders. - M_hist, z_hist, _, _, final_kv, final_z = phase_b_triton( - I_P_kv, - A, - I_P_z, - B_z, - decay, - F=F, - dot_precision=dot_precision, - direction=0, - init_state_kv=init_kv_padded, - init_state_z=init_z_padded, - return_final_state=True, - combined_history=True, - ) - - num, den = phase_c( - qkv, q_inv_rms, q_norm_weight, rope_cos, rope_sin, M_hist, z_hist, F=F, S=S, dot_precision=dot_precision - ) - - B = qkv.shape[0] - H = qkv.shape[3] - D = qkv.shape[4] - BLOCK_D = final_kv.shape[1] - state_kv = final_kv.view(B, H, BLOCK_D, BLOCK_D)[:, :, :D, :D].transpose(-1, -2).contiguous() - state_z = final_z.view(B, H, BLOCK_D)[:, :, :D].unsqueeze(-1).contiguous() - return num, den, state_kv, state_z - - -def fused_bigdn_stateful_chunkwise( - qkv, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - beta, - decay, - F, - S, - k_scale, - eps=1e-6, - return_final_state=False, - dot_precision=None, -): - """Drop-in replacement for `fused_gdn.fused_bigdn_stateful` using the - chunkwise pipeline. Same signature, same return shape: - output (B, N, H, D), and if return_final_state: + (state_kv, state_z). - dot_precision defaults to whatever `_resolve_launch_config` returns. - """ - if dot_precision is None: - dot_precision = _default_dot_prec() - if return_final_state: - out, state_kv, state_z = fused_bigdn_bidi_chunkwise( - qkv, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - beta, - decay, - F=F, - S=S, - k_scale=k_scale, - eps=eps, - dot_precision=dot_precision, - return_final_state=True, - ) - return out, state_kv, state_z - out = fused_bigdn_bidi_chunkwise( - qkv, - q_inv_rms, - k_inv_rms, - q_norm_weight, - k_norm_weight, - rope_cos, - rope_sin, - beta, - decay, - F=F, - S=S, - k_scale=k_scale, - eps=eps, - dot_precision=dot_precision, - ) - return out - - -# ───────────────────────────────────────────────────────────────────────────── -# Camera-branch wrapper — numerator-only single-path delta-rule scan via -# chunkwise. Drop-in for `diffusion.model.ops.fused_cam_gdn.cam_scan_func`. -# -# Cam math expanded: -# state = state * g # apply decay -# state += K^T @ ((V - K @ state) * β) # delta-rule -# Equivalently: -# state_new = g (I - K^T β K) state_old + K^T β V -# = g (I - P_kv) state_old + A -# This is bit-identical to chunkwise's Phase B M update, so the scan kernel -# is reusable. The only differences from main GDN: -# 1. Q/K/V come pre-prepped (cam_prep_kernel did RMSNorm+ReLU+UCPE+RoPE). -# We disable chunkwise's prep with identity tables (k_inv_rms=1, k_nw=1, -# k_scale=1, rope_cos=1, rope_sin=0) AND skip_relu=True (because cam -# applied ReLU BEFORE UCPE; the post-UCPE values can have legitimate -# negatives that re-applying ReLU would clobber). -# 2. No Z denominator scan; output is num-only (out = Q @ M, no /Z). -# skip_z=True elides Phase A Z; num_only=True elides Phase C den compute. -# ───────────────────────────────────────────────────────────────────────────── -def _cam_identity_tables( - *, - B: int, - N: int, - H: int, - D: int, - device: torch.device, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Cached identity RMS/RoPE tables used by ``cam_scan_chunkwise``.""" - device_index = device.index if device.type == "cuda" else None - key = (device.type, device_index, B, N, H * D, D) - cached = _CAM_IDENTITY_CACHE.get(key) - if cached is not None: - return cached - - ones_inv_rms = torch.ones(B, N, device=device, dtype=torch.float32) - ones_nw = torch.ones(H * D, device=device, dtype=torch.float32) - ones_cos = torch.ones(N, D, device=device, dtype=torch.float32) - zeros_sin = torch.zeros(N, D, device=device, dtype=torch.float32) - cached = (ones_inv_rms, ones_nw, ones_cos, zeros_sin) - _CAM_IDENTITY_CACHE[key] = cached - return cached - - -def cam_scan_chunkwise( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - beta: torch.Tensor, - decay: torch.Tensor, - *, - reverse: bool = False, - init_state: torch.Tensor | None = None, - save_final_state: bool = False, - dot_precision: int | None = None, -): - """Drop-in chunkwise replacement for `cam_scan_func`. - - Args mirror `cam_scan_func` exactly: - q, k, v: ``(B, H, D, N)`` fp32 contiguous (cam-prep'd: RMSNorm+ReLU+UCPE+RoPE) - beta: ``(B, H, F, S)`` fp32 contiguous - decay: ``(B, H, F)`` fp32 contiguous - reverse: bwd flip-and-shift semantics (autograd path); not yet supported. - init_state: optional ``(B*H, BLOCK_D, BLOCK_D)`` fp32 — cross-chunk AR state. - save_final_state: when True, also returns ``(out, final_state)``. - - Returns ``out`` of shape ``(B, H, D, N)`` fp32, or - ``(out, final_state: (B*H, BLOCK_D, BLOCK_D))`` if save_final_state=True. - """ - assert q.shape == k.shape == v.shape, f"q/k/v shape mismatch: {q.shape} {k.shape} {v.shape}" - assert q.is_contiguous() and k.is_contiguous() and v.is_contiguous() - assert beta.is_contiguous() and decay.is_contiguous() - assert q.dtype == torch.float32, f"cam_scan_chunkwise requires fp32 q/k/v (got {q.dtype})" - - if reverse and (init_state is not None or save_final_state): - raise NotImplementedError( - "cam_scan_chunkwise: state passing (init_state / save_final_state) is " - "only supported for the forward direction (reverse=False). The cam " - "branch's anti-causal pass resets per chunk; there is no global " - "cross-prefix state to cache for the reverse direction." - ) - - B, H, D, N = q.shape - F = beta.shape[2] - assert N % F == 0 - S = N // F - assert beta.shape == (B, H, F, S) - assert decay.shape == (B, H, F) - - BLOCK_D = triton.next_power_of_2(D) - - if dot_precision is None: - dot_precision = _default_dot_prec() - - # Repack (B, H, D, N) → (B, N, 3, H, D) for chunkwise's qkv layout. - # Avoid ``stack(...).permute(...).contiguous()`` because that materializes - # two large tensors. Direct packing allocates the destination once. - qkv = torch.empty(B, N, 3, H, D, device=q.device, dtype=q.dtype) - qkv[:, :, 0].copy_(q.permute(0, 3, 1, 2)) - qkv[:, :, 1].copy_(k.permute(0, 3, 1, 2)) - qkv[:, :, 2].copy_(v.permute(0, 3, 1, 2)) - - # Identity prep tables — make chunkwise's RMSNorm + RoPE no-ops. - ones_inv_rms, ones_nw, ones_cos, zeros_sin = _cam_identity_tables(B=B, N=N, H=H, D=D, device=q.device) - - # Phase A (skip_relu=True for cam-prep'd K; skip_z=True since cam has no Z scan). - # k_scale=1.0 because cam_prep already applied K-scale. - I_P_kv, A_, I_P_z, B_z = phase_a( - qkv, - beta, - ones_inv_rms, - ones_inv_rms, - ones_nw, - ones_nw, - ones_cos, - zeros_sin, - F=F, - S=S, - k_scale=1.0, - norm_eps=1e-5, - dot_precision=dot_precision, - skip_relu=True, - skip_z=True, - ) - - # Phase B (forward direction only; cam supports init_state on fwd, save_final - # on fwd; no rev). Pads (B*H, D, D) ↔ (B*H, BLOCK_D, BLOCK_D) inline. - init_kv_padded = None - init_z_padded = None - if init_state is not None: - if init_state.shape != (B * H, BLOCK_D, BLOCK_D): - raise ValueError( - f"cam_scan_chunkwise: init_state shape {tuple(init_state.shape)} " - f"!= expected (B*H, BLOCK_D, BLOCK_D) = {(B * H, BLOCK_D, BLOCK_D)}" - ) - if init_state.dtype != torch.float32: - raise ValueError(f"cam_scan_chunkwise: init_state must be fp32 (got {init_state.dtype}).") - if not init_state.is_contiguous(): - raise ValueError("cam_scan_chunkwise: init_state must be contiguous.") - # Cam stores state as M[K_feat, V_feat]. Chunkwise's Phase B kernel reads - # state with offs_dd = i*BLOCK_D + j where i is the fwd loop's M row. - # Storage layout matches cam's (row-major (D_K, D_V)), so a direct cast - # to fp32 contiguous is enough — no transpose needed. - init_kv_padded = init_state.to(torch.float32).contiguous() - # No Z state in cam — pass zeros to satisfy phase_b_triton. - init_z_padded = torch.zeros(B * H, BLOCK_D, device=q.device, dtype=torch.float32) - - direction = 2 if reverse else 1 - if save_final_state: - M_fwd, z_fwd_out, M_rev, z_rev_out, final_kv, _final_z = phase_b_triton( - I_P_kv, - A_, - I_P_z, - B_z, - decay, - F=F, - dot_precision=dot_precision, - direction=direction, - init_state_kv=init_kv_padded, - init_state_z=init_z_padded, - return_final_state=True, - skip_z=True, - ) - else: - M_fwd, z_fwd_out, M_rev, z_rev_out = phase_b_triton( - I_P_kv, - A_, - I_P_z, - B_z, - decay, - F=F, - dot_precision=dot_precision, - direction=direction, - init_state_kv=init_kv_padded, - init_state_z=init_z_padded, - skip_z=True, - ) - - # For reverse (flip-and-shift bwd), Phase B's reverse mode produces M_rev - # such that M_rev[F-1] = 0 and M_rev[t] = state computed from K/V at frames - # {F-1, F-2, ..., t+1} — exactly cam's REVERSE=1 semantics. - M_use = M_rev if reverse else M_fwd - z_use = z_rev_out if reverse else z_fwd_out - - # Phase C — num-only (NUM_ONLY=True skips den compute + store). - # z is unused with NUM_ONLY but still required by the kernel signature. - num_out, _ = phase_c( - qkv, - ones_inv_rms, - ones_nw, - ones_cos, - zeros_sin, - M_use, - z_use, - F=F, - S=S, - dot_precision=dot_precision, - skip_relu=True, - num_only=True, - ) - - # Convert chunkwise output (B, N, H, D) → cam's (B, H, D, N) layout, fp32. - out = num_out.permute(0, 2, 3, 1).contiguous().to(torch.float32) - - if save_final_state: - return out, final_kv # final_kv already (B*H, BLOCK_D, BLOCK_D) fp32 - return out - - -def cam_scan_bidi_chunkwise( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - beta: torch.Tensor, - decay: torch.Tensor, - *, - dot_precision: int | None = None, -) -> torch.Tensor: - """Bidirectional camera scan using shared chunkwise phases. - - This is equivalent to ``cam_scan_chunkwise(..., reverse=False) + - cam_scan_chunkwise(..., reverse=True)`` for full bidirectional attention, - but it packs QKV once, runs Phase A once, combines forward/reverse histories - inside Phase B, and runs Phase C once on the summed state. - """ - assert q.shape == k.shape == v.shape, f"q/k/v shape mismatch: {q.shape} {k.shape} {v.shape}" - assert q.is_contiguous() and k.is_contiguous() and v.is_contiguous() - assert beta.is_contiguous() and decay.is_contiguous() - assert q.dtype == torch.float32, f"cam_scan_bidi_chunkwise requires fp32 q/k/v (got {q.dtype})" - - B, H, D, N = q.shape - F = beta.shape[2] - assert N % F == 0 - S = N // F - assert beta.shape == (B, H, F, S) - assert decay.shape == (B, H, F) - - if dot_precision is None: - dot_precision = _default_dot_prec() - - qkv = torch.empty(B, N, 3, H, D, device=q.device, dtype=q.dtype) - qkv[:, :, 0].copy_(q.permute(0, 3, 1, 2)) - qkv[:, :, 1].copy_(k.permute(0, 3, 1, 2)) - qkv[:, :, 2].copy_(v.permute(0, 3, 1, 2)) - - ones_inv_rms, ones_nw, ones_cos, zeros_sin = _cam_identity_tables(B=B, N=N, H=H, D=D, device=q.device) - I_P_kv, A_, I_P_z, B_z = phase_a( - qkv, - beta, - ones_inv_rms, - ones_inv_rms, - ones_nw, - ones_nw, - ones_cos, - zeros_sin, - F=F, - S=S, - k_scale=1.0, - norm_eps=1e-5, - dot_precision=dot_precision, - skip_relu=True, - skip_z=True, - ) - M_hist, z_hist, _, _ = phase_b_triton( - I_P_kv, - A_, - I_P_z, - B_z, - decay, - F=F, - dot_precision=dot_precision, - direction=0, - combined_history=True, - skip_z=True, - ) - num_out, _ = phase_c( - qkv, - ones_inv_rms, - ones_nw, - ones_cos, - zeros_sin, - M_hist, - z_hist, - F=F, - S=S, - dot_precision=dot_precision, - skip_relu=True, - num_only=True, - ) - return num_out.permute(0, 2, 3, 1).contiguous().to(torch.float32) - - -def cam_scan_pair_chunkwise( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - beta_fwd: torch.Tensor, - decay_fwd: torch.Tensor, - beta_rev: torch.Tensor, - decay_rev: torch.Tensor, - *, - dot_precision: int | None = None, -) -> torch.Tensor: - """Sum a forward camera scan and a separately-gated reverse scan. - - Chunk-causal camera attention needs the reverse branch to use boundary-masked - gates while the forward branch uses the original gates. This wrapper keeps - that exact behavior but shares QKV packing, identity tables, and the final - output layout conversion across the two scans. - """ - assert q.shape == k.shape == v.shape, f"q/k/v shape mismatch: {q.shape} {k.shape} {v.shape}" - assert q.is_contiguous() and k.is_contiguous() and v.is_contiguous() - assert beta_fwd.is_contiguous() and decay_fwd.is_contiguous() - assert beta_rev.is_contiguous() and decay_rev.is_contiguous() - assert q.dtype == torch.float32, f"cam_scan_pair_chunkwise requires fp32 q/k/v (got {q.dtype})" - - B, H, D, N = q.shape - F = beta_fwd.shape[2] - assert N % F == 0 - S = N // F - assert beta_fwd.shape == beta_rev.shape == (B, H, F, S) - assert decay_fwd.shape == decay_rev.shape == (B, H, F) - - if dot_precision is None: - dot_precision = _default_dot_prec() - - qkv = torch.empty(B, N, 3, H, D, device=q.device, dtype=q.dtype) - qkv[:, :, 0].copy_(q.permute(0, 3, 1, 2)) - qkv[:, :, 1].copy_(k.permute(0, 3, 1, 2)) - qkv[:, :, 2].copy_(v.permute(0, 3, 1, 2)) - - ones_inv_rms, ones_nw, ones_cos, zeros_sin = _cam_identity_tables(B=B, N=N, H=H, D=D, device=q.device) - - I_P_kv, A_, I_P_z, B_z = phase_a( - qkv, - beta_fwd, - ones_inv_rms, - ones_inv_rms, - ones_nw, - ones_nw, - ones_cos, - zeros_sin, - F=F, - S=S, - k_scale=1.0, - norm_eps=1e-5, - dot_precision=dot_precision, - skip_relu=True, - skip_z=True, - ) - M_fwd, z_fwd, _, _ = phase_b_triton( - I_P_kv, - A_, - I_P_z, - B_z, - decay_fwd, - F=F, - dot_precision=dot_precision, - direction=1, - skip_z=True, - ) - num_out, _ = phase_c( - qkv, - ones_inv_rms, - ones_nw, - ones_cos, - zeros_sin, - M_fwd, - z_fwd, - F=F, - S=S, - dot_precision=dot_precision, - skip_relu=True, - num_only=True, - ) - del I_P_kv, A_, I_P_z, B_z, M_fwd, z_fwd - - I_P_kv, A_, I_P_z, B_z = phase_a( - qkv, - beta_rev, - ones_inv_rms, - ones_inv_rms, - ones_nw, - ones_nw, - ones_cos, - zeros_sin, - F=F, - S=S, - k_scale=1.0, - norm_eps=1e-5, - dot_precision=dot_precision, - skip_relu=True, - skip_z=True, - ) - _, _, M_rev, z_rev = phase_b_triton( - I_P_kv, - A_, - I_P_z, - B_z, - decay_rev, - F=F, - dot_precision=dot_precision, - direction=2, - skip_z=True, - ) - phase_c( - qkv, - ones_inv_rms, - ones_nw, - ones_cos, - zeros_sin, - M_rev, - z_rev, - F=F, - S=S, - dot_precision=dot_precision, - num_out=num_out, - accumulate=True, - skip_relu=True, - num_only=True, - ) - return num_out.permute(0, 2, 3, 1).contiguous().to(torch.float32) diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index 3ad510d00..f67e76154 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -24,33 +24,12 @@ from collections.abc import Callable from dataclasses import dataclass import math -import os import torch import torch.nn as nn import torch.nn.functional as F from torch import Tensor -try: - from .ops.fused_cam_gdn import ( - _prepare_ucpe_rope_tables as _fused_prepare_ucpe_rope_tables, - cam_prep_func as _fused_cam_prep_func, - ) - from .ops.fused_gdn import ( - fused_bigdn_func as _fused_bigdn_func, - fused_qk_inv_rms as _fused_qk_inv_rms, - prepare_rope_tables as _fused_prepare_rope_tables, - ) - from .ops.fused_gdn_chunkwise import ( - cam_scan_bidi_chunkwise as _fused_cam_scan_bidi_chunkwise, - ) -except ImportError: - _fused_bigdn_func = None - _fused_cam_prep_func = None - _fused_cam_scan_bidi_chunkwise = None - _fused_prepare_rope_tables = None - _fused_prepare_ucpe_rope_tables = None - @dataclass(frozen=True) class SanaWMStage1Spec: @@ -100,8 +79,6 @@ class _CameraProjectionCache: proj_q: Tensor proj_kv: Tensor rope_cam: Tensor | None - rope_cos: Tensor - rope_sin: Tensor class RMSNorm(nn.Module): @@ -472,14 +449,6 @@ def _forward_gdn_main( rotary_emb: Tensor | None, precomputed_gates: tuple[Tensor, Tensor], ) -> Tensor: - if _use_fused_gdn(x): - return self._forward_fused_gdn_main( - x, - HW=HW, - rotary_emb=rotary_emb, - precomputed_gates=precomputed_gates, - ) - batch, tokens, channels = x.shape qkv = self.qkv(x).reshape(batch, tokens, 3, self.heads, self.dim) if hasattr(self, "conv_k"): @@ -519,57 +488,6 @@ def _forward_gdn_main( ) return out.reshape(batch, tokens, channels).to(dtype=x.dtype) - def _forward_fused_gdn_main( - self, - x: Tensor, - *, - HW: tuple[int, int, int], - rotary_emb: Tensor | None, - precomputed_gates: tuple[Tensor, Tensor], - ) -> Tensor: - if ( - _fused_bigdn_func is None - or _fused_qk_inv_rms is None - or _fused_prepare_rope_tables is None - ): - raise RuntimeError("fused SANA-WM GDN kernels are not available.") - - batch, tokens, channels = x.shape - frames, height, width = HW - spatial = height * width - if tokens != frames * spatial: - raise ValueError(f"tokens={tokens} != T*H*W={frames * spatial}") - qkv = self.qkv(x).reshape(batch, tokens, 3, self.heads, self.dim).contiguous() - if hasattr(self, "conv_k"): - k_raw = qkv[:, :, 1].contiguous().reshape(batch, tokens, channels) - k_conv = _apply_bidirectional_temporal_conv(k_raw, self.conv_k, HW) - qkv[:, :, 1].copy_(k_conv.reshape(batch, tokens, self.heads, self.dim)) - - beta, decay = precomputed_gates - q_inv_rms, k_inv_rms = _fused_qk_inv_rms(qkv, eps=self.q_norm.eps) - rope_cos, rope_sin = _fused_prepare_rope_tables( - rotary_emb, - tokens, - self.dim, - x.device, - ) - out = _fused_bigdn_func( - qkv, - q_inv_rms, - k_inv_rms, - self.q_norm.weight.float().contiguous(), - self.k_norm.weight.float().contiguous(), - rope_cos, - rope_sin, - beta.contiguous(), - decay.contiguous(), - F=frames, - S=spatial, - k_scale=_gdn_key_scale(self.dim, HW), - eps=self.eps, - ) - return out.reshape(batch, tokens, channels).to(dtype=x.dtype) - def _forward_softmax_main( self, x: Tensor, @@ -618,16 +536,6 @@ def _forward_gdn_camera( camera_cache: _CameraProjectionCache | None, precomputed_gates: tuple[Tensor, Tensor], ) -> Tensor: - if _use_fused_gdn(x): - return self._forward_fused_gdn_camera( - x, - HW=HW, - rotary_emb=rotary_emb, - camera_conditions=camera_conditions, - camera_cache=camera_cache, - precomputed_gates=precomputed_gates, - ) - q_cam, k_cam, v_cam = _camera_qkv(self, x, HW) q_trans, k_trans, v_trans, inflation_sq, output_projector = _prepare_ucpe_qkv( q_cam, @@ -660,76 +568,6 @@ def _forward_gdn_camera( out = output_projector(out) return out.reshape(x.shape[0], x.shape[1], -1).to(dtype=x.dtype) - def _forward_fused_gdn_camera( - self, - x: Tensor, - *, - HW: tuple[int, int, int], - rotary_emb: Tensor | None, - camera_conditions: Tensor, - camera_cache: _CameraProjectionCache | None, - precomputed_gates: tuple[Tensor, Tensor], - ) -> Tensor: - if ( - _fused_cam_prep_func is None - or _fused_cam_scan_bidi_chunkwise is None - or _fused_prepare_ucpe_rope_tables is None - ): - raise RuntimeError("fused SANA-WM camera GDN kernels are not available.") - - q_raw, k_raw, v_raw = _camera_qkv(self, x, HW) - batch, tokens, heads, dim = q_raw.shape - frames, height, width = HW - spatial = height * width - if camera_cache is None: - camera_cache = _prepare_camera_projection_cache( - camera_conditions, - HW=HW, - rotary_emb=rotary_emb, - head_dim=dim, - ) - if camera_cache.rope_cam is None: - rope_cos = camera_cache.rope_cos - rope_sin = camera_cache.rope_sin - else: - rope_cos = camera_cache.rope_cos - rope_sin = camera_cache.rope_sin - - q_trans, k_trans, v_trans, inflation_sq = _fused_cam_prep_func( - q_raw.contiguous(), - k_raw.contiguous(), - v_raw.contiguous(), - q_norm_weight=self.q_norm_cam.weight.float().contiguous(), - k_norm_weight=self.k_norm_cam.weight.float().contiguous(), - proj_q=camera_cache.proj_q, - proj_kv=camera_cache.proj_kv, - rope_cos=rope_cos, - rope_sin=rope_sin, - k_scale=_gdn_key_scale(dim, HW), - norm_eps=self.q_norm_cam.eps, - ) - - beta, decay = precomputed_gates - frame_inflation = inflation_sq.reshape(batch, heads, frames, spatial).mean( - dim=-1, - ) - beta = beta / frame_inflation.unsqueeze(-1).clamp_min(1.0) - out = _fused_cam_scan_bidi_chunkwise( - q_trans.float().contiguous(), - k_trans.float().contiguous(), - v_trans.float().contiguous(), - beta.float().contiguous(), - decay.float().contiguous(), - ) - out = out.permute(0, 3, 1, 2).contiguous() - out = _ucpe_transform_apply( - out.to(dtype=x.dtype), - camera_cache.proj, - camera_cache.rope_cam, - inverse_rope=True, - ) - return out.reshape(batch, tokens, heads * dim).to(dtype=x.dtype) - def _forward_softmax_camera( self, x: Tensor, @@ -1158,24 +996,6 @@ def _chunk_index_from_chunk_size( ) -_GDN_PATH_LOGGED = False - - -def _use_fused_gdn(x: Tensor) -> bool: - available = _fused_bigdn_func is not None - disabled_by_env = os.environ.get("SANA_WM_DISABLE_FUSED_GDN", "0") == "1" - use = x.is_cuda and available and not disabled_by_env - global _GDN_PATH_LOGGED - if not _GDN_PATH_LOGGED and x.is_cuda: - _GDN_PATH_LOGGED = True - print( - f"[sana-wm] GDN path={'fused' if use else 'eager'} " - f"(fused_available={available}, disabled_by_env={disabled_by_env})", - flush=True, - ) - return use - - def _prepare_camera_projection_cache( camera_conditions: Tensor, *, @@ -1197,34 +1017,12 @@ def _prepare_camera_projection_cache( proj_q = proj.transpose(-1, -2).contiguous() proj_kv = _invert_se3(proj).contiguous() rope_cam = _slice_rope_for_camera(rotary_emb, head_dim) - if rope_cam is not None and _fused_prepare_ucpe_rope_tables is not None: - rope_cos, rope_sin = _fused_prepare_ucpe_rope_tables( - rope_cam, - tokens, - head_dim // 2, - camera_conditions.device, - ) - else: - rope_cos = torch.ones( - tokens, - head_dim // 2, - device=camera_conditions.device, - dtype=torch.float32, - ) - rope_sin = torch.zeros( - tokens, - head_dim // 2, - device=camera_conditions.device, - dtype=torch.float32, - ) return _CameraProjectionCache( raymats=raymats, proj=proj, proj_q=proj_q, proj_kv=proj_kv, rope_cam=rope_cam, - rope_cos=rope_cos, - rope_sin=rope_sin, ) @@ -1457,79 +1255,47 @@ def _gdn_histories( .float() ) v = v.reshape(batch, frames, spatial, heads, dim).permute(0, 3, 1, 2, 4).float() - eye = torch.eye(dim, dtype=torch.float32, device=k.device).view(1, 1, dim, dim) - m = torch.zeros(batch, heads, dim, dim, dtype=torch.float32, device=k.device) - z = torch.zeros(batch, heads, dim, dtype=torch.float32, device=k.device) - m_hist = torch.empty( - batch, - heads, - frames, - dim, - dim, - dtype=torch.float32, - device=k.device, - ) - z_hist = ( - torch.empty(batch, heads, frames, dim, dtype=torch.float32, device=k.device) - if include_denominator - else None - ) + eye = torch.eye(dim, dtype=torch.float32, device=k.device).view(1, 1, 1, dim, dim) + + # The per-frame GDN transition is linear in the running state, so the + # spatial reductions that build each frame's transition are batched over all + # frames in one einsum, leaving only the (light) sequential state scan as a + # loop — run forward and reverse to form the bidirectional history. + p_kv = torch.einsum("bhfsd,bhfse->bhfde", k_rot, beta[..., None] * k_rot) + a_kv = torch.einsum("bhfsd,bhfse->bhfde", k_rot, beta[..., None] * v) + trans_m = decay[..., None, None] * (eye - p_kv) + m_state = torch.zeros(batch, heads, dim, dim, dtype=torch.float32, device=k.device) + m_forward: list[Tensor] = [] for frame in range(frames): - beta_f = beta[:, :, frame] - decay_f = decay[:, :, frame] - p_kv = torch.einsum( - "bhsd,bhse->bhde", - k_rot[:, :, frame], - beta_f[..., None] * k_rot[:, :, frame], - ) - a_val = torch.einsum( - "bhsd,bhse->bhde", - k_rot[:, :, frame], - beta_f[..., None] * v[:, :, frame], - ) - m = decay_f[..., None, None] * torch.einsum("bhde,bhef->bhdf", eye - p_kv, m) - m = m + a_val - m_hist[:, :, frame] = m - if include_denominator and z_hist is not None: - p_z = torch.einsum( - "bhsd,bhse->bhde", - k[:, :, frame], - beta_f[..., None] * k[:, :, frame], - ) - b_z = (beta_f[..., None] * k[:, :, frame]).sum(dim=2) - z = decay_f[..., None] * torch.einsum("bhde,bhe->bhd", eye - p_z, z) - z = z + b_z - z_hist[:, :, frame] = z - - m = torch.zeros_like(m) - z = torch.zeros_like(z) + m_state = torch.einsum("bhde,bhef->bhdf", trans_m[:, :, frame], m_state) + m_state = m_state + a_kv[:, :, frame] + m_forward.append(m_state) + m_hist = torch.stack(m_forward, dim=2) + m_state = torch.zeros(batch, heads, dim, dim, dtype=torch.float32, device=k.device) for src in range(frames - 1, 0, -1): - dst = src - 1 - beta_f = beta[:, :, src] - decay_f = decay[:, :, src] - p_kv = torch.einsum( - "bhsd,bhse->bhde", - k_rot[:, :, src], - beta_f[..., None] * k_rot[:, :, src], - ) - a_val = torch.einsum( - "bhsd,bhse->bhde", - k_rot[:, :, src], - beta_f[..., None] * v[:, :, src], - ) - m = decay_f[..., None, None] * torch.einsum("bhde,bhef->bhdf", eye - p_kv, m) - m = m + a_val - m_hist[:, :, dst] += m - if include_denominator and z_hist is not None: - p_z = torch.einsum( - "bhsd,bhse->bhde", - k[:, :, src], - beta_f[..., None] * k[:, :, src], - ) - b_z = (beta_f[..., None] * k[:, :, src]).sum(dim=2) - z = decay_f[..., None] * torch.einsum("bhde,bhe->bhd", eye - p_z, z) - z = z + b_z - z_hist[:, :, dst] += z + m_state = torch.einsum("bhde,bhef->bhdf", trans_m[:, :, src], m_state) + m_state = m_state + a_kv[:, :, src] + m_hist[:, :, src - 1] = m_hist[:, :, src - 1] + m_state + + z_hist = None + if include_denominator: + beta_k = beta[..., None] * k + p_z = torch.einsum("bhfsd,bhfse->bhfde", k, beta_k) + b_z = beta_k.sum(dim=3) + trans_z = decay[..., None, None] * (eye - p_z) + z_state = torch.zeros(batch, heads, dim, dtype=torch.float32, device=k.device) + z_forward: list[Tensor] = [] + for frame in range(frames): + z_state = torch.einsum("bhde,bhe->bhd", trans_z[:, :, frame], z_state) + z_state = z_state + b_z[:, :, frame] + z_forward.append(z_state) + z_hist = torch.stack(z_forward, dim=2) + z_state = torch.zeros(batch, heads, dim, dtype=torch.float32, device=k.device) + for src in range(frames - 1, 0, -1): + z_state = torch.einsum("bhde,bhe->bhd", trans_z[:, :, src], z_state) + z_state = z_state + b_z[:, :, src] + z_hist[:, :, src - 1] = z_hist[:, :, src - 1] + z_state + return m_hist, z_hist From 36d2d8aa2cc86ee5596bb2590ddbf513b4f1e610 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Tue, 28 Jul 2026 13:40:37 -0700 Subject: [PATCH 35/64] Optimizations --- integrations/sana/sana_wm/stage1_model.py | 220 +++++++++++++++++++- integrations/sana/sana_wm/transformer.py | 21 +- integrations/sana/tests/test_smoke.py | 69 ++++++ integrations/sana/tests/test_stage1_cuda.py | 79 +++++++ 4 files changed, 382 insertions(+), 7 deletions(-) create mode 100644 integrations/sana/tests/test_stage1_cuda.py diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index f67e76154..388eb656a 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -30,6 +30,13 @@ import torch.nn.functional as F from torch import Tensor +try: + import triton + import triton.language as tl +except ImportError: # pragma: no cover - exercised in minimal CPU environments. + triton = None + tl = None + @dataclass(frozen=True) class SanaWMStage1Spec: @@ -782,10 +789,20 @@ def forward( x = self.x_embedder(x) x = x.permute(0, 2, 3, 4, 1).reshape(batch, frames * height * width, -1) - plucker_emb = None - if chunk_plucker is not None: - plucker_emb = self.plucker_embedder(chunk_plucker) - plucker_emb = plucker_emb.permute(0, 2, 3, 4, 1).reshape_as(x) + plucker_emb = kwargs.get("chunk_plucker_emb") + if isinstance(plucker_emb, Tensor): + if ( + plucker_emb.ndim != 3 + or plucker_emb.shape[1:] != x.shape[1:] + or plucker_emb.shape[0] not in (1, batch) + ): + raise ValueError( + "chunk_plucker_emb must have shape [1|B, T*H*W, hidden], " + f"got {tuple(plucker_emb.shape)} for x={tuple(x.shape)}." + ) + plucker_emb = plucker_emb.to(device=x.device, dtype=x.dtype) + elif chunk_plucker is not None: + plucker_emb = self.prepare_plucker_embedding(chunk_plucker) rotary_emb = _wan_rope_complex( self.spec.head_dim, @@ -811,7 +828,13 @@ def forward( else None ) block_kwargs = { - key: value for key, value in kwargs.items() if key != "camera_conditions" + key: value + for key, value in kwargs.items() + if key + not in { + "camera_conditions", + "chunk_plucker_emb", + } } y = self.y_embedder(y) @@ -850,6 +873,16 @@ def forward( x = self.final_layer(x, timestep_embed, frames=frames) return x.reshape(batch, frames, height, width, -1).permute(0, 4, 1, 2, 3) + def prepare_plucker_embedding(self, chunk_plucker: Tensor) -> Tensor: + """Project static Plucker conditioning into Stage-1 token space.""" + batch = chunk_plucker.shape[0] + plucker_emb = self.plucker_embedder(chunk_plucker) + return plucker_emb.permute(0, 2, 3, 4, 1).reshape( + batch, + -1, + self.spec.hidden_size, + ) + def _modulate(x: Tensor, shift: Tensor, scale: Tensor) -> Tensor: return x * (1 + scale) + shift @@ -1061,7 +1094,155 @@ def _compute_frame_gates( return beta, decay -def _apply_bidirectional_temporal_conv( +if triton is not None and tl is not None: + + @triton.jit + def _bidirectional_temporal_conv_kernel( + x_ptr, + weight_ptr, + out_ptr, + x_stride_b, + x_stride_t, + x_stride_c, + weight_stride_c, + weight_stride_k, + out_stride_b, + out_stride_t, + out_stride_c, + frames: tl.constexpr, + spatial: tl.constexpr, + channels, + kernel_size: tl.constexpr, + block_f: tl.constexpr, + block_s: tl.constexpr, + block_c: tl.constexpr, + ) -> None: + batch_idx = tl.program_id(0) + spatial_offsets = tl.program_id(1) * block_s + tl.arange(0, block_s) + channel_offsets = tl.program_id(2) * block_c + tl.arange(0, block_c) + frame_offsets = tl.arange(0, block_f) + + frame_mask = frame_offsets < frames + spatial_mask = spatial_offsets < spatial + channel_mask = channel_offsets < channels + element_mask = ( + frame_mask[:, None, None] + & spatial_mask[None, :, None] + & channel_mask[None, None, :] + ) + + token_offsets = frame_offsets[:, None, None] * spatial + spatial_offsets[ + None, + :, + None, + ] + x_offsets = ( + batch_idx * x_stride_b + + token_offsets * x_stride_t + + channel_offsets[None, None, :] * x_stride_c + ) + out_offsets = ( + batch_idx * out_stride_b + + token_offsets * out_stride_t + + channel_offsets[None, None, :] * out_stride_c + ) + + center = tl.load( + weight_ptr + + channel_offsets * weight_stride_c + + (kernel_size - 1) * weight_stride_k, + mask=channel_mask, + other=0.0, + ).to(tl.float32) + value = tl.load(x_ptr + x_offsets, mask=element_mask, other=0.0).to( + tl.float32 + ) + acc = value * center[None, None, :] + + for offset in range(1, kernel_size): + coeff = tl.load( + weight_ptr + + channel_offsets * weight_stride_c + + (kernel_size - 1 - offset) * weight_stride_k, + mask=channel_mask, + other=0.0, + ).to(tl.float32) + + prev_frame_offsets = frame_offsets - offset + prev_token_offsets = prev_frame_offsets[:, None, None] * spatial + ( + spatial_offsets[None, :, None] + ) + prev_mask = element_mask & (frame_offsets[:, None, None] >= offset) + prev_offsets = ( + batch_idx * x_stride_b + + prev_token_offsets * x_stride_t + + channel_offsets[None, None, :] * x_stride_c + ) + prev = tl.load(x_ptr + prev_offsets, mask=prev_mask, other=0.0).to( + tl.float32 + ) + + next_frame_offsets = frame_offsets + offset + next_token_offsets = next_frame_offsets[:, None, None] * spatial + ( + spatial_offsets[None, :, None] + ) + next_mask = element_mask & (next_frame_offsets[:, None, None] < frames) + next_offsets = ( + batch_idx * x_stride_b + + next_token_offsets * x_stride_t + + channel_offsets[None, None, :] * x_stride_c + ) + next_value = tl.load(x_ptr + next_offsets, mask=next_mask, other=0.0).to( + tl.float32 + ) + acc += (prev + next_value) * coeff[None, None, :] + + tl.store(out_ptr + out_offsets, acc, mask=element_mask) + +else: + _bidirectional_temporal_conv_kernel = None + + +def _apply_bidirectional_temporal_conv_fast( + x: Tensor, + conv: nn.Conv1d, + HW: tuple[int, int, int], +) -> Tensor: + batch, tokens, channels = x.shape + frames, height, width = HW + spatial = height * width + out = torch.empty_like(x) + assert triton is not None + assert _bidirectional_temporal_conv_kernel is not None + block_f = triton.next_power_of_2(frames) + block_s = 4 + block_c = 128 + grid = (batch, triton.cdiv(spatial, block_s), triton.cdiv(channels, block_c)) + weight = conv.weight[:, 0, :] + _bidirectional_temporal_conv_kernel[grid]( + x, + weight, + out, + x.stride(0), + x.stride(1), + x.stride(2), + weight.stride(0), + weight.stride(1), + out.stride(0), + out.stride(1), + out.stride(2), + frames, + spatial, + channels, + int(conv.kernel_size[0]), + block_f, + block_s, + block_c, + ) + return out + + +def _apply_bidirectional_temporal_conv_eager( x: Tensor, conv: nn.Conv1d, HW: tuple[int, int, int], @@ -1088,6 +1269,33 @@ def _apply_bidirectional_temporal_conv( ) +def _apply_bidirectional_temporal_conv( + x: Tensor, + conv: nn.Conv1d, + HW: tuple[int, int, int], +) -> Tensor: + if ( + x.is_cuda + and conv.weight.is_cuda + and conv.weight.dtype == x.dtype + and conv.bias is None + and conv.in_channels == x.shape[-1] + and conv.out_channels == x.shape[-1] + and conv.groups == x.shape[-1] + and conv.stride == (1,) + and conv.padding == (0,) + and conv.dilation == (1,) + and not torch.is_grad_enabled() + and triton is not None + and _bidirectional_temporal_conv_kernel is not None + ): + batch, tokens, _channels = x.shape + frames, height, width = HW + if tokens == frames * height * width: + return _apply_bidirectional_temporal_conv_fast(x, conv, HW) + return _apply_bidirectional_temporal_conv_eager(x, conv, HW) + + def _causal_depthwise_conv(x: Tensor, conv: nn.Conv1d) -> Tensor: kernel = int(conv.kernel_size[0]) padded = F.pad(x, (kernel - 1, 0)) diff --git a/integrations/sana/sana_wm/transformer.py b/integrations/sana/sana_wm/transformer.py index dd744772c..fb4c61092 100644 --- a/integrations/sana/sana_wm/transformer.py +++ b/integrations/sana/sana_wm/transformer.py @@ -314,6 +314,7 @@ def _predict_conditioned( timestep: Tensor, conditioning: SanaWMStage1Conditioning, ) -> Tensor: + self._prepare_static_model_kwargs(conditioning) model_timestep = _conditioned_frame_timestep( noisy_latent=noisy_latent, timestep=timestep, @@ -340,6 +341,24 @@ def _predict_conditioned( flow = _cfg_guidance(noise_pred, conditioning.cfg_scale) return _zero_conditioned_frame_flow(flow, conditioning) + def _prepare_static_model_kwargs( + self, + conditioning: SanaWMStage1Conditioning, + ) -> None: + """Cache model projections for conditioning that is constant per rollout.""" + model_kwargs = conditioning.model_kwargs + self._ensure_model() + chunk_plucker = model_kwargs.get("chunk_plucker") + prepare_plucker = getattr(self.model, "prepare_plucker_embedding", None) + if ( + "chunk_plucker_emb" not in model_kwargs + and isinstance(chunk_plucker, Tensor) + and callable(prepare_plucker) + ): + with torch.inference_mode(): + model_kwargs["chunk_plucker_emb"] = prepare_plucker(chunk_plucker) + del model_kwargs["chunk_plucker"] + def _predict_with_prompt( self, noisy_latent: Tensor, @@ -469,7 +488,7 @@ def _condition_model_kwargs(model_kwargs: dict[str, object]) -> dict[str, object return { key: value for key, value in model_kwargs.items() - if key != "negative_mask" + if key != "negative_mask" and not key.startswith("_") } diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index 38c08f97d..7b8f23398 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -465,6 +465,75 @@ def forward( assert call["data_info"] == {"condition_frame_info": {0: 0.0}} +def test_transformer_predict_flow_caches_static_plucker_embedding() -> None: + """Avoid re-projecting rollout-constant Plucker conditioning each step.""" + + class DummyModel(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.prepare_calls = 0 + self.forward_calls: list[dict[str, torch.Tensor]] = [] + + def prepare_plucker_embedding( + self, + chunk_plucker: torch.Tensor, + ) -> torch.Tensor: + self.prepare_calls += 1 + return chunk_plucker.flatten(2).transpose(1, 2) + + def forward( + self, + noisy_latent: torch.Tensor, + timestep: torch.Tensor, + prompt_embeds: torch.Tensor, + *, + chunk_plucker_emb: torch.Tensor, + **_kwargs: object, + ) -> torch.Tensor: + del timestep, prompt_embeds + assert "chunk_plucker" not in _kwargs + self.forward_calls.append({"chunk_plucker_emb": chunk_plucker_emb}) + return torch.ones_like(noisy_latent) + + transformer = SanaWMTransformerConfig().setup() + dummy_model = DummyModel() + transformer.model = dummy_model + transformer._model_built = True + chunk_plucker = torch.ones((1, 2, 2, 1, 1)) + conditioning = SanaWMStage1Conditioning( + condition=torch.ones((1, 1, 1)), + uncondition=None, + model_kwargs={ + "chunk_plucker": chunk_plucker, + "data_info": {"condition_frame_info": {0: 0.0}}, + }, + first_latent=torch.empty((1, 1, 1, 1, 1)), + latent_shape=(1, 1, 2, 1, 1), + cfg_scale=1.0, + flow_shift=1.0, + steps=1, + seed=0, + ) + cache = SanaWMTransformerCache() + + for _ in range(2): + transformer.predict_flow( + noisy_latent=torch.zeros((1, 1, 2, 1, 1)), + timestep=torch.full((1, 1, 2), 1000.0), + cache=cache, + input=conditioning, + ) + + assert dummy_model.prepare_calls == 1 + assert "chunk_plucker" not in conditioning.model_kwargs + assert len(dummy_model.forward_calls) == 2 + cached = conditioning.model_kwargs["chunk_plucker_emb"] + assert isinstance(cached, torch.Tensor) + assert all( + call["chunk_plucker_emb"] is cached for call in dummy_model.forward_calls + ) + + def test_inference_config_loads_yaml( tmp_path: Path, ) -> None: diff --git a/integrations/sana/tests/test_stage1_cuda.py b/integrations/sana/tests/test_stage1_cuda.py new file mode 100644 index 000000000..a72c40ada --- /dev/null +++ b/integrations/sana/tests/test_stage1_cuda.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CUDA tests for SANA-WM Stage-1 kernels.""" + +from __future__ import annotations + +import pytest +import torch + +import sana_wm.stage1_model as stage1 + +pytestmark = pytest.mark.ci_gpu + + +def _require_stage1_cuda() -> None: + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for Stage-1 kernel tests") + if stage1.triton is None: + pytest.skip("Triton is required for the Stage-1 temporal conv CUDA kernel") + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +def test_bidirectional_temporal_conv_fast_path_matches_eager( + dtype: torch.dtype, +) -> None: + """Compare the CUDA temporal-conv fast path against the eager conv path.""" + _require_stage1_cuda() + torch.manual_seed(1234) + batch, frames, height, width, channels = 2, 5, 3, 4, 16 + x = torch.randn( + batch, + frames * height * width, + channels, + device="cuda", + dtype=dtype, + ) + conv = torch.nn.Conv1d( + channels, + channels, + kernel_size=4, + groups=channels, + bias=False, + device="cuda", + dtype=dtype, + ) + + reference = stage1._apply_bidirectional_temporal_conv_eager( + x, + conv, + (frames, height, width), + ) + with torch.inference_mode(): + actual = stage1._apply_bidirectional_temporal_conv( + x, + conv, + (frames, height, width), + ) + + assert actual.shape == x.shape + assert actual.dtype == dtype + torch.testing.assert_close( + actual.float(), + reference.float(), + rtol=2e-2, + atol=8e-2 if dtype is torch.bfloat16 else 1e-5, + ) From 4c1d857e1ac6e4f3e09b81d5fb3b3066b04df3d6 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Tue, 28 Jul 2026 18:11:59 -0700 Subject: [PATCH 36/64] Optimize SANA-WM Stage 1 kernels Add guarded clean-room Triton kernels for UCPE transforms, UCPE norm reductions, GLU activation multiply, and RMSNorm/ReLU head prep. Precompute static RoPE and camera projection tensors for singleton camera conditioning so diffusion steps avoid rebuilding those tensors. Correct Stage-1 main attention routing to use the GDN path consistently while preserving the hybrid camera branch behavior. Add CPU and CUDA coverage for the cache behavior, routing, and fused kernel fallbacks. Final BF16 benchmark with refiner enabled, one warmup and three measured runs on GB202: FlashDreams generation median 92.39s versus upstream 107.55s, with Stage-1 DiT median 69.13s versus upstream 75.30s. Signed-off-by: Aidan Foster --- integrations/sana/sana_wm/stage1_model.py | 842 ++++++++++++++++++-- integrations/sana/sana_wm/transformer.py | 32 +- integrations/sana/tests/test_smoke.py | 187 ++++- integrations/sana/tests/test_stage1_cuda.py | 201 +++++ 4 files changed, 1179 insertions(+), 83 deletions(-) diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index 388eb656a..2f93cecce 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -24,6 +24,7 @@ from collections.abc import Callable from dataclasses import dataclass import math +import os import torch import torch.nn as nn @@ -79,6 +80,15 @@ def block_uses_gdn(self, index: int) -> bool: """Architecture spec for the public SANA-WM bidirectional Stage-1 checkpoint.""" +def _env_flag_enabled(name: str) -> bool: + return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"} + + +_DISABLE_UCPE_FAST = _env_flag_enabled("SANA_WM_STAGE1_DISABLE_UCPE_FAST") +_DISABLE_GLU_FAST = _env_flag_enabled("SANA_WM_STAGE1_DISABLE_GLU_FAST") +_DISABLE_RMS_RELU_FAST = _env_flag_enabled("SANA_WM_STAGE1_DISABLE_RMS_RELU_FAST") + + @dataclass(frozen=True) class _CameraProjectionCache: raymats: Tensor @@ -263,8 +273,9 @@ def forward(self, x: Tensor, *, frames: int, height: int, width: int) -> Tensor: x_2d = F.silu(self.inverted_conv(x_2d), inplace=True) x_2d = self.depth_conv(x_2d) value, gate = x_2d.chunk(2, dim=1) - gate = F.silu(gate, inplace=not torch.is_grad_enabled()) - x_2d = self.point_conv(value * gate) + x_2d = self.point_conv( + _silu_multiply(value, gate, inplace=not torch.is_grad_enabled()) + ) x_time = x_2d.view(batch, frames, channels, height * width).permute(0, 2, 1, 3) x_time = x_time + self.t_conv(x_time) @@ -392,25 +403,13 @@ def forward( f"channels={channels} != heads*dim={self.heads * self.dim}" ) - precomputed_gates = ( - self._compute_frame_gates(x, HW) if self.use_gdn_convs else None + precomputed_gates = self._compute_frame_gates(x, HW) + main_raw = self._forward_gdn_main( + x, + HW=HW, + rotary_emb=rotary_emb, + precomputed_gates=precomputed_gates, ) - if self.use_gdn_convs: - main_raw = self._forward_gdn_main( - x, - HW=HW, - rotary_emb=rotary_emb, - precomputed_gates=precomputed_gates, - ) - else: - main_raw = self._forward_softmax_main( - x, - HW=HW, - rotary_emb=rotary_emb, - chunk_size=kwargs.get("chunk_size"), - chunk_split_strategy=str(kwargs.get("chunk_split_strategy", "uniform")), - chunk_index=kwargs.get("chunk_index"), - ) cam_contrib: Tensor | int = 0 if camera_conditions is not None: @@ -463,21 +462,18 @@ def _forward_gdn_main( k_conv = _apply_bidirectional_temporal_conv(k_raw, self.conv_k, HW) qkv[:, :, 1] = k_conv.reshape(batch, tokens, self.heads, self.dim) - q = self.q_norm(qkv[:, :, 0].reshape(batch, tokens, channels)).reshape( - batch, - tokens, - self.heads, - self.dim, + q = _rmsnorm_relu_heads( + qkv[:, :, 0], + self.q_norm.weight, + self.q_norm.eps, ) - k = self.k_norm(qkv[:, :, 1].reshape(batch, tokens, channels)).reshape( - batch, - tokens, - self.heads, - self.dim, + k = _rmsnorm_relu_heads( + qkv[:, :, 1], + self.k_norm.weight, + self.k_norm.eps, + scale=_gdn_key_scale(self.dim, HW), ) v = qkv[:, :, 2] - q = F.relu(q.float()) - k = F.relu(k.float()) * _gdn_key_scale(self.dim, HW) v = v.float() q_rot = _apply_complex_rope(q, rotary_emb) k_rot = _apply_complex_rope(k, rotary_emb) @@ -804,28 +800,44 @@ def forward( elif chunk_plucker is not None: plucker_emb = self.prepare_plucker_embedding(chunk_plucker) - rotary_emb = _wan_rope_complex( - self.spec.head_dim, - frames, - height, - width, - x.device, - ) - camera_conditions = kwargs.get("camera_conditions") + rotary_emb = kwargs.get("rotary_emb") + if isinstance(rotary_emb, Tensor): + rotary_emb = rotary_emb.to(device=x.device) + else: + rotary_emb = _wan_rope_complex( + self.spec.head_dim, + frames, + height, + width, + x.device, + ) + raw_camera_conditions = kwargs.get("camera_conditions") camera_conditions = ( - camera_conditions.to(device=x.device, dtype=x.dtype) - if isinstance(camera_conditions, Tensor) + raw_camera_conditions.to(device=x.device, dtype=x.dtype) + if isinstance(raw_camera_conditions, Tensor) else None ) + camera_cache_value = kwargs.get("camera_cache") camera_cache = ( - _prepare_camera_projection_cache( - camera_conditions, - HW=(frames, height, width), - rotary_emb=rotary_emb, - head_dim=self.spec.head_dim, + _camera_projection_cache_to( + camera_cache_value, + device=x.device, + dtype=x.dtype, ) - if camera_conditions is not None - else None + if isinstance(camera_cache_value, _CameraProjectionCache) + else ( + _prepare_camera_projection_cache( + camera_conditions, + HW=(frames, height, width), + rotary_emb=rotary_emb, + head_dim=self.spec.head_dim, + ) + if camera_conditions is not None + else None + ) + ) + camera_signal = ( + camera_cache.raymats if camera_cache is not None else camera_conditions ) block_kwargs = { key: value @@ -834,6 +846,8 @@ def forward( not in { "camera_conditions", "chunk_plucker_emb", + "camera_cache", + "rotary_emb", } } @@ -863,7 +877,7 @@ def forward( plucker_emb=plucker_emb, HW=(frames, height, width), rotary_emb=rotary_emb, - camera_conditions=camera_conditions, + camera_conditions=camera_signal, camera_cache=camera_cache, chunk_size=self.spec.chunk_size, chunk_split_strategy=self.spec.chunk_split_strategy, @@ -883,6 +897,32 @@ def prepare_plucker_embedding(self, chunk_plucker: Tensor) -> Tensor: self.spec.hidden_size, ) + def prepare_camera_projection_cache( + self, + camera_conditions: Tensor, + *, + frames: int, + height: int, + width: int, + ) -> tuple[Tensor, _CameraProjectionCache]: + """Precompute static RoPE and camera projection tensors for a rollout.""" + param = next(self.parameters()) + camera_conditions = camera_conditions.to(device=param.device, dtype=param.dtype) + rotary_emb = _wan_rope_complex( + self.spec.head_dim, + frames, + height, + width, + param.device, + ) + camera_cache = _prepare_camera_projection_cache( + camera_conditions, + HW=(frames, height, width), + rotary_emb=rotary_emb, + head_dim=self.spec.head_dim, + ) + return rotary_emb, camera_cache + def _modulate(x: Tensor, shift: Tensor, scale: Tensor) -> Tensor: return x * (1 + scale) + shift @@ -1059,6 +1099,35 @@ def _prepare_camera_projection_cache( ) +def _camera_projection_cache_to( + cache: _CameraProjectionCache, + *, + device: torch.device, + dtype: torch.dtype, +) -> _CameraProjectionCache: + if ( + cache.raymats.device == device + and cache.proj.device == device + and cache.proj_q.device == device + and cache.proj_kv.device == device + and cache.raymats.dtype == dtype + and cache.proj.dtype == dtype + and cache.proj_q.dtype == dtype + and cache.proj_kv.dtype == dtype + and (cache.rope_cam is None or cache.rope_cam.device == device) + ): + return cache + return _CameraProjectionCache( + raymats=cache.raymats.to(device=device, dtype=dtype), + proj=cache.proj.to(device=device, dtype=dtype), + proj_q=cache.proj_q.to(device=device, dtype=dtype), + proj_kv=cache.proj_kv.to(device=device, dtype=dtype), + rope_cam=( + cache.rope_cam.to(device=device) if cache.rope_cam is not None else None + ), + ) + + @torch.compile def _apply_output_gate( out: Tensor, @@ -1199,8 +1268,359 @@ def _bidirectional_temporal_conv_kernel( tl.store(out_ptr + out_offsets, acc, mask=element_mask) + @triton.jit + def _ucpe_first_half_kernel( + x_ptr, + matrix_ptr, + out_ptr, + x_stride_b, + x_stride_n, + x_stride_h, + x_stride_d, + matrix_stride_b, + matrix_stride_n, + matrix_stride_i, + matrix_stride_j, + out_stride_b, + out_stride_n, + out_stride_h, + out_stride_d, + tokens: tl.constexpr, + groups4: tl.constexpr, + matrix_batch: tl.constexpr, + block_n: tl.constexpr, + ) -> None: + batch_idx = tl.program_id(0) + head_idx = tl.program_id(1) + packed_idx = tl.program_id(2) + group_idx = packed_idx % groups4 + token_block = packed_idx // groups4 + token_offsets = token_block * block_n + tl.arange(0, block_n) + token_mask = token_offsets < tokens + + matrix_batch_idx = 0 if matrix_batch == 1 else batch_idx + col0 = group_idx * 4 + col1 = col0 + 1 + col2 = col0 + 2 + col3 = col0 + 3 + + x_base = ( + batch_idx * x_stride_b + + token_offsets * x_stride_n + + head_idx * x_stride_h + ) + x0 = tl.load( + x_ptr + x_base + col0 * x_stride_d, + mask=token_mask, + other=0.0, + ).to( + tl.float32 + ) + x1 = tl.load( + x_ptr + x_base + col1 * x_stride_d, + mask=token_mask, + other=0.0, + ).to( + tl.float32 + ) + x2 = tl.load( + x_ptr + x_base + col2 * x_stride_d, + mask=token_mask, + other=0.0, + ).to( + tl.float32 + ) + x3 = tl.load( + x_ptr + x_base + col3 * x_stride_d, + mask=token_mask, + other=0.0, + ).to( + tl.float32 + ) + + matrix_base = ( + matrix_batch_idx * matrix_stride_b + token_offsets * matrix_stride_n + ) + out_base = ( + batch_idx * out_stride_b + + token_offsets * out_stride_n + + head_idx * out_stride_h + ) + for row in range(4): + row_base = matrix_base + row * matrix_stride_i + m0 = tl.load( + matrix_ptr + row_base + 0 * matrix_stride_j, + mask=token_mask, + other=0.0, + ).to(tl.float32) + m1 = tl.load( + matrix_ptr + row_base + 1 * matrix_stride_j, + mask=token_mask, + other=0.0, + ).to(tl.float32) + m2 = tl.load( + matrix_ptr + row_base + 2 * matrix_stride_j, + mask=token_mask, + other=0.0, + ).to(tl.float32) + m3 = tl.load( + matrix_ptr + row_base + 3 * matrix_stride_j, + mask=token_mask, + other=0.0, + ).to(tl.float32) + value = (x0 * m0 + x1 * m1) + (x2 * m2 + x3 * m3) + tl.store( + out_ptr + out_base + (col0 + row) * out_stride_d, + value, + mask=token_mask, + ) + + @triton.jit + def _ucpe_rope_second_half_kernel( + x_ptr, + rope_ptr, + out_ptr, + x_stride_b, + x_stride_n, + x_stride_h, + x_stride_d, + rope_stride_n, + rope_stride_p, + rope_stride_ri, + out_stride_b, + out_stride_n, + out_stride_h, + out_stride_d, + tokens: tl.constexpr, + half: tl.constexpr, + rope_pairs: tl.constexpr, + inverse_rope: tl.constexpr, + block_n: tl.constexpr, + ) -> None: + batch_idx = tl.program_id(0) + head_idx = tl.program_id(1) + packed_idx = tl.program_id(2) + pair_idx = packed_idx % rope_pairs + token_block = packed_idx // rope_pairs + token_offsets = token_block * block_n + tl.arange(0, block_n) + token_mask = token_offsets < tokens + + real_d = half + pair_idx * 2 + imag_d = real_d + 1 + x_base = ( + batch_idx * x_stride_b + + token_offsets * x_stride_n + + head_idx * x_stride_h + ) + real = tl.load( + x_ptr + x_base + real_d * x_stride_d, + mask=token_mask, + other=0.0, + ).to( + tl.float32 + ) + imag = tl.load( + x_ptr + x_base + imag_d * x_stride_d, + mask=token_mask, + other=0.0, + ).to( + tl.float32 + ) + rope_base = token_offsets * rope_stride_n + pair_idx * rope_stride_p + rope_real = tl.load( + rope_ptr + rope_base + 0 * rope_stride_ri, + mask=token_mask, + other=0.0, + ).to(tl.float32) + rope_imag = tl.load( + rope_ptr + rope_base + 1 * rope_stride_ri, + mask=token_mask, + other=0.0, + ).to(tl.float32) + if inverse_rope: + rope_imag = -rope_imag + + out_base = ( + batch_idx * out_stride_b + + token_offsets * out_stride_n + + head_idx * out_stride_h + ) + tl.store( + out_ptr + out_base + real_d * out_stride_d, + real * rope_real - imag * rope_imag, + mask=token_mask, + ) + tl.store( + out_ptr + out_base + imag_d * out_stride_d, + real * rope_imag + imag * rope_real, + mask=token_mask, + ) + + @triton.jit + def _ucpe_norms_kernel( + x_ptr, + out_ptr, + pre_ptr, + post_ptr, + x_stride_b, + x_stride_n, + x_stride_h, + x_stride_d, + out_stride_b, + out_stride_n, + out_stride_h, + out_stride_d, + pre_stride_b, + pre_stride_h, + pre_stride_n, + post_stride_b, + post_stride_h, + post_stride_n, + tokens: tl.constexpr, + dim: tl.constexpr, + block_n: tl.constexpr, + block_d: tl.constexpr, + ) -> None: + batch_idx = tl.program_id(0) + head_idx = tl.program_id(1) + token_block = tl.program_id(2) + token_offsets = token_block * block_n + tl.arange(0, block_n) + dim_offsets = tl.arange(0, block_d) + mask = (token_offsets[:, None] < tokens) & (dim_offsets[None, :] < dim) + + x_offsets = ( + batch_idx * x_stride_b + + token_offsets[:, None] * x_stride_n + + head_idx * x_stride_h + + dim_offsets[None, :] * x_stride_d + ) + out_offsets = ( + batch_idx * out_stride_b + + token_offsets[:, None] * out_stride_n + + head_idx * out_stride_h + + dim_offsets[None, :] * out_stride_d + ) + x_values = tl.load(x_ptr + x_offsets, mask=mask, other=0.0).to(tl.float32) + out_values = tl.load(out_ptr + out_offsets, mask=mask, other=0.0).to( + tl.float32 + ) + pre = tl.sum(x_values * x_values, axis=1) + post = tl.sum(out_values * out_values, axis=1) + token_mask = token_offsets < tokens + pre_offsets = ( + batch_idx * pre_stride_b + + head_idx * pre_stride_h + + token_offsets * pre_stride_n + ) + post_offsets = ( + batch_idx * post_stride_b + + head_idx * post_stride_h + + token_offsets * post_stride_n + ) + tl.store(pre_ptr + pre_offsets, pre, mask=token_mask) + tl.store(post_ptr + post_offsets, post, mask=token_mask) + + @triton.jit + def _silu_multiply_kernel( + value_ptr, + gate_ptr, + out_ptr, + value_stride_n, + value_stride_c, + value_stride_h, + value_stride_w, + gate_stride_n, + gate_stride_c, + gate_stride_h, + gate_stride_w, + channels: tl.constexpr, + height: tl.constexpr, + width: tl.constexpr, + total: tl.constexpr, + block: tl.constexpr, + ) -> None: + offsets = tl.program_id(0) * block + tl.arange(0, block) + mask = offsets < total + channel_offsets = offsets % channels + spatial_offsets = offsets // channels + width_offsets = spatial_offsets % width + height_offsets = (spatial_offsets // width) % height + batch_offsets = spatial_offsets // (height * width) + + value_offsets = ( + batch_offsets * value_stride_n + + channel_offsets * value_stride_c + + height_offsets * value_stride_h + + width_offsets * value_stride_w + ) + gate_offsets = ( + batch_offsets * gate_stride_n + + channel_offsets * gate_stride_c + + height_offsets * gate_stride_h + + width_offsets * gate_stride_w + ) + value = tl.load(value_ptr + value_offsets, mask=mask, other=0.0).to(tl.float32) + gate = tl.load(gate_ptr + gate_offsets, mask=mask, other=0.0).to(tl.float32) + silu_gate = gate / (1.0 + tl.exp(-gate)) + tl.store(out_ptr + offsets, value * silu_gate, mask=mask) + + @triton.jit + def _rmsnorm_relu_heads_kernel( + x_ptr, + weight_ptr, + out_ptr, + x_stride_b, + x_stride_n, + x_stride_h, + x_stride_d, + out_stride_b, + out_stride_n, + out_stride_h, + out_stride_d, + tokens: tl.constexpr, + heads: tl.constexpr, + dim: tl.constexpr, + channels: tl.constexpr, + eps: tl.constexpr, + scale: tl.constexpr, + block_c: tl.constexpr, + ) -> None: + row = tl.program_id(0) + batch_idx = row // tokens + token_idx = row - batch_idx * tokens + channel_offsets = tl.arange(0, block_c) + mask = channel_offsets < channels + head_offsets = channel_offsets // dim + dim_offsets = channel_offsets - head_offsets * dim + + x_offsets = ( + batch_idx * x_stride_b + + token_idx * x_stride_n + + head_offsets * x_stride_h + + dim_offsets * x_stride_d + ) + values = tl.load(x_ptr + x_offsets, mask=mask, other=0.0).to(tl.float32) + mean_sq = tl.sum(values * values, axis=0) / channels + inv_rms = tl.rsqrt(mean_sq + eps) + weights = tl.load(weight_ptr + channel_offsets, mask=mask, other=0.0).to( + tl.float32 + ) + out_values = tl.maximum(values * inv_rms * weights * scale, 0.0) + out_offsets = ( + batch_idx * out_stride_b + + token_idx * out_stride_n + + head_offsets * out_stride_h + + dim_offsets * out_stride_d + ) + tl.store(out_ptr + out_offsets, out_values, mask=mask) + else: _bidirectional_temporal_conv_kernel = None + _ucpe_first_half_kernel = None + _ucpe_rope_second_half_kernel = None + _ucpe_norms_kernel = None + _silu_multiply_kernel = None + _rmsnorm_relu_heads_kernel = None def _apply_bidirectional_temporal_conv_fast( @@ -1553,13 +1973,13 @@ def _prepare_ucpe_qkv( norm_eps: float, ) -> tuple[Tensor, Tensor, Tensor, Tensor, Callable[[Tensor], Tensor]]: batch, tokens, heads, dim = q_raw.shape - q_inv = _inv_rms(q_raw, norm_eps) - k_inv = _inv_rms(k_raw, norm_eps) - q_weight = q_norm_weight.float().view(heads, dim) - k_weight = k_norm_weight.float().view(heads, dim) - q_norm = F.relu(q_raw.float() * q_inv[:, :, None, None] * q_weight[None, None]) - k_norm = F.relu(k_raw.float() * k_inv[:, :, None, None] * k_weight[None, None]) - k_norm = k_norm * _gdn_key_scale(dim, HW) + q_norm = _rmsnorm_relu_heads(q_raw, q_norm_weight, norm_eps) + k_norm = _rmsnorm_relu_heads( + k_raw, + k_norm_weight, + norm_eps, + scale=_gdn_key_scale(dim, HW), + ) v_float = v_raw.float() if camera_cache is None: camera_cache = _prepare_camera_projection_cache( @@ -1664,6 +2084,75 @@ def _inv_rms(x: Tensor, eps: float) -> Tensor: return torch.rsqrt(x.float().pow(2).sum(dim=(-1, -2)) / channels + eps) +def _rmsnorm_relu_heads( + x: Tensor, + weight: Tensor, + eps: float, + *, + scale: float = 1.0, +) -> Tensor: + fast = _rmsnorm_relu_heads_fast(x, weight, eps, scale=scale) + if fast is not None: + return fast + batch, tokens, heads, dim = x.shape + inv_rms = _inv_rms(x, eps) + norm_weight = weight.float().view(heads, dim) + out = F.relu(x.float() * inv_rms[:, :, None, None] * norm_weight[None, None]) + if scale != 1.0: + out = out * scale + return out + + +def _rmsnorm_relu_heads_fast( + x: Tensor, + weight: Tensor, + eps: float, + *, + scale: float, +) -> Tensor | None: + batch, tokens, heads, dim = x.shape + channels = heads * dim + block_c = 1 << (channels - 1).bit_length() + if ( + _DISABLE_RMS_RELU_FAST + or not x.is_cuda + or not weight.is_cuda + or torch.is_grad_enabled() + or triton is None + or _rmsnorm_relu_heads_kernel is None + or weight.numel() != channels + or block_c > 4096 + ): + return None + out = torch.empty( + (batch, tokens, heads, dim), + device=x.device, + dtype=torch.float32, + ) + grid = (batch * tokens,) + _rmsnorm_relu_heads_kernel[grid]( + x, + weight, + out, + x.stride(0), + x.stride(1), + x.stride(2), + x.stride(3), + out.stride(0), + out.stride(1), + out.stride(2), + out.stride(3), + tokens, + heads, + dim, + channels, + float(eps), + float(scale), + block_c, + ) + return out + + def _ucpe_transform( x: Tensor, matrix: Tensor, @@ -1671,6 +2160,50 @@ def _ucpe_transform( *, inverse_rope: bool, ) -> tuple[Tensor, Tensor, Tensor]: + out = _ucpe_transform_apply(x, matrix, rotary_emb, inverse_rope=inverse_rope) + fast_norms = _ucpe_transform_norms_fast(x, out) + if fast_norms is not None: + pre_sq, post_sq = fast_norms + return out, pre_sq, post_sq + pre_sq = _ucpe_transform_norms_eager(x) + post_sq = _ucpe_transform_norms_eager(out) + return out, pre_sq, post_sq + + +def _ucpe_transform_norms_eager(x: Tensor) -> Tensor: + return x.float().pow(2).sum(dim=-1).transpose(1, 2).contiguous() + + +def _ucpe_transform_apply( + x: Tensor, + matrix: Tensor, + rotary_emb: Tensor | None, + *, + inverse_rope: bool, +) -> Tensor: + fast = _ucpe_transform_apply_fast( + x, + matrix, + rotary_emb, + inverse_rope=inverse_rope, + ) + if fast is not None: + return fast + return _ucpe_transform_apply_eager( + x, + matrix, + rotary_emb, + inverse_rope=inverse_rope, + ) + + +def _ucpe_transform_apply_eager( + x: Tensor, + matrix: Tensor, + rotary_emb: Tensor | None, + *, + inverse_rope: bool, +) -> Tensor: batch, tokens, heads, dim = x.shape half = dim // 2 if half % 4 != 0: @@ -1684,31 +2217,198 @@ def _ucpe_transform( else: second_out = _apply_complex_rope(second, rotary_emb) out = torch.cat([first_out, second_out], dim=-1) - pre_sq = x.float().pow(2).sum(dim=-1).transpose(1, 2).contiguous() - post_sq = out.float().pow(2).sum(dim=-1).transpose(1, 2).contiguous() - return out, pre_sq, post_sq + return out -def _ucpe_transform_apply( +def _ucpe_transform_apply_fast( x: Tensor, matrix: Tensor, rotary_emb: Tensor | None, *, inverse_rope: bool, -) -> Tensor: +) -> Tensor | None: batch, tokens, heads, dim = x.shape half = dim // 2 if half % 4 != 0: raise ValueError(f"UCPE requires head_dim/2 divisible by 4, got {half}.") - first = x[..., :half].reshape(batch, tokens, heads, half // 4, 4) - first_out = torch.einsum("bnij,bnhgj->bnhgi", matrix.float(), first.float()) - first_out = first_out.reshape(batch, tokens, heads, half) - second = x[..., half:] - if inverse_rope and rotary_emb is not None: - second_out = _apply_complex_rope(second, rotary_emb.conj()) - else: - second_out = _apply_complex_rope(second, rotary_emb) - return torch.cat([first_out, second_out], dim=-1) + if ( + _DISABLE_UCPE_FAST + or not x.is_cuda + or not matrix.is_cuda + or rotary_emb is None + or not rotary_emb.is_cuda + or torch.is_grad_enabled() + or triton is None + or _ucpe_first_half_kernel is None + or _ucpe_rope_second_half_kernel is None + or matrix.shape[-2:] != (4, 4) + or matrix.shape[1] != tokens + or matrix.shape[0] not in (1, batch) + ): + return None + rope = torch.view_as_real(rotary_emb.squeeze(0).squeeze(0)) + if rope.shape != (tokens, half // 2, 2): + return None + + out = torch.empty_like(x, dtype=torch.float32) + groups4 = half // 4 + rope_pairs = half // 2 + block_n = 128 + first_grid = ( + batch, + heads, + groups4 * triton.cdiv(tokens, block_n), + ) + _ucpe_first_half_kernel[first_grid]( + x, + matrix, + out, + x.stride(0), + x.stride(1), + x.stride(2), + x.stride(3), + matrix.stride(0), + matrix.stride(1), + matrix.stride(2), + matrix.stride(3), + out.stride(0), + out.stride(1), + out.stride(2), + out.stride(3), + tokens, + groups4, + matrix.shape[0], + block_n, + ) + rope_grid = ( + batch, + heads, + rope_pairs * triton.cdiv(tokens, block_n), + ) + _ucpe_rope_second_half_kernel[rope_grid]( + x, + rope, + out, + x.stride(0), + x.stride(1), + x.stride(2), + x.stride(3), + rope.stride(0), + rope.stride(1), + rope.stride(2), + out.stride(0), + out.stride(1), + out.stride(2), + out.stride(3), + tokens, + half, + rope_pairs, + inverse_rope, + block_n, + ) + return out + + +def _ucpe_transform_norms_fast(x: Tensor, out: Tensor) -> tuple[Tensor, Tensor] | None: + batch, tokens, heads, dim = x.shape + block_d = 1 << (dim - 1).bit_length() + if ( + _DISABLE_UCPE_FAST + or not x.is_cuda + or not out.is_cuda + or torch.is_grad_enabled() + or triton is None + or _ucpe_norms_kernel is None + or out.shape != x.shape + or block_d > 256 + ): + return None + + pre_sq = torch.empty((batch, heads, tokens), device=x.device, dtype=torch.float32) + post_sq = torch.empty_like(pre_sq) + block_n = 16 + grid = ( + batch, + heads, + triton.cdiv(tokens, block_n), + ) + _ucpe_norms_kernel[grid]( + x, + out, + pre_sq, + post_sq, + x.stride(0), + x.stride(1), + x.stride(2), + x.stride(3), + out.stride(0), + out.stride(1), + out.stride(2), + out.stride(3), + pre_sq.stride(0), + pre_sq.stride(1), + pre_sq.stride(2), + post_sq.stride(0), + post_sq.stride(1), + post_sq.stride(2), + tokens, + dim, + block_n, + block_d, + ) + return pre_sq, post_sq + + +def _silu_multiply(value: Tensor, gate: Tensor, *, inplace: bool) -> Tensor: + fast = _silu_multiply_fast(value, gate) + if fast is not None: + return fast + return value * F.silu(gate, inplace=inplace) + + +def _silu_multiply_fast(value: Tensor, gate: Tensor) -> Tensor | None: + if ( + _DISABLE_GLU_FAST + or value.shape != gate.shape + or value.dim() != 4 + or not value.is_cuda + or not gate.is_cuda + or torch.is_grad_enabled() + or triton is None + or _silu_multiply_kernel is None + ): + return None + batch, channels, height, width = value.shape + total = batch * channels * height * width + if total <= 0: + return torch.empty_like(value, memory_format=torch.channels_last) + out = torch.empty( + (batch, channels, height, width), + device=value.device, + dtype=value.dtype, + memory_format=torch.channels_last, + ) + block = 128 + grid = (triton.cdiv(total, block),) + _silu_multiply_kernel[grid]( + value, + gate, + out, + value.stride(0), + value.stride(1), + value.stride(2), + value.stride(3), + gate.stride(0), + gate.stride(1), + gate.stride(2), + gate.stride(3), + channels, + height, + width, + total, + block, + ) + return out def _camera_ray_mats( diff --git a/integrations/sana/sana_wm/transformer.py b/integrations/sana/sana_wm/transformer.py index fb4c61092..c05a71c59 100644 --- a/integrations/sana/sana_wm/transformer.py +++ b/integrations/sana/sana_wm/transformer.py @@ -359,6 +359,25 @@ def _prepare_static_model_kwargs( model_kwargs["chunk_plucker_emb"] = prepare_plucker(chunk_plucker) del model_kwargs["chunk_plucker"] + camera_conditions = model_kwargs.get("camera_conditions") + prepare_camera = getattr(self.model, "prepare_camera_projection_cache", None) + if ( + "camera_cache" not in model_kwargs + and isinstance(camera_conditions, Tensor) + and camera_conditions.shape[0] == 1 + and callable(prepare_camera) + ): + _batch, _channels, frames, height, width = conditioning.latent_shape + with torch.inference_mode(): + rotary_emb, camera_cache = prepare_camera( + camera_conditions, + frames=frames, + height=height, + width=width, + ) + model_kwargs["rotary_emb"] = rotary_emb + model_kwargs["camera_cache"] = camera_cache + def _predict_with_prompt( self, noisy_latent: Tensor, @@ -500,10 +519,15 @@ def _batched_cfg_model_kwargs(model_kwargs: dict[str, object]) -> dict[str, obje if isinstance(mask, Tensor): mask_uncond = negative_mask if isinstance(negative_mask, Tensor) else mask kwargs["mask"] = torch.cat([mask_uncond, mask], dim=0) - for key in ("camera_conditions", "chunk_plucker"): - value = kwargs.get(key) - if isinstance(value, Tensor): - kwargs[key] = torch.cat([value, value], dim=0) + camera_conditions = kwargs.get("camera_conditions") + if isinstance(camera_conditions, Tensor) and camera_conditions.shape[0] != 1: + kwargs["camera_conditions"] = torch.cat( + [camera_conditions, camera_conditions], + dim=0, + ) + chunk_plucker = kwargs.get("chunk_plucker") + if isinstance(chunk_plucker, Tensor): + kwargs["chunk_plucker"] = torch.cat([chunk_plucker, chunk_plucker], dim=0) return kwargs diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index 7b8f23398..c04a30158 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -454,10 +454,7 @@ def forward( assert call["timestep"].shape == (2, 1, 2) assert call["prompt_embeds"].shape == (2, 1, 1, 1) torch.testing.assert_close(call["mask"], torch.cat([neg_mask, cond_mask], dim=0)) - torch.testing.assert_close( - call["camera_conditions"], - torch.cat([camera, camera], dim=0), - ) + torch.testing.assert_close(call["camera_conditions"], camera) torch.testing.assert_close( call["chunk_plucker"], torch.cat([chunk_plucker, chunk_plucker], dim=0), @@ -472,7 +469,9 @@ class DummyModel(torch.nn.Module): def __init__(self) -> None: super().__init__() self.prepare_calls = 0 - self.forward_calls: list[dict[str, torch.Tensor]] = [] + self.prepare_camera_calls = 0 + self.forward_calls: list[dict[str, object]] = [] + self.camera_cache = object() def prepare_plucker_embedding( self, @@ -481,6 +480,18 @@ def prepare_plucker_embedding( self.prepare_calls += 1 return chunk_plucker.flatten(2).transpose(1, 2) + def prepare_camera_projection_cache( + self, + camera_conditions: torch.Tensor, + *, + frames: int, + height: int, + width: int, + ) -> tuple[torch.Tensor, object]: + del camera_conditions + self.prepare_camera_calls += 1 + return torch.empty(frames, height, width), self.camera_cache + def forward( self, noisy_latent: torch.Tensor, @@ -488,11 +499,19 @@ def forward( prompt_embeds: torch.Tensor, *, chunk_plucker_emb: torch.Tensor, + rotary_emb: torch.Tensor, + camera_cache: object, **_kwargs: object, ) -> torch.Tensor: del timestep, prompt_embeds assert "chunk_plucker" not in _kwargs - self.forward_calls.append({"chunk_plucker_emb": chunk_plucker_emb}) + self.forward_calls.append( + { + "chunk_plucker_emb": chunk_plucker_emb, + "rotary_emb": rotary_emb, + "camera_cache": camera_cache, + } + ) return torch.ones_like(noisy_latent) transformer = SanaWMTransformerConfig().setup() @@ -505,6 +524,7 @@ def forward( uncondition=None, model_kwargs={ "chunk_plucker": chunk_plucker, + "camera_conditions": torch.zeros((1, 2, 20)), "data_info": {"condition_frame_info": {0: 0.0}}, }, first_latent=torch.empty((1, 1, 1, 1, 1)), @@ -525,13 +545,19 @@ def forward( ) assert dummy_model.prepare_calls == 1 + assert dummy_model.prepare_camera_calls == 1 assert "chunk_plucker" not in conditioning.model_kwargs + assert "camera_cache" in conditioning.model_kwargs assert len(dummy_model.forward_calls) == 2 cached = conditioning.model_kwargs["chunk_plucker_emb"] assert isinstance(cached, torch.Tensor) assert all( call["chunk_plucker_emb"] is cached for call in dummy_model.forward_calls ) + assert all( + call["camera_cache"] is dummy_model.camera_cache + for call in dummy_model.forward_calls + ) def test_inference_config_loads_yaml( @@ -776,8 +802,8 @@ def test_stage1_forward_preserves_latent_shape() -> None: text_dim=12, timestep_dim=8, depth=2, - num_heads=4, - head_dim=4, + num_heads=2, + head_dim=8, max_text_length=5, latent_grid_size=(2, 2), mlp_ratio=1, @@ -799,6 +825,67 @@ def test_stage1_forward_preserves_latent_shape() -> None: assert out.shape == latents.shape +def test_stage1_forward_accepts_precomputed_camera_cache() -> None: + """Precomputed camera cache must match the raw camera-conditioning path.""" + torch.manual_seed(0) + spec = SanaWMStage1Spec( + latent_channels=4, + hidden_size=16, + text_dim=12, + timestep_dim=8, + depth=1, + num_heads=2, + head_dim=8, + max_text_length=5, + latent_grid_size=(2, 2), + mlp_ratio=1, + conv_kernel_size=3, + temporal_kernel_size=3, + plucker_channels=6, + raymap_channels=3, + softmax_every_n=1, + ) + model = SanaWMStage1Model(spec).eval() + with torch.no_grad(): + for param in model.parameters(): + param.normal_(mean=0.0, std=0.02) + latents = torch.randn(1, 4, 3, 2, 2) + timesteps = torch.ones(1, 1, 3) + text = torch.randn(1, 1, 5, 12) + mask = torch.ones(1, 5) + plucker = torch.randn(1, 6, 3, 2, 2) + camera = torch.zeros(1, 3, 20) + camera[..., :16] = torch.eye(4).flatten() + camera[..., 16:] = torch.tensor([1.0, 1.0, 0.5, 0.5]) + rotary_emb, camera_cache = model.prepare_camera_projection_cache( + camera, + frames=3, + height=2, + width=2, + ) + + raw = model( + latents, + timesteps, + text, + mask=mask, + chunk_plucker=plucker, + camera_conditions=camera, + ) + cached = model( + latents, + timesteps, + text, + mask=mask, + chunk_plucker=plucker, + camera_conditions=camera, + rotary_emb=rotary_emb, + camera_cache=camera_cache, + ) + + torch.testing.assert_close(cached, raw) + + def test_stage1_self_attention_uses_camera_conditions() -> None: """Changing camera conditions must affect camera attention.""" torch.manual_seed(0) @@ -834,6 +921,90 @@ def test_stage1_self_attention_uses_camera_conditions() -> None: assert not torch.allclose(base, shifted) +def test_stage1_self_attention_broadcasts_single_camera_batch() -> None: + """CFG can reuse one camera projection cache for both prompt branches.""" + torch.manual_seed(1) + spec = SanaWMStage1Spec( + latent_channels=4, + hidden_size=16, + text_dim=12, + timestep_dim=8, + depth=1, + num_heads=2, + head_dim=8, + max_text_length=5, + latent_grid_size=(2, 2), + mlp_ratio=1, + conv_kernel_size=3, + temporal_kernel_size=3, + plucker_channels=6, + raymap_channels=3, + softmax_every_n=2, + ) + attn = Stage1SelfAttention(spec, use_gdn_convs=True).eval() + hidden = torch.randn(2, 8, 16) + camera = torch.zeros(1, 2, 20) + camera[..., :16] = torch.eye(4).flatten() + camera[..., 16:] = torch.tensor([1.0, 1.0, 0.5, 0.5]) + + broadcast = attn(hidden, HW=(2, 2, 2), camera_conditions=camera) + duplicated = attn( + hidden, + HW=(2, 2, 2), + camera_conditions=torch.cat([camera, camera], dim=0), + ) + + assert broadcast.shape == hidden.shape + torch.testing.assert_close(broadcast, duplicated) + + +def test_stage1_self_attention_uses_gdn_main_without_gdn_convs() -> None: + """Blocks without GDN conv weights still route main attention through GDN.""" + spec = SanaWMStage1Spec( + latent_channels=4, + hidden_size=16, + text_dim=12, + timestep_dim=8, + depth=1, + num_heads=2, + head_dim=8, + max_text_length=5, + latent_grid_size=(2, 2), + mlp_ratio=1, + conv_kernel_size=3, + temporal_kernel_size=3, + plucker_channels=6, + raymap_channels=3, + softmax_every_n=1, + ) + attn = Stage1SelfAttention(spec, use_gdn_convs=False).eval() + hidden = torch.randn(1, 4, 16) + called = {"gdn_main": False} + + def fake_gdn_main( + x: torch.Tensor, + *, + HW: tuple[int, int, int], + rotary_emb: torch.Tensor | None, + precomputed_gates: tuple[torch.Tensor, torch.Tensor], + ) -> torch.Tensor: + del HW, rotary_emb + called["gdn_main"] = True + assert precomputed_gates[0].shape == (1, 2, 1, 4) + return torch.zeros_like(x) + + def fail_softmax_main(*_args: object, **_kwargs: object) -> torch.Tensor: + raise AssertionError("main attention should not use softmax") + + attn._forward_gdn_main = fake_gdn_main # type: ignore[method-assign] + attn._forward_softmax_main = fail_softmax_main # type: ignore[method-assign] + + out = attn(hidden, HW=(1, 2, 2), apply_output_gate=False) + + assert called["gdn_main"] + torch.testing.assert_close(out, torch.zeros_like(hidden)) + + def test_transformer_releases_stage1_runtime() -> None: """Free Stage-1-only modules and conditioning before decode/refine.""" transformer = SanaWMTransformerConfig().setup() diff --git a/integrations/sana/tests/test_stage1_cuda.py b/integrations/sana/tests/test_stage1_cuda.py index a72c40ada..d0f60ada6 100644 --- a/integrations/sana/tests/test_stage1_cuda.py +++ b/integrations/sana/tests/test_stage1_cuda.py @@ -77,3 +77,204 @@ def test_bidirectional_temporal_conv_fast_path_matches_eager( rtol=2e-2, atol=8e-2 if dtype is torch.bfloat16 else 1e-5, ) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("inverse_rope", [False, True]) +@pytest.mark.parametrize("matrix_batch", [1, 2]) +def test_ucpe_transform_apply_fast_path_matches_eager( + dtype: torch.dtype, + inverse_rope: bool, + matrix_batch: int, +) -> None: + """Compare the CUDA UCPE transform fast path against the eager path.""" + _require_stage1_cuda() + torch.manual_seed(4321) + batch, frames, height, width, heads, dim = 2, 3, 2, 3, 2, 16 + tokens = frames * height * width + x = torch.randn( + batch, + tokens, + heads, + dim, + device="cuda", + dtype=dtype, + ) + matrix = torch.randn( + matrix_batch, + tokens, + 4, + 4, + device="cuda", + dtype=dtype, + ) + rope = stage1._slice_rope_for_camera( + stage1._wan_rope_complex(dim, frames, height, width, x.device), + dim, + ) + + reference = stage1._ucpe_transform_apply_eager( + x, + matrix, + rope, + inverse_rope=inverse_rope, + ) + with torch.inference_mode(): + actual = stage1._ucpe_transform_apply( + x, + matrix, + rope, + inverse_rope=inverse_rope, + ) + + assert actual.shape == x.shape + assert actual.dtype is torch.float32 + torch.testing.assert_close( + actual, + reference, + rtol=2e-2 if dtype is torch.bfloat16 else 1e-5, + atol=8e-2 if dtype is torch.bfloat16 else 1e-5, + ) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("inverse_rope", [False, True]) +@pytest.mark.parametrize("matrix_batch", [1, 2]) +def test_ucpe_transform_fast_path_matches_eager_norms( + dtype: torch.dtype, + inverse_rope: bool, + matrix_batch: int, +) -> None: + """Compare the UCPE fast path including GDN norm buffers.""" + _require_stage1_cuda() + torch.manual_seed(2468) + batch, frames, height, width, heads, dim = 2, 3, 2, 3, 2, 16 + tokens = frames * height * width + x = torch.randn( + batch, + tokens, + heads, + dim, + device="cuda", + dtype=dtype, + ) + matrix = torch.randn( + matrix_batch, + tokens, + 4, + 4, + device="cuda", + dtype=dtype, + ) + rope = stage1._slice_rope_for_camera( + stage1._wan_rope_complex(dim, frames, height, width, x.device), + dim, + ) + + reference = stage1._ucpe_transform_apply_eager( + x, + matrix, + rope, + inverse_rope=inverse_rope, + ) + reference_pre = stage1._ucpe_transform_norms_eager(x) + reference_post = stage1._ucpe_transform_norms_eager(reference) + with torch.inference_mode(): + actual, actual_pre, actual_post = stage1._ucpe_transform( + x, + matrix, + rope, + inverse_rope=inverse_rope, + ) + + assert actual.shape == x.shape + assert actual_pre.shape == (batch, heads, tokens) + assert actual_post.shape == (batch, heads, tokens) + torch.testing.assert_close( + actual, + reference, + rtol=2e-2 if dtype is torch.bfloat16 else 1e-5, + atol=8e-2 if dtype is torch.bfloat16 else 1e-5, + ) + torch.testing.assert_close( + actual_pre, + reference_pre, + rtol=2e-2 if dtype is torch.bfloat16 else 1e-5, + atol=8e-2 if dtype is torch.bfloat16 else 1e-5, + ) + torch.testing.assert_close( + actual_post, + reference_post, + rtol=2e-2 if dtype is torch.bfloat16 else 1e-5, + atol=8e-2 if dtype is torch.bfloat16 else 1e-5, + ) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +def test_silu_multiply_fast_path_matches_eager(dtype: torch.dtype) -> None: + """Compare the fused GLU elementwise fast path against PyTorch eager.""" + _require_stage1_cuda() + torch.manual_seed(1357) + value = torch.randn( + 4, + 17, + 5, + 7, + device="cuda", + dtype=dtype, + ).contiguous(memory_format=torch.channels_last) + gate = torch.randn_like(value).contiguous(memory_format=torch.channels_last) + + reference = value * torch.nn.functional.silu(gate) + with torch.inference_mode(): + actual = stage1._silu_multiply(value, gate, inplace=False) + + assert actual.shape == value.shape + assert actual.dtype == value.dtype + assert actual.is_contiguous(memory_format=torch.channels_last) + torch.testing.assert_close( + actual.float(), + reference.float(), + rtol=2e-2 if dtype is torch.bfloat16 else 1e-5, + atol=8e-2 if dtype is torch.bfloat16 else 1e-5, + ) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("scale", [1.0, 0.0375]) +def test_rmsnorm_relu_heads_fast_path_matches_eager( + dtype: torch.dtype, + scale: float, +) -> None: + """Compare fused GDN RMSNorm/ReLU preparation against eager math.""" + _require_stage1_cuda() + torch.manual_seed(9753) + batch, tokens, heads, dim = 2, 13, 4, 8 + x = torch.randn( + batch, + tokens, + heads, + dim, + device="cuda", + dtype=dtype, + ) + weight = torch.randn(heads * dim, device="cuda", dtype=dtype) + eps = 1e-6 + inv_rms = stage1._inv_rms(x, eps) + reference = torch.nn.functional.relu( + x.float() * inv_rms[:, :, None, None] * weight.float().view(heads, dim) + ) + if scale != 1.0: + reference = reference * scale + + with torch.inference_mode(): + actual = stage1._rmsnorm_relu_heads(x, weight, eps, scale=scale) + + assert actual.shape == x.shape + assert actual.dtype is torch.float32 + torch.testing.assert_close( + actual, + reference, + rtol=2e-2 if dtype is torch.bfloat16 else 1e-5, + atol=8e-2 if dtype is torch.bfloat16 else 1e-5, + ) From 3eaaa8f274c0af8f5181231b1620653a53567bb4 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Wed, 29 Jul 2026 15:52:56 -0700 Subject: [PATCH 37/64] Update benchmark scripts for _streaming upstream --- .../sana/tests/parity_check/README.md | 61 +- integrations/sana/tests/parity_check/bench.sh | 385 +++++++++---- .../sana/tests/parity_check/bench_summary.py | 523 +++++++++++++++--- .../tests/parity_check/bench_sweep_summary.py | 86 ++- .../parity_check/changes_streaming.patch | 33 ++ .../sana/tests/parity_check/pyproject.toml | 9 +- integrations/sana/tests/parity_check/uv.lock | 42 +- .../tests/test_parity_benchmark_summary.py | 181 ++++++ 8 files changed, 1088 insertions(+), 232 deletions(-) create mode 100644 integrations/sana/tests/parity_check/changes_streaming.patch diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index 5171f7376..f5f4a9f46 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -10,13 +10,12 @@ This harness compares upstream FlashDreams `sana-wm-bidirectional` integration on the same demo image, prompt, camera trajectory, intrinsics, seed, resolution, and precision settings. -All dependencies live in this directory's isolated `./.venv`. `run.sh` reuses or -clones the upstream checkout, pins it to `6298508`, and applies `changes.patch` -idempotently. The patch is instrumentation-only: it adds upstream CLI flags for -frame dumps, stats JSON, precision selection, cuDNN SDPA selection, and Stage-1 -`torch.compile` without changing the generation algorithm. A small -`compat/mmcv` shim covers the registry/logging imports needed by the inference -path. +All dependencies live in this directory's isolated `./.venv`. `run.sh` and +`bench.sh` reuse or clone the upstream checkout, pin it to `6298508`, and apply +instrumentation patches idempotently. `changes.patch` covers the bidirectional +entrypoint. `changes_streaming.patch` covers the streaming entrypoint metadata. +The patches do not change generation algorithms. A small `compat/mmcv` shim +covers the registry/logging imports needed by the inference path. ## Run parity @@ -58,9 +57,11 @@ cd integrations/sana/tests/parity_check bash bench.sh ``` -Benchmark defaults discard one warmup run and measure three additional runs: -`WARMUP_RUNS=1 MEASURED_RUNS=3 COMPILE_STAGE1=0 NO_REFINER=1 FORCE_CUDNN_SDPA=0`. -Outputs are under `outputs/bench/`: +Benchmark defaults to the bidirectional comparison and discards one warmup run +before measuring three additional runs: +`SANA_WM_VARIANT=bidirectional BENCH_SIDE=both WARMUP_RUNS=1 MEASURED_RUNS=3 +COMPILE_STAGE1=0 NO_REFINER=1 FORCE_CUDNN_SDPA=0`. Outputs are under +`outputs/bench/`: - `bench.json` - machine-readable inputs, medians, p90s, memory, and stage timings. @@ -110,3 +111,43 @@ DEVICE_LABEL="RTX PRO 6000 Blackwell" BENCH_PRECISIONS=bf16,fp8,fp4 bash bench.s The scripts do not set allocator overrides or GPU wait loops. If GPU contention matters in your environment, handle it outside the harness. + +## Run upstream streaming benchmark + +Use the streaming variant when collecting upstream-only `SANA-WM_streaming` +numbers before the FlashDreams streaming runner exists: + +```bash +cd integrations/sana/tests/parity_check +SANA_WM_VARIANT=streaming BENCH_SIDE=upstream bash bench.sh +``` + +Streaming defaults use the upstream streaming entrypoint +`inference_video_scripts/wm/inference_sana_wm_streaming.py`, the +`Efficient-Large-Model/SANA-WM_streaming` weights, `NUM_FRAMES=241`, +`CFG_SCALE=1.0`, and `STREAMING_ACTION=w-80,dw-40,w-80,aw-40`. Set +`STREAMING_ACTION=` to use `CAMERA_PATH` instead. Outputs are precision-aware: + +- `outputs/bench/streaming//upstream/run_/stats.json` +- `outputs/bench/streaming//upstream/run_/command.txt` +- `outputs/bench/streaming//bench.json` +- `outputs/bench/streaming//bench.md` + +The streaming headline metric is steady-state generation milliseconds per +produced chunk. Warmup runs and the first decoded chunk are excluded; full-clip +wall time remains in `bench.json` and `bench.md` as supporting data. + +Run the upstream precision sweep in order with: + +```bash +SANA_WM_VARIANT=streaming BENCH_SIDE=upstream BENCH_PRECISIONS=bf16,fp8,fp4 bash bench.sh +``` + +FP8 and FP4 are passed through upstream streaming-only +`--stage1_precision`/`--refiner_precision` flags, and those precision runs sync +the isolated venv with the `quant` extra before launch. If Transformer Engine, +hardware, or upstream code rejects a precision, earlier precision outputs remain +in place and the failing command is recorded in the corresponding `command.txt`. + +`BENCH_SIDE=both` is intentionally rejected for `SANA_WM_VARIANT=streaming` +until the FlashDreams `sana-wm-streaming` runner is available. diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index c6f1a105f..ce2022a83 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -14,14 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Stack-matched SANA-WM upstream vs FlashDreams benchmark. The chart metric is -# post-load generation latency per generated clip. Defaults to Stage-1-only so -# conditioning, Stage-1 DiT, and SANA VAE decode are measured without the LTX-2 -# refiner. +# Stack-matched SANA-WM benchmark harness. Bidirectional defaults preserve the +# existing upstream-vs-FlashDreams clip-latency benchmark. Streaming mode can +# benchmark upstream only before the FlashDreams streaming runner lands. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" _abspath() { case "$1" in @@ -30,41 +30,126 @@ _abspath() { esac } +_is_true() { + case "${1,,}" in + 1|true|yes|on) return 0 ;; + *) return 1 ;; + esac +} + +_write_command() { + local path="$1" + shift + mkdir -p "$(dirname "${path}")" + printf "%q " "$@" > "${path}" + printf "\n" >> "${path}" +} + +_apply_patch_once() { + local patch_file="$1" + if git apply --reverse --check "${patch_file}" >/dev/null 2>&1; then + echo "[setup] patch already applied, skipping ${patch_file}" + elif git apply --check "${patch_file}" >/dev/null 2>&1; then + echo "[setup] applying ${patch_file}" + git apply "${patch_file}" + else + echo "[setup] ERROR: ${patch_file} neither cleanly applies nor is already applied." >&2 + exit 1 + fi +} + SANA_REPO="$(_abspath "${SANA_REPO:-${SCRIPT_DIR}/Sana}")" PATCH_FILE="${SCRIPT_DIR}/changes.patch" +STREAMING_PATCH_FILE="${SCRIPT_DIR}/changes_streaming.patch" REPO_URL="https://github.com/NVlabs/Sana.git" PIN_COMMIT="6298508" -OUTPUT_DIR="$(_abspath "${OUTPUT_DIR:-${SCRIPT_DIR}/outputs/bench}")" +SANA_WM_VARIANT="${SANA_WM_VARIANT:-bidirectional}" +BENCH_SIDE="${BENCH_SIDE:-both}" +case "${SANA_WM_VARIANT}" in + bidirectional|streaming) ;; + *) + echo "[bench] ERROR: SANA_WM_VARIANT must be bidirectional or streaming; got ${SANA_WM_VARIANT}" >&2 + exit 1 + ;; +esac +case "${BENCH_SIDE}" in + upstream|both) ;; + *) + echo "[bench] ERROR: BENCH_SIDE must be upstream or both; got ${BENCH_SIDE}" >&2 + exit 1 + ;; +esac +if [[ "${SANA_WM_VARIANT}" == "streaming" && "${BENCH_SIDE}" == "both" ]]; then + echo "[bench] ERROR: SANA_WM_VARIANT=streaming currently supports BENCH_SIDE=upstream only." >&2 + echo " Wire the FlashDreams sana-wm-streaming runner before running BENCH_SIDE=both." >&2 + exit 1 +fi + +DEVICE_LABEL="${DEVICE_LABEL:-GPU}" +STAGE1_PRECISION="${STAGE1_PRECISION:-bf16}" +REFINER_PRECISION="${REFINER_PRECISION:-${STAGE1_PRECISION}}" +QUANT_BACKEND="${QUANT_BACKEND:-auto}" +CHART_LABEL="${CHART_LABEL:-${DEVICE_LABEL}}" +BENCH_PRECISIONS="${BENCH_PRECISIONS:-}" +WARMUP_RUNS="${WARMUP_RUNS:-1}" +MEASURED_RUNS="${MEASURED_RUNS:-3}" +BENCH_DRY_RUN="${BENCH_DRY_RUN:-0}" + +if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then + DEFAULT_OUTPUT_DIR="${SCRIPT_DIR}/outputs/bench/streaming/${STAGE1_PRECISION}" + DEFAULT_SWEEP_OUTPUT_DIR="${SCRIPT_DIR}/outputs/bench/streaming" + NUM_FRAMES="${NUM_FRAMES:-241}" + CFG_SCALE="${CFG_SCALE:-1.0}" +else + DEFAULT_OUTPUT_DIR="${SCRIPT_DIR}/outputs/bench" + DEFAULT_SWEEP_OUTPUT_DIR="${SCRIPT_DIR}/outputs/bench" + NUM_FRAMES="${NUM_FRAMES:-121}" + CFG_SCALE="${CFG_SCALE:-5.0}" +fi +if [[ -n "${BENCH_PRECISIONS}" ]]; then + OUTPUT_DIR="$(_abspath "${OUTPUT_DIR:-${DEFAULT_SWEEP_OUTPUT_DIR}}")" +else + OUTPUT_DIR="$(_abspath "${OUTPUT_DIR:-${DEFAULT_OUTPUT_DIR}}")" +fi UPSTREAM_ROOT="${OUTPUT_DIR}/upstream" NATIVE_ROOT="${OUTPUT_DIR}/flashdreams" + IMAGE_PATH="$(_abspath "${IMAGE_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.png}")" PROMPT_PATH="$(_abspath "${PROMPT_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.txt}")" CAMERA_PATH="$(_abspath "${CAMERA_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_pose.npy}")" INTRINSICS_PATH="$(_abspath "${INTRINSICS_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_intrinsics.npy}")" -NUM_FRAMES="${NUM_FRAMES:-121}" +STREAMING_ACTION="${STREAMING_ACTION-w-80,dw-40,w-80,aw-40}" FPS="${FPS:-16}" STEP="${STEP:-60}" -CFG_SCALE="${CFG_SCALE:-5.0}" SEED="${SEED:-42}" NO_REFINER="${NO_REFINER:-1}" FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" COMPILE_STAGE1="${COMPILE_STAGE1:-0}" -WARMUP_RUNS="${WARMUP_RUNS:-1}" -MEASURED_RUNS="${MEASURED_RUNS:-3}" -DEVICE_LABEL="${DEVICE_LABEL:-GPU}" -STAGE1_PRECISION="${STAGE1_PRECISION:-bf16}" -REFINER_PRECISION="${REFINER_PRECISION:-${STAGE1_PRECISION}}" -QUANT_BACKEND="${QUANT_BACKEND:-auto}" -CHART_LABEL="${CHART_LABEL:-${DEVICE_LABEL}}" -BENCH_PRECISIONS="${BENCH_PRECISIONS:-}" -_is_true() { - case "${1,,}" in - 1|true|yes|on) return 0 ;; - *) return 1 ;; - esac -} +STREAMING_OUTPUT_MODE="${STREAMING_OUTPUT_MODE:-mp4}" +STREAMING_NO_MP4="${STREAMING_NO_MP4:-0}" +STREAMING_PROFILE_CUDA="${STREAMING_PROFILE_CUDA:-0}" +STREAMING_NO_COMPILE="${STREAMING_NO_COMPILE:-0}" +STREAMING_DENOISING_STEP_LIST="${STREAMING_DENOISING_STEP_LIST:-1000,960,889,727,0}" +STREAMING_NUM_FRAME_PER_BLOCK="${STREAMING_NUM_FRAME_PER_BLOCK:-3}" +STREAMING_REFINER_BLOCK_SIZE="${STREAMING_REFINER_BLOCK_SIZE:-3}" +STREAMING_REFINER_KV_MAX_FRAMES="${STREAMING_REFINER_KV_MAX_FRAMES:-11}" +STREAMING_REFINER_SEED="${STREAMING_REFINER_SEED:-${SEED}}" +STREAMING_SINK_SIZE="${STREAMING_SINK_SIZE:-1}" +STREAMING_NUM_CACHED_BLOCKS="${STREAMING_NUM_CACHED_BLOCKS:-2}" +STREAMING_CRF="${STREAMING_CRF:-18}" +STREAMING_PRESET="${STREAMING_PRESET:-medium}" +STREAMING_ENCODER="${STREAMING_ENCODER:-${SANA_WM_STREAMING_MP4_ENCODER:-libx264}}" + +if _is_true "${BENCH_DRY_RUN}"; then + echo "[bench] dry run" + echo " variant: ${SANA_WM_VARIANT}" + echo " side: ${BENCH_SIDE}" + echo " output: ${OUTPUT_DIR}" + echo " precision: stage1=${STAGE1_PRECISION} refiner=${REFINER_PRECISION}" + exit 0 +fi if [[ -n "${BENCH_PRECISIONS}" ]]; then mkdir -p "${OUTPUT_DIR}" @@ -86,10 +171,14 @@ if [[ -n "${BENCH_PRECISIONS}" ]]; then PRECISION_OUTPUT_DIR="${OUTPUT_DIR}/${PRECISION}" echo "[bench] precision sweep row ${PRECISION_LABEL} -> ${PRECISION_OUTPUT_DIR}" BENCH_PRECISIONS="" \ + SANA_REPO="${SANA_REPO}" \ + SANA_WM_VARIANT="${SANA_WM_VARIANT}" \ + BENCH_SIDE="${BENCH_SIDE}" \ OUTPUT_DIR="${PRECISION_OUTPUT_DIR}" \ STAGE1_PRECISION="${PRECISION}" \ REFINER_PRECISION="${PRECISION}" \ QUANT_BACKEND="${QUANT_BACKEND}" \ + DEVICE_LABEL="${DEVICE_LABEL}" \ CHART_LABEL="${CHART_LABEL}" \ bash "${SCRIPT_DIR}/bench.sh" SWEEP_ITEMS+=(--item "${PRECISION_LABEL}:${PRECISION_OUTPUT_DIR}/bench.json") @@ -108,7 +197,6 @@ if [[ -n "${BENCH_PRECISIONS}" ]]; then echo "[bench] done." echo " precision summary: ${OUTPUT_DIR}/bench.md" echo " precision data: ${OUTPUT_DIR}/perf.md" - echo " chart data: ${OUTPUT_DIR}/{bf16,fp8,fp4}/perf.md" exit 0 fi @@ -128,29 +216,54 @@ else echo "[setup] already at pinned commit ${PIN_COMMIT}, skipping checkout" fi -if git apply --reverse --check "${PATCH_FILE}" >/dev/null 2>&1; then - echo "[setup] patch already applied, skipping" -elif git apply --check "${PATCH_FILE}" >/dev/null 2>&1; then - echo "[setup] applying ${PATCH_FILE}" - git apply "${PATCH_FILE}" -else - echo "[setup] ERROR: ${PATCH_FILE} neither cleanly applies nor is already applied." >&2 - exit 1 +_apply_patch_once "${PATCH_FILE}" +if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then + if [[ ! -f "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm_streaming.py" ]]; then + echo "[setup] ERROR: pinned Sana checkout has no SANA-WM streaming entrypoint." >&2 + echo " Expected: inference_video_scripts/wm/inference_sana_wm_streaming.py" >&2 + exit 1 + fi + _apply_patch_once "${STREAMING_PATCH_FILE}" fi +UPSTREAM_COMMIT="$(git rev-parse HEAD)" +FLASHDREAMS_COMMIT="$(git -C "${REPO_ROOT}" rev-parse HEAD 2>/dev/null || printf "unknown")" -echo "[setup] ensuring Python deps via uv sync (isolated venv)" -( cd "${SCRIPT_DIR}" && uv sync ) +UV_SYNC_ARGS=(uv sync) +if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; then + UV_SYNC_ARGS+=(--extra quant) + export NVTE_FRAMEWORK="${NVTE_FRAMEWORK:-pytorch}" + export NVTE_WITH_NCCL_EP="${NVTE_WITH_NCCL_EP:-0}" + if [[ -z "${NVTE_CUDA_ARCHS:-}" ]]; then + NVTE_COMPUTE_CAP="$( + nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null \ + | head -n 1 \ + | tr -cd '0-9' + )" + if [[ -n "${NVTE_COMPUTE_CAP}" ]]; then + export NVTE_CUDA_ARCHS="${NVTE_COMPUTE_CAP}" + fi + fi + echo "[setup] TransformerEngine build env: NVTE_FRAMEWORK=${NVTE_FRAMEWORK} NVTE_CUDA_ARCHS=${NVTE_CUDA_ARCHS:-default} NVTE_WITH_NCCL_EP=${NVTE_WITH_NCCL_EP}" +fi +echo "[setup] ensuring Python deps via ${UV_SYNC_ARGS[*]} (isolated venv)" +( cd "${SCRIPT_DIR}" && "${UV_SYNC_ARGS[@]}" ) UPSTREAM_PYTHONPATH="${SCRIPT_DIR}/compat:${SANA_REPO}" if [[ -n "${PYTHONPATH:-}" ]]; then UPSTREAM_PYTHONPATH="${UPSTREAM_PYTHONPATH}:${PYTHONPATH}" fi -mkdir -p "${UPSTREAM_ROOT}" "${NATIVE_ROOT}" +mkdir -p "${UPSTREAM_ROOT}" +if [[ "${BENCH_SIDE}" == "both" ]]; then + mkdir -p "${NATIVE_ROOT}" +fi # Remove stale run_* dirs from prior (possibly longer) benchmarks. bench_summary # reads every run_* dir, so leftovers from an earlier run with more MEASURED_RUNS # would silently pollute this invocation's aggregate. -rm -rf "${UPSTREAM_ROOT:?}"/run_* "${NATIVE_ROOT:?}"/run_* +rm -rf "${UPSTREAM_ROOT:?}"/run_* +if [[ "${BENCH_SIDE}" == "both" ]]; then + rm -rf "${NATIVE_ROOT:?}"/run_* +fi UPSTREAM_REFINER_ARGS=() NATIVE_REFINER_ARGS=() @@ -180,53 +293,110 @@ NATIVE_PRECISION_ARGS=( --quant-backend "${QUANT_BACKEND}" ) +STREAMING_CAMERA_SOURCE="camera" +UPSTREAM_STREAMING_CAMERA_ARGS=(--camera "${CAMERA_PATH}") +if [[ -n "${STREAMING_ACTION}" ]]; then + STREAMING_CAMERA_SOURCE="action" + UPSTREAM_STREAMING_CAMERA_ARGS=(--action "${STREAMING_ACTION}") +fi +UPSTREAM_STREAMING_MODE_ARGS=(--output_mode "${STREAMING_OUTPUT_MODE}") +if _is_true "${STREAMING_NO_MP4}"; then + UPSTREAM_STREAMING_MODE_ARGS+=(--no_mp4) +fi +if _is_true "${STREAMING_PROFILE_CUDA}"; then + UPSTREAM_STREAMING_MODE_ARGS+=(--profile_cuda) +fi +if _is_true "${STREAMING_NO_COMPILE}"; then + UPSTREAM_STREAMING_MODE_ARGS+=(--no_compile) +fi + TOTAL_RUNS=$(( WARMUP_RUNS + MEASURED_RUNS )) for ((i = 0; i < TOTAL_RUNS; i++)); do UPSTREAM_OUT="${UPSTREAM_ROOT}/run_${i}" - NATIVE_OUT="${NATIVE_ROOT}/run_${i}" - mkdir -p "${UPSTREAM_OUT}" "${NATIVE_OUT}" + mkdir -p "${UPSTREAM_OUT}" - echo "[bench] upstream run ${i}/${TOTAL_RUNS}" - ( cd "${SCRIPT_DIR}" && \ - PYTHONPATH="${UPSTREAM_PYTHONPATH}" \ - uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm.py" \ - --image "${IMAGE_PATH}" \ - --prompt "${PROMPT_PATH}" \ - --camera "${CAMERA_PATH}" \ - --intrinsics "${INTRINSICS_PATH}" \ - --output_dir "${UPSTREAM_OUT}" \ - --name upstream \ - --num_frames "${NUM_FRAMES}" \ - --fps "${FPS}" \ - --step "${STEP}" \ - --cfg_scale "${CFG_SCALE}" \ - --seed "${SEED}" \ - --no_action_overlay \ - --stats_json "${UPSTREAM_OUT}/stats.json" \ - "${UPSTREAM_PRECISION_ARGS[@]}" \ - "${UPSTREAM_BACKEND_ARGS[@]}" \ - "${UPSTREAM_COMPILE_ARGS[@]}" \ - "${UPSTREAM_REFINER_ARGS[@]}" ) + if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then + UPSTREAM_CMD=( + env "PYTHONPATH=${UPSTREAM_PYTHONPATH}" + uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm_streaming.py" + --image "${IMAGE_PATH}" + --prompt "${PROMPT_PATH}" + "${UPSTREAM_STREAMING_CAMERA_ARGS[@]}" + --intrinsics "${INTRINSICS_PATH}" + --output_dir "${UPSTREAM_OUT}" + --name upstream + --num_frames "${NUM_FRAMES}" + --fps "${FPS}" + --cfg_scale "${CFG_SCALE}" + --seed "${SEED}" + --benchmark_json "${UPSTREAM_OUT}/stats.json" + --denoising_step_list "${STREAMING_DENOISING_STEP_LIST}" + --num_frame_per_block "${STREAMING_NUM_FRAME_PER_BLOCK}" + --refiner_block_size "${STREAMING_REFINER_BLOCK_SIZE}" + --refiner_kv_max_frames "${STREAMING_REFINER_KV_MAX_FRAMES}" + --refiner_seed "${STREAMING_REFINER_SEED}" + --sink_size "${STREAMING_SINK_SIZE}" + --num_cached_blocks "${STREAMING_NUM_CACHED_BLOCKS}" + --streaming_crf "${STREAMING_CRF}" + --streaming_preset "${STREAMING_PRESET}" + --streaming_encoder "${STREAMING_ENCODER}" + "${UPSTREAM_PRECISION_ARGS[@]}" + "${UPSTREAM_STREAMING_MODE_ARGS[@]}" + ) + else + UPSTREAM_CMD=( + env "PYTHONPATH=${UPSTREAM_PYTHONPATH}" + uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm.py" + --image "${IMAGE_PATH}" + --prompt "${PROMPT_PATH}" + --camera "${CAMERA_PATH}" + --intrinsics "${INTRINSICS_PATH}" + --output_dir "${UPSTREAM_OUT}" + --name upstream + --num_frames "${NUM_FRAMES}" + --fps "${FPS}" + --step "${STEP}" + --cfg_scale "${CFG_SCALE}" + --seed "${SEED}" + --no_action_overlay + --stats_json "${UPSTREAM_OUT}/stats.json" + "${UPSTREAM_PRECISION_ARGS[@]}" + "${UPSTREAM_BACKEND_ARGS[@]}" + "${UPSTREAM_COMPILE_ARGS[@]}" + "${UPSTREAM_REFINER_ARGS[@]}" + ) + fi - echo "[bench] FlashDreams run ${i}/${TOTAL_RUNS}" - ( cd "${SCRIPT_DIR}" && \ - uv run python "${SCRIPT_DIR}/run_native.py" \ - --image-path "${IMAGE_PATH}" \ - --prompt-path "${PROMPT_PATH}" \ - --camera-path "${CAMERA_PATH}" \ - --intrinsics-path "${INTRINSICS_PATH}" \ - --output-dir "${NATIVE_OUT}" \ - --name flashdreams \ - --num-frames "${NUM_FRAMES}" \ - --fps "${FPS}" \ - --step "${STEP}" \ - --cfg-scale "${CFG_SCALE}" \ - --seed "${SEED}" \ - --stats-json "${NATIVE_OUT}/stats.json" \ - "${NATIVE_PRECISION_ARGS[@]}" \ - "${NATIVE_BACKEND_ARGS[@]}" \ - "${NATIVE_COMPILE_ARGS[@]}" \ - "${NATIVE_REFINER_ARGS[@]}" ) + echo "[bench] upstream ${SANA_WM_VARIANT} run ${i}/${TOTAL_RUNS}" + _write_command "${UPSTREAM_OUT}/command.txt" "${UPSTREAM_CMD[@]}" + ( cd "${SCRIPT_DIR}" && "${UPSTREAM_CMD[@]}" ) + + if [[ "${BENCH_SIDE}" == "both" ]]; then + NATIVE_OUT="${NATIVE_ROOT}/run_${i}" + mkdir -p "${NATIVE_OUT}" + NATIVE_CMD=( + uv run python "${SCRIPT_DIR}/run_native.py" + --image-path "${IMAGE_PATH}" + --prompt-path "${PROMPT_PATH}" + --camera-path "${CAMERA_PATH}" + --intrinsics-path "${INTRINSICS_PATH}" + --output-dir "${NATIVE_OUT}" + --name flashdreams + --num-frames "${NUM_FRAMES}" + --fps "${FPS}" + --step "${STEP}" + --cfg-scale "${CFG_SCALE}" + --seed "${SEED}" + --stats-json "${NATIVE_OUT}/stats.json" + "${NATIVE_PRECISION_ARGS[@]}" + "${NATIVE_BACKEND_ARGS[@]}" + "${NATIVE_COMPILE_ARGS[@]}" + "${NATIVE_REFINER_ARGS[@]}" + ) + echo "[bench] FlashDreams ${SANA_WM_VARIANT} run ${i}/${TOTAL_RUNS}" + _write_command "${NATIVE_OUT}/command.txt" "${NATIVE_CMD[@]}" + ( cd "${SCRIPT_DIR}" && "${NATIVE_CMD[@]}" ) + fi done SUMMARY_JSON="${OUTPUT_DIR}/bench.json" @@ -242,28 +412,49 @@ fi if _is_true "${FORCE_CUDNN_SDPA}"; then SUMMARY_FLAGS+=(--force-cudnn-sdpa) fi +SUMMARY_ARGS=( + uv run python "${SCRIPT_DIR}/bench_summary.py" + --variant "${SANA_WM_VARIANT}" + --bench-side "${BENCH_SIDE}" + --upstream-dir "${UPSTREAM_ROOT}" + --warmup-runs "${WARMUP_RUNS}" + --image-path "${IMAGE_PATH}" + --prompt-path "${PROMPT_PATH}" + --camera-path "${CAMERA_PATH}" + --camera-source "${STREAMING_CAMERA_SOURCE}" + --intrinsics-path "${INTRINSICS_PATH}" + --num-frames "${NUM_FRAMES}" + --seed "${SEED}" + --device-label "${DEVICE_LABEL}" + --chart-label "${CHART_LABEL}" + --stage1-precision "${STAGE1_PRECISION}" + --refiner-precision "${REFINER_PRECISION}" + --quant-backend "${QUANT_BACKEND}" + --upstream-commit "${UPSTREAM_COMMIT}" + --flashdreams-commit "${FLASHDREAMS_COMMIT}" + --output-json "${SUMMARY_JSON}" + --output-md "${SUMMARY_MD}" + "${SUMMARY_FLAGS[@]}" +) +if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then + SUMMARY_ARGS+=( + --action "${STREAMING_ACTION}" + --output-mode "${STREAMING_OUTPUT_MODE}" + --denoising-step-list "${STREAMING_DENOISING_STEP_LIST}" + --num-frame-per-block "${STREAMING_NUM_FRAME_PER_BLOCK}" + --refiner-block-size "${STREAMING_REFINER_BLOCK_SIZE}" + --refiner-kv-max-frames "${STREAMING_REFINER_KV_MAX_FRAMES}" + ) +fi +if [[ "${BENCH_SIDE}" == "both" ]]; then + SUMMARY_ARGS+=(--flashdreams-dir "${NATIVE_ROOT}" --output-chart-md "${SUMMARY_CHART_MD}") +fi + echo "[bench] summarising -> ${SUMMARY_MD}" -( cd "${SCRIPT_DIR}" && \ - uv run python "${SCRIPT_DIR}/bench_summary.py" \ - --upstream-dir "${UPSTREAM_ROOT}" \ - --flashdreams-dir "${NATIVE_ROOT}" \ - --warmup-runs "${WARMUP_RUNS}" \ - --image-path "${IMAGE_PATH}" \ - --prompt-path "${PROMPT_PATH}" \ - --camera-path "${CAMERA_PATH}" \ - --intrinsics-path "${INTRINSICS_PATH}" \ - --num-frames "${NUM_FRAMES}" \ - --seed "${SEED}" \ - --device-label "${DEVICE_LABEL}" \ - --chart-label "${CHART_LABEL}" \ - --stage1-precision "${STAGE1_PRECISION}" \ - --refiner-precision "${REFINER_PRECISION}" \ - --quant-backend "${QUANT_BACKEND}" \ - "${SUMMARY_FLAGS[@]}" \ - --output-json "${SUMMARY_JSON}" \ - --output-md "${SUMMARY_MD}" \ - --output-chart-md "${SUMMARY_CHART_MD}" ) +( cd "${SCRIPT_DIR}" && "${SUMMARY_ARGS[@]}" ) echo "[bench] done." echo " summary: ${SUMMARY_MD}" -echo " chart data: ${SUMMARY_CHART_MD}" +if [[ -f "${SUMMARY_CHART_MD}" ]]; then + echo " chart data: ${SUMMARY_CHART_MD}" +fi diff --git a/integrations/sana/tests/parity_check/bench_summary.py b/integrations/sana/tests/parity_check/bench_summary.py index fa76ed17f..bc06b3c0b 100644 --- a/integrations/sana/tests/parity_check/bench_summary.py +++ b/integrations/sana/tests/parity_check/bench_summary.py @@ -28,7 +28,14 @@ def _load_stats(root: Path) -> list[dict[str, Any]]: paths = sorted(root.glob("run_*/stats.json")) if not paths and (root / "stats.json").exists(): paths = [root / "stats.json"] - return [json.loads(path.read_text(encoding="utf-8")) | {"_path": str(path)} for path in paths] + items = [] + for path in paths: + item = json.loads(path.read_text(encoding="utf-8")) | {"_path": str(path)} + command_path = path.with_name("command.txt") + if command_path.exists(): + item["_command"] = command_path.read_text(encoding="utf-8").strip() + items.append(item) + return items def _median(values: list[float]) -> float | None: @@ -63,7 +70,7 @@ def _native_stage_ms(item: dict[str, Any], key: str) -> float | None: def _upstream_mem_gib(item: dict[str, Any]) -> float | None: - value = item.get("mem_peak_gib") + value = item.get("mem_peak_gib", item.get("peak_mem_gb")) return float(value) if isinstance(value, (int, float)) else None @@ -88,6 +95,10 @@ def _collect( "warmup_runs": warmup_runs, "runs_measured": len(kept), "paths": [item.get("_path") for item in kept], + "run_records": [ + _run_record(item, index=index, warmup=index < warmup_runs) + for index, item in enumerate(items) + ], } wall = [ float(item["wall_s"]) @@ -114,6 +125,152 @@ def _collect( return rows +def _run_record( + item: dict[str, Any], + *, + index: int, + warmup: bool, +) -> dict[str, Any]: + record = { + key: value + for key, value in item.items() + if key not in {"_path", "_command"} + } + record["path"] = item.get("_path") + record["command"] = item.get("_command") + record["run_index"] = int(record.get("run_index", index)) + record["warmup"] = warmup + return record + + +def _first_command(items: list[dict[str, Any]]) -> str | None: + for item in items: + value = item.get("_command") + if isinstance(value, str) and value: + return value + return None + + +def _first_numeric(item: dict[str, Any], *keys: str) -> float | None: + for key in keys: + value = item.get(key) + if isinstance(value, (int, float)): + return float(value) + return None + + +def _streaming_steady_state_chunk_ms(item: dict[str, Any]) -> float | None: + steady_s = _first_numeric(item, "steady_state_seconds") + if steady_s is None: + return None + chunks_value = item.get("n_decode_chunks") + if not isinstance(chunks_value, int): + return None + # `steady_state_seconds` starts after the first decoded chunk, so exclude + # that chunk from the denominator to avoid mixing cache-fill/compile latency + # into the steady-state headline. + measured_chunks = chunks_value - 1 if item.get("first_chunk_seconds") is not None else chunks_value + if measured_chunks <= 0: + return None + return steady_s * 1000.0 / measured_chunks + + +def _collect_streaming( + items: list[dict[str, Any]], + warmup_runs: int, +) -> dict[str, Any]: + kept = items[warmup_runs:] + rows: dict[str, Any] = { + "runs_total": len(items), + "warmup_runs": warmup_runs, + "runs_measured": len(kept), + "paths": [item.get("_path") for item in kept], + "run_records": [ + _run_record(item, index=index, warmup=index < warmup_runs) + for index, item in enumerate(items) + ], + } + wall = [ + value + for item in kept + if (value := _first_numeric(item, "stream_wall_seconds", "wall_seconds", "wall_s")) is not None + ] + rows["wall_median_s"] = _median(wall) + rows["wall_p90_s"] = _p90(wall) + end_to_end = [ + value + for item in kept + if (value := _first_numeric(item, "end_to_end_seconds")) is not None + ] + rows["end_to_end_median_s"] = _median(end_to_end) + rows["end_to_end_p90_s"] = _p90(end_to_end) + first_chunk = [ + value + for item in kept + if (value := _first_numeric(item, "first_chunk_seconds")) is not None + ] + rows["first_chunk_median_s"] = _median(first_chunk) + rows["first_chunk_p90_s"] = _p90(first_chunk) + chunk_ms = [ + value + for item in kept + if (value := _streaming_steady_state_chunk_ms(item)) is not None + ] + rows["steady_state_chunk_median_ms"] = _median(chunk_ms) + rows["steady_state_chunk_p90_ms"] = _p90(chunk_ms) + steady_fps = [ + value + for item in kept + if (value := _first_numeric(item, "steady_state_frames_per_second")) is not None + ] + rows["steady_state_fps_median"] = _median(steady_fps) + rows["steady_state_fps_p90"] = _p90(steady_fps) + realtime = [ + value + for item in kept + if (value := _first_numeric(item, "steady_state_realtime_factor")) is not None + ] + rows["steady_state_realtime_median"] = _median(realtime) + rows["steady_state_realtime_p90"] = _p90(realtime) + memory = [ + value + for item in kept + if (value := _upstream_mem_gib(item)) is not None + ] + rows["mem_peak_median_gib"] = _median(memory) + rows["mem_peak_p90_gib"] = _p90(memory) + for label, key in { + "stage1_cuda": "stage1_cuda_seconds", + "refiner_cuda": "refiner_cuda_seconds", + "decode_cuda": "decode_cuda_seconds", + }.items(): + values = [ + value * 1000.0 + for item in kept + if (value := _first_numeric(item, key)) is not None + ] + rows[f"{label}_median_ms"] = _median(values) + rows[f"{label}_p90_ms"] = _p90(values) + for label, key in { + "n_pixel_frames": "n_pixel_frames", + "n_decode_chunks": "n_decode_chunks", + "n_refiner_blocks": "n_refiner_blocks", + }.items(): + values = [ + value + for item in kept + if (value := _first_numeric(item, key)) is not None + ] + rows[f"{label}_median"] = _median(values) + rows[f"{label}_p90"] = _p90(values) + rows["artifact_paths"] = [ + item.get("output_path") + for item in kept + if isinstance(item.get("output_path"), str) and item.get("output_path") + ] + return rows + + def _fmt(value: Any, suffix: str = "") -> str: if value is None: return "n/a" @@ -135,7 +292,10 @@ def _ms_per_clip(wall_s: float | None) -> float | None: def _metric_value(summary: dict[str, Any], side: str, key: str) -> float | None: - value = summary[side].get(key) + side_summary = summary.get(side) + if not isinstance(side_summary, dict): + return None + value = side_summary.get(key) return float(value) if isinstance(value, (int, float)) else None @@ -148,6 +308,19 @@ def _generation_ms_per_clip( return _ms_per_clip(_metric_value(summary, side, f"wall_{percentile}_s")) +def _streaming_generation_ms_per_chunk( + summary: dict[str, Any], + side: str, + *, + percentile: str = "median", +) -> float | None: + side_summary = summary.get(side) + if not isinstance(side_summary, dict): + return None + value = side_summary.get(f"steady_state_chunk_{percentile}_ms") + return float(value) if isinstance(value, (int, float)) else None + + def _generation_ms_per_frame( summary: dict[str, Any], side: str, @@ -167,9 +340,10 @@ def _sum_optional(*values: float | None) -> float | None: return sum(float(value) for value in values) -def _render_markdown(summary: dict[str, Any]) -> str: +def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: upstream = summary["upstream"] - native = summary["flashdreams"] + native = summary.get("flashdreams") + has_native = isinstance(native, dict) rows = [ "# SANA-WM parity harness benchmark", "", @@ -181,6 +355,8 @@ def _render_markdown(summary: dict[str, Any]) -> str: f"- intrinsics: `{summary['inputs']['intrinsics_path']}`", f"- num_frames: `{summary['inputs']['num_frames']}`", f"- seed: `{summary['inputs']['seed']}`", + f"- variant: `{summary['inputs'].get('variant', 'bidirectional')}`", + f"- bench_side: `{summary.get('bench_side', 'both')}`", f"- no_refiner: `{summary['inputs']['no_refiner']}`", f"- stage1_precision: `{summary['inputs']['stage1_precision']}`", f"- refiner_precision: `{summary['inputs']['refiner_precision']}`", @@ -196,54 +372,208 @@ def _render_markdown(summary: dict[str, Any]) -> str: "SANA-WM renders each requested bidirectional clip in one generation pass, not as independently timed frames.", "With the default `NO_REFINER=1`, the timed work covers conditioning, Stage-1 DiT, and SANA VAE decode.", "", - "| metric | upstream | FlashDreams |", - "| --- | ---: | ---: |", - f"| measured runs | {upstream['runs_measured']} | {native['runs_measured']} |", - f"| generation median / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream'), ' ms')} | {_fmt(_generation_ms_per_clip(summary, 'flashdreams'), ' ms')} |", - f"| generation p90 / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_generation_ms_per_clip(summary, 'flashdreams', percentile='p90'), ' ms')} |", - f"| generation median / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'upstream'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams'), ' ms')} |", - f"| generation p90 / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams', percentile='p90'), ' ms')} |", - f"| wall median | {_fmt(upstream['wall_median_s'], ' s')} | {_fmt(native['wall_median_s'], ' s')} |", - f"| wall p90 | {_fmt(upstream['wall_p90_s'], ' s')} | {_fmt(native['wall_p90_s'], ' s')} |", + ] + if has_native: + rows.extend( + [ + "| metric | upstream | FlashDreams |", + "| --- | ---: | ---: |", + f"| measured runs | {upstream['runs_measured']} | {native['runs_measured']} |", + f"| generation median / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream'), ' ms')} | {_fmt(_generation_ms_per_clip(summary, 'flashdreams'), ' ms')} |", + f"| generation p90 / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_generation_ms_per_clip(summary, 'flashdreams', percentile='p90'), ' ms')} |", + f"| generation median / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'upstream'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams'), ' ms')} |", + f"| generation p90 / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams', percentile='p90'), ' ms')} |", + f"| wall median | {_fmt(upstream['wall_median_s'], ' s')} | {_fmt(native['wall_median_s'], ' s')} |", + f"| wall p90 | {_fmt(upstream['wall_p90_s'], ' s')} | {_fmt(native['wall_p90_s'], ' s')} |", + "", + "## Timing breakdown", + "", + "| stage | upstream median | FlashDreams median |", + "| --- | ---: | ---: |", + f"| Stage-1 incl. conditioning median | {_fmt(upstream['stage1_total_median_ms'], ' ms')} | {_fmt(_sum_optional(native['encode_median_ms'], native['dit_median_ms']), ' ms')} |", + f"| Stage-1 DiT median | {_fmt(upstream['dit_median_ms'], ' ms')} | {_fmt(native['dit_median_ms'], ' ms')} |", + f"| conditioning/encode median | n/a | {_fmt(native['encode_median_ms'], ' ms')} |", + ] + ) + if summary["inputs"]["no_refiner"]: + # No refiner: FlashDreams `decode_ms` and upstream `vae_decode_s` are + # both the pure SANA VAE decode, so they compare directly. + rows.append( + f"| VAE decode median | {_fmt(upstream['vae_decode_median_ms'], ' ms')} | {_fmt(native['vae_decode_median_ms'], ' ms')} |" + ) + else: + # Refiner enabled: both sides bundle the refiner denoise together + # with its VAE decode into a single measurement (upstream + # `refiner_s`, FlashDreams `decode_ms`). They are only + # apples-to-apples as one combined row; the standalone upstream + # `vae_decode_s` is the Stage-1 decode and is NOT comparable to + # FlashDreams `decode_ms`. + rows.append( + f"| refiner + VAE decode median | {_fmt(upstream.get('refiner_median_ms'), ' ms')} | {_fmt(native['vae_decode_median_ms'], ' ms')} |" + ) + rows.append( + f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} | {_fmt(native['mem_peak_median_gib'], ' GiB')} |" + ) + else: + rows.extend( + [ + "| metric | upstream |", + "| --- | ---: |", + f"| measured runs | {upstream['runs_measured']} |", + f"| generation median / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream'), ' ms')} |", + f"| generation p90 / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream', percentile='p90'), ' ms')} |", + f"| generation median / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'upstream'), ' ms')} |", + f"| generation p90 / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'upstream', percentile='p90'), ' ms')} |", + f"| wall median | {_fmt(upstream['wall_median_s'], ' s')} |", + f"| wall p90 | {_fmt(upstream['wall_p90_s'], ' s')} |", + "", + "## Timing breakdown", + "", + "| stage | upstream median |", + "| --- | ---: |", + f"| Stage-1 incl. conditioning median | {_fmt(upstream['stage1_total_median_ms'], ' ms')} |", + f"| Stage-1 DiT median | {_fmt(upstream['dit_median_ms'], ' ms')} |", + f"| VAE decode median | {_fmt(upstream['vae_decode_median_ms'], ' ms')} |", + f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} |", + ] + ) + rows.append("") + return "\n".join(rows) + + +def _render_streaming_markdown(summary: dict[str, Any]) -> str: + upstream = summary["upstream"] + native = summary.get("flashdreams") + has_native = isinstance(native, dict) + title = ( + "# SANA-WM streaming benchmark" + if has_native + else "# SANA-WM streaming upstream benchmark" + ) + rows = [ + title, + "", + "## Inputs", + "", + f"- image: `{summary['inputs']['image_path']}`", + f"- prompt: `{summary['inputs']['prompt_path']}`", + f"- camera_source: `{summary['inputs'].get('camera_source')}`", + f"- camera: `{summary['inputs'].get('camera_path')}`", + f"- action: `{summary['inputs'].get('action')}`", + f"- intrinsics: `{summary['inputs']['intrinsics_path']}`", + f"- requested_num_frames: `{summary['inputs']['num_frames']}`", + f"- seed: `{summary['inputs']['seed']}`", + f"- stage1_precision: `{summary['inputs']['stage1_precision']}`", + f"- refiner_precision: `{summary['inputs']['refiner_precision']}`", + f"- warmup runs discarded: `{summary['inputs']['warmup_runs']}`", + f"- output_mode: `{summary['inputs'].get('output_mode')}`", + "", + "## Benchmark metric", "", - "## Timing breakdown", + "The headline metric is steady-state generation latency per produced chunk.", + "Warmup runs and the first decoded chunk are excluded so cold compile and cache-fill do not enter the headline.", + "Full-clip wall time is retained as supporting data.", "", - "| stage | upstream median | FlashDreams median |", - "| --- | ---: | ---: |", - f"| Stage-1 incl. conditioning median | {_fmt(upstream['stage1_total_median_ms'], ' ms')} | {_fmt(_sum_optional(native['encode_median_ms'], native['dit_median_ms']), ' ms')} |", - f"| Stage-1 DiT median | {_fmt(upstream['dit_median_ms'], ' ms')} | {_fmt(native['dit_median_ms'], ' ms')} |", - f"| conditioning/encode median | n/a | {_fmt(native['encode_median_ms'], ' ms')} |", ] - if summary["inputs"]["no_refiner"]: - # No refiner: FlashDreams `decode_ms` and upstream `vae_decode_s` are both - # the pure SANA VAE decode, so they compare directly. - rows.append( - f"| VAE decode median | {_fmt(upstream['vae_decode_median_ms'], ' ms')} | {_fmt(native['vae_decode_median_ms'], ' ms')} |" + if has_native: + rows.extend( + [ + "| metric | upstream | FlashDreams |", + "| --- | ---: | ---: |", + f"| measured runs | {upstream['runs_measured']} | {native['runs_measured']} |", + f"| steady-state generation median / chunk | {_fmt(_streaming_generation_ms_per_chunk(summary, 'upstream'), ' ms')} | {_fmt(_streaming_generation_ms_per_chunk(summary, 'flashdreams'), ' ms')} |", + f"| steady-state generation p90 / chunk | {_fmt(_streaming_generation_ms_per_chunk(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_streaming_generation_ms_per_chunk(summary, 'flashdreams', percentile='p90'), ' ms')} |", + f"| full-clip wall median | {_fmt(upstream['wall_median_s'], ' s')} | {_fmt(native['wall_median_s'], ' s')} |", + f"| first chunk median | {_fmt(upstream['first_chunk_median_s'], ' s')} | {_fmt(native['first_chunk_median_s'], ' s')} |", + f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} | {_fmt(native['mem_peak_median_gib'], ' GiB')} |", + ] ) else: - # Refiner enabled: both sides bundle the refiner denoise together with its - # VAE decode into a single measurement (upstream `refiner_s`, FlashDreams - # `decode_ms`). They are only apples-to-apples as one combined row; the - # standalone upstream `vae_decode_s` is the Stage-1 decode and is NOT - # comparable to FlashDreams `decode_ms`. - rows.append( - f"| refiner + VAE decode median | {_fmt(upstream.get('refiner_median_ms'), ' ms')} | {_fmt(native['vae_decode_median_ms'], ' ms')} |" + rows.extend( + [ + "| metric | upstream |", + "| --- | ---: |", + f"| measured runs | {upstream['runs_measured']} |", + f"| steady-state generation median / chunk | {_fmt(_streaming_generation_ms_per_chunk(summary, 'upstream'), ' ms')} |", + f"| steady-state generation p90 / chunk | {_fmt(_streaming_generation_ms_per_chunk(summary, 'upstream', percentile='p90'), ' ms')} |", + f"| full-clip wall median | {_fmt(upstream['wall_median_s'], ' s')} |", + f"| full-clip wall p90 | {_fmt(upstream['wall_p90_s'], ' s')} |", + f"| first chunk median | {_fmt(upstream['first_chunk_median_s'], ' s')} |", + f"| generated frames median | {_fmt(upstream['n_pixel_frames_median'])} |", + f"| decode chunks median | {_fmt(upstream['n_decode_chunks_median'])} |", + f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} |", + ] ) - rows.append( - f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} | {_fmt(native['mem_peak_median_gib'], ' GiB')} |" + rows.extend( + [ + "", + "## Timing breakdown", + "", + ] ) + if has_native: + rows.extend( + [ + "| stage | upstream median | FlashDreams median |", + "| --- | ---: | ---: |", + f"| Stage-1 CUDA total | {_fmt(upstream['stage1_cuda_median_ms'], ' ms')} | {_fmt(native['stage1_cuda_median_ms'], ' ms')} |", + f"| refiner CUDA total | {_fmt(upstream['refiner_cuda_median_ms'], ' ms')} | {_fmt(native['refiner_cuda_median_ms'], ' ms')} |", + f"| decode CUDA total | {_fmt(upstream['decode_cuda_median_ms'], ' ms')} | {_fmt(native['decode_cuda_median_ms'], ' ms')} |", + ] + ) + else: + rows.extend( + [ + "| stage | upstream median |", + "| --- | ---: |", + f"| Stage-1 CUDA total | {_fmt(upstream['stage1_cuda_median_ms'], ' ms')} |", + f"| refiner CUDA total | {_fmt(upstream['refiner_cuda_median_ms'], ' ms')} |", + f"| decode CUDA total | {_fmt(upstream['decode_cuda_median_ms'], ' ms')} |", + ] + ) rows.append("") return "\n".join(rows) +def _render_markdown(summary: dict[str, Any]) -> str: + if summary.get("variant") == "streaming" or summary["inputs"].get("variant") == "streaming": + return _render_streaming_markdown(summary) + return _render_bidirectional_markdown(summary) + + def _render_chart_markdown(summary: dict[str, Any], device_label: str) -> str: - official = _generation_ms_per_clip(summary, "upstream") - flashdreams = _generation_ms_per_clip(summary, "flashdreams") - if official is None or flashdreams is None: - raise ValueError("cannot render chart data without wall-clock stats") + streaming = summary.get("variant") == "streaming" or summary["inputs"].get("variant") == "streaming" + official = ( + _streaming_generation_ms_per_chunk(summary, "upstream") + if streaming + else _generation_ms_per_clip(summary, "upstream") + ) + flashdreams = ( + _streaming_generation_ms_per_chunk(summary, "flashdreams") + if streaming + else _generation_ms_per_clip(summary, "flashdreams") + ) + if official is None: + raise ValueError("cannot render chart data without upstream benchmark stats") + title = ( + "# SANA-WM Streaming Benchmark Data (ms/chunk)" + if streaming + else "# SANA-WM Benchmark Data (ms)" + ) + if flashdreams is None: + return "\n".join( + [ + title, + "", + "| device | official |", + "| --- | ---: |", + f"| {device_label} | {official:.2f} |", + "", + ] + ) return "\n".join( [ - "# SANA-WM Benchmark Data (ms)", + title, "", "| device | official | flashdreams |", "| --- | ---: | ---: |", @@ -255,12 +585,16 @@ def _render_chart_markdown(summary: dict[str, Any], device_label: str) -> str: def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--variant", choices=["bidirectional", "streaming"], default="bidirectional") + parser.add_argument("--bench-side", choices=["upstream", "both"], default="both") parser.add_argument("--upstream-dir", type=Path, required=True) - parser.add_argument("--flashdreams-dir", type=Path, required=True) + parser.add_argument("--flashdreams-dir", type=Path, default=None) parser.add_argument("--warmup-runs", type=int, default=1) parser.add_argument("--image-path", type=Path, required=True) parser.add_argument("--prompt-path", type=Path, required=True) parser.add_argument("--camera-path", type=Path, required=True) + parser.add_argument("--camera-source", choices=["camera", "action"], default="camera") + parser.add_argument("--action", default=None) parser.add_argument("--intrinsics-path", type=Path, required=True) parser.add_argument("--num-frames", type=int, required=True) parser.add_argument("--seed", type=int, required=True) @@ -276,21 +610,45 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--force-cudnn-sdpa", action="store_true") parser.add_argument("--device-label", default="GPU") parser.add_argument("--chart-label", default=None) + parser.add_argument("--upstream-commit", default=None) + parser.add_argument("--flashdreams-commit", default=None) + parser.add_argument("--output-mode", default=None) + parser.add_argument("--denoising-step-list", default=None) + parser.add_argument("--num-frame-per-block", type=int, default=None) + parser.add_argument("--refiner-block-size", type=int, default=None) + parser.add_argument("--refiner-kv-max-frames", type=int, default=None) parser.add_argument("--output-json", type=Path, required=True) parser.add_argument("--output-md", type=Path, required=True) parser.add_argument("--output-chart-md", type=Path, default=None) args = parser.parse_args(argv) + if args.bench_side == "both" and args.flashdreams_dir is None: + parser.error("--flashdreams-dir is required when --bench-side=both") + upstream_items = _load_stats(args.upstream_dir) - native_items = _load_stats(args.flashdreams_dir) + native_items = _load_stats(args.flashdreams_dir) if args.flashdreams_dir is not None else [] + if not upstream_items: + raise ValueError(f"no upstream stats found in {args.upstream_dir}") summary = { + "variant": args.variant, + "bench_side": args.bench_side, + "upstream_commit": args.upstream_commit, + "flashdreams_commit": args.flashdreams_commit, + "commands": { + "upstream": _first_command(upstream_items), + "flashdreams": _first_command(native_items), + }, "inputs": { "image_path": str(args.image_path), "prompt_path": str(args.prompt_path), "camera_path": str(args.camera_path), + "camera_source": args.camera_source, + "action": args.action, "intrinsics_path": str(args.intrinsics_path), "num_frames": args.num_frames, "seed": args.seed, + "variant": args.variant, + "bench_side": args.bench_side, "no_refiner": args.no_refiner, "stage1_precision": args.stage1_precision, "refiner_precision": args.refiner_precision, @@ -300,8 +658,37 @@ def main(argv: list[str] | None = None) -> None: "warmup_runs": args.warmup_runs, "device_label": args.device_label, "chart_label": args.chart_label or args.device_label, + "output_mode": args.output_mode, + "denoising_step_list": args.denoising_step_list, + "num_frame_per_block": args.num_frame_per_block, + "refiner_block_size": args.refiner_block_size, + "refiner_kv_max_frames": args.refiner_kv_max_frames, }, - "upstream": _collect( + } + if args.variant == "streaming": + summary["upstream"] = _collect_streaming(upstream_items, args.warmup_runs) + if args.bench_side == "both": + summary["flashdreams"] = _collect_streaming(native_items, args.warmup_runs) + summary["benchmark"] = { + "metric": "steady_state_generation_ms_per_chunk", + "unit": "ms", + "timing_boundary": ( + "streaming pipeline steady state after warmup runs and after the first " + "decoded chunk; excludes model construction and checkpoint loading" + ), + "device_label": args.device_label, + "chart_label": args.chart_label or args.device_label, + "official": _streaming_generation_ms_per_chunk(summary, "upstream"), + "official_p90": _streaming_generation_ms_per_chunk(summary, "upstream", percentile="p90"), + "flashdreams": _streaming_generation_ms_per_chunk(summary, "flashdreams") + if args.bench_side == "both" + else None, + "full_clip_wall_median_s": summary["upstream"]["wall_median_s"], + "variant": args.variant, + "precision": args.stage1_precision, + } + else: + summary["upstream"] = _collect( upstream_items, args.warmup_runs, _upstream_stage_ms, @@ -312,31 +699,33 @@ def main(argv: list[str] | None = None) -> None: "refiner": "refiner_s", "vae_decode": "vae_decode_s", }, - ), - "flashdreams": _collect( - native_items, - args.warmup_runs, - _native_stage_ms, - _native_mem_gib, - { - "encode": "encode_ms", - "dit": "diffuse_ms", - "vae_decode": "decode_ms", - }, - ), - } - summary["benchmark"] = { - "metric": "generation_ms_per_clip", - "unit": "ms", - "timing_boundary": ( - "pipeline.generate after model setup; excludes model construction, " - "checkpoint loading, video writing, and frame dumps" - ), - "device_label": args.device_label, - "chart_label": args.chart_label or args.device_label, - "official": _generation_ms_per_clip(summary, "upstream"), - "flashdreams": _generation_ms_per_clip(summary, "flashdreams"), - } + ) + if args.bench_side == "both": + summary["flashdreams"] = _collect( + native_items, + args.warmup_runs, + _native_stage_ms, + _native_mem_gib, + { + "encode": "encode_ms", + "dit": "diffuse_ms", + "vae_decode": "decode_ms", + }, + ) + summary["benchmark"] = { + "metric": "generation_ms_per_clip", + "unit": "ms", + "timing_boundary": ( + "pipeline.generate after model setup; excludes model construction, " + "checkpoint loading, video writing, and frame dumps" + ), + "device_label": args.device_label, + "chart_label": args.chart_label or args.device_label, + "official": _generation_ms_per_clip(summary, "upstream"), + "flashdreams": _generation_ms_per_clip(summary, "flashdreams") + if args.bench_side == "both" + else None, + } args.output_json.parent.mkdir(parents=True, exist_ok=True) args.output_json.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") args.output_md.parent.mkdir(parents=True, exist_ok=True) diff --git a/integrations/sana/tests/parity_check/bench_sweep_summary.py b/integrations/sana/tests/parity_check/bench_sweep_summary.py index 258b752ff..74f3c22c0 100644 --- a/integrations/sana/tests/parity_check/bench_sweep_summary.py +++ b/integrations/sana/tests/parity_check/bench_sweep_summary.py @@ -41,24 +41,44 @@ def _load_item(raw: str) -> dict[str, Any]: raise argparse.ArgumentTypeError(f"{path} has no benchmark object") official = benchmark.get("official") flashdreams = benchmark.get("flashdreams") - if not isinstance(official, (int, float)) or not isinstance( - flashdreams, (int, float) - ): + if not isinstance(official, (int, float)): raise argparse.ArgumentTypeError( - f"{path} benchmark must contain numeric official and flashdreams values" + f"{path} benchmark must contain a numeric official value" + ) + if flashdreams is not None and not isinstance(flashdreams, (int, float)): + raise argparse.ArgumentTypeError( + f"{path} benchmark flashdreams value must be numeric or null" ) return { "label": label, "path": str(path), "official": float(official), - "flashdreams": float(flashdreams), + "flashdreams": float(flashdreams) if isinstance(flashdreams, (int, float)) else None, + "metric": benchmark.get("metric", "generation_ms_per_clip"), + "unit": benchmark.get("unit", "ms"), + "variant": benchmark.get("variant") + or summary.get("variant") + or summary.get("inputs", {}).get("variant", "bidirectional"), "inputs": summary.get("inputs", {}), } def _render_chart(rows: list[dict[str, Any]]) -> str: + has_flashdreams = all(row["flashdreams"] is not None for row in rows) + unit_label = "ms/chunk" if rows and rows[0]["metric"] == "steady_state_generation_ms_per_chunk" else "ms" + if not has_flashdreams: + lines = [ + f"# SANA-WM Precision Sweep Upstream Summary ({unit_label})", + "", + "| precision | official |", + "| --- | ---: |", + ] + for row in rows: + lines.append(f"| {row['label']} | {row['official']:.2f} |") + lines.append("") + return "\n".join(lines) lines = [ - "# SANA-WM Precision Sweep Summary (ms)", + f"# SANA-WM Precision Sweep Summary ({unit_label})", "", "| precision | official | flashdreams |", "| --- | ---: | ---: |", @@ -72,20 +92,44 @@ def _render_chart(rows: list[dict[str, Any]]) -> str: def _render_report(rows: list[dict[str, Any]]) -> str: + has_flashdreams = all(row["flashdreams"] is not None for row in rows) + metric = rows[0]["metric"] if rows else "generation_ms_per_clip" + metric_text = ( + "steady-state generation latency per produced chunk" + if metric == "steady_state_generation_ms_per_chunk" + else "post-load generation latency per generated clip" + ) lines = [ "# SANA-WM precision benchmark sweep", "", - "The chart metric is post-load generation latency per generated clip.", + f"The chart metric is {metric_text}.", "Model-card chart data is grouped by GPU/device in each precision subdirectory's `perf.md`.", "", - "| precision | official | FlashDreams | source |", - "| --- | ---: | ---: | --- |", ] - for row in rows: - lines.append( - f"| {row['label']} | {row['official']:.2f} ms | " - f"{row['flashdreams']:.2f} ms | `{row['path']}` |" + if has_flashdreams: + lines.extend( + [ + "| precision | official | FlashDreams | source |", + "| --- | ---: | ---: | --- |", + ] + ) + else: + lines.extend( + [ + "| precision | official | source |", + "| --- | ---: | --- |", + ] ) + for row in rows: + if has_flashdreams: + lines.append( + f"| {row['label']} | {row['official']:.2f} ms | " + f"{row['flashdreams']:.2f} ms | `{row['path']}` |" + ) + else: + lines.append( + f"| {row['label']} | {row['official']:.2f} ms | `{row['path']}` |" + ) lines.append("") return "\n".join(lines) @@ -104,14 +148,18 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--output-chart-md", type=Path, required=True) args = parser.parse_args(argv) + metrics = {row["metric"] for row in args.item} + if len(metrics) != 1: + raise ValueError(f"cannot aggregate mixed benchmark metrics: {sorted(metrics)}") + units = {row["unit"] for row in args.item} + if len(units) != 1: + raise ValueError(f"cannot aggregate mixed benchmark units: {sorted(units)}") + has_flashdreams = all(row["flashdreams"] is not None for row in args.item) payload = { "benchmark": { - "metric": "generation_ms_per_clip", - "unit": "ms", - "timing_boundary": ( - "pipeline.generate after model setup; excludes model construction, " - "checkpoint loading, video writing, and frame dumps" - ), + "metric": args.item[0]["metric"], + "unit": args.item[0]["unit"], + "has_flashdreams": has_flashdreams, }, "rows": args.item, } diff --git a/integrations/sana/tests/parity_check/changes_streaming.patch b/integrations/sana/tests/parity_check/changes_streaming.patch new file mode 100644 index 000000000..e7ab9c605 --- /dev/null +++ b/integrations/sana/tests/parity_check/changes_streaming.patch @@ -0,0 +1,33 @@ +diff --git a/inference_video_scripts/wm/inference_sana_wm_streaming.py b/inference_video_scripts/wm/inference_sana_wm_streaming.py +index 2d61d4b..94f51c2 100644 +--- a/inference_video_scripts/wm/inference_sana_wm_streaming.py ++++ b/inference_video_scripts/wm/inference_sana_wm_streaming.py +@@ -523,6 +523,9 @@ def main() -> None: + if os.environ.get(name) is not None + } + payload = { ++ "backend": "upstream", ++ "variant": "streaming", ++ "entrypoint": "inference_video_scripts/wm/inference_sana_wm_streaming.py", + "output_mode": result["output_mode"], + "output_path": str(result["output_path"]) if result["output_path"] is not None else None, + "num_frames": int(num_frames), +@@ -559,6 +562,10 @@ def main() -> None: + "torch_compile": not bool(args.no_compile), + "profile_cuda": bool(args.profile_cuda), + "streaming_root": str(args.streaming_root), ++ "stage1_precision": args.stage1_precision, ++ "refiner_precision": args.refiner_precision, ++ "seed": int(args.seed), ++ "refiner_seed": int(args.refiner_seed), + "config": str(config_path), + "model_path": str(model_path), + "causal_vae_path": str(causal_vae_path), +@@ -574,6 +581,7 @@ def main() -> None: + "last_stream_wall_seconds": float(run_payloads[-1]["stream_wall_seconds"]), + "last_realtime_factor": float(run_payloads[-1]["realtime_factor"]), + "last_steady_state_realtime_factor": run_payloads[-1]["steady_state_realtime_factor"], ++ "mem_peak_gib": float(max(run["peak_mem_gb"] for run in run_payloads)), + } + args.benchmark_json.parent.mkdir(parents=True, exist_ok=True) + args.benchmark_json.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") diff --git a/integrations/sana/tests/parity_check/pyproject.toml b/integrations/sana/tests/parity_check/pyproject.toml index f92ac2738..eab2cbdd0 100644 --- a/integrations/sana/tests/parity_check/pyproject.toml +++ b/integrations/sana/tests/parity_check/pyproject.toml @@ -37,19 +37,20 @@ dependencies = [ [project.optional-dependencies] # Transformer Engine is only needed for the FP8/FP4 quant backends. It is gated # behind an opt-in extra so the default `uv sync` (BF16 benchmarking) does not -# try to build/import it. Install with `uv sync --extra quant` once a -# transformer-engine-torch build matching the local torch/CUDA ABI is available. +# try to build/import it. The git source avoids split-wheel ABI mismatches on +# bleeding-edge torch/CUDA stacks while keeping BF16 setup lightweight. quant = [ - "transformer-engine[pytorch,core-cu13]>=2.12; sys_platform != 'win32'", + "transformer-engine==2.17.0+2e559f06; sys_platform != 'win32'", ] [tool.uv.sources] flashdreams = { path = "../../../../flashdreams", editable = true } flashdreams-sana-wm = { path = "../..", editable = true } +transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", tag = "v2.17" } [tool.uv] managed = true override-dependencies = [ "nvidia-cublas>=13.4", ] -no-build-isolation-package = ["transformer-engine-torch"] +no-build-isolation-package = ["transformer-engine"] diff --git a/integrations/sana/tests/parity_check/uv.lock b/integrations/sana/tests/parity_check/uv.lock index 81c9f57a0..e982ddc53 100644 --- a/integrations/sana/tests/parity_check/uv.lock +++ b/integrations/sana/tests/parity_check/uv.lock @@ -387,6 +387,7 @@ dependencies = [ { name = "pillow" }, { name = "pyyaml" }, { name = "safetensors" }, + { name = "sentencepiece" }, { name = "torchvision" }, { name = "transformers" }, ] @@ -401,6 +402,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "safetensors", specifier = ">=0.5" }, + { name = "sentencepiece", specifier = ">=0.2" }, { name = "torchvision", specifier = ">=0.26" }, { name = "transformers", specifier = ">=5.0,<6" }, ] @@ -1245,7 +1247,7 @@ dependencies = [ [package.optional-dependencies] quant = [ - { name = "transformer-engine", extra = ["core-cu13", "pytorch"], marker = "sys_platform != 'win32'" }, + { name = "transformer-engine", marker = "sys_platform != 'win32'" }, ] [package.metadata] @@ -1276,7 +1278,7 @@ requires-dist = [ { name = "torch", specifier = ">=2.11" }, { name = "torchvision", specifier = ">=0.26" }, { name = "tqdm", specifier = ">=4.60" }, - { name = "transformer-engine", extras = ["pytorch", "core-cu13"], marker = "sys_platform != 'win32' and extra == 'quant'", specifier = ">=2.12" }, + { name = "transformer-engine", marker = "sys_platform != 'win32' and extra == 'quant'", git = "https://github.com/NVIDIA/TransformerEngine.git?tag=v2.17" }, { name = "transformers", specifier = ">=5.0,<6" }, { name = "triton", marker = "sys_platform == 'linux'", specifier = ">=3.6" }, ] @@ -1489,48 +1491,18 @@ wheels = [ [[package]] name = "transformer-engine" -version = "2.17.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/26/e501a474a3f36c2d439561fa4f2094a3c0344df439500dda55937a8fccf7/transformer_engine-2.17.0-py3-none-any.whl", hash = "sha256:c274fd74ea2e4caa7132921e1ecfa3847931abbed9f6b8cab61346cdb833bc76", size = 1001710, upload-time = "2026-07-09T00:36:15.453Z" }, -] - -[package.optional-dependencies] -core-cu13 = [ - { name = "transformer-engine-cu13" }, -] -pytorch = [ - { name = "transformer-engine-torch" }, -] - -[[package]] -name = "transformer-engine-cu13" -version = "2.17.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "importlib-metadata" }, - { name = "packaging" }, - { name = "pydantic" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/21/5e0ec562539798ffd375c218b8bd7612b387455ccc79ae14576f411c9609/transformer_engine_cu13-2.17.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:1c8e6627cf02358201f513ab902c7b4f82f179214bd775edc97ce0f2001d82da", size = 244570183, upload-time = "2026-07-09T00:36:35.507Z" }, -] - -[[package]] -name = "transformer-engine-torch" -version = "2.17.0" -source = { registry = "https://pypi.org/simple" } +version = "2.17.0+2e559f06" +source = { git = "https://github.com/NVIDIA/TransformerEngine.git?tag=v2.17#2e559f062497bef768dfbe9d7e45548fadeca80a" } dependencies = [ { name = "einops" }, + { name = "importlib-metadata" }, { name = "nvdlfw-inspect" }, { name = "onnx" }, { name = "onnxscript" }, { name = "packaging" }, { name = "pydantic" }, { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, - { name = "transformer-engine-cu13" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/f1/ad613157261ac71262f5f260acf07fe22ae5ee0bb0675622c4f19a3e8afb/transformer_engine_torch-2.17.0.tar.gz", hash = "sha256:52ca109cdbc987ca02f87735fa375da8a02e117acb87e25dcb7e27dca37aa87f", size = 369712, upload-time = "2026-07-09T00:36:28.065Z" } [[package]] name = "transformers" diff --git a/integrations/sana/tests/test_parity_benchmark_summary.py b/integrations/sana/tests/test_parity_benchmark_summary.py index 2a10e18b6..479b64e61 100644 --- a/integrations/sana/tests/test_parity_benchmark_summary.py +++ b/integrations/sana/tests/test_parity_benchmark_summary.py @@ -219,6 +219,122 @@ def test_benchmark_summary_keeps_frame_normalized_diagnostics() -> None: ) == 4000.0 +def test_streaming_upstream_summary_uses_steady_state_ms_per_chunk(tmp_path: Path) -> None: + module = _load_bench_summary() + warmup = tmp_path / "upstream" / "run_0" + measured = tmp_path / "upstream" / "run_1" + warmup.mkdir(parents=True) + measured.mkdir(parents=True) + (warmup / "stats.json").write_text( + json.dumps( + { + "variant": "streaming", + "stream_wall_seconds": 12.0, + "steady_state_seconds": 8.0, + "first_chunk_seconds": 4.0, + "n_decode_chunks": 5, + "n_pixel_frames": 97, + "peak_mem_gb": 40.0, + "output_path": "warmup.mp4", + } + ), + encoding="utf-8", + ) + (measured / "stats.json").write_text( + json.dumps( + { + "variant": "streaming", + "stream_wall_seconds": 10.0, + "end_to_end_seconds": 10.2, + "steady_state_seconds": 6.0, + "first_chunk_seconds": 4.0, + "first_chunk_frames": 25, + "n_decode_chunks": 4, + "n_refiner_blocks": 4, + "n_pixel_frames": 97, + "steady_state_frames_per_second": 12.0, + "steady_state_realtime_factor": 0.75, + "stage1_cuda_seconds": 2.0, + "refiner_cuda_seconds": 3.0, + "decode_cuda_seconds": 1.0, + "peak_mem_gb": 48.0, + "output_path": "measured.mp4", + } + ), + encoding="utf-8", + ) + (measured / "command.txt").write_text("uv run python upstream.py\n", encoding="utf-8") + out_json = tmp_path / "bench.json" + out_md = tmp_path / "bench.md" + + module.main( + [ + "--variant", + "streaming", + "--bench-side", + "upstream", + "--upstream-dir", + str(tmp_path / "upstream"), + "--warmup-runs", + "1", + "--image-path", + "image.png", + "--prompt-path", + "prompt.txt", + "--camera-path", + "pose.npy", + "--camera-source", + "action", + "--action", + "w-80", + "--intrinsics-path", + "intrinsics.npy", + "--num-frames", + "97", + "--seed", + "42", + "--stage1-precision", + "bf16", + "--refiner-precision", + "bf16", + "--device-label", + "GB202", + "--upstream-commit", + "6298508", + "--output-mode", + "mp4", + "--denoising-step-list", + "1000,960,889,727,0", + "--num-frame-per-block", + "3", + "--refiner-block-size", + "3", + "--refiner-kv-max-frames", + "11", + "--output-json", + str(out_json), + "--output-md", + str(out_md), + ] + ) + + summary = json.loads(out_json.read_text(encoding="utf-8")) + assert summary["variant"] == "streaming" + assert summary["bench_side"] == "upstream" + assert summary["commands"]["upstream"] == "uv run python upstream.py" + assert summary["upstream"]["steady_state_chunk_median_ms"] == 2000.0 + assert summary["upstream"]["wall_median_s"] == 10.0 + assert summary["upstream"]["mem_peak_median_gib"] == 48.0 + assert summary["upstream"]["stage1_cuda_median_ms"] == 2000.0 + assert summary["benchmark"]["metric"] == "steady_state_generation_ms_per_chunk" + assert summary["benchmark"]["official"] == 2000.0 + assert summary["benchmark"]["flashdreams"] is None + report = out_md.read_text(encoding="utf-8") + assert "# SANA-WM streaming upstream benchmark" in report + assert "steady-state generation median / chunk | 2000.00 ms" in report + assert "full-clip wall median | 10.00 s" in report + + def test_benchmark_sweep_summary_writes_precision_chart_data(tmp_path: Path) -> None: module = _load_bench_sweep_summary() bf16_json = tmp_path / "bf16.json" @@ -262,3 +378,68 @@ def test_benchmark_sweep_summary_writes_precision_chart_data(tmp_path: Path) -> assert payload["benchmark"]["metric"] == "generation_ms_per_clip" assert payload["benchmark"]["unit"] == "ms" assert [row["label"] for row in payload["rows"]] == ["BF16", "FP8"] + + +def test_benchmark_sweep_summary_accepts_streaming_upstream_only(tmp_path: Path) -> None: + module = _load_bench_sweep_summary() + bf16_json = tmp_path / "bf16.json" + fp8_json = tmp_path / "fp8.json" + bf16_json.write_text( + json.dumps( + { + "variant": "streaming", + "benchmark": { + "metric": "steady_state_generation_ms_per_chunk", + "unit": "ms", + "official": 1200.0, + "flashdreams": None, + }, + } + ), + encoding="utf-8", + ) + fp8_json.write_text( + json.dumps( + { + "variant": "streaming", + "benchmark": { + "metric": "steady_state_generation_ms_per_chunk", + "unit": "ms", + "official": 900.0, + "flashdreams": None, + }, + } + ), + encoding="utf-8", + ) + out_json = tmp_path / "bench.json" + out_md = tmp_path / "bench.md" + out_chart = tmp_path / "perf.md" + + module.main( + [ + "--item", + f"BF16:{bf16_json}", + "--item", + f"FP8:{fp8_json}", + "--output-json", + str(out_json), + "--output-md", + str(out_md), + "--output-chart-md", + str(out_chart), + ] + ) + + assert out_chart.read_text(encoding="utf-8") == ( + "# SANA-WM Precision Sweep Upstream Summary (ms/chunk)\n" + "\n" + "| precision | official |\n" + "| --- | ---: |\n" + "| BF16 | 1200.00 |\n" + "| FP8 | 900.00 |\n" + ) + payload = json.loads(out_json.read_text(encoding="utf-8")) + assert payload["benchmark"]["metric"] == "steady_state_generation_ms_per_chunk" + assert payload["benchmark"]["has_flashdreams"] is False + assert [row["flashdreams"] for row in payload["rows"]] == [None, None] From 0091ce1cd875a611eb1885dbf0b525ca0c05c4c1 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Wed, 29 Jul 2026 15:53:20 -0700 Subject: [PATCH 38/64] Add integration for SANA-WM_streaming variant --- integrations/sana/README.md | 54 ++- integrations/sana/pyproject.toml | 2 + integrations/sana/sana_wm/__init__.py | 9 +- integrations/sana/sana_wm/conditioning.py | 228 +++++++++++ integrations/sana/sana_wm/config.py | 46 ++- integrations/sana/sana_wm/constants.py | 42 +- integrations/sana/sana_wm/decoder.py | 212 +++++++++- integrations/sana/sana_wm/runner.py | 259 ++++++++++++- integrations/sana/sana_wm/scheduler.py | 44 +++ integrations/sana/sana_wm/stage1_model.py | 4 + integrations/sana/sana_wm/transformer.py | 448 +++++++++++++++++++++- integrations/sana/tests/test_smoke.py | 388 ++++++++++++++++++- uv.lock | 59 +++ 13 files changed, 1762 insertions(+), 33 deletions(-) diff --git a/integrations/sana/README.md b/integrations/sana/README.md index 2690a0e4e..335b44460 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -5,33 +5,45 @@ SPDX-License-Identifier: Apache-2.0 # `sana_wm` -FlashDreams SANA-WM integration for -[SANA-WM](https://huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional), -the 2.6B bidirectional camera-controlled world model released from NVlabs/Sana. - -The `sana-wm-bidirectional` runner uses FlashDreams config, runner, pipeline, -diffusion-model, scheduler, transformer, camera-conditioning, VAE decode, and -output-writing boundaries. The Stage-1 DiT module in this package loads the -public SANA-WM checkpoint directly. +FlashDreams SANA-WM integration for the +[SANA-WM bidirectional](https://huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional) +and +[SANA-WM streaming](https://huggingface.co/Efficient-Large-Model/SANA-WM_streaming) +camera-controlled world model releases from NVlabs/Sana. + +The `sana-wm-bidirectional` and `sana-wm-streaming` runners use FlashDreams +config, runner, pipeline, diffusion-model, scheduler, transformer, +camera-conditioning, VAE decode, and output-writing boundaries. The Stage-1 DiT +module in this package loads the public SANA-WM checkpoints directly. Streaming +uses a FlashDreams-owned config literal instead of vendoring the upstream YAML. Current scope: | component | status | | --- | --- | | Stage-1 BF16 | FlashDreams DiT execution. | +| Streaming Stage-1 BF16 | Chunked FlashDreams DiT execution with prefix recomputation. | | Stage-1 FP8 | PyTorch `_scaled_mm` backend. | | Stage-1 FP4 | Triton quantization plus PyTorch `_scaled_mm`. | | VAE decode | Direct `diffusers` LTX2 VAE use with upstream-matched tiling. | +| Streaming VAE decode | Direct causal LTX2 VAE use with prefix decode and new-frame emission. | | LTX-2 refiner | Direct `diffusers` LTX-2 transformer and Gemma connector use. | +| Streaming LTX-2 refiner | Chunk-causal refiner config with FlashDreams prefix refinement. | ## Runner | slug | description | | --- | --- | | `sana-wm-bidirectional` | SANA-WM Stage-1 + LTX-2 refiner runner. | +| `sana-wm-streaming` | SANA-WM streaming Stage-1 + streaming LTX-2 refiner/VAE runner. | The FlashDreams package is named `sana_wm`. +This integration does not use TransformerEngine at runtime. Its FP8/FP4 paths +use Torch/Triton replacement layers; avoid selecting the root `flashdreams` +`dev` extra when testing this package, because that extra contains +TransformerEngine for unrelated core parity tests. + ## Setup Install FlashDreams and the SANA integration into an environment with the @@ -47,6 +59,8 @@ well. ## Run +Bidirectional: + ```bash uv run flashdreams-run sana-wm-bidirectional \ --image-path ../Sana/asset/sana_wm/demo_0.png \ @@ -63,6 +77,28 @@ Expected output: outputs/sana_wm_bf16/sana-wm-bidirectional.mp4 ``` +Streaming: + +```bash +uv run flashdreams-run sana-wm-streaming \ + --image-path ../Sana/asset/sana_wm/demo_0.png \ + --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ + --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ + --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --num-frames 241 \ + --output-dir outputs/sana_wm_streaming_bf16 +``` + +Expected output: + +```text +outputs/sana_wm_streaming_bf16/sana-wm-streaming.mp4 +``` + +The streaming runner defaults to 3 latent frames per block and the distilled +schedule `[1000, 960, 889, 727, 0]`. Requested frame counts are snapped to +`8 * --num-frame-per-block * k + 1` before inference. + `--intrinsics-path` is optional. When omitted, intrinsics are derived from the first-frame size assuming a centered principal point and a horizontal field of view of `--intrinsics-hfov-deg` (default `90`, matching the demo intrinsics). @@ -164,7 +200,7 @@ camera conditioning, Stage-1 checkpoint schema, Stage-1 CPU forward shape, VAE tiling, and low-precision backend selection: ```bash -uv run --extra dev pytest integrations/sana/tests/test_smoke.py +uv run --package flashdreams-sana-wm --extra dev pytest integrations/sana/tests/test_smoke.py ``` GPU generation checks are heavyweight manual workflows and should stay out of diff --git a/integrations/sana/pyproject.toml b/integrations/sana/pyproject.toml index 2d63b6f77..861d69b9a 100644 --- a/integrations/sana/pyproject.toml +++ b/integrations/sana/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "Pillow>=10", "PyYAML>=6.0", "safetensors>=0.5", + "sentencepiece>=0.2", "torchvision>=0.26", "transformers>=5.0,<6", ] @@ -45,6 +46,7 @@ dev = [ [project.entry-points."flashdreams.runner_configs"] "sana-wm-bidirectional" = "sana_wm.config:RUNNER_SANA_WM_BIDIRECTIONAL" +"sana-wm-streaming" = "sana_wm.config:RUNNER_SANA_WM_STREAMING" [tool.setuptools.packages.find] include = ["sana_wm*"] diff --git a/integrations/sana/sana_wm/__init__.py b/integrations/sana/sana_wm/__init__.py index c59615ca6..5fd160d95 100644 --- a/integrations/sana/sana_wm/__init__.py +++ b/integrations/sana/sana_wm/__init__.py @@ -13,11 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""SANA-WM runner for FlashDreams.""" +"""SANA-WM runners for FlashDreams.""" -from sana_wm.config import RUNNER_CONFIGS, RUNNER_SANA_WM_BIDIRECTIONAL +from sana_wm.config import ( + RUNNER_CONFIGS, + RUNNER_SANA_WM_BIDIRECTIONAL, + RUNNER_SANA_WM_STREAMING, +) __all__ = [ "RUNNER_CONFIGS", "RUNNER_SANA_WM_BIDIRECTIONAL", + "RUNNER_SANA_WM_STREAMING", ] diff --git a/integrations/sana/sana_wm/conditioning.py b/integrations/sana/sana_wm/conditioning.py index 6be23039b..9f04dd9a3 100644 --- a/integrations/sana/sana_wm/conditioning.py +++ b/integrations/sana/sana_wm/conditioning.py @@ -38,8 +38,11 @@ DEFAULT_VIDEO_HEIGHT, DEFAULT_VIDEO_WIDTH, SANA_WM_CONFIG_PATH, + SANA_WM_STREAMING_CONFIG_PATH, + SANA_WM_STREAMING_LATENT_CHUNK_SIZE, ) from sana_wm.transformer import ( + SanaWMStreamingStage1Conditioning, QuantBackend, SanaWMStage1Conditioning, _chunk_index_from_config, @@ -103,6 +106,29 @@ class SanaWMI2VConditioningRequest: negative_prompt: str = "" +@dataclass(kw_only=True) +class SanaWMStreamingI2VConditioningRequest(SanaWMI2VConditioningRequest): + """Raw SANA-WM streaming rollout inputs.""" + + num_frame_per_block: int = SANA_WM_STREAMING_LATENT_CHUNK_SIZE + """Latent frames generated per steady-state AR block.""" + + +@dataclass(kw_only=True) +class SanaWMStreamingRolloutState: + """Cached full-rollout conditioning shared by every streaming AR step.""" + + text: SanaWMTextConditioning + first_latent: Tensor + model_kwargs: dict[str, object] + total_latent_shape: tuple[int, int, int, int, int] + chunk_boundaries: tuple[int, ...] + flow_shift: float + steps: int + seed: int + cfg_scale: float + + @dataclass(kw_only=True) class SanaWMTextPromptEncoderConfig(EncoderConfig): """Config for the Stage-1 prompt encoder component.""" @@ -528,6 +554,203 @@ def _resolve_flow_shift(self, override: float | None) -> float: return float(cfg.scheduler.flow_shift) +@dataclass(kw_only=True) +class SanaWMStreamingConditioningEncoderCache(StreamingEncoderCache): + """Per-rollout cache for streaming SANA-WM conditioning.""" + + rollout: SanaWMStreamingRolloutState | None = None + + +@dataclass(kw_only=True) +class SanaWMStreamingConditioningEncoderConfig(SanaWMConditioningEncoderConfig): + """Config for the pipeline-level streaming SANA-WM conditioning encoder.""" + + _target: type["SanaWMStreamingConditioningEncoder"] = field( + default_factory=lambda: SanaWMStreamingConditioningEncoder + ) + + config_path: str = SANA_WM_STREAMING_CONFIG_PATH + """SANA-WM streaming inference config path or built-in identifier.""" + + text_encoder: SanaWMTextPromptEncoderConfig = field( + default_factory=lambda: SanaWMTextPromptEncoderConfig( + config_path=SANA_WM_STREAMING_CONFIG_PATH + ) + ) + first_frame_encoder: SanaWMFirstFrameEncoderConfig = field( + default_factory=lambda: SanaWMFirstFrameEncoderConfig( + config_path=SANA_WM_STREAMING_CONFIG_PATH + ) + ) + + +class SanaWMStreamingConditioningEncoder(SanaWMConditioningEncoder): + """Prepare SANA-WM streaming rollout conditioning once, then slice AR chunks.""" + + config: SanaWMStreamingConditioningEncoderConfig + + def initialize_autoregressive_cache( + self, + **_context: Any, + ) -> SanaWMStreamingConditioningEncoderCache: + """Return an initially empty streaming conditioning cache.""" + return SanaWMStreamingConditioningEncoderCache() + + @torch.inference_mode() + def forward( + self, + input: SanaWMStreamingI2VConditioningRequest, + autoregressive_index: int = 0, + cache: SanaWMStreamingConditioningEncoderCache | None = None, + ) -> SanaWMStreamingStage1Conditioning: + """Return the Stage-1 conditioning payload for one streaming chunk.""" + if cache is None: + cache = SanaWMStreamingConditioningEncoderCache() + if cache.rollout is None: + if autoregressive_index != 0: + raise ValueError( + "SANA-WM streaming conditioning must start at AR step 0." + ) + cache.rollout = self._build_rollout(input) + + rollout = cache.rollout + chunk_count = len(rollout.chunk_boundaries) - 1 + if autoregressive_index < 0 or autoregressive_index >= chunk_count: + raise ValueError( + f"autoregressive_index={autoregressive_index} is outside the " + f"streaming rollout with {chunk_count} chunks." + ) + start = rollout.chunk_boundaries[autoregressive_index] + end = rollout.chunk_boundaries[autoregressive_index + 1] + latent_shape = ( + rollout.total_latent_shape[0], + rollout.total_latent_shape[1], + end - start, + rollout.total_latent_shape[3], + rollout.total_latent_shape[4], + ) + model_kwargs = dict(rollout.model_kwargs) + data_info = model_kwargs.get("data_info") + if isinstance(data_info, dict): + model_kwargs["data_info"] = dict(data_info) + + return SanaWMStreamingStage1Conditioning( + condition=rollout.text.condition, + uncondition=( + rollout.text.negative if rollout.cfg_scale > 1.0 else None + ), + model_kwargs=model_kwargs, + first_latent=rollout.first_latent, + latent_shape=latent_shape, + cfg_scale=rollout.cfg_scale, + flow_shift=rollout.flow_shift, + steps=rollout.steps, + seed=rollout.seed, + total_latent_shape=rollout.total_latent_shape, + start_frame=start, + end_frame=end, + chunk_index=autoregressive_index, + chunk_boundaries=rollout.chunk_boundaries, + ) + + def _build_rollout( + self, + input: SanaWMStreamingI2VConditioningRequest, + ) -> SanaWMStreamingRolloutState: + cfg = self._ensure_runtime_config() + text = self.text_encoder( + SanaWMTextPromptRequest( + prompt=input.prompt, + negative_prompt=input.negative_prompt, + ) + ) + first_latent = self.first_frame_encoder(input.image) + weight_dtype = _get_weight_dtype(cfg.model.mixed_precision) + camera = self.camera_encoder( + SanaWMCameraRequest( + poses_c2w=input.poses_c2w, + intrinsics_vec4=input.intrinsics_vec4, + ) + ) + raymap = camera["raymap"].unsqueeze(0).to( + first_latent.device, + dtype=weight_dtype, + ) + chunk_plucker = camera["chunk_plucker"].unsqueeze(0).to( + first_latent.device, + dtype=weight_dtype, + ) + + vae_stride = cfg.vae.vae_stride + latent_t = (input.num_frames - 1) // int(vae_stride[0]) + 1 + latent_h = self.config.height // int(vae_stride[-1]) + latent_w = self.config.width // int(vae_stride[-1]) + chunk_boundaries = streaming_chunk_boundaries( + latent_t, + int(input.num_frame_per_block), + ) + + model_kwargs_extra: dict[str, object] = {} + if input.cfg_scale > 1.0: + model_kwargs_extra["negative_mask"] = text.negative_mask + + chunk_index = _chunk_index_from_config(cfg, num_frames=latent_t) + model_kwargs: dict[str, object] = { + "data_info": { + "img_hw": torch.tensor( + [[self.config.height, self.config.width]], + dtype=torch.float, + device=first_latent.device, + ), + "condition_frame_info": {0: 0.0}, + }, + "mask": text.condition_mask, + "camera_conditions": raymap, + "chunk_plucker": chunk_plucker, + **model_kwargs_extra, + } + if chunk_index is not None: + model_kwargs["chunk_index"] = chunk_index + + return SanaWMStreamingRolloutState( + text=text, + first_latent=first_latent, + model_kwargs=model_kwargs, + total_latent_shape=( + 1, + int(first_latent.shape[1]), + latent_t, + latent_h, + latent_w, + ), + chunk_boundaries=chunk_boundaries, + flow_shift=self._resolve_flow_shift(input.flow_shift), + steps=int(input.steps), + seed=int(input.seed), + cfg_scale=float(input.cfg_scale), + ) + + +def streaming_chunk_boundaries(total_frames: int, chunk_size: int) -> tuple[int, ...]: + """Return SANA-WM streaming latent-frame AR chunk boundaries.""" + if total_frames <= 1: + raise ValueError( + f"SANA-WM streaming requires more than one latent frame; got {total_frames}." + ) + if chunk_size <= 0: + raise ValueError(f"chunk_size must be > 0, got {chunk_size}.") + active = total_frames - 1 + if active % chunk_size != 0: + raise ValueError( + "SANA-WM streaming active latent frames must divide the chunk size: " + f"active={active}, chunk_size={chunk_size}." + ) + boundaries = [0, 1 + chunk_size] + while boundaries[-1] < total_frames: + boundaries.append(boundaries[-1] + chunk_size) + return tuple(min(boundary, total_frames) for boundary in boundaries) + + def _module_device(module: nn.Module) -> torch.device: try: return next(module.parameters()).device @@ -545,6 +768,11 @@ def _module_device(module: nn.Module) -> torch.device: "SanaWMFirstFrameEncoderConfig", "SanaWMI2VConditioning", "SanaWMI2VConditioningRequest", + "SanaWMStreamingConditioningEncoder", + "SanaWMStreamingConditioningEncoderCache", + "SanaWMStreamingConditioningEncoderConfig", + "SanaWMStreamingI2VConditioningRequest", + "streaming_chunk_boundaries", "SanaWMTextConditioning", "SanaWMTextPromptEncoder", "SanaWMTextPromptEncoderConfig", diff --git a/integrations/sana/sana_wm/config.py b/integrations/sana/sana_wm/config.py index e08b2c679..9c2cd3951 100644 --- a/integrations/sana/sana_wm/config.py +++ b/integrations/sana/sana_wm/config.py @@ -19,12 +19,16 @@ from flashdreams.infra.pipeline import StreamInferencePipelineConfig from flashdreams.infra.runner import RunnerConfig -from sana_wm.conditioning import SanaWMConditioningEncoderConfig -from sana_wm.decoder import SanaWMVideoDecoderConfig +from sana_wm.conditioning import ( + SanaWMConditioningEncoderConfig, + SanaWMStreamingConditioningEncoderConfig, +) +from sana_wm.constants import DEFAULT_STREAMING_DENOISING_STEP_LIST +from sana_wm.decoder import SanaWMStreamingVideoDecoderConfig, SanaWMVideoDecoderConfig from sana_wm.diffusion import SanaWMDiffusionModelConfig -from sana_wm.runner import SanaWMRunnerConfig +from sana_wm.runner import SanaWMRunnerConfig, SanaWMStreamingRunnerConfig from sana_wm.scheduler import SanaWMLTXEulerSchedulerConfig -from sana_wm.transformer import SanaWMTransformerConfig +from sana_wm.transformer import SanaWMStreamingTransformerConfig, SanaWMTransformerConfig PIPELINE_SANA_WM_BIDIRECTIONAL = StreamInferencePipelineConfig( @@ -39,6 +43,22 @@ ) """FlashDreams SANA-WM pipeline.""" +PIPELINE_SANA_WM_STREAMING = StreamInferencePipelineConfig( + name="sana-wm-streaming", + encoder=SanaWMStreamingConditioningEncoderConfig(), + diffusion_model=SanaWMDiffusionModelConfig( + transformer=SanaWMStreamingTransformerConfig(), + scheduler=SanaWMLTXEulerSchedulerConfig( + num_inference_steps=len(DEFAULT_STREAMING_DENOISING_STEP_LIST) - 1, + shift=8.0, + denoising_step_list=DEFAULT_STREAMING_DENOISING_STEP_LIST, + ), + seed=42, + ), + decoder=SanaWMStreamingVideoDecoderConfig(), +) +"""FlashDreams SANA-WM streaming pipeline.""" + RUNNER_SANA_WM_BIDIRECTIONAL = SanaWMRunnerConfig( runner_name=PIPELINE_SANA_WM_BIDIRECTIONAL.name, description="SANA-WM bidirectional I2V runner (Stage-1 DiT + LTX-2 refiner).", @@ -46,13 +66,29 @@ ) """SANA-WM runner config.""" +RUNNER_SANA_WM_STREAMING = SanaWMStreamingRunnerConfig( + runner_name=PIPELINE_SANA_WM_STREAMING.name, + description=( + "SANA-WM streaming I2V runner (chunk-causal Stage-1 + streaming " + "LTX-2 refiner/VAE path)." + ), + pipeline=PIPELINE_SANA_WM_STREAMING, +) +"""SANA-WM streaming runner config.""" + RUNNER_CONFIGS: dict[str, RunnerConfig] = { - cfg.runner_name: cfg for cfg in (RUNNER_SANA_WM_BIDIRECTIONAL,) + cfg.runner_name: cfg + for cfg in ( + RUNNER_SANA_WM_BIDIRECTIONAL, + RUNNER_SANA_WM_STREAMING, + ) } """SANA-WM runner configs keyed by ``runner_name``.""" __all__ = [ "PIPELINE_SANA_WM_BIDIRECTIONAL", + "PIPELINE_SANA_WM_STREAMING", "RUNNER_CONFIGS", "RUNNER_SANA_WM_BIDIRECTIONAL", + "RUNNER_SANA_WM_STREAMING", ] diff --git a/integrations/sana/sana_wm/constants.py b/integrations/sana/sana_wm/constants.py index d45f0c974..9192f8654 100644 --- a/integrations/sana/sana_wm/constants.py +++ b/integrations/sana/sana_wm/constants.py @@ -13,28 +13,52 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Constants for the public SANA-WM bidirectional release.""" +"""Constants for the public SANA-WM releases.""" SANA_WM_HF_REPO = "Efficient-Large-Model/SANA-WM_bidirectional" """Hugging Face repository containing SANA-WM bidirectional artefacts.""" +SANA_WM_STREAMING_HF_REPO = "Efficient-Large-Model/SANA-WM_streaming" +"""Hugging Face repository containing SANA-WM streaming artefacts.""" + SANA_WM_MODEL_PATH = f"hf://{SANA_WM_HF_REPO}/dit/sana_wm_1600m_720p.safetensors" """Default Stage-1 SANA-WM DiT checkpoint.""" +SANA_WM_STREAMING_MODEL_PATH = f"hf://{SANA_WM_STREAMING_HF_REPO}/sana_dit/model.pt" +"""Default streaming Stage-1 SANA-WM DiT checkpoint.""" + SANA_WM_CONFIG_PATH = f"hf://{SANA_WM_HF_REPO}/config.yaml" """Default inference YAML.""" +SANA_WM_STREAMING_CONFIG_PATH = "flashdreams://sana-wm-streaming-1600m-720p" +"""Built-in FlashDreams config identifier for the streaming inference YAML.""" + SANA_WM_REFINER_ROOT = f"hf://{SANA_WM_HF_REPO}/refiner" """Default LTX-2 refiner root.""" SANA_WM_REFINER_GEMMA_ROOT = f"hf://{SANA_WM_HF_REPO}/refiner/text_encoder" """Default Gemma text-encoder root used by the refiner.""" +SANA_WM_STREAMING_CAUSAL_VAE_ROOT = ( + f"hf://{SANA_WM_STREAMING_HF_REPO}/ltx2_causal_vae" +) +"""Default causal LTX-2 VAE root used by streaming SANA-WM.""" + +SANA_WM_STREAMING_REFINER_ROOT = ( + f"hf://{SANA_WM_STREAMING_HF_REPO}/refiner_diffusers" +) +"""Default chunk-causal LTX-2 refiner root used by streaming SANA-WM.""" + +SANA_WM_STREAMING_REFINER_GEMMA_ROOT = ( + f"hf://{SANA_WM_STREAMING_HF_REPO}/gemma3_12b" +) +"""Default Gemma-3 text-encoder root used by the streaming refiner.""" + DEFAULT_VIDEO_HEIGHT = 704 -"""SANA-WM bidirectional output height in pixels.""" +"""SANA-WM output height in pixels.""" DEFAULT_VIDEO_WIDTH = 1280 -"""SANA-WM bidirectional output width in pixels.""" +"""SANA-WM output width in pixels.""" SANA_WM_VAE_TEMPORAL_COMPRESSION = 8 """Temporal compression ratio of the LTX2 VAE used by SANA-WM.""" @@ -47,3 +71,15 @@ DEFAULT_ACTION = "w-100,dw-60,w-100,aw-60" """Default SANA-WM demo action string.""" + +DEFAULT_STREAMING_NUM_FRAMES = 241 +"""Default streaming output frame count.""" + +SANA_WM_STREAMING_LATENT_CHUNK_SIZE = 3 +"""Default number of latent frames generated per streaming AR block.""" + +DEFAULT_STREAMING_DENOISING_STEP_LIST = (1000, 960, 889, 727, 0) +"""Default distilled Stage-1 timestep schedule for streaming SANA-WM.""" + +SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES = 11 +"""Default refiner sliding-window size in latent frames.""" diff --git a/integrations/sana/sana_wm/decoder.py b/integrations/sana/sana_wm/decoder.py index 38fcf5794..9343e7c64 100644 --- a/integrations/sana/sana_wm/decoder.py +++ b/integrations/sana/sana_wm/decoder.py @@ -41,6 +41,11 @@ SANA_WM_CONFIG_PATH, SANA_WM_REFINER_GEMMA_ROOT, SANA_WM_REFINER_ROOT, + SANA_WM_STREAMING_CAUSAL_VAE_ROOT, + SANA_WM_STREAMING_CONFIG_PATH, + SANA_WM_STREAMING_REFINER_GEMMA_ROOT, + SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES, + SANA_WM_STREAMING_REFINER_ROOT, SANA_WM_VAE_SPATIAL_COMPRESSION, SANA_WM_VAE_TEMPORAL_COMPRESSION, ) @@ -85,6 +90,9 @@ class SanaWMLTX2VAEDecoderConfig(InstantiateConfig): config_path: str = SANA_WM_CONFIG_PATH """SANA-WM inference YAML path or ``hf://`` URI.""" + vae_path: str | None = None + """Optional VAE root override. ``None`` uses the configured YAML path.""" + height: int = DEFAULT_VIDEO_HEIGHT width: int = DEFAULT_VIDEO_WIDTH @@ -208,7 +216,8 @@ def _ensure_vae(self) -> None: return cfg = self._ensure_runtime_config() self.vae_dtype = _get_weight_dtype(cfg.vae.weight_dtype) - cfg.vae.vae_pretrained = resolve_hf_path(cfg.vae.vae_pretrained) + vae_pretrained = self.config.vae_path or cfg.vae.vae_pretrained + cfg.vae.vae_pretrained = resolve_hf_path(vae_pretrained) self.vae = _get_vae( cfg.vae.vae_type, cfg.vae.vae_pretrained, @@ -514,12 +523,213 @@ def get_input_temporal_size( return ((output_temporal_size - 1) // ratio) + 1 +@dataclass(kw_only=True) +class SanaWMStreamingVideoDecoderCache(SanaWMVideoDecoderCache): + """Per-rollout cache for streaming SANA-WM decode/refine.""" + + block_size: int = 3 + refiner_kv_max_frames: int = SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES + stage1_chunks: list[Tensor] = field(default_factory=list) + emitted_frames: int = 0 + emitted_stage1_frames: int = 0 + + +@dataclass(kw_only=True) +class SanaWMStreamingLTX2VAEDecoderConfig(SanaWMLTX2VAEDecoderConfig): + """Config for the streaming causal LTX-2 VAE decode path.""" + + config_path: str = SANA_WM_STREAMING_CONFIG_PATH + vae_path: str | None = SANA_WM_STREAMING_CAUSAL_VAE_ROOT + + +@dataclass(kw_only=True) +class SanaWMStreamingLTX2LatentRefinerConfig(SanaWMLTX2LatentRefinerConfig): + """Config for the streaming chunk-causal LTX-2 refiner path.""" + + _target: type["SanaWMStreamingLTX2LatentRefiner"] = field( + default_factory=lambda: SanaWMStreamingLTX2LatentRefiner + ) + + refiner_root: str = SANA_WM_STREAMING_REFINER_ROOT + refiner_gemma_root: str = SANA_WM_STREAMING_REFINER_GEMMA_ROOT + kv_max_frames: int = SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES + block_size: int = 3 + + +class SanaWMStreamingLTX2LatentRefiner(SanaWMLTX2LatentRefiner): + """Streaming refiner adapter using FlashDreams-owned prefix recomputation.""" + + config: SanaWMStreamingLTX2LatentRefinerConfig + + @torch.inference_mode() + def refine_prefix( + self, + *, + latents: Tensor, + prompt: str, + fps: int, + sink_size: int, + seed: int, + ) -> Tensor: + """Refine the currently visible latent prefix.""" + if latents.shape[2] <= sink_size: + return latents + return self.refine_latents( + latents=latents, + prompt=prompt, + fps=fps, + sink_size=sink_size, + seed=seed, + ) + + +@dataclass(kw_only=True) +class SanaWMStreamingVideoDecoderConfig(DecoderConfig): + """Config for streaming SANA-WM latent refinement and chunk decode.""" + + _target: type["SanaWMStreamingVideoDecoder"] = field( + default_factory=lambda: SanaWMStreamingVideoDecoder + ) + + vae_decoder: SanaWMStreamingLTX2VAEDecoderConfig = field( + default_factory=SanaWMStreamingLTX2VAEDecoderConfig + ) + refiner: SanaWMStreamingLTX2LatentRefinerConfig | None = field( + default_factory=SanaWMStreamingLTX2LatentRefinerConfig + ) + + +class SanaWMStreamingVideoDecoder(StreamingVideoDecoder[SanaWMStreamingVideoDecoderCache]): + """Decode SANA-WM streaming latent chunks into newly available frames.""" + + config: SanaWMStreamingVideoDecoderConfig + + def __init__(self, config: SanaWMStreamingVideoDecoderConfig) -> None: + super().__init__(config) + self.config = config + self.vae_decoder = config.vae_decoder.setup() + self.refiner = config.refiner.setup() if config.refiner is not None else None + + def initialize_autoregressive_cache( + self, + **context: Any, + ) -> SanaWMStreamingVideoDecoderCache: + """Build per-rollout streaming decode/refiner settings.""" + return SanaWMStreamingVideoDecoderCache(**context) + + @torch.inference_mode() + def forward( + self, + input: Tensor, + autoregressive_index: int = 0, + cache: SanaWMStreamingVideoDecoderCache | None = None, + ) -> SanaWMDecodedVideo: + """Refine/decode one SANA-WM streaming latent chunk.""" + del autoregressive_index + cache = cache or SanaWMStreamingVideoDecoderCache() + cache.stage1_chunks.append(input) + stage1_prefix = torch.cat(cache.stage1_chunks, dim=2) + output_prefix = stage1_prefix + if self.refiner is not None: + output_prefix = self.refiner.refine_prefix( + latents=stage1_prefix, + prompt=cache.prompt, + fps=cache.fps, + sink_size=cache.sink_size, + seed=cache.refiner_seed, + ) + elif cache.save_stage1: + logger.info( + "SANA-WM streaming is already running without the refiner; " + "--save-stage1 does not create an extra output." + ) + + video_hwc, cache.emitted_frames = _decode_new_frames( + self.vae_decoder, + output_prefix, + emitted_frames=cache.emitted_frames, + ) + stage1_video_hwc = None + if cache.save_stage1 and self.refiner is not None: + stage1_video_hwc, cache.emitted_stage1_frames = _decode_new_frames( + self.vae_decoder, + stage1_prefix, + emitted_frames=cache.emitted_stage1_frames, + ) + return SanaWMDecodedVideo( + video_hwc=video_hwc, + stage1_video_hwc=stage1_video_hwc, + ) + + @property + def spatial_compression_ratio(self) -> int: + """Pixel side divided by latent side.""" + return SANA_WM_VAE_SPATIAL_COMPRESSION + + @property + def temporal_compression_ratio(self) -> int: + """Pixel frames emitted per steady-state latent frame.""" + return SANA_WM_VAE_TEMPORAL_COMPRESSION + + def get_output_temporal_size( + self, + autoregressive_index: int, + input_temporal_size: int, + ) -> int: + """Return newly emitted pixel frames for one streaming latent chunk.""" + if input_temporal_size <= 0: + raise ValueError( + f"input_temporal_size must be positive, got {input_temporal_size}." + ) + ratio = self.temporal_compression_ratio + if autoregressive_index == 0: + if input_temporal_size <= 1: + raise ValueError("Streaming AR step 0 must include sink + active frames.") + return (input_temporal_size - 1) * ratio + return input_temporal_size * ratio + + def get_input_temporal_size( + self, + autoregressive_index: int, + output_temporal_size: int, + ) -> int: + """Return latent frames needed to emit ``output_temporal_size`` pixels.""" + ratio = self.temporal_compression_ratio + if output_temporal_size <= 0 or output_temporal_size % ratio != 0: + raise ValueError( + "SANA-WM streaming output frame count must be a positive " + f"multiple of {ratio}; got {output_temporal_size}." + ) + latent_frames = output_temporal_size // ratio + return latent_frames + 1 if autoregressive_index == 0 else latent_frames + + +def _decode_new_frames( + vae_decoder: SanaWMLTX2VAEDecoder, + latent_prefix: Tensor, + *, + emitted_frames: int, +) -> tuple[np.ndarray, int]: + """Decode a prefix and return only frames not emitted by prior chunks.""" + video = vae_decoder.decode_latents(latent_prefix) + start = emitted_frames + if start == 0 and video.shape[0] > 0: + start = 1 + return video[start:], int(video.shape[0]) + + __all__ = [ "SanaWMDecodedVideo", "SanaWMLTX2LatentRefiner", "SanaWMLTX2LatentRefinerConfig", "SanaWMLTX2VAEDecoder", "SanaWMLTX2VAEDecoderConfig", + "SanaWMStreamingLTX2LatentRefiner", + "SanaWMStreamingLTX2LatentRefinerConfig", + "SanaWMStreamingLTX2VAEDecoderConfig", + "SanaWMStreamingVideoDecoder", + "SanaWMStreamingVideoDecoderCache", + "SanaWMStreamingVideoDecoderConfig", "SanaWMVideoDecoder", "SanaWMVideoDecoderCache", "SanaWMVideoDecoderConfig", diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index ec7c73161..63af8236b 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -29,7 +29,11 @@ from flashdreams.infra.config import derive_config from flashdreams.infra.pipeline import StreamInferencePipelineConfig from flashdreams.infra.runner import Runner, RunnerConfig -from flashdreams.infra.runner_io import ensure_output_dir, resolve_prompt_value, runner_artifact_path +from flashdreams.infra.runner_io import ( + ensure_output_dir, + resolve_prompt_value, + runner_artifact_path, +) from sana_wm.camera import ( action_string_to_c2w, default_intrinsics_vec4, @@ -39,16 +43,29 @@ snap_num_frames, transform_intrinsics_for_crop, ) -from sana_wm.conditioning import SanaWMI2VConditioningRequest +from sana_wm.conditioning import ( + SanaWMI2VConditioningRequest, + SanaWMStreamingI2VConditioningRequest, + streaming_chunk_boundaries, +) from sana_wm.constants import ( DEFAULT_ACTION, DEFAULT_FPS, + DEFAULT_STREAMING_DENOISING_STEP_LIST, + DEFAULT_STREAMING_NUM_FRAMES, DEFAULT_VIDEO_HEIGHT, DEFAULT_VIDEO_WIDTH, SANA_WM_CONFIG_PATH, SANA_WM_MODEL_PATH, SANA_WM_REFINER_GEMMA_ROOT, SANA_WM_REFINER_ROOT, + SANA_WM_STREAMING_CAUSAL_VAE_ROOT, + SANA_WM_STREAMING_CONFIG_PATH, + SANA_WM_STREAMING_LATENT_CHUNK_SIZE, + SANA_WM_STREAMING_MODEL_PATH, + SANA_WM_STREAMING_REFINER_GEMMA_ROOT, + SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES, + SANA_WM_STREAMING_REFINER_ROOT, SANA_WM_VAE_TEMPORAL_COMPRESSION, ) from sana_wm.decoder import SanaWMDecodedVideo @@ -185,6 +202,60 @@ class SanaWMRunnerConfig(RunnerConfig): """Move the Stage-1 text encoder to CPU between prompt encodes.""" +@dataclass(kw_only=True) +class SanaWMStreamingRunnerConfig(SanaWMRunnerConfig): + """Runner config for the SANA-WM streaming release.""" + + _target: type["SanaWMStreamingRunner"] = field( + default_factory=lambda: SanaWMStreamingRunner + ) + + num_frames: int = DEFAULT_STREAMING_NUM_FRAMES + """Requested streaming output frames before chunk-stride snapping.""" + + step: int = len(DEFAULT_STREAMING_DENOISING_STEP_LIST) - 1 + """Stage-1 streaming denoising steps.""" + + cfg_scale: float = 1.0 + """Classifier-free guidance scale for streaming Stage 1.""" + + flow_shift: float | None = 8.0 + """Streaming scheduler flow-shift override.""" + + config_path: str = SANA_WM_STREAMING_CONFIG_PATH + """SANA-WM streaming config path or built-in identifier.""" + + model_path: str = SANA_WM_STREAMING_MODEL_PATH + """Streaming Stage-1 checkpoint path or ``hf://`` URI.""" + + causal_vae_path: str = SANA_WM_STREAMING_CAUSAL_VAE_ROOT + """Streaming causal LTX-2 VAE root path or ``hf://`` URI.""" + + refiner_root: str = SANA_WM_STREAMING_REFINER_ROOT + """Streaming LTX-2 refiner root path or ``hf://`` URI.""" + + refiner_gemma_root: str = SANA_WM_STREAMING_REFINER_GEMMA_ROOT + """Streaming Gemma-3 text-encoder root path or ``hf://`` URI.""" + + num_frame_per_block: int = SANA_WM_STREAMING_LATENT_CHUNK_SIZE + """Latent frames generated per steady-state AR block.""" + + num_cached_blocks: int = 2 + """Stage-1 streaming context-window hint passed to the transformer.""" + + no_sink_token: bool = False + """Disable the Stage-1 streaming sink-token context hint.""" + + denoising_step_list: tuple[int, ...] = DEFAULT_STREAMING_DENOISING_STEP_LIST + """Explicit distilled Stage-1 timestep schedule.""" + + refiner_block_size: int = SANA_WM_STREAMING_LATENT_CHUNK_SIZE + """Latent frames refined per streaming AR block.""" + + refiner_kv_max_frames: int = SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES + """Refiner sliding-window size in latent frames.""" + + class SanaWMRunner(Runner[SanaWMRunnerConfig, object]): """CLI driver for SANA-WM configs.""" @@ -340,13 +411,15 @@ def _prepare_inputs(self) -> tuple[object, np.ndarray, np.ndarray, int]: cfg = self.config assert cfg.image_path is not None image = Image.open(cfg.image_path).convert("RGB") + stride = self._frame_snap_stride() snapped = snap_num_frames( cfg.num_frames, - stride=SANA_WM_VAE_TEMPORAL_COMPRESSION, + stride=stride, ) if snapped != cfg.num_frames and self.is_rank_zero: logger.warning( - "SANA-WM requires num_frames = 8k+1; requested {} snapped to {}.", + "SANA-WM requires num_frames = {}k+1; requested {} snapped to {}.", + stride, cfg.num_frames, snapped, ) @@ -395,6 +468,114 @@ def _sampling_algo(self) -> SamplingAlgo: return "flow_euler_ltx" return self.config.sampling_algo + def _frame_snap_stride(self) -> int: + """Return the required pixel-frame stride for this runner.""" + return SANA_WM_VAE_TEMPORAL_COMPRESSION + + +class SanaWMStreamingRunner(SanaWMRunner): + """CLI driver for SANA-WM streaming configs.""" + + config: SanaWMStreamingRunnerConfig + + def run(self) -> None: + """Run SANA-WM streaming inference and write outputs.""" + cfg = self.config + if cfg.image_path is None: + raise ValueError("SanaWMStreamingRunner requires --image-path.") + + device = self._resolve_device() + quant_backend = _resolve_quant_backend( + cfg.quant_backend, + _active_quantized_precisions( + stage1_precision=cfg.stage1_precision, + refiner_precision=cfg.refiner_precision, + refiner_enabled=not cfg.no_refiner, + ), + ) + _validate_precision_request( + device=device, + stage1_precision=cfg.stage1_precision, + refiner_precision=cfg.refiner_precision, + refiner_enabled=not cfg.no_refiner, + quant_backend=cfg.quant_backend, + ) + prompt = self._resolve_prompt() + image, c2w, intrinsics_vec4, num_frames = self._prepare_inputs() + pipeline_cfg = _streaming_pipeline_config( + cfg, + quant_backend=quant_backend, + ) + pipeline = pipeline_cfg.setup().to(device).eval() + latent_frames = (num_frames - 1) // SANA_WM_VAE_TEMPORAL_COMPRESSION + 1 + chunk_boundaries = streaming_chunk_boundaries( + latent_frames, + cfg.num_frame_per_block, + ) + cache = pipeline.initialize_cache( + decoder_context={ + "prompt": prompt, + "fps": cfg.fps, + "save_stage1": cfg.save_stage1, + "refiner_seed": cfg.refiner_seed, + "sink_size": cfg.sink_size, + "block_size": cfg.refiner_block_size, + "refiner_kv_max_frames": cfg.refiner_kv_max_frames, + } + ) + request = SanaWMStreamingI2VConditioningRequest( + image=image, + prompt=prompt, + poses_c2w=c2w, + intrinsics_vec4=intrinsics_vec4, + num_frames=num_frames, + fps=cfg.fps, + steps=cfg.step, + cfg_scale=cfg.cfg_scale, + flow_shift=cfg.flow_shift, + seed=cfg.seed, + negative_prompt=cfg.negative_prompt, + num_frame_per_block=cfg.num_frame_per_block, + ) + + decoded_chunks: list[np.ndarray] = [] + stage1_chunks: list[np.ndarray] = [] + with torch.inference_mode(): + for ar_idx in range(len(chunk_boundaries) - 1): + decoded = pipeline.generate(ar_idx, cache, input=request) + pipeline.finalize(ar_idx, cache) + if not isinstance(decoded, SanaWMDecodedVideo): + raise TypeError( + "SANA-WM streaming pipeline decoder returned " + f"{type(decoded).__name__}, expected SanaWMDecodedVideo." + ) + if decoded.video_hwc.size: + decoded_chunks.append(decoded.video_hwc) + if decoded.stage1_video_hwc is not None and decoded.stage1_video_hwc.size: + stage1_chunks.append(decoded.stage1_video_hwc) + + if not self.is_rank_zero: + return + ensure_output_dir(cfg.output_dir) + if not decoded_chunks: + raise RuntimeError("SANA-WM streaming produced no decoded frames.") + video_hwc = np.concatenate(decoded_chunks, axis=0) + _write_video( + runner_artifact_path(cfg.output_dir, cfg.runner_name, "mp4"), + video_hwc, + cfg.fps, + ) + if stage1_chunks: + _write_video( + runner_artifact_path(cfg.output_dir, f"{cfg.runner_name}_stage1", "mp4"), + np.concatenate(stage1_chunks, axis=0), + cfg.fps, + ) + + def _frame_snap_stride(self) -> int: + """Return the streaming pixel-frame chunk stride.""" + return SANA_WM_VAE_TEMPORAL_COMPRESSION * self.config.num_frame_per_block + def _pipeline_config( cfg: SanaWMRunnerConfig, @@ -457,6 +638,74 @@ def _pipeline_config( ) +def _streaming_pipeline_config( + cfg: SanaWMStreamingRunnerConfig, + *, + quant_backend: ResolvedQuantBackend, +) -> StreamInferencePipelineConfig: + """Apply CLI runtime fields to the SANA-WM streaming pipeline literal.""" + return derive_config( + cfg.pipeline, + diffusion_model=dict( + seed=cfg.seed, + scheduler=dict( + num_inference_steps=cfg.step, + shift=cfg.flow_shift if cfg.flow_shift is not None else 8.0, + denoising_step_list=cfg.denoising_step_list, + ), + transformer=dict( + config_path=cfg.config_path, + checkpoint_path=cfg.model_path, + stage1_precision=cfg.stage1_precision, + quant_backend=quant_backend, + offload_stage1=cfg.offload_stage1, + num_frame_per_block=cfg.num_frame_per_block, + num_cached_blocks=cfg.num_cached_blocks, + sink_token=not cfg.no_sink_token, + ), + ), + encoder=dict( + config_path=cfg.config_path, + text_encoder=dict( + config_path=cfg.config_path, + stage1_precision=cfg.stage1_precision, + quant_backend=quant_backend, + offload_text_encoder=cfg.offload_text_encoder, + ), + first_frame_encoder=dict( + config_path=cfg.config_path, + offload_vae=cfg.offload_vae, + ), + camera_encoder=dict( + height=DEFAULT_VIDEO_HEIGHT, + width=DEFAULT_VIDEO_WIDTH, + ), + height=DEFAULT_VIDEO_HEIGHT, + width=DEFAULT_VIDEO_WIDTH, + ), + decoder=dict( + vae_decoder=dict( + config_path=cfg.config_path, + vae_path=cfg.causal_vae_path, + offload_vae=cfg.offload_vae, + ), + refiner=( + None + if cfg.no_refiner + else dict( + refiner_root=cfg.refiner_root, + refiner_gemma_root=cfg.refiner_gemma_root, + refiner_precision=cfg.refiner_precision, + quant_backend=quant_backend, + offload_refiner=cfg.offload_refiner, + kv_max_frames=cfg.refiner_kv_max_frames, + block_size=cfg.refiner_block_size, + ) + ), + ), + ) + + def _write_video(path: Path, video_hwc: np.ndarray, fps: int) -> Path: """Write an HWC uint8 video to ``path``.""" import imageio.v3 as iio @@ -599,4 +848,6 @@ def _validate_torch_fp4_primitives() -> None: "SamplingAlgo", "SanaWMRunner", "SanaWMRunnerConfig", + "SanaWMStreamingRunner", + "SanaWMStreamingRunnerConfig", ] diff --git a/integrations/sana/sana_wm/scheduler.py b/integrations/sana/sana_wm/scheduler.py index 168c377db..3f58e2ecd 100644 --- a/integrations/sana/sana_wm/scheduler.py +++ b/integrations/sana/sana_wm/scheduler.py @@ -47,6 +47,14 @@ class SanaWMLTXEulerSchedulerConfig(SchedulerConfig): num_train_timesteps: int = 1000 """Training timestep scale used by the public SANA-WM release.""" + denoising_step_list: tuple[int, ...] | None = None + """Optional explicit training-scale timestep schedule. + + Streaming SANA-WM uses a distilled student with a fixed schedule that + already includes the intended flow shift. When this is set, the scheduler + uses these values verbatim and requires the final entry to be ``0``. + """ + class SanaWMLTXEulerScheduler(Scheduler): """Euler scheduler with SANA-WM per-token timestep support.""" @@ -65,6 +73,14 @@ def timesteps( device: torch.device | str, ) -> Tensor: """Return diffusers-compatible FlowMatch Euler timesteps.""" + if self.config.denoising_step_list is not None: + del num_inference_steps, shift + return _explicit_timesteps( + self.config.denoising_step_list, + device=device, + num_train_timesteps=self.config.num_train_timesteps, + ) + steps = int(num_inference_steps) if steps <= 0: raise ValueError(f"num_inference_steps must be > 0, got {steps}.") @@ -194,3 +210,31 @@ def add_noise( "SanaWMLTXEulerScheduler", "SanaWMLTXEulerSchedulerConfig", ] + + +def _explicit_timesteps( + values: tuple[int, ...], + *, + device: torch.device | str, + num_train_timesteps: int, +) -> Tensor: + """Validate and materialize a fixed training-scale timestep schedule.""" + if len(values) < 2: + raise ValueError("denoising_step_list must contain at least two timesteps.") + if int(values[-1]) != 0: + raise ValueError("denoising_step_list must end with 0.") + if any(int(value) < 0 for value in values): + raise ValueError("denoising_step_list cannot contain negative timesteps.") + if any(int(a) < int(b) for a, b in zip(values, values[1:])): + raise ValueError("denoising_step_list must be monotonically non-increasing.") + upper = int(num_train_timesteps) + if any(int(value) > upper for value in values): + raise ValueError( + "denoising_step_list cannot exceed num_train_timesteps=" + f"{num_train_timesteps}." + ) + return torch.tensor( + tuple(float(value) for value in values), + dtype=torch.float32, + device=device, + ) diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index 2f93cecce..b729c7172 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -79,6 +79,9 @@ def block_uses_gdn(self, index: int) -> bool: SANA_WM_STAGE1_SPEC = SanaWMStage1Spec() """Architecture spec for the public SANA-WM bidirectional Stage-1 checkpoint.""" +SANA_WM_STREAMING_STAGE1_SPEC = SanaWMStage1Spec(chunk_size=3) +"""Architecture spec for the public SANA-WM streaming Stage-1 checkpoint.""" + def _env_flag_enabled(name: str) -> bool: return os.environ.get(name, "").lower() in {"1", "true", "yes", "on"} @@ -2487,6 +2490,7 @@ def _invert_se3(transforms: Tensor) -> Tensor: __all__ = [ "SANA_WM_STAGE1_SPEC", + "SANA_WM_STREAMING_STAGE1_SPEC", "GLUMBConvTemp", "RMSNorm", "SanaWMStage1Block", diff --git a/integrations/sana/sana_wm/transformer.py b/integrations/sana/sana_wm/transformer.py index c05a71c59..0ffea0343 100644 --- a/integrations/sana/sana_wm/transformer.py +++ b/integrations/sana/sana_wm/transformer.py @@ -40,13 +40,21 @@ DEFAULT_VIDEO_WIDTH, SANA_WM_CONFIG_PATH, SANA_WM_MODEL_PATH, + SANA_WM_STREAMING_CAUSAL_VAE_ROOT, + SANA_WM_STREAMING_CONFIG_PATH, + SANA_WM_STREAMING_LATENT_CHUNK_SIZE, + SANA_WM_STREAMING_MODEL_PATH, ) from sana_wm.quant import ( TorchScaledMMFP4Recipe, TorchScaledMMFP8Recipe, replace_linear_with_quant, ) -from sana_wm.stage1_model import SanaWMStage1Model, linearize_stage1_ffn_for_quant +from sana_wm.stage1_model import ( + SANA_WM_STREAMING_STAGE1_SPEC, + SanaWMStage1Model, + linearize_stage1_ffn_for_quant, +) Precision = Literal["bf16", "fp8", "fp4"] QuantBackend = Literal["auto", "torch", "torch-fp8", "torch-fp4"] @@ -87,6 +95,22 @@ class SanaWMStage1Conditioning: seed: int +@dataclass(kw_only=True) +class SanaWMStreamingStage1Conditioning(SanaWMStage1Conditioning): + """Per-chunk Stage-1 inputs for SANA-WM streaming inference.""" + + total_latent_shape: tuple[int, int, int, int, int] + start_frame: int + end_frame: int + chunk_index: int + chunk_boundaries: tuple[int, ...] + + @property + def num_chunks(self) -> int: + """Return the number of AR chunks in this rollout.""" + return len(self.chunk_boundaries) - 1 + + @dataclass(kw_only=True) class SanaWMTransformerCache(TransformerAutoregressiveCache): """AR cache for the one-shot SANA-WM Stage-1 rollout.""" @@ -94,6 +118,14 @@ class SanaWMTransformerCache(TransformerAutoregressiveCache): conditioning: SanaWMStage1Conditioning | None = None +@dataclass(kw_only=True) +class SanaWMStreamingTransformerCache(SanaWMTransformerCache): + """AR cache for streaming SANA-WM Stage-1 prefix sampling.""" + + latent_state: Tensor | None = None + initial_latent_state: Tensor | None = None + + @dataclass(kw_only=True) class SanaWMTransformerConfig(TransformerConfig): """Config for the SANA-WM Stage-1 transformer adapter.""" @@ -126,6 +158,30 @@ class SanaWMTransformerConfig(TransformerConfig): Stage-1 memory for the VAE/refiner on memory-constrained GPUs.""" +@dataclass(kw_only=True) +class SanaWMStreamingTransformerConfig(SanaWMTransformerConfig): + """Config for the SANA-WM streaming Stage-1 transformer adapter.""" + + _target: type["SanaWMStreamingTransformer"] = field( + default_factory=lambda: SanaWMStreamingTransformer + ) + + config_path: str = SANA_WM_STREAMING_CONFIG_PATH + """SANA-WM streaming inference config path or built-in identifier.""" + + checkpoint_path: str = SANA_WM_STREAMING_MODEL_PATH + """SANA-WM streaming Stage-1 checkpoint path or ``hf://`` URI.""" + + num_frame_per_block: int = SANA_WM_STREAMING_LATENT_CHUNK_SIZE + """Latent frames generated per steady-state AR block.""" + + num_cached_blocks: int = 2 + """Stage-1 streaming context-window hint retained on the config surface.""" + + sink_token: bool = True + """Keep the first chunk as the Stage-1 streaming sink anchor.""" + + class SanaWMTransformer(Transformer[SanaWMTransformerCache]): """FlashDreams adapter for the SANA-WM Stage-1 model call.""" @@ -386,12 +442,15 @@ def _predict_with_prompt( model_kwargs: dict[str, object], ) -> Tensor: self._ensure_model() - return self.model( + output = self.model( noisy_latent, timestep, prompt_embeds, **model_kwargs, ) + if isinstance(output, tuple): + output = output[0] + return cast(Tensor, output) def _ensure_runtime_config(self) -> Any: if self._runtime_config is not None: @@ -489,6 +548,246 @@ def _prepare_stage1_quant(self) -> None: torch.cuda.empty_cache() +class SanaWMStreamingTransformer(SanaWMTransformer): + """FlashDreams adapter for SANA-WM streaming Stage-1 chunks. + + This implementation keeps the full latent rollout in the transformer + cache and recomputes the available prefix for each chunk. That preserves + the FlashDreams AR lifecycle without depending on upstream cached module + classes. + """ + + config: SanaWMStreamingTransformerConfig + + def __init__(self, config: SanaWMStreamingTransformerConfig) -> None: + super().__init__(config) + self.config = config + self._current_latent_shape: tuple[int, ...] | None = None + + @property + def latent_shape(self) -> tuple[int, ...]: + """Return the current streaming chunk latent shape.""" + if self._current_latent_shape is not None: + return self._current_latent_shape + return ( + 1, + 128, + int(self.config.num_frame_per_block) + 1, + self.config.height // 32, + self.config.width // 32, + ) + + def initialize_autoregressive_cache( + self, + *, + conditioning: SanaWMStreamingStage1Conditioning | None = None, + **_: Any, + ) -> SanaWMStreamingTransformerCache: + """Build a cache for a streaming SANA-WM rollout.""" + self._current_latent_shape = ( + conditioning.latent_shape if conditioning is not None else None + ) + return SanaWMStreamingTransformerCache(conditioning=conditioning) + + def patchify_and_maybe_split_cp(self, x: Any) -> Any: + """SANA-WM streaming latents are already in model layout.""" + if isinstance(x, SanaWMStreamingStage1Conditioning): + self._current_latent_shape = x.latent_shape + return x + + def initial_noise( + self, + *, + latent_shape: tuple[int, ...], + rng: torch.Generator | None, + cache: SanaWMTransformerCache, + input: Any = None, + ) -> Tensor: + """Return this AR chunk's initial noisy latent.""" + if not isinstance(input, SanaWMStreamingStage1Conditioning): + return super().initial_noise( + latent_shape=latent_shape, + rng=rng, + cache=cache, + input=input, + ) + streaming_cache = _require_streaming_cache(cache) + streaming_cache.conditioning = input + self._current_latent_shape = input.latent_shape + latent_state = self._ensure_streaming_latent_state(streaming_cache, input) + return latent_state[:, :, input.start_frame : input.end_frame].clone() + + def predict_flow( + self, + noisy_latent: Tensor, + timestep: Tensor, + cache: SanaWMTransformerCache, + input: Any = None, + model_kwargs: dict[str, object] | None = None, + ) -> Tensor: + """Execute one streaming SANA-WM Stage-1 flow prediction.""" + if not isinstance(input, SanaWMStreamingStage1Conditioning): + return super().predict_flow( + noisy_latent=noisy_latent, + timestep=timestep, + cache=cache, + input=input, + model_kwargs=model_kwargs, + ) + del model_kwargs + streaming_cache = _require_streaming_cache(cache) + streaming_cache.conditioning = input + return self._predict_streaming_conditioned( + noisy_latent=noisy_latent, + timestep=timestep, + cache=streaming_cache, + conditioning=input, + ) + + def postprocess_clean_latent( + self, + clean_latent: Tensor, + cache: SanaWMTransformerCache, + input: Any = None, + ) -> Tensor: + """Commit a finished streaming chunk to the prefix latent cache.""" + if not isinstance(input, SanaWMStreamingStage1Conditioning): + return super().postprocess_clean_latent(clean_latent, cache, input) + streaming_cache = _require_streaming_cache(cache) + latent_state = self._ensure_streaming_latent_state(streaming_cache, input) + latent_state[:, :, input.start_frame : input.end_frame] = clean_latent + if input.start_frame == 0: + latent_state[:, :, :1] = input.first_latent.to( + device=latent_state.device, + dtype=latent_state.dtype, + ) + clean_latent[:, :, :1] = latent_state[:, :, :1] + + if input.chunk_index == input.num_chunks - 1 and self.config.offload_stage1: + self.release_stage1_runtime(streaming_cache) + else: + streaming_cache.conditioning = None + return clean_latent + + def finalize_kv_cache( + self, + noisy_latent: Tensor, + timestep: Tensor, + cache: SanaWMTransformerCache, + input: Any = None, + ) -> None: + """Prefix recomputation keeps streaming context in ``latent_state``.""" + del noisy_latent, timestep, cache, input + + def _ensure_model(self) -> None: + if self._model_built: + self._prepare_stage1_quant() + return + t0 = time.perf_counter() + cfg = self._ensure_runtime_config() + weight_dtype = self._ensure_weight_dtype() + model = SanaWMStage1Model(SANA_WM_STREAMING_STAGE1_SPEC).to(self.device) + logger.info( + "[Sana] Loaded {} ({:,} params)", + cfg.model.model, + sum(p.numel() for p in model.parameters()), + ) + self._model_path = resolve_hf_path(self.config.checkpoint_path) + state = find_model(self._model_path) + if "generator" in state: + state = state["generator"] + if "state_dict" not in state: + state = { + "state_dict": { + (k[len("model.") :] if k.startswith("model.") else k): v + for k, v in state.items() + } + } + missing, unexpected = model.load_state_dict(state["state_dict"], strict=True) + if missing: + logger.warning("[Sana] Missing keys: {}", missing) + if unexpected: + logger.warning("[Sana] Unexpected keys: {}", unexpected) + self.model = model.eval().to(weight_dtype) + self._model_built = True + self._prepare_stage1_quant() + logger.info( + "[timing] streaming stage1 build+load+quant: {:.3f}s (precision={})", + time.perf_counter() - t0, + self.config.stage1_precision, + ) + + def _ensure_streaming_latent_state( + self, + cache: SanaWMStreamingTransformerCache, + conditioning: SanaWMStreamingStage1Conditioning, + ) -> Tensor: + if cache.latent_state is None: + generator = torch.Generator(device=self.device).manual_seed( + conditioning.seed + ) + cache.latent_state = torch.randn( + conditioning.total_latent_shape, + dtype=self._ensure_weight_dtype(), + device=self.device, + generator=generator, + ) + cache.latent_state[:, :, :1] = conditioning.first_latent.to( + device=self.device, + dtype=cache.latent_state.dtype, + ) + cache.initial_latent_state = cache.latent_state.clone() + return cache.latent_state + + def _predict_streaming_conditioned( + self, + *, + noisy_latent: Tensor, + timestep: Tensor, + cache: SanaWMStreamingTransformerCache, + conditioning: SanaWMStreamingStage1Conditioning, + ) -> Tensor: + latent_state = self._ensure_streaming_latent_state(cache, conditioning) + start = int(conditioning.start_frame) + end = int(conditioning.end_frame) + prefix = latent_state[:, :, :end].clone() + prefix[:, :, start:end] = noisy_latent + if start == 0: + prefix[:, :, :1] = conditioning.first_latent.to( + device=prefix.device, + dtype=prefix.dtype, + ) + model_timestep = _streaming_prefix_timestep( + noisy_latent=noisy_latent, + timestep=timestep, + conditioning=conditioning, + prefix_frames=end, + ) + kwargs = _streaming_prefix_model_kwargs(conditioning.model_kwargs, end) + if conditioning.cfg_scale <= 1.0: + flow_prefix = self._predict_with_prompt( + prefix, + model_timestep, + conditioning.condition, + _condition_model_kwargs(kwargs), + ) + else: + if conditioning.uncondition is None: + raise RuntimeError("CFG was requested without negative prompt embeds.") + flow_prefix = self._predict_with_prompt( + torch.cat([prefix, prefix], dim=0), + torch.cat([model_timestep, model_timestep], dim=0), + torch.cat([conditioning.uncondition, conditioning.condition], dim=0), + _batched_cfg_model_kwargs(kwargs), + ) + flow_prefix = _cfg_guidance(flow_prefix, conditioning.cfg_scale) + + flow = flow_prefix[:, :, start:end] + if start == 0: + flow[:, :, :1] = 0 + return flow.to(dtype=noisy_latent.dtype) + + def _stage1_quant_include_patterns() -> tuple[str, ...]: """Return Stage-1 Linear names eligible for FP8/FP4 quantization.""" return _STAGE1_QUANT_INCLUDE_DEFAULTS @@ -502,6 +801,17 @@ def _require_conditioning( return cache.conditioning +def _require_streaming_cache( + cache: SanaWMTransformerCache, +) -> SanaWMStreamingTransformerCache: + if not isinstance(cache, SanaWMStreamingTransformerCache): + raise TypeError( + "SANA-WM streaming transformer requires " + "SanaWMStreamingTransformerCache." + ) + return cache + + def _condition_model_kwargs(model_kwargs: dict[str, object]) -> dict[str, object]: """Return model kwargs for the positive prompt branch.""" return { @@ -531,6 +841,82 @@ def _batched_cfg_model_kwargs(model_kwargs: dict[str, object]) -> dict[str, obje return kwargs +def _streaming_prefix_model_kwargs( + model_kwargs: dict[str, object], + prefix_frames: int, +) -> dict[str, object]: + """Slice full-rollout model kwargs to the currently visible prefix.""" + result: dict[str, object] = {} + for key, value in model_kwargs.items(): + if isinstance(value, Tensor): + if value.ndim == 5 and value.shape[2] >= prefix_frames: + result[key] = value[:, :, :prefix_frames].contiguous() + elif value.ndim >= 3 and value.shape[1] >= prefix_frames: + result[key] = value[:, :prefix_frames].contiguous() + else: + result[key] = value + elif isinstance(value, dict): + result[key] = dict(value) + else: + result[key] = value + return result + + +def _streaming_prefix_timestep( + *, + noisy_latent: Tensor, + timestep: Tensor, + conditioning: SanaWMStreamingStage1Conditioning, + prefix_frames: int, +) -> Tensor: + """Build a prefix timestep table with clean context and noisy current chunk.""" + batch = noisy_latent.shape[0] + chunk_frames = int(conditioning.end_frame - conditioning.start_frame) + if timestep.ndim == 0: + current = timestep.reshape(1, 1, 1).expand(batch, 1, chunk_frames) + elif timestep.ndim == 1: + if timestep.numel() == 1: + current = timestep.reshape(1, 1, 1).expand(batch, 1, chunk_frames) + else: + current = timestep.reshape(batch, 1, 1).expand(batch, 1, chunk_frames) + elif timestep.ndim == 3: + current = timestep + elif timestep.ndim == noisy_latent.ndim: + current = timestep[:, :1, :, 0, 0] + else: + raise ValueError( + "SANA-WM streaming timestep must be scalar, [B], [B, 1, T], " + f"or latent-shaped; got shape={tuple(timestep.shape)}." + ) + if current.shape != (batch, 1, chunk_frames): + raise ValueError( + "SANA-WM streaming timestep shape is incompatible with the chunk: " + f"got {tuple(current.shape)}, expected {(batch, 1, chunk_frames)}." + ) + current = current.to(device=noisy_latent.device, dtype=torch.float32).clone() + data_info = conditioning.model_kwargs.get("data_info", {}) + condition_frame_info = ( + data_info.get("condition_frame_info", {}) + if isinstance(data_info, dict) + else {} + ) + if isinstance(condition_frame_info, dict): + for frame_idx in condition_frame_info: + index = int(frame_idx) + if conditioning.start_frame <= index < conditioning.end_frame: + current[:, :, index - conditioning.start_frame] = 0 + + prefix = torch.zeros( + batch, + 1, + prefix_frames, + dtype=torch.float32, + device=noisy_latent.device, + ) + prefix[:, :, conditioning.start_frame : conditioning.end_frame] = current + return prefix + + def _cfg_guidance(noise_pred: Tensor, cfg_scale: float) -> Tensor: noise_pred_uncond, noise_pred_text = noise_pred.chunk(2, dim=0) return noise_pred_uncond + cfg_scale * (noise_pred_text - noise_pred_uncond) @@ -651,6 +1037,8 @@ def tail_size(latent_stride: int) -> int: def _load_inference_config(config_path: str) -> Any: + if config_path == SANA_WM_STREAMING_CONFIG_PATH: + return _to_namespace(_builtin_streaming_config()) with open(resolve_hf_path(config_path), encoding="utf-8") as handle: raw = yaml.safe_load(handle) or {} if not isinstance(raw, dict): @@ -659,6 +1047,43 @@ def _load_inference_config(config_path: str) -> Any: return _to_namespace(raw) +def _builtin_streaming_config() -> dict[str, object]: + """Return the FlashDreams-owned SANA-WM streaming config data.""" + return { + "work_dir": "", + "model": { + "model": "SanaMSVideoCamCtrlStreaming_1600M_P1_D20", + "mixed_precision": "bf16", + "chunk_size": SANA_WM_STREAMING_LATENT_CHUNK_SIZE, + "chunk_split_strategy": "first_chunk_plus_one", + "softmax_every_n": 4, + }, + "vae": { + "vae_type": "LTX2VAE_diffusers_causal", + "vae_pretrained": SANA_WM_STREAMING_CAUSAL_VAE_ROOT, + "weight_dtype": "bfloat16", + "vae_latent_dim": 128, + "vae_stride": [8, 32, 32], + "use_framewise_encoding": True, + "use_framewise_decoding": True, + }, + "text_encoder": { + "text_encoder_name": "gemma-2-2b-it", + "model_max_length": 300, + "chi_prompt": [ + ( + "Given a user prompt, generate an enhanced visual " + "description suitable for image generation. User Prompt: " + ) + ], + }, + "scheduler": { + "flow_shift": 9.95, + "inference_flow_shift": 8.0, + }, + } + + def _get_vae(*args: Any, **kwargs: Any) -> nn.Module: name, model_path = args[:2] device = kwargs["device"] @@ -667,15 +1092,18 @@ def _get_vae(*args: Any, **kwargs: Any) -> nn.Module: raise ValueError(f"Unsupported SANA-WM VAE type: {name!r}") from diffusers import AutoencoderKLLTX2Video - return ( - AutoencoderKLLTX2Video.from_pretrained( + try: + vae = AutoencoderKLLTX2Video.from_pretrained( model_path, subfolder="vae", torch_dtype=dtype, ) - .to(device) - .eval() - ) + except OSError: + vae = AutoencoderKLLTX2Video.from_pretrained( + model_path, + torch_dtype=dtype, + ) + return vae.to(device).eval() def _get_tokenizer_and_text_encoder(*args: Any, **kwargs: Any) -> tuple[Any, nn.Module]: @@ -847,8 +1275,12 @@ def _chunk_index_from_chunk_size( __all__ = [ + "SanaWMStage1Conditioning", + "SanaWMStreamingStage1Conditioning", + "SanaWMStreamingTransformer", + "SanaWMStreamingTransformerCache", + "SanaWMStreamingTransformerConfig", "SanaWMTransformer", "SanaWMTransformerCache", "SanaWMTransformerConfig", - "SanaWMStage1Conditioning", ] diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index c04a30158..df9e569c2 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -35,19 +35,32 @@ from sana_wm.config import ( PIPELINE_SANA_WM_BIDIRECTIONAL, + PIPELINE_SANA_WM_STREAMING, RUNNER_CONFIGS, RUNNER_SANA_WM_BIDIRECTIONAL, + RUNNER_SANA_WM_STREAMING, ) from sana_wm.constants import ( + DEFAULT_STREAMING_DENOISING_STEP_LIST, SANA_WM_CONFIG_PATH, SANA_WM_HF_REPO, SANA_WM_MODEL_PATH, + SANA_WM_STREAMING_CAUSAL_VAE_ROOT, + SANA_WM_STREAMING_CONFIG_PATH, + SANA_WM_STREAMING_HF_REPO, + SANA_WM_STREAMING_MODEL_PATH, + SANA_WM_STREAMING_REFINER_GEMMA_ROOT, + SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES, + SANA_WM_STREAMING_REFINER_ROOT, ) from sana_wm.runner import ( SanaWMRunner, SanaWMRunnerConfig, + SanaWMStreamingRunner, + SanaWMStreamingRunnerConfig, _pipeline_config, _resolve_quant_backend, + _streaming_pipeline_config, _validate_precision_request, ) from sana_wm.conditioning import ( @@ -55,11 +68,18 @@ SanaWMCameraRequest, SanaWMConditioningEncoderConfig, SanaWMFirstFrameEncoderConfig, + SanaWMStreamingConditioningEncoderConfig, + SanaWMStreamingI2VConditioningRequest, SanaWMTextPromptEncoderConfig, SanaWMTextPromptRequest, + streaming_chunk_boundaries, ) from sana_wm.decoder import ( SanaWMDecodedVideo, + SanaWMStreamingLTX2LatentRefinerConfig, + SanaWMStreamingLTX2VAEDecoderConfig, + SanaWMStreamingVideoDecoder, + SanaWMStreamingVideoDecoderConfig, SanaWMLTX2LatentRefinerConfig, SanaWMLTX2VAEDecoderConfig, SanaWMVideoDecoderConfig, @@ -70,6 +90,9 @@ SanaWMLTXEulerSchedulerConfig, ) from sana_wm.transformer import ( + SanaWMStreamingStage1Conditioning, + SanaWMStreamingTransformerCache, + SanaWMStreamingTransformerConfig, SanaWMTransformerCache, SanaWMTransformerConfig, SanaWMStage1Conditioning, @@ -89,6 +112,7 @@ from sana_wm.stage1_model import ( GLUMBConvTemp, SANA_WM_STAGE1_SPEC, + SANA_WM_STREAMING_STAGE1_SPEC, SanaWMStage1Model, SanaWMStage1Spec, Stage1SelfAttention, @@ -105,9 +129,10 @@ def test_runner_config_is_registered() -> None: - """Expose the SANA-WM runner slug.""" + """Expose the SANA-WM runner slugs.""" assert RUNNER_CONFIGS == { "sana-wm-bidirectional": RUNNER_SANA_WM_BIDIRECTIONAL, + "sana-wm-streaming": RUNNER_SANA_WM_STREAMING, } @@ -116,11 +141,13 @@ def test_runner_name_mirrors_pipeline_name() -> None: assert RUNNER_SANA_WM_BIDIRECTIONAL.runner_name == ( PIPELINE_SANA_WM_BIDIRECTIONAL.name ) + assert RUNNER_SANA_WM_STREAMING.runner_name == PIPELINE_SANA_WM_STREAMING.name def test_runner_has_description() -> None: """Provide non-empty CLI help text for the runner registry.""" assert RUNNER_SANA_WM_BIDIRECTIONAL.description.strip() + assert RUNNER_SANA_WM_STREAMING.description.strip() def test_pipeline_uses_sana_diffusion_model() -> None: @@ -137,6 +164,24 @@ def test_pipeline_uses_sana_diffusion_model() -> None: assert isinstance(pipeline.decoder.refiner, SanaWMLTX2LatentRefinerConfig) +def test_streaming_pipeline_uses_sana_streaming_components() -> None: + """Keep the streaming runner wired to explicit streaming boundaries.""" + pipeline = PIPELINE_SANA_WM_STREAMING + transformer = pipeline.diffusion_model.transformer + + assert isinstance(pipeline.encoder, SanaWMStreamingConditioningEncoderConfig) + assert isinstance(pipeline.decoder, SanaWMStreamingVideoDecoderConfig) + assert isinstance(pipeline.diffusion_model, SanaWMDiffusionModelConfig) + assert pipeline.diffusion_model._target is DiffusionModel + assert isinstance(transformer, SanaWMStreamingTransformerConfig) + assert isinstance(pipeline.diffusion_model.scheduler, SanaWMLTXEulerSchedulerConfig) + assert pipeline.diffusion_model.scheduler.denoising_step_list == ( + DEFAULT_STREAMING_DENOISING_STEP_LIST + ) + assert isinstance(pipeline.decoder.vae_decoder, SanaWMStreamingLTX2VAEDecoderConfig) + assert isinstance(pipeline.decoder.refiner, SanaWMStreamingLTX2LatentRefinerConfig) + + def test_sana_decoder_uses_video_decoder_contract() -> None: """Expose SANA-WM VAE temporal sizing through the FlashDreams decoder API.""" decoder = SanaWMVideoDecoderConfig().setup() @@ -152,6 +197,22 @@ def test_sana_decoder_uses_video_decoder_contract() -> None: decoder.get_output_temporal_size(1, 21) +def test_streaming_decoder_uses_video_decoder_contract() -> None: + """Expose streaming chunk temporal sizing through the decoder API.""" + decoder = SanaWMStreamingVideoDecoderConfig(refiner=None).setup() + + assert isinstance(decoder, StreamingVideoDecoder) + assert isinstance(decoder, SanaWMStreamingVideoDecoder) + assert decoder.spatial_compression_ratio == 32 + assert decoder.temporal_compression_ratio == 8 + assert decoder.get_output_temporal_size(0, 4) == 24 + assert decoder.get_input_temporal_size(0, 24) == 4 + assert decoder.get_output_temporal_size(1, 3) == 24 + assert decoder.get_input_temporal_size(1, 24) == 3 + with pytest.raises(ValueError, match="multiple of 8"): + decoder.get_input_temporal_size(1, 25) + + def test_sana_diffusion_config_instantiates_base_model() -> None: """Use FlashDreams' shared diffusion model rather than a Sana-only runner.""" model = PIPELINE_SANA_WM_BIDIRECTIONAL.diffusion_model.setup() @@ -161,6 +222,18 @@ def test_sana_diffusion_config_instantiates_base_model() -> None: assert isinstance(model.scheduler, SanaWMLTXEulerScheduler) +def test_streaming_diffusion_config_instantiates_base_model() -> None: + """Keep streaming on the shared diffusion loop with streaming components.""" + model = PIPELINE_SANA_WM_STREAMING.diffusion_model.setup() + + assert type(model) is DiffusionModel + assert isinstance(model.transformer.config, SanaWMStreamingTransformerConfig) + assert isinstance(model.scheduler, SanaWMLTXEulerScheduler) + assert model.scheduler.config.denoising_step_list == ( + DEFAULT_STREAMING_DENOISING_STEP_LIST + ) + + def test_runner_pipeline_config_routes_runtime_fields_to_components() -> None: """Apply CLI overrides to the component that owns each runtime field.""" cfg = derive_config( @@ -197,6 +270,64 @@ def test_runner_pipeline_config_routes_runtime_fields_to_components() -> None: assert pipeline.diffusion_model.scheduler.shift == 6.5 +def test_streaming_runner_pipeline_config_routes_runtime_fields_to_components() -> None: + """Apply streaming CLI overrides to the component that owns each field.""" + cfg = derive_config( + RUNNER_SANA_WM_STREAMING, + config_path="stream_config.yaml", + model_path="stream_model.safetensors", + causal_vae_path="local_causal_vae", + refiner_root="local_refiner", + refiner_gemma_root="local_gemma", + stage1_precision="fp8", + step=4, + flow_shift=8.0, + no_sink_token=True, + num_frame_per_block=5, + num_cached_blocks=3, + denoising_step_list=(1000, 500, 100, 0), + refiner_block_size=5, + refiner_kv_max_frames=17, + offload_vae=True, + offload_text_encoder=True, + offload_stage1=True, + ) + + pipeline = _streaming_pipeline_config(cfg, quant_backend="torch-fp8") + + assert isinstance(pipeline.encoder, SanaWMStreamingConditioningEncoderConfig) + assert isinstance(pipeline.decoder, SanaWMStreamingVideoDecoderConfig) + assert pipeline.encoder.config_path == "stream_config.yaml" + assert pipeline.encoder.text_encoder.config_path == "stream_config.yaml" + assert pipeline.encoder.text_encoder.stage1_precision == "fp8" + assert pipeline.encoder.text_encoder.quant_backend == "torch-fp8" + assert pipeline.encoder.text_encoder.offload_text_encoder is True + assert pipeline.encoder.first_frame_encoder.offload_vae is True + assert pipeline.decoder.vae_decoder.config_path == "stream_config.yaml" + assert pipeline.decoder.vae_decoder.vae_path == "local_causal_vae" + assert pipeline.decoder.vae_decoder.offload_vae is True + assert isinstance(pipeline.decoder.refiner, SanaWMStreamingLTX2LatentRefinerConfig) + assert pipeline.decoder.refiner.refiner_root == "local_refiner" + assert pipeline.decoder.refiner.refiner_gemma_root == "local_gemma" + assert pipeline.decoder.refiner.kv_max_frames == 17 + assert pipeline.decoder.refiner.block_size == 5 + assert pipeline.diffusion_model.transformer.checkpoint_path == ( + "stream_model.safetensors" + ) + assert pipeline.diffusion_model.transformer.offload_stage1 is True + assert pipeline.diffusion_model.transformer.num_frame_per_block == 5 + assert pipeline.diffusion_model.transformer.num_cached_blocks == 3 + assert pipeline.diffusion_model.transformer.sink_token is False + assert pipeline.diffusion_model.scheduler.num_inference_steps == 4 + assert pipeline.diffusion_model.scheduler.shift == 8.0 + assert pipeline.diffusion_model.scheduler.denoising_step_list == ( + 1000, + 500, + 100, + 0, + ) + + def test_sana_ltx_scheduler_step_pins_zero_timestep_tokens() -> None: """Keep first-frame tokens fixed in the per-token LTX Euler step.""" scheduler = SanaWMLTXEulerSchedulerConfig(num_inference_steps=4).setup() @@ -274,6 +405,33 @@ def test_sana_ltx_scheduler_matches_diffusers_per_token_step() -> None: torch.testing.assert_close(actual, expected) +def test_sana_ltx_scheduler_uses_explicit_streaming_timesteps() -> None: + """Use the distilled streaming timestep list verbatim when configured.""" + scheduler = SanaWMLTXEulerSchedulerConfig( + denoising_step_list=DEFAULT_STREAMING_DENOISING_STEP_LIST, + ).setup() + + timesteps = scheduler.timesteps( + num_inference_steps=999, + shift=1.0, + device=torch.device("cpu"), + ) + + assert timesteps.dtype == torch.float32 + torch.testing.assert_close( + timesteps, + torch.tensor(DEFAULT_STREAMING_DENOISING_STEP_LIST, dtype=torch.float32), + ) + with pytest.raises(ValueError, match="end with 0"): + SanaWMLTXEulerSchedulerConfig( + denoising_step_list=(1000, 500), + ).setup().timesteps( + num_inference_steps=2, + shift=1.0, + device=torch.device("cpu"), + ) + + def test_sana_transformer_keeps_conditioned_frame_fixed_with_generic_scheduler() -> None: """Keep SANA conditioning out of the scheduler and inside the transformer.""" scheduler = SanaWMLTXEulerSchedulerConfig(num_inference_steps=1).setup() @@ -371,6 +529,120 @@ def test_transformer_initial_noise_uses_conditioning_payload() -> None: torch.testing.assert_close(noise[:, :, :1], first_latent) +def test_streaming_transformer_commits_chunks_into_prefix_cache() -> None: + """Keep streaming Stage-1 chunks in one rollout-level latent cache.""" + + class DummyModel(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.calls: list[dict[str, torch.Tensor]] = [] + + def forward( + self, + noisy_latent: torch.Tensor, + timestep: torch.Tensor, + prompt_embeds: torch.Tensor, + **_kwargs: object, + ) -> tuple[torch.Tensor]: + del prompt_embeds + self.calls.append( + { + "noisy_latent": noisy_latent.detach().clone(), + "timestep": timestep.detach().clone(), + } + ) + return (torch.ones_like(noisy_latent),) + + def conditioning_for_chunk( + *, + start: int, + end: int, + chunk_index: int, + ) -> SanaWMStreamingStage1Conditioning: + return SanaWMStreamingStage1Conditioning( + condition=torch.ones((1, 1, 1)), + uncondition=None, + model_kwargs={ + "data_info": {"condition_frame_info": {0: 0.0}}, + "camera_conditions": torch.zeros((1, 7, 20)), + "chunk_plucker": torch.zeros((1, 2, 7, 1, 1)), + }, + first_latent=torch.full((1, 1, 1, 1, 1), 7.0), + latent_shape=(1, 1, end - start, 1, 1), + cfg_scale=1.0, + flow_shift=8.0, + steps=4, + seed=123, + total_latent_shape=(1, 1, 7, 1, 1), + start_frame=start, + end_frame=end, + chunk_index=chunk_index, + chunk_boundaries=(0, 4, 7), + ) + + transformer = SanaWMStreamingTransformerConfig().setup() + transformer.weight_dtype = torch.float32 + dummy_model = DummyModel() + transformer.model = dummy_model + transformer._model_built = True + chunk0 = conditioning_for_chunk(start=0, end=4, chunk_index=0) + chunk1 = conditioning_for_chunk(start=4, end=7, chunk_index=1) + cache = transformer.initialize_autoregressive_cache(conditioning=chunk0) + assert isinstance(cache, SanaWMStreamingTransformerCache) + + noise0 = transformer.initial_noise( + latent_shape=chunk0.latent_shape, + rng=None, + cache=cache, + input=chunk0, + ) + flow0 = transformer.predict_flow( + noisy_latent=noise0, + timestep=torch.tensor(1000.0), + cache=cache, + input=chunk0, + ) + clean0 = torch.full_like(noise0, 2.0) + transformer.postprocess_clean_latent(clean0, cache, input=chunk0) + + noise1 = transformer.initial_noise( + latent_shape=chunk1.latent_shape, + rng=None, + cache=cache, + input=chunk1, + ) + flow1 = transformer.predict_flow( + noisy_latent=noise1, + timestep=torch.tensor(500.0), + cache=cache, + input=chunk1, + ) + clean1 = torch.full_like(noise1, 4.0) + transformer.postprocess_clean_latent(clean1, cache, input=chunk1) + + assert len(dummy_model.calls) == 2 + assert dummy_model.calls[0]["noisy_latent"].shape == (1, 1, 4, 1, 1) + assert dummy_model.calls[1]["noisy_latent"].shape == (1, 1, 7, 1, 1) + torch.testing.assert_close( + dummy_model.calls[0]["timestep"], + torch.tensor([[[0.0, 1000.0, 1000.0, 1000.0]]]), + ) + torch.testing.assert_close( + dummy_model.calls[1]["timestep"], + torch.tensor([[[0.0, 0.0, 0.0, 0.0, 500.0, 500.0, 500.0]]]), + ) + torch.testing.assert_close(flow0[:, :, :1], torch.zeros_like(flow0[:, :, :1])) + torch.testing.assert_close(flow0[:, :, 1:], torch.ones_like(flow0[:, :, 1:])) + torch.testing.assert_close(flow1, torch.ones_like(flow1)) + assert cache.latent_state is not None + torch.testing.assert_close( + cache.latent_state[:, :, :1], + torch.full((1, 1, 1, 1, 1), 7.0), + ) + torch.testing.assert_close(cache.latent_state[:, :, 1:4], clean0[:, :, 1:]) + torch.testing.assert_close(cache.latent_state[:, :, 4:7], clean1) + + def test_transformer_predict_flow_applies_cfg_from_conditioning_input() -> None: """Keep CFG inside the transformer boundary used by base diffusion.""" @@ -598,6 +870,18 @@ def test_inference_config_loads_yaml( assert cfg.work_dir == "" +def test_inference_config_loads_builtin_streaming_config() -> None: + """Provide FlashDreams-owned config data for the streaming release.""" + cfg = _load_inference_config(SANA_WM_STREAMING_CONFIG_PATH) + + assert cfg.work_dir == "" + assert cfg.model.model == "SanaMSVideoCamCtrlStreaming_1600M_P1_D20" + assert cfg.model.chunk_size == 3 + assert cfg.vae.vae_type == "LTX2VAE_diffusers_causal" + assert cfg.vae.vae_pretrained == SANA_WM_STREAMING_CAUSAL_VAE_ROOT + assert cfg.scheduler.inference_flow_shift == 8.0 + + def test_text_prompt_encoder_outputs_padded_prompt_schema( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -742,6 +1026,20 @@ def test_camera_conditioning_encoder_outputs_sana_schema() -> None: assert camera["chunk_plucker"].shape == (48, 3, 22, 40) +def test_streaming_chunk_boundaries_use_sink_plus_fixed_blocks() -> None: + """Represent streaming AR chunks as sink+block, then block-only steps.""" + assert streaming_chunk_boundaries(total_frames=10, chunk_size=3) == ( + 0, + 4, + 7, + 10, + ) + with pytest.raises(ValueError, match="more than one latent frame"): + streaming_chunk_boundaries(total_frames=1, chunk_size=3) + with pytest.raises(ValueError, match="divide the chunk size"): + streaming_chunk_boundaries(total_frames=9, chunk_size=3) + + def test_video_decoder_returns_structured_video(monkeypatch: pytest.MonkeyPatch) -> None: """Decode Stage-1 latents through the explicit decoder component.""" decoder = SanaWMVideoDecoderConfig(refiner=None).setup() @@ -762,12 +1060,53 @@ def test_video_decoder_returns_structured_video(monkeypatch: pytest.MonkeyPatch) assert decoded.stage1_video_hwc is None +def test_streaming_video_decoder_emits_only_new_frames( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Decode prefixes while returning only frames produced by the new chunk.""" + decoder = SanaWMStreamingVideoDecoderConfig(refiner=None).setup() + + def decode_prefix(latents: torch.Tensor) -> np.ndarray: + frames = 1 + (latents.shape[2] - 1) * 8 + values = np.arange(frames, dtype=np.uint8) + return np.broadcast_to(values[:, None, None, None], (frames, 2, 2, 3)).copy() + + monkeypatch.setattr(decoder.vae_decoder, "decode_latents", decode_prefix) + cache = decoder.initialize_autoregressive_cache() + + first = decoder( + torch.zeros((1, 4, 4, 1, 1)), + autoregressive_index=0, + cache=cache, + ) + second = decoder( + torch.zeros((1, 4, 3, 1, 1)), + autoregressive_index=1, + cache=cache, + ) + + assert first.video_hwc.shape == (24, 2, 2, 3) + assert second.video_hwc.shape == (24, 2, 2, 3) + assert first.video_hwc[0, 0, 0, 0] == 1 + assert second.video_hwc[0, 0, 0, 0] == 25 + assert cache.emitted_frames == 49 + assert cache.stage1_chunks[0].shape[2] == 4 + assert cache.stage1_chunks[1].shape[2] == 3 + assert first.stage1_video_hwc is None + assert second.stage1_video_hwc is None + + def test_stage1_model_matches_checkpoint_schema() -> None: """Pin the Stage-1 module to the public checkpoint schema.""" state = SanaWMStage1Model().state_dict() assert SANA_WM_STAGE1_SPEC.chunk_size is None assert SANA_WM_STAGE1_SPEC.chunk_split_strategy == "first_chunk_plus_one" + assert SANA_WM_STREAMING_STAGE1_SPEC.chunk_size == 3 + assert SANA_WM_STREAMING_STAGE1_SPEC.chunk_split_strategy == ( + "first_chunk_plus_one" + ) + assert SANA_WM_STREAMING_STAGE1_SPEC.depth == SANA_WM_STAGE1_SPEC.depth assert len(state) == 872 assert tuple(state["x_embedder.proj.weight"].shape) == (2240, 128, 1, 1, 1) assert tuple(state["raymap_embedder.proj.weight"].shape) == (2240, 3, 1, 1, 1) @@ -1218,6 +1557,27 @@ def test_hf_defaults_point_at_bidirectional_release() -> None: ) +def test_hf_defaults_point_at_streaming_release() -> None: + """Pin every default streaming artefact to the streaming HF repo.""" + assert SANA_WM_STREAMING_HF_REPO == "Efficient-Large-Model/SANA-WM_streaming" + assert SANA_WM_STREAMING_MODEL_PATH == ( + "hf://Efficient-Large-Model/SANA-WM_streaming/sana_dit/model.pt" + ) + assert SANA_WM_STREAMING_CONFIG_PATH == ( + "flashdreams://sana-wm-streaming-1600m-720p" + ) + assert SANA_WM_STREAMING_CAUSAL_VAE_ROOT == ( + "hf://Efficient-Large-Model/SANA-WM_streaming/ltx2_causal_vae" + ) + assert SANA_WM_STREAMING_REFINER_ROOT == ( + "hf://Efficient-Large-Model/SANA-WM_streaming/refiner_diffusers" + ) + assert SANA_WM_STREAMING_REFINER_GEMMA_ROOT == ( + "hf://Efficient-Large-Model/SANA-WM_streaming/gemma3_12b" + ) + assert SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES == 11 + + def test_runner_setup_preserves_cli_fields() -> None: """Construct the runner and preserve CLI override fields.""" cfg = derive_config( @@ -1232,6 +1592,22 @@ def test_runner_setup_preserves_cli_fields() -> None: assert runner.config.image_path == Path("missing.png") +def test_streaming_runner_setup_preserves_cli_fields() -> None: + """Construct the streaming runner and preserve CLI override fields.""" + cfg = derive_config( + RUNNER_SANA_WM_STREAMING, + image_path=Path("missing.png"), + prompt="demo", + num_frame_per_block=5, + ) + + runner = cfg.setup() + + assert isinstance(runner, SanaWMStreamingRunner) + assert runner.config.image_path == Path("missing.png") + assert runner.config.num_frame_per_block == 5 + + def test_runner_derives_intrinsics_when_omitted(tmp_path: Path) -> None: """Omitting --intrinsics-path derives per-frame intrinsics from the frame.""" from PIL import Image @@ -1289,6 +1665,7 @@ def test_runner_fits_camera_path_to_requested_frames(tmp_path: Path) -> None: def test_runner_config_type() -> None: """Keep the exported literal on the SANA-WM runner config subclass.""" assert isinstance(RUNNER_SANA_WM_BIDIRECTIONAL, SanaWMRunnerConfig) + assert isinstance(RUNNER_SANA_WM_STREAMING, SanaWMStreamingRunnerConfig) def test_runner_defaults_to_bf16_precision() -> None: @@ -1297,6 +1674,14 @@ def test_runner_defaults_to_bf16_precision() -> None: assert RUNNER_SANA_WM_BIDIRECTIONAL.refiner_precision == "bf16" assert RUNNER_SANA_WM_BIDIRECTIONAL.quant_backend == "auto" assert RUNNER_SANA_WM_BIDIRECTIONAL.no_refiner is False + assert RUNNER_SANA_WM_STREAMING.stage1_precision == "bf16" + assert RUNNER_SANA_WM_STREAMING.refiner_precision == "bf16" + assert RUNNER_SANA_WM_STREAMING.quant_backend == "auto" + assert RUNNER_SANA_WM_STREAMING.no_refiner is False + assert RUNNER_SANA_WM_STREAMING.num_frame_per_block == 3 + assert RUNNER_SANA_WM_STREAMING.denoising_step_list == ( + DEFAULT_STREAMING_DENOISING_STEP_LIST + ) def test_stage1_quant_scope_matches_upstream_precision_cli() -> None: @@ -1740,6 +2125,7 @@ def test_pyproject_entry_point_matches_runner_literal() -> None: assert entry_points == { "sana-wm-bidirectional": "sana_wm.config:RUNNER_SANA_WM_BIDIRECTIONAL", + "sana-wm-streaming": "sana_wm.config:RUNNER_SANA_WM_STREAMING", } diff --git a/uv.lock b/uv.lock index 9b271241a..e752fb27d 100644 --- a/uv.lock +++ b/uv.lock @@ -1603,6 +1603,7 @@ dependencies = [ { name = "pillow" }, { name = "pyyaml" }, { name = "safetensors" }, + { name = "sentencepiece" }, { name = "torchvision", version = "0.26.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform != 'win32' and extra == 'group-11-flashdreams-cuda12') or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, { name = "torchvision", version = "0.27.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'win32' and extra == 'extra-11-flashdreams-dev') or (sys_platform != 'win32' and extra != 'group-11-flashdreams-cuda12') or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, { name = "torchvision", version = "0.27.1+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "sys_platform == 'win32' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, @@ -1624,6 +1625,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "safetensors", specifier = ">=0.5" }, + { name = "sentencepiece", specifier = ">=0.2" }, { name = "torchvision", specifier = ">=0.26" }, { name = "transformers", specifier = ">=5.0,<6" }, ] @@ -5230,6 +5232,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/19/969dc072906c84dd0a3b05dcf57ea750936087d7873549e408b35cfc3f97/scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8", size = 25279661, upload-time = "2026-06-19T15:01:40.817Z" }, ] +[[package]] +name = "sentencepiece" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/33/ea3cb3839607eb175da835244a798f797f478c5ddf0e8ecdf57ea85a4c70/sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6", size = 8218435, upload-time = "2026-07-12T08:39:34.701Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/1b/e6c69e4c2026ed575d68dda2847a404468ca7b5fa684bb0b19f71d82d29d/sentencepiece-0.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e", size = 2180607, upload-time = "2026-07-12T08:38:01.018Z" }, + { url = "https://files.pythonhosted.org/packages/36/5a/2a1d84c87dc075d4f8cf1a2470a95399e59834e219ffb5f4285533e750d0/sentencepiece-0.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b", size = 1437502, upload-time = "2026-07-12T08:38:02.899Z" }, + { url = "https://files.pythonhosted.org/packages/1b/39/3d43a75dd5a22503ca5074d0d37707cabb2e4a71b4bc6e6c61be3643cc7a/sentencepiece-0.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914", size = 1345667, upload-time = "2026-07-12T08:38:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/90/d5/a69a8cc896e7de3fe2061b08c2f33e28656f243bed8af6a2df9f5d8c3124/sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711", size = 1322864, upload-time = "2026-07-12T08:38:06.49Z" }, + { url = "https://files.pythonhosted.org/packages/e4/79/dd1836df32971d4eb14ff5cb4a8b3fe4419adbeada8e81d09dc53c5c0ef0/sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d", size = 1392757, upload-time = "2026-07-12T08:38:08.559Z" }, + { url = "https://files.pythonhosted.org/packages/26/83/c3547715c29b7e4c84a180a240267f7685dde6f9b981396f16b95405ec9d/sentencepiece-0.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0", size = 1245044, upload-time = "2026-07-12T08:38:10.21Z" }, + { url = "https://files.pythonhosted.org/packages/1f/55/7da03b35582a4eb276f99051109f3e3e8f176835b6d6837422e4c3a013dd/sentencepiece-0.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936", size = 1190467, upload-time = "2026-07-12T08:38:12.07Z" }, + { url = "https://files.pythonhosted.org/packages/20/31/f23a2efaa0210b883574001b88fa64e499f798f0848a0b610fb9b384d162/sentencepiece-0.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0", size = 2184255, upload-time = "2026-07-12T08:38:14.855Z" }, + { url = "https://files.pythonhosted.org/packages/96/f2/1ee0ccb772d71e822f625d6cb5f0ea825835e877f28a9ef299a1291df19e/sentencepiece-0.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790", size = 1438545, upload-time = "2026-07-12T08:38:16.674Z" }, + { url = "https://files.pythonhosted.org/packages/2a/92/3a6ea4a2c6dd9e7062698a5a33534ca0e20844883338ae9c6b9c122c1a9f/sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e", size = 1346997, upload-time = "2026-07-12T08:38:18.499Z" }, + { url = "https://files.pythonhosted.org/packages/f3/3a/7839048997c7bc0c34c57526f539f835e20c7a57dc2a99f99579b11cdbef/sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e", size = 1324282, upload-time = "2026-07-12T08:38:20.342Z" }, + { url = "https://files.pythonhosted.org/packages/06/5f/9117bf854aef817ad0d0ee9310eed0308a7e529e7eaf2e80ad9cd281ef82/sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107", size = 1394242, upload-time = "2026-07-12T08:38:22.976Z" }, + { url = "https://files.pythonhosted.org/packages/ab/62/9e2569867e3dcff7ad6d89642a9615b9801b5cd698abe7df3b490361f66e/sentencepiece-0.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e", size = 1246268, upload-time = "2026-07-12T08:38:24.857Z" }, + { url = "https://files.pythonhosted.org/packages/96/c9/5d781d4ef1124564a45c98b9ff25d531c10cdf568ec6314a2d1946f9251c/sentencepiece-0.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c", size = 1190702, upload-time = "2026-07-12T08:38:26.789Z" }, + { url = "https://files.pythonhosted.org/packages/b8/13/7a562289c8d5b49ebdf3f9c1e8ab67cf14a8743b1d90c8f406bfdec36b72/sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b", size = 2188384, upload-time = "2026-07-12T08:38:28.486Z" }, + { url = "https://files.pythonhosted.org/packages/85/d1/912f14fd5eae168aba726ffb6a9a2dc1c71fe7676c53da6f5c442b886d4a/sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820", size = 1441553, upload-time = "2026-07-12T08:38:30.552Z" }, + { url = "https://files.pythonhosted.org/packages/bd/44/caa9cab5f261a019e2808bc5046152775dc57352ba9cbae7525e9e7a1ed4/sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838", size = 1347176, upload-time = "2026-07-12T08:38:32.211Z" }, + { url = "https://files.pythonhosted.org/packages/19/90/cd798935668cff71d309d8ff10385844ecf216b1fe454f1993ed8bf2cb91/sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a", size = 1325200, upload-time = "2026-07-12T08:38:33.689Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2d/37e3da037318a70066ded0d51bc2a7f35491ae6338dd993d5eb1503fc3b5/sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da", size = 1397736, upload-time = "2026-07-12T08:38:35.771Z" }, + { url = "https://files.pythonhosted.org/packages/8d/11/753fca2e6b109be3ab7867abf357dfe48677fe726ae5a5363d0b54ca9450/sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383", size = 1248030, upload-time = "2026-07-12T08:38:37.604Z" }, + { url = "https://files.pythonhosted.org/packages/e2/0a/70efbe861ca182d7d4b6e1a20f58e043400848fa9f2915229f082e221648/sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d", size = 1187325, upload-time = "2026-07-12T08:38:39.348Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a3/b3b05095c174d6e80d37d5ddc2f57c2c56237333e7bbd6079cf3243c2a8a/sentencepiece-0.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:77c3ce990b23441e5ecfa5bce181fd6f408b564aeb6d7e1d1e7de9c5612501c8", size = 2188346, upload-time = "2026-07-12T08:38:41.089Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f3/72ebc4acb10a06bcf7503fbc6091c8f5db68300f6aac4356c09e6c76e0e1/sentencepiece-0.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd523c4992041faa5c2b3cde62253d11a96c30d73a34afe48a486e8e2254cd1c", size = 1441434, upload-time = "2026-07-12T08:38:42.56Z" }, + { url = "https://files.pythonhosted.org/packages/34/db/f9ea1a6844b4fa5dfe2312095cd866a1f724cd0905054ab9d5991778ba50/sentencepiece-0.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:201a8e0f55501a76e08dbf2c54bc45f4642b379271e89c667d517bfbc2191f2a", size = 1347267, upload-time = "2026-07-12T08:38:44.389Z" }, + { url = "https://files.pythonhosted.org/packages/32/4f/31c1073314ad94466bca37d29581761d70110237ee3d46b0efece59a8c1e/sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0", size = 1324980, upload-time = "2026-07-12T08:38:46.304Z" }, + { url = "https://files.pythonhosted.org/packages/59/b4/a0356fa04d6a14337a6e0e443556785a0422c53ec58baae6b9568120eb0f/sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb", size = 1397593, upload-time = "2026-07-12T08:38:48.302Z" }, + { url = "https://files.pythonhosted.org/packages/09/fa/d2d6369257fd2f0de616b1c7110b73fab409ef61b14f1b9e0010ed325914/sentencepiece-0.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:74f0ee601047c0c12a783088b51be4e6214a62ecd9e02278c477433cd16e0ed9", size = 1247987, upload-time = "2026-07-12T08:38:50.15Z" }, + { url = "https://files.pythonhosted.org/packages/17/ee/2bb594da6fd95e32f29057f1aa7fa996701b8980090923c2d8711fdc0a24/sentencepiece-0.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:b23fe17779834d3c27aaf2edac9486d04cca1a7deb8f5facda35150ac6263a91", size = 1187250, upload-time = "2026-07-12T08:38:52.246Z" }, + { url = "https://files.pythonhosted.org/packages/58/9c/dfc82846460e7a712310f5613f23d8b553cabb4e2e648663c11d8382af56/sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:72b7825b331b1b7e7c45be2e674b3e3c65af608fa376bad2d851b20aaf0cdc78", size = 2223080, upload-time = "2026-07-12T08:38:54.391Z" }, + { url = "https://files.pythonhosted.org/packages/8d/4e/3ff12cebe6d31662d9ceeabfb282de20bd0d6098fa282b4a3b8305abc7e8/sentencepiece-0.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d795c4ac689a57f9d4ba2288126ec7901d389ad5827d2f8b8533c883974fe563", size = 1458511, upload-time = "2026-07-12T08:38:56.811Z" }, + { url = "https://files.pythonhosted.org/packages/59/5a/16d51d05360be4cee3ebfe4837c184054c4eed16cabaeb3b039524e9a000/sentencepiece-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ab3f1ae98970b5590e2209341522718900ba19bcc2c207ffaa6bd417ad960c5", size = 1361138, upload-time = "2026-07-12T08:38:58.808Z" }, + { url = "https://files.pythonhosted.org/packages/0f/af/c30ee2a9f99d51db9844acaa8fa0b611a97c2fa7116646fa43db3300b187/sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d", size = 1328625, upload-time = "2026-07-12T08:39:00.849Z" }, + { url = "https://files.pythonhosted.org/packages/3e/1a/4c6b39d03f5ba8439509adbd5a23c9538088a3cb679e7a47b911e8442bc6/sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b", size = 1398595, upload-time = "2026-07-12T08:39:02.86Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bc/9eedddcec1fd57bc70200fa3ebf792d18fa63527a5369581cd416c81f97f/sentencepiece-0.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:89625fb43765cccaa1443b9adb61f283e5fe4cb1536728205d06bada730caa53", size = 1259346, upload-time = "2026-07-12T08:39:04.559Z" }, + { url = "https://files.pythonhosted.org/packages/41/15/7e74c8533848866ff560b29f7d8719921b76c4ec7149592d6d28e0deee75/sentencepiece-0.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:4f0603267cd15b92b68c2c0e852a441507614b70dc7773659baa6b8c214a91fd", size = 1196596, upload-time = "2026-07-12T08:39:06.454Z" }, + { url = "https://files.pythonhosted.org/packages/0b/7e/f5df63edb6bcb46c1343cfa5d9192d73a4eb61af2e800d9402efff387523/sentencepiece-0.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c62bd361cec1f5b556eb8210264ecfff37486cd990c3386cc00310f26c54090a", size = 2190240, upload-time = "2026-07-12T08:39:08.178Z" }, + { url = "https://files.pythonhosted.org/packages/52/0a/095d183b453b2a2e20b016829029c58eca90adc1c9911113e5d26fff45ed/sentencepiece-0.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:46ba07b543add034de0ff47ac5f907e9a06682f91d85121a972764628933be6b", size = 1442220, upload-time = "2026-07-12T08:39:09.91Z" }, + { url = "https://files.pythonhosted.org/packages/d1/18/823954c9c90e74eba09fb96752dc37a5555df00d69866cb9406d1725dc7e/sentencepiece-0.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:79bac5a251f23a7341e28fda9ce0d5319edf45328239ce037c0682936f137906", size = 1348056, upload-time = "2026-07-12T08:39:11.744Z" }, + { url = "https://files.pythonhosted.org/packages/10/ca/1b6c251321901cbf8a2d2e48b8b70eb82a449011b766af52a228d0a90b6b/sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719", size = 1325463, upload-time = "2026-07-12T08:39:13.413Z" }, + { url = "https://files.pythonhosted.org/packages/24/b3/718847349da7b25c8220ed86d85b89080af94740b2d87a59198104ae5c51/sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab", size = 1398138, upload-time = "2026-07-12T08:39:15.564Z" }, + { url = "https://files.pythonhosted.org/packages/33/fe/4906f12c458274edd96387e4baaad7c6f064a2b7c11a1cc2401c8a7bd483/sentencepiece-0.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:63250cfab8b80a1ef82a614eb2b3cadfec2c405f870cedc139d08e2f063eb708", size = 1356144, upload-time = "2026-07-12T08:39:17.313Z" }, + { url = "https://files.pythonhosted.org/packages/d3/eb/22f89b6542aba400b0007cf0b1697cc3f99be8fb682fdb4c05eec450e33f/sentencepiece-0.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:65d84ec36888de4a848eee5f910e67fbc79b064685ef1e10a502e14520ead9c9", size = 1294351, upload-time = "2026-07-12T08:39:18.967Z" }, + { url = "https://files.pythonhosted.org/packages/84/c4/7afe8c2315b76e46818851a057e50a378a0382aa00b970a1fa444181b6f6/sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d254c98ca6387655400b3959c33c83efd807f5edeb608e3aca45800ceaa77151", size = 2223281, upload-time = "2026-07-12T08:39:20.978Z" }, + { url = "https://files.pythonhosted.org/packages/98/42/fb678e472c554ef086be6375d20060ca610a2c4218854d4c091001fc6f91/sentencepiece-0.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3fd9ce2ab4460c713cfdeb4aca693ca6732a11538e05fb332d5af42e3d7fde25", size = 1458779, upload-time = "2026-07-12T08:39:22.812Z" }, + { url = "https://files.pythonhosted.org/packages/78/52/ffe402b13bce1889228a98dc6cd86ae8afac1112362236be3468be784441/sentencepiece-0.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7fc14c1585139fa6b68775e616a6b90cf622ebf219f9558c0aeaf5d253ee6c9b", size = 1361736, upload-time = "2026-07-12T08:39:24.602Z" }, + { url = "https://files.pythonhosted.org/packages/78/4a/2288f60e7283583ec0a0f16e72f9c8e68557d7e7a4b585d2cda4f9f47e64/sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811", size = 1328155, upload-time = "2026-07-12T08:39:26.422Z" }, + { url = "https://files.pythonhosted.org/packages/26/31/5dd6882ebe899f741a5cfe40ff56c6efc06bc26ee287abdb723b671f409c/sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf", size = 1398307, upload-time = "2026-07-12T08:39:28.637Z" }, + { url = "https://files.pythonhosted.org/packages/da/05/7d7780fa63f4b8c1821953b916e25f89ae8f14d4da6ba91e10f6d06dc2b4/sentencepiece-0.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:046b15ea22d8042e2e173561d464ec3b64a9c2081324df70ebce7bf7ebb3e497", size = 1367133, upload-time = "2026-07-12T08:39:30.546Z" }, + { url = "https://files.pythonhosted.org/packages/49/a1/70007fef3f818c688de4a730f98024a671599ab67f20270f8efb03d69dcc/sentencepiece-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fa9f5ef0e2a82233dd0b8b32ea3f5710e0c44afbc07ed3620219f32601e56090", size = 1302760, upload-time = "2026-07-12T08:39:32.457Z" }, +] + [[package]] name = "setuptools" version = "81.0.0" From ec508c540ca09eea2df3f9d1a05487c90f675cad Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Wed, 29 Jul 2026 16:34:19 -0700 Subject: [PATCH 39/64] Rename SANA benchmark native helper to FlashDreams --- integrations/sana/tests/parity_check/bench.sh | 46 ++++++------- .../sana/tests/parity_check/bench_summary.py | 64 +++++++++---------- integrations/sana/tests/parity_check/run.sh | 44 ++++++------- .../{run_native.py => run_flashdreams.py} | 2 +- 4 files changed, 78 insertions(+), 78 deletions(-) rename integrations/sana/tests/parity_check/{run_native.py => run_flashdreams.py} (99%) diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index ce2022a83..846cf399a 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -113,7 +113,7 @@ else OUTPUT_DIR="$(_abspath "${OUTPUT_DIR:-${DEFAULT_OUTPUT_DIR}}")" fi UPSTREAM_ROOT="${OUTPUT_DIR}/upstream" -NATIVE_ROOT="${OUTPUT_DIR}/flashdreams" +FLASHDREAMS_ROOT="${OUTPUT_DIR}/flashdreams" IMAGE_PATH="$(_abspath "${IMAGE_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.png}")" PROMPT_PATH="$(_abspath "${PROMPT_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.txt}")" @@ -254,7 +254,7 @@ fi mkdir -p "${UPSTREAM_ROOT}" if [[ "${BENCH_SIDE}" == "both" ]]; then - mkdir -p "${NATIVE_ROOT}" + mkdir -p "${FLASHDREAMS_ROOT}" fi # Remove stale run_* dirs from prior (possibly longer) benchmarks. bench_summary @@ -262,32 +262,32 @@ fi # would silently pollute this invocation's aggregate. rm -rf "${UPSTREAM_ROOT:?}"/run_* if [[ "${BENCH_SIDE}" == "both" ]]; then - rm -rf "${NATIVE_ROOT:?}"/run_* + rm -rf "${FLASHDREAMS_ROOT:?}"/run_* fi UPSTREAM_REFINER_ARGS=() -NATIVE_REFINER_ARGS=() +FLASHDREAMS_REFINER_ARGS=() if _is_true "${NO_REFINER}"; then UPSTREAM_REFINER_ARGS+=(--no_refiner) - NATIVE_REFINER_ARGS+=(--no-refiner) + FLASHDREAMS_REFINER_ARGS+=(--no-refiner) fi UPSTREAM_BACKEND_ARGS=() -NATIVE_BACKEND_ARGS=() +FLASHDREAMS_BACKEND_ARGS=() if _is_true "${FORCE_CUDNN_SDPA}"; then UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) - NATIVE_BACKEND_ARGS+=(--force-cudnn-sdpa) + FLASHDREAMS_BACKEND_ARGS+=(--force-cudnn-sdpa) fi UPSTREAM_COMPILE_ARGS=() -NATIVE_COMPILE_ARGS=() +FLASHDREAMS_COMPILE_ARGS=() if _is_true "${COMPILE_STAGE1}"; then UPSTREAM_COMPILE_ARGS+=(--compile_stage1) - NATIVE_COMPILE_ARGS+=(--compile-stage1) + FLASHDREAMS_COMPILE_ARGS+=(--compile-stage1) fi UPSTREAM_PRECISION_ARGS=( --stage1_precision "${STAGE1_PRECISION}" --refiner_precision "${REFINER_PRECISION}" ) -NATIVE_PRECISION_ARGS=( +FLASHDREAMS_PRECISION_ARGS=( --stage1-precision "${STAGE1_PRECISION}" --refiner-precision "${REFINER_PRECISION}" --quant-backend "${QUANT_BACKEND}" @@ -372,30 +372,30 @@ for ((i = 0; i < TOTAL_RUNS; i++)); do ( cd "${SCRIPT_DIR}" && "${UPSTREAM_CMD[@]}" ) if [[ "${BENCH_SIDE}" == "both" ]]; then - NATIVE_OUT="${NATIVE_ROOT}/run_${i}" - mkdir -p "${NATIVE_OUT}" - NATIVE_CMD=( - uv run python "${SCRIPT_DIR}/run_native.py" + FLASHDREAMS_OUT="${FLASHDREAMS_ROOT}/run_${i}" + mkdir -p "${FLASHDREAMS_OUT}" + FLASHDREAMS_CMD=( + uv run python "${SCRIPT_DIR}/run_flashdreams.py" --image-path "${IMAGE_PATH}" --prompt-path "${PROMPT_PATH}" --camera-path "${CAMERA_PATH}" --intrinsics-path "${INTRINSICS_PATH}" - --output-dir "${NATIVE_OUT}" + --output-dir "${FLASHDREAMS_OUT}" --name flashdreams --num-frames "${NUM_FRAMES}" --fps "${FPS}" --step "${STEP}" --cfg-scale "${CFG_SCALE}" --seed "${SEED}" - --stats-json "${NATIVE_OUT}/stats.json" - "${NATIVE_PRECISION_ARGS[@]}" - "${NATIVE_BACKEND_ARGS[@]}" - "${NATIVE_COMPILE_ARGS[@]}" - "${NATIVE_REFINER_ARGS[@]}" + --stats-json "${FLASHDREAMS_OUT}/stats.json" + "${FLASHDREAMS_PRECISION_ARGS[@]}" + "${FLASHDREAMS_BACKEND_ARGS[@]}" + "${FLASHDREAMS_COMPILE_ARGS[@]}" + "${FLASHDREAMS_REFINER_ARGS[@]}" ) echo "[bench] FlashDreams ${SANA_WM_VARIANT} run ${i}/${TOTAL_RUNS}" - _write_command "${NATIVE_OUT}/command.txt" "${NATIVE_CMD[@]}" - ( cd "${SCRIPT_DIR}" && "${NATIVE_CMD[@]}" ) + _write_command "${FLASHDREAMS_OUT}/command.txt" "${FLASHDREAMS_CMD[@]}" + ( cd "${SCRIPT_DIR}" && "${FLASHDREAMS_CMD[@]}" ) fi done @@ -447,7 +447,7 @@ if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then ) fi if [[ "${BENCH_SIDE}" == "both" ]]; then - SUMMARY_ARGS+=(--flashdreams-dir "${NATIVE_ROOT}" --output-chart-md "${SUMMARY_CHART_MD}") + SUMMARY_ARGS+=(--flashdreams-dir "${FLASHDREAMS_ROOT}" --output-chart-md "${SUMMARY_CHART_MD}") fi echo "[bench] summarising -> ${SUMMARY_MD}" diff --git a/integrations/sana/tests/parity_check/bench_summary.py b/integrations/sana/tests/parity_check/bench_summary.py index bc06b3c0b..1811b0255 100644 --- a/integrations/sana/tests/parity_check/bench_summary.py +++ b/integrations/sana/tests/parity_check/bench_summary.py @@ -61,7 +61,7 @@ def _upstream_stage_ms(item: dict[str, Any], key: str) -> float | None: return None -def _native_stage_ms(item: dict[str, Any], key: str) -> float | None: +def _flashdreams_stage_ms(item: dict[str, Any], key: str) -> float | None: stats = item.get("stats_ms") if not isinstance(stats, dict): return None @@ -74,7 +74,7 @@ def _upstream_mem_gib(item: dict[str, Any]) -> float | None: return float(value) if isinstance(value, (int, float)) else None -def _native_mem_gib(item: dict[str, Any]) -> float | None: +def _flashdreams_mem_gib(item: dict[str, Any]) -> float | None: stats = item.get("stats_ms") if not isinstance(stats, dict): return None @@ -342,8 +342,8 @@ def _sum_optional(*values: float | None) -> float | None: def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: upstream = summary["upstream"] - native = summary.get("flashdreams") - has_native = isinstance(native, dict) + flashdreams = summary.get("flashdreams") + has_flashdreams = isinstance(flashdreams, dict) rows = [ "# SANA-WM parity harness benchmark", "", @@ -373,33 +373,33 @@ def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: "With the default `NO_REFINER=1`, the timed work covers conditioning, Stage-1 DiT, and SANA VAE decode.", "", ] - if has_native: + if has_flashdreams: rows.extend( [ "| metric | upstream | FlashDreams |", "| --- | ---: | ---: |", - f"| measured runs | {upstream['runs_measured']} | {native['runs_measured']} |", + f"| measured runs | {upstream['runs_measured']} | {flashdreams['runs_measured']} |", f"| generation median / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream'), ' ms')} | {_fmt(_generation_ms_per_clip(summary, 'flashdreams'), ' ms')} |", f"| generation p90 / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_generation_ms_per_clip(summary, 'flashdreams', percentile='p90'), ' ms')} |", f"| generation median / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'upstream'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams'), ' ms')} |", f"| generation p90 / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams', percentile='p90'), ' ms')} |", - f"| wall median | {_fmt(upstream['wall_median_s'], ' s')} | {_fmt(native['wall_median_s'], ' s')} |", - f"| wall p90 | {_fmt(upstream['wall_p90_s'], ' s')} | {_fmt(native['wall_p90_s'], ' s')} |", + f"| wall median | {_fmt(upstream['wall_median_s'], ' s')} | {_fmt(flashdreams['wall_median_s'], ' s')} |", + f"| wall p90 | {_fmt(upstream['wall_p90_s'], ' s')} | {_fmt(flashdreams['wall_p90_s'], ' s')} |", "", "## Timing breakdown", "", "| stage | upstream median | FlashDreams median |", "| --- | ---: | ---: |", - f"| Stage-1 incl. conditioning median | {_fmt(upstream['stage1_total_median_ms'], ' ms')} | {_fmt(_sum_optional(native['encode_median_ms'], native['dit_median_ms']), ' ms')} |", - f"| Stage-1 DiT median | {_fmt(upstream['dit_median_ms'], ' ms')} | {_fmt(native['dit_median_ms'], ' ms')} |", - f"| conditioning/encode median | n/a | {_fmt(native['encode_median_ms'], ' ms')} |", + f"| Stage-1 incl. conditioning median | {_fmt(upstream['stage1_total_median_ms'], ' ms')} | {_fmt(_sum_optional(flashdreams['encode_median_ms'], flashdreams['dit_median_ms']), ' ms')} |", + f"| Stage-1 DiT median | {_fmt(upstream['dit_median_ms'], ' ms')} | {_fmt(flashdreams['dit_median_ms'], ' ms')} |", + f"| conditioning/encode median | n/a | {_fmt(flashdreams['encode_median_ms'], ' ms')} |", ] ) if summary["inputs"]["no_refiner"]: # No refiner: FlashDreams `decode_ms` and upstream `vae_decode_s` are # both the pure SANA VAE decode, so they compare directly. rows.append( - f"| VAE decode median | {_fmt(upstream['vae_decode_median_ms'], ' ms')} | {_fmt(native['vae_decode_median_ms'], ' ms')} |" + f"| VAE decode median | {_fmt(upstream['vae_decode_median_ms'], ' ms')} | {_fmt(flashdreams['vae_decode_median_ms'], ' ms')} |" ) else: # Refiner enabled: both sides bundle the refiner denoise together @@ -409,10 +409,10 @@ def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: # `vae_decode_s` is the Stage-1 decode and is NOT comparable to # FlashDreams `decode_ms`. rows.append( - f"| refiner + VAE decode median | {_fmt(upstream.get('refiner_median_ms'), ' ms')} | {_fmt(native['vae_decode_median_ms'], ' ms')} |" + f"| refiner + VAE decode median | {_fmt(upstream.get('refiner_median_ms'), ' ms')} | {_fmt(flashdreams['vae_decode_median_ms'], ' ms')} |" ) rows.append( - f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} | {_fmt(native['mem_peak_median_gib'], ' GiB')} |" + f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} | {_fmt(flashdreams['mem_peak_median_gib'], ' GiB')} |" ) else: rows.extend( @@ -443,11 +443,11 @@ def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: def _render_streaming_markdown(summary: dict[str, Any]) -> str: upstream = summary["upstream"] - native = summary.get("flashdreams") - has_native = isinstance(native, dict) + flashdreams = summary.get("flashdreams") + has_flashdreams = isinstance(flashdreams, dict) title = ( "# SANA-WM streaming benchmark" - if has_native + if has_flashdreams else "# SANA-WM streaming upstream benchmark" ) rows = [ @@ -475,17 +475,17 @@ def _render_streaming_markdown(summary: dict[str, Any]) -> str: "Full-clip wall time is retained as supporting data.", "", ] - if has_native: + if has_flashdreams: rows.extend( [ "| metric | upstream | FlashDreams |", "| --- | ---: | ---: |", - f"| measured runs | {upstream['runs_measured']} | {native['runs_measured']} |", + f"| measured runs | {upstream['runs_measured']} | {flashdreams['runs_measured']} |", f"| steady-state generation median / chunk | {_fmt(_streaming_generation_ms_per_chunk(summary, 'upstream'), ' ms')} | {_fmt(_streaming_generation_ms_per_chunk(summary, 'flashdreams'), ' ms')} |", f"| steady-state generation p90 / chunk | {_fmt(_streaming_generation_ms_per_chunk(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_streaming_generation_ms_per_chunk(summary, 'flashdreams', percentile='p90'), ' ms')} |", - f"| full-clip wall median | {_fmt(upstream['wall_median_s'], ' s')} | {_fmt(native['wall_median_s'], ' s')} |", - f"| first chunk median | {_fmt(upstream['first_chunk_median_s'], ' s')} | {_fmt(native['first_chunk_median_s'], ' s')} |", - f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} | {_fmt(native['mem_peak_median_gib'], ' GiB')} |", + f"| full-clip wall median | {_fmt(upstream['wall_median_s'], ' s')} | {_fmt(flashdreams['wall_median_s'], ' s')} |", + f"| first chunk median | {_fmt(upstream['first_chunk_median_s'], ' s')} | {_fmt(flashdreams['first_chunk_median_s'], ' s')} |", + f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} | {_fmt(flashdreams['mem_peak_median_gib'], ' GiB')} |", ] ) else: @@ -511,14 +511,14 @@ def _render_streaming_markdown(summary: dict[str, Any]) -> str: "", ] ) - if has_native: + if has_flashdreams: rows.extend( [ "| stage | upstream median | FlashDreams median |", "| --- | ---: | ---: |", - f"| Stage-1 CUDA total | {_fmt(upstream['stage1_cuda_median_ms'], ' ms')} | {_fmt(native['stage1_cuda_median_ms'], ' ms')} |", - f"| refiner CUDA total | {_fmt(upstream['refiner_cuda_median_ms'], ' ms')} | {_fmt(native['refiner_cuda_median_ms'], ' ms')} |", - f"| decode CUDA total | {_fmt(upstream['decode_cuda_median_ms'], ' ms')} | {_fmt(native['decode_cuda_median_ms'], ' ms')} |", + f"| Stage-1 CUDA total | {_fmt(upstream['stage1_cuda_median_ms'], ' ms')} | {_fmt(flashdreams['stage1_cuda_median_ms'], ' ms')} |", + f"| refiner CUDA total | {_fmt(upstream['refiner_cuda_median_ms'], ' ms')} | {_fmt(flashdreams['refiner_cuda_median_ms'], ' ms')} |", + f"| decode CUDA total | {_fmt(upstream['decode_cuda_median_ms'], ' ms')} | {_fmt(flashdreams['decode_cuda_median_ms'], ' ms')} |", ] ) else: @@ -626,7 +626,7 @@ def main(argv: list[str] | None = None) -> None: parser.error("--flashdreams-dir is required when --bench-side=both") upstream_items = _load_stats(args.upstream_dir) - native_items = _load_stats(args.flashdreams_dir) if args.flashdreams_dir is not None else [] + flashdreams_items = _load_stats(args.flashdreams_dir) if args.flashdreams_dir is not None else [] if not upstream_items: raise ValueError(f"no upstream stats found in {args.upstream_dir}") summary = { @@ -636,7 +636,7 @@ def main(argv: list[str] | None = None) -> None: "flashdreams_commit": args.flashdreams_commit, "commands": { "upstream": _first_command(upstream_items), - "flashdreams": _first_command(native_items), + "flashdreams": _first_command(flashdreams_items), }, "inputs": { "image_path": str(args.image_path), @@ -668,7 +668,7 @@ def main(argv: list[str] | None = None) -> None: if args.variant == "streaming": summary["upstream"] = _collect_streaming(upstream_items, args.warmup_runs) if args.bench_side == "both": - summary["flashdreams"] = _collect_streaming(native_items, args.warmup_runs) + summary["flashdreams"] = _collect_streaming(flashdreams_items, args.warmup_runs) summary["benchmark"] = { "metric": "steady_state_generation_ms_per_chunk", "unit": "ms", @@ -702,10 +702,10 @@ def main(argv: list[str] | None = None) -> None: ) if args.bench_side == "both": summary["flashdreams"] = _collect( - native_items, + flashdreams_items, args.warmup_runs, - _native_stage_ms, - _native_mem_gib, + _flashdreams_stage_ms, + _flashdreams_mem_gib, { "encode": "encode_ms", "dit": "diffuse_ms", diff --git a/integrations/sana/tests/parity_check/run.sh b/integrations/sana/tests/parity_check/run.sh index a5460210d..07540fde8 100644 --- a/integrations/sana/tests/parity_check/run.sh +++ b/integrations/sana/tests/parity_check/run.sh @@ -36,7 +36,7 @@ PIN_COMMIT="6298508" OUTPUT_DIR="$(_abspath "${OUTPUT_DIR:-${SCRIPT_DIR}/outputs/parity}")" UPSTREAM_OUT="${OUTPUT_DIR}/upstream" -NATIVE_OUT="${OUTPUT_DIR}/flashdreams" +FLASHDREAMS_OUT="${OUTPUT_DIR}/flashdreams" IMAGE_PATH="$(_abspath "${IMAGE_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.png}")" PROMPT_PATH="$(_abspath "${PROMPT_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.txt}")" CAMERA_PATH="$(_abspath "${CAMERA_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_pose.npy}")" @@ -93,36 +93,36 @@ if [[ -n "${PYTHONPATH:-}" ]]; then UPSTREAM_PYTHONPATH="${UPSTREAM_PYTHONPATH}:${PYTHONPATH}" fi -mkdir -p "${UPSTREAM_OUT}" "${NATIVE_OUT}" +mkdir -p "${UPSTREAM_OUT}" "${FLASHDREAMS_OUT}" UPSTREAM_FRAMES="${UPSTREAM_OUT}/frames.npy" -NATIVE_FRAMES="${NATIVE_OUT}/frames.npy" +FLASHDREAMS_FRAMES="${FLASHDREAMS_OUT}/frames.npy" UPSTREAM_STATS="${UPSTREAM_OUT}/stats.json" -NATIVE_STATS="${NATIVE_OUT}/stats.json" +FLASHDREAMS_STATS="${FLASHDREAMS_OUT}/stats.json" UPSTREAM_REFINER_ARGS=() -NATIVE_REFINER_ARGS=() +FLASHDREAMS_REFINER_ARGS=() if _is_true "${NO_REFINER}"; then UPSTREAM_REFINER_ARGS+=(--no_refiner) - NATIVE_REFINER_ARGS+=(--no-refiner) + FLASHDREAMS_REFINER_ARGS+=(--no-refiner) fi UPSTREAM_BACKEND_ARGS=() -NATIVE_BACKEND_ARGS=() +FLASHDREAMS_BACKEND_ARGS=() if _is_true "${FORCE_CUDNN_SDPA}"; then UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) - NATIVE_BACKEND_ARGS+=(--force-cudnn-sdpa) + FLASHDREAMS_BACKEND_ARGS+=(--force-cudnn-sdpa) fi UPSTREAM_COMPILE_ARGS=() -NATIVE_COMPILE_ARGS=() +FLASHDREAMS_COMPILE_ARGS=() if _is_true "${COMPILE_STAGE1}"; then UPSTREAM_COMPILE_ARGS+=(--compile_stage1) - NATIVE_COMPILE_ARGS+=(--compile-stage1) + FLASHDREAMS_COMPILE_ARGS+=(--compile-stage1) fi UPSTREAM_PRECISION_ARGS=( --stage1_precision "${STAGE1_PRECISION}" --refiner_precision "${REFINER_PRECISION}" ) -NATIVE_PRECISION_ARGS=( +FLASHDREAMS_PRECISION_ARGS=( --stage1-precision "${STAGE1_PRECISION}" --refiner-precision "${REFINER_PRECISION}" --quant-backend "${QUANT_BACKEND}" @@ -151,35 +151,35 @@ echo "[run] upstream SANA-WM -> ${UPSTREAM_OUT}" "${UPSTREAM_COMPILE_ARGS[@]}" \ "${UPSTREAM_REFINER_ARGS[@]}" ) -echo "[run] FlashDreams SANA-WM -> ${NATIVE_OUT}" +echo "[run] FlashDreams SANA-WM -> ${FLASHDREAMS_OUT}" ( cd "${SCRIPT_DIR}" && \ - uv run python "${SCRIPT_DIR}/run_native.py" \ + uv run python "${SCRIPT_DIR}/run_flashdreams.py" \ --image-path "${IMAGE_PATH}" \ --prompt-path "${PROMPT_PATH}" \ --camera-path "${CAMERA_PATH}" \ --intrinsics-path "${INTRINSICS_PATH}" \ - --output-dir "${NATIVE_OUT}" \ + --output-dir "${FLASHDREAMS_OUT}" \ --name flashdreams \ --num-frames "${NUM_FRAMES}" \ --fps "${FPS}" \ --step "${STEP}" \ --cfg-scale "${CFG_SCALE}" \ --seed "${SEED}" \ - --dump-frames "${NATIVE_FRAMES}" \ - --stats-json "${NATIVE_STATS}" \ - "${NATIVE_PRECISION_ARGS[@]}" \ - "${NATIVE_BACKEND_ARGS[@]}" \ - "${NATIVE_COMPILE_ARGS[@]}" \ - "${NATIVE_REFINER_ARGS[@]}" ) + --dump-frames "${FLASHDREAMS_FRAMES}" \ + --stats-json "${FLASHDREAMS_STATS}" \ + "${FLASHDREAMS_PRECISION_ARGS[@]}" \ + "${FLASHDREAMS_BACKEND_ARGS[@]}" \ + "${FLASHDREAMS_COMPILE_ARGS[@]}" \ + "${FLASHDREAMS_REFINER_ARGS[@]}" ) echo "[diff] summarising parity -> ${OUTPUT_DIR}/parity.json" ( cd "${SCRIPT_DIR}" && \ uv run python "${SCRIPT_DIR}/diff_parity.py" \ --upstream "${UPSTREAM_FRAMES}" \ - --flashdreams "${NATIVE_FRAMES}" \ + --flashdreams "${FLASHDREAMS_FRAMES}" \ --output "${OUTPUT_DIR}/parity.json" ) echo "[run] done." echo " upstream frames : ${UPSTREAM_FRAMES}" -echo " flashdreams frames: ${NATIVE_FRAMES}" +echo " flashdreams frames: ${FLASHDREAMS_FRAMES}" echo " parity JSON : ${OUTPUT_DIR}/parity.json" diff --git a/integrations/sana/tests/parity_check/run_native.py b/integrations/sana/tests/parity_check/run_flashdreams.py similarity index 99% rename from integrations/sana/tests/parity_check/run_native.py rename to integrations/sana/tests/parity_check/run_flashdreams.py index d49601ab7..1ec0adbb2 100644 --- a/integrations/sana/tests/parity_check/run_native.py +++ b/integrations/sana/tests/parity_check/run_flashdreams.py @@ -315,7 +315,7 @@ def main() -> None: "stats_ms": stats, }, ) - print(f"[native] wrote {video_path}") + print(f"[flashdreams] wrote {video_path}") if __name__ == "__main__": From a89e40ff36d28c384cd99067d00ae2049e8b7a6b Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 30 Jul 2026 08:31:53 -0700 Subject: [PATCH 40/64] Fix benchmarks hopefully --- .../sana/tests/parity_check/README.md | 70 +++- integrations/sana/tests/parity_check/bench.sh | 102 +++-- .../tests/parity_check/bench_sweep_summary.py | 13 +- .../sana/tests/parity_check/diff_parity.py | 11 +- integrations/sana/tests/parity_check/run.sh | 180 +------- .../tests/parity_check/run_bidirectional.sh | 140 +++++++ .../sana/tests/parity_check/run_common.sh | 87 ++++ .../tests/parity_check/run_flashdreams.py | 306 +------------- .../run_flashdreams_bidirectional.py | 209 ++++++++++ .../parity_check/run_flashdreams_common.py | 270 ++++++++++++ .../parity_check/run_flashdreams_streaming.py | 388 ++++++++++++++++++ .../sana/tests/parity_check/run_streaming.sh | 199 +++++++++ .../parity_check/streaming_continuity.py | 149 +++++++ .../tests/test_parity_benchmark_summary.py | 193 +++++++++ .../sana/tests/test_parity_quality_tools.py | 103 +++++ 15 files changed, 1912 insertions(+), 508 deletions(-) create mode 100644 integrations/sana/tests/parity_check/run_bidirectional.sh create mode 100644 integrations/sana/tests/parity_check/run_common.sh create mode 100644 integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py create mode 100644 integrations/sana/tests/parity_check/run_flashdreams_common.py create mode 100644 integrations/sana/tests/parity_check/run_flashdreams_streaming.py create mode 100644 integrations/sana/tests/parity_check/run_streaming.sh create mode 100644 integrations/sana/tests/parity_check/streaming_continuity.py create mode 100644 integrations/sana/tests/test_parity_quality_tools.py diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index f5f4a9f46..b3203f808 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -7,8 +7,10 @@ SPDX-License-Identifier: Apache-2.0 This harness compares upstream [`NVlabs/Sana`](https://github.com/NVlabs/Sana) SANA-WM against the in-tree -FlashDreams `sana-wm-bidirectional` integration on the same demo image, prompt, -camera trajectory, intrinsics, seed, resolution, and precision settings. +FlashDreams integrations on matched demo inputs, seeds, resolution, and +precision settings. `run.sh` dispatches to separate bidirectional and streaming +parity runners. `bench.sh` supports bidirectional by default and streaming when +`SANA_WM_VARIANT=streaming` is set. All dependencies live in this directory's isolated `./.venv`. `run.sh` and `bench.sh` reuse or clone the upstream checkout, pin it to `6298508`, and apply @@ -24,6 +26,13 @@ cd integrations/sana/tests/parity_check bash run.sh ``` +`run.sh` defaults to `SANA_WM_VARIANT=bidirectional`. Set +`SANA_WM_VARIANT=streaming` to run the streaming parity and continuity path: + +```bash +SANA_WM_VARIANT=streaming bash run.sh +``` + Defaults: - `SANA_REPO=$HOME/dev/Sana` @@ -45,11 +54,27 @@ Outputs are written under `outputs/parity/`: - `flashdreams/stats.json` - `parity.json` +Streaming parity writes to `outputs/parity/streaming/` by default: + +- `upstream/frames.npz` +- `upstream/stats.json` +- `flashdreams/frames.npy` +- `flashdreams/stats.json` +- `parity.json` +- `continuity.json` + Set `NO_REFINER=0` to compare the full Stage-1 + LTX-2 refiner path. Set `COMPILE_STAGE1=1` to wrap each Stage-1 DiT with `torch.compile`; this is opt-in because the pinned upstream SANA-WM stack can fail during TorchInductor Triton compilation on current PyTorch/Triton builds. +Streaming parity always uses the full upstream streaming stack; `NO_REFINER=1` +is rejected because upstream streaming has no no-refiner entrypoint. It saves +every decoded frame and checks both frame parity and chunk-boundary continuity. +`STREAMING_NO_COMPILE=1` is the streaming parity default to keep quality checks +debug-friendly; set `STREAMING_NO_COMPILE=0` when explicitly checking compiled +streaming behavior. + ## Run benchmark ```bash @@ -60,7 +85,7 @@ bash bench.sh Benchmark defaults to the bidirectional comparison and discards one warmup run before measuring three additional runs: `SANA_WM_VARIANT=bidirectional BENCH_SIDE=both WARMUP_RUNS=1 MEASURED_RUNS=3 -COMPILE_STAGE1=0 NO_REFINER=1 FORCE_CUDNN_SDPA=0`. Outputs are under +COMPILE_STAGE1=0 NO_REFINER=0 FORCE_CUDNN_SDPA=0`. Outputs are under `outputs/bench/`: - `bench.json` - machine-readable inputs, medians, p90s, memory, and stage @@ -72,9 +97,9 @@ The benchmark metric is post-load generation latency per generated clip. Model construction, checkpoint loading, video writing, and frame dumps are outside the timing boundary. SANA-WM renders each requested bidirectional clip in one generation pass, rather than as independently timed frames. With the default -`NO_REFINER=1`, the timed work covers conditioning, Stage-1 DiT, and SANA VAE -decode. Set `NO_REFINER=0` to benchmark the full Stage-1 + LTX-2 refiner path -with the same timing boundary. +`NO_REFINER=0`, the timed work covers conditioning, Stage-1 DiT, LTX-2 refiner, +and SANA VAE decode. Set `NO_REFINER=1` only for a diagnostic Stage-1 plus SANA +VAE decode benchmark. `bench.md` also reports Stage-1 DiT, conditioning/encode, VAE decode, optional refiner, memory, and frame-normalized diagnostic breakdowns. Those rows explain @@ -112,35 +137,49 @@ DEVICE_LABEL="RTX PRO 6000 Blackwell" BENCH_PRECISIONS=bf16,fp8,fp4 bash bench.s The scripts do not set allocator overrides or GPU wait loops. If GPU contention matters in your environment, handle it outside the harness. -## Run upstream streaming benchmark +## Run streaming benchmark -Use the streaming variant when collecting upstream-only `SANA-WM_streaming` -numbers before the FlashDreams streaming runner exists: +Use the streaming variant to compare upstream `SANA-WM_streaming` with the +FlashDreams `sana-wm-streaming` runner: ```bash cd integrations/sana/tests/parity_check -SANA_WM_VARIANT=streaming BENCH_SIDE=upstream bash bench.sh +SANA_WM_VARIANT=streaming BENCH_SIDE=both bash bench.sh ``` Streaming defaults use the upstream streaming entrypoint `inference_video_scripts/wm/inference_sana_wm_streaming.py`, the `Efficient-Large-Model/SANA-WM_streaming` weights, `NUM_FRAMES=241`, -`CFG_SCALE=1.0`, and `STREAMING_ACTION=w-80,dw-40,w-80,aw-40`. Set -`STREAMING_ACTION=` to use `CAMERA_PATH` instead. Outputs are precision-aware: +`CFG_SCALE=1.0`, `NO_REFINER=0`, and +`STREAMING_ACTION=w-80,dw-40,w-80,aw-40`. Set `STREAMING_ACTION=` to use +`CAMERA_PATH` instead. Outputs are precision-aware: - `outputs/bench/streaming//upstream/run_/stats.json` - `outputs/bench/streaming//upstream/run_/command.txt` +- `outputs/bench/streaming//flashdreams/run_/stats.json` +- `outputs/bench/streaming//flashdreams/run_/command.txt` - `outputs/bench/streaming//bench.json` - `outputs/bench/streaming//bench.md` +- `outputs/bench/streaming//perf.md` The streaming headline metric is steady-state generation milliseconds per produced chunk. Warmup runs and the first decoded chunk are excluded; full-clip wall time remains in `bench.json` and `bench.md` as supporting data. -Run the upstream precision sweep in order with: +Do not use FlashDreams streaming benchmark numbers for reporting until the +FlashDreams-owned streaming decoder/refiner state machine preserves chunk +continuity and validates against the official upstream baseline. Current +FlashDreams streaming artifacts show chunk-boundary discontinuities aligned +with the 24-frame output chunk cadence. + +By default, upstream compiles the streaming refiner transformer. The benchmark +passes the equivalent `torch.compile` wrapper to FlashDreams. Set +`STREAMING_NO_COMPILE=1` to disable refiner compile on both sides. + +Run the comparison precision sweep in order with: ```bash -SANA_WM_VARIANT=streaming BENCH_SIDE=upstream BENCH_PRECISIONS=bf16,fp8,fp4 bash bench.sh +SANA_WM_VARIANT=streaming BENCH_SIDE=both BENCH_PRECISIONS=bf16,fp8,fp4 bash bench.sh ``` FP8 and FP4 are passed through upstream streaming-only @@ -148,6 +187,3 @@ FP8 and FP4 are passed through upstream streaming-only the isolated venv with the `quant` extra before launch. If Transformer Engine, hardware, or upstream code rejects a precision, earlier precision outputs remain in place and the failing command is recorded in the corresponding `command.txt`. - -`BENCH_SIDE=both` is intentionally rejected for `SANA_WM_VARIANT=streaming` -until the FlashDreams `sana-wm-streaming` runner is available. diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index 846cf399a..45a34ebd6 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -14,9 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Stack-matched SANA-WM benchmark harness. Bidirectional defaults preserve the -# existing upstream-vs-FlashDreams clip-latency benchmark. Streaming mode can -# benchmark upstream only before the FlashDreams streaming runner lands. +# Stack-matched SANA-WM benchmark harness. By default this runs the +# bidirectional upstream-vs-FlashDreams clip-latency comparison. Set +# SANA_WM_VARIANT=streaming to compare upstream streaming with the FlashDreams +# streaming integration. set -euo pipefail @@ -80,11 +81,6 @@ case "${BENCH_SIDE}" in exit 1 ;; esac -if [[ "${SANA_WM_VARIANT}" == "streaming" && "${BENCH_SIDE}" == "both" ]]; then - echo "[bench] ERROR: SANA_WM_VARIANT=streaming currently supports BENCH_SIDE=upstream only." >&2 - echo " Wire the FlashDreams sana-wm-streaming runner before running BENCH_SIDE=both." >&2 - exit 1 -fi DEVICE_LABEL="${DEVICE_LABEL:-GPU}" STAGE1_PRECISION="${STAGE1_PRECISION:-bf16}" @@ -101,11 +97,13 @@ if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then DEFAULT_SWEEP_OUTPUT_DIR="${SCRIPT_DIR}/outputs/bench/streaming" NUM_FRAMES="${NUM_FRAMES:-241}" CFG_SCALE="${CFG_SCALE:-1.0}" + NO_REFINER="${NO_REFINER:-0}" else DEFAULT_OUTPUT_DIR="${SCRIPT_DIR}/outputs/bench" DEFAULT_SWEEP_OUTPUT_DIR="${SCRIPT_DIR}/outputs/bench" NUM_FRAMES="${NUM_FRAMES:-121}" CFG_SCALE="${CFG_SCALE:-5.0}" + NO_REFINER="${NO_REFINER:-0}" fi if [[ -n "${BENCH_PRECISIONS}" ]]; then OUTPUT_DIR="$(_abspath "${OUTPUT_DIR:-${DEFAULT_SWEEP_OUTPUT_DIR}}")" @@ -123,7 +121,6 @@ STREAMING_ACTION="${STREAMING_ACTION-w-80,dw-40,w-80,aw-40}" FPS="${FPS:-16}" STEP="${STEP:-60}" SEED="${SEED:-42}" -NO_REFINER="${NO_REFINER:-1}" FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" COMPILE_STAGE1="${COMPILE_STAGE1:-0}" @@ -142,6 +139,15 @@ STREAMING_CRF="${STREAMING_CRF:-18}" STREAMING_PRESET="${STREAMING_PRESET:-medium}" STREAMING_ENCODER="${STREAMING_ENCODER:-${SANA_WM_STREAMING_MP4_ENCODER:-libx264}}" +if [[ "${SANA_WM_VARIANT}" == "streaming" ]] && _is_true "${NO_REFINER}"; then + echo "[bench] ERROR: SANA_WM_VARIANT=streaming benchmarks the full upstream streaming stack; NO_REFINER=1 is not supported." >&2 + exit 1 +fi +if [[ "${SANA_WM_VARIANT}" == "streaming" ]] && _is_true "${COMPILE_STAGE1}"; then + echo "[bench] ERROR: COMPILE_STAGE1=1 is bidirectional-only. Streaming compile parity is controlled by STREAMING_NO_COMPILE." >&2 + exit 1 +fi + if _is_true "${BENCH_DRY_RUN}"; then echo "[bench] dry run" echo " variant: ${SANA_WM_VARIANT}" @@ -274,7 +280,9 @@ fi UPSTREAM_BACKEND_ARGS=() FLASHDREAMS_BACKEND_ARGS=() if _is_true "${FORCE_CUDNN_SDPA}"; then - UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) + if [[ "${SANA_WM_VARIANT}" != "streaming" ]]; then + UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) + fi FLASHDREAMS_BACKEND_ARGS+=(--force-cudnn-sdpa) fi UPSTREAM_COMPILE_ARGS=() @@ -295,11 +303,14 @@ FLASHDREAMS_PRECISION_ARGS=( STREAMING_CAMERA_SOURCE="camera" UPSTREAM_STREAMING_CAMERA_ARGS=(--camera "${CAMERA_PATH}") +FLASHDREAMS_STREAMING_CAMERA_ARGS=(--camera-source camera --camera-path "${CAMERA_PATH}") if [[ -n "${STREAMING_ACTION}" ]]; then STREAMING_CAMERA_SOURCE="action" UPSTREAM_STREAMING_CAMERA_ARGS=(--action "${STREAMING_ACTION}") + FLASHDREAMS_STREAMING_CAMERA_ARGS=(--camera-source action --action "${STREAMING_ACTION}" --camera-path "${CAMERA_PATH}") fi UPSTREAM_STREAMING_MODE_ARGS=(--output_mode "${STREAMING_OUTPUT_MODE}") +FLASHDREAMS_STREAMING_MODE_ARGS=(--output-mode "${STREAMING_OUTPUT_MODE}") if _is_true "${STREAMING_NO_MP4}"; then UPSTREAM_STREAMING_MODE_ARGS+=(--no_mp4) fi @@ -309,6 +320,10 @@ fi if _is_true "${STREAMING_NO_COMPILE}"; then UPSTREAM_STREAMING_MODE_ARGS+=(--no_compile) fi +FLASHDREAMS_STREAMING_COMPILE_ARGS=() +if [[ "${SANA_WM_VARIANT}" == "streaming" ]] && ! _is_true "${STREAMING_NO_COMPILE}"; then + FLASHDREAMS_STREAMING_COMPILE_ARGS+=(--compile-streaming-refiner) +fi TOTAL_RUNS=$(( WARMUP_RUNS + MEASURED_RUNS )) for ((i = 0; i < TOTAL_RUNS; i++)); do @@ -374,25 +389,54 @@ for ((i = 0; i < TOTAL_RUNS; i++)); do if [[ "${BENCH_SIDE}" == "both" ]]; then FLASHDREAMS_OUT="${FLASHDREAMS_ROOT}/run_${i}" mkdir -p "${FLASHDREAMS_OUT}" - FLASHDREAMS_CMD=( - uv run python "${SCRIPT_DIR}/run_flashdreams.py" - --image-path "${IMAGE_PATH}" - --prompt-path "${PROMPT_PATH}" - --camera-path "${CAMERA_PATH}" - --intrinsics-path "${INTRINSICS_PATH}" - --output-dir "${FLASHDREAMS_OUT}" - --name flashdreams - --num-frames "${NUM_FRAMES}" - --fps "${FPS}" - --step "${STEP}" - --cfg-scale "${CFG_SCALE}" - --seed "${SEED}" - --stats-json "${FLASHDREAMS_OUT}/stats.json" - "${FLASHDREAMS_PRECISION_ARGS[@]}" - "${FLASHDREAMS_BACKEND_ARGS[@]}" - "${FLASHDREAMS_COMPILE_ARGS[@]}" - "${FLASHDREAMS_REFINER_ARGS[@]}" - ) + if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then + FLASHDREAMS_CMD=( + uv run python "${SCRIPT_DIR}/run_flashdreams_streaming.py" + --image-path "${IMAGE_PATH}" + --prompt-path "${PROMPT_PATH}" + "${FLASHDREAMS_STREAMING_CAMERA_ARGS[@]}" + --intrinsics-path "${INTRINSICS_PATH}" + --output-dir "${FLASHDREAMS_OUT}" + --name flashdreams + --num-frames "${NUM_FRAMES}" + --fps "${FPS}" + --cfg-scale "${CFG_SCALE}" + --flow-shift "8.0" + --seed "${SEED}" + --refiner-seed "${STREAMING_REFINER_SEED}" + --stats-json "${FLASHDREAMS_OUT}/stats.json" + --denoising-step-list "${STREAMING_DENOISING_STEP_LIST}" + --num-frame-per-block "${STREAMING_NUM_FRAME_PER_BLOCK}" + --num-cached-blocks "${STREAMING_NUM_CACHED_BLOCKS}" + --sink-size "${STREAMING_SINK_SIZE}" + --refiner-block-size "${STREAMING_REFINER_BLOCK_SIZE}" + --refiner-kv-max-frames "${STREAMING_REFINER_KV_MAX_FRAMES}" + "${FLASHDREAMS_STREAMING_MODE_ARGS[@]}" + "${FLASHDREAMS_STREAMING_COMPILE_ARGS[@]}" + "${FLASHDREAMS_PRECISION_ARGS[@]}" + "${FLASHDREAMS_BACKEND_ARGS[@]}" + ) + else + FLASHDREAMS_CMD=( + uv run python "${SCRIPT_DIR}/run_flashdreams_bidirectional.py" + --image-path "${IMAGE_PATH}" + --prompt-path "${PROMPT_PATH}" + --camera-path "${CAMERA_PATH}" + --intrinsics-path "${INTRINSICS_PATH}" + --output-dir "${FLASHDREAMS_OUT}" + --name flashdreams + --num-frames "${NUM_FRAMES}" + --fps "${FPS}" + --step "${STEP}" + --cfg-scale "${CFG_SCALE}" + --seed "${SEED}" + --stats-json "${FLASHDREAMS_OUT}/stats.json" + "${FLASHDREAMS_PRECISION_ARGS[@]}" + "${FLASHDREAMS_BACKEND_ARGS[@]}" + "${FLASHDREAMS_COMPILE_ARGS[@]}" + "${FLASHDREAMS_REFINER_ARGS[@]}" + ) + fi echo "[bench] FlashDreams ${SANA_WM_VARIANT} run ${i}/${TOTAL_RUNS}" _write_command "${FLASHDREAMS_OUT}/command.txt" "${FLASHDREAMS_CMD[@]}" ( cd "${SCRIPT_DIR}" && "${FLASHDREAMS_CMD[@]}" ) diff --git a/integrations/sana/tests/parity_check/bench_sweep_summary.py b/integrations/sana/tests/parity_check/bench_sweep_summary.py index 74f3c22c0..c65a3fe04 100644 --- a/integrations/sana/tests/parity_check/bench_sweep_summary.py +++ b/integrations/sana/tests/parity_check/bench_sweep_summary.py @@ -53,7 +53,9 @@ def _load_item(raw: str) -> dict[str, Any]: "label": label, "path": str(path), "official": float(official), - "flashdreams": float(flashdreams) if isinstance(flashdreams, (int, float)) else None, + "flashdreams": ( + float(flashdreams) if isinstance(flashdreams, (int, float)) else None + ), "metric": benchmark.get("metric", "generation_ms_per_clip"), "unit": benchmark.get("unit", "ms"), "variant": benchmark.get("variant") @@ -65,7 +67,11 @@ def _load_item(raw: str) -> dict[str, Any]: def _render_chart(rows: list[dict[str, Any]]) -> str: has_flashdreams = all(row["flashdreams"] is not None for row in rows) - unit_label = "ms/chunk" if rows and rows[0]["metric"] == "steady_state_generation_ms_per_chunk" else "ms" + unit_label = ( + "ms/chunk" + if rows and rows[0]["metric"] == "steady_state_generation_ms_per_chunk" + else "ms" + ) if not has_flashdreams: lines = [ f"# SANA-WM Precision Sweep Upstream Summary ({unit_label})", @@ -103,7 +109,8 @@ def _render_report(rows: list[dict[str, Any]]) -> str: "# SANA-WM precision benchmark sweep", "", f"The chart metric is {metric_text}.", - "Model-card chart data is grouped by GPU/device in each precision subdirectory's `perf.md`.", + "Model-card chart data is grouped by GPU/device in each precision " + "subdirectory's `perf.md`.", "", ] if has_flashdreams: diff --git a/integrations/sana/tests/parity_check/diff_parity.py b/integrations/sana/tests/parity_check/diff_parity.py index db9d395e2..8fb19b009 100644 --- a/integrations/sana/tests/parity_check/diff_parity.py +++ b/integrations/sana/tests/parity_check/diff_parity.py @@ -26,7 +26,16 @@ def _load_frames(path: Path) -> np.ndarray: - frames = np.load(path) + loaded = np.load(path) + if isinstance(loaded, np.lib.npyio.NpzFile): + try: + if "frames" not in loaded: + raise ValueError(f"{path} must contain a 'frames' array.") + frames = loaded["frames"] + finally: + loaded.close() + else: + frames = loaded if frames.ndim != 4 or frames.shape[-1] != 3: raise ValueError(f"{path} must contain [T,H,W,3] frames; got {frames.shape}.") if frames.dtype != np.uint8: diff --git a/integrations/sana/tests/parity_check/run.sh b/integrations/sana/tests/parity_check/run.sh index 07540fde8..33eb9bad0 100644 --- a/integrations/sana/tests/parity_check/run.sh +++ b/integrations/sana/tests/parity_check/run.sh @@ -14,172 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Patch pinned upstream SANA-WM with instrumentation, run it and FlashDreams on -# the same demo input, dump decoded uint8 frames, and compute mean |Delta| / 255. +# Dispatch to the bidirectional or streaming SANA-WM parity runner. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" - -_abspath() { - case "$1" in - /*) printf '%s\n' "$1" ;; - *) printf '%s/%s\n' "${PWD}" "$1" ;; - esac -} - -SANA_REPO="$(_abspath "${SANA_REPO:-${HOME}/dev/Sana}")" -PATCH_FILE="${SCRIPT_DIR}/changes.patch" -REPO_URL="https://github.com/NVlabs/Sana.git" -PIN_COMMIT="6298508" - -OUTPUT_DIR="$(_abspath "${OUTPUT_DIR:-${SCRIPT_DIR}/outputs/parity}")" -UPSTREAM_OUT="${OUTPUT_DIR}/upstream" -FLASHDREAMS_OUT="${OUTPUT_DIR}/flashdreams" -IMAGE_PATH="$(_abspath "${IMAGE_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.png}")" -PROMPT_PATH="$(_abspath "${PROMPT_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.txt}")" -CAMERA_PATH="$(_abspath "${CAMERA_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_pose.npy}")" -INTRINSICS_PATH="$(_abspath "${INTRINSICS_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_intrinsics.npy}")" -NUM_FRAMES="${NUM_FRAMES:-121}" -FPS="${FPS:-16}" -STEP="${STEP:-60}" -CFG_SCALE="${CFG_SCALE:-5.0}" -SEED="${SEED:-42}" -NO_REFINER="${NO_REFINER:-1}" -FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" -COMPILE_STAGE1="${COMPILE_STAGE1:-0}" -STAGE1_PRECISION="${STAGE1_PRECISION:-bf16}" -REFINER_PRECISION="${REFINER_PRECISION:-${STAGE1_PRECISION}}" -QUANT_BACKEND="${QUANT_BACKEND:-auto}" - -_is_true() { - case "${1,,}" in - 1|true|yes|on) return 0 ;; - *) return 1 ;; - esac -} - -if [[ ! -d "${SANA_REPO}/.git" ]]; then - echo "[setup] cloning ${REPO_URL} -> ${SANA_REPO}" - git clone "${REPO_URL}" "${SANA_REPO}" -else - echo "[setup] repo already present at ${SANA_REPO}, skipping clone" -fi - -cd "${SANA_REPO}" -CURRENT_COMMIT="$(git rev-parse --short HEAD)" -if [[ "${CURRENT_COMMIT}" != "${PIN_COMMIT}" ]]; then - echo "[setup] checking out pinned commit ${PIN_COMMIT}" - git checkout "${PIN_COMMIT}" -else - echo "[setup] already at pinned commit ${PIN_COMMIT}, skipping checkout" -fi - -if git apply --reverse --check "${PATCH_FILE}" >/dev/null 2>&1; then - echo "[setup] patch already applied, skipping" -elif git apply --check "${PATCH_FILE}" >/dev/null 2>&1; then - echo "[setup] applying ${PATCH_FILE}" - git apply "${PATCH_FILE}" -else - echo "[setup] ERROR: ${PATCH_FILE} neither cleanly applies nor is already applied." >&2 - exit 1 -fi - -echo "[setup] ensuring Python deps via uv sync (isolated venv)" -( cd "${SCRIPT_DIR}" && uv sync ) -UPSTREAM_PYTHONPATH="${SCRIPT_DIR}/compat:${SANA_REPO}" -if [[ -n "${PYTHONPATH:-}" ]]; then - UPSTREAM_PYTHONPATH="${UPSTREAM_PYTHONPATH}:${PYTHONPATH}" -fi - -mkdir -p "${UPSTREAM_OUT}" "${FLASHDREAMS_OUT}" - -UPSTREAM_FRAMES="${UPSTREAM_OUT}/frames.npy" -FLASHDREAMS_FRAMES="${FLASHDREAMS_OUT}/frames.npy" -UPSTREAM_STATS="${UPSTREAM_OUT}/stats.json" -FLASHDREAMS_STATS="${FLASHDREAMS_OUT}/stats.json" - -UPSTREAM_REFINER_ARGS=() -FLASHDREAMS_REFINER_ARGS=() -if _is_true "${NO_REFINER}"; then - UPSTREAM_REFINER_ARGS+=(--no_refiner) - FLASHDREAMS_REFINER_ARGS+=(--no-refiner) -fi -UPSTREAM_BACKEND_ARGS=() -FLASHDREAMS_BACKEND_ARGS=() -if _is_true "${FORCE_CUDNN_SDPA}"; then - UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) - FLASHDREAMS_BACKEND_ARGS+=(--force-cudnn-sdpa) -fi -UPSTREAM_COMPILE_ARGS=() -FLASHDREAMS_COMPILE_ARGS=() -if _is_true "${COMPILE_STAGE1}"; then - UPSTREAM_COMPILE_ARGS+=(--compile_stage1) - FLASHDREAMS_COMPILE_ARGS+=(--compile-stage1) -fi -UPSTREAM_PRECISION_ARGS=( - --stage1_precision "${STAGE1_PRECISION}" - --refiner_precision "${REFINER_PRECISION}" -) -FLASHDREAMS_PRECISION_ARGS=( - --stage1-precision "${STAGE1_PRECISION}" - --refiner-precision "${REFINER_PRECISION}" - --quant-backend "${QUANT_BACKEND}" -) - -echo "[run] upstream SANA-WM -> ${UPSTREAM_OUT}" -( cd "${SCRIPT_DIR}" && \ - PYTHONPATH="${UPSTREAM_PYTHONPATH}" \ - uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm.py" \ - --image "${IMAGE_PATH}" \ - --prompt "${PROMPT_PATH}" \ - --camera "${CAMERA_PATH}" \ - --intrinsics "${INTRINSICS_PATH}" \ - --output_dir "${UPSTREAM_OUT}" \ - --name upstream \ - --num_frames "${NUM_FRAMES}" \ - --fps "${FPS}" \ - --step "${STEP}" \ - --cfg_scale "${CFG_SCALE}" \ - --seed "${SEED}" \ - --no_action_overlay \ - --dump_frames "${UPSTREAM_FRAMES}" \ - --stats_json "${UPSTREAM_STATS}" \ - "${UPSTREAM_PRECISION_ARGS[@]}" \ - "${UPSTREAM_BACKEND_ARGS[@]}" \ - "${UPSTREAM_COMPILE_ARGS[@]}" \ - "${UPSTREAM_REFINER_ARGS[@]}" ) - -echo "[run] FlashDreams SANA-WM -> ${FLASHDREAMS_OUT}" -( cd "${SCRIPT_DIR}" && \ - uv run python "${SCRIPT_DIR}/run_flashdreams.py" \ - --image-path "${IMAGE_PATH}" \ - --prompt-path "${PROMPT_PATH}" \ - --camera-path "${CAMERA_PATH}" \ - --intrinsics-path "${INTRINSICS_PATH}" \ - --output-dir "${FLASHDREAMS_OUT}" \ - --name flashdreams \ - --num-frames "${NUM_FRAMES}" \ - --fps "${FPS}" \ - --step "${STEP}" \ - --cfg-scale "${CFG_SCALE}" \ - --seed "${SEED}" \ - --dump-frames "${FLASHDREAMS_FRAMES}" \ - --stats-json "${FLASHDREAMS_STATS}" \ - "${FLASHDREAMS_PRECISION_ARGS[@]}" \ - "${FLASHDREAMS_BACKEND_ARGS[@]}" \ - "${FLASHDREAMS_COMPILE_ARGS[@]}" \ - "${FLASHDREAMS_REFINER_ARGS[@]}" ) - -echo "[diff] summarising parity -> ${OUTPUT_DIR}/parity.json" -( cd "${SCRIPT_DIR}" && \ - uv run python "${SCRIPT_DIR}/diff_parity.py" \ - --upstream "${UPSTREAM_FRAMES}" \ - --flashdreams "${FLASHDREAMS_FRAMES}" \ - --output "${OUTPUT_DIR}/parity.json" ) - -echo "[run] done." -echo " upstream frames : ${UPSTREAM_FRAMES}" -echo " flashdreams frames: ${FLASHDREAMS_FRAMES}" -echo " parity JSON : ${OUTPUT_DIR}/parity.json" +SANA_WM_VARIANT="${SANA_WM_VARIANT:-bidirectional}" + +case "${SANA_WM_VARIANT}" in + bidirectional) + exec bash "${SCRIPT_DIR}/run_bidirectional.sh" "$@" + ;; + streaming) + exec bash "${SCRIPT_DIR}/run_streaming.sh" "$@" + ;; + *) + echo "[run] ERROR: SANA_WM_VARIANT must be bidirectional or streaming; got ${SANA_WM_VARIANT}" >&2 + exit 1 + ;; +esac diff --git a/integrations/sana/tests/parity_check/run_bidirectional.sh b/integrations/sana/tests/parity_check/run_bidirectional.sh new file mode 100644 index 000000000..ed5b75292 --- /dev/null +++ b/integrations/sana/tests/parity_check/run_bidirectional.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Patch pinned upstream SANA-WM with bidirectional instrumentation, run it and +# the FlashDreams bidirectional integration on the same demo input, dump +# decoded uint8 frames, and compute mean |Delta| / 255. + +set -euo pipefail + +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/run_common.sh" + +SANA_REPO="$(parity_abspath "${SANA_REPO:-${HOME}/dev/Sana}")" +PATCH_FILE="${PARITY_SCRIPT_DIR}/changes.patch" + +OUTPUT_DIR="$(parity_abspath "${OUTPUT_DIR:-${PARITY_SCRIPT_DIR}/outputs/parity}")" +UPSTREAM_OUT="${OUTPUT_DIR}/upstream" +FLASHDREAMS_OUT="${OUTPUT_DIR}/flashdreams" +IMAGE_PATH="$(parity_abspath "${IMAGE_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.png}")" +PROMPT_PATH="$(parity_abspath "${PROMPT_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.txt}")" +CAMERA_PATH="$(parity_abspath "${CAMERA_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_pose.npy}")" +INTRINSICS_PATH="$(parity_abspath "${INTRINSICS_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_intrinsics.npy}")" +NUM_FRAMES="${NUM_FRAMES:-121}" +FPS="${FPS:-16}" +STEP="${STEP:-60}" +CFG_SCALE="${CFG_SCALE:-5.0}" +SEED="${SEED:-42}" +NO_REFINER="${NO_REFINER:-1}" +FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" +COMPILE_STAGE1="${COMPILE_STAGE1:-0}" +STAGE1_PRECISION="${STAGE1_PRECISION:-bf16}" +REFINER_PRECISION="${REFINER_PRECISION:-${STAGE1_PRECISION}}" +QUANT_BACKEND="${QUANT_BACKEND:-auto}" + +parity_prepare_sana_repo "${SANA_REPO}" "${PATCH_FILE}" +parity_sync_venv +UPSTREAM_PYTHONPATH="$(parity_upstream_pythonpath "${SANA_REPO}")" + +mkdir -p "${UPSTREAM_OUT}" "${FLASHDREAMS_OUT}" + +UPSTREAM_FRAMES="${UPSTREAM_OUT}/frames.npy" +FLASHDREAMS_FRAMES="${FLASHDREAMS_OUT}/frames.npy" +UPSTREAM_STATS="${UPSTREAM_OUT}/stats.json" +FLASHDREAMS_STATS="${FLASHDREAMS_OUT}/stats.json" + +UPSTREAM_REFINER_ARGS=() +FLASHDREAMS_REFINER_ARGS=() +if parity_is_true "${NO_REFINER}"; then + UPSTREAM_REFINER_ARGS+=(--no_refiner) + FLASHDREAMS_REFINER_ARGS+=(--no-refiner) +fi +UPSTREAM_BACKEND_ARGS=() +FLASHDREAMS_BACKEND_ARGS=() +if parity_is_true "${FORCE_CUDNN_SDPA}"; then + UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) + FLASHDREAMS_BACKEND_ARGS+=(--force-cudnn-sdpa) +fi +UPSTREAM_COMPILE_ARGS=() +FLASHDREAMS_COMPILE_ARGS=() +if parity_is_true "${COMPILE_STAGE1}"; then + UPSTREAM_COMPILE_ARGS+=(--compile_stage1) + FLASHDREAMS_COMPILE_ARGS+=(--compile-stage1) +fi +UPSTREAM_PRECISION_ARGS=( + --stage1_precision "${STAGE1_PRECISION}" + --refiner_precision "${REFINER_PRECISION}" +) +FLASHDREAMS_PRECISION_ARGS=( + --stage1-precision "${STAGE1_PRECISION}" + --refiner-precision "${REFINER_PRECISION}" + --quant-backend "${QUANT_BACKEND}" +) + +echo "[run] upstream SANA-WM bidirectional -> ${UPSTREAM_OUT}" +( cd "${PARITY_SCRIPT_DIR}" && \ + PYTHONPATH="${UPSTREAM_PYTHONPATH}" \ + uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm.py" \ + --image "${IMAGE_PATH}" \ + --prompt "${PROMPT_PATH}" \ + --camera "${CAMERA_PATH}" \ + --intrinsics "${INTRINSICS_PATH}" \ + --output_dir "${UPSTREAM_OUT}" \ + --name upstream \ + --num_frames "${NUM_FRAMES}" \ + --fps "${FPS}" \ + --step "${STEP}" \ + --cfg_scale "${CFG_SCALE}" \ + --seed "${SEED}" \ + --no_action_overlay \ + --dump_frames "${UPSTREAM_FRAMES}" \ + --stats_json "${UPSTREAM_STATS}" \ + "${UPSTREAM_PRECISION_ARGS[@]}" \ + "${UPSTREAM_BACKEND_ARGS[@]}" \ + "${UPSTREAM_COMPILE_ARGS[@]}" \ + "${UPSTREAM_REFINER_ARGS[@]}" ) + +echo "[run] FlashDreams SANA-WM bidirectional -> ${FLASHDREAMS_OUT}" +( cd "${PARITY_SCRIPT_DIR}" && \ + uv run python "${PARITY_SCRIPT_DIR}/run_flashdreams_bidirectional.py" \ + --image-path "${IMAGE_PATH}" \ + --prompt-path "${PROMPT_PATH}" \ + --camera-path "${CAMERA_PATH}" \ + --intrinsics-path "${INTRINSICS_PATH}" \ + --output-dir "${FLASHDREAMS_OUT}" \ + --name flashdreams \ + --num-frames "${NUM_FRAMES}" \ + --fps "${FPS}" \ + --step "${STEP}" \ + --cfg-scale "${CFG_SCALE}" \ + --seed "${SEED}" \ + --dump-frames "${FLASHDREAMS_FRAMES}" \ + --stats-json "${FLASHDREAMS_STATS}" \ + "${FLASHDREAMS_PRECISION_ARGS[@]}" \ + "${FLASHDREAMS_BACKEND_ARGS[@]}" \ + "${FLASHDREAMS_COMPILE_ARGS[@]}" \ + "${FLASHDREAMS_REFINER_ARGS[@]}" ) + +echo "[diff] summarising parity -> ${OUTPUT_DIR}/parity.json" +( cd "${PARITY_SCRIPT_DIR}" && \ + uv run python "${PARITY_SCRIPT_DIR}/diff_parity.py" \ + --upstream "${UPSTREAM_FRAMES}" \ + --flashdreams "${FLASHDREAMS_FRAMES}" \ + --output "${OUTPUT_DIR}/parity.json" ) + +echo "[run] done." +echo " upstream frames : ${UPSTREAM_FRAMES}" +echo " flashdreams frames: ${FLASHDREAMS_FRAMES}" +echo " parity JSON : ${OUTPUT_DIR}/parity.json" diff --git a/integrations/sana/tests/parity_check/run_common.sh b/integrations/sana/tests/parity_check/run_common.sh new file mode 100644 index 000000000..7a2b8299f --- /dev/null +++ b/integrations/sana/tests/parity_check/run_common.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PARITY_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PARITY_REPO_ROOT="$(cd "${PARITY_SCRIPT_DIR}/../../../.." && pwd)" +PARITY_REPO_URL="https://github.com/NVlabs/Sana.git" +PARITY_PIN_COMMIT="6298508" + +parity_abspath() { + case "$1" in + /*) printf '%s\n' "$1" ;; + *) printf '%s/%s\n' "${PWD}" "$1" ;; + esac +} + +parity_is_true() { + case "${1,,}" in + 1|true|yes|on) return 0 ;; + *) return 1 ;; + esac +} + +parity_apply_patch_once() { + local patch_file="$1" + if git apply --reverse --check "${patch_file}" >/dev/null 2>&1; then + echo "[setup] patch already applied, skipping ${patch_file}" + elif git apply --check "${patch_file}" >/dev/null 2>&1; then + echo "[setup] applying ${patch_file}" + git apply "${patch_file}" + else + echo "[setup] ERROR: ${patch_file} neither cleanly applies nor is already applied." >&2 + exit 1 + fi +} + +parity_prepare_sana_repo() { + local sana_repo="$1" + shift + if [[ ! -d "${sana_repo}/.git" ]]; then + echo "[setup] cloning ${PARITY_REPO_URL} -> ${sana_repo}" + git clone "${PARITY_REPO_URL}" "${sana_repo}" + else + echo "[setup] repo already present at ${sana_repo}, skipping clone" + fi + + cd "${sana_repo}" + local current_commit + current_commit="$(git rev-parse --short HEAD)" + if [[ "${current_commit}" != "${PARITY_PIN_COMMIT}" ]]; then + echo "[setup] checking out pinned commit ${PARITY_PIN_COMMIT}" + git checkout "${PARITY_PIN_COMMIT}" + else + echo "[setup] already at pinned commit ${PARITY_PIN_COMMIT}, skipping checkout" + fi + + local patch_file + for patch_file in "$@"; do + parity_apply_patch_once "${patch_file}" + done +} + +parity_sync_venv() { + echo "[setup] ensuring Python deps via uv sync (isolated venv)" + ( cd "${PARITY_SCRIPT_DIR}" && uv sync ) +} + +parity_upstream_pythonpath() { + local sana_repo="$1" + local upstream_pythonpath="${PARITY_SCRIPT_DIR}/compat:${sana_repo}" + if [[ -n "${PYTHONPATH:-}" ]]; then + upstream_pythonpath="${upstream_pythonpath}:${PYTHONPATH}" + fi + printf '%s\n' "${upstream_pythonpath}" +} diff --git a/integrations/sana/tests/parity_check/run_flashdreams.py b/integrations/sana/tests/parity_check/run_flashdreams.py index 1ec0adbb2..2901ceb7a 100644 --- a/integrations/sana/tests/parity_check/run_flashdreams.py +++ b/integrations/sana/tests/parity_check/run_flashdreams.py @@ -13,309 +13,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Run the FlashDreams SANA-WM pipeline with frame dumps and profiler JSON.""" +"""Compatibility dispatcher for FlashDreams SANA-WM benchmark helpers.""" from __future__ import annotations import argparse -from contextlib import nullcontext -import json -import os -import time -from pathlib import Path -from typing import Any -import imageio.v3 as iio -import numpy as np -import torch -from PIL import Image -from flashdreams.infra.config import derive_config -from sana_wm.camera import ( - default_intrinsics_vec4, - load_intrinsics, - resize_center_crop_geometry, - snap_num_frames, - transform_intrinsics_for_crop, -) -from sana_wm.conditioning import SanaWMI2VConditioningRequest -from sana_wm.config import RUNNER_SANA_WM_BIDIRECTIONAL -from sana_wm.constants import DEFAULT_VIDEO_HEIGHT, DEFAULT_VIDEO_WIDTH -from sana_wm.decoder import SanaWMDecodedVideo -from sana_wm.runner import ( - _active_quantized_precisions, - _pipeline_config, - _resolve_quant_backend, - _validate_precision_request, -) - - -def _apply_backend_defaults() -> None: - torch.backends.cudnn.benchmark = True - torch.set_float32_matmul_precision("high") - if torch.cuda.is_available(): - torch.backends.cuda.enable_flash_sdp(False) - torch.backends.cuda.enable_math_sdp(True) - torch.backends.cuda.enable_mem_efficient_sdp(False) - torch.backends.cuda.enable_cudnn_sdp(True) - try: - import torch._inductor.config as inductor_config - - inductor_config.coordinate_descent_tuning = True - inductor_config.epilogue_fusion = True - except Exception: - pass - - -def _resolve_device(device: str) -> torch.device: - if device == "auto": - if torch.cuda.is_available(): - return torch.device(f"cuda:{int(os.environ.get('LOCAL_RANK', '0'))}") - return torch.device("cpu") - if device == "cuda" and torch.cuda.is_available(): - return torch.device(f"cuda:{int(os.environ.get('LOCAL_RANK', '0'))}") - return torch.device(device) - - -def _prepare_inputs( - *, - image_path: Path, - camera_path: Path, - intrinsics_path: Path | None, - intrinsics_hfov_deg: float, - num_frames_requested: int, -) -> tuple[Image.Image, np.ndarray, np.ndarray, int]: - image = Image.open(image_path).convert("RGB") - c2w_full = np.load(camera_path).astype(np.float32) - if c2w_full.ndim != 3 or c2w_full.shape[1:] != (4, 4): - raise ValueError(f"--camera-path must be [F,4,4]; got {c2w_full.shape}.") - - num_frames = min(num_frames_requested, c2w_full.shape[0]) - num_frames = snap_num_frames(num_frames, stride=8, upper_bound=c2w_full.shape[0]) - c2w = c2w_full[:num_frames] - - resized_size, crop_offset = resize_center_crop_geometry( - image.size, - target_h=DEFAULT_VIDEO_HEIGHT, - target_w=DEFAULT_VIDEO_WIDTH, - ) - resized = image.resize(resized_size, Image.LANCZOS) - left, top = crop_offset - cropped = resized.crop( - (left, top, left + DEFAULT_VIDEO_WIDTH, top + DEFAULT_VIDEO_HEIGHT) - ) - if intrinsics_path is None: - intrinsics_src = default_intrinsics_vec4( - image.size, - num_frames, - hfov_deg=intrinsics_hfov_deg, - ) - else: - intrinsics_src = load_intrinsics(intrinsics_path, num_frames) - intrinsics_vec4 = transform_intrinsics_for_crop( - intrinsics_src, - image.size, - resized_size, - crop_offset, - ) - return cropped, c2w, intrinsics_vec4, num_frames - - -def _install_stage1_compile_hook(pipeline: Any) -> None: - transformer = pipeline.diffusion_model.transformer - original_ensure_model = transformer._ensure_model - compiled = {"done": False} - - def _ensure_model_with_compile() -> None: - original_ensure_model() - if compiled["done"]: - return - transformer.model = torch.compile( - transformer.model, - mode="max-autotune-no-cudagraphs", - ) - compiled["done"] = True - - transformer._ensure_model = _ensure_model_with_compile - - -def _write_video(output_dir: Path, name: str, frames: np.ndarray, fps: int) -> Path: - output_dir.mkdir(parents=True, exist_ok=True) - path = output_dir / f"{name}_generated.mp4" - iio.imwrite(path, frames, fps=fps) - return path - - -def _write_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--image-path", type=Path, required=True) - parser.add_argument("--prompt-path", type=Path, required=True) - parser.add_argument("--camera-path", type=Path, required=True) - parser.add_argument("--intrinsics-path", type=Path, default=None) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--name", default="flashdreams") - parser.add_argument("--dump-frames", type=Path, default=None) - parser.add_argument("--stats-json", type=Path, default=None) - parser.add_argument("--num-frames", type=int, default=161) - parser.add_argument("--fps", type=int, default=16) - parser.add_argument("--step", type=int, default=60) - parser.add_argument("--cfg-scale", type=float, default=5.0) - parser.add_argument("--flow-shift", type=float, default=None) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--negative-prompt", default="") - parser.add_argument("--no-refiner", action="store_true") - parser.add_argument("--save-stage1", action="store_true") - parser.add_argument("--device", default="auto") - parser.add_argument("--intrinsics-hfov-deg", type=float, default=90.0) - parser.add_argument("--stage1-precision", choices=["bf16", "fp8", "fp4"], default="bf16") - parser.add_argument("--refiner-precision", choices=["bf16", "fp8", "fp4"], default="bf16") +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(add_help=False) parser.add_argument( - "--quant-backend", - choices=["auto", "torch", "torch-fp8", "torch-fp4"], - default="auto", + "--variant", + choices=["bidirectional", "streaming"], + default="bidirectional", ) - parser.add_argument("--force-cudnn-sdpa", action="store_true") - parser.add_argument("--compile-stage1", action="store_true") - args = parser.parse_args() - - if args.force_cudnn_sdpa: - _apply_backend_defaults() + args, rest = parser.parse_known_args(argv) + if args.variant == "streaming": + from run_flashdreams_streaming import main as run_streaming - prompt = args.prompt_path.read_text(encoding="utf-8", errors="replace").strip() - if not prompt: - raise ValueError(f"Prompt file is empty: {args.prompt_path}") - - device = _resolve_device(args.device) - quantized = _active_quantized_precisions( - stage1_precision=args.stage1_precision, - refiner_precision=args.refiner_precision, - refiner_enabled=not args.no_refiner, - ) - quant_backend = _resolve_quant_backend(args.quant_backend, quantized) - _validate_precision_request( - device=device, - stage1_precision=args.stage1_precision, - refiner_precision=args.refiner_precision, - refiner_enabled=not args.no_refiner, - quant_backend=args.quant_backend, - ) - with nullcontext(): - image, c2w, intrinsics_vec4, num_frames = _prepare_inputs( - image_path=args.image_path, - camera_path=args.camera_path, - intrinsics_path=args.intrinsics_path, - intrinsics_hfov_deg=args.intrinsics_hfov_deg, - num_frames_requested=args.num_frames, - ) - runner_cfg = derive_config( - RUNNER_SANA_WM_BIDIRECTIONAL, - output_dir=args.output_dir, - runner_name=args.name, - num_frames=num_frames, - fps=args.fps, - step=args.step, - cfg_scale=args.cfg_scale, - flow_shift=args.flow_shift, - seed=args.seed, - negative_prompt=args.negative_prompt, - no_refiner=args.no_refiner, - save_stage1=args.save_stage1, - refiner_seed=args.seed, - stage1_precision=args.stage1_precision, - refiner_precision=args.refiner_precision, - quant_backend=args.quant_backend, - ) - pipeline_cfg = _pipeline_config(runner_cfg, quant_backend=quant_backend) - pipeline_cfg = derive_config(pipeline_cfg, enable_sync_and_profile=True) - pipeline = pipeline_cfg.setup().to(device).eval() - if args.compile_stage1: - _install_stage1_compile_hook(pipeline) - encoder = getattr(pipeline, "encoder", None) - if encoder is not None: - text_encoder = getattr(encoder, "text_encoder", None) - if text_encoder is not None: - text_encoder._ensure_text_encoder() - first_frame_encoder = getattr(encoder, "first_frame_encoder", None) - if first_frame_encoder is not None: - first_frame_encoder._ensure_vae() - pipeline.diffusion_model.transformer._ensure_model() - decoder = getattr(pipeline, "decoder", None) - vae_decoder = getattr(decoder, "vae_decoder", None) - if vae_decoder is not None: - vae_decoder._ensure_vae() - - request = SanaWMI2VConditioningRequest( - image=image, - prompt=prompt, - poses_c2w=c2w, - intrinsics_vec4=intrinsics_vec4, - num_frames=num_frames, - fps=args.fps, - steps=args.step, - cfg_scale=args.cfg_scale, - flow_shift=args.flow_shift, - seed=args.seed, - negative_prompt=args.negative_prompt, - ) - - cache = pipeline.initialize_cache( - decoder_context={ - "prompt": prompt, - "fps": args.fps, - "save_stage1": args.save_stage1, - "refiner_seed": args.seed, - "sink_size": runner_cfg.sink_size, - } - ) - if torch.cuda.is_available(): - torch.cuda.reset_peak_memory_stats(device) - torch.cuda.synchronize(device) - generation_start = time.perf_counter() - with torch.inference_mode(): - decoded = pipeline.generate( - 0, - cache, - input=request, - ) - if torch.cuda.is_available(): - torch.cuda.synchronize(device) - wall_s = time.perf_counter() - generation_start - stats = pipeline.finalize(0, cache) or {} - - if not isinstance(decoded, SanaWMDecodedVideo): - raise TypeError(f"expected SanaWMDecodedVideo, got {type(decoded).__name__}") + run_streaming(rest) + else: + from run_flashdreams_bidirectional import main as run_bidirectional - frames = np.asarray(decoded.video_hwc, dtype=np.uint8) - video_path = _write_video(args.output_dir, args.name, frames, args.fps) - if args.dump_frames is not None: - args.dump_frames.parent.mkdir(parents=True, exist_ok=True) - np.save(args.dump_frames, frames) - if args.stats_json is not None: - _write_json( - args.stats_json, - { - "backend": "flashdreams", - "runner": "sana-wm-bidirectional", - "video_path": str(video_path), - "video_shape": list(frames.shape), - "num_frames": num_frames, - "seed": args.seed, - "fps": args.fps, - "step": args.step, - "cfg_scale": args.cfg_scale, - "no_refiner": args.no_refiner, - "compile_stage1": args.compile_stage1, - "force_cudnn_sdpa": args.force_cudnn_sdpa, - "wall_s": wall_s, - "stats_ms": stats, - }, - ) - print(f"[flashdreams] wrote {video_path}") + run_bidirectional(rest) if __name__ == "__main__": diff --git a/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py b/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py new file mode 100644 index 000000000..401104809 --- /dev/null +++ b/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py @@ -0,0 +1,209 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run the FlashDreams SANA-WM bidirectional benchmark helper.""" + +from __future__ import annotations + +import argparse +import time +from pathlib import Path + +import numpy as np +import torch + +from flashdreams.infra.config import derive_config +from run_flashdreams_common import ( + apply_backend_defaults, + install_stage1_compile_hook, + preload_pipeline_components, + prepare_bidirectional_inputs, + resolve_device, + write_json, + write_video, +) +from sana_wm.conditioning import SanaWMI2VConditioningRequest +from sana_wm.config import RUNNER_SANA_WM_BIDIRECTIONAL +from sana_wm.decoder import SanaWMDecodedVideo +from sana_wm.runner import ( + _active_quantized_precisions, + _pipeline_config, + _resolve_quant_backend, + _validate_precision_request, +) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--image-path", type=Path, required=True) + parser.add_argument("--prompt-path", type=Path, required=True) + parser.add_argument("--camera-path", type=Path, required=True) + parser.add_argument("--intrinsics-path", type=Path, default=None) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--name", default="flashdreams") + parser.add_argument("--dump-frames", type=Path, default=None) + parser.add_argument("--stats-json", type=Path, default=None) + parser.add_argument("--num-frames", type=int, default=161) + parser.add_argument("--fps", type=int, default=16) + parser.add_argument("--step", type=int, default=60) + parser.add_argument("--cfg-scale", type=float, default=5.0) + parser.add_argument("--flow-shift", type=float, default=None) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--negative-prompt", default="") + parser.add_argument("--no-refiner", action="store_true") + parser.add_argument("--save-stage1", action="store_true") + parser.add_argument("--device", default="auto") + parser.add_argument("--intrinsics-hfov-deg", type=float, default=90.0) + parser.add_argument( + "--stage1-precision", + choices=["bf16", "fp8", "fp4"], + default="bf16", + ) + parser.add_argument( + "--refiner-precision", + choices=["bf16", "fp8", "fp4"], + default="bf16", + ) + parser.add_argument( + "--quant-backend", + choices=["auto", "torch", "torch-fp8", "torch-fp4"], + default="auto", + ) + parser.add_argument("--force-cudnn-sdpa", action="store_true") + parser.add_argument("--compile-stage1", action="store_true") + args = parser.parse_args(argv) + + if args.force_cudnn_sdpa: + apply_backend_defaults() + + prompt = args.prompt_path.read_text(encoding="utf-8", errors="replace").strip() + if not prompt: + raise ValueError(f"Prompt file is empty: {args.prompt_path}") + + device = resolve_device(args.device) + quantized = _active_quantized_precisions( + stage1_precision=args.stage1_precision, + refiner_precision=args.refiner_precision, + refiner_enabled=not args.no_refiner, + ) + quant_backend = _resolve_quant_backend(args.quant_backend, quantized) + _validate_precision_request( + device=device, + stage1_precision=args.stage1_precision, + refiner_precision=args.refiner_precision, + refiner_enabled=not args.no_refiner, + quant_backend=args.quant_backend, + ) + image, c2w, intrinsics_vec4, num_frames = prepare_bidirectional_inputs( + image_path=args.image_path, + camera_path=args.camera_path, + intrinsics_path=args.intrinsics_path, + intrinsics_hfov_deg=args.intrinsics_hfov_deg, + num_frames_requested=args.num_frames, + ) + runner_cfg = derive_config( + RUNNER_SANA_WM_BIDIRECTIONAL, + output_dir=args.output_dir, + runner_name=args.name, + num_frames=num_frames, + fps=args.fps, + step=args.step, + cfg_scale=args.cfg_scale, + flow_shift=args.flow_shift, + seed=args.seed, + negative_prompt=args.negative_prompt, + no_refiner=args.no_refiner, + save_stage1=args.save_stage1, + refiner_seed=args.seed, + stage1_precision=args.stage1_precision, + refiner_precision=args.refiner_precision, + quant_backend=args.quant_backend, + ) + pipeline_cfg = _pipeline_config(runner_cfg, quant_backend=quant_backend) + pipeline_cfg = derive_config(pipeline_cfg, enable_sync_and_profile=True) + pipeline = pipeline_cfg.setup().to(device).eval() + if args.compile_stage1: + install_stage1_compile_hook(pipeline) + preload_pipeline_components(pipeline) + + request = SanaWMI2VConditioningRequest( + image=image, + prompt=prompt, + poses_c2w=c2w, + intrinsics_vec4=intrinsics_vec4, + num_frames=num_frames, + fps=args.fps, + steps=args.step, + cfg_scale=args.cfg_scale, + flow_shift=args.flow_shift, + seed=args.seed, + negative_prompt=args.negative_prompt, + ) + cache = pipeline.initialize_cache( + decoder_context={ + "prompt": prompt, + "fps": args.fps, + "save_stage1": args.save_stage1, + "refiner_seed": args.seed, + "sink_size": runner_cfg.sink_size, + } + ) + if torch.cuda.is_available(): + torch.cuda.reset_peak_memory_stats(device) + torch.cuda.synchronize(device) + generation_start = time.perf_counter() + with torch.inference_mode(): + decoded = pipeline.generate(0, cache, input=request) + if torch.cuda.is_available(): + torch.cuda.synchronize(device) + wall_s = time.perf_counter() - generation_start + stats = pipeline.finalize(0, cache) or {} + + if not isinstance(decoded, SanaWMDecodedVideo): + raise TypeError(f"expected SanaWMDecodedVideo, got {type(decoded).__name__}") + + frames = np.asarray(decoded.video_hwc, dtype=np.uint8) + video_path = write_video(args.output_dir, args.name, frames, args.fps) + if args.dump_frames is not None: + args.dump_frames.parent.mkdir(parents=True, exist_ok=True) + np.save(args.dump_frames, frames) + if args.stats_json is not None: + write_json( + args.stats_json, + { + "backend": "flashdreams", + "variant": "bidirectional", + "runner": "sana-wm-bidirectional", + "video_path": str(video_path), + "video_shape": list(frames.shape), + "num_frames": num_frames, + "seed": args.seed, + "fps": args.fps, + "step": args.step, + "cfg_scale": args.cfg_scale, + "no_refiner": args.no_refiner, + "stage1_precision": args.stage1_precision, + "refiner_precision": args.refiner_precision, + "compile_stage1": args.compile_stage1, + "force_cudnn_sdpa": args.force_cudnn_sdpa, + "wall_s": wall_s, + "stats_ms": stats, + }, + ) + print(f"[flashdreams] wrote {video_path}") + + +if __name__ == "__main__": + main() diff --git a/integrations/sana/tests/parity_check/run_flashdreams_common.py b/integrations/sana/tests/parity_check/run_flashdreams_common.py new file mode 100644 index 000000000..41608ca07 --- /dev/null +++ b/integrations/sana/tests/parity_check/run_flashdreams_common.py @@ -0,0 +1,270 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared helpers for FlashDreams SANA-WM benchmark runners.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +import imageio.v3 as iio +import numpy as np +import torch +from PIL import Image + +from sana_wm.camera import ( + action_string_to_c2w, + default_intrinsics_vec4, + fit_camera_trajectory, + load_intrinsics, + resize_center_crop_geometry, + snap_num_frames, + transform_intrinsics_for_crop, +) +from sana_wm.constants import ( + DEFAULT_VIDEO_HEIGHT, + DEFAULT_VIDEO_WIDTH, + SANA_WM_VAE_TEMPORAL_COMPRESSION, +) + + +def apply_backend_defaults() -> None: + """Apply the stack-matched CUDA defaults used by the benchmark harness.""" + torch.backends.cudnn.benchmark = True + torch.set_float32_matmul_precision("high") + if torch.cuda.is_available(): + torch.backends.cuda.enable_flash_sdp(False) + torch.backends.cuda.enable_math_sdp(True) + torch.backends.cuda.enable_mem_efficient_sdp(False) + torch.backends.cuda.enable_cudnn_sdp(True) + try: + import torch._inductor.config as inductor_config + + inductor_config.coordinate_descent_tuning = True + inductor_config.epilogue_fusion = True + except Exception: + pass + + +def resolve_device(device: str) -> torch.device: + """Resolve ``auto``/``cuda`` to the local rank's CUDA device when present.""" + if device == "auto": + if torch.cuda.is_available(): + return torch.device(f"cuda:{int(os.environ.get('LOCAL_RANK', '0'))}") + return torch.device("cpu") + if device == "cuda" and torch.cuda.is_available(): + return torch.device(f"cuda:{int(os.environ.get('LOCAL_RANK', '0'))}") + return torch.device(device) + + +def prepare_bidirectional_inputs( + *, + image_path: Path, + camera_path: Path, + intrinsics_path: Path | None, + intrinsics_hfov_deg: float, + num_frames_requested: int, +) -> tuple[Image.Image, np.ndarray, np.ndarray, int]: + """Prepare the legacy bidirectional input contract.""" + image = Image.open(image_path).convert("RGB") + c2w_full = np.load(camera_path).astype(np.float32) + if c2w_full.ndim != 3 or c2w_full.shape[1:] != (4, 4): + raise ValueError(f"--camera-path must be [F,4,4]; got {c2w_full.shape}.") + + num_frames = min(num_frames_requested, c2w_full.shape[0]) + num_frames = snap_num_frames( + num_frames, + stride=SANA_WM_VAE_TEMPORAL_COMPRESSION, + upper_bound=c2w_full.shape[0], + ) + c2w = c2w_full[:num_frames] + cropped, intrinsics_vec4 = _prepare_image_and_intrinsics( + image, + intrinsics_path=intrinsics_path, + intrinsics_hfov_deg=intrinsics_hfov_deg, + num_frames=num_frames, + ) + return cropped, c2w, intrinsics_vec4, num_frames + + +def prepare_streaming_inputs( + *, + image_path: Path, + camera_path: Path | None, + camera_source: str, + action: str | None, + translation_speed: float, + rotation_speed_deg: float, + intrinsics_path: Path | None, + intrinsics_hfov_deg: float, + num_frames_requested: int, + snap_stride: int, +) -> tuple[Image.Image, np.ndarray, np.ndarray, int]: + """Prepare the streaming action/camera input contract.""" + image = Image.open(image_path).convert("RGB") + if camera_source == "action": + if not action: + raise ValueError("--camera-source=action requires --action.") + num_frames = snap_num_frames(num_frames_requested, stride=snap_stride) + c2w = action_string_to_c2w( + action, + translation_speed=translation_speed, + rotation_speed_deg=rotation_speed_deg, + num_frames=num_frames, + ) + else: + if camera_path is None: + raise ValueError("--camera-source=camera requires --camera-path.") + c2w_full = np.load(camera_path).astype(np.float32) + if c2w_full.ndim != 3 or c2w_full.shape[1:] != (4, 4): + raise ValueError(f"--camera-path must be [F,4,4]; got {c2w_full.shape}.") + num_frames = min(num_frames_requested, c2w_full.shape[0]) + num_frames = snap_num_frames( + num_frames, + stride=snap_stride, + upper_bound=c2w_full.shape[0], + ) + c2w = fit_camera_trajectory(c2w_full, num_frames) + + cropped, intrinsics_vec4 = _prepare_image_and_intrinsics( + image, + intrinsics_path=intrinsics_path, + intrinsics_hfov_deg=intrinsics_hfov_deg, + num_frames=num_frames, + ) + return cropped, c2w, intrinsics_vec4, num_frames + + +def preload_pipeline_components(pipeline: Any) -> None: + """Load model components before timed generation begins.""" + encoder = getattr(pipeline, "encoder", None) + if encoder is not None: + text_encoder = getattr(encoder, "text_encoder", None) + if text_encoder is not None: + text_encoder._ensure_text_encoder() + first_frame_encoder = getattr(encoder, "first_frame_encoder", None) + if first_frame_encoder is not None: + first_frame_encoder._ensure_vae() + pipeline.diffusion_model.transformer._ensure_model() + decoder = getattr(pipeline, "decoder", None) + vae_decoder = getattr(decoder, "vae_decoder", None) + if vae_decoder is not None: + vae_decoder._ensure_vae() + refiner = getattr(decoder, "refiner", None) + if refiner is not None: + refiner._ensure_refiner() + + +def install_stage1_compile_hook(pipeline: Any) -> None: + """Compile Stage 1 lazily after checkpoint load.""" + transformer = pipeline.diffusion_model.transformer + original_ensure_model = transformer._ensure_model + compiled = {"done": False} + + def _ensure_model_with_compile() -> None: + original_ensure_model() + if compiled["done"]: + return + transformer.model = torch.compile( + transformer.model, + mode="max-autotune-no-cudagraphs", + ) + compiled["done"] = True + + transformer._ensure_model = _ensure_model_with_compile + + +def compile_streaming_refiner(pipeline: Any) -> None: + """Match upstream streaming's default refiner ``torch.compile`` path.""" + decoder = getattr(pipeline, "decoder", None) + refiner = getattr(decoder, "refiner", None) + if refiner is None: + return + refiner._ensure_refiner() + compiled = getattr(refiner.refiner, "_flashdreams_compiled", False) + if compiled: + return + compile_mode = os.environ.get( + "SANA_WM_TORCH_COMPILE_MODE", + "max-autotune-no-cudagraphs", + ).strip() + compile_dynamic_raw = os.environ.get( + "SANA_WM_TORCH_COMPILE_DYNAMIC", + "1", + ).strip().lower() + compile_dynamic = compile_dynamic_raw not in {"0", "false", "no", "off"} + refiner.refiner.transformer = torch.compile( + refiner.refiner.transformer, + mode=compile_mode, + dynamic=compile_dynamic, + ) + refiner.refiner._flashdreams_compiled = True + + +def sum_stage_ms(rows: list[dict[str, float]], key: str) -> float | None: + """Sum optional per-chunk CUDA stage timings.""" + values = [row[key] for row in rows if isinstance(row.get(key), (int, float))] + return float(sum(values)) if values else None + + +def write_video(output_dir: Path, name: str, frames: np.ndarray, fps: int) -> Path: + """Write HWC uint8 frames to an MP4 file.""" + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / f"{name}_generated.mp4" + iio.imwrite(path, frames, fps=fps) + return path + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + """Write a benchmark JSON payload.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") + + +def _prepare_image_and_intrinsics( + image: Image.Image, + *, + intrinsics_path: Path | None, + intrinsics_hfov_deg: float, + num_frames: int, +) -> tuple[Image.Image, np.ndarray]: + resized_size, crop_offset = resize_center_crop_geometry( + image.size, + target_h=DEFAULT_VIDEO_HEIGHT, + target_w=DEFAULT_VIDEO_WIDTH, + ) + resized = image.resize(resized_size, Image.LANCZOS) + left, top = crop_offset + cropped = resized.crop( + (left, top, left + DEFAULT_VIDEO_WIDTH, top + DEFAULT_VIDEO_HEIGHT) + ) + if intrinsics_path is None: + intrinsics_src = default_intrinsics_vec4( + image.size, + num_frames, + hfov_deg=intrinsics_hfov_deg, + ) + else: + intrinsics_src = load_intrinsics(intrinsics_path, num_frames) + intrinsics_vec4 = transform_intrinsics_for_crop( + intrinsics_src, + image.size, + resized_size, + crop_offset, + ) + return cropped, intrinsics_vec4 diff --git a/integrations/sana/tests/parity_check/run_flashdreams_streaming.py b/integrations/sana/tests/parity_check/run_flashdreams_streaming.py new file mode 100644 index 000000000..828949628 --- /dev/null +++ b/integrations/sana/tests/parity_check/run_flashdreams_streaming.py @@ -0,0 +1,388 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run the FlashDreams SANA-WM streaming benchmark helper.""" + +from __future__ import annotations + +import argparse +import time +from pathlib import Path + +import numpy as np +import torch + +from flashdreams.infra.config import derive_config +from run_flashdreams_common import ( + apply_backend_defaults, + compile_streaming_refiner, + install_stage1_compile_hook, + preload_pipeline_components, + prepare_streaming_inputs, + resolve_device, + sum_stage_ms, + write_json, + write_video, +) +from sana_wm.conditioning import ( + SanaWMStreamingI2VConditioningRequest, + streaming_chunk_boundaries, +) +from sana_wm.config import RUNNER_SANA_WM_STREAMING +from sana_wm.constants import ( + SANA_WM_STREAMING_LATENT_CHUNK_SIZE, + SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES, + SANA_WM_VAE_TEMPORAL_COMPRESSION, +) +from sana_wm.decoder import SanaWMDecodedVideo +from sana_wm.runner import ( + _active_quantized_precisions, + _resolve_quant_backend, + _streaming_pipeline_config, + _validate_precision_request, +) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--image-path", type=Path, required=True) + parser.add_argument("--prompt-path", type=Path, required=True) + parser.add_argument("--camera-path", type=Path, default=None) + parser.add_argument( + "--camera-source", + choices=["camera", "action"], + default="camera", + ) + parser.add_argument("--action", default=None) + parser.add_argument("--translation-speed", type=float, default=0.025) + parser.add_argument("--rotation-speed-deg", type=float, default=0.6) + parser.add_argument("--intrinsics-path", type=Path, default=None) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--name", default="flashdreams") + parser.add_argument("--dump-frames", type=Path, default=None) + parser.add_argument("--stats-json", type=Path, default=None) + parser.add_argument("--num-frames", type=int, default=241) + parser.add_argument("--fps", type=int, default=16) + parser.add_argument("--cfg-scale", type=float, default=1.0) + parser.add_argument("--flow-shift", type=float, default=8.0) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--negative-prompt", default="") + parser.add_argument("--no-refiner", action="store_true") + parser.add_argument("--save-stage1", action="store_true") + parser.add_argument("--device", default="auto") + parser.add_argument("--intrinsics-hfov-deg", type=float, default=90.0) + parser.add_argument( + "--stage1-precision", + choices=["bf16", "fp8", "fp4"], + default="bf16", + ) + parser.add_argument( + "--refiner-precision", + choices=["bf16", "fp8", "fp4"], + default="bf16", + ) + parser.add_argument( + "--quant-backend", + choices=["auto", "torch", "torch-fp8", "torch-fp4"], + default="auto", + ) + parser.add_argument("--force-cudnn-sdpa", action="store_true") + parser.add_argument("--compile-stage1", action="store_true") + parser.add_argument("--compile-streaming-refiner", action="store_true") + parser.add_argument("--output-mode", choices=["mp4", "discard"], default="mp4") + parser.add_argument( + "--denoising-step-list", + default="1000,960,889,727,0", + ) + parser.add_argument( + "--num-frame-per-block", + type=int, + default=SANA_WM_STREAMING_LATENT_CHUNK_SIZE, + ) + parser.add_argument("--num-cached-blocks", type=int, default=2) + parser.add_argument("--sink-size", type=int, default=1) + parser.add_argument("--no-sink-token", action="store_true") + parser.add_argument( + "--refiner-block-size", + type=int, + default=SANA_WM_STREAMING_LATENT_CHUNK_SIZE, + ) + parser.add_argument( + "--refiner-kv-max-frames", + type=int, + default=SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES, + ) + parser.add_argument("--refiner-seed", type=int, default=None) + args = parser.parse_args(argv) + + if args.force_cudnn_sdpa: + apply_backend_defaults() + + prompt = args.prompt_path.read_text(encoding="utf-8", errors="replace").strip() + if not prompt: + raise ValueError(f"Prompt file is empty: {args.prompt_path}") + if args.refiner_seed is None: + args.refiner_seed = args.seed + + device = resolve_device(args.device) + quantized = _active_quantized_precisions( + stage1_precision=args.stage1_precision, + refiner_precision=args.refiner_precision, + refiner_enabled=not args.no_refiner, + ) + quant_backend = _resolve_quant_backend(args.quant_backend, quantized) + _validate_precision_request( + device=device, + stage1_precision=args.stage1_precision, + refiner_precision=args.refiner_precision, + refiner_enabled=not args.no_refiner, + quant_backend=args.quant_backend, + ) + + snap_stride = SANA_WM_VAE_TEMPORAL_COMPRESSION * args.refiner_block_size + image, c2w, intrinsics_vec4, num_frames = prepare_streaming_inputs( + image_path=args.image_path, + camera_path=args.camera_path, + camera_source=args.camera_source, + action=args.action, + translation_speed=args.translation_speed, + rotation_speed_deg=args.rotation_speed_deg, + intrinsics_path=args.intrinsics_path, + intrinsics_hfov_deg=args.intrinsics_hfov_deg, + num_frames_requested=args.num_frames, + snap_stride=snap_stride, + ) + denoising_step_list = tuple( + int(timestep.strip()) + for timestep in args.denoising_step_list.split(",") + if timestep.strip() + ) + if not denoising_step_list or denoising_step_list[-1] != 0: + raise ValueError("--denoising-step-list must end with 0.") + + runner_cfg = derive_config( + RUNNER_SANA_WM_STREAMING, + output_dir=args.output_dir, + runner_name=args.name, + num_frames=num_frames, + fps=args.fps, + step=len(denoising_step_list) - 1, + cfg_scale=args.cfg_scale, + flow_shift=args.flow_shift, + seed=args.seed, + negative_prompt=args.negative_prompt, + no_refiner=args.no_refiner, + save_stage1=args.save_stage1, + refiner_seed=args.refiner_seed, + sink_size=args.sink_size, + stage1_precision=args.stage1_precision, + refiner_precision=args.refiner_precision, + quant_backend=args.quant_backend, + num_frame_per_block=args.num_frame_per_block, + num_cached_blocks=args.num_cached_blocks, + no_sink_token=args.no_sink_token, + denoising_step_list=denoising_step_list, + refiner_block_size=args.refiner_block_size, + refiner_kv_max_frames=args.refiner_kv_max_frames, + ) + pipeline_cfg = _streaming_pipeline_config(runner_cfg, quant_backend=quant_backend) + pipeline_cfg = derive_config(pipeline_cfg, enable_sync_and_profile=True) + pipeline = pipeline_cfg.setup().to(device).eval() + if args.compile_stage1: + install_stage1_compile_hook(pipeline) + preload_pipeline_components(pipeline) + if args.compile_streaming_refiner: + compile_streaming_refiner(pipeline) + + latent_frames = (num_frames - 1) // SANA_WM_VAE_TEMPORAL_COMPRESSION + 1 + chunk_boundaries = streaming_chunk_boundaries( + latent_frames, + args.num_frame_per_block, + ) + request = SanaWMStreamingI2VConditioningRequest( + image=image, + prompt=prompt, + poses_c2w=c2w, + intrinsics_vec4=intrinsics_vec4, + num_frames=num_frames, + fps=args.fps, + steps=len(denoising_step_list) - 1, + cfg_scale=args.cfg_scale, + flow_shift=args.flow_shift, + seed=args.seed, + negative_prompt=args.negative_prompt, + num_frame_per_block=args.num_frame_per_block, + ) + cache = pipeline.initialize_cache( + decoder_context={ + "prompt": prompt, + "fps": args.fps, + "save_stage1": args.save_stage1, + "refiner_seed": args.refiner_seed, + "sink_size": runner_cfg.sink_size, + "block_size": args.refiner_block_size, + "refiner_kv_max_frames": args.refiner_kv_max_frames, + } + ) + + decoded_chunks: list[np.ndarray] = [] + stage1_chunks: list[np.ndarray] = [] + per_chunk_stats: list[dict[str, float]] = [] + per_chunk_wall_s: list[float] = [] + if torch.cuda.is_available(): + torch.cuda.reset_peak_memory_stats(device) + torch.cuda.synchronize(device) + generation_start = time.perf_counter() + with torch.inference_mode(): + for ar_idx in range(len(chunk_boundaries) - 1): + if torch.cuda.is_available(): + torch.cuda.synchronize(device) + chunk_start = time.perf_counter() + decoded = pipeline.generate(ar_idx, cache, input=request) + stats = pipeline.finalize(ar_idx, cache) or {} + if torch.cuda.is_available(): + torch.cuda.synchronize(device) + per_chunk_wall_s.append(time.perf_counter() - chunk_start) + per_chunk_stats.append(stats) + if not isinstance(decoded, SanaWMDecodedVideo): + raise TypeError( + "expected SanaWMDecodedVideo, " + f"got {type(decoded).__name__}" + ) + if decoded.video_hwc.size: + decoded_chunks.append(np.asarray(decoded.video_hwc, dtype=np.uint8)) + if decoded.stage1_video_hwc is not None and decoded.stage1_video_hwc.size: + stage1_chunks.append(np.asarray(decoded.stage1_video_hwc, dtype=np.uint8)) + if torch.cuda.is_available(): + torch.cuda.synchronize(device) + stream_wall_seconds = time.perf_counter() - generation_start + first_chunk_seconds = per_chunk_wall_s[0] if per_chunk_wall_s else None + steady_state_seconds = ( + sum(per_chunk_wall_s[1:]) if len(per_chunk_wall_s) > 1 else None + ) + + if not decoded_chunks: + raise RuntimeError("FlashDreams SANA-WM streaming produced no decoded frames.") + video_hwc = np.concatenate(decoded_chunks, axis=0) + first_chunk_frames = int(decoded_chunks[0].shape[0]) if decoded_chunks else 0 + n_pixel_frames = int(video_hwc.shape[0]) + video_path = None + if args.output_mode != "discard": + video_path = write_video(args.output_dir, args.name, video_hwc, args.fps) + if args.dump_frames is not None: + args.dump_frames.parent.mkdir(parents=True, exist_ok=True) + np.save(args.dump_frames, video_hwc) + if stage1_chunks and args.output_mode != "discard": + write_video( + args.output_dir, + f"{args.name}_stage1", + np.concatenate(stage1_chunks, axis=0), + args.fps, + ) + + steady_frames = max(0, n_pixel_frames - first_chunk_frames) + frames_per_second = ( + n_pixel_frames / stream_wall_seconds if stream_wall_seconds > 0 else None + ) + realtime_factor = ( + frames_per_second / args.fps + if frames_per_second is not None and args.fps > 0 + else None + ) + steady_state_frames_per_second = ( + steady_frames / steady_state_seconds + if steady_state_seconds and steady_state_seconds > 0 + else None + ) + steady_state_realtime_factor = ( + steady_state_frames_per_second / args.fps + if steady_state_frames_per_second is not None and args.fps > 0 + else None + ) + stage1_cuda_ms = sum_stage_ms(per_chunk_stats, "diffuse_ms") + decode_cuda_ms = sum_stage_ms(per_chunk_stats, "decode_ms") + if args.stats_json is not None: + write_json( + args.stats_json, + { + "backend": "flashdreams", + "variant": "streaming", + "runner": "sana-wm-streaming", + "output_mode": args.output_mode, + "output_path": str(video_path) if video_path is not None else None, + "video_shape": list(video_hwc.shape), + "requested_num_frames": args.num_frames, + "actual_num_frames": num_frames, + "num_frames": num_frames, + "seed": args.seed, + "fps": args.fps, + "cfg_scale": args.cfg_scale, + "flow_shift": args.flow_shift, + "no_refiner": args.no_refiner, + "save_stage1": args.save_stage1, + "stage1_precision": args.stage1_precision, + "refiner_precision": args.refiner_precision, + "quant_backend": args.quant_backend, + "compile_stage1": args.compile_stage1, + "compile_streaming_refiner": args.compile_streaming_refiner, + "force_cudnn_sdpa": args.force_cudnn_sdpa, + "denoising_step_list": list(denoising_step_list), + "num_frame_per_block": args.num_frame_per_block, + "num_cached_blocks": args.num_cached_blocks, + "sink_token": not args.no_sink_token, + "refiner_block_size": args.refiner_block_size, + "refiner_kv_max_frames": args.refiner_kv_max_frames, + "refiner_seed": args.refiner_seed, + "stream_wall_seconds": stream_wall_seconds, + "wall_s": stream_wall_seconds, + "end_to_end_seconds": stream_wall_seconds, + "first_chunk_seconds": first_chunk_seconds, + "first_chunk_frames": first_chunk_frames, + "steady_state_seconds": steady_state_seconds, + "steady_state_frames_per_second": steady_state_frames_per_second, + "steady_state_realtime_factor": steady_state_realtime_factor, + "frames_per_second": frames_per_second, + "realtime_factor": realtime_factor, + "n_decode_chunks": len(chunk_boundaries) - 1, + "n_refiner_blocks": len(chunk_boundaries) - 1, + "n_pixel_frames": n_pixel_frames, + "stage1_cuda_seconds": ( + stage1_cuda_ms / 1000.0 + if stage1_cuda_ms is not None + else None + ), + "refiner_cuda_seconds": None, + "decode_cuda_seconds": ( + decode_cuda_ms / 1000.0 + if decode_cuda_ms is not None + else None + ), + "per_chunk_wall_seconds": per_chunk_wall_s, + "per_chunk_stats_ms": per_chunk_stats, + "mem_peak_gib": ( + torch.cuda.max_memory_allocated(device) / (1024**3) + if torch.cuda.is_available() + else None + ), + }, + ) + if video_path is not None: + print(f"[flashdreams] wrote {video_path}") + else: + print(f"[flashdreams] streaming output discarded ({n_pixel_frames} frames)") + + +if __name__ == "__main__": + main() diff --git a/integrations/sana/tests/parity_check/run_streaming.sh b/integrations/sana/tests/parity_check/run_streaming.sh new file mode 100644 index 000000000..42910e18d --- /dev/null +++ b/integrations/sana/tests/parity_check/run_streaming.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Patch pinned upstream SANA-WM with streaming instrumentation, run it and the +# FlashDreams streaming integration on the same demo input, dump decoded uint8 +# frames, compute frame parity, and report chunk-boundary continuity. + +set -euo pipefail + +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/run_common.sh" + +SANA_REPO="$(parity_abspath "${SANA_REPO:-${HOME}/dev/Sana}")" +PATCH_FILE="${PARITY_SCRIPT_DIR}/changes_streaming.patch" + +OUTPUT_DIR="$(parity_abspath "${OUTPUT_DIR:-${PARITY_SCRIPT_DIR}/outputs/parity/streaming}")" +UPSTREAM_OUT="${OUTPUT_DIR}/upstream" +FLASHDREAMS_OUT="${OUTPUT_DIR}/flashdreams" +IMAGE_PATH="$(parity_abspath "${IMAGE_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.png}")" +PROMPT_PATH="$(parity_abspath "${PROMPT_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.txt}")" +CAMERA_PATH="$(parity_abspath "${CAMERA_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_pose.npy}")" +INTRINSICS_PATH="$(parity_abspath "${INTRINSICS_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_intrinsics.npy}")" + +NUM_FRAMES="${NUM_FRAMES:-241}" +FPS="${FPS:-16}" +CFG_SCALE="${CFG_SCALE:-1.0}" +FLOW_SHIFT="${FLOW_SHIFT:-8.0}" +SEED="${SEED:-42}" +REFINER_SEED="${REFINER_SEED:-${STREAMING_REFINER_SEED:-${SEED}}}" +NO_REFINER="${NO_REFINER:-0}" +FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" +STREAMING_NO_COMPILE="${STREAMING_NO_COMPILE:-1}" +STAGE1_PRECISION="${STAGE1_PRECISION:-bf16}" +REFINER_PRECISION="${REFINER_PRECISION:-${STAGE1_PRECISION}}" +QUANT_BACKEND="${QUANT_BACKEND:-auto}" +STREAMING_ACTION="${STREAMING_ACTION-w-80,dw-40,w-80,aw-40}" +TRANSLATION_SPEED="${TRANSLATION_SPEED:-0.025}" +ROTATION_SPEED_DEG="${ROTATION_SPEED_DEG:-0.6}" +STREAMING_DENOISING_STEP_LIST="${STREAMING_DENOISING_STEP_LIST:-1000,960,889,727,0}" +STREAMING_NUM_FRAME_PER_BLOCK="${STREAMING_NUM_FRAME_PER_BLOCK:-3}" +STREAMING_NUM_CACHED_BLOCKS="${STREAMING_NUM_CACHED_BLOCKS:-2}" +STREAMING_REFINER_BLOCK_SIZE="${STREAMING_REFINER_BLOCK_SIZE:-3}" +STREAMING_REFINER_KV_MAX_FRAMES="${STREAMING_REFINER_KV_MAX_FRAMES:-11}" +STREAMING_SINK_SIZE="${STREAMING_SINK_SIZE:-1}" +STREAMING_CRF="${STREAMING_CRF:-18}" +STREAMING_PRESET="${STREAMING_PRESET:-medium}" +STREAMING_ENCODER="${STREAMING_ENCODER:-${SANA_WM_STREAMING_MP4_ENCODER:-libx264}}" +STREAMING_OUTPUT_MODE="${STREAMING_OUTPUT_MODE:-mp4}" +STREAMING_SAMPLE_FRAME_STRIDE="${STREAMING_SAMPLE_FRAME_STRIDE:-1}" + +if parity_is_true "${NO_REFINER}"; then + echo "[run] ERROR: streaming parity uses the full upstream streaming stack; NO_REFINER=1 is not supported." >&2 + exit 1 +fi +if [[ "${STREAMING_OUTPUT_MODE}" != "mp4" && "${STREAMING_OUTPUT_MODE}" != "cpu" && "${STREAMING_OUTPUT_MODE}" != "discard" ]]; then + echo "[run] ERROR: STREAMING_OUTPUT_MODE must be mp4, cpu, or discard; got ${STREAMING_OUTPUT_MODE}" >&2 + exit 1 +fi +if [[ "${STREAMING_SAMPLE_FRAME_STRIDE}" != "1" ]]; then + echo "[run] ERROR: streaming parity and continuity checks require STREAMING_SAMPLE_FRAME_STRIDE=1." >&2 + exit 1 +fi + +parity_prepare_sana_repo "${SANA_REPO}" "${PATCH_FILE}" +if [[ ! -f "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm_streaming.py" ]]; then + echo "[run] ERROR: pinned Sana checkout has no SANA-WM streaming entrypoint." >&2 + echo " Expected: inference_video_scripts/wm/inference_sana_wm_streaming.py" >&2 + exit 1 +fi +parity_sync_venv +UPSTREAM_PYTHONPATH="$(parity_upstream_pythonpath "${SANA_REPO}")" + +mkdir -p "${UPSTREAM_OUT}" "${FLASHDREAMS_OUT}" + +UPSTREAM_FRAMES="${UPSTREAM_OUT}/frames.npz" +FLASHDREAMS_FRAMES="${FLASHDREAMS_OUT}/frames.npy" +UPSTREAM_STATS="${UPSTREAM_OUT}/stats.json" +FLASHDREAMS_STATS="${FLASHDREAMS_OUT}/stats.json" + +UPSTREAM_PRECISION_ARGS=( + --stage1_precision "${STAGE1_PRECISION}" + --refiner_precision "${REFINER_PRECISION}" +) +FLASHDREAMS_PRECISION_ARGS=( + --stage1-precision "${STAGE1_PRECISION}" + --refiner-precision "${REFINER_PRECISION}" + --quant-backend "${QUANT_BACKEND}" +) +UPSTREAM_CAMERA_ARGS=(--camera "${CAMERA_PATH}") +FLASHDREAMS_CAMERA_ARGS=(--camera-source camera --camera-path "${CAMERA_PATH}") +if [[ -n "${STREAMING_ACTION}" ]]; then + UPSTREAM_CAMERA_ARGS=(--action "${STREAMING_ACTION}") + FLASHDREAMS_CAMERA_ARGS=(--camera-source action --action "${STREAMING_ACTION}" --camera-path "${CAMERA_PATH}") +fi +UPSTREAM_MODE_ARGS=(--output_mode "${STREAMING_OUTPUT_MODE}") +FLASHDREAMS_MODE_ARGS=(--output-mode "${STREAMING_OUTPUT_MODE}") +if parity_is_true "${STREAMING_NO_COMPILE}"; then + UPSTREAM_MODE_ARGS+=(--no_compile) +fi +FLASHDREAMS_COMPILE_ARGS=() +if ! parity_is_true "${STREAMING_NO_COMPILE}"; then + FLASHDREAMS_COMPILE_ARGS+=(--compile-streaming-refiner) +fi +FLASHDREAMS_BACKEND_ARGS=() +if parity_is_true "${FORCE_CUDNN_SDPA}"; then + FLASHDREAMS_BACKEND_ARGS+=(--force-cudnn-sdpa) +fi + +echo "[run] upstream SANA-WM streaming -> ${UPSTREAM_OUT}" +( cd "${PARITY_SCRIPT_DIR}" && \ + PYTHONPATH="${UPSTREAM_PYTHONPATH}" \ + uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm_streaming.py" \ + --image "${IMAGE_PATH}" \ + --prompt "${PROMPT_PATH}" \ + "${UPSTREAM_CAMERA_ARGS[@]}" \ + --intrinsics "${INTRINSICS_PATH}" \ + --output_dir "${UPSTREAM_OUT}" \ + --name upstream \ + --num_frames "${NUM_FRAMES}" \ + --fps "${FPS}" \ + --cfg_scale "${CFG_SCALE}" \ + --flow_shift "${FLOW_SHIFT}" \ + --seed "${SEED}" \ + --benchmark_json "${UPSTREAM_STATS}" \ + --sample_frames_npz "${UPSTREAM_FRAMES}" \ + --sample_frame_stride "${STREAMING_SAMPLE_FRAME_STRIDE}" \ + --denoising_step_list "${STREAMING_DENOISING_STEP_LIST}" \ + --num_frame_per_block "${STREAMING_NUM_FRAME_PER_BLOCK}" \ + --num_cached_blocks "${STREAMING_NUM_CACHED_BLOCKS}" \ + --sink_size "${STREAMING_SINK_SIZE}" \ + --refiner_block_size "${STREAMING_REFINER_BLOCK_SIZE}" \ + --refiner_kv_max_frames "${STREAMING_REFINER_KV_MAX_FRAMES}" \ + --refiner_seed "${REFINER_SEED}" \ + --streaming_crf "${STREAMING_CRF}" \ + --streaming_preset "${STREAMING_PRESET}" \ + --streaming_encoder "${STREAMING_ENCODER}" \ + "${UPSTREAM_PRECISION_ARGS[@]}" \ + "${UPSTREAM_MODE_ARGS[@]}" ) + +echo "[run] FlashDreams SANA-WM streaming -> ${FLASHDREAMS_OUT}" +( cd "${PARITY_SCRIPT_DIR}" && \ + uv run python "${PARITY_SCRIPT_DIR}/run_flashdreams_streaming.py" \ + --image-path "${IMAGE_PATH}" \ + --prompt-path "${PROMPT_PATH}" \ + "${FLASHDREAMS_CAMERA_ARGS[@]}" \ + --intrinsics-path "${INTRINSICS_PATH}" \ + --output-dir "${FLASHDREAMS_OUT}" \ + --name flashdreams \ + --num-frames "${NUM_FRAMES}" \ + --fps "${FPS}" \ + --cfg-scale "${CFG_SCALE}" \ + --flow-shift "${FLOW_SHIFT}" \ + --seed "${SEED}" \ + --refiner-seed "${REFINER_SEED}" \ + --stats-json "${FLASHDREAMS_STATS}" \ + --dump-frames "${FLASHDREAMS_FRAMES}" \ + --denoising-step-list "${STREAMING_DENOISING_STEP_LIST}" \ + --num-frame-per-block "${STREAMING_NUM_FRAME_PER_BLOCK}" \ + --num-cached-blocks "${STREAMING_NUM_CACHED_BLOCKS}" \ + --sink-size "${STREAMING_SINK_SIZE}" \ + --refiner-block-size "${STREAMING_REFINER_BLOCK_SIZE}" \ + --refiner-kv-max-frames "${STREAMING_REFINER_KV_MAX_FRAMES}" \ + "${FLASHDREAMS_MODE_ARGS[@]}" \ + "${FLASHDREAMS_COMPILE_ARGS[@]}" \ + "${FLASHDREAMS_PRECISION_ARGS[@]}" \ + "${FLASHDREAMS_BACKEND_ARGS[@]}" ) + +echo "[diff] summarising streaming frame parity -> ${OUTPUT_DIR}/parity.json" +( cd "${PARITY_SCRIPT_DIR}" && \ + uv run python "${PARITY_SCRIPT_DIR}/diff_parity.py" \ + --upstream "${UPSTREAM_FRAMES}" \ + --flashdreams "${FLASHDREAMS_FRAMES}" \ + --output "${OUTPUT_DIR}/parity.json" ) + +echo "[diff] summarising streaming continuity -> ${OUTPUT_DIR}/continuity.json" +( cd "${PARITY_SCRIPT_DIR}" && \ + uv run python "${PARITY_SCRIPT_DIR}/streaming_continuity.py" \ + --upstream "${UPSTREAM_FRAMES}" \ + --flashdreams "${FLASHDREAMS_FRAMES}" \ + --chunk-size "$(( STREAMING_REFINER_BLOCK_SIZE * 8 ))" \ + --output "${OUTPUT_DIR}/continuity.json" ) + +echo "[run] done." +echo " upstream frames : ${UPSTREAM_FRAMES}" +echo " flashdreams frames: ${FLASHDREAMS_FRAMES}" +echo " parity JSON : ${OUTPUT_DIR}/parity.json" +echo " continuity JSON : ${OUTPUT_DIR}/continuity.json" diff --git a/integrations/sana/tests/parity_check/streaming_continuity.py b/integrations/sana/tests/parity_check/streaming_continuity.py new file mode 100644 index 000000000..cf0dc9fc6 --- /dev/null +++ b/integrations/sana/tests/parity_check/streaming_continuity.py @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Summarize SANA-WM streaming chunk-boundary continuity.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import numpy as np + +from diff_parity import _load_frames + + +def summarize_frames(frames: np.ndarray, *, chunk_size: int) -> dict[str, Any]: + """Return consecutive-frame deltas with chunk-boundary highlights.""" + if chunk_size <= 0: + raise ValueError(f"chunk_size must be positive, got {chunk_size}.") + if frames.shape[0] < 2: + raise ValueError(f"At least two frames are required, got {frames.shape[0]}.") + + diffs = np.abs(frames[1:].astype(np.int16) - frames[:-1].astype(np.int16)) + per_transition = diffs.reshape(diffs.shape[0], -1).mean(axis=1) + boundary_indices = list(range(chunk_size - 1, len(per_transition), chunk_size)) + boundary_set = set(boundary_indices) + nonboundary = np.asarray( + [value for idx, value in enumerate(per_transition) if idx not in boundary_set], + dtype=np.float64, + ) + boundary = np.asarray([per_transition[idx] for idx in boundary_indices], dtype=np.float64) + + nonboundary_p95 = float(np.percentile(nonboundary, 95)) if nonboundary.size else None + boundary_mean = float(boundary.mean()) if boundary.size else None + ratio = None + if nonboundary_p95 is not None and nonboundary_p95 > 0.0 and boundary_mean is not None: + ratio = boundary_mean / nonboundary_p95 + + return { + "shape": list(frames.shape), + "chunk_size": int(chunk_size), + "mean_abs_delta": float(per_transition.mean()), + "p95_abs_delta": float(np.percentile(per_transition, 95)), + "nonboundary_mean_abs_delta": float(nonboundary.mean()) if nonboundary.size else None, + "nonboundary_p95_abs_delta": nonboundary_p95, + "boundary_mean_abs_delta": boundary_mean, + "boundary_max_abs_delta": float(boundary.max()) if boundary.size else None, + "boundary_to_nonboundary_p95_ratio": ratio, + "boundary_transitions": [ + { + "from_frame": int(idx), + "to_frame": int(idx + 1), + "mean_abs_delta": float(per_transition[idx]), + } + for idx in boundary_indices + ], + } + + +def _comparison( + upstream: dict[str, Any] | None, + flashdreams: dict[str, Any] | None, +) -> dict[str, Any]: + if upstream is None or flashdreams is None: + return {} + upstream_boundary = upstream.get("boundary_mean_abs_delta") + flashdreams_boundary = flashdreams.get("boundary_mean_abs_delta") + upstream_ratio = upstream.get("boundary_to_nonboundary_p95_ratio") + flashdreams_ratio = flashdreams.get("boundary_to_nonboundary_p95_ratio") + return { + "shape_match": upstream.get("shape") == flashdreams.get("shape"), + "flashdreams_boundary_mean_minus_upstream": ( + float(flashdreams_boundary - upstream_boundary) + if isinstance(upstream_boundary, (int, float)) + and isinstance(flashdreams_boundary, (int, float)) + else None + ), + "flashdreams_boundary_ratio_minus_upstream": ( + float(flashdreams_ratio - upstream_ratio) + if isinstance(upstream_ratio, (int, float)) + and isinstance(flashdreams_ratio, (int, float)) + else None + ), + } + + +def _format_optional_float(value: object) -> str: + return f"{value:.4f}" if isinstance(value, (int, float)) else "n/a" + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--upstream", type=Path, default=None) + parser.add_argument("--flashdreams", type=Path, default=None) + parser.add_argument("--chunk-size", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + + if args.upstream is None and args.flashdreams is None: + raise ValueError("At least one of --upstream or --flashdreams is required.") + + upstream = ( + summarize_frames(_load_frames(args.upstream), chunk_size=args.chunk_size) + if args.upstream is not None + else None + ) + flashdreams = ( + summarize_frames(_load_frames(args.flashdreams), chunk_size=args.chunk_size) + if args.flashdreams is not None + else None + ) + payload = { + "chunk_size": int(args.chunk_size), + "upstream": upstream, + "flashdreams": flashdreams, + "comparison": _comparison(upstream, flashdreams), + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + for label, summary in (("upstream", upstream), ("flashdreams", flashdreams)): + if summary is None: + continue + print( + f"{label}: boundary mean " + f"{_format_optional_float(summary['boundary_mean_abs_delta'])}, " + f"nonboundary p95 " + f"{_format_optional_float(summary['nonboundary_p95_abs_delta'])}, " + f"ratio " + f"{_format_optional_float(summary['boundary_to_nonboundary_p95_ratio'])}" + ) + + +if __name__ == "__main__": + main() diff --git a/integrations/sana/tests/test_parity_benchmark_summary.py b/integrations/sana/tests/test_parity_benchmark_summary.py index 479b64e61..7901b10ae 100644 --- a/integrations/sana/tests/test_parity_benchmark_summary.py +++ b/integrations/sana/tests/test_parity_benchmark_summary.py @@ -335,6 +335,134 @@ def test_streaming_upstream_summary_uses_steady_state_ms_per_chunk(tmp_path: Pat assert "full-clip wall median | 10.00 s" in report +def test_streaming_comparison_summary_writes_chart_data(tmp_path: Path) -> None: + module = _load_bench_summary() + upstream = tmp_path / "upstream" / "run_0" + flashdreams = tmp_path / "flashdreams" / "run_0" + upstream.mkdir(parents=True) + flashdreams.mkdir(parents=True) + (upstream / "stats.json").write_text( + json.dumps( + { + "variant": "streaming", + "stream_wall_seconds": 12.0, + "steady_state_seconds": 8.0, + "first_chunk_seconds": 4.0, + "first_chunk_frames": 24, + "n_decode_chunks": 5, + "n_refiner_blocks": 5, + "n_pixel_frames": 120, + "stage1_cuda_seconds": 2.0, + "refiner_cuda_seconds": 4.0, + "decode_cuda_seconds": 1.0, + "peak_mem_gb": 48.0, + "output_path": "upstream.mp4", + } + ), + encoding="utf-8", + ) + (flashdreams / "stats.json").write_text( + json.dumps( + { + "variant": "streaming", + "stream_wall_seconds": 8.0, + "steady_state_seconds": 4.0, + "first_chunk_seconds": 4.0, + "first_chunk_frames": 24, + "n_decode_chunks": 5, + "n_refiner_blocks": 5, + "n_pixel_frames": 120, + "stage1_cuda_seconds": 1.5, + "refiner_cuda_seconds": None, + "decode_cuda_seconds": 2.0, + "mem_peak_gib": 44.0, + "output_path": "flashdreams.mp4", + } + ), + encoding="utf-8", + ) + (upstream / "command.txt").write_text("uv run python upstream.py\n", encoding="utf-8") + (flashdreams / "command.txt").write_text( + "uv run python run_flashdreams_streaming.py\n", + encoding="utf-8", + ) + out_json = tmp_path / "bench.json" + out_md = tmp_path / "bench.md" + out_chart = tmp_path / "perf.md" + + module.main( + [ + "--variant", + "streaming", + "--bench-side", + "both", + "--upstream-dir", + str(tmp_path / "upstream"), + "--flashdreams-dir", + str(tmp_path / "flashdreams"), + "--warmup-runs", + "0", + "--image-path", + "image.png", + "--prompt-path", + "prompt.txt", + "--camera-path", + "pose.npy", + "--camera-source", + "action", + "--action", + "w-80", + "--intrinsics-path", + "intrinsics.npy", + "--num-frames", + "121", + "--seed", + "42", + "--stage1-precision", + "bf16", + "--refiner-precision", + "bf16", + "--device-label", + "GB202", + "--output-mode", + "mp4", + "--denoising-step-list", + "1000,960,889,727,0", + "--num-frame-per-block", + "3", + "--refiner-block-size", + "3", + "--refiner-kv-max-frames", + "11", + "--output-json", + str(out_json), + "--output-md", + str(out_md), + "--output-chart-md", + str(out_chart), + ] + ) + + assert out_chart.read_text(encoding="utf-8") == ( + "# SANA-WM Streaming Benchmark Data (ms/chunk)\n" + "\n" + "| device | official | flashdreams |\n" + "| --- | ---: | ---: |\n" + "| GB202 | 2000.00 | 1000.00 |\n" + ) + summary = json.loads(out_json.read_text(encoding="utf-8")) + assert summary["bench_side"] == "both" + assert summary["commands"] == { + "upstream": "uv run python upstream.py", + "flashdreams": "uv run python run_flashdreams_streaming.py", + } + assert summary["benchmark"]["official"] == 2000.0 + assert summary["benchmark"]["flashdreams"] == 1000.0 + report = out_md.read_text(encoding="utf-8") + assert "# SANA-WM streaming benchmark" in report + assert "steady-state generation median / chunk | 2000.00 ms | 1000.00 ms" in report + + def test_benchmark_sweep_summary_writes_precision_chart_data(tmp_path: Path) -> None: module = _load_bench_sweep_summary() bf16_json = tmp_path / "bf16.json" @@ -443,3 +571,68 @@ def test_benchmark_sweep_summary_accepts_streaming_upstream_only(tmp_path: Path) assert payload["benchmark"]["metric"] == "steady_state_generation_ms_per_chunk" assert payload["benchmark"]["has_flashdreams"] is False assert [row["flashdreams"] for row in payload["rows"]] == [None, None] + + +def test_benchmark_sweep_summary_reports_streaming_comparison_values( + tmp_path: Path, +) -> None: + module = _load_bench_sweep_summary() + items = { + "BF16": (100.0, 80.0), + "FP8": (90.0, 95.0), + "FP4": (70.0, 35.0), + } + paths: list[tuple[str, Path]] = [] + for label, (official, flashdreams) in items.items(): + path = tmp_path / f"{label.lower()}.json" + path.write_text( + json.dumps( + { + "variant": "streaming", + "benchmark": { + "metric": "steady_state_generation_ms_per_chunk", + "unit": "ms", + "official": official, + "flashdreams": flashdreams, + }, + } + ), + encoding="utf-8", + ) + paths.append((label, path)) + out_json = tmp_path / "bench.json" + out_md = tmp_path / "bench.md" + out_chart = tmp_path / "perf.md" + + module.main( + [ + *[ + value + for label, path in paths + for value in ("--item", f"{label}:{path}") + ], + "--output-json", + str(out_json), + "--output-md", + str(out_md), + "--output-chart-md", + str(out_chart), + ] + ) + + assert out_chart.read_text(encoding="utf-8") == ( + "# SANA-WM Precision Sweep Summary (ms/chunk)\n" + "\n" + "| precision | official | flashdreams |\n" + "| --- | ---: | ---: |\n" + "| BF16 | 100.00 | 80.00 |\n" + "| FP8 | 90.00 | 95.00 |\n" + "| FP4 | 70.00 | 35.00 |\n" + ) + payload = json.loads(out_json.read_text(encoding="utf-8")) + assert [row["official"] for row in payload["rows"]] == [100.0, 90.0, 70.0] + assert [row["flashdreams"] for row in payload["rows"]] == [80.0, 95.0, 35.0] + report = out_md.read_text(encoding="utf-8") + assert "| precision | official | FlashDreams | source |" in report + assert "| BF16 | 100.00 ms | 80.00 ms |" in report + assert "| FP8 | 90.00 ms | 95.00 ms |" in report diff --git a/integrations/sana/tests/test_parity_quality_tools.py b/integrations/sana/tests/test_parity_quality_tools.py new file mode 100644 index 000000000..296857331 --- /dev/null +++ b/integrations/sana/tests/test_parity_quality_tools.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CPU-safe tests for SANA-WM parity quality tools.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from types import ModuleType + +import numpy as np +import pytest + +pytestmark = pytest.mark.ci_cpu + +TOOLS_DIR = Path("integrations/sana/tests/parity_check") + + +def _load_tool(module_name: str) -> ModuleType: + path = TOOLS_DIR / f"{module_name}.py" + spec = importlib.util.spec_from_file_location(module_name, path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.path.insert(0, str(TOOLS_DIR)) + try: + spec.loader.exec_module(module) + finally: + sys.path.remove(str(TOOLS_DIR)) + return module + + +def test_diff_parity_loads_upstream_streaming_npz(tmp_path: Path) -> None: + module = _load_tool("diff_parity") + frames = np.arange(2 * 3 * 4 * 3, dtype=np.uint8).reshape(2, 3, 4, 3) + path = tmp_path / "frames.npz" + np.savez_compressed(path, frames=frames, frame_indices=np.arange(2)) + + loaded = module._load_frames(path) + + assert loaded.dtype == np.uint8 + np.testing.assert_array_equal(loaded, frames) + + +def test_streaming_continuity_reports_boundary_spikes(tmp_path: Path) -> None: + module = _load_tool("streaming_continuity") + values = np.asarray([0, 1, 2, 20, 21, 22, 40], dtype=np.uint8) + frames = np.repeat(values[:, None, None, None], 3, axis=3) + + summary = module.summarize_frames(frames, chunk_size=3) + + assert summary["boundary_mean_abs_delta"] == 18.0 + assert summary["nonboundary_p95_abs_delta"] == 1.0 + assert summary["boundary_to_nonboundary_p95_ratio"] == 18.0 + assert summary["boundary_transitions"] == [ + {"from_frame": 2, "to_frame": 3, "mean_abs_delta": 18.0}, + {"from_frame": 5, "to_frame": 6, "mean_abs_delta": 18.0}, + ] + + +def test_streaming_continuity_cli_writes_comparison_json(tmp_path: Path) -> None: + module = _load_tool("streaming_continuity") + upstream_frames = np.zeros((4, 1, 1, 3), dtype=np.uint8) + flashdreams_frames = upstream_frames.copy() + flashdreams_frames[2:] = 10 + upstream = tmp_path / "upstream.npz" + flashdreams = tmp_path / "flashdreams.npy" + output = tmp_path / "continuity.json" + np.savez_compressed(upstream, frames=upstream_frames) + np.save(flashdreams, flashdreams_frames) + + module.main( + [ + "--upstream", + str(upstream), + "--flashdreams", + str(flashdreams), + "--chunk-size", + "2", + "--output", + str(output), + ] + ) + + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["comparison"]["shape_match"] is True + assert payload["upstream"]["boundary_mean_abs_delta"] == 0.0 + assert payload["flashdreams"]["boundary_mean_abs_delta"] == 10.0 From 48b6f8bc86184c05fa30194bbb5802d5b77603cf Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 30 Jul 2026 10:13:21 -0700 Subject: [PATCH 41/64] Wire up streaming variant to benchmark scripts --- integrations/sana/sana_wm/decoder.py | 203 +++++++++++++++++++++----- integrations/sana/sana_wm/refiner.py | 87 ++++++++++- integrations/sana/tests/test_smoke.py | 74 +++++++++- 3 files changed, 313 insertions(+), 51 deletions(-) diff --git a/integrations/sana/sana_wm/decoder.py b/integrations/sana/sana_wm/decoder.py index 9343e7c64..2b21e9190 100644 --- a/integrations/sana/sana_wm/decoder.py +++ b/integrations/sana/sana_wm/decoder.py @@ -530,8 +530,11 @@ class SanaWMStreamingVideoDecoderCache(SanaWMVideoDecoderCache): block_size: int = 3 refiner_kv_max_frames: int = SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES stage1_chunks: list[Tensor] = field(default_factory=list) - emitted_frames: int = 0 - emitted_stage1_frames: int = 0 + refined_chunks: list[Tensor] = field(default_factory=list) + stage1_sink: Tensor | None = None + refiner_prompt_embeds: Tensor | None = None + refiner_prompt_attention_mask: Tensor | None = None + refiner_generator: torch.Generator | None = None @dataclass(kw_only=True) @@ -557,30 +560,35 @@ class SanaWMStreamingLTX2LatentRefinerConfig(SanaWMLTX2LatentRefinerConfig): class SanaWMStreamingLTX2LatentRefiner(SanaWMLTX2LatentRefiner): - """Streaming refiner adapter using FlashDreams-owned prefix recomputation.""" + """Streaming refiner adapter using FlashDreams-owned chunk state.""" config: SanaWMStreamingLTX2LatentRefinerConfig @torch.inference_mode() - def refine_prefix( + def refine_chunk( self, *, - latents: Tensor, + context_latents: Tensor, + active_latents: Tensor, prompt: str, + prompt_embeds: Tensor | None, + prompt_attention_mask: Tensor | None, fps: int, - sink_size: int, - seed: int, - ) -> Tensor: - """Refine the currently visible latent prefix.""" - if latents.shape[2] <= sink_size: - return latents - return self.refine_latents( - latents=latents, - prompt=prompt, + generator: torch.Generator, + ) -> tuple[Tensor, Tensor, Tensor]: + """Refine one active chunk against clean context latents.""" + self._ensure_refiner() + if prompt_embeds is None or prompt_attention_mask is None: + prompt_embeds, prompt_attention_mask = self.refiner.encode_prompt(prompt) + refined = self.refiner.refine_active_latents( + context_latents=context_latents, + active_latents=active_latents, + prompt_embeds=prompt_embeds, + prompt_attention_mask=prompt_attention_mask, fps=fps, - sink_size=sink_size, - seed=seed, + generator=generator, ) + return refined, prompt_embeds, prompt_attention_mask @dataclass(kw_only=True) @@ -625,18 +633,38 @@ def forward( cache: SanaWMStreamingVideoDecoderCache | None = None, ) -> SanaWMDecodedVideo: """Refine/decode one SANA-WM streaming latent chunk.""" - del autoregressive_index cache = cache or SanaWMStreamingVideoDecoderCache() - cache.stage1_chunks.append(input) - stage1_prefix = torch.cat(cache.stage1_chunks, dim=2) - output_prefix = stage1_prefix + stage1_active = _split_streaming_stage1_chunk( + input, + autoregressive_index=autoregressive_index, + cache=cache, + ) + refiner_context = _latent_context( + sink=cache.stage1_sink, + chunks=cache.refined_chunks, + max_frames=_max_streaming_context_frames( + cache, + active_frames=int(stage1_active.shape[2]), + ), + ) + output_active = stage1_active if self.refiner is not None: - output_prefix = self.refiner.refine_prefix( - latents=stage1_prefix, + if cache.refiner_generator is None: + cache.refiner_generator = torch.Generator( + device=stage1_active.device + ).manual_seed(int(cache.refiner_seed)) + ( + output_active, + cache.refiner_prompt_embeds, + cache.refiner_prompt_attention_mask, + ) = self.refiner.refine_chunk( + context_latents=refiner_context, + active_latents=stage1_active, prompt=cache.prompt, + prompt_embeds=cache.refiner_prompt_embeds, + prompt_attention_mask=cache.refiner_prompt_attention_mask, fps=cache.fps, - sink_size=cache.sink_size, - seed=cache.refiner_seed, + generator=cache.refiner_generator, ) elif cache.save_stage1: logger.info( @@ -644,18 +672,30 @@ def forward( "--save-stage1 does not create an extra output." ) - video_hwc, cache.emitted_frames = _decode_new_frames( + video_hwc = _decode_streaming_active_frames( self.vae_decoder, - output_prefix, - emitted_frames=cache.emitted_frames, + context_latents=refiner_context, + active_latents=output_active, + temporal_compression_ratio=self.temporal_compression_ratio, ) + cache.refined_chunks.append(output_active.detach().contiguous()) stage1_video_hwc = None if cache.save_stage1 and self.refiner is not None: - stage1_video_hwc, cache.emitted_stage1_frames = _decode_new_frames( + stage1_context = _latent_context( + sink=cache.stage1_sink, + chunks=cache.stage1_chunks, + max_frames=_max_streaming_context_frames( + cache, + active_frames=int(stage1_active.shape[2]), + ), + ) + stage1_video_hwc = _decode_streaming_active_frames( self.vae_decoder, - stage1_prefix, - emitted_frames=cache.emitted_stage1_frames, + context_latents=stage1_context, + active_latents=stage1_active, + temporal_compression_ratio=self.temporal_compression_ratio, ) + cache.stage1_chunks.append(stage1_active.detach().contiguous()) return SanaWMDecodedVideo( video_hwc=video_hwc, stage1_video_hwc=stage1_video_hwc, @@ -704,18 +744,101 @@ def get_input_temporal_size( return latent_frames + 1 if autoregressive_index == 0 else latent_frames -def _decode_new_frames( +def _split_streaming_stage1_chunk( + chunk: Tensor, + *, + autoregressive_index: int, + cache: SanaWMStreamingVideoDecoderCache, +) -> Tensor: + """Return active latent frames and initialize the sink on the first chunk.""" + if autoregressive_index < 0: + raise ValueError(f"autoregressive_index must be >= 0, got {autoregressive_index}.") + if chunk.shape[2] <= 0: + raise ValueError("SANA-WM streaming latent chunk must contain frames.") + if autoregressive_index == 0: + if cache.stage1_sink is not None: + raise RuntimeError("SANA-WM streaming sink was already initialized.") + if chunk.shape[2] <= cache.sink_size: + raise ValueError("Streaming AR step 0 must include sink + active frames.") + cache.stage1_sink = chunk[:, :, : cache.sink_size].detach().contiguous() + return chunk[:, :, cache.sink_size :].contiguous() + if cache.stage1_sink is None: + raise RuntimeError("SANA-WM streaming AR step 0 must run before later chunks.") + return chunk.contiguous() + + +def _max_streaming_context_frames( + cache: SanaWMStreamingVideoDecoderCache, + *, + active_frames: int, +) -> int: + """Return sink plus rolling-history context length for one active block.""" + if active_frames <= 0: + raise ValueError(f"active_frames must be positive, got {active_frames}.") + return max(int(cache.sink_size), int(cache.refiner_kv_max_frames) - active_frames) + + +def _latent_context( + *, + sink: Tensor | None, + chunks: list[Tensor], + max_frames: int, +) -> Tensor: + """Build clean sink-plus-history context capped to ``max_frames`` latents.""" + if sink is None: + raise RuntimeError("SANA-WM streaming sink has not been initialized.") + if max_frames <= sink.shape[2]: + return sink + history_budget = max_frames - int(sink.shape[2]) + history = _tail_latent_frames(chunks, history_budget) + if history is None: + return sink + return torch.cat([sink, history], dim=2) + + +def _tail_latent_frames(chunks: list[Tensor], max_frames: int) -> Tensor | None: + """Return the last ``max_frames`` frames from a latent chunk list.""" + if max_frames <= 0 or not chunks: + return None + selected: list[Tensor] = [] + remaining = int(max_frames) + for chunk in reversed(chunks): + if remaining <= 0: + break + take = min(int(chunk.shape[2]), remaining) + selected.append(chunk[:, :, -take:]) + remaining -= take + if not selected: + return None + return torch.cat(list(reversed(selected)), dim=2).contiguous() + + +def _decode_streaming_active_frames( vae_decoder: SanaWMLTX2VAEDecoder, - latent_prefix: Tensor, *, - emitted_frames: int, -) -> tuple[np.ndarray, int]: - """Decode a prefix and return only frames not emitted by prior chunks.""" - video = vae_decoder.decode_latents(latent_prefix) - start = emitted_frames - if start == 0 and video.shape[0] > 0: - start = 1 - return video[start:], int(video.shape[0]) + context_latents: Tensor, + active_latents: Tensor, + temporal_compression_ratio: int, +) -> np.ndarray: + """Decode active frames with latent context and drop context pixels.""" + decode_latents = torch.cat([context_latents, active_latents], dim=2) + video = vae_decoder.decode_latents(decode_latents) + start = _pixel_frames_for_latents( + int(context_latents.shape[2]), + temporal_compression_ratio=temporal_compression_ratio, + ) + return video[start:] + + +def _pixel_frames_for_latents( + latent_frames: int, + *, + temporal_compression_ratio: int, +) -> int: + """Return decoded pixel frames for a non-empty latent prefix.""" + if latent_frames <= 0: + return 0 + return 1 + (latent_frames - 1) * int(temporal_compression_ratio) __all__ = [ diff --git a/integrations/sana/sana_wm/refiner.py b/integrations/sana/sana_wm/refiner.py index 8a1246281..dae92c097 100644 --- a/integrations/sana/sana_wm/refiner.py +++ b/integrations/sana/sana_wm/refiner.py @@ -159,12 +159,8 @@ def refine_latents( f"sink_size={sink_size}." ) - prompt_embeds, prompt_attention_mask = self._encode_prompt(prompt) - - _move_ltx2_video_modules_to(self.transformer, self.device) - _offload_video_unused_audio_modules(self.transformer, "cpu") - self.transformer.eval() - self._prepare_quantization() + prompt_embeds, prompt_attention_mask = self.encode_prompt(prompt) + self._prepare_video_runtime() z = sana_latent.to(device=self.device, dtype=self.dtype) sigmas_t = torch.tensor(sigmas, dtype=torch.float32, device=self.device) @@ -217,6 +213,85 @@ def refine_latents( return torch.cat([sink, noisy], dim=2) + @torch.inference_mode() + def encode_prompt(self, prompt: str) -> tuple[Tensor, Tensor]: + """Encode a prompt for one or more LTX-2 refinement calls.""" + return self._encode_prompt(prompt) + + @torch.inference_mode() + def refine_active_latents( + self, + *, + context_latents: Tensor, + active_latents: Tensor, + prompt_embeds: Tensor, + prompt_attention_mask: Tensor, + fps: float, + generator: torch.Generator, + sigmas: tuple[float, ...] = (0.909375, 0.725, 0.421875, 0.0), + ) -> Tensor: + """Refine one active latent block against frozen context latents. + + This is the FlashDreams-owned streaming refinement primitive. The + context is treated as clean, fixed latent history; only ``active_latents`` + are noised and updated by the deterministic Euler steps. + """ + if context_latents.shape[2] <= 0: + raise ValueError("context_latents must contain at least one frame.") + if active_latents.shape[2] <= 0: + raise ValueError("active_latents must contain at least one frame.") + + self._prepare_video_runtime() + context = context_latents.to(device=self.device, dtype=self.dtype).contiguous() + active = active_latents.to(device=self.device, dtype=self.dtype).contiguous() + sigmas_t = torch.tensor(sigmas, dtype=torch.float32, device=self.device) + start_sigma = float(sigmas_t[0]) + eps = torch.randn( + active.shape, + generator=generator, + device=self.device, + dtype=self.dtype, + ) + noisy = (1.0 - start_sigma) * active + start_sigma * eps + + for step_index in range(len(sigmas_t) - 1): + sigma = sigmas_t[step_index] + denoised = self._predict_current_x0( + sink=context, + noisy_current=noisy, + prompt_embeds=prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + sigma=sigma, + fps=fps, + ) + noisy_tokens = _pack_latents( + noisy, + patch_size=self.transformer.config.patch_size, + patch_size_t=self.transformer.config.patch_size_t, + ) + velocity = (noisy_tokens.float() - denoised.float()) / sigma.float() + next_tokens = ( + noisy_tokens.float() + + velocity * (sigmas_t[step_index + 1] - sigma).float() + ) + noisy = _unpack_latents( + next_tokens.to(self.dtype), + num_frames=noisy.shape[2], + height=noisy.shape[3], + width=noisy.shape[4], + patch_size=self.transformer.config.patch_size, + patch_size_t=self.transformer.config.patch_size_t, + ) + + return noisy + + def _prepare_video_runtime(self) -> None: + """Prepare video-only refiner modules for latent refinement.""" + _move_ltx2_video_modules_to(self.transformer, self.device) + _offload_video_unused_audio_modules(self.transformer, "cpu") + self.transformer.eval() + self._prepare_quantization() + def _load_diffusers_components(self) -> tuple[nn.Module, nn.Module]: from diffusers.models.transformers.transformer_ltx2 import ( LTX2VideoTransformer3DModel, diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index df9e569c2..d8f754866 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -1063,15 +1063,15 @@ def test_video_decoder_returns_structured_video(monkeypatch: pytest.MonkeyPatch) def test_streaming_video_decoder_emits_only_new_frames( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Decode prefixes while returning only frames produced by the new chunk.""" + """Decode chunks with context while returning only the active frames.""" decoder = SanaWMStreamingVideoDecoderConfig(refiner=None).setup() - def decode_prefix(latents: torch.Tensor) -> np.ndarray: + def decode_with_context(latents: torch.Tensor) -> np.ndarray: frames = 1 + (latents.shape[2] - 1) * 8 values = np.arange(frames, dtype=np.uint8) return np.broadcast_to(values[:, None, None, None], (frames, 2, 2, 3)).copy() - monkeypatch.setattr(decoder.vae_decoder, "decode_latents", decode_prefix) + monkeypatch.setattr(decoder.vae_decoder, "decode_latents", decode_with_context) cache = decoder.initialize_autoregressive_cache() first = decoder( @@ -1089,13 +1089,77 @@ def decode_prefix(latents: torch.Tensor) -> np.ndarray: assert second.video_hwc.shape == (24, 2, 2, 3) assert first.video_hwc[0, 0, 0, 0] == 1 assert second.video_hwc[0, 0, 0, 0] == 25 - assert cache.emitted_frames == 49 - assert cache.stage1_chunks[0].shape[2] == 4 + assert cache.stage1_sink is not None + assert cache.stage1_sink.shape[2] == 1 + assert cache.stage1_chunks[0].shape[2] == 3 assert cache.stage1_chunks[1].shape[2] == 3 + assert cache.refined_chunks[0].shape[2] == 3 + assert cache.refined_chunks[1].shape[2] == 3 assert first.stage1_video_hwc is None assert second.stage1_video_hwc is None +def test_streaming_video_decoder_refines_against_rolling_history( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Use prior refined chunks as clean context for later streaming chunks.""" + decoder = SanaWMStreamingVideoDecoderConfig(refiner=None).setup() + calls: list[torch.Tensor] = [] + + class DummyRefiner: + def refine_chunk( + self, + *, + context_latents: torch.Tensor, + active_latents: torch.Tensor, + prompt: str, + prompt_embeds: torch.Tensor | None, + prompt_attention_mask: torch.Tensor | None, + fps: int, + generator: torch.Generator, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + del prompt, prompt_embeds, prompt_attention_mask, fps, generator + calls.append(context_latents.clone()) + return active_latents + 1, torch.ones((1, 1, 1)), torch.ones((1, 1)) + + def decode_with_context(latents: torch.Tensor) -> np.ndarray: + frames = 1 + (latents.shape[2] - 1) * 8 + return np.zeros((frames, 2, 2, 3), dtype=np.uint8) + + decoder.refiner = DummyRefiner() + monkeypatch.setattr(decoder.vae_decoder, "decode_latents", decode_with_context) + cache = decoder.initialize_autoregressive_cache( + prompt="demo", + refiner_kv_max_frames=11, + ) + + decoder(torch.zeros((1, 1, 4, 1, 1)), autoregressive_index=0, cache=cache) + decoder(torch.full((1, 1, 3, 1, 1), 2.0), autoregressive_index=1, cache=cache) + decoder(torch.full((1, 1, 3, 1, 1), 4.0), autoregressive_index=2, cache=cache) + decoder(torch.full((1, 1, 3, 1, 1), 6.0), autoregressive_index=3, cache=cache) + + assert [call.shape[2] for call in calls] == [1, 4, 7, 8] + torch.testing.assert_close(calls[0], torch.zeros((1, 1, 1, 1, 1))) + torch.testing.assert_close( + calls[1], + torch.cat( + [ + torch.zeros((1, 1, 1, 1, 1)), + torch.ones((1, 1, 3, 1, 1)), + ], + dim=2, + ), + ) + torch.testing.assert_close( + calls[3], + torch.tensor([0.0, 1.0, 3.0, 3.0, 3.0, 5.0, 5.0, 5.0]).reshape( + 1, 1, 8, 1, 1 + ), + ) + assert cache.refiner_prompt_embeds is not None + assert cache.refiner_prompt_attention_mask is not None + + def test_stage1_model_matches_checkpoint_schema() -> None: """Pin the Stage-1 module to the public checkpoint schema.""" state = SanaWMStage1Model().state_dict() From 1d4fdd857061d8299c96b963f64c2bf95799c07f Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 30 Jul 2026 11:01:12 -0700 Subject: [PATCH 42/64] Tighten scripts, split streaming and bidirectional model cards --- .../performance/sana_wm/perf-bf16-0727.md | 5 - .../sana_wm_bidirectional/perf-bf16-gb202.md | 5 + .../sana_wm_streaming/perf-bf16-gb202.md | 5 + .../sana_wm_streaming/perf-fp4-gb202.md | 5 + .../sana_wm_streaming/perf-fp8-gb202.md | 5 + docs/source/index.rst | 11 +- docs/source/models/index.rst | 15 +- ...{sana_wm.rst => sana_wm_bidirectional.rst} | 83 ++++---- docs/source/models/sana_wm_streaming.rst | 200 ++++++++++++++++++ integrations/sana/README.md | 40 ++-- .../sana/tests/parity_check/README.md | 53 +++-- integrations/sana/tests/parity_check/bench.sh | 29 +++ .../sana/tests/parity_check/bench_summary.py | 16 +- .../tests/parity_check/bench_sweep_summary.py | 21 ++ .../tests/test_parity_benchmark_summary.py | 127 +++++++++-- 15 files changed, 505 insertions(+), 115 deletions(-) delete mode 100644 docs/source/_static/performance/sana_wm/perf-bf16-0727.md create mode 100644 docs/source/_static/performance/sana_wm_bidirectional/perf-bf16-gb202.md create mode 100644 docs/source/_static/performance/sana_wm_streaming/perf-bf16-gb202.md create mode 100644 docs/source/_static/performance/sana_wm_streaming/perf-fp4-gb202.md create mode 100644 docs/source/_static/performance/sana_wm_streaming/perf-fp8-gb202.md rename docs/source/models/{sana_wm.rst => sana_wm_bidirectional.rst} (59%) create mode 100644 docs/source/models/sana_wm_streaming.rst diff --git a/docs/source/_static/performance/sana_wm/perf-bf16-0727.md b/docs/source/_static/performance/sana_wm/perf-bf16-0727.md deleted file mode 100644 index 64ba5dca3..000000000 --- a/docs/source/_static/performance/sana_wm/perf-bf16-0727.md +++ /dev/null @@ -1,5 +0,0 @@ -# SANA-WM BF16 Benchmark Data (ms) - -| device | official | flashdreams | -| --- | ---: | ---: | -| GB300 | 45823.92 | 42399.31 | diff --git a/docs/source/_static/performance/sana_wm_bidirectional/perf-bf16-gb202.md b/docs/source/_static/performance/sana_wm_bidirectional/perf-bf16-gb202.md new file mode 100644 index 000000000..ba71900dc --- /dev/null +++ b/docs/source/_static/performance/sana_wm_bidirectional/perf-bf16-gb202.md @@ -0,0 +1,5 @@ +# SANA-WM_bidirectional BF16 Benchmark Data (ms) + +| device | official | flashdreams | +| --- | ---: | ---: | +| GB202 | 107868.83 | 98432.36 | diff --git a/docs/source/_static/performance/sana_wm_streaming/perf-bf16-gb202.md b/docs/source/_static/performance/sana_wm_streaming/perf-bf16-gb202.md new file mode 100644 index 000000000..0b6442046 --- /dev/null +++ b/docs/source/_static/performance/sana_wm_streaming/perf-bf16-gb202.md @@ -0,0 +1,5 @@ +# SANA-WM_streaming BF16 Benchmark Data (ms/chunk) + +| device | official | flashdreams | +| --- | ---: | ---: | +| GB202 | 1846.87 | 7084.36 | diff --git a/docs/source/_static/performance/sana_wm_streaming/perf-fp4-gb202.md b/docs/source/_static/performance/sana_wm_streaming/perf-fp4-gb202.md new file mode 100644 index 000000000..f3d1e5643 --- /dev/null +++ b/docs/source/_static/performance/sana_wm_streaming/perf-fp4-gb202.md @@ -0,0 +1,5 @@ +# SANA-WM_streaming FP4 Benchmark Data (ms/chunk) + +| device | official | flashdreams | +| --- | ---: | ---: | +| GB202 | 1420.96 | 16182.13 | diff --git a/docs/source/_static/performance/sana_wm_streaming/perf-fp8-gb202.md b/docs/source/_static/performance/sana_wm_streaming/perf-fp8-gb202.md new file mode 100644 index 000000000..8889c97f3 --- /dev/null +++ b/docs/source/_static/performance/sana_wm_streaming/perf-fp8-gb202.md @@ -0,0 +1,5 @@ +# SANA-WM_streaming FP8 Benchmark Data (ms/chunk) + +| device | official | flashdreams | +| --- | ---: | ---: | +| GB202 | 1635.46 | 8964.06 | diff --git a/docs/source/index.rst b/docs/source/index.rst index 49971cc80..ac7cf9267 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -239,6 +239,13 @@ invocation, the checkpoint source, and the per-implementation knobs. Camera-controllable image-to-video world model. + .. grid-item-card:: SANA-WM_streaming + :class-card: fd-feature + :link: models/sana_wm_streaming + :link-type: doc + + Chunk-causal camera-controlled world model. + .. grid-item-card:: FlashVSR :class-card: fd-feature :link: models/flashvsr @@ -261,9 +268,9 @@ invocation, the checkpoint source, and the per-implementation knobs. Bidirectional Cosmos-Predict2 reference implementations (T2V / I2V, 2B). - .. grid-item-card:: SANA-WM (bidirectional) + .. grid-item-card:: SANA-WM_bidirectional :class-card: fd-feature - :link: models/sana_wm + :link: models/sana_wm_bidirectional :link-type: doc Bidirectional camera-controlled world model (Stage-1 DiT + LTX-2 diff --git a/docs/source/models/index.rst b/docs/source/models/index.rst index 3a68a014a..08bd72901 100644 --- a/docs/source/models/index.rst +++ b/docs/source/models/index.rst @@ -28,7 +28,8 @@ Models flashvsr hy_worldplay lingbot_world - sana_wm + sana_wm_streaming + sana_wm_bidirectional wan21 FlashDreams runs a growing family of world and video models (text-to-video, @@ -139,6 +140,14 @@ uses, and the settings you can tune. Action- and camera-controllable image-to-video world model. + .. grid-item-card:: SANA-WM_streaming + :class-card: fd-feature + :link: /models/sana_wm_streaming + :link-type: doc + + Chunk-causal camera-controlled world model with streaming Stage-1, + refiner, and VAE paths. + .. container:: fd-eyebrow Bidirectional Video Generation @@ -173,9 +182,9 @@ uses, and the settings you can tune. Bidirectional Cosmos-Predict2 reference implementations (T2V / I2V, 2B). - .. grid-item-card:: SANA-WM + .. grid-item-card:: SANA-WM_bidirectional :class-card: fd-feature - :link: /models/sana_wm + :link: /models/sana_wm_bidirectional :link-type: doc Bidirectional camera-controlled world model (Stage-1 DiT + LTX-2 diff --git a/docs/source/models/sana_wm.rst b/docs/source/models/sana_wm_bidirectional.rst similarity index 59% rename from docs/source/models/sana_wm.rst rename to docs/source/models/sana_wm_bidirectional.rst index a25299bda..70c49a46d 100644 --- a/docs/source/models/sana_wm.rst +++ b/docs/source/models/sana_wm_bidirectional.rst @@ -13,7 +13,7 @@ .. See the License for the specific language governing permissions and .. limitations under the License. -SANA-WM +SANA-WM_bidirectional =================================== .. container:: fd-cta-row @@ -31,27 +31,30 @@ SANA-WM .. button-link:: https://huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional :color: primary - Model page + Checkpoint .. button-link:: https://github.com/NVlabs/Sana :color: primary Official code -SANA-WM is a 2.6B bidirectional, camera-controlled world model from the -`NVlabs/Sana `_ family. Given a first frame, a -text prompt, and a camera trajectory it renders a video clip in a single -bidirectional pass. FlashDreams runs it through the ``sana-wm-bidirectional`` -runner, which pairs a native Stage-1 DiT (loading the public SANA-WM checkpoint -directly) with an LTX-2 refiner for the final decode. +``SANA-WM_bidirectional`` is the full-sequence, camera-controlled +`NVlabs/Sana `_ world model release. Given a +first frame, a text prompt, and a camera trajectory, it renders a video clip in +a single bidirectional pass. FlashDreams runs it through the +``sana-wm-bidirectional`` runner with a native Stage-1 DiT and an LTX-2 refiner. + +The sibling streaming release has a separate model card: +:doc:`sana_wm_streaming`. Requirements ------------ - **PyTorch**: >= 2.9. -- **Precision**: BF16 by default. FP8 Stage-1/refiner inference is available on - Hopper or newer GPUs (``sm_90+``) and FP4 on Blackwell (``sm_100+``); both - lower the memory footprint relative to the BF16 default. +- **Precision**: BF16 by default. The FlashDreams runner also exposes opt-in + FP8 and FP4 execution paths, but the upstream-vs-FlashDreams benchmark for + ``SANA-WM_bidirectional`` is BF16-only because upstream + ``SANA-WM_bidirectional`` does not support those precision flags. Installation ------------ @@ -64,8 +67,8 @@ Installation Running the method ------------------ -To run SANA-WM, launch the ``sana-wm-bidirectional`` runner with a first-frame -image, a prompt, and a camera trajectory: +Launch the ``sana-wm-bidirectional`` runner with a first-frame image, a prompt, +and a camera trajectory: .. code-block:: bash @@ -75,20 +78,19 @@ image, a prompt, and a camera trajectory: --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ --num-frames 161 \ - --output-dir outputs/sana_wm_bf16 + --output-dir outputs/sana_wm_bidirectional_bf16 -The demo image, prompt, camera, and intrinsics files ship with the SANA-WM -release; any inputs with matching shapes work as well. +The demo image, prompt, camera, and intrinsics files ship with the +``SANA-WM_bidirectional`` release; equivalent inputs with matching shapes work +as well. Optional inputs and knobs ~~~~~~~~~~~~~~~~~~~~~~~~~~~ - ``--intrinsics-path`` is optional. When omitted, intrinsics are derived from the first-frame size, assuming a centered principal point and a horizontal - field of view of ``--intrinsics-hfov-deg`` (default ``90``, matching the demo - intrinsics). -- ``--camera-path`` can be replaced by an ``--action`` DSL string, so a minimal - run needs only an image and a prompt: + field of view of ``--intrinsics-hfov-deg``. +- ``--camera-path`` can be replaced by an ``--action`` DSL string: .. code-block:: bash @@ -100,9 +102,6 @@ Optional inputs and knobs --output-dir outputs/mine - ``--no-refiner True`` runs Stage-1 only, for diagnostics. -- Quantized Stage-1/refiner inference is opt-in via ``--stage1-precision`` / - ``--refiner-precision`` (``fp8`` on ``sm_90+``, ``fp4`` on ``sm_100+``), with - ``--quant-backend`` (``torch-fp8`` / ``torch-fp4``) to force a backend. To inspect all supported CLI arguments and their default values, run: @@ -114,43 +113,37 @@ What to expect -------------- - **Model checkpoint**: pulled from - ``huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional`` on first run and - cached under ``$HF_HOME``. The LTX-2 refiner and VAE weights are fetched from - their respective ``diffusers`` repositories on first use. -- **First launch**: a few minutes for the download and warmup; subsequent - launches reuse the caches. -- **Outputs**: ``outputs//sana_wm_generated.mp4``. - -See :doc:`/developer_guides/inference_pipeline_overview` for what one pass does -end-to-end. + ``huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional`` on first run. +- **First launch**: a few minutes for download and warmup; subsequent launches + reuse local caches. +- **Outputs**: ``outputs//sana-wm-bidirectional.mp4``. Profiling benchmark ------------------- -Here is the BF16 profiling benchmark on post-load generation latency per -generated clip for FlashDreams SANA-WM compared to the -`official SANA-WM implementation `_ under -matched settings. On GB300, FlashDreams is about 7% faster than the official -implementation at median clip latency (full pipeline: Stage-1 DiT + LTX-2 -refiner, 10 measured runs). +The BF16 chart below compares post-load generation latency per generated clip +for FlashDreams ``SANA-WM_bidirectional`` and the official +``SANA-WM_bidirectional`` implementation under matched settings. On GB202, +FlashDreams measured 98,432.36 ms per clip versus 107,868.83 ms for the +official implementation. .. raw:: html

- This chart shows post-load generation latency per generated clip in milliseconds for a 121-frame - full-pipeline BF16 run (Stage-1 DiT + LTX-2 refiner). The measured row used an NVIDIA GB300. + This chart shows post-load generation latency per generated clip in milliseconds for a + 121-frame full-pipeline BF16 run (Stage-1 DiT + LTX-2 refiner + SANA VAE decode). + The measured row used one NVIDIA GB202 GPU, one discarded warmup run, and three measured runs. Model construction, checkpoint loading, video writing, and frame dumps are outside the timing boundary. - For the official SANA-WM implementation, see - this instruction. + The upstream checkout was pinned to commit 6298508.

diff --git a/docs/source/models/sana_wm_streaming.rst b/docs/source/models/sana_wm_streaming.rst new file mode 100644 index 000000000..e1c2fa97c --- /dev/null +++ b/docs/source/models/sana_wm_streaming.rst @@ -0,0 +1,200 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 +.. +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. + +SANA-WM_streaming +=================================== + +.. container:: fd-cta-row + + .. button-link:: https://nvlabs.github.io/Sana/ + :color: primary + + Project page + + .. button-link:: https://arxiv.org/abs/2410.10629 + :color: primary + + arXiv paper + + .. button-link:: https://huggingface.co/Efficient-Large-Model/SANA-WM_streaming + :color: primary + + Checkpoint + + .. button-link:: https://github.com/NVlabs/Sana + :color: primary + + Official code + +``SANA-WM_streaming`` is the chunk-causal, camera-controlled +`NVlabs/Sana `_ world model release. It +produces video progressively across autoregressive chunks with a chunk-causal +Stage-1 DiT, streaming LTX-2 refiner, and streaming VAE decode path. +FlashDreams runs it through the ``sana-wm-streaming`` runner. + +The sibling full-sequence release has a separate model card: +:doc:`sana_wm_bidirectional`. + +Requirements +------------ + +- **PyTorch**: >= 2.9. +- **Precision**: BF16 by default. FP8 Stage-1/refiner inference is available on + Hopper or newer GPUs (``sm_90+``), and FP4 is available on Blackwell + (``sm_100+``). These upstream precision flags belong to + ``SANA-WM_streaming``. + +Installation +------------ + +.. code-block:: bash + + # from the repo root + uv sync --package flashdreams-sana-wm --extra dev + +Running the method +------------------ + +Launch the ``sana-wm-streaming`` runner with a first-frame image, a prompt, and +a camera trajectory: + +.. code-block:: bash + + uv run flashdreams-run sana-wm-streaming \ + --image-path ../Sana/asset/sana_wm/demo_0.png \ + --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ + --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ + --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --num-frames 241 \ + --output-dir outputs/sana_wm_streaming_bf16 + +The runner defaults to 3 latent frames per block and the distilled schedule +``1000,960,889,727,0``. Requested frame counts are snapped to +``8 * --num-frame-per-block * k + 1`` before inference. + +Optional inputs and knobs +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``--intrinsics-path`` is optional. When omitted, intrinsics are derived from + the first-frame size. +- ``--camera-path`` can be replaced by an ``--action`` DSL string: + + .. code-block:: bash + + uv run flashdreams-run sana-wm-streaming \ + --image-path my_frame.png \ + --prompt "a scene description; describe the world's own motion" \ + --action "w-80,dw-40,w-80,aw-40" \ + --num-frames 241 \ + --output-dir outputs/mine_streaming + +- ``--stage1-precision`` and ``--refiner-precision`` accept ``bf16``, ``fp8``, + or ``fp4`` when the selected hardware supports the requested precision. + +To inspect all supported CLI arguments and their default values, run: + +.. code-block:: bash + + uv run flashdreams-run sana-wm-streaming --help + +What to expect +-------------- + +- **Model checkpoint**: pulled from + ``huggingface.co/Efficient-Large-Model/SANA-WM_streaming`` on first run. +- **First launch**: a few minutes for download and warmup; subsequent launches + reuse local caches. +- **Outputs**: ``outputs//sana-wm-streaming.mp4``. + +Profiling benchmark +------------------- + +The charts below compare steady-state generation latency per produced chunk for +FlashDreams ``SANA-WM_streaming`` and the official ``SANA-WM_streaming`` +implementation under matched settings. Warmup runs and the first decoded chunk +are excluded from the headline metric; full-clip wall time remains supporting +benchmark data. These GB202 latency runs show the official implementation +faster than FlashDreams for BF16, FP8, and FP4. + +.. raw:: html + +
+
+
+

+ BF16 steady-state milliseconds per produced chunk on one NVIDIA GB202 GPU: + official 1,846.87 ms, FlashDreams 7,084.36 ms. +

+
+
+ +
+
+
+

+ FP8 steady-state milliseconds per produced chunk on one NVIDIA GB202 GPU: + official 1,635.46 ms, FlashDreams 8,964.06 ms. +

+
+
+ +
+
+
+

+ FP4 steady-state milliseconds per produced chunk on one NVIDIA GB202 GPU: + official 1,420.96 ms, FlashDreams 16,182.13 ms. +

+
+
+ + +All three rows use the same demo image/prompt, ``w-80,dw-40,w-80,aw-40`` +action path, 241 requested frames, one discarded warmup run, and three measured +runs. The upstream checkout was pinned to commit 6298508. + +Citation +-------- + +If you use SANA-WM, please cite the original SANA work: + +.. code-block:: bibtex + + @misc{xie2024sana, + title={SANA: Efficient High-Resolution Image Synthesis with Linear Diffusion Transformers}, + author={Enze Xie and Junsong Chen and Junyu Chen and Han Cai and Haotian Tang and Yujun Lin and Zhekai Zhang and Muyang Li and Ligeng Zhu and Yao Lu and Song Han}, + year={2024}, + eprint={2410.10629}, + archivePrefix={arXiv}, + primaryClass={cs.CV} + } diff --git a/integrations/sana/README.md b/integrations/sana/README.md index 335b44460..1a1dbef32 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -6,9 +6,9 @@ SPDX-License-Identifier: Apache-2.0 # `sana_wm` FlashDreams SANA-WM integration for the -[SANA-WM bidirectional](https://huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional) +[SANA-WM_bidirectional](https://huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional) and -[SANA-WM streaming](https://huggingface.co/Efficient-Large-Model/SANA-WM_streaming) +[SANA-WM_streaming](https://huggingface.co/Efficient-Large-Model/SANA-WM_streaming) camera-controlled world model releases from NVlabs/Sana. The `sana-wm-bidirectional` and `sana-wm-streaming` runners use FlashDreams @@ -34,8 +34,8 @@ Current scope: | slug | description | | --- | --- | -| `sana-wm-bidirectional` | SANA-WM Stage-1 + LTX-2 refiner runner. | -| `sana-wm-streaming` | SANA-WM streaming Stage-1 + streaming LTX-2 refiner/VAE runner. | +| `sana-wm-bidirectional` | `SANA-WM_bidirectional` Stage-1 + LTX-2 refiner runner. | +| `sana-wm-streaming` | `SANA-WM_streaming` Stage-1 + streaming LTX-2 refiner/VAE runner. | The FlashDreams package is named `sana_wm`. @@ -174,24 +174,30 @@ pattern used by the other benchmarked FlashDreams integrations. ```bash cd integrations/sana/tests/parity_check -# Upstream + FlashDreams parity artifacts and frame diff. +# Upstream + FlashDreams SANA-WM_bidirectional parity artifacts and frame diff. bash run.sh -# Matched upstream-vs-FlashDreams benchmark report. +# Matched SANA-WM_bidirectional upstream-vs-FlashDreams BF16 benchmark report. DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh + +# Matched SANA-WM_streaming upstream-vs-FlashDreams benchmark report. +SANA_WM_VARIANT=streaming BENCH_SIDE=both DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh + +# SANA-WM_streaming BF16/FP8/FP4 precision sweep. +SANA_WM_VARIANT=streaming BENCH_SIDE=both BENCH_PRECISIONS=bf16,fp8,fp4 \ + DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh ``` -`bench.sh` writes `outputs/bench/bench.md` for review and `outputs/bench/perf.md` -for chart-ready data. Both use the same benchmark metric: post-load generation -latency per generated clip. SANA-WM renders each requested bidirectional clip in -one generation pass, rather than as independently timed frames. Stage-1 DiT, -conditioning/encode, VAE decode, optional refiner, memory, and frame-normalized -diagnostic rows are reported as breakdowns of that metric. - -For precision sweeps, copy the per-precision chart data from -`outputs/bench//perf.md` into the docs. Each model-card chart should -cover one precision and use GPU/device as the differentiating row, matching the -other FlashDreams model-card benchmark charts. +For `SANA_WM_VARIANT=bidirectional`, `bench.sh` writes `outputs/bench/bench.md` +for review and `outputs/bench/perf.md` for chart-ready data. The metric is +post-load generation latency per generated clip. The upstream comparison is +BF16-only because upstream `SANA-WM_bidirectional` does not support FP8/FP4 +precision flags. + +For `SANA_WM_VARIANT=streaming`, benchmark outputs are written under +`outputs/bench/streaming//`. The metric is steady-state generation +latency per produced chunk. BF16, FP8, and FP4 comparison sweeps belong to +`SANA-WM_streaming`. ## Tests diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index b3203f808..0476d36c9 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -3,13 +3,14 @@ SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# SANA-WM parity and benchmark harness +# SANA-WM_bidirectional and SANA-WM_streaming parity and benchmark harness This harness compares upstream -[`NVlabs/Sana`](https://github.com/NVlabs/Sana) SANA-WM against the in-tree -FlashDreams integrations on matched demo inputs, seeds, resolution, and -precision settings. `run.sh` dispatches to separate bidirectional and streaming -parity runners. `bench.sh` supports bidirectional by default and streaming when +[`NVlabs/Sana`](https://github.com/NVlabs/Sana) `SANA-WM_bidirectional` and +`SANA-WM_streaming` against the matching in-tree FlashDreams integrations on +matched demo inputs, seeds, resolution, and precision settings. `run.sh` +dispatches to separate bidirectional and streaming parity runners. `bench.sh` +supports bidirectional by default and streaming when `SANA_WM_VARIANT=streaming` is set. All dependencies live in this directory's isolated `./.venv`. `run.sh` and @@ -65,7 +66,7 @@ Streaming parity writes to `outputs/parity/streaming/` by default: Set `NO_REFINER=0` to compare the full Stage-1 + LTX-2 refiner path. Set `COMPILE_STAGE1=1` to wrap each Stage-1 DiT with `torch.compile`; this is opt-in -because the pinned upstream SANA-WM stack can fail during TorchInductor Triton +because the pinned upstream Sana stack can fail during TorchInductor Triton compilation on current PyTorch/Triton builds. Streaming parity always uses the full upstream streaming stack; `NO_REFINER=1` @@ -75,15 +76,15 @@ every decoded frame and checks both frame parity and chunk-boundary continuity. debug-friendly; set `STREAMING_NO_COMPILE=0` when explicitly checking compiled streaming behavior. -## Run benchmark +## Run SANA-WM_bidirectional benchmark ```bash cd integrations/sana/tests/parity_check bash bench.sh ``` -Benchmark defaults to the bidirectional comparison and discards one warmup run -before measuring three additional runs: +Benchmark defaults to the `SANA-WM_bidirectional` comparison and discards one +warmup run before measuring three additional runs: `SANA_WM_VARIANT=bidirectional BENCH_SIDE=both WARMUP_RUNS=1 MEASURED_RUNS=3 COMPILE_STAGE1=0 NO_REFINER=0 FORCE_CUDNN_SDPA=0`. Outputs are under `outputs/bench/`: @@ -95,7 +96,7 @@ COMPILE_STAGE1=0 NO_REFINER=0 FORCE_CUDNN_SDPA=0`. Outputs are under The benchmark metric is post-load generation latency per generated clip. Model construction, checkpoint loading, video writing, and frame dumps are outside the -timing boundary. SANA-WM renders each requested bidirectional clip in one +timing boundary. `SANA-WM_bidirectional` renders each requested clip in one generation pass, rather than as independently timed frames. With the default `NO_REFINER=0`, the timed work covers conditioning, Stage-1 DiT, LTX-2 refiner, and SANA VAE decode. Set `NO_REFINER=1` only for a diagnostic Stage-1 plus SANA @@ -105,9 +106,8 @@ VAE decode benchmark. refiner, memory, and frame-normalized diagnostic breakdowns. Those rows explain the benchmark result; they are not a second benchmark metric. -The model card should use one chart per precision. Each chart file should have -GPU/device as the first column and implementation as the series columns, for -example: +Each model-card chart file should have GPU/device as the first column and +implementation as the series columns, for example: ```markdown | device | official | flashdreams | @@ -115,12 +115,13 @@ example: | GB202 | 77826.74 | 76938.72 | ``` -In precision-sweep mode, the chart-ready files are -`outputs/bench//perf.md`. The top-level `outputs/bench/perf.md` is a -precision summary, not the model-card chart data. +Bidirectional upstream comparisons are BF16-only because upstream +`SANA-WM_bidirectional` does not support FP8/FP4 precision flags. Use +`outputs/bench/perf.md` from a BF16 bidirectional run as the model-card chart +data. Set `FORCE_CUDNN_SDPA=1` only for backend-isolation probes. It is not the -default SANA-WM benchmark setting. +default `SANA-WM_bidirectional` benchmark setting. Set `DEVICE_LABEL` when generating chart data for docs: @@ -128,19 +129,19 @@ Set `DEVICE_LABEL` when generating chart data for docs: DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh ``` -Run the precision sweep used by the SANA-WM model card with: +Run the BF16 benchmark used by the `SANA-WM_bidirectional` model card with: ```bash -DEVICE_LABEL="RTX PRO 6000 Blackwell" BENCH_PRECISIONS=bf16,fp8,fp4 bash bench.sh +DEVICE_LABEL="GB202" bash bench.sh ``` The scripts do not set allocator overrides or GPU wait loops. If GPU contention matters in your environment, handle it outside the harness. -## Run streaming benchmark +## Run SANA-WM_streaming benchmark -Use the streaming variant to compare upstream `SANA-WM_streaming` with the -FlashDreams `sana-wm-streaming` runner: +Use `SANA_WM_VARIANT=streaming` to compare upstream `SANA-WM_streaming` with +the FlashDreams `sana-wm-streaming` runner: ```bash cd integrations/sana/tests/parity_check @@ -166,11 +167,9 @@ The streaming headline metric is steady-state generation milliseconds per produced chunk. Warmup runs and the first decoded chunk are excluded; full-clip wall time remains in `bench.json` and `bench.md` as supporting data. -Do not use FlashDreams streaming benchmark numbers for reporting until the -FlashDreams-owned streaming decoder/refiner state machine preserves chunk -continuity and validates against the official upstream baseline. Current -FlashDreams streaming artifacts show chunk-boundary discontinuities aligned -with the 24-frame output chunk cadence. +These streaming benchmark rows measure latency only. Run the streaming parity +and continuity checks before making quality claims about a FlashDreams +streaming change. By default, upstream compiles the streaming refiner transformer. The benchmark passes the equivalent `torch.compile` wrapper to FlashDreams. Set diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index 45a34ebd6..b23f5fa4e 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -92,6 +92,13 @@ WARMUP_RUNS="${WARMUP_RUNS:-1}" MEASURED_RUNS="${MEASURED_RUNS:-3}" BENCH_DRY_RUN="${BENCH_DRY_RUN:-0}" +_reject_bidirectional_low_precision() { + echo "[bench] ERROR: upstream SANA-WM_bidirectional benchmarks are BF16-only." >&2 + echo " FP8 and FP4 precision flags are only supported by upstream SANA-WM_streaming." >&2 + echo " Use STAGE1_PRECISION=bf16 REFINER_PRECISION=bf16, or set SANA_WM_VARIANT=streaming." >&2 + exit 1 +} + if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then DEFAULT_OUTPUT_DIR="${SCRIPT_DIR}/outputs/bench/streaming/${STAGE1_PRECISION}" DEFAULT_SWEEP_OUTPUT_DIR="${SCRIPT_DIR}/outputs/bench/streaming" @@ -147,6 +154,28 @@ if [[ "${SANA_WM_VARIANT}" == "streaming" ]] && _is_true "${COMPILE_STAGE1}"; th echo "[bench] ERROR: COMPILE_STAGE1=1 is bidirectional-only. Streaming compile parity is controlled by STREAMING_NO_COMPILE." >&2 exit 1 fi +if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then + if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; then + _reject_bidirectional_low_precision + fi + if [[ -n "${BENCH_PRECISIONS}" ]]; then + IFS=',' read -r -a BIDIRECTIONAL_PRECISION_LIST <<< "${BENCH_PRECISIONS}" + for PRECISION_RAW in "${BIDIRECTIONAL_PRECISION_LIST[@]}"; do + PRECISION="${PRECISION_RAW//[[:space:]]/}" + if [[ -z "${PRECISION}" ]]; then + continue + fi + case "${PRECISION}" in + bf16) ;; + fp8|fp4) _reject_bidirectional_low_precision ;; + *) + echo "[bench] ERROR: unsupported BENCH_PRECISIONS entry: ${PRECISION}" >&2 + exit 1 + ;; + esac + done + fi +fi if _is_true "${BENCH_DRY_RUN}"; then echo "[bench] dry run" diff --git a/integrations/sana/tests/parity_check/bench_summary.py b/integrations/sana/tests/parity_check/bench_summary.py index 1811b0255..488522d35 100644 --- a/integrations/sana/tests/parity_check/bench_summary.py +++ b/integrations/sana/tests/parity_check/bench_summary.py @@ -344,6 +344,11 @@ def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: upstream = summary["upstream"] flashdreams = summary.get("flashdreams") has_flashdreams = isinstance(flashdreams, dict) + refiner_scope = ( + "With `NO_REFINER=1`, the timed work covers conditioning, Stage-1 DiT, and SANA VAE decode." + if summary["inputs"]["no_refiner"] + else "With `NO_REFINER=0`, the timed work covers conditioning, Stage-1 DiT, LTX-2 refiner, and SANA VAE decode." + ) rows = [ "# SANA-WM parity harness benchmark", "", @@ -370,7 +375,7 @@ def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: "The chart metric is post-load generation latency per generated clip.", "Model construction, checkpoint loading, video writing, and frame dumps are outside this timing boundary.", "SANA-WM renders each requested bidirectional clip in one generation pass, not as independently timed frames.", - "With the default `NO_REFINER=1`, the timed work covers conditioning, Stage-1 DiT, and SANA VAE decode.", + refiner_scope, "", ] if has_flashdreams: @@ -624,6 +629,15 @@ def main(argv: list[str] | None = None) -> None: if args.bench_side == "both" and args.flashdreams_dir is None: parser.error("--flashdreams-dir is required when --bench-side=both") + if args.variant == "bidirectional" and ( + args.stage1_precision != "bf16" or args.refiner_precision != "bf16" + ): + parser.error( + "upstream SANA-WM_bidirectional benchmarks are BF16-only; " + "FP8 and FP4 precision flags are only supported by upstream " + "SANA-WM_streaming. Use --stage1-precision bf16 " + "--refiner-precision bf16, or set --variant streaming." + ) upstream_items = _load_stats(args.upstream_dir) flashdreams_items = _load_stats(args.flashdreams_dir) if args.flashdreams_dir is not None else [] diff --git a/integrations/sana/tests/parity_check/bench_sweep_summary.py b/integrations/sana/tests/parity_check/bench_sweep_summary.py index c65a3fe04..e7ea802f6 100644 --- a/integrations/sana/tests/parity_check/bench_sweep_summary.py +++ b/integrations/sana/tests/parity_check/bench_sweep_summary.py @@ -141,6 +141,26 @@ def _render_report(rows: list[dict[str, Any]]) -> str: return "\n".join(lines) +def _validate_rows(rows: list[dict[str, Any]]) -> None: + for row in rows: + if row["variant"] == "streaming": + continue + inputs = row.get("inputs", {}) + stage1_precision = inputs.get("stage1_precision") + refiner_precision = inputs.get("refiner_precision") + label = row["label"].strip().lower() + if ( + label in {"fp8", "fp4"} + or stage1_precision in {"fp8", "fp4"} + or refiner_precision in {"fp8", "fp4"} + ): + raise ValueError( + "upstream SANA-WM_bidirectional benchmarks are BF16-only; " + "FP8 and FP4 precision comparisons belong to " + "SANA_WM_VARIANT=streaming." + ) + + def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -161,6 +181,7 @@ def main(argv: list[str] | None = None) -> None: units = {row["unit"] for row in args.item} if len(units) != 1: raise ValueError(f"cannot aggregate mixed benchmark units: {sorted(units)}") + _validate_rows(args.item) has_flashdreams = all(row["flashdreams"] is not None for row in args.item) payload = { "benchmark": { diff --git a/integrations/sana/tests/test_parity_benchmark_summary.py b/integrations/sana/tests/test_parity_benchmark_summary.py index 7901b10ae..e81638de7 100644 --- a/integrations/sana/tests/test_parity_benchmark_summary.py +++ b/integrations/sana/tests/test_parity_benchmark_summary.py @@ -19,6 +19,8 @@ import importlib.util import json +import os +import subprocess from pathlib import Path from types import ModuleType @@ -58,9 +60,9 @@ def test_benchmark_summary_uses_generation_ms_per_clip_for_chart() -> None: "num_frames": 20, "seed": 42, "no_refiner": True, - "stage1_precision": "fp8", - "refiner_precision": "fp8", - "quant_backend": "torch-fp8", + "stage1_precision": "bf16", + "refiner_precision": "bf16", + "quant_backend": "auto", "compile_stage1": False, "force_cudnn_sdpa": True, "warmup_runs": 1, @@ -89,7 +91,7 @@ def test_benchmark_summary_uses_generation_ms_per_clip_for_chart() -> None: chart = module._render_chart_markdown(summary, "Test GPU") assert "## Benchmark metric" in report - assert "- stage1_precision: `fp8`" in report + assert "- stage1_precision: `bf16`" in report assert "generation median / clip | 2000.00 ms | 1500.00 ms" in report assert "generation median / frame, diagnostic | 100.00 ms | 75.00 ms" in report assert "## Timing breakdown" in report @@ -201,6 +203,66 @@ def test_benchmark_summary_writes_chart_data(tmp_path: Path) -> None: } +def test_bidirectional_summary_rejects_low_precision_comparison(tmp_path: Path) -> None: + module = _load_bench_summary() + with pytest.raises(SystemExit): + module.main( + [ + "--variant", + "bidirectional", + "--bench-side", + "both", + "--upstream-dir", + str(tmp_path / "upstream"), + "--flashdreams-dir", + str(tmp_path / "flashdreams"), + "--image-path", + "image.png", + "--prompt-path", + "prompt.txt", + "--camera-path", + "pose.npy", + "--intrinsics-path", + "intrinsics.npy", + "--num-frames", + "40", + "--seed", + "42", + "--stage1-precision", + "fp8", + "--refiner-precision", + "fp8", + "--output-json", + str(tmp_path / "bench.json"), + "--output-md", + str(tmp_path / "bench.md"), + ] + ) + + +def test_bidirectional_bench_script_rejects_low_precision_dry_run() -> None: + env = os.environ.copy() + env.update( + { + "BENCH_DRY_RUN": "1", + "SANA_WM_VARIANT": "bidirectional", + "STAGE1_PRECISION": "fp4", + "REFINER_PRECISION": "fp4", + } + ) + result = subprocess.run( + ["bash", "integrations/sana/tests/parity_check/bench.sh"], + check=False, + cwd=Path.cwd(), + env=env, + text=True, + capture_output=True, + ) + + assert result.returncode != 0 + assert "upstream SANA-WM_bidirectional benchmarks are BF16-only" in result.stderr + + def test_benchmark_summary_keeps_frame_normalized_diagnostics() -> None: module = _load_bench_summary() assert module._generation_ms_per_frame( @@ -463,16 +525,20 @@ def test_streaming_comparison_summary_writes_chart_data(tmp_path: Path) -> None: assert "steady-state generation median / chunk | 2000.00 ms | 1000.00 ms" in report -def test_benchmark_sweep_summary_writes_precision_chart_data(tmp_path: Path) -> None: +def test_benchmark_sweep_summary_accepts_bidirectional_bf16_chart_data(tmp_path: Path) -> None: module = _load_bench_sweep_summary() bf16_json = tmp_path / "bf16.json" - fp8_json = tmp_path / "fp8.json" bf16_json.write_text( - json.dumps({"benchmark": {"official": 1000.0, "flashdreams": 900.0}}), - encoding="utf-8", - ) - fp8_json.write_text( - json.dumps({"benchmark": {"official": 800.0, "flashdreams": 700.0}}), + json.dumps( + { + "variant": "bidirectional", + "inputs": { + "stage1_precision": "bf16", + "refiner_precision": "bf16", + }, + "benchmark": {"official": 1000.0, "flashdreams": 900.0}, + } + ), encoding="utf-8", ) out_json = tmp_path / "bench.json" @@ -483,8 +549,6 @@ def test_benchmark_sweep_summary_writes_precision_chart_data(tmp_path: Path) -> [ "--item", f"BF16:{bf16_json}", - "--item", - f"FP8:{fp8_json}", "--output-json", str(out_json), "--output-md", @@ -500,12 +564,45 @@ def test_benchmark_sweep_summary_writes_precision_chart_data(tmp_path: Path) -> "| precision | official | flashdreams |\n" "| --- | ---: | ---: |\n" "| BF16 | 1000.00 | 900.00 |\n" - "| FP8 | 800.00 | 700.00 |\n" ) payload = json.loads(out_json.read_text(encoding="utf-8")) assert payload["benchmark"]["metric"] == "generation_ms_per_clip" assert payload["benchmark"]["unit"] == "ms" - assert [row["label"] for row in payload["rows"]] == ["BF16", "FP8"] + assert [row["label"] for row in payload["rows"]] == ["BF16"] + + +def test_benchmark_sweep_summary_rejects_bidirectional_low_precision( + tmp_path: Path, +) -> None: + module = _load_bench_sweep_summary() + fp8_json = tmp_path / "fp8.json" + fp8_json.write_text( + json.dumps( + { + "variant": "bidirectional", + "inputs": { + "stage1_precision": "fp8", + "refiner_precision": "fp8", + }, + "benchmark": {"official": 800.0, "flashdreams": 700.0}, + } + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="BF16-only"): + module.main( + [ + "--item", + f"FP8:{fp8_json}", + "--output-json", + str(tmp_path / "bench.json"), + "--output-md", + str(tmp_path / "bench.md"), + "--output-chart-md", + str(tmp_path / "perf.md"), + ] + ) def test_benchmark_sweep_summary_accepts_streaming_upstream_only(tmp_path: Path) -> None: From 8cc3bf7f28c400b4077651f03387ae6aaec50cb7 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 30 Jul 2026 12:53:36 -0700 Subject: [PATCH 43/64] Add note to model cards --- docs/source/models/sana_wm_bidirectional.rst | 5 +++++ docs/source/models/sana_wm_streaming.rst | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/docs/source/models/sana_wm_bidirectional.rst b/docs/source/models/sana_wm_bidirectional.rst index 70c49a46d..bb0e81d67 100644 --- a/docs/source/models/sana_wm_bidirectional.rst +++ b/docs/source/models/sana_wm_bidirectional.rst @@ -127,6 +127,11 @@ for FlashDreams ``SANA-WM_bidirectional`` and the official FlashDreams measured 98,432.36 ms per clip versus 107,868.83 ms for the official implementation. +In this chart, ``Official Impl`` means the pinned NVlabs/Sana upstream +implementation measured by the FlashDreams parity harness under matched +settings. It is not the SANA-WM 80-scene benchmark result published by the +model authors. + .. raw:: html
diff --git a/docs/source/models/sana_wm_streaming.rst b/docs/source/models/sana_wm_streaming.rst index e1c2fa97c..f264bcb5a 100644 --- a/docs/source/models/sana_wm_streaming.rst +++ b/docs/source/models/sana_wm_streaming.rst @@ -128,6 +128,11 @@ are excluded from the headline metric; full-clip wall time remains supporting benchmark data. These GB202 latency runs show the official implementation faster than FlashDreams for BF16, FP8, and FP4. +In these charts, ``Official Impl`` means the pinned NVlabs/Sana upstream +implementation measured by the FlashDreams parity harness under matched +settings. It is not the SANA-WM 80-scene benchmark result published by the +model authors. + .. raw:: html
From 3cad01d0b1b964175e15939aa83861e13d59fa60 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 30 Jul 2026 12:53:54 -0700 Subject: [PATCH 44/64] Optimize streaming --- integrations/sana/sana_wm/decoder.py | 150 +++++++- integrations/sana/sana_wm/refiner.py | 435 +++++++++++++++++++++-- integrations/sana/sana_wm/transformer.py | 259 ++++++++++++-- integrations/sana/tests/test_smoke.py | 179 +++++++--- 4 files changed, 906 insertions(+), 117 deletions(-) diff --git a/integrations/sana/sana_wm/decoder.py b/integrations/sana/sana_wm/decoder.py index 2b21e9190..7c2c6458d 100644 --- a/integrations/sana/sana_wm/decoder.py +++ b/integrations/sana/sana_wm/decoder.py @@ -189,23 +189,68 @@ def decode_latents(self, latents: Tensor) -> np.ndarray: latents.shape[2], tuple(decoded.shape) if isinstance(decoded, Tensor) else "list", ) - if isinstance(decoded, list): - decoded = torch.stack(decoded, dim=0) - video = ( - torch.clamp(127.5 * decoded + 127.5, 0, 255) - .to(torch.uint8) - .permute(0, 2, 3, 4, 1) - .contiguous() - .cpu() - .numpy()[0] - ) + video = _decoded_video_to_hwc_uint8(decoded) if self.config.offload_vae: self.vae.to("cpu") del samples, decoded - if torch.cuda.is_available(): + if self.config.offload_vae and torch.cuda.is_available(): torch.cuda.empty_cache() return video + @torch.inference_mode() + def decode_streaming_chunk( + self, + latents: Tensor, + *, + reset_cache: bool, + ) -> np.ndarray | None: + """Decode one causal VAE chunk when the runtime exposes chunk caching.""" + if not bool(getattr(self.config, "use_streaming_decode_cache", False)): + return None + self._ensure_vae() + if not _supports_causal_vae_chunk_decode(self.vae): + return None + if self.config.offload_vae: + self.vae.to(self.device) + + samples = latents.to(device=self.device, dtype=self.vae_dtype) + samples = _prepare_ltx2_vae_decode_samples(self.vae, samples) + decode_per_frame = getattr(self.vae, "decode_per_frame_with_cache") + if reset_cache: + self.vae.clear_decoder_cache() + if torch.cuda.is_available(): + torch.cuda.synchronize() + t0 = time.perf_counter() + try: + decoded = decode_per_frame( + samples, + temb=None, + causal=True, + reset_cache=reset_cache, + ) + except TypeError as exc: + logger.debug("[sana-vae] streaming chunk decode unavailable: {}", exc) + if self.config.offload_vae: + self.vae.to("cpu") + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return None + if torch.cuda.is_available(): + torch.cuda.synchronize() + logger.info( + "[timing] vae streaming decode: {:.3f}s (latent T={} reset={})", + time.perf_counter() - t0, + latents.shape[2], + reset_cache, + ) + video = _decoded_video_to_hwc_uint8(decoded) + if self.config.offload_vae: + self.vae.to("cpu") + if torch.cuda.is_available(): + torch.cuda.empty_cache() + del samples, decoded + return video + def _ensure_runtime_config(self) -> Any: if self._runtime_config is None: self._runtime_config = _load_inference_config(self.config.config_path) @@ -312,6 +357,61 @@ def _vae_decode(self, latents: Tensor) -> Tensor: ) +def _prepare_ltx2_vae_decode_samples(vae: nn.Module, latents: Tensor) -> Tensor: + """Convert normalized SANA-WM latents to the VAE decoder latent domain.""" + latents_mean = vae.latents_mean.view(1, -1, 1, 1, 1).to( + latents.device, + latents.dtype, + ) + latents_std = vae.latents_std.view(1, -1, 1, 1, 1).to( + latents.device, + latents.dtype, + ) + scale = float(vae.config.scaling_factor) + return latents * latents_std / scale + latents_mean + + +def _supports_causal_vae_chunk_decode(vae: nn.Module) -> bool: + """Return whether ``vae`` exposes the causal chunk decode contract.""" + decoder = getattr(vae, "decoder", None) + if ( + decoder is not None + and hasattr(decoder, "is_causal") + and not bool(getattr(decoder, "is_causal")) + ): + return False + return callable(getattr(vae, "clear_decoder_cache", None)) and callable( + getattr(vae, "decode_per_frame_with_cache", None) + ) + + +def _decoded_video_to_hwc_uint8(decoded: Tensor | list[Tensor]) -> np.ndarray: + """Convert VAE decode output to a single ``uint8`` HWC video array.""" + if isinstance(decoded, list): + if not decoded: + raise ValueError("VAE decode returned no frames.") + first = decoded[0] + if first.ndim == 5: + decoded = torch.cat(decoded, dim=2) + elif first.ndim == 4: + decoded = torch.stack(decoded, dim=2) + else: + decoded = torch.stack(decoded, dim=0) + if decoded.ndim != 5: + raise ValueError( + "SANA-WM VAE decode output must have shape [B, C, T, H, W]; " + f"got {tuple(decoded.shape)}." + ) + return ( + torch.clamp(127.5 * decoded + 127.5, 0, 255) + .to(torch.uint8) + .permute(0, 2, 3, 4, 1) + .contiguous() + .cpu() + .numpy()[0] + ) + + @dataclass(kw_only=True) class SanaWMLTX2LatentRefinerConfig(InstantiateConfig): """Config for the optional LTX-2 latent refiner component.""" @@ -529,6 +629,7 @@ class SanaWMStreamingVideoDecoderCache(SanaWMVideoDecoderCache): block_size: int = 3 refiner_kv_max_frames: int = SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES + vae_streaming_cache_ready: bool = False stage1_chunks: list[Tensor] = field(default_factory=list) refined_chunks: list[Tensor] = field(default_factory=list) stage1_sink: Tensor | None = None @@ -543,6 +644,8 @@ class SanaWMStreamingLTX2VAEDecoderConfig(SanaWMLTX2VAEDecoderConfig): config_path: str = SANA_WM_STREAMING_CONFIG_PATH vae_path: str | None = SANA_WM_STREAMING_CAUSAL_VAE_ROOT + use_streaming_decode_cache: bool = True + """Use the causal VAE's per-chunk decoder cache when available.""" @dataclass(kw_only=True) @@ -677,6 +780,7 @@ def forward( context_latents=refiner_context, active_latents=output_active, temporal_compression_ratio=self.temporal_compression_ratio, + cache=None if cache.save_stage1 else cache, ) cache.refined_chunks.append(output_active.detach().contiguous()) stage1_video_hwc = None @@ -694,6 +798,7 @@ def forward( context_latents=stage1_context, active_latents=stage1_active, temporal_compression_ratio=self.temporal_compression_ratio, + cache=None, ) cache.stage1_chunks.append(stage1_active.detach().contiguous()) return SanaWMDecodedVideo( @@ -819,8 +924,31 @@ def _decode_streaming_active_frames( context_latents: Tensor, active_latents: Tensor, temporal_compression_ratio: int, + cache: SanaWMStreamingVideoDecoderCache | None = None, ) -> np.ndarray: """Decode active frames with latent context and drop context pixels.""" + if cache is not None: + reset_cache = not cache.vae_streaming_cache_ready + streaming_latents = ( + torch.cat([context_latents, active_latents], dim=2) + if reset_cache + else active_latents + ) + video = vae_decoder.decode_streaming_chunk( + streaming_latents, + reset_cache=reset_cache, + ) + if video is not None: + cache.vae_streaming_cache_ready = True + if not reset_cache: + return video + start = _pixel_frames_for_latents( + int(context_latents.shape[2]), + temporal_compression_ratio=temporal_compression_ratio, + ) + return video[start:] + cache.vae_streaming_cache_ready = False + decode_latents = torch.cat([context_latents, active_latents], dim=2) video = vae_decoder.decode_latents(decode_latents) start = _pixel_frames_for_latents( diff --git a/integrations/sana/sana_wm/refiner.py b/integrations/sana/sana_wm/refiner.py index dae92c097..3bda7bcc1 100644 --- a/integrations/sana/sana_wm/refiner.py +++ b/integrations/sana/sana_wm/refiner.py @@ -17,7 +17,9 @@ from __future__ import annotations +import os import time +from dataclasses import dataclass from pathlib import Path from typing import Literal @@ -48,6 +50,25 @@ ) +@dataclass(frozen=True) +class _RefinerBlockContext: + """Cached context K/V for one refiner transformer block.""" + + key: Tensor + value: Tensor + + +@dataclass(frozen=True) +class _RefinerContextCache: + """Per-chunk refiner context state reused across denoise steps.""" + + blocks: tuple[_RefinerBlockContext, ...] + encoder_hidden_states: Tensor + encoder_attention_mask: Tensor | None + video_rotary_emb: tuple[Tensor, Tensor] + n_context_tokens: int + + def _move_tensor_attr(module: nn.Module, name: str, device: torch.device | str) -> None: tensor = getattr(module, name, None) if isinstance(tensor, nn.Parameter): @@ -175,6 +196,16 @@ def refine_latents( dtype=self.dtype, ) noisy = (1.0 - start_sigma) * current + start_sigma * eps + packed_context = self._pack_refiner_context(sink) + context_cache = self._build_refiner_context_cache( + context_tokens=packed_context, + prompt_embeds=prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + num_frames=int(z.shape[2]), + height=int(z.shape[3]), + width=int(z.shape[4]), + fps=fps, + ) iterator = range(len(sigmas_t) - 1) if progress: @@ -191,6 +222,8 @@ def refine_latents( prompt_attention_mask=prompt_attention_mask, sigma=sigma, fps=fps, + packed_context_tokens=packed_context, + context_cache=context_cache, ) noisy_tokens = _pack_latents( noisy, @@ -253,6 +286,16 @@ def refine_active_latents( dtype=self.dtype, ) noisy = (1.0 - start_sigma) * active + start_sigma * eps + packed_context = self._pack_refiner_context(context) + context_cache = self._build_refiner_context_cache( + context_tokens=packed_context, + prompt_embeds=prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + num_frames=int(context.shape[2] + active.shape[2]), + height=int(context.shape[3]), + width=int(context.shape[4]), + fps=fps, + ) for step_index in range(len(sigmas_t) - 1): sigma = sigmas_t[step_index] @@ -263,6 +306,8 @@ def refine_active_latents( prompt_attention_mask=prompt_attention_mask, sigma=sigma, fps=fps, + packed_context_tokens=packed_context, + context_cache=context_cache, ) noisy_tokens = _pack_latents( noisy, @@ -292,6 +337,96 @@ def _prepare_video_runtime(self) -> None: self.transformer.eval() self._prepare_quantization() + def _pack_refiner_context(self, context: Tensor) -> Tensor | None: + """Pack fixed context once when temporal patches do not cross the split.""" + if int(self.transformer.config.patch_size_t) != 1: + return None + return _pack_latents( + context, + patch_size=self.transformer.config.patch_size, + patch_size_t=self.transformer.config.patch_size_t, + ) + + def _build_refiner_context_cache( + self, + *, + context_tokens: Tensor | None, + prompt_embeds: Tensor, + prompt_attention_mask: Tensor, + num_frames: int, + height: int, + width: int, + fps: float, + ) -> _RefinerContextCache | None: + """Precompute clean context block state for active-only refinement.""" + if context_tokens is None or not _refiner_context_cache_enabled(): + return None + transformer = self.transformer + for name in ("rope", "proj_in", "time_embed", "caption_projection"): + if not hasattr(transformer, name): + return None + if not hasattr(transformer, "transformer_blocks"): + return None + batch_size = context_tokens.size(0) + encoder_attention_mask = _prepare_encoder_attention_mask( + prompt_attention_mask, + context_tokens.dtype, + ) + video_coords = transformer.rope.prepare_video_coords( + batch_size, + num_frames, + height, + width, + context_tokens.device, + fps=fps, + ) + video_rotary_emb = transformer.rope(video_coords, device=context_tokens.device) + context_rotary_emb = _slice_refiner_rotary_emb( + video_rotary_emb, + 0, + context_tokens.shape[1], + ) + + hidden_states = transformer.proj_in(context_tokens) + context_timestep = torch.zeros( + batch_size, + context_tokens.shape[1], + dtype=torch.float32, + device=context_tokens.device, + ) + temb, _embedded_timestep = transformer.time_embed( + context_timestep.flatten(), + batch_size=batch_size, + hidden_dtype=hidden_states.dtype, + ) + temb = temb.view(batch_size, -1, temb.size(-1)) + encoder_hidden_states = transformer.caption_projection(prompt_embeds) + encoder_hidden_states = encoder_hidden_states.view( + batch_size, + -1, + hidden_states.size(-1), + ) + + blocks: list[_RefinerBlockContext] = [] + for block in transformer.transformer_blocks: + hidden_states, block_context = _forward_refiner_context_block( + block=block, + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + video_rotary_emb=context_rotary_emb, + encoder_attention_mask=encoder_attention_mask, + ) + blocks.append(block_context) + + return _RefinerContextCache( + blocks=tuple(blocks), + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + video_rotary_emb=video_rotary_emb, + n_context_tokens=int(context_tokens.shape[1]), + ) + def _load_diffusers_components(self) -> tuple[nn.Module, nn.Module]: from diffusers.models.transformers.transformer_ltx2 import ( LTX2VideoTransformer3DModel, @@ -448,19 +583,45 @@ def _predict_current_x0( prompt_attention_mask: Tensor, sigma: Tensor, fps: float, + packed_context_tokens: Tensor | None = None, + context_cache: _RefinerContextCache | None = None, ) -> Tensor: - full_latent = torch.cat([sink, noisy_current], dim=2) - batch_size, _, num_frames, height, width = full_latent.shape - latent_tokens = _pack_latents( - full_latent, + batch_size, _, context_frames, height, width = sink.shape + current_tokens = _pack_latents( + noisy_current, patch_size=self.transformer.config.patch_size, patch_size_t=self.transformer.config.patch_size_t, ) - n_context_tokens = _pack_latents( - sink, - patch_size=self.transformer.config.patch_size, - patch_size_t=self.transformer.config.patch_size_t, - ).shape[1] + if context_cache is not None and packed_context_tokens is not None: + raw_timestep = torch.full( + (batch_size, current_tokens.shape[1], 1), + float(sigma), + dtype=torch.float32, + device=self.device, + ) + model_timestep = ( + raw_timestep.squeeze(-1) + * float(self.transformer.config.timestep_scale_multiplier) + ) + velocity = self._forward_video_current_only( + hidden_states=current_tokens, + timestep=model_timestep, + context_cache=context_cache, + ) + denoised = current_tokens.float() - velocity.float() * raw_timestep + return denoised.to(self.dtype) + + if packed_context_tokens is None: + latent_tokens = _pack_latents( + torch.cat([sink, noisy_current], dim=2), + patch_size=self.transformer.config.patch_size, + patch_size_t=self.transformer.config.patch_size_t, + ) + n_context_tokens = latent_tokens.shape[1] - current_tokens.shape[1] + else: + latent_tokens = torch.cat([packed_context_tokens, current_tokens], dim=1) + n_context_tokens = packed_context_tokens.shape[1] + num_frames = context_frames + int(noisy_current.shape[2]) raw_timestep = torch.zeros( batch_size, @@ -558,6 +719,67 @@ def _forward_video_only( hidden_states = hidden_states * (1 + scale) + shift return transformer.proj_out(hidden_states) + def _forward_video_current_only( + self, + *, + hidden_states: Tensor, + timestep: Tensor, + context_cache: _RefinerContextCache, + ) -> Tensor: + """Run the refiner over active tokens with cached clean context K/V.""" + transformer = self.transformer + batch_size = hidden_states.size(0) + if batch_size != context_cache.encoder_hidden_states.size(0): + raise ValueError( + "Refiner context cache batch size does not match active tokens: " + f"{context_cache.encoder_hidden_states.size(0)} != {batch_size}." + ) + current_start = int(context_cache.n_context_tokens) + current_end = current_start + int(hidden_states.shape[1]) + current_rotary_emb = _slice_refiner_rotary_emb( + context_cache.video_rotary_emb, + current_start, + current_end, + ) + + hidden_states = transformer.proj_in(hidden_states) + temb, embedded_timestep = transformer.time_embed( + timestep.flatten(), + batch_size=batch_size, + hidden_dtype=hidden_states.dtype, + ) + temb = temb.view(batch_size, -1, temb.size(-1)) + embedded_timestep = embedded_timestep.view( + batch_size, + -1, + embedded_timestep.size(-1), + ) + + if len(context_cache.blocks) != len(transformer.transformer_blocks): + raise ValueError("Refiner context cache block count is stale.") + for block, block_context in zip( + transformer.transformer_blocks, + context_cache.blocks, + ): + hidden_states = _forward_refiner_current_block( + block=block, + hidden_states=hidden_states, + encoder_hidden_states=context_cache.encoder_hidden_states, + temb=temb, + video_rotary_emb=current_rotary_emb, + encoder_attention_mask=context_cache.encoder_attention_mask, + context=block_context, + ) + + scale_shift_values = ( + transformer.scale_shift_table[None, None] + + embedded_timestep[:, :, None] + ) + shift, scale = scale_shift_values[:, :, 0], scale_shift_values[:, :, 1] + hidden_states = transformer.norm_out(hidden_states) + hidden_states = hidden_states * (1 + scale) + shift + return transformer.proj_out(hidden_states) + def _forward_video_block( *, @@ -604,6 +826,111 @@ def _forward_video_block( return hidden_states + block.ff(norm_hidden_states) * gate_mlp +def _forward_refiner_context_block( + *, + block: nn.Module, + hidden_states: Tensor, + encoder_hidden_states: Tensor, + temb: Tensor, + video_rotary_emb: tuple[Tensor, Tensor], + encoder_attention_mask: Tensor | None, +) -> tuple[Tensor, _RefinerBlockContext]: + """Run one block for clean context and cache its self-attention K/V.""" + batch_size = hidden_states.size(0) + norm_hidden_states = block.norm1(hidden_states) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + _refiner_block_ada_values(block, temb, batch_size) + ) + norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa + + query, key, value, gate_logits = _refiner_self_attention_qkv( + attn=block.attn1, + hidden_states=norm_hidden_states, + query_rotary_emb=video_rotary_emb, + ) + attn_hidden_states = _finish_refiner_self_attention( + attn=block.attn1, + hidden_states=_refiner_attention(query, key, value), + gate_logits=gate_logits, + dtype=query.dtype, + ) + hidden_states = hidden_states + attn_hidden_states * gate_msa + + norm_hidden_states = block.norm2(hidden_states) + attn_hidden_states = block.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + query_rotary_emb=None, + attention_mask=encoder_attention_mask, + ) + hidden_states = hidden_states + attn_hidden_states + + norm_hidden_states = block.norm3(hidden_states) * (1 + scale_mlp) + shift_mlp + hidden_states = hidden_states + block.ff(norm_hidden_states) * gate_mlp + return hidden_states, _RefinerBlockContext(key=key, value=value) + + +def _forward_refiner_current_block( + *, + block: nn.Module, + hidden_states: Tensor, + encoder_hidden_states: Tensor, + temb: Tensor, + video_rotary_emb: tuple[Tensor, Tensor], + encoder_attention_mask: Tensor | None, + context: _RefinerBlockContext, +) -> Tensor: + """Run one active-token block using cached clean context K/V.""" + batch_size = hidden_states.size(0) + norm_hidden_states = block.norm1(hidden_states) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + _refiner_block_ada_values(block, temb, batch_size) + ) + norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa + + query, key, value, gate_logits = _refiner_self_attention_qkv( + attn=block.attn1, + hidden_states=norm_hidden_states, + query_rotary_emb=video_rotary_emb, + ) + key = torch.cat([context.key, key], dim=1) + value = torch.cat([context.value, value], dim=1) + attn_hidden_states = _finish_refiner_self_attention( + attn=block.attn1, + hidden_states=_refiner_attention(query, key, value), + gate_logits=gate_logits, + dtype=query.dtype, + ) + hidden_states = hidden_states + attn_hidden_states * gate_msa + + norm_hidden_states = block.norm2(hidden_states) + attn_hidden_states = block.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + query_rotary_emb=None, + attention_mask=encoder_attention_mask, + ) + hidden_states = hidden_states + attn_hidden_states + + norm_hidden_states = block.norm3(hidden_states) * (1 + scale_mlp) + shift_mlp + return hidden_states + block.ff(norm_hidden_states) * gate_mlp + + +def _refiner_block_ada_values( + block: nn.Module, + temb: Tensor, + batch_size: int, +) -> tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: + num_ada_params = block.scale_shift_table.shape[0] + ada_values = block.scale_shift_table[None, None].to(temb.device) + temb.reshape( + batch_size, + temb.size(1), + num_ada_params, + -1, + ) + return ada_values[:, :, :6].unbind(dim=2) + + def _streaming_self_attention( *, attn: nn.Module, @@ -611,6 +938,44 @@ def _streaming_self_attention( query_rotary_emb: tuple[Tensor, Tensor], n_context_tokens: int, ) -> Tensor: + query, key, value, gate_logits = _refiner_self_attention_qkv( + attn=attn, + hidden_states=hidden_states, + query_rotary_emb=query_rotary_emb, + ) + + if n_context_tokens <= 0 or n_context_tokens >= query.shape[1]: + hidden_states = _refiner_attention(query, key, value) + else: + context_hidden_states = _refiner_attention( + query[:, :n_context_tokens], + key[:, :n_context_tokens], + value[:, :n_context_tokens], + ) + current_hidden_states = _refiner_attention( + query[:, n_context_tokens:], + key, + value, + ) + hidden_states = torch.cat( + [context_hidden_states, current_hidden_states], + dim=1, + ) + + return _finish_refiner_self_attention( + attn=attn, + hidden_states=hidden_states, + gate_logits=gate_logits, + dtype=query.dtype, + ) + + +def _refiner_self_attention_qkv( + *, + attn: nn.Module, + hidden_states: Tensor, + query_rotary_emb: tuple[Tensor, Tensor], +) -> tuple[Tensor, Tensor, Tensor, Tensor | None]: from diffusers.models.transformers.transformer_ltx2 import ( apply_interleaved_rotary_emb, apply_split_rotary_emb, @@ -639,26 +1004,17 @@ def _streaming_self_attention( query = query.unflatten(2, (attn.heads, -1)) key = key.unflatten(2, (attn.heads, -1)) value = value.unflatten(2, (attn.heads, -1)) + return query, key, value, gate_logits - if n_context_tokens <= 0 or n_context_tokens >= query.shape[1]: - hidden_states = _refiner_attention(query, key, value) - else: - context_hidden_states = _refiner_attention( - query[:, :n_context_tokens], - key[:, :n_context_tokens], - value[:, :n_context_tokens], - ) - current_hidden_states = _refiner_attention( - query[:, n_context_tokens:], - key, - value, - ) - hidden_states = torch.cat( - [context_hidden_states, current_hidden_states], - dim=1, - ) - hidden_states = hidden_states.flatten(2, 3).to(query.dtype) +def _finish_refiner_self_attention( + *, + attn: nn.Module, + hidden_states: Tensor, + gate_logits: Tensor | None, + dtype: torch.dtype, +) -> Tensor: + hidden_states = hidden_states.flatten(2, 3).to(dtype) if gate_logits is not None: hidden_states = hidden_states.unflatten(2, (attn.heads, -1)) gates = (2.0 * torch.sigmoid(gate_logits)).unsqueeze(-1) @@ -668,6 +1024,19 @@ def _streaming_self_attention( return attn.to_out[1](hidden_states) +def _slice_refiner_rotary_emb( + rotary_emb: tuple[Tensor, Tensor], + start: int, + end: int, +) -> tuple[Tensor, Tensor]: + cos, sin = rotary_emb + if cos.ndim == 4: + return cos[:, :, start:end], sin[:, :, start:end] + if cos.ndim >= 3: + return cos[:, start:end], sin[:, start:end] + return cos[start:end], sin[start:end] + + def _refiner_attention(query: Tensor, key: Tensor, value: Tensor) -> Tensor: hidden_states = F.scaled_dot_product_attention( query.transpose(1, 2), @@ -765,11 +1134,19 @@ def _unpack_latents( return latents.flatten(6, 7).flatten(4, 5).flatten(2, 3) -def _prepare_encoder_attention_mask(mask: Tensor | None, dtype: torch.dtype) -> Tensor | None: +def _prepare_encoder_attention_mask( + mask: Tensor | None, + dtype: torch.dtype, +) -> Tensor | None: if mask is None: return None if mask.ndim != 2: return mask - if bool(torch.all(mask)): + if not torch.compiler.is_compiling() and bool(torch.all(mask)): return None return ((1 - mask.to(dtype)) * -10000.0).unsqueeze(1) + + +def _refiner_context_cache_enabled() -> bool: + value = os.environ.get("SANA_WM_REFINER_CONTEXT_CACHE", "1").strip().lower() + return value not in {"0", "false", "no", "off"} diff --git a/integrations/sana/sana_wm/transformer.py b/integrations/sana/sana_wm/transformer.py index 0ffea0343..633c34d3f 100644 --- a/integrations/sana/sana_wm/transformer.py +++ b/integrations/sana/sana_wm/transformer.py @@ -126,6 +126,15 @@ class SanaWMStreamingTransformerCache(SanaWMTransformerCache): initial_latent_state: Tensor | None = None +@dataclass(frozen=True) +class _StreamingStage1Window: + """Local Stage-1 frame window for one streaming AR step.""" + + frame_indices: tuple[int, ...] + active_start: int + active_end: int + + @dataclass(kw_only=True) class SanaWMTransformerConfig(TransformerConfig): """Config for the SANA-WM Stage-1 transformer adapter.""" @@ -750,23 +759,39 @@ def _predict_streaming_conditioned( latent_state = self._ensure_streaming_latent_state(cache, conditioning) start = int(conditioning.start_frame) end = int(conditioning.end_frame) - prefix = latent_state[:, :, :end].clone() - prefix[:, :, start:end] = noisy_latent - if start == 0: - prefix[:, :, :1] = conditioning.first_latent.to( - device=prefix.device, - dtype=prefix.dtype, + window = _streaming_stage1_window(conditioning, self.config) + window_index = torch.tensor( + window.frame_indices, + dtype=torch.long, + device=latent_state.device, + ) + model_latent = latent_state.index_select(2, window_index).clone() + model_latent[:, :, window.active_start : window.active_end] = noisy_latent + if 0 in window.frame_indices: + sink_local = window.frame_indices.index(0) + model_latent[:, :, sink_local : sink_local + 1] = ( + conditioning.first_latent.to( + device=model_latent.device, + dtype=model_latent.dtype, + ) ) - model_timestep = _streaming_prefix_timestep( + model_timestep = _streaming_window_timestep( noisy_latent=noisy_latent, timestep=timestep, conditioning=conditioning, - prefix_frames=end, + frame_indices=window.frame_indices, + active_start=window.active_start, + active_end=window.active_end, + ) + kwargs = _streaming_window_model_kwargs( + conditioning.model_kwargs, + frame_indices=window.frame_indices, + total_frames=int(conditioning.total_latent_shape[2]), + chunk_size=int(self.config.num_frame_per_block), ) - kwargs = _streaming_prefix_model_kwargs(conditioning.model_kwargs, end) if conditioning.cfg_scale <= 1.0: - flow_prefix = self._predict_with_prompt( - prefix, + flow_window = self._predict_with_prompt( + model_latent, model_timestep, conditioning.condition, _condition_model_kwargs(kwargs), @@ -774,15 +799,15 @@ def _predict_streaming_conditioned( else: if conditioning.uncondition is None: raise RuntimeError("CFG was requested without negative prompt embeds.") - flow_prefix = self._predict_with_prompt( - torch.cat([prefix, prefix], dim=0), + flow_window = self._predict_with_prompt( + torch.cat([model_latent, model_latent], dim=0), torch.cat([model_timestep, model_timestep], dim=0), torch.cat([conditioning.uncondition, conditioning.condition], dim=0), _batched_cfg_model_kwargs(kwargs), ) - flow_prefix = _cfg_guidance(flow_prefix, conditioning.cfg_scale) + flow_window = _cfg_guidance(flow_window, conditioning.cfg_scale) - flow = flow_prefix[:, :, start:end] + flow = flow_window[:, :, window.active_start : window.active_end] if start == 0: flow[:, :, :1] = 0 return flow.to(dtype=noisy_latent.dtype) @@ -862,6 +887,112 @@ def _streaming_prefix_model_kwargs( return result +def _streaming_stage1_window( + conditioning: SanaWMStreamingStage1Conditioning, + config: SanaWMStreamingTransformerConfig, +) -> _StreamingStage1Window: + """Return the sink-plus-tail frame window for one streaming Stage-1 call.""" + start = int(conditioning.start_frame) + end = int(conditioning.end_frame) + active_frames = end - start + if start < 0 or active_frames <= 0: + raise ValueError( + "SANA-WM streaming chunk bounds must describe a positive chunk; " + f"got start={start}, end={end}." + ) + if start == 0: + return _StreamingStage1Window( + frame_indices=tuple(range(end)), + active_start=0, + active_end=end, + ) + + sink_frames = 1 if bool(config.sink_token) else 0 + sink_frames = min(sink_frames, start) + cached_frames = max(0, int(config.num_cached_blocks)) * max( + 1, + int(config.num_frame_per_block), + ) + history_start = max(sink_frames, start - cached_frames) + if history_start <= sink_frames: + frame_indices = tuple(range(end)) + active_start = start + else: + frame_indices = tuple(range(sink_frames)) + tuple(range(history_start, end)) + active_start = len(frame_indices) - active_frames + return _StreamingStage1Window( + frame_indices=frame_indices, + active_start=active_start, + active_end=active_start + active_frames, + ) + + +def _streaming_window_model_kwargs( + model_kwargs: dict[str, object], + *, + frame_indices: tuple[int, ...], + total_frames: int, + chunk_size: int, +) -> dict[str, object]: + """Slice full-rollout model kwargs to a possibly non-contiguous frame window.""" + result: dict[str, object] = {} + for key, value in model_kwargs.items(): + if key in {"camera_cache", "rotary_emb", "chunk_plucker_emb"}: + continue + if key == "data_info" and isinstance(value, dict): + result[key] = _streaming_window_data_info(value, frame_indices) + elif key == "chunk_index": + result[key] = _chunk_index_from_chunk_size( + len(frame_indices), + chunk_size, + strategy="first_chunk_plus_one", + ) + elif isinstance(value, Tensor): + result[key] = _slice_streaming_window_tensor( + value, + frame_indices=frame_indices, + total_frames=total_frames, + ) + elif isinstance(value, dict): + result[key] = dict(value) + else: + result[key] = value + return result + + +def _slice_streaming_window_tensor( + value: Tensor, + *, + frame_indices: tuple[int, ...], + total_frames: int, +) -> Tensor: + """Index latent-frame-aligned tensors along their temporal axis.""" + index = torch.tensor(frame_indices, dtype=torch.long, device=value.device) + if value.ndim == 5 and value.shape[2] >= total_frames: + return value.index_select(2, index).contiguous() + if value.ndim >= 3 and value.shape[1] >= total_frames: + return value.index_select(1, index).contiguous() + return value + + +def _streaming_window_data_info( + data_info: dict[object, object], + frame_indices: tuple[int, ...], +) -> dict[object, object]: + """Remap absolute conditioned-frame metadata into the local frame window.""" + result = dict(data_info) + frame_to_local = {frame: local for local, frame in enumerate(frame_indices)} + condition_frame_info = data_info.get("condition_frame_info") + if isinstance(condition_frame_info, dict): + remapped = {} + for frame, value in condition_frame_info.items(): + local = frame_to_local.get(int(frame)) + if local is not None: + remapped[local] = value + result["condition_frame_info"] = remapped + return result + + def _streaming_prefix_timestep( *, noisy_latent: Tensor, @@ -870,6 +1001,76 @@ def _streaming_prefix_timestep( prefix_frames: int, ) -> Tensor: """Build a prefix timestep table with clean context and noisy current chunk.""" + current = _streaming_current_timestep( + noisy_latent=noisy_latent, + timestep=timestep, + conditioning=conditioning, + ) + batch = noisy_latent.shape[0] + chunk_frames = current.shape[2] + active_start = int(conditioning.start_frame) + active_end = active_start + chunk_frames + if active_end > prefix_frames: + raise ValueError( + "SANA-WM streaming prefix is shorter than the active chunk end: " + f"prefix_frames={prefix_frames}, active_end={active_end}." + ) + prefix = torch.zeros( + batch, + 1, + prefix_frames, + dtype=torch.float32, + device=noisy_latent.device, + ) + prefix[:, :, active_start:active_end] = current + return prefix + + +def _streaming_window_timestep( + *, + noisy_latent: Tensor, + timestep: Tensor, + conditioning: SanaWMStreamingStage1Conditioning, + frame_indices: tuple[int, ...], + active_start: int, + active_end: int, +) -> Tensor: + """Build a local timestep table for a bounded streaming frame window.""" + current = _streaming_current_timestep( + noisy_latent=noisy_latent, + timestep=timestep, + conditioning=conditioning, + ) + batch = noisy_latent.shape[0] + if active_end - active_start != current.shape[2]: + raise ValueError( + "SANA-WM streaming window active span does not match the latent chunk: " + f"active=({active_start}, {active_end}), chunk={current.shape[2]}." + ) + window = torch.zeros( + batch, + 1, + len(frame_indices), + dtype=torch.float32, + device=noisy_latent.device, + ) + window[:, :, active_start:active_end] = current + condition_frame_info = _streaming_condition_frame_info(conditioning) + frame_to_local = {frame: local for local, frame in enumerate(frame_indices)} + for frame_idx in condition_frame_info: + local = frame_to_local.get(int(frame_idx)) + if local is not None: + window[:, :, local] = 0 + return window + + +def _streaming_current_timestep( + *, + noisy_latent: Tensor, + timestep: Tensor, + conditioning: SanaWMStreamingStage1Conditioning, +) -> Tensor: + """Normalize scheduler timestep input to ``[B, 1, active_T]``.""" batch = noisy_latent.shape[0] chunk_frames = int(conditioning.end_frame - conditioning.start_frame) if timestep.ndim == 0: @@ -894,27 +1095,25 @@ def _streaming_prefix_timestep( f"got {tuple(current.shape)}, expected {(batch, 1, chunk_frames)}." ) current = current.to(device=noisy_latent.device, dtype=torch.float32).clone() + condition_frame_info = _streaming_condition_frame_info(conditioning) + for frame_idx in condition_frame_info: + index = int(frame_idx) + if conditioning.start_frame <= index < conditioning.end_frame: + current[:, :, index - conditioning.start_frame] = 0 + return current + + +def _streaming_condition_frame_info( + conditioning: SanaWMStreamingStage1Conditioning, +) -> dict[object, object]: + """Return conditioned-frame metadata from a streaming conditioning payload.""" data_info = conditioning.model_kwargs.get("data_info", {}) condition_frame_info = ( data_info.get("condition_frame_info", {}) if isinstance(data_info, dict) else {} ) - if isinstance(condition_frame_info, dict): - for frame_idx in condition_frame_info: - index = int(frame_idx) - if conditioning.start_frame <= index < conditioning.end_frame: - current[:, :, index - conditioning.start_frame] = 0 - - prefix = torch.zeros( - batch, - 1, - prefix_frames, - dtype=torch.float32, - device=noisy_latent.device, - ) - prefix[:, :, conditioning.start_frame : conditioning.end_frame] = current - return prefix + return condition_frame_info if isinstance(condition_frame_info, dict) else {} def _cfg_guidance(noise_pred: Tensor, cfg_scale: float) -> Tensor: diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index d8f754866..acb93f6e4 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -530,12 +530,12 @@ def test_transformer_initial_noise_uses_conditioning_payload() -> None: def test_streaming_transformer_commits_chunks_into_prefix_cache() -> None: - """Keep streaming Stage-1 chunks in one rollout-level latent cache.""" + """Keep chunk state while bounding the per-step Stage-1 model window.""" class DummyModel(torch.nn.Module): def __init__(self) -> None: super().__init__() - self.calls: list[dict[str, torch.Tensor]] = [] + self.calls: list[dict[str, object]] = [] def forward( self, @@ -549,10 +549,18 @@ def forward( { "noisy_latent": noisy_latent.detach().clone(), "timestep": timestep.detach().clone(), + "camera_conditions": _kwargs["camera_conditions"] + .detach() + .clone(), + "chunk_plucker": _kwargs["chunk_plucker"].detach().clone(), + "data_info": dict(_kwargs["data_info"]), + "chunk_index": list(_kwargs["chunk_index"]), } ) return (torch.ones_like(noisy_latent),) + total_frames = 13 + def conditioning_for_chunk( *, start: int, @@ -564,8 +572,11 @@ def conditioning_for_chunk( uncondition=None, model_kwargs={ "data_info": {"condition_frame_info": {0: 0.0}}, - "camera_conditions": torch.zeros((1, 7, 20)), - "chunk_plucker": torch.zeros((1, 2, 7, 1, 1)), + "camera_conditions": torch.arange(total_frames, dtype=torch.float32) + .reshape(1, total_frames, 1), + "chunk_plucker": torch.arange(total_frames, dtype=torch.float32) + .reshape(1, 1, total_frames, 1, 1), + "chunk_index": [0, 4, 7, 10], }, first_latent=torch.full((1, 1, 1, 1, 1), 7.0), latent_shape=(1, 1, end - start, 1, 1), @@ -573,11 +584,11 @@ def conditioning_for_chunk( flow_shift=8.0, steps=4, seed=123, - total_latent_shape=(1, 1, 7, 1, 1), + total_latent_shape=(1, 1, total_frames, 1, 1), start_frame=start, end_frame=end, chunk_index=chunk_index, - chunk_boundaries=(0, 4, 7), + chunk_boundaries=(0, 4, 7, 10, 13), ) transformer = SanaWMStreamingTransformerConfig().setup() @@ -585,44 +596,46 @@ def conditioning_for_chunk( dummy_model = DummyModel() transformer.model = dummy_model transformer._model_built = True - chunk0 = conditioning_for_chunk(start=0, end=4, chunk_index=0) - chunk1 = conditioning_for_chunk(start=4, end=7, chunk_index=1) - cache = transformer.initialize_autoregressive_cache(conditioning=chunk0) - assert isinstance(cache, SanaWMStreamingTransformerCache) - - noise0 = transformer.initial_noise( - latent_shape=chunk0.latent_shape, - rng=None, - cache=cache, - input=chunk0, + chunks = ( + conditioning_for_chunk(start=0, end=4, chunk_index=0), + conditioning_for_chunk(start=4, end=7, chunk_index=1), + conditioning_for_chunk(start=7, end=10, chunk_index=2), + conditioning_for_chunk(start=10, end=13, chunk_index=3), ) - flow0 = transformer.predict_flow( - noisy_latent=noise0, - timestep=torch.tensor(1000.0), - cache=cache, - input=chunk0, - ) - clean0 = torch.full_like(noise0, 2.0) - transformer.postprocess_clean_latent(clean0, cache, input=chunk0) + cache = transformer.initialize_autoregressive_cache(conditioning=chunks[0]) + assert isinstance(cache, SanaWMStreamingTransformerCache) - noise1 = transformer.initial_noise( - latent_shape=chunk1.latent_shape, - rng=None, - cache=cache, - input=chunk1, - ) - flow1 = transformer.predict_flow( - noisy_latent=noise1, - timestep=torch.tensor(500.0), - cache=cache, - input=chunk1, - ) - clean1 = torch.full_like(noise1, 4.0) - transformer.postprocess_clean_latent(clean1, cache, input=chunk1) + flows: list[torch.Tensor] = [] + cleans: list[torch.Tensor] = [] + for chunk, timestep, clean_value in ( + (chunks[0], 1000.0, 2.0), + (chunks[1], 500.0, 4.0), + (chunks[2], 250.0, 6.0), + (chunks[3], 125.0, 8.0), + ): + noise = transformer.initial_noise( + latent_shape=chunk.latent_shape, + rng=None, + cache=cache, + input=chunk, + ) + flows.append( + transformer.predict_flow( + noisy_latent=noise, + timestep=torch.tensor(timestep), + cache=cache, + input=chunk, + ) + ) + clean = torch.full_like(noise, clean_value) + cleans.append(clean) + transformer.postprocess_clean_latent(clean, cache, input=chunk) - assert len(dummy_model.calls) == 2 + assert len(dummy_model.calls) == 4 assert dummy_model.calls[0]["noisy_latent"].shape == (1, 1, 4, 1, 1) assert dummy_model.calls[1]["noisy_latent"].shape == (1, 1, 7, 1, 1) + assert dummy_model.calls[2]["noisy_latent"].shape == (1, 1, 10, 1, 1) + assert dummy_model.calls[3]["noisy_latent"].shape == (1, 1, 10, 1, 1) torch.testing.assert_close( dummy_model.calls[0]["timestep"], torch.tensor([[[0.0, 1000.0, 1000.0, 1000.0]]]), @@ -631,16 +644,36 @@ def conditioning_for_chunk( dummy_model.calls[1]["timestep"], torch.tensor([[[0.0, 0.0, 0.0, 0.0, 500.0, 500.0, 500.0]]]), ) - torch.testing.assert_close(flow0[:, :, :1], torch.zeros_like(flow0[:, :, :1])) - torch.testing.assert_close(flow0[:, :, 1:], torch.ones_like(flow0[:, :, 1:])) - torch.testing.assert_close(flow1, torch.ones_like(flow1)) + torch.testing.assert_close( + dummy_model.calls[3]["timestep"], + torch.tensor( + [[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 125.0, 125.0, 125.0]]] + ), + ) + camera3 = dummy_model.calls[3]["camera_conditions"] + assert isinstance(camera3, torch.Tensor) + torch.testing.assert_close( + camera3.flatten(), + torch.tensor([0.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0]), + ) + plucker3 = dummy_model.calls[3]["chunk_plucker"] + assert isinstance(plucker3, torch.Tensor) + torch.testing.assert_close(plucker3.flatten(), camera3.flatten()) + assert dummy_model.calls[3]["data_info"] == {"condition_frame_info": {0: 0.0}} + assert dummy_model.calls[3]["chunk_index"] == [0, 4, 7] + torch.testing.assert_close(flows[0][:, :, :1], torch.zeros_like(flows[0][:, :, :1])) + torch.testing.assert_close(flows[0][:, :, 1:], torch.ones_like(flows[0][:, :, 1:])) + torch.testing.assert_close(flows[1], torch.ones_like(flows[1])) + torch.testing.assert_close(flows[3], torch.ones_like(flows[3])) assert cache.latent_state is not None torch.testing.assert_close( cache.latent_state[:, :, :1], torch.full((1, 1, 1, 1, 1), 7.0), ) - torch.testing.assert_close(cache.latent_state[:, :, 1:4], clean0[:, :, 1:]) - torch.testing.assert_close(cache.latent_state[:, :, 4:7], clean1) + torch.testing.assert_close(cache.latent_state[:, :, 1:4], cleans[0][:, :, 1:]) + torch.testing.assert_close(cache.latent_state[:, :, 4:7], cleans[1]) + torch.testing.assert_close(cache.latent_state[:, :, 7:10], cleans[2]) + torch.testing.assert_close(cache.latent_state[:, :, 10:13], cleans[3]) def test_transformer_predict_flow_applies_cfg_from_conditioning_input() -> None: @@ -1064,7 +1097,12 @@ def test_streaming_video_decoder_emits_only_new_frames( monkeypatch: pytest.MonkeyPatch, ) -> None: """Decode chunks with context while returning only the active frames.""" - decoder = SanaWMStreamingVideoDecoderConfig(refiner=None).setup() + decoder = SanaWMStreamingVideoDecoderConfig( + vae_decoder=SanaWMStreamingLTX2VAEDecoderConfig( + use_streaming_decode_cache=False, + ), + refiner=None, + ).setup() def decode_with_context(latents: torch.Tensor) -> np.ndarray: frames = 1 + (latents.shape[2] - 1) * 8 @@ -1099,11 +1137,55 @@ def decode_with_context(latents: torch.Tensor) -> np.ndarray: assert second.stage1_video_hwc is None +def test_streaming_video_decoder_uses_causal_vae_chunk_cache( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Decode only active latents after the causal VAE cache is initialized.""" + decoder = SanaWMStreamingVideoDecoderConfig(refiner=None).setup() + calls: list[tuple[torch.Tensor, bool]] = [] + + def decode_chunk(latents: torch.Tensor, *, reset_cache: bool) -> np.ndarray | None: + calls.append((latents.clone(), reset_cache)) + frames = 1 + (latents.shape[2] - 1) * 8 if reset_cache else latents.shape[2] * 8 + value = 10 if reset_cache else 20 + return np.full((frames, 2, 2, 3), value, dtype=np.uint8) + + monkeypatch.setattr(decoder.vae_decoder, "decode_streaming_chunk", decode_chunk) + cache = decoder.initialize_autoregressive_cache() + + first = decoder( + torch.zeros((1, 4, 4, 1, 1)), + autoregressive_index=0, + cache=cache, + ) + second = decoder( + torch.zeros((1, 4, 3, 1, 1)), + autoregressive_index=1, + cache=cache, + ) + + assert [tuple(call[0].shape) for call in calls] == [ + (1, 4, 4, 1, 1), + (1, 4, 3, 1, 1), + ] + assert [call[1] for call in calls] == [True, False] + assert first.video_hwc.shape == (24, 2, 2, 3) + assert second.video_hwc.shape == (24, 2, 2, 3) + assert first.video_hwc[0, 0, 0, 0] == 10 + assert second.video_hwc[0, 0, 0, 0] == 20 + assert cache.vae_streaming_cache_ready is True + + def test_streaming_video_decoder_refines_against_rolling_history( monkeypatch: pytest.MonkeyPatch, ) -> None: """Use prior refined chunks as clean context for later streaming chunks.""" - decoder = SanaWMStreamingVideoDecoderConfig(refiner=None).setup() + decoder = SanaWMStreamingVideoDecoderConfig( + vae_decoder=SanaWMStreamingLTX2VAEDecoderConfig( + use_streaming_decode_cache=False, + ), + refiner=None, + ).setup() calls: list[torch.Tensor] = [] class DummyRefiner: @@ -1910,8 +1992,11 @@ def predict_current_x0( prompt_attention_mask: torch.Tensor, sigma: torch.Tensor, fps: float, + packed_context_tokens: torch.Tensor | None, + context_cache: object | None, ) -> torch.Tensor: - del sink, prompt_embeds, prompt_attention_mask, sigma, fps + del sink, prompt_embeds, prompt_attention_mask, sigma, fps, context_cache + assert packed_context_tokens is not None inference_modes.append(torch.is_inference_mode_enabled()) return torch.zeros_like(_pack_latents(noisy_current)) From 2c5304fc2c6b29a3809820314a6d5a9f9ea7ded3 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 30 Jul 2026 13:35:18 -0700 Subject: [PATCH 45/64] Handle TE depedencies, re-enforce naming convention --- .../sana/tests/parity_check/README.md | 58 +++++++++++++------ integrations/sana/tests/parity_check/bench.sh | 57 +++++++++++++++--- ...nges.patch => changes_bidirectional.patch} | 0 .../sana/tests/parity_check/pyproject.toml | 6 ++ .../tests/parity_check/run_bidirectional.sh | 2 +- integrations/sana/tests/parity_check/uv.lock | 49 +++++++++++++--- 6 files changed, 137 insertions(+), 35 deletions(-) rename integrations/sana/tests/parity_check/{changes.patch => changes_bidirectional.patch} (100%) diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index 0476d36c9..7e613ed82 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -9,16 +9,28 @@ This harness compares upstream [`NVlabs/Sana`](https://github.com/NVlabs/Sana) `SANA-WM_bidirectional` and `SANA-WM_streaming` against the matching in-tree FlashDreams integrations on matched demo inputs, seeds, resolution, and precision settings. `run.sh` -dispatches to separate bidirectional and streaming parity runners. `bench.sh` -supports bidirectional by default and streaming when -`SANA_WM_VARIANT=streaming` is set. +dispatches to separate bidirectional and streaming parity runners. By default, +`bench.sh` runs both benchmark comparisons; set +`SANA_WM_VARIANT=bidirectional` or `SANA_WM_VARIANT=streaming` to run only one +variant. All dependencies live in this directory's isolated `./.venv`. `run.sh` and -`bench.sh` reuse or clone the upstream checkout, pin it to `6298508`, and apply -instrumentation patches idempotently. `changes.patch` covers the bidirectional -entrypoint. `changes_streaming.patch` covers the streaming entrypoint metadata. -The patches do not change generation algorithms. A small `compat/mmcv` shim -covers the registry/logging imports needed by the inference path. +`bench.sh` run `uv sync` for that environment, reuse or clone the upstream +checkout, pin it to `6298508`, and apply instrumentation patches idempotently. +`changes_bidirectional.patch` covers the bidirectional entrypoint. +`changes_streaming.patch` covers the streaming entrypoint metadata. The patches +do not change generation algorithms. A small `compat/mmcv` shim covers the +registry/logging imports needed by the inference path. + +The BF16 path uses the base isolated environment. FP8 and FP4 streaming +benchmarks opt into the `quant` extra, which builds TransformerEngine from the +pinned git source against the local CUDA/PyTorch stack and explicitly installs +the ONNX runtime packages TransformerEngine imports. `bench.sh` also seeds +`setuptools`, `wheel`, and `pybind11` into `./.venv` before that build because +upstream TransformerEngine does not declare all metadata/build requirements. A +new checkout therefore needs network access for `uv sync`, GitHub access for +the pinned upstream checkout and TransformerEngine source, and a CUDA build +toolchain for low-precision runs. ## Run parity @@ -76,18 +88,18 @@ every decoded frame and checks both frame parity and chunk-boundary continuity. debug-friendly; set `STREAMING_NO_COMPILE=0` when explicitly checking compiled streaming behavior. -## Run SANA-WM_bidirectional benchmark +## Run SANA-WM benchmarks ```bash cd integrations/sana/tests/parity_check bash bench.sh ``` -Benchmark defaults to the `SANA-WM_bidirectional` comparison and discards one -warmup run before measuring three additional runs: -`SANA_WM_VARIANT=bidirectional BENCH_SIDE=both WARMUP_RUNS=1 MEASURED_RUNS=3 -COMPILE_STAGE1=0 NO_REFINER=0 FORCE_CUDNN_SDPA=0`. Outputs are under -`outputs/bench/`: +Benchmark defaults to both sibling variants and discards one warmup run before +measuring three additional runs: +`SANA_WM_VARIANT=both BENCH_SIDE=both WARMUP_RUNS=1 MEASURED_RUNS=3 +COMPILE_STAGE1=0 NO_REFINER=0 FORCE_CUDNN_SDPA=0`. Bidirectional outputs are +under `outputs/bench/`: - `bench.json` - machine-readable inputs, medians, p90s, memory, and stage timings. @@ -117,7 +129,7 @@ implementation as the series columns, for example: Bidirectional upstream comparisons are BF16-only because upstream `SANA-WM_bidirectional` does not support FP8/FP4 precision flags. Use -`outputs/bench/perf.md` from a BF16 bidirectional run as the model-card chart +`outputs/bench/perf.md` from the BF16 bidirectional run as the model-card chart data. Set `FORCE_CUDNN_SDPA=1` only for backend-isolation probes. It is not the @@ -129,10 +141,10 @@ Set `DEVICE_LABEL` when generating chart data for docs: DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh ``` -Run the BF16 benchmark used by the `SANA-WM_bidirectional` model card with: +Run only the BF16 benchmark used by the `SANA-WM_bidirectional` model card with: ```bash -DEVICE_LABEL="GB202" bash bench.sh +SANA_WM_VARIANT=bidirectional DEVICE_LABEL="GB202" bash bench.sh ``` The scripts do not set allocator overrides or GPU wait loops. If GPU contention @@ -140,8 +152,8 @@ matters in your environment, handle it outside the harness. ## Run SANA-WM_streaming benchmark -Use `SANA_WM_VARIANT=streaming` to compare upstream `SANA-WM_streaming` with -the FlashDreams `sana-wm-streaming` runner: +Use `SANA_WM_VARIANT=streaming` to compare only upstream `SANA-WM_streaming` +with the FlashDreams `sana-wm-streaming` runner: ```bash cd integrations/sana/tests/parity_check @@ -181,6 +193,14 @@ Run the comparison precision sweep in order with: SANA_WM_VARIANT=streaming BENCH_SIDE=both BENCH_PRECISIONS=bf16,fp8,fp4 bash bench.sh ``` +Omit `SANA_WM_VARIANT` to regenerate both model-card datasets in one command. +The bidirectional benchmark stays BF16-only, and the streaming benchmark uses +the requested precision sweep: + +```bash +BENCH_SIDE=both BENCH_PRECISIONS=bf16,fp8,fp4 DEVICE_LABEL="GB202" bash bench.sh +``` + FP8 and FP4 are passed through upstream streaming-only `--stage1_precision`/`--refiner_precision` flags, and those precision runs sync the isolated venv with the `quant` extra before launch. If Transformer Engine, diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index b23f5fa4e..82f33e387 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -14,10 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Stack-matched SANA-WM benchmark harness. By default this runs the -# bidirectional upstream-vs-FlashDreams clip-latency comparison. Set -# SANA_WM_VARIANT=streaming to compare upstream streaming with the FlashDreams -# streaming integration. +# Stack-matched SANA-WM benchmark harness. By default this runs both sibling +# SANA-WM variants. Set SANA_WM_VARIANT=bidirectional or +# SANA_WM_VARIANT=streaming to run only one comparison. set -euo pipefail @@ -60,17 +59,17 @@ _apply_patch_once() { } SANA_REPO="$(_abspath "${SANA_REPO:-${SCRIPT_DIR}/Sana}")" -PATCH_FILE="${SCRIPT_DIR}/changes.patch" +PATCH_FILE="${SCRIPT_DIR}/changes_bidirectional.patch" STREAMING_PATCH_FILE="${SCRIPT_DIR}/changes_streaming.patch" REPO_URL="https://github.com/NVlabs/Sana.git" PIN_COMMIT="6298508" -SANA_WM_VARIANT="${SANA_WM_VARIANT:-bidirectional}" +SANA_WM_VARIANT="${SANA_WM_VARIANT:-both}" BENCH_SIDE="${BENCH_SIDE:-both}" case "${SANA_WM_VARIANT}" in - bidirectional|streaming) ;; + both|bidirectional|streaming) ;; *) - echo "[bench] ERROR: SANA_WM_VARIANT must be bidirectional or streaming; got ${SANA_WM_VARIANT}" >&2 + echo "[bench] ERROR: SANA_WM_VARIANT must be both, bidirectional, or streaming; got ${SANA_WM_VARIANT}" >&2 exit 1 ;; esac @@ -92,6 +91,44 @@ WARMUP_RUNS="${WARMUP_RUNS:-1}" MEASURED_RUNS="${MEASURED_RUNS:-3}" BENCH_DRY_RUN="${BENCH_DRY_RUN:-0}" +if [[ "${SANA_WM_VARIANT}" == "both" ]]; then + BIDIRECTIONAL_OUTPUT_ENV=() + STREAMING_OUTPUT_ENV=() + if [[ -n "${OUTPUT_DIR:-}" ]]; then + OUTPUT_BASE="$(_abspath "${OUTPUT_DIR}")" + BIDIRECTIONAL_OUTPUT_ENV=(OUTPUT_DIR="${OUTPUT_BASE}/bidirectional") + if [[ -n "${BENCH_PRECISIONS}" ]]; then + STREAMING_OUTPUT_ENV=(OUTPUT_DIR="${OUTPUT_BASE}/streaming") + else + STREAMING_OUTPUT_ENV=(OUTPUT_DIR="${OUTPUT_BASE}/streaming/${STAGE1_PRECISION}") + fi + fi + + STREAMING_PRECISION_ENV=(BENCH_PRECISIONS=) + if [[ -n "${BENCH_PRECISIONS}" ]]; then + STREAMING_PRECISION_ENV=(BENCH_PRECISIONS="${BENCH_PRECISIONS}") + fi + + echo "[bench] running SANA-WM_bidirectional benchmark" + env \ + SANA_WM_VARIANT=bidirectional \ + BENCH_PRECISIONS= \ + STAGE1_PRECISION=bf16 \ + REFINER_PRECISION=bf16 \ + "${BIDIRECTIONAL_OUTPUT_ENV[@]}" \ + bash "${SCRIPT_DIR}/bench.sh" + + echo "[bench] running SANA-WM_streaming benchmark" + env \ + SANA_WM_VARIANT=streaming \ + "${STREAMING_PRECISION_ENV[@]}" \ + "${STREAMING_OUTPUT_ENV[@]}" \ + bash "${SCRIPT_DIR}/bench.sh" + + echo "[bench] all requested SANA-WM benchmarks done." + exit 0 +fi + _reject_bidirectional_low_precision() { echo "[bench] ERROR: upstream SANA-WM_bidirectional benchmarks are BF16-only." >&2 echo " FP8 and FP4 precision flags are only supported by upstream SANA-WM_streaming." >&2 @@ -279,6 +316,10 @@ if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; th fi fi echo "[setup] TransformerEngine build env: NVTE_FRAMEWORK=${NVTE_FRAMEWORK} NVTE_CUDA_ARCHS=${NVTE_CUDA_ARCHS:-default} NVTE_WITH_NCCL_EP=${NVTE_WITH_NCCL_EP}" + echo "[setup] seeding TransformerEngine build tools in isolated venv" + ( cd "${SCRIPT_DIR}" && \ + uv venv --allow-existing --python "3.12" .venv >/dev/null && \ + uv pip install --python .venv/bin/python setuptools wheel pybind11 ) fi echo "[setup] ensuring Python deps via ${UV_SYNC_ARGS[*]} (isolated venv)" ( cd "${SCRIPT_DIR}" && "${UV_SYNC_ARGS[@]}" ) diff --git a/integrations/sana/tests/parity_check/changes.patch b/integrations/sana/tests/parity_check/changes_bidirectional.patch similarity index 100% rename from integrations/sana/tests/parity_check/changes.patch rename to integrations/sana/tests/parity_check/changes_bidirectional.patch diff --git a/integrations/sana/tests/parity_check/pyproject.toml b/integrations/sana/tests/parity_check/pyproject.toml index eab2cbdd0..5b5a3f788 100644 --- a/integrations/sana/tests/parity_check/pyproject.toml +++ b/integrations/sana/tests/parity_check/pyproject.toml @@ -40,7 +40,13 @@ dependencies = [ # try to build/import it. The git source avoids split-wheel ABI mismatches on # bleeding-edge torch/CUDA stacks while keeping BF16 setup lightweight. quant = [ + "ml-dtypes>=0.5", + "nvdlfw-inspect>=0.2", + "onnx>=1.22", + "onnxscript>=0.7", + "pybind11>=3.0", "transformer-engine==2.17.0+2e559f06; sys_platform != 'win32'", + "wheel>=0.45", ] [tool.uv.sources] diff --git a/integrations/sana/tests/parity_check/run_bidirectional.sh b/integrations/sana/tests/parity_check/run_bidirectional.sh index ed5b75292..67faf566d 100644 --- a/integrations/sana/tests/parity_check/run_bidirectional.sh +++ b/integrations/sana/tests/parity_check/run_bidirectional.sh @@ -23,7 +23,7 @@ set -euo pipefail source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/run_common.sh" SANA_REPO="$(parity_abspath "${SANA_REPO:-${HOME}/dev/Sana}")" -PATCH_FILE="${PARITY_SCRIPT_DIR}/changes.patch" +PATCH_FILE="${PARITY_SCRIPT_DIR}/changes_bidirectional.patch" OUTPUT_DIR="$(parity_abspath "${OUTPUT_DIR:-${PARITY_SCRIPT_DIR}/outputs/parity}")" UPSTREAM_OUT="${OUTPUT_DIR}/upstream" diff --git a/integrations/sana/tests/parity_check/uv.lock b/integrations/sana/tests/parity_check/uv.lock index e982ddc53..af4d99550 100644 --- a/integrations/sana/tests/parity_check/uv.lock +++ b/integrations/sana/tests/parity_check/uv.lock @@ -342,6 +342,7 @@ requires-dist = [ { name = "mediapy", marker = "extra == 'runners'", specifier = ">=1.1" }, { name = "numpy", specifier = ">=1.24,<2.5" }, { name = "nvidia-ml-py", specifier = ">=12.0" }, + { name = "nvidia-vfx", marker = "extra == 'rtx-postprocess'", specifier = "==0.1.0.1" }, { name = "opencv-python-headless", marker = "extra == 'examples'", specifier = ">=4.5" }, { name = "opencv-python-headless", marker = "extra == 'runners'", specifier = ">=4.5" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, @@ -359,7 +360,7 @@ requires-dist = [ { name = "tyro", specifier = ">=1.0" }, { name = "urllib3", specifier = ">=2.7.0" }, ] -provides-extras = ["dev", "examples", "runners", "serving"] +provides-extras = ["dev", "examples", "runners", "rtx-postprocess", "serving"] [package.metadata.requires-dev] cuda12 = [ @@ -642,6 +643,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/b8/3c70881695e056f8a32f8b941126cf78775d9a4d7feba8abcb52cb7b04f2/ml_dtypes-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac", size = 676927, upload-time = "2025-11-17T22:31:48.182Z" }, { url = "https://files.pythonhosted.org/packages/54/0f/428ef6881782e5ebb7eca459689448c0394fa0a80bea3aa9262cba5445ea/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900", size = 5028464, upload-time = "2025-11-17T22:31:50.135Z" }, { url = "https://files.pythonhosted.org/packages/3a/cb/28ce52eb94390dda42599c98ea0204d74799e4d8047a0eb559b6fd648056/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff", size = 5009002, upload-time = "2025-11-17T22:31:52.001Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f0/0cfadd537c5470378b1b32bd859cf2824972174b51b873c9d95cfd7475a5/ml_dtypes-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7", size = 212222, upload-time = "2025-11-17T22:31:53.742Z" }, + { url = "https://files.pythonhosted.org/packages/16/2e/9acc86985bfad8f2c2d30291b27cd2bb4c74cea08695bd540906ed744249/ml_dtypes-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460", size = 160793, upload-time = "2025-11-17T22:31:55.358Z" }, ] [[package]] @@ -696,7 +699,8 @@ version = "0.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, - { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'win32'" }, + { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "sys_platform == 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8a/86/94188e03e5d4dd7b73c390b0cddcde5618b3799c18e327b2bf15763f6137/nvdlfw_inspect-0.2.2-py3-none-any.whl", hash = "sha256:8a4dc2814c5a4cd19ae304170b9bfa514538ef3c3eb243a45a82404ec3cb279d", size = 30964, upload-time = "2025-12-03T10:52:01.933Z" }, @@ -892,6 +896,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/55/b34fc2aa30aa54b4a775402d24c4082242c720283a274fe976ac8eb94480/onnx-1.22.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae5a563f281cd9d2845622cecf6c092a57e4ee1b138f66fdbbdd4200567a5e16", size = 18889249, upload-time = "2026-06-15T12:49:34.7Z" }, { url = "https://files.pythonhosted.org/packages/09/a6/bd32357e6cc1ecb473afd78193d7231724f284435d2db25696ecfaaa1503/onnx-1.22.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:955e02e1f6d385b53d52f9cd7b9cdf5caf417c300bcfe3c64c6d542be763845b", size = 19106514, upload-time = "2026-06-15T12:49:37.424Z" }, { url = "https://files.pythonhosted.org/packages/5a/9d/3af461ac6c714b8b369cb71499659932f4f12cfb066250b62f7567c3d530/onnx-1.22.0-cp312-abi3-pyemscripten_2025_0_wasm32.whl", hash = "sha256:82e9f27fc1223cb06d68a56bed6f9d3caf3d0dad1b61bce45006d529b15bd94c", size = 16966387, upload-time = "2026-06-15T12:49:40.918Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f0/68195b5e5a53e333faf2660f5352ee43738d0e42fc5216cc6b1871a9fbfb/onnx-1.22.0-cp312-abi3-win32.whl", hash = "sha256:cc8b66b312f8f03a53e268afb67180a2d97dd12cc79e2b61361c6c0073448016", size = 17081568, upload-time = "2026-06-15T12:49:43.398Z" }, + { url = "https://files.pythonhosted.org/packages/13/a8/734725bb703c5fabb687f79c79e51249475212b3eb37771ac4a4ac9b487f/onnx-1.22.0-cp312-abi3-win_amd64.whl", hash = "sha256:72ccebab3bac07215c204ce8848d42e78eaaa666badbf72d25cd359b9f269e3a", size = 17213290, upload-time = "2026-06-15T12:49:45.933Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2a/8ce48d8ae26a8761ad4e5dc771961b155c5c3c7c8540ec7f2f2d71b69af0/onnx-1.22.0-cp312-abi3-win_arm64.whl", hash = "sha256:f3c120dcdb70ad738f3c061b32798f408ea299eb69f84dd69ab4a6bf3c2ec01f", size = 17207030, upload-time = "2026-06-15T12:49:48.635Z" }, ] [[package]] @@ -1003,6 +1010,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, ] +[[package]] +name = "pybind11" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/f0/35145a3c3baffeef55d4b8324caa33abaa8fa56ab345ecd4b2211d09163e/pybind11-3.0.4.tar.gz", hash = "sha256:3286b59c8a774b9ee650169302dd5a4eedc30a8617905a0560dd8ee44775130c", size = 589533, upload-time = "2026-04-19T03:08:15.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/06/c3a23c9a0263b136c519f033a58d4641e73065fefc7754e9667ec206d992/pybind11-3.0.4-py3-none-any.whl", hash = "sha256:961720ee652da51d531b7b2451a6bd2bc042b0106e6d9baa48ecb7d58034ce63", size = 314166, upload-time = "2026-04-19T03:08:14.091Z" }, +] + [[package]] name = "pydantic" version = "2.13.4" @@ -1247,7 +1263,13 @@ dependencies = [ [package.optional-dependencies] quant = [ + { name = "ml-dtypes" }, + { name = "nvdlfw-inspect" }, + { name = "onnx" }, + { name = "onnxscript" }, + { name = "pybind11" }, { name = "transformer-engine", marker = "sys_platform != 'win32'" }, + { name = "wheel" }, ] [package.metadata] @@ -1261,11 +1283,16 @@ requires-dist = [ { name = "ftfy", specifier = ">=6.0" }, { name = "huggingface-hub", specifier = ">=0.36" }, { name = "imageio", extras = ["ffmpeg"], specifier = ">=2.31" }, + { name = "ml-dtypes", marker = "extra == 'quant'", specifier = ">=0.5" }, { name = "numpy", specifier = ">=1.24,<2.5" }, + { name = "nvdlfw-inspect", marker = "extra == 'quant'", specifier = ">=0.2" }, { name = "omegaconf", specifier = ">=2.3" }, + { name = "onnx", marker = "extra == 'quant'", specifier = ">=1.22" }, + { name = "onnxscript", marker = "extra == 'quant'", specifier = ">=0.7" }, { name = "opencv-python-headless", specifier = ">=4.8" }, { name = "pillow", specifier = ">=10" }, { name = "protobuf", specifier = ">=7.35.0,<8" }, + { name = "pybind11", marker = "extra == 'quant'", specifier = ">=3.0" }, { name = "pyrallis", specifier = ">=0.3" }, { name = "pytz", specifier = ">=2024.0" }, { name = "pyyaml", specifier = ">=6.0" }, @@ -1281,6 +1308,7 @@ requires-dist = [ { name = "transformer-engine", marker = "sys_platform != 'win32' and extra == 'quant'", git = "https://github.com/NVIDIA/TransformerEngine.git?tag=v2.17" }, { name = "transformers", specifier = ">=5.0,<6" }, { name = "triton", marker = "sys_platform == 'linux'", specifier = ">=3.6" }, + { name = "wheel", marker = "extra == 'quant'", specifier = ">=0.45" }, ] provides-extras = ["quant"] @@ -1494,14 +1522,9 @@ name = "transformer-engine" version = "2.17.0+2e559f06" source = { git = "https://github.com/NVIDIA/TransformerEngine.git?tag=v2.17#2e559f062497bef768dfbe9d7e45548fadeca80a" } dependencies = [ - { name = "einops" }, { name = "importlib-metadata" }, - { name = "nvdlfw-inspect" }, - { name = "onnx" }, - { name = "onnxscript" }, { name = "packaging" }, { name = "pydantic" }, - { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, ] [[package]] @@ -1634,6 +1657,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl", hash = "sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85", size = 323166, upload-time = "2026-06-29T18:11:09.888Z" }, ] +[[package]] +name = "wheel" +version = "0.47.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/62/75f18a0f03b4219c456652c7780e4d749b929eb605c098ce3a5b6b6bc081/wheel-0.47.0.tar.gz", hash = "sha256:cc72bd1009ba0cf63922e28f94d9d83b920aa2bb28f798a31d0691b02fa3c9b3", size = 63854, upload-time = "2026-04-22T15:51:27.727Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/1b/9e33c09813d65e248f7f773119148a612516a4bea93e9c6f545f78455b7c/wheel-0.47.0-py3-none-any.whl", hash = "sha256:212281cab4dff978f6cedd499cd893e1f620791ca6ff7107cf270781e587eced", size = 32218, upload-time = "2026-04-22T15:51:26.296Z" }, +] + [[package]] name = "win32-setctime" version = "1.2.0" From 22347c29e5ac8b7a11a468b7070bcfb75349652a Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 30 Jul 2026 16:46:06 -0700 Subject: [PATCH 46/64] Run benchmark runs in proper sequence --- integrations/sana/tests/parity_check/bench.sh | 151 ++++++++++-------- .../sana/tests/parity_check/bench_summary.py | 2 + 2 files changed, 87 insertions(+), 66 deletions(-) diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index 82f33e387..ef5892b89 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -396,25 +396,29 @@ if [[ "${SANA_WM_VARIANT}" == "streaming" ]] && ! _is_true "${STREAMING_NO_COMPI fi TOTAL_RUNS=$(( WARMUP_RUNS + MEASURED_RUNS )) -for ((i = 0; i < TOTAL_RUNS; i++)); do - UPSTREAM_OUT="${UPSTREAM_ROOT}/run_${i}" - mkdir -p "${UPSTREAM_OUT}" + +_run_upstream_once() { + local i="$1" + local upstream_out="${UPSTREAM_ROOT}/run_${i}" + local -a upstream_cmd + + mkdir -p "${upstream_out}" if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then - UPSTREAM_CMD=( + upstream_cmd=( env "PYTHONPATH=${UPSTREAM_PYTHONPATH}" uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm_streaming.py" --image "${IMAGE_PATH}" --prompt "${PROMPT_PATH}" "${UPSTREAM_STREAMING_CAMERA_ARGS[@]}" --intrinsics "${INTRINSICS_PATH}" - --output_dir "${UPSTREAM_OUT}" + --output_dir "${upstream_out}" --name upstream --num_frames "${NUM_FRAMES}" --fps "${FPS}" --cfg_scale "${CFG_SCALE}" --seed "${SEED}" - --benchmark_json "${UPSTREAM_OUT}/stats.json" + --benchmark_json "${upstream_out}/stats.json" --denoising_step_list "${STREAMING_DENOISING_STEP_LIST}" --num_frame_per_block "${STREAMING_NUM_FRAME_PER_BLOCK}" --refiner_block_size "${STREAMING_REFINER_BLOCK_SIZE}" @@ -429,14 +433,14 @@ for ((i = 0; i < TOTAL_RUNS; i++)); do "${UPSTREAM_STREAMING_MODE_ARGS[@]}" ) else - UPSTREAM_CMD=( + upstream_cmd=( env "PYTHONPATH=${UPSTREAM_PYTHONPATH}" uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm.py" --image "${IMAGE_PATH}" --prompt "${PROMPT_PATH}" --camera "${CAMERA_PATH}" --intrinsics "${INTRINSICS_PATH}" - --output_dir "${UPSTREAM_OUT}" + --output_dir "${upstream_out}" --name upstream --num_frames "${NUM_FRAMES}" --fps "${FPS}" @@ -444,7 +448,7 @@ for ((i = 0; i < TOTAL_RUNS; i++)); do --cfg_scale "${CFG_SCALE}" --seed "${SEED}" --no_action_overlay - --stats_json "${UPSTREAM_OUT}/stats.json" + --stats_json "${upstream_out}/stats.json" "${UPSTREAM_PRECISION_ARGS[@]}" "${UPSTREAM_BACKEND_ARGS[@]}" "${UPSTREAM_COMPILE_ARGS[@]}" @@ -453,66 +457,81 @@ for ((i = 0; i < TOTAL_RUNS; i++)); do fi echo "[bench] upstream ${SANA_WM_VARIANT} run ${i}/${TOTAL_RUNS}" - _write_command "${UPSTREAM_OUT}/command.txt" "${UPSTREAM_CMD[@]}" - ( cd "${SCRIPT_DIR}" && "${UPSTREAM_CMD[@]}" ) - - if [[ "${BENCH_SIDE}" == "both" ]]; then - FLASHDREAMS_OUT="${FLASHDREAMS_ROOT}/run_${i}" - mkdir -p "${FLASHDREAMS_OUT}" - if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then - FLASHDREAMS_CMD=( - uv run python "${SCRIPT_DIR}/run_flashdreams_streaming.py" - --image-path "${IMAGE_PATH}" - --prompt-path "${PROMPT_PATH}" - "${FLASHDREAMS_STREAMING_CAMERA_ARGS[@]}" - --intrinsics-path "${INTRINSICS_PATH}" - --output-dir "${FLASHDREAMS_OUT}" - --name flashdreams - --num-frames "${NUM_FRAMES}" - --fps "${FPS}" - --cfg-scale "${CFG_SCALE}" - --flow-shift "8.0" - --seed "${SEED}" - --refiner-seed "${STREAMING_REFINER_SEED}" - --stats-json "${FLASHDREAMS_OUT}/stats.json" - --denoising-step-list "${STREAMING_DENOISING_STEP_LIST}" - --num-frame-per-block "${STREAMING_NUM_FRAME_PER_BLOCK}" - --num-cached-blocks "${STREAMING_NUM_CACHED_BLOCKS}" - --sink-size "${STREAMING_SINK_SIZE}" - --refiner-block-size "${STREAMING_REFINER_BLOCK_SIZE}" - --refiner-kv-max-frames "${STREAMING_REFINER_KV_MAX_FRAMES}" - "${FLASHDREAMS_STREAMING_MODE_ARGS[@]}" - "${FLASHDREAMS_STREAMING_COMPILE_ARGS[@]}" - "${FLASHDREAMS_PRECISION_ARGS[@]}" - "${FLASHDREAMS_BACKEND_ARGS[@]}" - ) - else - FLASHDREAMS_CMD=( - uv run python "${SCRIPT_DIR}/run_flashdreams_bidirectional.py" - --image-path "${IMAGE_PATH}" - --prompt-path "${PROMPT_PATH}" - --camera-path "${CAMERA_PATH}" - --intrinsics-path "${INTRINSICS_PATH}" - --output-dir "${FLASHDREAMS_OUT}" - --name flashdreams - --num-frames "${NUM_FRAMES}" - --fps "${FPS}" - --step "${STEP}" - --cfg-scale "${CFG_SCALE}" - --seed "${SEED}" - --stats-json "${FLASHDREAMS_OUT}/stats.json" - "${FLASHDREAMS_PRECISION_ARGS[@]}" - "${FLASHDREAMS_BACKEND_ARGS[@]}" - "${FLASHDREAMS_COMPILE_ARGS[@]}" - "${FLASHDREAMS_REFINER_ARGS[@]}" - ) - fi - echo "[bench] FlashDreams ${SANA_WM_VARIANT} run ${i}/${TOTAL_RUNS}" - _write_command "${FLASHDREAMS_OUT}/command.txt" "${FLASHDREAMS_CMD[@]}" - ( cd "${SCRIPT_DIR}" && "${FLASHDREAMS_CMD[@]}" ) + _write_command "${upstream_out}/command.txt" "${upstream_cmd[@]}" + ( cd "${SCRIPT_DIR}" && "${upstream_cmd[@]}" ) +} + +_run_flashdreams_once() { + local i="$1" + local flashdreams_out="${FLASHDREAMS_ROOT}/run_${i}" + local -a flashdreams_cmd + + mkdir -p "${flashdreams_out}" + + if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then + flashdreams_cmd=( + uv run python "${SCRIPT_DIR}/run_flashdreams_streaming.py" + --image-path "${IMAGE_PATH}" + --prompt-path "${PROMPT_PATH}" + "${FLASHDREAMS_STREAMING_CAMERA_ARGS[@]}" + --intrinsics-path "${INTRINSICS_PATH}" + --output-dir "${flashdreams_out}" + --name flashdreams + --num-frames "${NUM_FRAMES}" + --fps "${FPS}" + --cfg-scale "${CFG_SCALE}" + --flow-shift "8.0" + --seed "${SEED}" + --refiner-seed "${STREAMING_REFINER_SEED}" + --stats-json "${flashdreams_out}/stats.json" + --denoising-step-list "${STREAMING_DENOISING_STEP_LIST}" + --num-frame-per-block "${STREAMING_NUM_FRAME_PER_BLOCK}" + --num-cached-blocks "${STREAMING_NUM_CACHED_BLOCKS}" + --sink-size "${STREAMING_SINK_SIZE}" + --refiner-block-size "${STREAMING_REFINER_BLOCK_SIZE}" + --refiner-kv-max-frames "${STREAMING_REFINER_KV_MAX_FRAMES}" + "${FLASHDREAMS_STREAMING_MODE_ARGS[@]}" + "${FLASHDREAMS_STREAMING_COMPILE_ARGS[@]}" + "${FLASHDREAMS_PRECISION_ARGS[@]}" + "${FLASHDREAMS_BACKEND_ARGS[@]}" + ) + else + flashdreams_cmd=( + uv run python "${SCRIPT_DIR}/run_flashdreams_bidirectional.py" + --image-path "${IMAGE_PATH}" + --prompt-path "${PROMPT_PATH}" + --camera-path "${CAMERA_PATH}" + --intrinsics-path "${INTRINSICS_PATH}" + --output-dir "${flashdreams_out}" + --name flashdreams + --num-frames "${NUM_FRAMES}" + --fps "${FPS}" + --step "${STEP}" + --cfg-scale "${CFG_SCALE}" + --seed "${SEED}" + --stats-json "${flashdreams_out}/stats.json" + "${FLASHDREAMS_PRECISION_ARGS[@]}" + "${FLASHDREAMS_BACKEND_ARGS[@]}" + "${FLASHDREAMS_COMPILE_ARGS[@]}" + "${FLASHDREAMS_REFINER_ARGS[@]}" + ) fi + + echo "[bench] FlashDreams ${SANA_WM_VARIANT} run ${i}/${TOTAL_RUNS}" + _write_command "${flashdreams_out}/command.txt" "${flashdreams_cmd[@]}" + ( cd "${SCRIPT_DIR}" && "${flashdreams_cmd[@]}" ) +} + +for ((i = 0; i < TOTAL_RUNS; i++)); do + _run_upstream_once "${i}" done +if [[ "${BENCH_SIDE}" == "both" ]]; then + for ((i = 0; i < TOTAL_RUNS; i++)); do + _run_flashdreams_once "${i}" + done +fi + SUMMARY_JSON="${OUTPUT_DIR}/bench.json" SUMMARY_MD="${OUTPUT_DIR}/bench.md" SUMMARY_CHART_MD="${OUTPUT_DIR}/perf.md" diff --git a/integrations/sana/tests/parity_check/bench_summary.py b/integrations/sana/tests/parity_check/bench_summary.py index 488522d35..64ba10718 100644 --- a/integrations/sana/tests/parity_check/bench_summary.py +++ b/integrations/sana/tests/parity_check/bench_summary.py @@ -374,6 +374,7 @@ def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: "", "The chart metric is post-load generation latency per generated clip.", "Model construction, checkpoint loading, video writing, and frame dumps are outside this timing boundary.", + "Each sample is a separate Python process; discarded warmup samples prime process-external state, not a live model instance.", "SANA-WM renders each requested bidirectional clip in one generation pass, not as independently timed frames.", refiner_scope, "", @@ -477,6 +478,7 @@ def _render_streaming_markdown(summary: dict[str, Any]) -> str: "", "The headline metric is steady-state generation latency per produced chunk.", "Warmup runs and the first decoded chunk are excluded so cold compile and cache-fill do not enter the headline.", + "Each sample is a separate Python process; discarded warmup samples prime process-external state, not a live model instance.", "Full-clip wall time is retained as supporting data.", "", ] From 38e84be4e1369731a00cf2214b954540fa0d424c Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 30 Jul 2026 16:46:44 -0700 Subject: [PATCH 47/64] Update model card data --- .../performance/sana_wm_bidirectional/perf-bf16-gb202.md | 2 +- .../performance/sana_wm_streaming/perf-bf16-gb202.md | 2 +- .../_static/performance/sana_wm_streaming/perf-fp4-gb202.md | 2 +- .../_static/performance/sana_wm_streaming/perf-fp8-gb202.md | 2 +- docs/source/models/sana_wm_bidirectional.rst | 2 +- docs/source/models/sana_wm_streaming.rst | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/_static/performance/sana_wm_bidirectional/perf-bf16-gb202.md b/docs/source/_static/performance/sana_wm_bidirectional/perf-bf16-gb202.md index ba71900dc..2eb8b9790 100644 --- a/docs/source/_static/performance/sana_wm_bidirectional/perf-bf16-gb202.md +++ b/docs/source/_static/performance/sana_wm_bidirectional/perf-bf16-gb202.md @@ -2,4 +2,4 @@ | device | official | flashdreams | | --- | ---: | ---: | -| GB202 | 107868.83 | 98432.36 | +| GB202 | 101874.59 | 96353.88 | diff --git a/docs/source/_static/performance/sana_wm_streaming/perf-bf16-gb202.md b/docs/source/_static/performance/sana_wm_streaming/perf-bf16-gb202.md index 0b6442046..26c33ec84 100644 --- a/docs/source/_static/performance/sana_wm_streaming/perf-bf16-gb202.md +++ b/docs/source/_static/performance/sana_wm_streaming/perf-bf16-gb202.md @@ -2,4 +2,4 @@ | device | official | flashdreams | | --- | ---: | ---: | -| GB202 | 1846.87 | 7084.36 | +| GB202 | 1849.65 | 3660.15 | diff --git a/docs/source/_static/performance/sana_wm_streaming/perf-fp4-gb202.md b/docs/source/_static/performance/sana_wm_streaming/perf-fp4-gb202.md index f3d1e5643..603e6580c 100644 --- a/docs/source/_static/performance/sana_wm_streaming/perf-fp4-gb202.md +++ b/docs/source/_static/performance/sana_wm_streaming/perf-fp4-gb202.md @@ -2,4 +2,4 @@ | device | official | flashdreams | | --- | ---: | ---: | -| GB202 | 1420.96 | 16182.13 | +| GB202 | 1308.22 | 6520.72 | diff --git a/docs/source/_static/performance/sana_wm_streaming/perf-fp8-gb202.md b/docs/source/_static/performance/sana_wm_streaming/perf-fp8-gb202.md index 8889c97f3..46c703500 100644 --- a/docs/source/_static/performance/sana_wm_streaming/perf-fp8-gb202.md +++ b/docs/source/_static/performance/sana_wm_streaming/perf-fp8-gb202.md @@ -2,4 +2,4 @@ | device | official | flashdreams | | --- | ---: | ---: | -| GB202 | 1635.46 | 8964.06 | +| GB202 | 1552.48 | 3993.07 | diff --git a/docs/source/models/sana_wm_bidirectional.rst b/docs/source/models/sana_wm_bidirectional.rst index bb0e81d67..6f20d69c6 100644 --- a/docs/source/models/sana_wm_bidirectional.rst +++ b/docs/source/models/sana_wm_bidirectional.rst @@ -124,7 +124,7 @@ Profiling benchmark The BF16 chart below compares post-load generation latency per generated clip for FlashDreams ``SANA-WM_bidirectional`` and the official ``SANA-WM_bidirectional`` implementation under matched settings. On GB202, -FlashDreams measured 98,432.36 ms per clip versus 107,868.83 ms for the +FlashDreams measured 96,353.88 ms per clip versus 101,874.59 ms for the official implementation. In this chart, ``Official Impl`` means the pinned NVlabs/Sana upstream diff --git a/docs/source/models/sana_wm_streaming.rst b/docs/source/models/sana_wm_streaming.rst index f264bcb5a..3904a9493 100644 --- a/docs/source/models/sana_wm_streaming.rst +++ b/docs/source/models/sana_wm_streaming.rst @@ -146,7 +146,7 @@ model authors.

BF16 steady-state milliseconds per produced chunk on one NVIDIA GB202 GPU: - official 1,846.87 ms, FlashDreams 7,084.36 ms. + official 1,849.65 ms, FlashDreams 3,660.15 ms.

@@ -162,7 +162,7 @@ model authors.

FP8 steady-state milliseconds per produced chunk on one NVIDIA GB202 GPU: - official 1,635.46 ms, FlashDreams 8,964.06 ms. + official 1,552.48 ms, FlashDreams 3,993.07 ms.

@@ -178,7 +178,7 @@ model authors.

FP4 steady-state milliseconds per produced chunk on one NVIDIA GB202 GPU: - official 1,420.96 ms, FlashDreams 16,182.13 ms. + official 1,308.22 ms, FlashDreams 6,520.72 ms.

From 174e4b50a091e156f8a73d385d031056e459f6c2 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 30 Jul 2026 17:34:56 -0700 Subject: [PATCH 48/64] Benchmark changes/fixes --- docs/source/models/sana_wm_bidirectional.rst | 8 +- .../sana/tests/parity_check/README.md | 57 +-- integrations/sana/tests/parity_check/bench.sh | 102 +++-- .../sana/tests/parity_check/bench_summary.py | 228 ++++++++-- .../tests/parity_check/bench_sweep_summary.py | 139 +++--- .../parity_check/changes_bidirectional.patch | 72 +++- .../sana/tests/parity_check/run_common.sh | 14 +- .../run_flashdreams_bidirectional.py | 93 ++-- .../tests/test_parity_benchmark_summary.py | 396 +++++++++++++++++- 9 files changed, 892 insertions(+), 217 deletions(-) diff --git a/docs/source/models/sana_wm_bidirectional.rst b/docs/source/models/sana_wm_bidirectional.rst index 6f20d69c6..aec7c65ea 100644 --- a/docs/source/models/sana_wm_bidirectional.rst +++ b/docs/source/models/sana_wm_bidirectional.rst @@ -121,8 +121,8 @@ What to expect Profiling benchmark ------------------- -The BF16 chart below compares post-load generation latency per generated clip -for FlashDreams ``SANA-WM_bidirectional`` and the official +The BF16 chart below compares steady-state in-process generation latency per +generated clip for FlashDreams ``SANA-WM_bidirectional`` and the official ``SANA-WM_bidirectional`` implementation under matched settings. On GB202, FlashDreams measured 96,353.88 ms per clip versus 101,874.59 ms for the official implementation. @@ -144,9 +144,9 @@ model authors. >

- This chart shows post-load generation latency per generated clip in milliseconds for a + This chart shows steady-state in-process generation latency per generated clip in milliseconds for a 121-frame full-pipeline BF16 run (Stage-1 DiT + LTX-2 refiner + SANA VAE decode). - The measured row used one NVIDIA GB202 GPU, one discarded warmup run, and three measured runs. + The measured row used one NVIDIA GB202 GPU, one live warmup generation, and three measured generations. Model construction, checkpoint loading, video writing, and frame dumps are outside the timing boundary. The upstream checkout was pinned to commit 6298508.

diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index 7e613ed82..f712c2f4e 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -95,24 +95,29 @@ cd integrations/sana/tests/parity_check bash bench.sh ``` -Benchmark defaults to both sibling variants and discards one warmup run before -measuring three additional runs: +Benchmark defaults to both sibling variants: `SANA_WM_VARIANT=both BENCH_SIDE=both WARMUP_RUNS=1 MEASURED_RUNS=3 -COMPILE_STAGE1=0 NO_REFINER=0 FORCE_CUDNN_SDPA=0`. Bidirectional outputs are -under `outputs/bench/`: +BENCH_PRECISIONS=bf16,fp8,fp4 COMPILE_STAGE1=0 NO_REFINER=0 +FORCE_CUDNN_SDPA=0`. For bidirectional, the warmup/measured counts mean one +live warmup generation and three measured generations inside each measured +process. For streaming, those counts remain process-level warmup/measured runs +because steady state is measured from chunks inside each process. +Bidirectional outputs are under `outputs/bench/`: - `bench.json` - machine-readable inputs, medians, p90s, memory, and stage timings. - `bench.md` - human-readable report. - `perf.md` - chart-ready data using the same benchmark metric as `bench.md`. -The benchmark metric is post-load generation latency per generated clip. Model -construction, checkpoint loading, video writing, and frame dumps are outside the -timing boundary. `SANA-WM_bidirectional` renders each requested clip in one -generation pass, rather than as independently timed frames. With the default -`NO_REFINER=0`, the timed work covers conditioning, Stage-1 DiT, LTX-2 refiner, -and SANA VAE decode. Set `NO_REFINER=1` only for a diagnostic Stage-1 plus SANA -VAE decode benchmark. +The bidirectional benchmark metric is steady-state in-process generation +latency per generated clip. Warmup generations run on the live model and are +excluded from the headline metric. Model construction, checkpoint loading, +video writing, and frame dumps are outside the timing boundary. +`SANA-WM_bidirectional` renders each requested clip in one generation pass, +rather than as independently timed frames. With the default `NO_REFINER=0`, the +timed work covers conditioning, Stage-1 DiT, LTX-2 refiner, and SANA VAE decode. +Set `NO_REFINER=1` only for a diagnostic Stage-1 plus SANA VAE decode +benchmark. `bench.md` also reports Stage-1 DiT, conditioning/encode, VAE decode, optional refiner, memory, and frame-normalized diagnostic breakdowns. Those rows explain @@ -128,9 +133,11 @@ implementation as the series columns, for example: ``` Bidirectional upstream comparisons are BF16-only because upstream -`SANA-WM_bidirectional` does not support FP8/FP4 precision flags. Use -`outputs/bench/perf.md` from the BF16 bidirectional run as the model-card chart -data. +`SANA-WM_bidirectional` does not support FP8/FP4 precision flags. The default +bidirectional precision sweep therefore runs BF16 as upstream-vs-FlashDreams, +then records FP8 and FP4 as FlashDreams-only diagnostic rows. Use +`outputs/bench/bf16/perf.md` from the BF16 bidirectional row as the model-card +chart data. Set `FORCE_CUDNN_SDPA=1` only for backend-isolation probes. It is not the default `SANA-WM_bidirectional` benchmark setting. @@ -144,7 +151,7 @@ DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh Run only the BF16 benchmark used by the `SANA-WM_bidirectional` model card with: ```bash -SANA_WM_VARIANT=bidirectional DEVICE_LABEL="GB202" bash bench.sh +SANA_WM_VARIANT=bidirectional BENCH_PRECISIONS=bf16 DEVICE_LABEL="GB202" bash bench.sh ``` The scripts do not set allocator overrides or GPU wait loops. If GPU contention @@ -190,19 +197,21 @@ passes the equivalent `torch.compile` wrapper to FlashDreams. Set Run the comparison precision sweep in order with: ```bash -SANA_WM_VARIANT=streaming BENCH_SIDE=both BENCH_PRECISIONS=bf16,fp8,fp4 bash bench.sh +SANA_WM_VARIANT=streaming BENCH_SIDE=both bash bench.sh ``` Omit `SANA_WM_VARIANT` to regenerate both model-card datasets in one command. -The bidirectional benchmark stays BF16-only, and the streaming benchmark uses -the requested precision sweep: +The default run uses the BF16/FP8/FP4 precision sweep for streaming and for +FlashDreams-only bidirectional diagnostics, while the official bidirectional +comparison stays BF16-only: ```bash -BENCH_SIDE=both BENCH_PRECISIONS=bf16,fp8,fp4 DEVICE_LABEL="GB202" bash bench.sh +DEVICE_LABEL="GB202" bash bench.sh ``` -FP8 and FP4 are passed through upstream streaming-only -`--stage1_precision`/`--refiner_precision` flags, and those precision runs sync -the isolated venv with the `quant` extra before launch. If Transformer Engine, -hardware, or upstream code rejects a precision, earlier precision outputs remain -in place and the failing command is recorded in the corresponding `command.txt`. +FP8 and FP4 are passed through the precision flags supported by each measured +side. For upstream this means streaming only; for FlashDreams this also includes +bidirectional diagnostic rows. Those precision runs sync the isolated venv with +the `quant` extra before launch. If Transformer Engine, hardware, or upstream +code rejects a precision, earlier precision outputs remain in place and the +failing command is recorded in the corresponding `command.txt`. diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index ef5892b89..76b59157b 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -53,8 +53,18 @@ _apply_patch_once() { echo "[setup] applying ${patch_file}" git apply "${patch_file}" else - echo "[setup] ERROR: ${patch_file} neither cleanly applies nor is already applied." >&2 - exit 1 + echo "[setup] refreshing stale patched targets for ${patch_file}" + while IFS=$'\t' read -r _added _deleted path; do + [[ -n "${path}" ]] || continue + git checkout -- "${path}" + done < <(git apply --numstat "${patch_file}") + if git apply --check "${patch_file}" >/dev/null 2>&1; then + echo "[setup] applying ${patch_file}" + git apply "${patch_file}" + else + echo "[setup] ERROR: ${patch_file} neither cleanly applies nor is already applied." >&2 + exit 1 + fi fi } @@ -74,9 +84,9 @@ case "${SANA_WM_VARIANT}" in ;; esac case "${BENCH_SIDE}" in - upstream|both) ;; + upstream|flashdreams|both) ;; *) - echo "[bench] ERROR: BENCH_SIDE must be upstream or both; got ${BENCH_SIDE}" >&2 + echo "[bench] ERROR: BENCH_SIDE must be upstream, flashdreams, or both; got ${BENCH_SIDE}" >&2 exit 1 ;; esac @@ -86,14 +96,20 @@ STAGE1_PRECISION="${STAGE1_PRECISION:-bf16}" REFINER_PRECISION="${REFINER_PRECISION:-${STAGE1_PRECISION}}" QUANT_BACKEND="${QUANT_BACKEND:-auto}" CHART_LABEL="${CHART_LABEL:-${DEVICE_LABEL}}" -BENCH_PRECISIONS="${BENCH_PRECISIONS:-}" +BENCH_PRECISIONS="${BENCH_PRECISIONS-bf16,fp8,fp4}" WARMUP_RUNS="${WARMUP_RUNS:-1}" MEASURED_RUNS="${MEASURED_RUNS:-3}" +BIDIRECTIONAL_WARMUP_GENERATIONS="${BIDIRECTIONAL_WARMUP_GENERATIONS:-${WARMUP_RUNS}}" +BIDIRECTIONAL_MEASURED_GENERATIONS="${BIDIRECTIONAL_MEASURED_GENERATIONS:-${MEASURED_RUNS}}" BENCH_DRY_RUN="${BENCH_DRY_RUN:-0}" if [[ "${SANA_WM_VARIANT}" == "both" ]]; then BIDIRECTIONAL_OUTPUT_ENV=() STREAMING_OUTPUT_ENV=() + BIDIRECTIONAL_GENERATION_ENV=( + BIDIRECTIONAL_WARMUP_GENERATIONS="${BIDIRECTIONAL_WARMUP_GENERATIONS}" + BIDIRECTIONAL_MEASURED_GENERATIONS="${BIDIRECTIONAL_MEASURED_GENERATIONS}" + ) if [[ -n "${OUTPUT_DIR:-}" ]]; then OUTPUT_BASE="$(_abspath "${OUTPUT_DIR}")" BIDIRECTIONAL_OUTPUT_ENV=(OUTPUT_DIR="${OUTPUT_BASE}/bidirectional") @@ -112,9 +128,10 @@ if [[ "${SANA_WM_VARIANT}" == "both" ]]; then echo "[bench] running SANA-WM_bidirectional benchmark" env \ SANA_WM_VARIANT=bidirectional \ - BENCH_PRECISIONS= \ + BENCH_PRECISIONS="${BENCH_PRECISIONS}" \ STAGE1_PRECISION=bf16 \ REFINER_PRECISION=bf16 \ + "${BIDIRECTIONAL_GENERATION_ENV[@]}" \ "${BIDIRECTIONAL_OUTPUT_ENV[@]}" \ bash "${SCRIPT_DIR}/bench.sh" @@ -131,8 +148,8 @@ fi _reject_bidirectional_low_precision() { echo "[bench] ERROR: upstream SANA-WM_bidirectional benchmarks are BF16-only." >&2 - echo " FP8 and FP4 precision flags are only supported by upstream SANA-WM_streaming." >&2 - echo " Use STAGE1_PRECISION=bf16 REFINER_PRECISION=bf16, or set SANA_WM_VARIANT=streaming." >&2 + echo " Use BENCH_SIDE=flashdreams for FlashDreams-only SANA-WM_bidirectional FP8/FP4 diagnostics." >&2 + echo " For upstream comparisons, use STAGE1_PRECISION=bf16 REFINER_PRECISION=bf16, or set SANA_WM_VARIANT=streaming." >&2 exit 1 } @@ -192,7 +209,7 @@ if [[ "${SANA_WM_VARIANT}" == "streaming" ]] && _is_true "${COMPILE_STAGE1}"; th exit 1 fi if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then - if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; then + if [[ "${BENCH_SIDE}" != "flashdreams" ]] && [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; then _reject_bidirectional_low_precision fi if [[ -n "${BENCH_PRECISIONS}" ]]; then @@ -203,8 +220,7 @@ if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then continue fi case "${PRECISION}" in - bf16) ;; - fp8|fp4) _reject_bidirectional_low_precision ;; + bf16|fp8|fp4) ;; *) echo "[bench] ERROR: unsupported BENCH_PRECISIONS entry: ${PRECISION}" >&2 exit 1 @@ -220,6 +236,16 @@ if _is_true "${BENCH_DRY_RUN}"; then echo " side: ${BENCH_SIDE}" echo " output: ${OUTPUT_DIR}" echo " precision: stage1=${STAGE1_PRECISION} refiner=${REFINER_PRECISION}" + if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then + echo " bidirectional generations: warmup=${BIDIRECTIONAL_WARMUP_GENERATIONS} measured=${BIDIRECTIONAL_MEASURED_GENERATIONS}" + if [[ -n "${BENCH_PRECISIONS}" ]]; then + echo " precision sweep: ${BENCH_PRECISIONS} (non-BF16 rows are FlashDreams-only)" + fi + elif [[ -n "${BENCH_PRECISIONS}" ]]; then + echo " precision sweep: ${BENCH_PRECISIONS}" + else + echo " runs: warmup=${WARMUP_RUNS} measured=${MEASURED_RUNS}" + fi exit 0 fi @@ -241,11 +267,19 @@ if [[ -n "${BENCH_PRECISIONS}" ]]; then esac PRECISION_LABEL="${PRECISION^^}" PRECISION_OUTPUT_DIR="${OUTPUT_DIR}/${PRECISION}" + ROW_BENCH_SIDE="${BENCH_SIDE}" + if [[ "${SANA_WM_VARIANT}" == "bidirectional" && "${PRECISION}" != "bf16" ]]; then + if [[ "${BENCH_SIDE}" == "upstream" ]]; then + echo "[bench] skipping upstream SANA-WM_bidirectional ${PRECISION_LABEL}; upstream is BF16-only" + continue + fi + ROW_BENCH_SIDE="flashdreams" + fi echo "[bench] precision sweep row ${PRECISION_LABEL} -> ${PRECISION_OUTPUT_DIR}" BENCH_PRECISIONS="" \ SANA_REPO="${SANA_REPO}" \ SANA_WM_VARIANT="${SANA_WM_VARIANT}" \ - BENCH_SIDE="${BENCH_SIDE}" \ + BENCH_SIDE="${ROW_BENCH_SIDE}" \ OUTPUT_DIR="${PRECISION_OUTPUT_DIR}" \ STAGE1_PRECISION="${PRECISION}" \ REFINER_PRECISION="${PRECISION}" \ @@ -328,16 +362,20 @@ if [[ -n "${PYTHONPATH:-}" ]]; then UPSTREAM_PYTHONPATH="${UPSTREAM_PYTHONPATH}:${PYTHONPATH}" fi -mkdir -p "${UPSTREAM_ROOT}" -if [[ "${BENCH_SIDE}" == "both" ]]; then +if [[ "${BENCH_SIDE}" != "flashdreams" ]]; then + mkdir -p "${UPSTREAM_ROOT}" +fi +if [[ "${BENCH_SIDE}" != "upstream" ]]; then mkdir -p "${FLASHDREAMS_ROOT}" fi # Remove stale run_* dirs from prior (possibly longer) benchmarks. bench_summary # reads every run_* dir, so leftovers from an earlier run with more MEASURED_RUNS # would silently pollute this invocation's aggregate. -rm -rf "${UPSTREAM_ROOT:?}"/run_* -if [[ "${BENCH_SIDE}" == "both" ]]; then +if [[ "${BENCH_SIDE}" != "flashdreams" ]]; then + rm -rf "${UPSTREAM_ROOT:?}"/run_* +fi +if [[ "${BENCH_SIDE}" != "upstream" ]]; then rm -rf "${FLASHDREAMS_ROOT:?}"/run_* fi @@ -395,7 +433,11 @@ if [[ "${SANA_WM_VARIANT}" == "streaming" ]] && ! _is_true "${STREAMING_NO_COMPI FLASHDREAMS_STREAMING_COMPILE_ARGS+=(--compile-streaming-refiner) fi -TOTAL_RUNS=$(( WARMUP_RUNS + MEASURED_RUNS )) +if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then + TOTAL_RUNS=1 +else + TOTAL_RUNS=$(( WARMUP_RUNS + MEASURED_RUNS )) +fi _run_upstream_once() { local i="$1" @@ -449,6 +491,8 @@ _run_upstream_once() { --seed "${SEED}" --no_action_overlay --stats_json "${upstream_out}/stats.json" + --warmup_generations "${BIDIRECTIONAL_WARMUP_GENERATIONS}" + --measured_generations "${BIDIRECTIONAL_MEASURED_GENERATIONS}" "${UPSTREAM_PRECISION_ARGS[@]}" "${UPSTREAM_BACKEND_ARGS[@]}" "${UPSTREAM_COMPILE_ARGS[@]}" @@ -510,6 +554,8 @@ _run_flashdreams_once() { --cfg-scale "${CFG_SCALE}" --seed "${SEED}" --stats-json "${flashdreams_out}/stats.json" + --warmup-generations "${BIDIRECTIONAL_WARMUP_GENERATIONS}" + --measured-generations "${BIDIRECTIONAL_MEASURED_GENERATIONS}" "${FLASHDREAMS_PRECISION_ARGS[@]}" "${FLASHDREAMS_BACKEND_ARGS[@]}" "${FLASHDREAMS_COMPILE_ARGS[@]}" @@ -522,11 +568,13 @@ _run_flashdreams_once() { ( cd "${SCRIPT_DIR}" && "${flashdreams_cmd[@]}" ) } -for ((i = 0; i < TOTAL_RUNS; i++)); do - _run_upstream_once "${i}" -done +if [[ "${BENCH_SIDE}" != "flashdreams" ]]; then + for ((i = 0; i < TOTAL_RUNS; i++)); do + _run_upstream_once "${i}" + done +fi -if [[ "${BENCH_SIDE}" == "both" ]]; then +if [[ "${BENCH_SIDE}" != "upstream" ]]; then for ((i = 0; i < TOTAL_RUNS; i++)); do _run_flashdreams_once "${i}" done @@ -545,12 +593,15 @@ fi if _is_true "${FORCE_CUDNN_SDPA}"; then SUMMARY_FLAGS+=(--force-cudnn-sdpa) fi +SUMMARY_WARMUP_RUNS="${WARMUP_RUNS}" +if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then + SUMMARY_WARMUP_RUNS="${BIDIRECTIONAL_WARMUP_GENERATIONS}" +fi SUMMARY_ARGS=( uv run python "${SCRIPT_DIR}/bench_summary.py" --variant "${SANA_WM_VARIANT}" --bench-side "${BENCH_SIDE}" - --upstream-dir "${UPSTREAM_ROOT}" - --warmup-runs "${WARMUP_RUNS}" + --warmup-runs "${SUMMARY_WARMUP_RUNS}" --image-path "${IMAGE_PATH}" --prompt-path "${PROMPT_PATH}" --camera-path "${CAMERA_PATH}" @@ -569,6 +620,9 @@ SUMMARY_ARGS=( --output-md "${SUMMARY_MD}" "${SUMMARY_FLAGS[@]}" ) +if [[ "${BENCH_SIDE}" != "flashdreams" ]]; then + SUMMARY_ARGS+=(--upstream-dir "${UPSTREAM_ROOT}") +fi if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then SUMMARY_ARGS+=( --action "${STREAMING_ACTION}" @@ -579,7 +633,7 @@ if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then --refiner-kv-max-frames "${STREAMING_REFINER_KV_MAX_FRAMES}" ) fi -if [[ "${BENCH_SIDE}" == "both" ]]; then +if [[ "${BENCH_SIDE}" != "upstream" ]]; then SUMMARY_ARGS+=(--flashdreams-dir "${FLASHDREAMS_ROOT}" --output-chart-md "${SUMMARY_CHART_MD}") fi diff --git a/integrations/sana/tests/parity_check/bench_summary.py b/integrations/sana/tests/parity_check/bench_summary.py index 64ba10718..c5d8f847d 100644 --- a/integrations/sana/tests/parity_check/bench_summary.py +++ b/integrations/sana/tests/parity_check/bench_summary.py @@ -82,22 +82,79 @@ def _flashdreams_mem_gib(item: dict[str, Any]) -> float | None: return float(value) if isinstance(value, (int, float)) else None -def _collect( +def _expand_generation_records( items: list[dict[str, Any]], - warmup_runs: int, + warmup_generations: int, +) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for process_index, item in enumerate(items): + generation_records = item.get("generation_records") + if not isinstance(generation_records, list) or not generation_records: + path = item.get("_path", "") + raise ValueError( + "bidirectional benchmark stats must contain generation_records; " + f"discard stale benchmark output and rerun bench.sh ({path})" + ) + + process_metadata = { + key: value + for key, value in item.items() + if key + not in { + "generation_records", + "wall_s", + "timings_s", + "stats_ms", + "video_path", + "video_shape", + } + } + for fallback_index, generation in enumerate(generation_records): + if not isinstance(generation, dict): + continue + generation_index = int( + generation.get("generation_index", fallback_index) + ) + merged = process_metadata | generation + parent_path = item.get("_path") + if isinstance(parent_path, str): + merged["_path"] = f"{parent_path}#generation_{generation_index}" + merged["_process_path"] = parent_path + merged["_command"] = item.get("_command") + merged["process_index"] = process_index + merged["generation_index"] = generation_index + merged["_warmup"] = bool( + generation.get("warmup", generation_index < warmup_generations) + ) + records.append(merged) + return records + + +def _collect_bidirectional( + items: list[dict[str, Any]], + warmup_generations: int, stage_reader, mem_reader, stages: dict[str, str], ) -> dict[str, Any]: - kept = items[warmup_runs:] + records = _expand_generation_records(items, warmup_generations) + kept = [item for item in records if not item.get("_warmup", False)] + warmup_count = len(records) - len(kept) rows: dict[str, Any] = { - "runs_total": len(items), - "warmup_runs": warmup_runs, + "runs_total": len(records), + "warmup_runs": warmup_count, "runs_measured": len(kept), + "generations_total": len(records), + "warmup_generations": warmup_count, + "generations_measured": len(kept), "paths": [item.get("_path") for item in kept], "run_records": [ - _run_record(item, index=index, warmup=index < warmup_runs) - for index, item in enumerate(items) + _run_record( + item, + index=index, + warmup=bool(item.get("_warmup", index < warmup_generations)), + ) + for index, item in enumerate(records) ], } wall = [ @@ -134,7 +191,7 @@ def _run_record( record = { key: value for key, value in item.items() - if key not in {"_path", "_command"} + if key not in {"_path", "_command", "_warmup"} } record["path"] = item.get("_path") record["command"] = item.get("_command") @@ -341,8 +398,9 @@ def _sum_optional(*values: float | None) -> float | None: def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: - upstream = summary["upstream"] + upstream = summary.get("upstream") flashdreams = summary.get("flashdreams") + has_upstream = isinstance(upstream, dict) has_flashdreams = isinstance(flashdreams, dict) refiner_scope = ( "With `NO_REFINER=1`, the timed work covers conditioning, Stage-1 DiT, and SANA VAE decode." @@ -368,23 +426,23 @@ def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: f"- quant_backend: `{summary['inputs']['quant_backend']}`", f"- compile_stage1: `{summary['inputs']['compile_stage1']}`", f"- force_cudnn_sdpa: `{summary['inputs']['force_cudnn_sdpa']}`", - f"- warmup runs discarded: `{summary['inputs']['warmup_runs']}`", + f"- warmup generations discarded: `{summary['inputs']['warmup_runs']}`", "", "## Benchmark metric", "", - "The chart metric is post-load generation latency per generated clip.", + "The chart metric is steady-state in-process generation latency per generated clip.", + "Warmup generations run on the live model and are excluded from the headline metric.", "Model construction, checkpoint loading, video writing, and frame dumps are outside this timing boundary.", - "Each sample is a separate Python process; discarded warmup samples prime process-external state, not a live model instance.", "SANA-WM renders each requested bidirectional clip in one generation pass, not as independently timed frames.", refiner_scope, "", ] - if has_flashdreams: + if has_upstream and has_flashdreams: rows.extend( [ "| metric | upstream | FlashDreams |", "| --- | ---: | ---: |", - f"| measured runs | {upstream['runs_measured']} | {flashdreams['runs_measured']} |", + f"| measured generations | {upstream['generations_measured']} | {flashdreams['generations_measured']} |", f"| generation median / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream'), ' ms')} | {_fmt(_generation_ms_per_clip(summary, 'flashdreams'), ' ms')} |", f"| generation p90 / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream', percentile='p90'), ' ms')} | {_fmt(_generation_ms_per_clip(summary, 'flashdreams', percentile='p90'), ' ms')} |", f"| generation median / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'upstream'), ' ms')} | {_fmt(_generation_ms_per_frame(summary, 'flashdreams'), ' ms')} |", @@ -420,12 +478,12 @@ def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: rows.append( f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} | {_fmt(flashdreams['mem_peak_median_gib'], ' GiB')} |" ) - else: + elif has_upstream: rows.extend( [ "| metric | upstream |", "| --- | ---: |", - f"| measured runs | {upstream['runs_measured']} |", + f"| measured generations | {upstream['generations_measured']} |", f"| generation median / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream'), ' ms')} |", f"| generation p90 / clip | {_fmt(_generation_ms_per_clip(summary, 'upstream', percentile='p90'), ' ms')} |", f"| generation median / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'upstream'), ' ms')} |", @@ -443,6 +501,37 @@ def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: f"| peak memory median | {_fmt(upstream['mem_peak_median_gib'], ' GiB')} |", ] ) + elif has_flashdreams: + decode_label = ( + "VAE decode median" + if summary["inputs"]["no_refiner"] + else "refiner + VAE decode median" + ) + rows.extend( + [ + "| metric | FlashDreams |", + "| --- | ---: |", + f"| measured generations | {flashdreams['generations_measured']} |", + f"| generation median / clip | {_fmt(_generation_ms_per_clip(summary, 'flashdreams'), ' ms')} |", + f"| generation p90 / clip | {_fmt(_generation_ms_per_clip(summary, 'flashdreams', percentile='p90'), ' ms')} |", + f"| generation median / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'flashdreams'), ' ms')} |", + f"| generation p90 / frame, diagnostic | {_fmt(_generation_ms_per_frame(summary, 'flashdreams', percentile='p90'), ' ms')} |", + f"| wall median | {_fmt(flashdreams['wall_median_s'], ' s')} |", + f"| wall p90 | {_fmt(flashdreams['wall_p90_s'], ' s')} |", + "", + "## Timing breakdown", + "", + "| stage | FlashDreams median |", + "| --- | ---: |", + f"| Stage-1 incl. conditioning median | {_fmt(_sum_optional(flashdreams['encode_median_ms'], flashdreams['dit_median_ms']), ' ms')} |", + f"| Stage-1 DiT median | {_fmt(flashdreams['dit_median_ms'], ' ms')} |", + f"| conditioning/encode median | {_fmt(flashdreams['encode_median_ms'], ' ms')} |", + f"| {decode_label} | {_fmt(flashdreams['vae_decode_median_ms'], ' ms')} |", + f"| peak memory median | {_fmt(flashdreams['mem_peak_median_gib'], ' GiB')} |", + ] + ) + else: + raise ValueError("cannot render bidirectional benchmark without any stats") rows.append("") return "\n".join(rows) @@ -560,8 +649,8 @@ def _render_chart_markdown(summary: dict[str, Any], device_label: str) -> str: if streaming else _generation_ms_per_clip(summary, "flashdreams") ) - if official is None: - raise ValueError("cannot render chart data without upstream benchmark stats") + if official is None and flashdreams is None: + raise ValueError("cannot render chart data without benchmark stats") title = ( "# SANA-WM Streaming Benchmark Data (ms/chunk)" if streaming @@ -578,6 +667,17 @@ def _render_chart_markdown(summary: dict[str, Any], device_label: str) -> str: "", ] ) + if official is None: + return "\n".join( + [ + title, + "", + "| device | flashdreams |", + "| --- | ---: |", + f"| {device_label} | {flashdreams:.2f} |", + "", + ] + ) return "\n".join( [ title, @@ -593,8 +693,12 @@ def _render_chart_markdown(summary: dict[str, Any], device_label: str) -> str: def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--variant", choices=["bidirectional", "streaming"], default="bidirectional") - parser.add_argument("--bench-side", choices=["upstream", "both"], default="both") - parser.add_argument("--upstream-dir", type=Path, required=True) + parser.add_argument( + "--bench-side", + choices=["upstream", "flashdreams", "both"], + default="both", + ) + parser.add_argument("--upstream-dir", type=Path, default=None) parser.add_argument("--flashdreams-dir", type=Path, default=None) parser.add_argument("--warmup-runs", type=int, default=1) parser.add_argument("--image-path", type=Path, required=True) @@ -629,22 +733,37 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--output-chart-md", type=Path, default=None) args = parser.parse_args(argv) - if args.bench_side == "both" and args.flashdreams_dir is None: - parser.error("--flashdreams-dir is required when --bench-side=both") + if args.bench_side != "flashdreams" and args.upstream_dir is None: + parser.error("--upstream-dir is required unless --bench-side=flashdreams") + if args.bench_side != "upstream" and args.flashdreams_dir is None: + parser.error("--flashdreams-dir is required unless --bench-side=upstream") if args.variant == "bidirectional" and ( args.stage1_precision != "bf16" or args.refiner_precision != "bf16" - ): + ) and args.bench_side != "flashdreams": parser.error( "upstream SANA-WM_bidirectional benchmarks are BF16-only; " "FP8 and FP4 precision flags are only supported by upstream " - "SANA-WM_streaming. Use --stage1-precision bf16 " + "SANA-WM_streaming. Use --bench-side flashdreams for FlashDreams-only " + "bidirectional diagnostics, use --stage1-precision bf16 " "--refiner-precision bf16, or set --variant streaming." ) + if args.variant == "streaming" and args.bench_side == "flashdreams": + parser.error("--bench-side=flashdreams is only implemented for bidirectional") - upstream_items = _load_stats(args.upstream_dir) - flashdreams_items = _load_stats(args.flashdreams_dir) if args.flashdreams_dir is not None else [] - if not upstream_items: + upstream_items = ( + _load_stats(args.upstream_dir) + if args.upstream_dir is not None and args.bench_side != "flashdreams" + else [] + ) + flashdreams_items = ( + _load_stats(args.flashdreams_dir) + if args.flashdreams_dir is not None and args.bench_side != "upstream" + else [] + ) + if args.bench_side != "flashdreams" and not upstream_items: raise ValueError(f"no upstream stats found in {args.upstream_dir}") + if args.bench_side != "upstream" and not flashdreams_items: + raise ValueError(f"no FlashDreams stats found in {args.flashdreams_dir}") summary = { "variant": args.variant, "bench_side": args.bench_side, @@ -672,6 +791,9 @@ def main(argv: list[str] | None = None) -> None: "compile_stage1": args.compile_stage1, "force_cudnn_sdpa": args.force_cudnn_sdpa, "warmup_runs": args.warmup_runs, + "warmup_generations": ( + args.warmup_runs if args.variant == "bidirectional" else None + ), "device_label": args.device_label, "chart_label": args.chart_label or args.device_label, "output_mode": args.output_mode, @@ -704,20 +826,21 @@ def main(argv: list[str] | None = None) -> None: "precision": args.stage1_precision, } else: - summary["upstream"] = _collect( - upstream_items, - args.warmup_runs, - _upstream_stage_ms, - _upstream_mem_gib, - { - "dit": "stage1_dit_s|stage1_sample_s", - "stage1_total": "stage1_sample_s", - "refiner": "refiner_s", - "vae_decode": "vae_decode_s", - }, - ) - if args.bench_side == "both": - summary["flashdreams"] = _collect( + if args.bench_side != "flashdreams": + summary["upstream"] = _collect_bidirectional( + upstream_items, + args.warmup_runs, + _upstream_stage_ms, + _upstream_mem_gib, + { + "dit": "stage1_dit_s|stage1_sample_s", + "stage1_total": "stage1_sample_s", + "refiner": "refiner_s", + "vae_decode": "vae_decode_s", + }, + ) + if args.bench_side != "upstream": + summary["flashdreams"] = _collect_bidirectional( flashdreams_items, args.warmup_runs, _flashdreams_stage_ms, @@ -729,18 +852,31 @@ def main(argv: list[str] | None = None) -> None: }, ) summary["benchmark"] = { - "metric": "generation_ms_per_clip", + "metric": "steady_state_generation_ms_per_clip", "unit": "ms", "timing_boundary": ( - "pipeline.generate after model setup; excludes model construction, " - "checkpoint loading, video writing, and frame dumps" + "pipeline.generate after model setup with live warmup generations " + "excluded; excludes model construction, checkpoint loading, video " + "writing, and frame dumps" ), "device_label": args.device_label, "chart_label": args.chart_label or args.device_label, - "official": _generation_ms_per_clip(summary, "upstream"), + "official": _generation_ms_per_clip(summary, "upstream") + if args.bench_side != "flashdreams" + else None, "flashdreams": _generation_ms_per_clip(summary, "flashdreams") - if args.bench_side == "both" + if args.bench_side != "upstream" else None, + "warmup_generations": ( + summary.get("upstream", summary.get("flashdreams"))[ + "warmup_generations" + ] + ), + "measured_generations": ( + summary.get("upstream", summary.get("flashdreams"))[ + "generations_measured" + ] + ), } args.output_json.parent.mkdir(parents=True, exist_ok=True) args.output_json.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") diff --git a/integrations/sana/tests/parity_check/bench_sweep_summary.py b/integrations/sana/tests/parity_check/bench_sweep_summary.py index e7ea802f6..635a03f6d 100644 --- a/integrations/sana/tests/parity_check/bench_sweep_summary.py +++ b/integrations/sana/tests/parity_check/bench_sweep_summary.py @@ -41,22 +41,29 @@ def _load_item(raw: str) -> dict[str, Any]: raise argparse.ArgumentTypeError(f"{path} has no benchmark object") official = benchmark.get("official") flashdreams = benchmark.get("flashdreams") - if not isinstance(official, (int, float)): + if official is not None and not isinstance(official, (int, float)): raise argparse.ArgumentTypeError( - f"{path} benchmark must contain a numeric official value" + f"{path} benchmark official value must be numeric or null" ) if flashdreams is not None and not isinstance(flashdreams, (int, float)): raise argparse.ArgumentTypeError( f"{path} benchmark flashdreams value must be numeric or null" ) + if official is None and flashdreams is None: + raise argparse.ArgumentTypeError( + f"{path} benchmark must contain an official or flashdreams value" + ) + metric = benchmark.get("metric") + if not isinstance(metric, str): + raise argparse.ArgumentTypeError(f"{path} benchmark must contain a metric") return { "label": label, "path": str(path), - "official": float(official), + "official": float(official) if isinstance(official, (int, float)) else None, "flashdreams": ( float(flashdreams) if isinstance(flashdreams, (int, float)) else None ), - "metric": benchmark.get("metric", "generation_ms_per_clip"), + "metric": metric, "unit": benchmark.get("unit", "ms"), "variant": benchmark.get("variant") or summary.get("variant") @@ -66,44 +73,58 @@ def _load_item(raw: str) -> dict[str, Any]: def _render_chart(rows: list[dict[str, Any]]) -> str: - has_flashdreams = all(row["flashdreams"] is not None for row in rows) + has_official = any(row["official"] is not None for row in rows) + has_flashdreams = any(row["flashdreams"] is not None for row in rows) unit_label = ( "ms/chunk" if rows and rows[0]["metric"] == "steady_state_generation_ms_per_chunk" else "ms" ) - if not has_flashdreams: - lines = [ - f"# SANA-WM Precision Sweep Upstream Summary ({unit_label})", - "", - "| precision | official |", - "| --- | ---: |", - ] - for row in rows: - lines.append(f"| {row['label']} | {row['official']:.2f} |") - lines.append("") - return "\n".join(lines) lines = [ - f"# SANA-WM Precision Sweep Summary ({unit_label})", + ( + f"# SANA-WM Precision Sweep Summary ({unit_label})" + if has_flashdreams + else f"# SANA-WM Precision Sweep Upstream Summary ({unit_label})" + ), "", - "| precision | official | flashdreams |", - "| --- | ---: | ---: |", ] + columns = ["precision"] + aligns = ["---"] + if has_official: + columns.append("official") + aligns.append("---:") + if has_flashdreams: + columns.append("flashdreams") + aligns.append("---:") + lines.append("| " + " | ".join(columns) + " |") + lines.append("| " + " | ".join(aligns) + " |") for row in rows: - lines.append( - f"| {row['label']} | {row['official']:.2f} | {row['flashdreams']:.2f} |" - ) + values = [row["label"]] + if has_official: + values.append( + f"{row['official']:.2f}" + if row["official"] is not None + else "n/a" + ) + if has_flashdreams: + values.append( + f"{row['flashdreams']:.2f}" + if row["flashdreams"] is not None + else "n/a" + ) + lines.append("| " + " | ".join(values) + " |") lines.append("") return "\n".join(lines) def _render_report(rows: list[dict[str, Any]]) -> str: - has_flashdreams = all(row["flashdreams"] is not None for row in rows) - metric = rows[0]["metric"] if rows else "generation_ms_per_clip" + has_official = any(row["official"] is not None for row in rows) + has_flashdreams = any(row["flashdreams"] is not None for row in rows) + metric = rows[0]["metric"] if rows else "steady_state_generation_ms_per_clip" metric_text = ( "steady-state generation latency per produced chunk" if metric == "steady_state_generation_ms_per_chunk" - else "post-load generation latency per generated clip" + else "steady-state in-process generation latency per generated clip" ) lines = [ "# SANA-WM precision benchmark sweep", @@ -113,30 +134,34 @@ def _render_report(rows: list[dict[str, Any]]) -> str: "subdirectory's `perf.md`.", "", ] + columns = ["precision"] + aligns = ["---"] + if has_official: + columns.append("official") + aligns.append("---:") if has_flashdreams: - lines.extend( - [ - "| precision | official | FlashDreams | source |", - "| --- | ---: | ---: | --- |", - ] - ) - else: - lines.extend( - [ - "| precision | official | source |", - "| --- | ---: | --- |", - ] - ) + columns.append("FlashDreams") + aligns.append("---:") + columns.append("source") + aligns.append("---") + lines.append("| " + " | ".join(columns) + " |") + lines.append("| " + " | ".join(aligns) + " |") for row in rows: - if has_flashdreams: - lines.append( - f"| {row['label']} | {row['official']:.2f} ms | " - f"{row['flashdreams']:.2f} ms | `{row['path']}` |" + values = [row["label"]] + if has_official: + values.append( + f"{row['official']:.2f} ms" + if row["official"] is not None + else "n/a" ) - else: - lines.append( - f"| {row['label']} | {row['official']:.2f} ms | `{row['path']}` |" + if has_flashdreams: + values.append( + f"{row['flashdreams']:.2f} ms" + if row["flashdreams"] is not None + else "n/a" ) + values.append(f"`{row['path']}`") + lines.append("| " + " | ".join(values) + " |") lines.append("") return "\n".join(lines) @@ -145,6 +170,12 @@ def _validate_rows(rows: list[dict[str, Any]]) -> None: for row in rows: if row["variant"] == "streaming": continue + if row["metric"] != "steady_state_generation_ms_per_clip": + raise ValueError( + "SANA-WM_bidirectional benchmark summaries must use " + "steady_state_generation_ms_per_clip; discard stale " + "generation_ms_per_clip data and rerun bench.sh." + ) inputs = row.get("inputs", {}) stage1_precision = inputs.get("stage1_precision") refiner_precision = inputs.get("refiner_precision") @@ -154,11 +185,15 @@ def _validate_rows(rows: list[dict[str, Any]]) -> None: or stage1_precision in {"fp8", "fp4"} or refiner_precision in {"fp8", "fp4"} ): - raise ValueError( - "upstream SANA-WM_bidirectional benchmarks are BF16-only; " - "FP8 and FP4 precision comparisons belong to " - "SANA_WM_VARIANT=streaming." - ) + if row["official"] is not None: + raise ValueError( + "upstream SANA-WM_bidirectional benchmarks are BF16-only; " + "FP8 and FP4 bidirectional rows must be FlashDreams-only." + ) + if row["flashdreams"] is None: + raise ValueError( + "FP8 and FP4 bidirectional rows must contain a FlashDreams value." + ) def main(argv: list[str] | None = None) -> None: @@ -182,11 +217,13 @@ def main(argv: list[str] | None = None) -> None: if len(units) != 1: raise ValueError(f"cannot aggregate mixed benchmark units: {sorted(units)}") _validate_rows(args.item) - has_flashdreams = all(row["flashdreams"] is not None for row in args.item) + has_official = any(row["official"] is not None for row in args.item) + has_flashdreams = any(row["flashdreams"] is not None for row in args.item) payload = { "benchmark": { "metric": args.item[0]["metric"], "unit": args.item[0]["unit"], + "has_official": has_official, "has_flashdreams": has_flashdreams, }, "rows": args.item, diff --git a/integrations/sana/tests/parity_check/changes_bidirectional.patch b/integrations/sana/tests/parity_check/changes_bidirectional.patch index 10f349d4c..f9a6e94c9 100644 --- a/integrations/sana/tests/parity_check/changes_bidirectional.patch +++ b/integrations/sana/tests/parity_check/changes_bidirectional.patch @@ -1,8 +1,6 @@ -diff --git a/inference_video_scripts/wm/inference_sana_wm.py b/inference_video_scripts/wm/inference_sana_wm.py -index 5e07197..17db817 100644 --- a/inference_video_scripts/wm/inference_sana_wm.py +++ b/inference_video_scripts/wm/inference_sana_wm.py -@@ -937,6 +937,101 @@ def write_video(output_dir: Path, name: str, video_hwc: np.ndarray, fps: int, lo +@@ -937,6 +937,101 @@ return video_path @@ -104,7 +102,7 @@ index 5e07197..17db817 100644 # ============================================================================ # Pipeline # ============================================================================ -@@ -2132,6 +2227,30 @@ def _build_parser() -> argparse.ArgumentParser: +@@ -2132,6 +2227,32 @@ action="store_true", help="Skip rendering the WASD + joystick overlay on the output video.", ) @@ -132,22 +130,28 @@ index 5e07197..17db817 100644 + default="bf16", + help="LTX-2 refiner precision for benchmark parity.", + ) ++ p.add_argument("--warmup_generations", type=int, default=0) ++ p.add_argument("--measured_generations", type=int, default=1) # Weights and config. p.add_argument( -@@ -2222,7 +2341,11 @@ def _snap_num_frames(n: int, stride: int = 8, *, upper_bound: int | None = None) +@@ -2222,7 +2343,15 @@ def main() -> None: args = _build_parser().parse_args() + if args.force_cudnn_sdpa: + apply_backend_defaults() ++ if args.warmup_generations < 0: ++ raise ValueError("--warmup_generations must be >= 0") ++ if args.measured_generations < 1: ++ raise ValueError("--measured_generations must be >= 1") + logger = get_root_logger() + apply_precision_args(args, logger) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") image = Image.open(args.image).convert("RGB") -@@ -2276,6 +2399,15 @@ def main() -> None: +@@ -2276,6 +2405,15 @@ offload_refiner=args.offload_refiner, logger=logger, ) @@ -163,18 +167,45 @@ index 5e07197..17db817 100644 denoising_step_list: list[int] | None = None if args.denoising_step_list: -@@ -2308,14 +2440,48 @@ def main() -> None: +@@ -2308,14 +2446,79 @@ save_stage1=args.save_stage1, ) -+ if torch.cuda.is_available(): -+ torch.cuda.reset_peak_memory_stats(device) -+ torch.cuda.synchronize(device) -+ wall_start = time.perf_counter() - out = pipeline.generate(cropped, prompt, c2w, intrinsics_vec4, params) -+ if torch.cuda.is_available(): -+ torch.cuda.synchronize(device) -+ wall_s = time.perf_counter() - wall_start +- out = pipeline.generate(cropped, prompt, c2w, intrinsics_vec4, params) ++ generation_records: list[dict[str, object]] = [] ++ final_out: dict[str, object] | None = None ++ final_record: dict[str, object] | None = None ++ total_generations = args.warmup_generations + args.measured_generations ++ for generation_index in range(total_generations): ++ warmup = generation_index < args.warmup_generations ++ timings.clear() ++ if torch.cuda.is_available(): ++ torch.cuda.reset_peak_memory_stats(device) ++ torch.cuda.synchronize(device) ++ wall_start = time.perf_counter() ++ out = pipeline.generate(cropped, prompt, c2w, intrinsics_vec4, params) ++ if torch.cuda.is_available(): ++ torch.cuda.synchronize(device) ++ wall_s = time.perf_counter() - wall_start ++ record: dict[str, object] = { ++ "generation_index": generation_index, ++ "warmup": warmup, ++ "wall_s": wall_s, ++ "timings_s": dict(timings), ++ "num_frames": num_frames, ++ "seed": args.seed, ++ } ++ if torch.cuda.is_available(): ++ record["mem_peak_gib"] = torch.cuda.max_memory_allocated(device) / (1024**3) ++ generation_records.append(record) ++ if not warmup: ++ final_out = out ++ final_record = record ++ ++ if final_out is None or final_record is None: ++ raise RuntimeError("bidirectional benchmark produced no measured generations") ++ ++ out = final_out video_hwc = out["video"] if not args.no_action_overlay: @@ -183,6 +214,8 @@ index 5e07197..17db817 100644 - write_video(args.output_dir, args.name, video_hwc, params.fps, logger) + video_path = write_video(args.output_dir, args.name, video_hwc, params.fps, logger) ++ final_record["video_path"] = str(video_path) ++ final_record["video_shape"] = list(np.asarray(video_hwc).shape) + + if args.dump_frames is not None: + args.dump_frames.parent.mkdir(parents=True, exist_ok=True) @@ -194,7 +227,7 @@ index 5e07197..17db817 100644 + "entrypoint": "inference_video_scripts/wm/inference_sana_wm.py", + "video_path": str(video_path), + "video_shape": list(np.asarray(video_hwc).shape), -+ "timings_s": timings, ++ "timings_s": final_record["timings_s"], + "num_frames": num_frames, + "seed": args.seed, + "fps": params.fps, @@ -205,10 +238,13 @@ index 5e07197..17db817 100644 + "refiner_precision": args.refiner_precision, + "compile_stage1": args.compile_stage1, + "force_cudnn_sdpa": args.force_cudnn_sdpa, -+ "wall_s": wall_s, ++ "warmup_generations": args.warmup_generations, ++ "measured_generations": args.measured_generations, ++ "wall_s": final_record["wall_s"], ++ "generation_records": generation_records, + } + if torch.cuda.is_available(): -+ stats["mem_peak_gib"] = torch.cuda.max_memory_allocated(device) / (1024**3) ++ stats["mem_peak_gib"] = final_record.get("mem_peak_gib") + write_json(args.stats_json, stats) stage1_video = out.get("stage1_video") diff --git a/integrations/sana/tests/parity_check/run_common.sh b/integrations/sana/tests/parity_check/run_common.sh index 7a2b8299f..808d36750 100644 --- a/integrations/sana/tests/parity_check/run_common.sh +++ b/integrations/sana/tests/parity_check/run_common.sh @@ -41,8 +41,18 @@ parity_apply_patch_once() { echo "[setup] applying ${patch_file}" git apply "${patch_file}" else - echo "[setup] ERROR: ${patch_file} neither cleanly applies nor is already applied." >&2 - exit 1 + echo "[setup] refreshing stale patched targets for ${patch_file}" + while IFS=$'\t' read -r _added _deleted path; do + [[ -n "${path}" ]] || continue + git checkout -- "${path}" + done < <(git apply --numstat "${patch_file}") + if git apply --check "${patch_file}" >/dev/null 2>&1; then + echo "[setup] applying ${patch_file}" + git apply "${patch_file}" + else + echo "[setup] ERROR: ${patch_file} neither cleanly applies nor is already applied." >&2 + exit 1 + fi fi } diff --git a/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py b/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py index 401104809..bbc627b0a 100644 --- a/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py +++ b/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py @@ -83,8 +83,15 @@ def main(argv: list[str] | None = None) -> None: ) parser.add_argument("--force-cudnn-sdpa", action="store_true") parser.add_argument("--compile-stage1", action="store_true") + parser.add_argument("--warmup-generations", type=int, default=0) + parser.add_argument("--measured-generations", type=int, default=1) args = parser.parse_args(argv) + if args.warmup_generations < 0: + raise ValueError("--warmup-generations must be >= 0") + if args.measured_generations < 1: + raise ValueError("--measured-generations must be >= 1") + if args.force_cudnn_sdpa: apply_backend_defaults() @@ -151,34 +158,65 @@ def main(argv: list[str] | None = None) -> None: seed=args.seed, negative_prompt=args.negative_prompt, ) - cache = pipeline.initialize_cache( - decoder_context={ - "prompt": prompt, - "fps": args.fps, - "save_stage1": args.save_stage1, - "refiner_seed": args.seed, - "sink_size": runner_cfg.sink_size, - } - ) - if torch.cuda.is_available(): - torch.cuda.reset_peak_memory_stats(device) - torch.cuda.synchronize(device) - generation_start = time.perf_counter() + generation_records: list[dict[str, object]] = [] + final_record: dict[str, object] | None = None + final_frames: np.ndarray | None = None + total_generations = args.warmup_generations + args.measured_generations with torch.inference_mode(): - decoded = pipeline.generate(0, cache, input=request) - if torch.cuda.is_available(): - torch.cuda.synchronize(device) - wall_s = time.perf_counter() - generation_start - stats = pipeline.finalize(0, cache) or {} + for generation_index in range(total_generations): + warmup = generation_index < args.warmup_generations + cache = pipeline.initialize_cache( + decoder_context={ + "prompt": prompt, + "fps": args.fps, + "save_stage1": args.save_stage1, + "refiner_seed": args.seed, + "sink_size": runner_cfg.sink_size, + } + ) + if torch.cuda.is_available(): + torch.cuda.reset_peak_memory_stats(device) + torch.cuda.synchronize(device) + generation_start = time.perf_counter() + decoded = pipeline.generate(0, cache, input=request) + if torch.cuda.is_available(): + torch.cuda.synchronize(device) + wall_s = time.perf_counter() - generation_start + stats = pipeline.finalize(0, cache) or {} + + if not isinstance(decoded, SanaWMDecodedVideo): + raise TypeError( + f"expected SanaWMDecodedVideo, got {type(decoded).__name__}" + ) + + if torch.cuda.is_available(): + stats.setdefault( + "mem_peak_gib", + torch.cuda.max_memory_allocated(device) / (1024**3), + ) + frames = np.asarray(decoded.video_hwc, dtype=np.uint8) + record: dict[str, object] = { + "generation_index": generation_index, + "warmup": warmup, + "wall_s": wall_s, + "stats_ms": stats, + "video_shape": list(frames.shape), + "num_frames": num_frames, + "seed": args.seed, + } + generation_records.append(record) + if not warmup: + final_record = record + final_frames = frames - if not isinstance(decoded, SanaWMDecodedVideo): - raise TypeError(f"expected SanaWMDecodedVideo, got {type(decoded).__name__}") + if final_record is None or final_frames is None: + raise RuntimeError("bidirectional benchmark produced no measured generations") - frames = np.asarray(decoded.video_hwc, dtype=np.uint8) - video_path = write_video(args.output_dir, args.name, frames, args.fps) + video_path = write_video(args.output_dir, args.name, final_frames, args.fps) + final_record["video_path"] = str(video_path) if args.dump_frames is not None: args.dump_frames.parent.mkdir(parents=True, exist_ok=True) - np.save(args.dump_frames, frames) + np.save(args.dump_frames, final_frames) if args.stats_json is not None: write_json( args.stats_json, @@ -187,7 +225,7 @@ def main(argv: list[str] | None = None) -> None: "variant": "bidirectional", "runner": "sana-wm-bidirectional", "video_path": str(video_path), - "video_shape": list(frames.shape), + "video_shape": list(final_frames.shape), "num_frames": num_frames, "seed": args.seed, "fps": args.fps, @@ -198,8 +236,11 @@ def main(argv: list[str] | None = None) -> None: "refiner_precision": args.refiner_precision, "compile_stage1": args.compile_stage1, "force_cudnn_sdpa": args.force_cudnn_sdpa, - "wall_s": wall_s, - "stats_ms": stats, + "warmup_generations": args.warmup_generations, + "measured_generations": args.measured_generations, + "wall_s": final_record["wall_s"], + "stats_ms": final_record["stats_ms"], + "generation_records": generation_records, }, ) print(f"[flashdreams] wrote {video_path}") diff --git a/integrations/sana/tests/test_parity_benchmark_summary.py b/integrations/sana/tests/test_parity_benchmark_summary.py index e81638de7..5aaa17285 100644 --- a/integrations/sana/tests/test_parity_benchmark_summary.py +++ b/integrations/sana/tests/test_parity_benchmark_summary.py @@ -49,7 +49,7 @@ def _load_bench_sweep_summary() -> ModuleType: return module -def test_benchmark_summary_uses_generation_ms_per_clip_for_chart() -> None: +def test_bidirectional_summary_uses_steady_state_generation_ms_per_clip_for_chart() -> None: module = _load_bench_summary() summary = { "inputs": { @@ -69,6 +69,7 @@ def test_benchmark_summary_uses_generation_ms_per_clip_for_chart() -> None: }, "upstream": { "runs_measured": 3, + "generations_measured": 3, "wall_median_s": 2.0, "wall_p90_s": 2.2, "stage1_total_median_ms": 1600.0, @@ -78,6 +79,7 @@ def test_benchmark_summary_uses_generation_ms_per_clip_for_chart() -> None: }, "flashdreams": { "runs_measured": 3, + "generations_measured": 3, "wall_median_s": 1.5, "wall_p90_s": 1.8, "encode_median_ms": 100.0, @@ -92,6 +94,8 @@ def test_benchmark_summary_uses_generation_ms_per_clip_for_chart() -> None: assert "## Benchmark metric" in report assert "- stage1_precision: `bf16`" in report + assert "steady-state in-process generation latency per generated clip" in report + assert "measured generations | 3 | 3" in report assert "generation median / clip | 2000.00 ms | 1500.00 ms" in report assert "generation median / frame, diagnostic | 100.00 ms | 75.00 ms" in report assert "## Timing breakdown" in report @@ -116,13 +120,52 @@ def test_benchmark_summary_writes_chart_data(tmp_path: Path) -> None: (upstream / "stats.json").write_text( json.dumps( { - "wall_s": 4.0, - "mem_peak_gib": 10.0, - "timings_s": { - "stage1_sample_s": 3.5, - "stage1_dit_s": 3.0, - "vae_decode_s": 0.5, - }, + "generation_records": [ + { + "generation_index": 0, + "warmup": True, + "wall_s": 99.0, + "mem_peak_gib": 99.0, + "timings_s": { + "stage1_sample_s": 90.0, + "stage1_dit_s": 80.0, + "vae_decode_s": 9.0, + }, + }, + { + "generation_index": 1, + "warmup": False, + "wall_s": 3.0, + "mem_peak_gib": 9.0, + "timings_s": { + "stage1_sample_s": 2.5, + "stage1_dit_s": 2.0, + "vae_decode_s": 0.5, + }, + }, + { + "generation_index": 2, + "warmup": False, + "wall_s": 4.0, + "mem_peak_gib": 10.0, + "timings_s": { + "stage1_sample_s": 3.5, + "stage1_dit_s": 3.0, + "vae_decode_s": 0.5, + }, + }, + { + "generation_index": 3, + "warmup": False, + "wall_s": 5.0, + "mem_peak_gib": 11.0, + "timings_s": { + "stage1_sample_s": 4.5, + "stage1_dit_s": 4.0, + "vae_decode_s": 0.5, + }, + }, + ], } ), encoding="utf-8", @@ -130,13 +173,52 @@ def test_benchmark_summary_writes_chart_data(tmp_path: Path) -> None: (flashdreams / "stats.json").write_text( json.dumps( { - "wall_s": 2.0, - "stats_ms": { - "encode_ms": 100.0, - "diffuse_ms": 1500.0, - "decode_ms": 400.0, - "mem_peak_gib": 12.0, - }, + "generation_records": [ + { + "generation_index": 0, + "warmup": True, + "wall_s": 77.0, + "stats_ms": { + "encode_ms": 700.0, + "diffuse_ms": 70000.0, + "decode_ms": 6000.0, + "mem_peak_gib": 70.0, + }, + }, + { + "generation_index": 1, + "warmup": False, + "wall_s": 1.5, + "stats_ms": { + "encode_ms": 80.0, + "diffuse_ms": 1200.0, + "decode_ms": 300.0, + "mem_peak_gib": 11.0, + }, + }, + { + "generation_index": 2, + "warmup": False, + "wall_s": 2.0, + "stats_ms": { + "encode_ms": 100.0, + "diffuse_ms": 1500.0, + "decode_ms": 400.0, + "mem_peak_gib": 12.0, + }, + }, + { + "generation_index": 3, + "warmup": False, + "wall_s": 2.5, + "stats_ms": { + "encode_ms": 120.0, + "diffuse_ms": 1800.0, + "decode_ms": 500.0, + "mem_peak_gib": 13.0, + }, + }, + ], } ), encoding="utf-8", @@ -152,7 +234,7 @@ def test_benchmark_summary_writes_chart_data(tmp_path: Path) -> None: "--flashdreams-dir", str(tmp_path / "flashdreams"), "--warmup-runs", - "0", + "1", "--image-path", "image.png", "--prompt-path", @@ -182,6 +264,8 @@ def test_benchmark_summary_writes_chart_data(tmp_path: Path) -> None: assert "| Test GPU | 4000.00 | 2000.00 |" in out_chart.read_text(encoding="utf-8") report = out_md.read_text(encoding="utf-8") + assert "warmup generations discarded: `1`" in report + assert "measured generations | 3 | 3" in report assert "generation median / clip | 4000.00 ms | 2000.00 ms" in report assert "generation median / frame, diagnostic | 100.00 ms | 50.00 ms" in report assert "generation median / frame |" not in report @@ -190,17 +274,163 @@ def test_benchmark_summary_writes_chart_data(tmp_path: Path) -> None: assert "per generated frame" not in report summary = json.loads(out_json.read_text(encoding="utf-8")) assert summary["benchmark"] == { - "metric": "generation_ms_per_clip", + "metric": "steady_state_generation_ms_per_clip", "unit": "ms", "timing_boundary": ( - "pipeline.generate after model setup; excludes model construction, " - "checkpoint loading, video writing, and frame dumps" + "pipeline.generate after model setup with live warmup generations " + "excluded; excludes model construction, checkpoint loading, video " + "writing, and frame dumps" ), "device_label": "Test GPU", "chart_label": "Test GPU", "official": 4000.0, "flashdreams": 2000.0, + "warmup_generations": 1, + "measured_generations": 3, } + assert summary["upstream"]["generations_measured"] == 3 + assert summary["upstream"]["run_records"][0]["warmup"] is True + assert summary["upstream"]["run_records"][1]["path"].endswith( + "stats.json#generation_1" + ) + + +def test_bidirectional_summary_rejects_stale_single_record_stats( + tmp_path: Path, +) -> None: + module = _load_bench_summary() + upstream = tmp_path / "upstream" / "run_0" + upstream.mkdir(parents=True) + (upstream / "stats.json").write_text( + json.dumps({"wall_s": 4.0, "timings_s": {"stage1_sample_s": 3.5}}), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="generation_records"): + module.main( + [ + "--variant", + "bidirectional", + "--bench-side", + "upstream", + "--upstream-dir", + str(tmp_path / "upstream"), + "--image-path", + "image.png", + "--prompt-path", + "prompt.txt", + "--camera-path", + "pose.npy", + "--intrinsics-path", + "intrinsics.npy", + "--num-frames", + "40", + "--seed", + "42", + "--stage1-precision", + "bf16", + "--refiner-precision", + "bf16", + "--output-json", + str(tmp_path / "bench.json"), + "--output-md", + str(tmp_path / "bench.md"), + ] + ) + + +def test_bidirectional_summary_accepts_flashdreams_only_low_precision( + tmp_path: Path, +) -> None: + module = _load_bench_summary() + flashdreams = tmp_path / "flashdreams" / "run_0" + flashdreams.mkdir(parents=True) + (flashdreams / "stats.json").write_text( + json.dumps( + { + "generation_records": [ + { + "generation_index": 0, + "warmup": True, + "wall_s": 99.0, + "stats_ms": { + "encode_ms": 900.0, + "diffuse_ms": 90000.0, + "decode_ms": 9000.0, + "mem_peak_gib": 70.0, + }, + }, + { + "generation_index": 1, + "warmup": False, + "wall_s": 2.0, + "stats_ms": { + "encode_ms": 100.0, + "diffuse_ms": 1500.0, + "decode_ms": 400.0, + "mem_peak_gib": 12.0, + }, + }, + ], + } + ), + encoding="utf-8", + ) + out_json = tmp_path / "bench.json" + out_md = tmp_path / "bench.md" + out_chart = tmp_path / "perf.md" + + module.main( + [ + "--variant", + "bidirectional", + "--bench-side", + "flashdreams", + "--flashdreams-dir", + str(tmp_path / "flashdreams"), + "--warmup-runs", + "1", + "--image-path", + "image.png", + "--prompt-path", + "prompt.txt", + "--camera-path", + "pose.npy", + "--intrinsics-path", + "intrinsics.npy", + "--num-frames", + "40", + "--seed", + "42", + "--stage1-precision", + "fp8", + "--refiner-precision", + "fp8", + "--device-label", + "Test GPU", + "--output-json", + str(out_json), + "--output-md", + str(out_md), + "--output-chart-md", + str(out_chart), + ] + ) + + assert out_chart.read_text(encoding="utf-8") == ( + "# SANA-WM Benchmark Data (ms)\n" + "\n" + "| device | flashdreams |\n" + "| --- | ---: |\n" + "| Test GPU | 2000.00 |\n" + ) + summary = json.loads(out_json.read_text(encoding="utf-8")) + assert summary["benchmark"]["official"] is None + assert summary["benchmark"]["flashdreams"] == 2000.0 + assert summary["benchmark"]["measured_generations"] == 1 + report = out_md.read_text(encoding="utf-8") + assert "| metric | FlashDreams |" in report + assert "generation median / clip | 2000.00 ms" in report def test_bidirectional_summary_rejects_low_precision_comparison(tmp_path: Path) -> None: @@ -536,7 +766,11 @@ def test_benchmark_sweep_summary_accepts_bidirectional_bf16_chart_data(tmp_path: "stage1_precision": "bf16", "refiner_precision": "bf16", }, - "benchmark": {"official": 1000.0, "flashdreams": 900.0}, + "benchmark": { + "metric": "steady_state_generation_ms_per_clip", + "official": 1000.0, + "flashdreams": 900.0, + }, } ), encoding="utf-8", @@ -566,11 +800,124 @@ def test_benchmark_sweep_summary_accepts_bidirectional_bf16_chart_data(tmp_path: "| BF16 | 1000.00 | 900.00 |\n" ) payload = json.loads(out_json.read_text(encoding="utf-8")) - assert payload["benchmark"]["metric"] == "generation_ms_per_clip" + assert payload["benchmark"]["metric"] == "steady_state_generation_ms_per_clip" assert payload["benchmark"]["unit"] == "ms" + assert payload["benchmark"]["has_official"] is True + assert payload["benchmark"]["has_flashdreams"] is True assert [row["label"] for row in payload["rows"]] == ["BF16"] +def test_benchmark_sweep_summary_rejects_stale_bidirectional_metric( + tmp_path: Path, +) -> None: + module = _load_bench_sweep_summary() + stale_json = tmp_path / "stale.json" + stale_json.write_text( + json.dumps( + { + "variant": "bidirectional", + "inputs": { + "stage1_precision": "bf16", + "refiner_precision": "bf16", + }, + "benchmark": { + "metric": "generation_ms_per_clip", + "official": 1000.0, + "flashdreams": 900.0, + }, + } + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="discard stale"): + module.main( + [ + "--item", + f"BF16:{stale_json}", + "--output-json", + str(tmp_path / "bench.json"), + "--output-md", + str(tmp_path / "bench.md"), + "--output-chart-md", + str(tmp_path / "perf.md"), + ] + ) + + +def test_benchmark_sweep_summary_accepts_bidirectional_flashdreams_only_low_precision( + tmp_path: Path, +) -> None: + module = _load_bench_sweep_summary() + bf16_json = tmp_path / "bf16.json" + fp8_json = tmp_path / "fp8.json" + bf16_json.write_text( + json.dumps( + { + "variant": "bidirectional", + "inputs": { + "stage1_precision": "bf16", + "refiner_precision": "bf16", + }, + "benchmark": { + "metric": "steady_state_generation_ms_per_clip", + "official": 1000.0, + "flashdreams": 900.0, + }, + } + ), + encoding="utf-8", + ) + fp8_json.write_text( + json.dumps( + { + "variant": "bidirectional", + "inputs": { + "stage1_precision": "fp8", + "refiner_precision": "fp8", + }, + "benchmark": { + "metric": "steady_state_generation_ms_per_clip", + "official": None, + "flashdreams": 700.0, + }, + } + ), + encoding="utf-8", + ) + out_json = tmp_path / "bench.json" + out_md = tmp_path / "bench.md" + out_chart = tmp_path / "perf.md" + + module.main( + [ + "--item", + f"BF16:{bf16_json}", + "--item", + f"FP8:{fp8_json}", + "--output-json", + str(out_json), + "--output-md", + str(out_md), + "--output-chart-md", + str(out_chart), + ] + ) + + assert out_chart.read_text(encoding="utf-8") == ( + "# SANA-WM Precision Sweep Summary (ms)\n" + "\n" + "| precision | official | flashdreams |\n" + "| --- | ---: | ---: |\n" + "| BF16 | 1000.00 | 900.00 |\n" + "| FP8 | n/a | 700.00 |\n" + ) + payload = json.loads(out_json.read_text(encoding="utf-8")) + assert payload["benchmark"]["has_official"] is True + assert payload["benchmark"]["has_flashdreams"] is True + assert payload["rows"][1]["official"] is None + + def test_benchmark_sweep_summary_rejects_bidirectional_low_precision( tmp_path: Path, ) -> None: @@ -584,7 +931,11 @@ def test_benchmark_sweep_summary_rejects_bidirectional_low_precision( "stage1_precision": "fp8", "refiner_precision": "fp8", }, - "benchmark": {"official": 800.0, "flashdreams": 700.0}, + "benchmark": { + "metric": "steady_state_generation_ms_per_clip", + "official": 800.0, + "flashdreams": 700.0, + }, } ), encoding="utf-8", @@ -666,6 +1017,7 @@ def test_benchmark_sweep_summary_accepts_streaming_upstream_only(tmp_path: Path) ) payload = json.loads(out_json.read_text(encoding="utf-8")) assert payload["benchmark"]["metric"] == "steady_state_generation_ms_per_chunk" + assert payload["benchmark"]["has_official"] is True assert payload["benchmark"]["has_flashdreams"] is False assert [row["flashdreams"] for row in payload["rows"]] == [None, None] From 3fede3f9ffac6babfb18b4e462603f6f9c91d445 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 30 Jul 2026 18:11:21 -0700 Subject: [PATCH 49/64] Fix issues running on fresh machine? --- .../sana/tests/parity_check/README.md | 17 ++++++++++--- integrations/sana/tests/parity_check/bench.sh | 25 ++++++++++++++++++- .../sana/tests/parity_check/bench_summary.py | 3 +++ .../parity_check/changes_bidirectional.patch | 25 +++++++++++++++---- .../parity_check/compat/sitecustomize.py | 19 ++++++++++++++ .../tests/parity_check/run_bidirectional.sh | 10 ++++++++ .../run_flashdreams_bidirectional.py | 7 ++++++ .../parity_check/run_flashdreams_common.py | 6 +++++ .../parity_check/run_flashdreams_streaming.py | 7 ++++++ .../sana/tests/parity_check/run_streaming.sh | 9 +++++++ .../tests/test_parity_benchmark_summary.py | 1 + 11 files changed, 119 insertions(+), 10 deletions(-) create mode 100644 integrations/sana/tests/parity_check/compat/sitecustomize.py diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index f712c2f4e..0cb1c66e9 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -58,6 +58,7 @@ Defaults: - `REFINER_PRECISION=$STAGE1_PRECISION` - `QUANT_BACKEND=auto` - `FORCE_CUDNN_SDPA=0` +- `DISABLE_CUDNN_SDPA=0` Outputs are written under `outputs/parity/`: @@ -98,10 +99,11 @@ bash bench.sh Benchmark defaults to both sibling variants: `SANA_WM_VARIANT=both BENCH_SIDE=both WARMUP_RUNS=1 MEASURED_RUNS=3 BENCH_PRECISIONS=bf16,fp8,fp4 COMPILE_STAGE1=0 NO_REFINER=0 -FORCE_CUDNN_SDPA=0`. For bidirectional, the warmup/measured counts mean one -live warmup generation and three measured generations inside each measured -process. For streaming, those counts remain process-level warmup/measured runs -because steady state is measured from chunks inside each process. +FORCE_CUDNN_SDPA=0 DISABLE_CUDNN_SDPA=0`. For bidirectional, the +warmup/measured counts mean one live warmup generation and three measured +generations inside each measured process. For streaming, those counts remain +process-level warmup/measured runs because steady state is measured from chunks +inside each process. Bidirectional outputs are under `outputs/bench/`: - `bench.json` - machine-readable inputs, medians, p90s, memory, and stage @@ -142,6 +144,13 @@ chart data. Set `FORCE_CUDNN_SDPA=1` only for backend-isolation probes. It is not the default `SANA-WM_bidirectional` benchmark setting. +Set `DISABLE_CUDNN_SDPA=1` when the local PyTorch/cuDNN stack auto-selects a +cuDNN scaled-dot-product-attention backend that fails on the benchmark GPU. This +is mainly a compatibility escape hatch for upstream runs on newer GPU stacks; +do not combine it with `FORCE_CUDNN_SDPA=1`. If the same host also fails in +cuDNN frontend convolution setup, combine it with +`TORCH_CUDNN_V8_API_DISABLED=1`. + Set `DEVICE_LABEL` when generating chart data for docs: ```bash diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index 76b59157b..e74b22c12 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -101,6 +101,8 @@ WARMUP_RUNS="${WARMUP_RUNS:-1}" MEASURED_RUNS="${MEASURED_RUNS:-3}" BIDIRECTIONAL_WARMUP_GENERATIONS="${BIDIRECTIONAL_WARMUP_GENERATIONS:-${WARMUP_RUNS}}" BIDIRECTIONAL_MEASURED_GENERATIONS="${BIDIRECTIONAL_MEASURED_GENERATIONS:-${MEASURED_RUNS}}" +FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" +DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA:-0}" BENCH_DRY_RUN="${BENCH_DRY_RUN:-0}" if [[ "${SANA_WM_VARIANT}" == "both" ]]; then @@ -131,6 +133,8 @@ if [[ "${SANA_WM_VARIANT}" == "both" ]]; then BENCH_PRECISIONS="${BENCH_PRECISIONS}" \ STAGE1_PRECISION=bf16 \ REFINER_PRECISION=bf16 \ + FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA}" \ + DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA}" \ "${BIDIRECTIONAL_GENERATION_ENV[@]}" \ "${BIDIRECTIONAL_OUTPUT_ENV[@]}" \ bash "${SCRIPT_DIR}/bench.sh" @@ -138,6 +142,8 @@ if [[ "${SANA_WM_VARIANT}" == "both" ]]; then echo "[bench] running SANA-WM_streaming benchmark" env \ SANA_WM_VARIANT=streaming \ + FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA}" \ + DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA}" \ "${STREAMING_PRECISION_ENV[@]}" \ "${STREAMING_OUTPUT_ENV[@]}" \ bash "${SCRIPT_DIR}/bench.sh" @@ -182,7 +188,6 @@ STREAMING_ACTION="${STREAMING_ACTION-w-80,dw-40,w-80,aw-40}" FPS="${FPS:-16}" STEP="${STEP:-60}" SEED="${SEED:-42}" -FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" COMPILE_STAGE1="${COMPILE_STAGE1:-0}" STREAMING_OUTPUT_MODE="${STREAMING_OUTPUT_MODE:-mp4}" @@ -208,6 +213,10 @@ if [[ "${SANA_WM_VARIANT}" == "streaming" ]] && _is_true "${COMPILE_STAGE1}"; th echo "[bench] ERROR: COMPILE_STAGE1=1 is bidirectional-only. Streaming compile parity is controlled by STREAMING_NO_COMPILE." >&2 exit 1 fi +if _is_true "${FORCE_CUDNN_SDPA}" && _is_true "${DISABLE_CUDNN_SDPA}"; then + echo "[bench] ERROR: FORCE_CUDNN_SDPA=1 and DISABLE_CUDNN_SDPA=1 are mutually exclusive." >&2 + exit 1 +fi if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then if [[ "${BENCH_SIDE}" != "flashdreams" ]] && [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; then _reject_bidirectional_low_precision @@ -236,6 +245,7 @@ if _is_true "${BENCH_DRY_RUN}"; then echo " side: ${BENCH_SIDE}" echo " output: ${OUTPUT_DIR}" echo " precision: stage1=${STAGE1_PRECISION} refiner=${REFINER_PRECISION}" + echo " disable_cudnn_sdpa: ${DISABLE_CUDNN_SDPA}" if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then echo " bidirectional generations: warmup=${BIDIRECTIONAL_WARMUP_GENERATIONS} measured=${BIDIRECTIONAL_MEASURED_GENERATIONS}" if [[ -n "${BENCH_PRECISIONS}" ]]; then @@ -286,6 +296,8 @@ if [[ -n "${BENCH_PRECISIONS}" ]]; then QUANT_BACKEND="${QUANT_BACKEND}" \ DEVICE_LABEL="${DEVICE_LABEL}" \ CHART_LABEL="${CHART_LABEL}" \ + FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA}" \ + DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA}" \ bash "${SCRIPT_DIR}/bench.sh" SWEEP_ITEMS+=(--item "${PRECISION_LABEL}:${PRECISION_OUTPUT_DIR}/bench.json") done @@ -387,6 +399,12 @@ if _is_true "${NO_REFINER}"; then fi UPSTREAM_BACKEND_ARGS=() FLASHDREAMS_BACKEND_ARGS=() +if _is_true "${DISABLE_CUDNN_SDPA}"; then + if [[ "${SANA_WM_VARIANT}" != "streaming" ]]; then + UPSTREAM_BACKEND_ARGS+=(--disable_cudnn_sdpa) + fi + FLASHDREAMS_BACKEND_ARGS+=(--disable-cudnn-sdpa) +fi if _is_true "${FORCE_CUDNN_SDPA}"; then if [[ "${SANA_WM_VARIANT}" != "streaming" ]]; then UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) @@ -449,6 +467,7 @@ _run_upstream_once() { if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then upstream_cmd=( env "PYTHONPATH=${UPSTREAM_PYTHONPATH}" + "SANA_WM_DISABLE_CUDNN_SDPA=${DISABLE_CUDNN_SDPA}" uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm_streaming.py" --image "${IMAGE_PATH}" --prompt "${PROMPT_PATH}" @@ -477,6 +496,7 @@ _run_upstream_once() { else upstream_cmd=( env "PYTHONPATH=${UPSTREAM_PYTHONPATH}" + "SANA_WM_DISABLE_CUDNN_SDPA=${DISABLE_CUDNN_SDPA}" uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm.py" --image "${IMAGE_PATH}" --prompt "${PROMPT_PATH}" @@ -593,6 +613,9 @@ fi if _is_true "${FORCE_CUDNN_SDPA}"; then SUMMARY_FLAGS+=(--force-cudnn-sdpa) fi +if _is_true "${DISABLE_CUDNN_SDPA}"; then + SUMMARY_FLAGS+=(--disable-cudnn-sdpa) +fi SUMMARY_WARMUP_RUNS="${WARMUP_RUNS}" if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then SUMMARY_WARMUP_RUNS="${BIDIRECTIONAL_WARMUP_GENERATIONS}" diff --git a/integrations/sana/tests/parity_check/bench_summary.py b/integrations/sana/tests/parity_check/bench_summary.py index c5d8f847d..e959c0188 100644 --- a/integrations/sana/tests/parity_check/bench_summary.py +++ b/integrations/sana/tests/parity_check/bench_summary.py @@ -426,6 +426,7 @@ def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: f"- quant_backend: `{summary['inputs']['quant_backend']}`", f"- compile_stage1: `{summary['inputs']['compile_stage1']}`", f"- force_cudnn_sdpa: `{summary['inputs']['force_cudnn_sdpa']}`", + f"- disable_cudnn_sdpa: `{summary['inputs']['disable_cudnn_sdpa']}`", f"- warmup generations discarded: `{summary['inputs']['warmup_runs']}`", "", "## Benchmark metric", @@ -719,6 +720,7 @@ def main(argv: list[str] | None = None) -> None: ) parser.add_argument("--compile-stage1", action="store_true") parser.add_argument("--force-cudnn-sdpa", action="store_true") + parser.add_argument("--disable-cudnn-sdpa", action="store_true") parser.add_argument("--device-label", default="GPU") parser.add_argument("--chart-label", default=None) parser.add_argument("--upstream-commit", default=None) @@ -790,6 +792,7 @@ def main(argv: list[str] | None = None) -> None: "quant_backend": args.quant_backend, "compile_stage1": args.compile_stage1, "force_cudnn_sdpa": args.force_cudnn_sdpa, + "disable_cudnn_sdpa": args.disable_cudnn_sdpa, "warmup_runs": args.warmup_runs, "warmup_generations": ( args.warmup_runs if args.variant == "bidirectional" else None diff --git a/integrations/sana/tests/parity_check/changes_bidirectional.patch b/integrations/sana/tests/parity_check/changes_bidirectional.patch index f9a6e94c9..2aa9b0b72 100644 --- a/integrations/sana/tests/parity_check/changes_bidirectional.patch +++ b/integrations/sana/tests/parity_check/changes_bidirectional.patch @@ -1,6 +1,6 @@ --- a/inference_video_scripts/wm/inference_sana_wm.py +++ b/inference_video_scripts/wm/inference_sana_wm.py -@@ -937,6 +937,101 @@ +@@ -937,6 +937,106 @@ return video_path @@ -26,6 +26,11 @@ + pass + + ++def disable_cudnn_sdpa() -> None: ++ if torch.cuda.is_available(): ++ torch.backends.cuda.enable_cudnn_sdp(False) ++ ++ +def apply_precision_args(args: argparse.Namespace, logger: logging.Logger) -> None: + stage1_precision = args.stage1_precision + refiner_precision = "bf16" if args.no_refiner else args.refiner_precision @@ -102,7 +107,7 @@ # ============================================================================ # Pipeline # ============================================================================ -@@ -2132,6 +2227,32 @@ +@@ -2132,6 +2232,37 @@ action="store_true", help="Skip rendering the WASD + joystick overlay on the output video.", ) @@ -114,6 +119,11 @@ + help="Force PyTorch scaled_dot_product_attention to use the cuDNN backend.", + ) + p.add_argument( ++ "--disable_cudnn_sdpa", ++ action="store_true", ++ help="Disable PyTorch's cuDNN scaled_dot_product_attention backend.", ++ ) ++ p.add_argument( + "--compile_stage1", + action="store_true", + help="Wrap the Stage-1 DiT with torch.compile(mode='max-autotune-no-cudagraphs').", @@ -135,12 +145,16 @@ # Weights and config. p.add_argument( -@@ -2222,7 +2343,15 @@ +@@ -2222,7 +2353,19 @@ def main() -> None: args = _build_parser().parse_args() ++ if args.force_cudnn_sdpa and args.disable_cudnn_sdpa: ++ raise ValueError("--force_cudnn_sdpa and --disable_cudnn_sdpa are mutually exclusive") + if args.force_cudnn_sdpa: + apply_backend_defaults() ++ if args.disable_cudnn_sdpa: ++ disable_cudnn_sdpa() + if args.warmup_generations < 0: + raise ValueError("--warmup_generations must be >= 0") + if args.measured_generations < 1: @@ -151,7 +165,7 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") image = Image.open(args.image).convert("RGB") -@@ -2276,6 +2405,15 @@ +@@ -2276,6 +2419,15 @@ offload_refiner=args.offload_refiner, logger=logger, ) @@ -167,7 +181,7 @@ denoising_step_list: list[int] | None = None if args.denoising_step_list: -@@ -2308,14 +2446,79 @@ +@@ -2308,14 +2460,80 @@ save_stage1=args.save_stage1, ) @@ -238,6 +252,7 @@ + "refiner_precision": args.refiner_precision, + "compile_stage1": args.compile_stage1, + "force_cudnn_sdpa": args.force_cudnn_sdpa, ++ "disable_cudnn_sdpa": args.disable_cudnn_sdpa, + "warmup_generations": args.warmup_generations, + "measured_generations": args.measured_generations, + "wall_s": final_record["wall_s"], diff --git a/integrations/sana/tests/parity_check/compat/sitecustomize.py b/integrations/sana/tests/parity_check/compat/sitecustomize.py new file mode 100644 index 000000000..9d6ae2119 --- /dev/null +++ b/integrations/sana/tests/parity_check/compat/sitecustomize.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Python startup hooks for the upstream SANA benchmark subprocesses.""" + +from __future__ import annotations + +import os + + +def _is_true(value: str | None) -> bool: + return value is not None and value.lower() in {"1", "true", "yes", "on"} + + +if _is_true(os.environ.get("SANA_WM_DISABLE_CUDNN_SDPA")): + import torch + + if torch.cuda.is_available(): + torch.backends.cuda.enable_cudnn_sdp(False) diff --git a/integrations/sana/tests/parity_check/run_bidirectional.sh b/integrations/sana/tests/parity_check/run_bidirectional.sh index 67faf566d..adc626e12 100644 --- a/integrations/sana/tests/parity_check/run_bidirectional.sh +++ b/integrations/sana/tests/parity_check/run_bidirectional.sh @@ -39,6 +39,7 @@ CFG_SCALE="${CFG_SCALE:-5.0}" SEED="${SEED:-42}" NO_REFINER="${NO_REFINER:-1}" FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" +DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA:-0}" COMPILE_STAGE1="${COMPILE_STAGE1:-0}" STAGE1_PRECISION="${STAGE1_PRECISION:-bf16}" REFINER_PRECISION="${REFINER_PRECISION:-${STAGE1_PRECISION}}" @@ -63,6 +64,14 @@ if parity_is_true "${NO_REFINER}"; then fi UPSTREAM_BACKEND_ARGS=() FLASHDREAMS_BACKEND_ARGS=() +if parity_is_true "${FORCE_CUDNN_SDPA}" && parity_is_true "${DISABLE_CUDNN_SDPA}"; then + echo "[run] ERROR: FORCE_CUDNN_SDPA=1 and DISABLE_CUDNN_SDPA=1 are mutually exclusive." >&2 + exit 1 +fi +if parity_is_true "${DISABLE_CUDNN_SDPA}"; then + UPSTREAM_BACKEND_ARGS+=(--disable_cudnn_sdpa) + FLASHDREAMS_BACKEND_ARGS+=(--disable-cudnn-sdpa) +fi if parity_is_true "${FORCE_CUDNN_SDPA}"; then UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) FLASHDREAMS_BACKEND_ARGS+=(--force-cudnn-sdpa) @@ -86,6 +95,7 @@ FLASHDREAMS_PRECISION_ARGS=( echo "[run] upstream SANA-WM bidirectional -> ${UPSTREAM_OUT}" ( cd "${PARITY_SCRIPT_DIR}" && \ PYTHONPATH="${UPSTREAM_PYTHONPATH}" \ + SANA_WM_DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA}" \ uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm.py" \ --image "${IMAGE_PATH}" \ --prompt "${PROMPT_PATH}" \ diff --git a/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py b/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py index bbc627b0a..68ee9326c 100644 --- a/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py +++ b/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py @@ -27,6 +27,7 @@ from flashdreams.infra.config import derive_config from run_flashdreams_common import ( apply_backend_defaults, + disable_cudnn_sdpa, install_stage1_compile_hook, preload_pipeline_components, prepare_bidirectional_inputs, @@ -82,11 +83,14 @@ def main(argv: list[str] | None = None) -> None: default="auto", ) parser.add_argument("--force-cudnn-sdpa", action="store_true") + parser.add_argument("--disable-cudnn-sdpa", action="store_true") parser.add_argument("--compile-stage1", action="store_true") parser.add_argument("--warmup-generations", type=int, default=0) parser.add_argument("--measured-generations", type=int, default=1) args = parser.parse_args(argv) + if args.force_cudnn_sdpa and args.disable_cudnn_sdpa: + raise ValueError("--force-cudnn-sdpa and --disable-cudnn-sdpa are mutually exclusive") if args.warmup_generations < 0: raise ValueError("--warmup-generations must be >= 0") if args.measured_generations < 1: @@ -94,6 +98,8 @@ def main(argv: list[str] | None = None) -> None: if args.force_cudnn_sdpa: apply_backend_defaults() + if args.disable_cudnn_sdpa: + disable_cudnn_sdpa() prompt = args.prompt_path.read_text(encoding="utf-8", errors="replace").strip() if not prompt: @@ -236,6 +242,7 @@ def main(argv: list[str] | None = None) -> None: "refiner_precision": args.refiner_precision, "compile_stage1": args.compile_stage1, "force_cudnn_sdpa": args.force_cudnn_sdpa, + "disable_cudnn_sdpa": args.disable_cudnn_sdpa, "warmup_generations": args.warmup_generations, "measured_generations": args.measured_generations, "wall_s": final_record["wall_s"], diff --git a/integrations/sana/tests/parity_check/run_flashdreams_common.py b/integrations/sana/tests/parity_check/run_flashdreams_common.py index 41608ca07..5d8c2e34c 100644 --- a/integrations/sana/tests/parity_check/run_flashdreams_common.py +++ b/integrations/sana/tests/parity_check/run_flashdreams_common.py @@ -61,6 +61,12 @@ def apply_backend_defaults() -> None: pass +def disable_cudnn_sdpa() -> None: + """Disable PyTorch's cuDNN SDPA backend while leaving other SDPA choices available.""" + if torch.cuda.is_available(): + torch.backends.cuda.enable_cudnn_sdp(False) + + def resolve_device(device: str) -> torch.device: """Resolve ``auto``/``cuda`` to the local rank's CUDA device when present.""" if device == "auto": diff --git a/integrations/sana/tests/parity_check/run_flashdreams_streaming.py b/integrations/sana/tests/parity_check/run_flashdreams_streaming.py index 828949628..d0b598e47 100644 --- a/integrations/sana/tests/parity_check/run_flashdreams_streaming.py +++ b/integrations/sana/tests/parity_check/run_flashdreams_streaming.py @@ -28,6 +28,7 @@ from run_flashdreams_common import ( apply_backend_defaults, compile_streaming_refiner, + disable_cudnn_sdpa, install_stage1_compile_hook, preload_pipeline_components, prepare_streaming_inputs, @@ -99,6 +100,7 @@ def main(argv: list[str] | None = None) -> None: default="auto", ) parser.add_argument("--force-cudnn-sdpa", action="store_true") + parser.add_argument("--disable-cudnn-sdpa", action="store_true") parser.add_argument("--compile-stage1", action="store_true") parser.add_argument("--compile-streaming-refiner", action="store_true") parser.add_argument("--output-mode", choices=["mp4", "discard"], default="mp4") @@ -127,8 +129,12 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--refiner-seed", type=int, default=None) args = parser.parse_args(argv) + if args.force_cudnn_sdpa and args.disable_cudnn_sdpa: + raise ValueError("--force-cudnn-sdpa and --disable-cudnn-sdpa are mutually exclusive") if args.force_cudnn_sdpa: apply_backend_defaults() + if args.disable_cudnn_sdpa: + disable_cudnn_sdpa() prompt = args.prompt_path.read_text(encoding="utf-8", errors="replace").strip() if not prompt: @@ -338,6 +344,7 @@ def main(argv: list[str] | None = None) -> None: "compile_stage1": args.compile_stage1, "compile_streaming_refiner": args.compile_streaming_refiner, "force_cudnn_sdpa": args.force_cudnn_sdpa, + "disable_cudnn_sdpa": args.disable_cudnn_sdpa, "denoising_step_list": list(denoising_step_list), "num_frame_per_block": args.num_frame_per_block, "num_cached_blocks": args.num_cached_blocks, diff --git a/integrations/sana/tests/parity_check/run_streaming.sh b/integrations/sana/tests/parity_check/run_streaming.sh index 42910e18d..d3d9fd2ea 100644 --- a/integrations/sana/tests/parity_check/run_streaming.sh +++ b/integrations/sana/tests/parity_check/run_streaming.sh @@ -114,6 +114,14 @@ if ! parity_is_true "${STREAMING_NO_COMPILE}"; then FLASHDREAMS_COMPILE_ARGS+=(--compile-streaming-refiner) fi FLASHDREAMS_BACKEND_ARGS=() +DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA:-0}" +if parity_is_true "${FORCE_CUDNN_SDPA}" && parity_is_true "${DISABLE_CUDNN_SDPA}"; then + echo "[run] ERROR: FORCE_CUDNN_SDPA=1 and DISABLE_CUDNN_SDPA=1 are mutually exclusive." >&2 + exit 1 +fi +if parity_is_true "${DISABLE_CUDNN_SDPA}"; then + FLASHDREAMS_BACKEND_ARGS+=(--disable-cudnn-sdpa) +fi if parity_is_true "${FORCE_CUDNN_SDPA}"; then FLASHDREAMS_BACKEND_ARGS+=(--force-cudnn-sdpa) fi @@ -121,6 +129,7 @@ fi echo "[run] upstream SANA-WM streaming -> ${UPSTREAM_OUT}" ( cd "${PARITY_SCRIPT_DIR}" && \ PYTHONPATH="${UPSTREAM_PYTHONPATH}" \ + SANA_WM_DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA}" \ uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm_streaming.py" \ --image "${IMAGE_PATH}" \ --prompt "${PROMPT_PATH}" \ diff --git a/integrations/sana/tests/test_parity_benchmark_summary.py b/integrations/sana/tests/test_parity_benchmark_summary.py index 5aaa17285..07f18bea3 100644 --- a/integrations/sana/tests/test_parity_benchmark_summary.py +++ b/integrations/sana/tests/test_parity_benchmark_summary.py @@ -65,6 +65,7 @@ def test_bidirectional_summary_uses_steady_state_generation_ms_per_clip_for_char "quant_backend": "auto", "compile_stage1": False, "force_cudnn_sdpa": True, + "disable_cudnn_sdpa": False, "warmup_runs": 1, }, "upstream": { From 416fbb5e053f85a7250b7479c8b4bd99a1704048 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 30 Jul 2026 19:04:21 -0700 Subject: [PATCH 50/64] Revert "Fix issues running on fresh machine?" This reverts commit 3fede3f9ffac6babfb18b4e462603f6f9c91d445. --- .../sana/tests/parity_check/README.md | 17 +++---------- integrations/sana/tests/parity_check/bench.sh | 25 +------------------ .../sana/tests/parity_check/bench_summary.py | 3 --- .../parity_check/changes_bidirectional.patch | 25 ++++--------------- .../parity_check/compat/sitecustomize.py | 19 -------------- .../tests/parity_check/run_bidirectional.sh | 10 -------- .../run_flashdreams_bidirectional.py | 7 ------ .../parity_check/run_flashdreams_common.py | 6 ----- .../parity_check/run_flashdreams_streaming.py | 7 ------ .../sana/tests/parity_check/run_streaming.sh | 9 ------- .../tests/test_parity_benchmark_summary.py | 1 - 11 files changed, 10 insertions(+), 119 deletions(-) delete mode 100644 integrations/sana/tests/parity_check/compat/sitecustomize.py diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index 0cb1c66e9..f712c2f4e 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -58,7 +58,6 @@ Defaults: - `REFINER_PRECISION=$STAGE1_PRECISION` - `QUANT_BACKEND=auto` - `FORCE_CUDNN_SDPA=0` -- `DISABLE_CUDNN_SDPA=0` Outputs are written under `outputs/parity/`: @@ -99,11 +98,10 @@ bash bench.sh Benchmark defaults to both sibling variants: `SANA_WM_VARIANT=both BENCH_SIDE=both WARMUP_RUNS=1 MEASURED_RUNS=3 BENCH_PRECISIONS=bf16,fp8,fp4 COMPILE_STAGE1=0 NO_REFINER=0 -FORCE_CUDNN_SDPA=0 DISABLE_CUDNN_SDPA=0`. For bidirectional, the -warmup/measured counts mean one live warmup generation and three measured -generations inside each measured process. For streaming, those counts remain -process-level warmup/measured runs because steady state is measured from chunks -inside each process. +FORCE_CUDNN_SDPA=0`. For bidirectional, the warmup/measured counts mean one +live warmup generation and three measured generations inside each measured +process. For streaming, those counts remain process-level warmup/measured runs +because steady state is measured from chunks inside each process. Bidirectional outputs are under `outputs/bench/`: - `bench.json` - machine-readable inputs, medians, p90s, memory, and stage @@ -144,13 +142,6 @@ chart data. Set `FORCE_CUDNN_SDPA=1` only for backend-isolation probes. It is not the default `SANA-WM_bidirectional` benchmark setting. -Set `DISABLE_CUDNN_SDPA=1` when the local PyTorch/cuDNN stack auto-selects a -cuDNN scaled-dot-product-attention backend that fails on the benchmark GPU. This -is mainly a compatibility escape hatch for upstream runs on newer GPU stacks; -do not combine it with `FORCE_CUDNN_SDPA=1`. If the same host also fails in -cuDNN frontend convolution setup, combine it with -`TORCH_CUDNN_V8_API_DISABLED=1`. - Set `DEVICE_LABEL` when generating chart data for docs: ```bash diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index e74b22c12..76b59157b 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -101,8 +101,6 @@ WARMUP_RUNS="${WARMUP_RUNS:-1}" MEASURED_RUNS="${MEASURED_RUNS:-3}" BIDIRECTIONAL_WARMUP_GENERATIONS="${BIDIRECTIONAL_WARMUP_GENERATIONS:-${WARMUP_RUNS}}" BIDIRECTIONAL_MEASURED_GENERATIONS="${BIDIRECTIONAL_MEASURED_GENERATIONS:-${MEASURED_RUNS}}" -FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" -DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA:-0}" BENCH_DRY_RUN="${BENCH_DRY_RUN:-0}" if [[ "${SANA_WM_VARIANT}" == "both" ]]; then @@ -133,8 +131,6 @@ if [[ "${SANA_WM_VARIANT}" == "both" ]]; then BENCH_PRECISIONS="${BENCH_PRECISIONS}" \ STAGE1_PRECISION=bf16 \ REFINER_PRECISION=bf16 \ - FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA}" \ - DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA}" \ "${BIDIRECTIONAL_GENERATION_ENV[@]}" \ "${BIDIRECTIONAL_OUTPUT_ENV[@]}" \ bash "${SCRIPT_DIR}/bench.sh" @@ -142,8 +138,6 @@ if [[ "${SANA_WM_VARIANT}" == "both" ]]; then echo "[bench] running SANA-WM_streaming benchmark" env \ SANA_WM_VARIANT=streaming \ - FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA}" \ - DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA}" \ "${STREAMING_PRECISION_ENV[@]}" \ "${STREAMING_OUTPUT_ENV[@]}" \ bash "${SCRIPT_DIR}/bench.sh" @@ -188,6 +182,7 @@ STREAMING_ACTION="${STREAMING_ACTION-w-80,dw-40,w-80,aw-40}" FPS="${FPS:-16}" STEP="${STEP:-60}" SEED="${SEED:-42}" +FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" COMPILE_STAGE1="${COMPILE_STAGE1:-0}" STREAMING_OUTPUT_MODE="${STREAMING_OUTPUT_MODE:-mp4}" @@ -213,10 +208,6 @@ if [[ "${SANA_WM_VARIANT}" == "streaming" ]] && _is_true "${COMPILE_STAGE1}"; th echo "[bench] ERROR: COMPILE_STAGE1=1 is bidirectional-only. Streaming compile parity is controlled by STREAMING_NO_COMPILE." >&2 exit 1 fi -if _is_true "${FORCE_CUDNN_SDPA}" && _is_true "${DISABLE_CUDNN_SDPA}"; then - echo "[bench] ERROR: FORCE_CUDNN_SDPA=1 and DISABLE_CUDNN_SDPA=1 are mutually exclusive." >&2 - exit 1 -fi if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then if [[ "${BENCH_SIDE}" != "flashdreams" ]] && [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; then _reject_bidirectional_low_precision @@ -245,7 +236,6 @@ if _is_true "${BENCH_DRY_RUN}"; then echo " side: ${BENCH_SIDE}" echo " output: ${OUTPUT_DIR}" echo " precision: stage1=${STAGE1_PRECISION} refiner=${REFINER_PRECISION}" - echo " disable_cudnn_sdpa: ${DISABLE_CUDNN_SDPA}" if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then echo " bidirectional generations: warmup=${BIDIRECTIONAL_WARMUP_GENERATIONS} measured=${BIDIRECTIONAL_MEASURED_GENERATIONS}" if [[ -n "${BENCH_PRECISIONS}" ]]; then @@ -296,8 +286,6 @@ if [[ -n "${BENCH_PRECISIONS}" ]]; then QUANT_BACKEND="${QUANT_BACKEND}" \ DEVICE_LABEL="${DEVICE_LABEL}" \ CHART_LABEL="${CHART_LABEL}" \ - FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA}" \ - DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA}" \ bash "${SCRIPT_DIR}/bench.sh" SWEEP_ITEMS+=(--item "${PRECISION_LABEL}:${PRECISION_OUTPUT_DIR}/bench.json") done @@ -399,12 +387,6 @@ if _is_true "${NO_REFINER}"; then fi UPSTREAM_BACKEND_ARGS=() FLASHDREAMS_BACKEND_ARGS=() -if _is_true "${DISABLE_CUDNN_SDPA}"; then - if [[ "${SANA_WM_VARIANT}" != "streaming" ]]; then - UPSTREAM_BACKEND_ARGS+=(--disable_cudnn_sdpa) - fi - FLASHDREAMS_BACKEND_ARGS+=(--disable-cudnn-sdpa) -fi if _is_true "${FORCE_CUDNN_SDPA}"; then if [[ "${SANA_WM_VARIANT}" != "streaming" ]]; then UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) @@ -467,7 +449,6 @@ _run_upstream_once() { if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then upstream_cmd=( env "PYTHONPATH=${UPSTREAM_PYTHONPATH}" - "SANA_WM_DISABLE_CUDNN_SDPA=${DISABLE_CUDNN_SDPA}" uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm_streaming.py" --image "${IMAGE_PATH}" --prompt "${PROMPT_PATH}" @@ -496,7 +477,6 @@ _run_upstream_once() { else upstream_cmd=( env "PYTHONPATH=${UPSTREAM_PYTHONPATH}" - "SANA_WM_DISABLE_CUDNN_SDPA=${DISABLE_CUDNN_SDPA}" uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm.py" --image "${IMAGE_PATH}" --prompt "${PROMPT_PATH}" @@ -613,9 +593,6 @@ fi if _is_true "${FORCE_CUDNN_SDPA}"; then SUMMARY_FLAGS+=(--force-cudnn-sdpa) fi -if _is_true "${DISABLE_CUDNN_SDPA}"; then - SUMMARY_FLAGS+=(--disable-cudnn-sdpa) -fi SUMMARY_WARMUP_RUNS="${WARMUP_RUNS}" if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then SUMMARY_WARMUP_RUNS="${BIDIRECTIONAL_WARMUP_GENERATIONS}" diff --git a/integrations/sana/tests/parity_check/bench_summary.py b/integrations/sana/tests/parity_check/bench_summary.py index e959c0188..c5d8f847d 100644 --- a/integrations/sana/tests/parity_check/bench_summary.py +++ b/integrations/sana/tests/parity_check/bench_summary.py @@ -426,7 +426,6 @@ def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: f"- quant_backend: `{summary['inputs']['quant_backend']}`", f"- compile_stage1: `{summary['inputs']['compile_stage1']}`", f"- force_cudnn_sdpa: `{summary['inputs']['force_cudnn_sdpa']}`", - f"- disable_cudnn_sdpa: `{summary['inputs']['disable_cudnn_sdpa']}`", f"- warmup generations discarded: `{summary['inputs']['warmup_runs']}`", "", "## Benchmark metric", @@ -720,7 +719,6 @@ def main(argv: list[str] | None = None) -> None: ) parser.add_argument("--compile-stage1", action="store_true") parser.add_argument("--force-cudnn-sdpa", action="store_true") - parser.add_argument("--disable-cudnn-sdpa", action="store_true") parser.add_argument("--device-label", default="GPU") parser.add_argument("--chart-label", default=None) parser.add_argument("--upstream-commit", default=None) @@ -792,7 +790,6 @@ def main(argv: list[str] | None = None) -> None: "quant_backend": args.quant_backend, "compile_stage1": args.compile_stage1, "force_cudnn_sdpa": args.force_cudnn_sdpa, - "disable_cudnn_sdpa": args.disable_cudnn_sdpa, "warmup_runs": args.warmup_runs, "warmup_generations": ( args.warmup_runs if args.variant == "bidirectional" else None diff --git a/integrations/sana/tests/parity_check/changes_bidirectional.patch b/integrations/sana/tests/parity_check/changes_bidirectional.patch index 2aa9b0b72..f9a6e94c9 100644 --- a/integrations/sana/tests/parity_check/changes_bidirectional.patch +++ b/integrations/sana/tests/parity_check/changes_bidirectional.patch @@ -1,6 +1,6 @@ --- a/inference_video_scripts/wm/inference_sana_wm.py +++ b/inference_video_scripts/wm/inference_sana_wm.py -@@ -937,6 +937,106 @@ +@@ -937,6 +937,101 @@ return video_path @@ -26,11 +26,6 @@ + pass + + -+def disable_cudnn_sdpa() -> None: -+ if torch.cuda.is_available(): -+ torch.backends.cuda.enable_cudnn_sdp(False) -+ -+ +def apply_precision_args(args: argparse.Namespace, logger: logging.Logger) -> None: + stage1_precision = args.stage1_precision + refiner_precision = "bf16" if args.no_refiner else args.refiner_precision @@ -107,7 +102,7 @@ # ============================================================================ # Pipeline # ============================================================================ -@@ -2132,6 +2232,37 @@ +@@ -2132,6 +2227,32 @@ action="store_true", help="Skip rendering the WASD + joystick overlay on the output video.", ) @@ -119,11 +114,6 @@ + help="Force PyTorch scaled_dot_product_attention to use the cuDNN backend.", + ) + p.add_argument( -+ "--disable_cudnn_sdpa", -+ action="store_true", -+ help="Disable PyTorch's cuDNN scaled_dot_product_attention backend.", -+ ) -+ p.add_argument( + "--compile_stage1", + action="store_true", + help="Wrap the Stage-1 DiT with torch.compile(mode='max-autotune-no-cudagraphs').", @@ -145,16 +135,12 @@ # Weights and config. p.add_argument( -@@ -2222,7 +2353,19 @@ +@@ -2222,7 +2343,15 @@ def main() -> None: args = _build_parser().parse_args() -+ if args.force_cudnn_sdpa and args.disable_cudnn_sdpa: -+ raise ValueError("--force_cudnn_sdpa and --disable_cudnn_sdpa are mutually exclusive") + if args.force_cudnn_sdpa: + apply_backend_defaults() -+ if args.disable_cudnn_sdpa: -+ disable_cudnn_sdpa() + if args.warmup_generations < 0: + raise ValueError("--warmup_generations must be >= 0") + if args.measured_generations < 1: @@ -165,7 +151,7 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") image = Image.open(args.image).convert("RGB") -@@ -2276,6 +2419,15 @@ +@@ -2276,6 +2405,15 @@ offload_refiner=args.offload_refiner, logger=logger, ) @@ -181,7 +167,7 @@ denoising_step_list: list[int] | None = None if args.denoising_step_list: -@@ -2308,14 +2460,80 @@ +@@ -2308,14 +2446,79 @@ save_stage1=args.save_stage1, ) @@ -252,7 +238,6 @@ + "refiner_precision": args.refiner_precision, + "compile_stage1": args.compile_stage1, + "force_cudnn_sdpa": args.force_cudnn_sdpa, -+ "disable_cudnn_sdpa": args.disable_cudnn_sdpa, + "warmup_generations": args.warmup_generations, + "measured_generations": args.measured_generations, + "wall_s": final_record["wall_s"], diff --git a/integrations/sana/tests/parity_check/compat/sitecustomize.py b/integrations/sana/tests/parity_check/compat/sitecustomize.py deleted file mode 100644 index 9d6ae2119..000000000 --- a/integrations/sana/tests/parity_check/compat/sitecustomize.py +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Python startup hooks for the upstream SANA benchmark subprocesses.""" - -from __future__ import annotations - -import os - - -def _is_true(value: str | None) -> bool: - return value is not None and value.lower() in {"1", "true", "yes", "on"} - - -if _is_true(os.environ.get("SANA_WM_DISABLE_CUDNN_SDPA")): - import torch - - if torch.cuda.is_available(): - torch.backends.cuda.enable_cudnn_sdp(False) diff --git a/integrations/sana/tests/parity_check/run_bidirectional.sh b/integrations/sana/tests/parity_check/run_bidirectional.sh index adc626e12..67faf566d 100644 --- a/integrations/sana/tests/parity_check/run_bidirectional.sh +++ b/integrations/sana/tests/parity_check/run_bidirectional.sh @@ -39,7 +39,6 @@ CFG_SCALE="${CFG_SCALE:-5.0}" SEED="${SEED:-42}" NO_REFINER="${NO_REFINER:-1}" FORCE_CUDNN_SDPA="${FORCE_CUDNN_SDPA:-0}" -DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA:-0}" COMPILE_STAGE1="${COMPILE_STAGE1:-0}" STAGE1_PRECISION="${STAGE1_PRECISION:-bf16}" REFINER_PRECISION="${REFINER_PRECISION:-${STAGE1_PRECISION}}" @@ -64,14 +63,6 @@ if parity_is_true "${NO_REFINER}"; then fi UPSTREAM_BACKEND_ARGS=() FLASHDREAMS_BACKEND_ARGS=() -if parity_is_true "${FORCE_CUDNN_SDPA}" && parity_is_true "${DISABLE_CUDNN_SDPA}"; then - echo "[run] ERROR: FORCE_CUDNN_SDPA=1 and DISABLE_CUDNN_SDPA=1 are mutually exclusive." >&2 - exit 1 -fi -if parity_is_true "${DISABLE_CUDNN_SDPA}"; then - UPSTREAM_BACKEND_ARGS+=(--disable_cudnn_sdpa) - FLASHDREAMS_BACKEND_ARGS+=(--disable-cudnn-sdpa) -fi if parity_is_true "${FORCE_CUDNN_SDPA}"; then UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) FLASHDREAMS_BACKEND_ARGS+=(--force-cudnn-sdpa) @@ -95,7 +86,6 @@ FLASHDREAMS_PRECISION_ARGS=( echo "[run] upstream SANA-WM bidirectional -> ${UPSTREAM_OUT}" ( cd "${PARITY_SCRIPT_DIR}" && \ PYTHONPATH="${UPSTREAM_PYTHONPATH}" \ - SANA_WM_DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA}" \ uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm.py" \ --image "${IMAGE_PATH}" \ --prompt "${PROMPT_PATH}" \ diff --git a/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py b/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py index 68ee9326c..bbc627b0a 100644 --- a/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py +++ b/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py @@ -27,7 +27,6 @@ from flashdreams.infra.config import derive_config from run_flashdreams_common import ( apply_backend_defaults, - disable_cudnn_sdpa, install_stage1_compile_hook, preload_pipeline_components, prepare_bidirectional_inputs, @@ -83,14 +82,11 @@ def main(argv: list[str] | None = None) -> None: default="auto", ) parser.add_argument("--force-cudnn-sdpa", action="store_true") - parser.add_argument("--disable-cudnn-sdpa", action="store_true") parser.add_argument("--compile-stage1", action="store_true") parser.add_argument("--warmup-generations", type=int, default=0) parser.add_argument("--measured-generations", type=int, default=1) args = parser.parse_args(argv) - if args.force_cudnn_sdpa and args.disable_cudnn_sdpa: - raise ValueError("--force-cudnn-sdpa and --disable-cudnn-sdpa are mutually exclusive") if args.warmup_generations < 0: raise ValueError("--warmup-generations must be >= 0") if args.measured_generations < 1: @@ -98,8 +94,6 @@ def main(argv: list[str] | None = None) -> None: if args.force_cudnn_sdpa: apply_backend_defaults() - if args.disable_cudnn_sdpa: - disable_cudnn_sdpa() prompt = args.prompt_path.read_text(encoding="utf-8", errors="replace").strip() if not prompt: @@ -242,7 +236,6 @@ def main(argv: list[str] | None = None) -> None: "refiner_precision": args.refiner_precision, "compile_stage1": args.compile_stage1, "force_cudnn_sdpa": args.force_cudnn_sdpa, - "disable_cudnn_sdpa": args.disable_cudnn_sdpa, "warmup_generations": args.warmup_generations, "measured_generations": args.measured_generations, "wall_s": final_record["wall_s"], diff --git a/integrations/sana/tests/parity_check/run_flashdreams_common.py b/integrations/sana/tests/parity_check/run_flashdreams_common.py index 5d8c2e34c..41608ca07 100644 --- a/integrations/sana/tests/parity_check/run_flashdreams_common.py +++ b/integrations/sana/tests/parity_check/run_flashdreams_common.py @@ -61,12 +61,6 @@ def apply_backend_defaults() -> None: pass -def disable_cudnn_sdpa() -> None: - """Disable PyTorch's cuDNN SDPA backend while leaving other SDPA choices available.""" - if torch.cuda.is_available(): - torch.backends.cuda.enable_cudnn_sdp(False) - - def resolve_device(device: str) -> torch.device: """Resolve ``auto``/``cuda`` to the local rank's CUDA device when present.""" if device == "auto": diff --git a/integrations/sana/tests/parity_check/run_flashdreams_streaming.py b/integrations/sana/tests/parity_check/run_flashdreams_streaming.py index d0b598e47..828949628 100644 --- a/integrations/sana/tests/parity_check/run_flashdreams_streaming.py +++ b/integrations/sana/tests/parity_check/run_flashdreams_streaming.py @@ -28,7 +28,6 @@ from run_flashdreams_common import ( apply_backend_defaults, compile_streaming_refiner, - disable_cudnn_sdpa, install_stage1_compile_hook, preload_pipeline_components, prepare_streaming_inputs, @@ -100,7 +99,6 @@ def main(argv: list[str] | None = None) -> None: default="auto", ) parser.add_argument("--force-cudnn-sdpa", action="store_true") - parser.add_argument("--disable-cudnn-sdpa", action="store_true") parser.add_argument("--compile-stage1", action="store_true") parser.add_argument("--compile-streaming-refiner", action="store_true") parser.add_argument("--output-mode", choices=["mp4", "discard"], default="mp4") @@ -129,12 +127,8 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--refiner-seed", type=int, default=None) args = parser.parse_args(argv) - if args.force_cudnn_sdpa and args.disable_cudnn_sdpa: - raise ValueError("--force-cudnn-sdpa and --disable-cudnn-sdpa are mutually exclusive") if args.force_cudnn_sdpa: apply_backend_defaults() - if args.disable_cudnn_sdpa: - disable_cudnn_sdpa() prompt = args.prompt_path.read_text(encoding="utf-8", errors="replace").strip() if not prompt: @@ -344,7 +338,6 @@ def main(argv: list[str] | None = None) -> None: "compile_stage1": args.compile_stage1, "compile_streaming_refiner": args.compile_streaming_refiner, "force_cudnn_sdpa": args.force_cudnn_sdpa, - "disable_cudnn_sdpa": args.disable_cudnn_sdpa, "denoising_step_list": list(denoising_step_list), "num_frame_per_block": args.num_frame_per_block, "num_cached_blocks": args.num_cached_blocks, diff --git a/integrations/sana/tests/parity_check/run_streaming.sh b/integrations/sana/tests/parity_check/run_streaming.sh index d3d9fd2ea..42910e18d 100644 --- a/integrations/sana/tests/parity_check/run_streaming.sh +++ b/integrations/sana/tests/parity_check/run_streaming.sh @@ -114,14 +114,6 @@ if ! parity_is_true "${STREAMING_NO_COMPILE}"; then FLASHDREAMS_COMPILE_ARGS+=(--compile-streaming-refiner) fi FLASHDREAMS_BACKEND_ARGS=() -DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA:-0}" -if parity_is_true "${FORCE_CUDNN_SDPA}" && parity_is_true "${DISABLE_CUDNN_SDPA}"; then - echo "[run] ERROR: FORCE_CUDNN_SDPA=1 and DISABLE_CUDNN_SDPA=1 are mutually exclusive." >&2 - exit 1 -fi -if parity_is_true "${DISABLE_CUDNN_SDPA}"; then - FLASHDREAMS_BACKEND_ARGS+=(--disable-cudnn-sdpa) -fi if parity_is_true "${FORCE_CUDNN_SDPA}"; then FLASHDREAMS_BACKEND_ARGS+=(--force-cudnn-sdpa) fi @@ -129,7 +121,6 @@ fi echo "[run] upstream SANA-WM streaming -> ${UPSTREAM_OUT}" ( cd "${PARITY_SCRIPT_DIR}" && \ PYTHONPATH="${UPSTREAM_PYTHONPATH}" \ - SANA_WM_DISABLE_CUDNN_SDPA="${DISABLE_CUDNN_SDPA}" \ uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm_streaming.py" \ --image "${IMAGE_PATH}" \ --prompt "${PROMPT_PATH}" \ diff --git a/integrations/sana/tests/test_parity_benchmark_summary.py b/integrations/sana/tests/test_parity_benchmark_summary.py index 07f18bea3..5aaa17285 100644 --- a/integrations/sana/tests/test_parity_benchmark_summary.py +++ b/integrations/sana/tests/test_parity_benchmark_summary.py @@ -65,7 +65,6 @@ def test_bidirectional_summary_uses_steady_state_generation_ms_per_clip_for_char "quant_backend": "auto", "compile_stage1": False, "force_cudnn_sdpa": True, - "disable_cudnn_sdpa": False, "warmup_runs": 1, }, "upstream": { From 284202d90d208c66b75c924060c2e0f55718f29b Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Thu, 30 Jul 2026 19:09:39 -0700 Subject: [PATCH 51/64] Fix issues running on fresh machine (for real) --- .../sana/tests/parity_check/README.md | 15 +++++++-- .../sana/tests/parity_check/pyproject.toml | 8 ++++- integrations/sana/tests/parity_check/uv.lock | 33 +++++++++++-------- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index f712c2f4e..e2ed877f3 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -142,7 +142,16 @@ chart data. Set `FORCE_CUDNN_SDPA=1` only for backend-isolation probes. It is not the default `SANA-WM_bidirectional` benchmark setting. -Set `DEVICE_LABEL` when generating chart data for docs: +On shared or mixed-GPU hosts, set `CUDA_VISIBLE_DEVICES` to the benchmark GPU's +UUID before running the harness. This keeps PyTorch's NVML and CUDA runtime +device enumeration consistent without changing the benchmark backend policy: + +```bash +CUDA_VISIBLE_DEVICES=GPU-... bash bench.sh +``` + +Set `DEVICE_LABEL` to label chart data with a device name that is not just +`GPU`: ```bash DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh @@ -151,7 +160,7 @@ DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh Run only the BF16 benchmark used by the `SANA-WM_bidirectional` model card with: ```bash -SANA_WM_VARIANT=bidirectional BENCH_PRECISIONS=bf16 DEVICE_LABEL="GB202" bash bench.sh +SANA_WM_VARIANT=bidirectional BENCH_PRECISIONS=bf16 bash bench.sh ``` The scripts do not set allocator overrides or GPU wait loops. If GPU contention @@ -206,7 +215,7 @@ FlashDreams-only bidirectional diagnostics, while the official bidirectional comparison stays BF16-only: ```bash -DEVICE_LABEL="GB202" bash bench.sh +bash bench.sh ``` FP8 and FP4 are passed through the precision flags supported by each measured diff --git a/integrations/sana/tests/parity_check/pyproject.toml b/integrations/sana/tests/parity_check/pyproject.toml index 5b5a3f788..6479ab92f 100644 --- a/integrations/sana/tests/parity_check/pyproject.toml +++ b/integrations/sana/tests/parity_check/pyproject.toml @@ -57,6 +57,12 @@ transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", [tool.uv] managed = true override-dependencies = [ - "nvidia-cublas>=13.4", + # PyTorch 2.13.0's default CUDA 13 dependency set can resolve cuDNN + # 9.20.0.48 on Linux. That stack fails standalone BF16 Conv3d and SDPA on + # GB300 with CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH. Pin the CUDA runtime + # packages that keep the default cuDNN backends working on GB202 and GB300. + "nvidia-cublas==13.6.1.10", + "nvidia-cuda-nvrtc==13.3.33", + "nvidia-cudnn-cu13==9.24.0.43", ] no-build-isolation-package = ["transformer-engine"] diff --git a/integrations/sana/tests/parity_check/uv.lock b/integrations/sana/tests/parity_check/uv.lock index af4d99550..4fc781db9 100644 --- a/integrations/sana/tests/parity_check/uv.lock +++ b/integrations/sana/tests/parity_check/uv.lock @@ -7,7 +7,11 @@ resolution-markers = [ ] [manifest] -overrides = [{ name = "nvidia-cublas", specifier = ">=13.4" }] +overrides = [ + { name = "nvidia-cublas", specifier = "==13.6.1.10" }, + { name = "nvidia-cuda-nvrtc", specifier = "==13.3.33" }, + { name = "nvidia-cudnn-cu13", specifier = "==9.24.0.43" }, +] [[package]] name = "accelerate" @@ -193,7 +197,7 @@ wheels = [ [package.optional-dependencies] cublas = [ { name = "nvidia-cublas" }, - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc" }, ] cudart = [ { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, @@ -225,7 +229,7 @@ nvjitlink = [ { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc" }, ] nvtx = [ { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, @@ -708,14 +712,14 @@ wheels = [ [[package]] name = "nvidia-cublas" -version = "13.6.0.2" +version = "13.6.1.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nvidia-cuda-nvrtc" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/22/7c08d8e93f6a2e4879ac83c12696aecaf17a0fc2c9e8d204caceaa3b8426/nvidia_cublas-13.6.0.2-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:946f6a252b1cc72d8de912c75975fd6d8ba44f67d4e5044fe764ddb909f4a688", size = 518599300, upload-time = "2026-06-29T16:54:56.859Z" }, - { url = "https://files.pythonhosted.org/packages/fd/6c/173c7a3db77a6592210f73f194f0f8ed5e51b6ec61cfed7b1eee06ac5fd3/nvidia_cublas-13.6.0.2-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b82c80c886cea6da6e149a5c3bdba274f12b7e4ec4b00a050b916b0446fb4153", size = 410473152, upload-time = "2026-06-29T16:55:54.297Z" }, + { url = "https://files.pythonhosted.org/packages/42/51/a174a0e79793e528ef4645a9d554e3aa48fd8da3f9c9cf523c0176bb121b/nvidia_cublas-13.6.1.10-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e05a431062a17cb9b02e2f37e67817b637051ce8fad57b388482c594396ddbb4", size = 518610411, upload-time = "2026-07-30T18:18:32.969Z" }, + { url = "https://files.pythonhosted.org/packages/dd/e9/288a93d8234b8f8ea0ccac7b8cc5d7aa4c663d9676c64f4a2eb3a1f9ffc4/nvidia_cublas-13.6.1.10-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:feb2ed8a1e211bc5774413efc0f1a08c4d5269b56f68b4ac6fe5408e57f7dc1c", size = 410475904, upload-time = "2026-07-30T18:19:26.388Z" }, ] [[package]] @@ -729,11 +733,12 @@ wheels = [ [[package]] name = "nvidia-cuda-nvrtc" -version = "13.0.88" +version = "13.3.33" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, - { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, + { url = "https://files.pythonhosted.org/packages/8b/2c/86916c8a34dcdb0c3ddd1c0e30545041bd781184e437b9cb76fcda70560b/nvidia_cuda_nvrtc-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:82530788b8c6164a54d3fd9ae8bcca8893d397c4aeb998861982a03bbe41e204", size = 51110910, upload-time = "2026-05-26T16:38:16.116Z" }, + { url = "https://files.pythonhosted.org/packages/e7/b6/60a3641111d39ebfcfcd8b8bfd0290d7623c4b8b5f90952c2d84776f8ca4/nvidia_cuda_nvrtc-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7b05ecda494c6dabc44231a608b060a71008a730d9dfda932cc508e6d29159e0", size = 49260054, upload-time = "2026-05-26T16:37:51.177Z" }, + { url = "https://files.pythonhosted.org/packages/a1/42/edce72f2c5a0f587168109c867f25f4a9a6cd7289ecf0d68ed2b1070f273/nvidia_cuda_nvrtc-13.3.33-py3-none-win_amd64.whl", hash = "sha256:7d2af818851c0c224d5f92221e9226e51ee23c236df4b51f9194563979c888be", size = 45319163, upload-time = "2026-05-26T17:02:49.217Z" }, ] [[package]] @@ -747,14 +752,15 @@ wheels = [ [[package]] name = "nvidia-cudnn-cu13" -version = "9.20.0.48" +version = "9.24.0.43" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nvidia-cublas" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, - { url = "https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304", size = 366173588, upload-time = "2026-03-09T19:29:34.474Z" }, + { url = "https://files.pythonhosted.org/packages/ca/30/7c257e3d5cb4fecb147b93895c66e29c93f8e76d74b45bb418ff0587c4ec/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:a6812a554a1ff0413e9c52b84c26c050380649ab9615f9c16bded368ce9f421f", size = 650976863, upload-time = "2026-07-02T16:23:39.248Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ba/791cffd048fe5b044e620df55267e3e95c0e6e07d50b41e377c03dfc910f/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:71f181cd810e90f9b6023b01186fe82d13d65f0ec098581ee201d39fad769e4b", size = 553099438, upload-time = "2026-07-02T16:27:42.58Z" }, + { url = "https://files.pythonhosted.org/packages/31/23/1dd3aa15cc4ab62c8fc88f8049ef137bc44c17892f5577bc80d994941f77/nvidia_cudnn_cu13-9.24.0.43-py3-none-win_amd64.whl", hash = "sha256:67a7273b5cf062f9446fd76cf464351a1c0f66501e6cd78f6675c0d604d8ac87", size = 412314771, upload-time = "2026-07-02T16:31:07.973Z" }, ] [[package]] @@ -1453,7 +1459,7 @@ dependencies = [ { name = "fsspec" }, { name = "jinja2" }, { name = "networkx" }, - { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu13" }, { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, @@ -1480,6 +1486,7 @@ dependencies = [ { name = "fsspec" }, { name = "jinja2" }, { name = "networkx" }, + { name = "nvidia-cudnn-cu13" }, { name = "setuptools" }, { name = "sympy" }, { name = "typing-extensions" }, From 04a90b8a5457287bedab2c21a340ccb9d2e87491 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 08:27:51 -0700 Subject: [PATCH 52/64] Fix upstream TE version issue on multi-GPU systems --- .../sana/tests/parity_check/README.md | 10 +++-- integrations/sana/tests/parity_check/bench.sh | 39 +++++++++++++++++-- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index e2ed877f3..e3452d6f6 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -221,6 +221,10 @@ bash bench.sh FP8 and FP4 are passed through the precision flags supported by each measured side. For upstream this means streaming only; for FlashDreams this also includes bidirectional diagnostic rows. Those precision runs sync the isolated venv with -the `quant` extra before launch. If Transformer Engine, hardware, or upstream -code rejects a precision, earlier precision outputs remain in place and the -failing command is recorded in the corresponding `command.txt`. +the `quant` extra before launch. `bench.sh` compiles Transformer Engine for the +current CUDA-visible PyTorch device; set `CUDA_VISIBLE_DEVICES` before the run +on mixed-GPU hosts. If the selected CUDA architecture changes, the script +rebuilds Transformer Engine once and records the architecture in the isolated +venv. If Transformer Engine, hardware, or upstream code rejects a precision, +earlier precision outputs remain in place and the failing command is recorded in +the corresponding `command.txt`. diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index 76b59157b..95d6c2269 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -335,20 +335,48 @@ UPSTREAM_COMMIT="$(git rev-parse HEAD)" FLASHDREAMS_COMMIT="$(git -C "${REPO_ROOT}" rev-parse HEAD 2>/dev/null || printf "unknown")" UV_SYNC_ARGS=(uv sync) +UV_SYNC_ENV=() +NVTE_ARCH_MARKER="${SCRIPT_DIR}/.venv/.flashdreams_nvte_cuda_archs" +NVTE_MARK_ARCH=0 if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; then UV_SYNC_ARGS+=(--extra quant) export NVTE_FRAMEWORK="${NVTE_FRAMEWORK:-pytorch}" export NVTE_WITH_NCCL_EP="${NVTE_WITH_NCCL_EP:-0}" + if [[ ! -x "${SCRIPT_DIR}/.venv/bin/python" ]] || ! ( cd "${SCRIPT_DIR}" && .venv/bin/python -c "import torch" >/dev/null 2>&1 ); then + echo "[setup] ensuring base Python deps before TransformerEngine arch detection" + ( cd "${SCRIPT_DIR}" && uv sync ) + fi if [[ -z "${NVTE_CUDA_ARCHS:-}" ]]; then NVTE_COMPUTE_CAP="$( - nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null \ - | head -n 1 \ - | tr -cd '0-9' + ( cd "${SCRIPT_DIR}" && .venv/bin/python - <<'PY' ) +import torch + +if torch.cuda.is_available(): + major, minor = torch.cuda.get_device_capability(torch.cuda.current_device()) + print(f"{major}{minor}") +PY )" if [[ -n "${NVTE_COMPUTE_CAP}" ]]; then export NVTE_CUDA_ARCHS="${NVTE_COMPUTE_CAP}" fi fi + if [[ -n "${NVTE_CUDA_ARCHS:-}" ]]; then + NVTE_MARK_ARCH=1 + NVTE_INSTALLED=0 + if ( cd "${SCRIPT_DIR}" && .venv/bin/python - <<'PY' ) +import importlib.util + +raise SystemExit(0 if importlib.util.find_spec("transformer_engine") else 1) +PY + then + NVTE_INSTALLED=1 + fi + if [[ "${NVTE_INSTALLED}" != "1" ]] || [[ ! -f "${NVTE_ARCH_MARKER}" ]] || [[ "$(cat "${NVTE_ARCH_MARKER}")" != "${NVTE_CUDA_ARCHS}" ]]; then + echo "[setup] refreshing TransformerEngine build for CUDA archs ${NVTE_CUDA_ARCHS}" + UV_SYNC_ARGS+=(--reinstall-package transformer-engine) + UV_SYNC_ENV=(env UV_NO_CACHE=1) + fi + fi echo "[setup] TransformerEngine build env: NVTE_FRAMEWORK=${NVTE_FRAMEWORK} NVTE_CUDA_ARCHS=${NVTE_CUDA_ARCHS:-default} NVTE_WITH_NCCL_EP=${NVTE_WITH_NCCL_EP}" echo "[setup] seeding TransformerEngine build tools in isolated venv" ( cd "${SCRIPT_DIR}" && \ @@ -356,7 +384,10 @@ if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; th uv pip install --python .venv/bin/python setuptools wheel pybind11 ) fi echo "[setup] ensuring Python deps via ${UV_SYNC_ARGS[*]} (isolated venv)" -( cd "${SCRIPT_DIR}" && "${UV_SYNC_ARGS[@]}" ) +( cd "${SCRIPT_DIR}" && "${UV_SYNC_ENV[@]}" "${UV_SYNC_ARGS[@]}" ) +if [[ "${NVTE_MARK_ARCH}" == "1" ]]; then + printf "%s\n" "${NVTE_CUDA_ARCHS}" > "${NVTE_ARCH_MARKER}" +fi UPSTREAM_PYTHONPATH="${SCRIPT_DIR}/compat:${SANA_REPO}" if [[ -n "${PYTHONPATH:-}" ]]; then UPSTREAM_PYTHONPATH="${UPSTREAM_PYTHONPATH}:${PYTHONPATH}" From 0bfa88c6b822e336276fec7c9df2fa8de8097996 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 08:54:13 -0700 Subject: [PATCH 53/64] Fix datacenter TE building issues --- .../sana/tests/parity_check/README.md | 9 ++-- integrations/sana/tests/parity_check/bench.sh | 41 ++++++++++++++++++- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index e3452d6f6..6c9ddb1f7 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -225,6 +225,9 @@ the `quant` extra before launch. `bench.sh` compiles Transformer Engine for the current CUDA-visible PyTorch device; set `CUDA_VISIBLE_DEVICES` before the run on mixed-GPU hosts. If the selected CUDA architecture changes, the script rebuilds Transformer Engine once and records the architecture in the isolated -venv. If Transformer Engine, hardware, or upstream code rejects a precision, -earlier precision outputs remain in place and the failing command is recorded in -the corresponding `command.txt`. +venv. When the installed CUDA compiler reports an architecture-specific target +for that device, the script uses that target because Transformer Engine's +low-precision kernels may use architecture-specific CUDA instructions. If +Transformer Engine, hardware, or upstream code rejects a precision, earlier +precision outputs remain in place and the failing command is recorded in the +corresponding `command.txt`. diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index 95d6c2269..5322a47d0 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -349,11 +349,50 @@ if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; th if [[ -z "${NVTE_CUDA_ARCHS:-}" ]]; then NVTE_COMPUTE_CAP="$( ( cd "${SCRIPT_DIR}" && .venv/bin/python - <<'PY' ) +import os +import shutil +import subprocess + import torch + +def _find_nvcc(): + candidates = [] + for env_name in ("CUDAToolkit_ROOT", "CUDA_HOME", "CUDA_PATH"): + cuda_root = os.environ.get(env_name) + if cuda_root: + candidates.append(os.path.join(cuda_root, "bin", "nvcc")) + candidates.append(shutil.which("nvcc")) + candidates.append("/usr/local/cuda/bin/nvcc") + for candidate in candidates: + if candidate and os.path.exists(candidate): + return candidate + return None + + +def _nvcc_supports_arch(nvcc, arch): + try: + result = subprocess.run( + [nvcc, "--list-gpu-arch"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + except OSError: + return False + return result.returncode == 0 and f"compute_{arch}" in result.stdout.split() + + if torch.cuda.is_available(): major, minor = torch.cuda.get_device_capability(torch.cuda.current_device()) - print(f"{major}{minor}") + compute_cap = f"{major}{minor}" + nvcc = _find_nvcc() + arch_specific = f"{compute_cap}a" + if nvcc and _nvcc_supports_arch(nvcc, arch_specific): + print(arch_specific) + else: + print(compute_cap) PY )" if [[ -n "${NVTE_COMPUTE_CAP}" ]]; then From e12e9e1c71f3deb46afb9c5c2e4e1ebaccb13972 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 09:13:59 -0700 Subject: [PATCH 54/64] More TE fix --- integrations/sana/tests/parity_check/bench.sh | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index 5322a47d0..6b423ef5e 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -352,6 +352,7 @@ if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; th import os import shutil import subprocess +import tempfile import torch @@ -370,18 +371,29 @@ def _find_nvcc(): return None -def _nvcc_supports_arch(nvcc, arch): +def _nvcc_accepts_arch(nvcc, arch): try: - result = subprocess.run( - [nvcc, "--list-gpu-arch"], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - ) + with tempfile.TemporaryDirectory() as tmpdir: + src = os.path.join(tmpdir, "arch_probe.cu") + obj = os.path.join(tmpdir, "arch_probe.o") + with open(src, "w", encoding="utf-8") as handle: + handle.write('extern "C" __global__ void flashdreams_arch_probe() {}\\n') + result = subprocess.run( + [ + nvcc, + "-c", + src, + "-o", + obj, + f"--generate-code=arch=compute_{arch},code=sm_{arch}", + ], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) except OSError: return False - return result.returncode == 0 and f"compute_{arch}" in result.stdout.split() + return result.returncode == 0 if torch.cuda.is_available(): @@ -389,7 +401,7 @@ if torch.cuda.is_available(): compute_cap = f"{major}{minor}" nvcc = _find_nvcc() arch_specific = f"{compute_cap}a" - if nvcc and _nvcc_supports_arch(nvcc, arch_specific): + if nvcc and _nvcc_accepts_arch(nvcc, arch_specific): print(arch_specific) else: print(compute_cap) From 64a6e481e9906ebe0ca1eae66bf962cae37033d4 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 09:34:17 -0700 Subject: [PATCH 55/64] TE fixes --- .../sana/tests/parity_check/README.md | 24 ++++---- integrations/sana/tests/parity_check/bench.sh | 55 ++++++++++++------- .../sana/tests/parity_check/pyproject.toml | 4 ++ integrations/sana/tests/parity_check/uv.lock | 41 ++++++++++++++ 4 files changed, 93 insertions(+), 31 deletions(-) diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index 6c9ddb1f7..72533e047 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -26,11 +26,12 @@ The BF16 path uses the base isolated environment. FP8 and FP4 streaming benchmarks opt into the `quant` extra, which builds TransformerEngine from the pinned git source against the local CUDA/PyTorch stack and explicitly installs the ONNX runtime packages TransformerEngine imports. `bench.sh` also seeds -`setuptools`, `wheel`, and `pybind11` into `./.venv` before that build because -upstream TransformerEngine does not declare all metadata/build requirements. A -new checkout therefore needs network access for `uv sync`, GitHub access for -the pinned upstream checkout and TransformerEngine source, and a CUDA build -toolchain for low-precision runs. +`setuptools`, `wheel`, `pybind11`, and the pinned NVIDIA CUDA compiler wheels +into `./.venv` before that build because upstream TransformerEngine does not +declare all metadata/build requirements. A new checkout therefore needs network +access for `uv sync`, GitHub access for the pinned upstream checkout and +TransformerEngine source, and a host compiler/CMake toolchain for +low-precision runs. ## Run parity @@ -225,9 +226,10 @@ the `quant` extra before launch. `bench.sh` compiles Transformer Engine for the current CUDA-visible PyTorch device; set `CUDA_VISIBLE_DEVICES` before the run on mixed-GPU hosts. If the selected CUDA architecture changes, the script rebuilds Transformer Engine once and records the architecture in the isolated -venv. When the installed CUDA compiler reports an architecture-specific target -for that device, the script uses that target because Transformer Engine's -low-precision kernels may use architecture-specific CUDA instructions. If -Transformer Engine, hardware, or upstream code rejects a precision, earlier -precision outputs remain in place and the failing command is recorded in the -corresponding `command.txt`. +venv. Quant builds prefer the pinned `nvidia-cuda-nvcc` compiler from that venv +via `cuda.pathfinder`, then fall back to the system CUDA compiler. When the +selected compiler accepts an architecture-specific target for the CUDA-visible +device, the script uses that target because Transformer Engine's low-precision +kernels may use architecture-specific CUDA instructions. If Transformer Engine, +hardware, or upstream code rejects a precision, earlier precision outputs remain +in place and the failing command is recorded in the corresponding `command.txt`. diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index 6b423ef5e..cfcd741d5 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -346,31 +346,50 @@ if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; th echo "[setup] ensuring base Python deps before TransformerEngine arch detection" ( cd "${SCRIPT_DIR}" && uv sync ) fi + echo "[setup] seeding TransformerEngine build tools in isolated venv" + ( cd "${SCRIPT_DIR}" && \ + uv pip install --python .venv/bin/python \ + setuptools wheel pybind11 \ + "nvidia-cuda-crt==13.3.33" \ + "nvidia-cuda-nvcc==13.3.33" \ + "nvidia-nvvm==13.3.33" ) + if [[ -z "${CUDACXX:-}" ]]; then + NVTE_NVCC="$( + ( cd "${SCRIPT_DIR}" && .venv/bin/python - <<'PY' ) +from cuda.pathfinder import find_nvidia_binary_utility + +nvcc = find_nvidia_binary_utility("nvcc") +if nvcc: + print(nvcc) +PY + )" + if [[ -n "${NVTE_NVCC}" ]]; then + export CUDACXX="${NVTE_NVCC}" + fi + fi + if [[ -z "${CUDACXX:-}" ]]; then + NVTE_NVCC="$(find "${SCRIPT_DIR}/.venv" -type f -name nvcc -print -quit 2>/dev/null || true)" + if [[ -n "${NVTE_NVCC}" ]]; then + export CUDACXX="${NVTE_NVCC}" + fi + fi + if [[ -n "${CUDACXX:-}" ]]; then + NVTE_CUDA_ROOT="$(cd "$(dirname "${CUDACXX}")/.." && pwd)" + export PATH="$(dirname "${CUDACXX}"):${PATH}" + export CUDA_HOME="${CUDA_HOME:-${NVTE_CUDA_ROOT}}" + export CUDA_PATH="${CUDA_PATH:-${CUDA_HOME}}" + echo "[setup] TransformerEngine nvcc: ${CUDACXX}" + fi if [[ -z "${NVTE_CUDA_ARCHS:-}" ]]; then NVTE_COMPUTE_CAP="$( ( cd "${SCRIPT_DIR}" && .venv/bin/python - <<'PY' ) import os -import shutil import subprocess import tempfile import torch -def _find_nvcc(): - candidates = [] - for env_name in ("CUDAToolkit_ROOT", "CUDA_HOME", "CUDA_PATH"): - cuda_root = os.environ.get(env_name) - if cuda_root: - candidates.append(os.path.join(cuda_root, "bin", "nvcc")) - candidates.append(shutil.which("nvcc")) - candidates.append("/usr/local/cuda/bin/nvcc") - for candidate in candidates: - if candidate and os.path.exists(candidate): - return candidate - return None - - def _nvcc_accepts_arch(nvcc, arch): try: with tempfile.TemporaryDirectory() as tmpdir: @@ -399,7 +418,7 @@ def _nvcc_accepts_arch(nvcc, arch): if torch.cuda.is_available(): major, minor = torch.cuda.get_device_capability(torch.cuda.current_device()) compute_cap = f"{major}{minor}" - nvcc = _find_nvcc() + nvcc = os.environ.get("CUDACXX") arch_specific = f"{compute_cap}a" if nvcc and _nvcc_accepts_arch(nvcc, arch_specific): print(arch_specific) @@ -429,10 +448,6 @@ PY fi fi echo "[setup] TransformerEngine build env: NVTE_FRAMEWORK=${NVTE_FRAMEWORK} NVTE_CUDA_ARCHS=${NVTE_CUDA_ARCHS:-default} NVTE_WITH_NCCL_EP=${NVTE_WITH_NCCL_EP}" - echo "[setup] seeding TransformerEngine build tools in isolated venv" - ( cd "${SCRIPT_DIR}" && \ - uv venv --allow-existing --python "3.12" .venv >/dev/null && \ - uv pip install --python .venv/bin/python setuptools wheel pybind11 ) fi echo "[setup] ensuring Python deps via ${UV_SYNC_ARGS[*]} (isolated venv)" ( cd "${SCRIPT_DIR}" && "${UV_SYNC_ENV[@]}" "${UV_SYNC_ARGS[@]}" ) diff --git a/integrations/sana/tests/parity_check/pyproject.toml b/integrations/sana/tests/parity_check/pyproject.toml index 6479ab92f..9b04b4533 100644 --- a/integrations/sana/tests/parity_check/pyproject.toml +++ b/integrations/sana/tests/parity_check/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ quant = [ "ml-dtypes>=0.5", "nvdlfw-inspect>=0.2", + "nvidia-cuda-nvcc==13.3.33; sys_platform == 'linux'", "onnx>=1.22", "onnxscript>=0.7", "pybind11>=3.0", @@ -62,7 +63,10 @@ override-dependencies = [ # GB300 with CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH. Pin the CUDA runtime # packages that keep the default cuDNN backends working on GB202 and GB300. "nvidia-cublas==13.6.1.10", + "nvidia-cuda-crt==13.3.33", + "nvidia-cuda-nvcc==13.3.33", "nvidia-cuda-nvrtc==13.3.33", "nvidia-cudnn-cu13==9.24.0.43", + "nvidia-nvvm==13.3.33", ] no-build-isolation-package = ["transformer-engine"] diff --git a/integrations/sana/tests/parity_check/uv.lock b/integrations/sana/tests/parity_check/uv.lock index 4fc781db9..85a6297e0 100644 --- a/integrations/sana/tests/parity_check/uv.lock +++ b/integrations/sana/tests/parity_check/uv.lock @@ -9,8 +9,11 @@ resolution-markers = [ [manifest] overrides = [ { name = "nvidia-cublas", specifier = "==13.6.1.10" }, + { name = "nvidia-cuda-crt", specifier = "==13.3.33" }, + { name = "nvidia-cuda-nvcc", specifier = "==13.3.33" }, { name = "nvidia-cuda-nvrtc", specifier = "==13.3.33" }, { name = "nvidia-cudnn-cu13", specifier = "==9.24.0.43" }, + { name = "nvidia-nvvm", specifier = "==13.3.33" }, ] [[package]] @@ -722,6 +725,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/e9/288a93d8234b8f8ea0ccac7b8cc5d7aa4c663d9676c64f4a2eb3a1f9ffc4/nvidia_cublas-13.6.1.10-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:feb2ed8a1e211bc5774413efc0f1a08c4d5269b56f68b4ac6fe5408e57f7dc1c", size = 410475904, upload-time = "2026-07-30T18:19:26.388Z" }, ] +[[package]] +name = "nvidia-cuda-crt" +version = "13.3.33" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/32/5ea57f8cd6ad5df2173d175ac5db4e06edde40028b1b1f6c539ea4c10290/nvidia_cuda_crt-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c8c257393f9c9146a85d3644f352be8154843d760031f756e673222c768a4930", size = 157348, upload-time = "2026-05-26T16:28:40.446Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a7/998af901511d5efdc6e42fc597d32a69f34eecf86f1591a9d230ab3ab951/nvidia_cuda_crt-13.3.33-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:01ff37600c7b880a14cab4ade763b4c10c0ff92f25cc9dca30f0881ce52693c4", size = 157350, upload-time = "2026-05-26T16:29:22.315Z" }, + { url = "https://files.pythonhosted.org/packages/a4/5f/fc8ce6b7719c825e0e519d2922e3b7630238e860222ad3f972dd9b8b7fa9/nvidia_cuda_crt-13.3.33-py3-none-win_amd64.whl", hash = "sha256:7e89c6dbb807a47ee0628907488b158e57c36fa31af3756a8f826a9ec482715f", size = 158284, upload-time = "2026-05-26T16:59:37.309Z" }, +] + [[package]] name = "nvidia-cuda-cupti" version = "13.0.85" @@ -731,6 +744,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, ] +[[package]] +name = "nvidia-cuda-nvcc" +version = "13.3.33" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cuda-crt" }, + { name = "nvidia-cuda-runtime" }, + { name = "nvidia-nvvm" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/b6/bb07a3a63b5b7b55516366747892abbf3ee62d616684c40bb51e6cbfe956/nvidia_cuda_nvcc-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c348623b1434aebd234da9ec1f81022587ae4995d65c3dc8a7743245cc441f7", size = 39515074, upload-time = "2026-05-26T16:34:28.489Z" }, + { url = "https://files.pythonhosted.org/packages/3f/af/e1b107f034f7c133255c162b922bbad3da5be20ebf76df17662ae4bd31f6/nvidia_cuda_nvcc-13.3.33-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:53b5f1be1731574368b8be931b77b6313492266c464aef3dd3f431569ce90deb", size = 44943276, upload-time = "2026-05-26T16:35:05.912Z" }, + { url = "https://files.pythonhosted.org/packages/47/c2/831fa54020621a64d44cff47f1ed5eb0611794495fce01c857f6999d76b1/nvidia_cuda_nvcc-13.3.33-py3-none-win_amd64.whl", hash = "sha256:21c93aeef695a81b688137119f9120fe08a67292bf0ad730d94dc2b18bec23f0", size = 32723421, upload-time = "2026-05-26T17:01:47.511Z" }, +] + [[package]] name = "nvidia-cuda-nvrtc" version = "13.3.33" @@ -748,6 +776,7 @@ source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, + { url = "https://files.pythonhosted.org/packages/b7/94/6b867483bec07da24ffa32736c79fabb94ef3a7af4d787a9d4a974868576/nvidia_cuda_runtime-13.0.96-py3-none-win_amd64.whl", hash = "sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492", size = 2927037, upload-time = "2025-10-09T09:04:23.782Z" }, ] [[package]] @@ -873,6 +902,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, ] +[[package]] +name = "nvidia-nvvm" +version = "13.3.33" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/8a/f767031dcd0d24c2bbab4b696dbcf004da4f3284e5e4649fc47bc0e2bb78/nvidia_nvvm-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:aafaf73246b6126bc88f521e5dab1d196395ee87739d9f5b7c39c9fee0ead9c7", size = 69250604, upload-time = "2026-05-26T16:57:56.875Z" }, + { url = "https://files.pythonhosted.org/packages/83/36/ce0d42d3a4465c858c379932f0080d29d22f04383ab79119c7c4f4cdd5ef/nvidia_nvvm-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd74a1c5ef284ba04c1ba75f886404dff953c54731a3a9c7b45e9aedaf1a226b", size = 66984524, upload-time = "2026-05-26T16:57:30.778Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/4de7a37803d168337ab36f81ecbc496c7c21c9b06ec68ce0ecc381af88d4/nvidia_nvvm-13.3.33-py3-none-win_amd64.whl", hash = "sha256:b1c63cf8972d8a1ff153c5ac4cc7038fe6ef705aa38415f12007b0e5e4c31b79", size = 60175824, upload-time = "2026-05-26T17:13:27.588Z" }, +] + [[package]] name = "omegaconf" version = "2.3.1" @@ -1271,6 +1310,7 @@ dependencies = [ quant = [ { name = "ml-dtypes" }, { name = "nvdlfw-inspect" }, + { name = "nvidia-cuda-nvcc" }, { name = "onnx" }, { name = "onnxscript" }, { name = "pybind11" }, @@ -1292,6 +1332,7 @@ requires-dist = [ { name = "ml-dtypes", marker = "extra == 'quant'", specifier = ">=0.5" }, { name = "numpy", specifier = ">=1.24,<2.5" }, { name = "nvdlfw-inspect", marker = "extra == 'quant'", specifier = ">=0.2" }, + { name = "nvidia-cuda-nvcc", marker = "sys_platform == 'linux' and extra == 'quant'", specifier = "==13.3.33" }, { name = "omegaconf", specifier = ">=2.3" }, { name = "onnx", marker = "extra == 'quant'", specifier = ">=1.22" }, { name = "onnxscript", marker = "extra == 'quant'", specifier = ">=0.7" }, From 60911d51c30e1fa2a8295278d04362215a1968d3 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 09:45:52 -0700 Subject: [PATCH 56/64] TE Fixes --- .../sana/tests/parity_check/README.md | 21 ++---- integrations/sana/tests/parity_check/bench.sh | 74 +------------------ .../sana/tests/parity_check/pyproject.toml | 10 +-- integrations/sana/tests/parity_check/uv.lock | 41 ---------- 4 files changed, 15 insertions(+), 131 deletions(-) diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index 72533e047..e3452d6f6 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -26,12 +26,11 @@ The BF16 path uses the base isolated environment. FP8 and FP4 streaming benchmarks opt into the `quant` extra, which builds TransformerEngine from the pinned git source against the local CUDA/PyTorch stack and explicitly installs the ONNX runtime packages TransformerEngine imports. `bench.sh` also seeds -`setuptools`, `wheel`, `pybind11`, and the pinned NVIDIA CUDA compiler wheels -into `./.venv` before that build because upstream TransformerEngine does not -declare all metadata/build requirements. A new checkout therefore needs network -access for `uv sync`, GitHub access for the pinned upstream checkout and -TransformerEngine source, and a host compiler/CMake toolchain for -low-precision runs. +`setuptools`, `wheel`, and `pybind11` into `./.venv` before that build because +upstream TransformerEngine does not declare all metadata/build requirements. A +new checkout therefore needs network access for `uv sync`, GitHub access for +the pinned upstream checkout and TransformerEngine source, and a CUDA build +toolchain for low-precision runs. ## Run parity @@ -226,10 +225,6 @@ the `quant` extra before launch. `bench.sh` compiles Transformer Engine for the current CUDA-visible PyTorch device; set `CUDA_VISIBLE_DEVICES` before the run on mixed-GPU hosts. If the selected CUDA architecture changes, the script rebuilds Transformer Engine once and records the architecture in the isolated -venv. Quant builds prefer the pinned `nvidia-cuda-nvcc` compiler from that venv -via `cuda.pathfinder`, then fall back to the system CUDA compiler. When the -selected compiler accepts an architecture-specific target for the CUDA-visible -device, the script uses that target because Transformer Engine's low-precision -kernels may use architecture-specific CUDA instructions. If Transformer Engine, -hardware, or upstream code rejects a precision, earlier precision outputs remain -in place and the failing command is recorded in the corresponding `command.txt`. +venv. If Transformer Engine, hardware, or upstream code rejects a precision, +earlier precision outputs remain in place and the failing command is recorded in +the corresponding `command.txt`. diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index cfcd741d5..bc30462d4 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -346,84 +346,15 @@ if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; th echo "[setup] ensuring base Python deps before TransformerEngine arch detection" ( cd "${SCRIPT_DIR}" && uv sync ) fi - echo "[setup] seeding TransformerEngine build tools in isolated venv" - ( cd "${SCRIPT_DIR}" && \ - uv pip install --python .venv/bin/python \ - setuptools wheel pybind11 \ - "nvidia-cuda-crt==13.3.33" \ - "nvidia-cuda-nvcc==13.3.33" \ - "nvidia-nvvm==13.3.33" ) - if [[ -z "${CUDACXX:-}" ]]; then - NVTE_NVCC="$( - ( cd "${SCRIPT_DIR}" && .venv/bin/python - <<'PY' ) -from cuda.pathfinder import find_nvidia_binary_utility - -nvcc = find_nvidia_binary_utility("nvcc") -if nvcc: - print(nvcc) -PY - )" - if [[ -n "${NVTE_NVCC}" ]]; then - export CUDACXX="${NVTE_NVCC}" - fi - fi - if [[ -z "${CUDACXX:-}" ]]; then - NVTE_NVCC="$(find "${SCRIPT_DIR}/.venv" -type f -name nvcc -print -quit 2>/dev/null || true)" - if [[ -n "${NVTE_NVCC}" ]]; then - export CUDACXX="${NVTE_NVCC}" - fi - fi - if [[ -n "${CUDACXX:-}" ]]; then - NVTE_CUDA_ROOT="$(cd "$(dirname "${CUDACXX}")/.." && pwd)" - export PATH="$(dirname "${CUDACXX}"):${PATH}" - export CUDA_HOME="${CUDA_HOME:-${NVTE_CUDA_ROOT}}" - export CUDA_PATH="${CUDA_PATH:-${CUDA_HOME}}" - echo "[setup] TransformerEngine nvcc: ${CUDACXX}" - fi if [[ -z "${NVTE_CUDA_ARCHS:-}" ]]; then NVTE_COMPUTE_CAP="$( ( cd "${SCRIPT_DIR}" && .venv/bin/python - <<'PY' ) -import os -import subprocess -import tempfile - import torch -def _nvcc_accepts_arch(nvcc, arch): - try: - with tempfile.TemporaryDirectory() as tmpdir: - src = os.path.join(tmpdir, "arch_probe.cu") - obj = os.path.join(tmpdir, "arch_probe.o") - with open(src, "w", encoding="utf-8") as handle: - handle.write('extern "C" __global__ void flashdreams_arch_probe() {}\\n') - result = subprocess.run( - [ - nvcc, - "-c", - src, - "-o", - obj, - f"--generate-code=arch=compute_{arch},code=sm_{arch}", - ], - check=False, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - except OSError: - return False - return result.returncode == 0 - - if torch.cuda.is_available(): major, minor = torch.cuda.get_device_capability(torch.cuda.current_device()) - compute_cap = f"{major}{minor}" - nvcc = os.environ.get("CUDACXX") - arch_specific = f"{compute_cap}a" - if nvcc and _nvcc_accepts_arch(nvcc, arch_specific): - print(arch_specific) - else: - print(compute_cap) + print(f"{major}{minor}") PY )" if [[ -n "${NVTE_COMPUTE_CAP}" ]]; then @@ -448,6 +379,9 @@ PY fi fi echo "[setup] TransformerEngine build env: NVTE_FRAMEWORK=${NVTE_FRAMEWORK} NVTE_CUDA_ARCHS=${NVTE_CUDA_ARCHS:-default} NVTE_WITH_NCCL_EP=${NVTE_WITH_NCCL_EP}" + echo "[setup] seeding TransformerEngine build tools in isolated venv" + ( cd "${SCRIPT_DIR}" && \ + uv pip install --python .venv/bin/python setuptools wheel pybind11 ) fi echo "[setup] ensuring Python deps via ${UV_SYNC_ARGS[*]} (isolated venv)" ( cd "${SCRIPT_DIR}" && "${UV_SYNC_ENV[@]}" "${UV_SYNC_ARGS[@]}" ) diff --git a/integrations/sana/tests/parity_check/pyproject.toml b/integrations/sana/tests/parity_check/pyproject.toml index 9b04b4533..b853466a5 100644 --- a/integrations/sana/tests/parity_check/pyproject.toml +++ b/integrations/sana/tests/parity_check/pyproject.toml @@ -42,7 +42,6 @@ dependencies = [ quant = [ "ml-dtypes>=0.5", "nvdlfw-inspect>=0.2", - "nvidia-cuda-nvcc==13.3.33; sys_platform == 'linux'", "onnx>=1.22", "onnxscript>=0.7", "pybind11>=3.0", @@ -59,14 +58,11 @@ transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", managed = true override-dependencies = [ # PyTorch 2.13.0's default CUDA 13 dependency set can resolve cuDNN - # 9.20.0.48 on Linux. That stack fails standalone BF16 Conv3d and SDPA on - # GB300 with CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH. Pin the CUDA runtime - # packages that keep the default cuDNN backends working on GB202 and GB300. + # 9.20.0.48 on Linux. That stack can fail standalone BF16 Conv3d and SDPA + # with CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH. Pin the CUDA runtime + # packages that keep the default cuDNN backends working. "nvidia-cublas==13.6.1.10", - "nvidia-cuda-crt==13.3.33", - "nvidia-cuda-nvcc==13.3.33", "nvidia-cuda-nvrtc==13.3.33", "nvidia-cudnn-cu13==9.24.0.43", - "nvidia-nvvm==13.3.33", ] no-build-isolation-package = ["transformer-engine"] diff --git a/integrations/sana/tests/parity_check/uv.lock b/integrations/sana/tests/parity_check/uv.lock index 85a6297e0..4fc781db9 100644 --- a/integrations/sana/tests/parity_check/uv.lock +++ b/integrations/sana/tests/parity_check/uv.lock @@ -9,11 +9,8 @@ resolution-markers = [ [manifest] overrides = [ { name = "nvidia-cublas", specifier = "==13.6.1.10" }, - { name = "nvidia-cuda-crt", specifier = "==13.3.33" }, - { name = "nvidia-cuda-nvcc", specifier = "==13.3.33" }, { name = "nvidia-cuda-nvrtc", specifier = "==13.3.33" }, { name = "nvidia-cudnn-cu13", specifier = "==9.24.0.43" }, - { name = "nvidia-nvvm", specifier = "==13.3.33" }, ] [[package]] @@ -725,16 +722,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/e9/288a93d8234b8f8ea0ccac7b8cc5d7aa4c663d9676c64f4a2eb3a1f9ffc4/nvidia_cublas-13.6.1.10-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:feb2ed8a1e211bc5774413efc0f1a08c4d5269b56f68b4ac6fe5408e57f7dc1c", size = 410475904, upload-time = "2026-07-30T18:19:26.388Z" }, ] -[[package]] -name = "nvidia-cuda-crt" -version = "13.3.33" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/32/5ea57f8cd6ad5df2173d175ac5db4e06edde40028b1b1f6c539ea4c10290/nvidia_cuda_crt-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c8c257393f9c9146a85d3644f352be8154843d760031f756e673222c768a4930", size = 157348, upload-time = "2026-05-26T16:28:40.446Z" }, - { url = "https://files.pythonhosted.org/packages/8d/a7/998af901511d5efdc6e42fc597d32a69f34eecf86f1591a9d230ab3ab951/nvidia_cuda_crt-13.3.33-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:01ff37600c7b880a14cab4ade763b4c10c0ff92f25cc9dca30f0881ce52693c4", size = 157350, upload-time = "2026-05-26T16:29:22.315Z" }, - { url = "https://files.pythonhosted.org/packages/a4/5f/fc8ce6b7719c825e0e519d2922e3b7630238e860222ad3f972dd9b8b7fa9/nvidia_cuda_crt-13.3.33-py3-none-win_amd64.whl", hash = "sha256:7e89c6dbb807a47ee0628907488b158e57c36fa31af3756a8f826a9ec482715f", size = 158284, upload-time = "2026-05-26T16:59:37.309Z" }, -] - [[package]] name = "nvidia-cuda-cupti" version = "13.0.85" @@ -744,21 +731,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, ] -[[package]] -name = "nvidia-cuda-nvcc" -version = "13.3.33" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cuda-crt" }, - { name = "nvidia-cuda-runtime" }, - { name = "nvidia-nvvm" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/b6/bb07a3a63b5b7b55516366747892abbf3ee62d616684c40bb51e6cbfe956/nvidia_cuda_nvcc-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c348623b1434aebd234da9ec1f81022587ae4995d65c3dc8a7743245cc441f7", size = 39515074, upload-time = "2026-05-26T16:34:28.489Z" }, - { url = "https://files.pythonhosted.org/packages/3f/af/e1b107f034f7c133255c162b922bbad3da5be20ebf76df17662ae4bd31f6/nvidia_cuda_nvcc-13.3.33-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:53b5f1be1731574368b8be931b77b6313492266c464aef3dd3f431569ce90deb", size = 44943276, upload-time = "2026-05-26T16:35:05.912Z" }, - { url = "https://files.pythonhosted.org/packages/47/c2/831fa54020621a64d44cff47f1ed5eb0611794495fce01c857f6999d76b1/nvidia_cuda_nvcc-13.3.33-py3-none-win_amd64.whl", hash = "sha256:21c93aeef695a81b688137119f9120fe08a67292bf0ad730d94dc2b18bec23f0", size = 32723421, upload-time = "2026-05-26T17:01:47.511Z" }, -] - [[package]] name = "nvidia-cuda-nvrtc" version = "13.3.33" @@ -776,7 +748,6 @@ source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, - { url = "https://files.pythonhosted.org/packages/b7/94/6b867483bec07da24ffa32736c79fabb94ef3a7af4d787a9d4a974868576/nvidia_cuda_runtime-13.0.96-py3-none-win_amd64.whl", hash = "sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492", size = 2927037, upload-time = "2025-10-09T09:04:23.782Z" }, ] [[package]] @@ -902,16 +873,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, ] -[[package]] -name = "nvidia-nvvm" -version = "13.3.33" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/01/8a/f767031dcd0d24c2bbab4b696dbcf004da4f3284e5e4649fc47bc0e2bb78/nvidia_nvvm-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:aafaf73246b6126bc88f521e5dab1d196395ee87739d9f5b7c39c9fee0ead9c7", size = 69250604, upload-time = "2026-05-26T16:57:56.875Z" }, - { url = "https://files.pythonhosted.org/packages/83/36/ce0d42d3a4465c858c379932f0080d29d22f04383ab79119c7c4f4cdd5ef/nvidia_nvvm-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd74a1c5ef284ba04c1ba75f886404dff953c54731a3a9c7b45e9aedaf1a226b", size = 66984524, upload-time = "2026-05-26T16:57:30.778Z" }, - { url = "https://files.pythonhosted.org/packages/8f/96/4de7a37803d168337ab36f81ecbc496c7c21c9b06ec68ce0ecc381af88d4/nvidia_nvvm-13.3.33-py3-none-win_amd64.whl", hash = "sha256:b1c63cf8972d8a1ff153c5ac4cc7038fe6ef705aa38415f12007b0e5e4c31b79", size = 60175824, upload-time = "2026-05-26T17:13:27.588Z" }, -] - [[package]] name = "omegaconf" version = "2.3.1" @@ -1310,7 +1271,6 @@ dependencies = [ quant = [ { name = "ml-dtypes" }, { name = "nvdlfw-inspect" }, - { name = "nvidia-cuda-nvcc" }, { name = "onnx" }, { name = "onnxscript" }, { name = "pybind11" }, @@ -1332,7 +1292,6 @@ requires-dist = [ { name = "ml-dtypes", marker = "extra == 'quant'", specifier = ">=0.5" }, { name = "numpy", specifier = ">=1.24,<2.5" }, { name = "nvdlfw-inspect", marker = "extra == 'quant'", specifier = ">=0.2" }, - { name = "nvidia-cuda-nvcc", marker = "sys_platform == 'linux' and extra == 'quant'", specifier = "==13.3.33" }, { name = "omegaconf", specifier = ">=2.3" }, { name = "onnx", marker = "extra == 'quant'", specifier = ">=1.22" }, { name = "onnxscript", marker = "extra == 'quant'", specifier = ">=0.7" }, From bd0816edff84a2b266fcd29bfed5741b783756ee Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 09:54:43 -0700 Subject: [PATCH 57/64] TE fixes --- .../sana/tests/parity_check/README.md | 14 +++---- integrations/sana/tests/parity_check/bench.sh | 39 ++++++------------- 2 files changed, 18 insertions(+), 35 deletions(-) diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index e3452d6f6..de14cd146 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -221,10 +221,10 @@ bash bench.sh FP8 and FP4 are passed through the precision flags supported by each measured side. For upstream this means streaming only; for FlashDreams this also includes bidirectional diagnostic rows. Those precision runs sync the isolated venv with -the `quant` extra before launch. `bench.sh` compiles Transformer Engine for the -current CUDA-visible PyTorch device; set `CUDA_VISIBLE_DEVICES` before the run -on mixed-GPU hosts. If the selected CUDA architecture changes, the script -rebuilds Transformer Engine once and records the architecture in the isolated -venv. If Transformer Engine, hardware, or upstream code rejects a precision, -earlier precision outputs remain in place and the failing command is recorded in -the corresponding `command.txt`. +the `quant` extra before launch. `bench.sh` lets Transformer Engine use its +upstream CUDA-version-aware default architecture set unless `NVTE_CUDA_ARCHS` is +set explicitly. The script records that setting in the isolated venv and +rebuilds Transformer Engine once when it changes, so stale extension wheels from +an earlier architecture setting are not reused. If Transformer Engine, hardware, +or upstream code rejects a precision, earlier precision outputs remain in place +and the failing command is recorded in the corresponding `command.txt`. diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/parity_check/bench.sh index bc30462d4..6274e829c 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/parity_check/bench.sh @@ -342,28 +342,10 @@ if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; th UV_SYNC_ARGS+=(--extra quant) export NVTE_FRAMEWORK="${NVTE_FRAMEWORK:-pytorch}" export NVTE_WITH_NCCL_EP="${NVTE_WITH_NCCL_EP:-0}" - if [[ ! -x "${SCRIPT_DIR}/.venv/bin/python" ]] || ! ( cd "${SCRIPT_DIR}" && .venv/bin/python -c "import torch" >/dev/null 2>&1 ); then - echo "[setup] ensuring base Python deps before TransformerEngine arch detection" - ( cd "${SCRIPT_DIR}" && uv sync ) - fi - if [[ -z "${NVTE_CUDA_ARCHS:-}" ]]; then - NVTE_COMPUTE_CAP="$( - ( cd "${SCRIPT_DIR}" && .venv/bin/python - <<'PY' ) -import torch - - -if torch.cuda.is_available(): - major, minor = torch.cuda.get_device_capability(torch.cuda.current_device()) - print(f"{major}{minor}") -PY - )" - if [[ -n "${NVTE_COMPUTE_CAP}" ]]; then - export NVTE_CUDA_ARCHS="${NVTE_COMPUTE_CAP}" - fi - fi - if [[ -n "${NVTE_CUDA_ARCHS:-}" ]]; then - NVTE_MARK_ARCH=1 - NVTE_INSTALLED=0 + NVTE_MARK_ARCH=1 + NVTE_ARCH_SETTING="${NVTE_CUDA_ARCHS:-default}" + NVTE_INSTALLED=0 + if [[ -x "${SCRIPT_DIR}/.venv/bin/python" ]]; then if ( cd "${SCRIPT_DIR}" && .venv/bin/python - <<'PY' ) import importlib.util @@ -372,21 +354,22 @@ PY then NVTE_INSTALLED=1 fi - if [[ "${NVTE_INSTALLED}" != "1" ]] || [[ ! -f "${NVTE_ARCH_MARKER}" ]] || [[ "$(cat "${NVTE_ARCH_MARKER}")" != "${NVTE_CUDA_ARCHS}" ]]; then - echo "[setup] refreshing TransformerEngine build for CUDA archs ${NVTE_CUDA_ARCHS}" - UV_SYNC_ARGS+=(--reinstall-package transformer-engine) - UV_SYNC_ENV=(env UV_NO_CACHE=1) - fi + fi + if [[ "${NVTE_INSTALLED}" != "1" ]] || [[ ! -f "${NVTE_ARCH_MARKER}" ]] || [[ "$(cat "${NVTE_ARCH_MARKER}")" != "${NVTE_ARCH_SETTING}" ]]; then + echo "[setup] refreshing TransformerEngine build for CUDA archs ${NVTE_ARCH_SETTING}" + UV_SYNC_ARGS+=(--reinstall-package transformer-engine) + UV_SYNC_ENV=(env UV_NO_CACHE=1) fi echo "[setup] TransformerEngine build env: NVTE_FRAMEWORK=${NVTE_FRAMEWORK} NVTE_CUDA_ARCHS=${NVTE_CUDA_ARCHS:-default} NVTE_WITH_NCCL_EP=${NVTE_WITH_NCCL_EP}" echo "[setup] seeding TransformerEngine build tools in isolated venv" ( cd "${SCRIPT_DIR}" && \ + uv venv --allow-existing --python "3.12" .venv >/dev/null && \ uv pip install --python .venv/bin/python setuptools wheel pybind11 ) fi echo "[setup] ensuring Python deps via ${UV_SYNC_ARGS[*]} (isolated venv)" ( cd "${SCRIPT_DIR}" && "${UV_SYNC_ENV[@]}" "${UV_SYNC_ARGS[@]}" ) if [[ "${NVTE_MARK_ARCH}" == "1" ]]; then - printf "%s\n" "${NVTE_CUDA_ARCHS}" > "${NVTE_ARCH_MARKER}" + printf "%s\n" "${NVTE_ARCH_SETTING}" > "${NVTE_ARCH_MARKER}" fi UPSTREAM_PYTHONPATH="${SCRIPT_DIR}/compat:${SANA_REPO}" if [[ -n "${PYTHONPATH:-}" ]]; then From 009ddae7e41e19b9d7ed29b5d56ae0c0de668e41 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 12:33:55 -0700 Subject: [PATCH 58/64] Finish model card data --- .../{perf-bf16-gb202.md => perf-0801-bf16.md} | 2 +- .../{perf-bf16-gb202.md => perf-0801-bf16.md} | 2 +- .../{perf-fp4-gb202.md => perf-0801-fp4.md} | 2 +- .../{perf-fp8-gb202.md => perf-0801-fp8.md} | 2 +- docs/source/models/sana_wm_bidirectional.rst | 26 +++---- docs/source/models/sana_wm_streaming.rst | 39 +++++----- integrations/sana/README.md | 76 ++++++------------- .../sana/tests/parity_check/README.md | 47 +++++++----- 8 files changed, 92 insertions(+), 104 deletions(-) rename docs/source/_static/performance/sana_wm_bidirectional/{perf-bf16-gb202.md => perf-0801-bf16.md} (76%) rename docs/source/_static/performance/sana_wm_streaming/{perf-bf16-gb202.md => perf-0801-bf16.md} (78%) rename docs/source/_static/performance/sana_wm_streaming/{perf-fp4-gb202.md => perf-0801-fp4.md} (78%) rename docs/source/_static/performance/sana_wm_streaming/{perf-fp8-gb202.md => perf-0801-fp8.md} (78%) diff --git a/docs/source/_static/performance/sana_wm_bidirectional/perf-bf16-gb202.md b/docs/source/_static/performance/sana_wm_bidirectional/perf-0801-bf16.md similarity index 76% rename from docs/source/_static/performance/sana_wm_bidirectional/perf-bf16-gb202.md rename to docs/source/_static/performance/sana_wm_bidirectional/perf-0801-bf16.md index 2eb8b9790..650203a2f 100644 --- a/docs/source/_static/performance/sana_wm_bidirectional/perf-bf16-gb202.md +++ b/docs/source/_static/performance/sana_wm_bidirectional/perf-0801-bf16.md @@ -2,4 +2,4 @@ | device | official | flashdreams | | --- | ---: | ---: | -| GB202 | 101874.59 | 96353.88 | +| GB300 | 56932.83 | 34182.39 | diff --git a/docs/source/_static/performance/sana_wm_streaming/perf-bf16-gb202.md b/docs/source/_static/performance/sana_wm_streaming/perf-0801-bf16.md similarity index 78% rename from docs/source/_static/performance/sana_wm_streaming/perf-bf16-gb202.md rename to docs/source/_static/performance/sana_wm_streaming/perf-0801-bf16.md index 26c33ec84..fbf72a305 100644 --- a/docs/source/_static/performance/sana_wm_streaming/perf-bf16-gb202.md +++ b/docs/source/_static/performance/sana_wm_streaming/perf-0801-bf16.md @@ -2,4 +2,4 @@ | device | official | flashdreams | | --- | ---: | ---: | -| GB202 | 1849.65 | 3660.15 | +| GB300 | 1170.29 | 1957.93 | diff --git a/docs/source/_static/performance/sana_wm_streaming/perf-fp4-gb202.md b/docs/source/_static/performance/sana_wm_streaming/perf-0801-fp4.md similarity index 78% rename from docs/source/_static/performance/sana_wm_streaming/perf-fp4-gb202.md rename to docs/source/_static/performance/sana_wm_streaming/perf-0801-fp4.md index 603e6580c..646e492d3 100644 --- a/docs/source/_static/performance/sana_wm_streaming/perf-fp4-gb202.md +++ b/docs/source/_static/performance/sana_wm_streaming/perf-0801-fp4.md @@ -2,4 +2,4 @@ | device | official | flashdreams | | --- | ---: | ---: | -| GB202 | 1308.22 | 6520.72 | +| GB300 | 1594.33 | 4118.36 | diff --git a/docs/source/_static/performance/sana_wm_streaming/perf-fp8-gb202.md b/docs/source/_static/performance/sana_wm_streaming/perf-0801-fp8.md similarity index 78% rename from docs/source/_static/performance/sana_wm_streaming/perf-fp8-gb202.md rename to docs/source/_static/performance/sana_wm_streaming/perf-0801-fp8.md index 46c703500..d54c152dd 100644 --- a/docs/source/_static/performance/sana_wm_streaming/perf-fp8-gb202.md +++ b/docs/source/_static/performance/sana_wm_streaming/perf-0801-fp8.md @@ -2,4 +2,4 @@ | device | official | flashdreams | | --- | ---: | ---: | -| GB202 | 1552.48 | 3993.07 | +| GB300 | 1482.92 | 2392.67 | diff --git a/docs/source/models/sana_wm_bidirectional.rst b/docs/source/models/sana_wm_bidirectional.rst index aec7c65ea..c4bcd033a 100644 --- a/docs/source/models/sana_wm_bidirectional.rst +++ b/docs/source/models/sana_wm_bidirectional.rst @@ -73,16 +73,15 @@ and a camera trajectory: .. code-block:: bash uv run flashdreams-run sana-wm-bidirectional \ - --image-path ../Sana/asset/sana_wm/demo_0.png \ - --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ - --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ - --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --image-path \ + --prompt-path \ + --camera-path \ + --intrinsics-path \ --num-frames 161 \ --output-dir outputs/sana_wm_bidirectional_bf16 -The demo image, prompt, camera, and intrinsics files ship with the -``SANA-WM_bidirectional`` release; equivalent inputs with matching shapes work -as well. +The first frame, prompt, camera, and intrinsics inputs must follow the same +shape conventions as the ``SANA-WM_bidirectional`` release examples. Optional inputs and knobs ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -123,9 +122,9 @@ Profiling benchmark The BF16 chart below compares steady-state in-process generation latency per generated clip for FlashDreams ``SANA-WM_bidirectional`` and the official -``SANA-WM_bidirectional`` implementation under matched settings. On GB202, -FlashDreams measured 96,353.88 ms per clip versus 101,874.59 ms for the -official implementation. +``SANA-WM_bidirectional`` implementation under matched settings on one NVIDIA +GB300 GPU. FlashDreams measured 34,182.39 ms per clip versus 56,932.83 ms for +the official implementation. In this chart, ``Official Impl`` means the pinned NVlabs/Sana upstream implementation measured by the FlashDreams parity harness under matched @@ -138,7 +137,7 @@ model authors.
@@ -146,9 +145,10 @@ model authors.

This chart shows steady-state in-process generation latency per generated clip in milliseconds for a 121-frame full-pipeline BF16 run (Stage-1 DiT + LTX-2 refiner + SANA VAE decode). - The measured row used one NVIDIA GB202 GPU, one live warmup generation, and three measured generations. + The measured row used one NVIDIA GB300 GPU, one live warmup generation, + and three measured generations. Model construction, checkpoint loading, video writing, and frame dumps are outside the timing boundary. - The upstream checkout was pinned to commit 6298508. + The benchmark runs recorded FlashDreams commit bd0816e and upstream commit 6298508.

diff --git a/docs/source/models/sana_wm_streaming.rst b/docs/source/models/sana_wm_streaming.rst index 3904a9493..a9fff764f 100644 --- a/docs/source/models/sana_wm_streaming.rst +++ b/docs/source/models/sana_wm_streaming.rst @@ -73,13 +73,16 @@ a camera trajectory: .. code-block:: bash uv run flashdreams-run sana-wm-streaming \ - --image-path ../Sana/asset/sana_wm/demo_0.png \ - --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ - --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ - --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --image-path \ + --prompt-path \ + --camera-path \ + --intrinsics-path \ --num-frames 241 \ --output-dir outputs/sana_wm_streaming_bf16 +The first frame, prompt, camera, and intrinsics inputs must follow the same +shape conventions as the ``SANA-WM_streaming`` release examples. + The runner defaults to 3 latent frames per block and the distilled schedule ``1000,960,889,727,0``. Requested frame counts are snapped to ``8 * --num-frame-per-block * k + 1`` before inference. @@ -124,9 +127,8 @@ Profiling benchmark The charts below compare steady-state generation latency per produced chunk for FlashDreams ``SANA-WM_streaming`` and the official ``SANA-WM_streaming`` implementation under matched settings. Warmup runs and the first decoded chunk -are excluded from the headline metric; full-clip wall time remains supporting -benchmark data. These GB202 latency runs show the official implementation -faster than FlashDreams for BF16, FP8, and FP4. +are excluded from the headline metric. These GB300 latency runs show the +official implementation faster than FlashDreams for BF16, FP8, and FP4. In these charts, ``Official Impl`` means the pinned NVlabs/Sana upstream implementation measured by the FlashDreams parity harness under matched @@ -139,14 +141,14 @@ model authors.

- BF16 steady-state milliseconds per produced chunk on one NVIDIA GB202 GPU: - official 1,849.65 ms, FlashDreams 3,660.15 ms. + BF16 steady-state milliseconds per produced chunk on one NVIDIA GB300 GPU: + official 1,170.29 ms, FlashDreams 1,957.93 ms.

@@ -155,14 +157,14 @@ model authors.

- FP8 steady-state milliseconds per produced chunk on one NVIDIA GB202 GPU: - official 1,552.48 ms, FlashDreams 3,993.07 ms. + FP8 steady-state milliseconds per produced chunk on one NVIDIA GB300 GPU: + official 1,482.92 ms, FlashDreams 2,392.67 ms.

@@ -171,22 +173,23 @@ model authors.

- FP4 steady-state milliseconds per produced chunk on one NVIDIA GB202 GPU: - official 1,308.22 ms, FlashDreams 6,520.72 ms. + FP4 steady-state milliseconds per produced chunk on one NVIDIA GB300 GPU: + official 1,594.33 ms, FlashDreams 4,118.36 ms.

-All three rows use the same demo image/prompt, ``w-80,dw-40,w-80,aw-40`` +All charts use the same demo image/prompt, ``w-80,dw-40,w-80,aw-40`` action path, 241 requested frames, one discarded warmup run, and three measured -runs. The upstream checkout was pinned to commit 6298508. +runs. The benchmark runs recorded FlashDreams commit bd0816e and upstream +commit 6298508. Citation -------- diff --git a/integrations/sana/README.md b/integrations/sana/README.md index 1a1dbef32..897de8a61 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -53,9 +53,9 @@ project's GPU runtime dependencies: uv sync --package flashdreams-sana-wm --extra dev ``` -The examples below use the public demo image, prompt, camera, and intrinsics -files from the SANA-WM release. Equivalent inputs with the same shapes work as -well. +The examples below use placeholder paths for a first-frame image, prompt, +camera trajectory, and intrinsics. Inputs must follow the same shape +conventions as the public SANA-WM release examples. ## Run @@ -63,10 +63,10 @@ Bidirectional: ```bash uv run flashdreams-run sana-wm-bidirectional \ - --image-path ../Sana/asset/sana_wm/demo_0.png \ - --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ - --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ - --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --image-path \ + --prompt-path \ + --camera-path \ + --intrinsics-path \ --num-frames 161 \ --output-dir outputs/sana_wm_bf16 ``` @@ -81,10 +81,10 @@ Streaming: ```bash uv run flashdreams-run sana-wm-streaming \ - --image-path ../Sana/asset/sana_wm/demo_0.png \ - --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ - --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ - --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --image-path \ + --prompt-path \ + --camera-path \ + --intrinsics-path \ --num-frames 241 \ --output-dir outputs/sana_wm_streaming_bf16 ``` @@ -142,10 +142,10 @@ FP8 smoke: ```bash uv run flashdreams-run sana-wm-bidirectional \ - --image-path ../Sana/asset/sana_wm/demo_0.png \ - --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ - --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ - --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --image-path \ + --prompt-path \ + --camera-path \ + --intrinsics-path \ --num-frames 161 \ --output-dir outputs/sana_wm_fp8 \ --stage1-precision fp8 \ @@ -156,50 +156,22 @@ FP4 smoke: ```bash uv run flashdreams-run sana-wm-bidirectional \ - --image-path ../Sana/asset/sana_wm/demo_0.png \ - --prompt-path ../Sana/asset/sana_wm/demo_0.txt \ - --camera-path ../Sana/asset/sana_wm/demo_0_pose.npy \ - --intrinsics-path ../Sana/asset/sana_wm/demo_0_intrinsics.npy \ + --image-path \ + --prompt-path \ + --camera-path \ + --intrinsics-path \ --num-frames 161 \ --output-dir outputs/sana_wm_fp4 \ --stage1-precision fp4 \ --refiner-precision fp4 ``` -## Parity and benchmark +## Development checks -The upstream comparison harness lives under `tests/parity_check/`, matching the -pattern used by the other benchmarked FlashDreams integrations. - -```bash -cd integrations/sana/tests/parity_check - -# Upstream + FlashDreams SANA-WM_bidirectional parity artifacts and frame diff. -bash run.sh - -# Matched SANA-WM_bidirectional upstream-vs-FlashDreams BF16 benchmark report. -DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh - -# Matched SANA-WM_streaming upstream-vs-FlashDreams benchmark report. -SANA_WM_VARIANT=streaming BENCH_SIDE=both DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh - -# SANA-WM_streaming BF16/FP8/FP4 precision sweep. -SANA_WM_VARIANT=streaming BENCH_SIDE=both BENCH_PRECISIONS=bf16,fp8,fp4 \ - DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh -``` - -For `SANA_WM_VARIANT=bidirectional`, `bench.sh` writes `outputs/bench/bench.md` -for review and `outputs/bench/perf.md` for chart-ready data. The metric is -post-load generation latency per generated clip. The upstream comparison is -BF16-only because upstream `SANA-WM_bidirectional` does not support FP8/FP4 -precision flags. - -For `SANA_WM_VARIANT=streaming`, benchmark outputs are written under -`outputs/bench/streaming//`. The metric is steady-state generation -latency per produced chunk. BF16, FP8, and FP4 comparison sweeps belong to -`SANA-WM_streaming`. - -## Tests +Contributor parity and benchmark workflows live in +[`tests/parity_check/`](tests/parity_check/README.md). That harness compares +the FlashDreams runners against the pinned upstream Sana checkout under matched +inputs. CPU-safe tests cover import, config boundaries, action parsing, intrinsics, camera conditioning, Stage-1 checkpoint schema, Stage-1 CPU forward shape, VAE diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index de14cd146..4de3af450 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -22,14 +22,16 @@ checkout, pin it to `6298508`, and apply instrumentation patches idempotently. do not change generation algorithms. A small `compat/mmcv` shim covers the registry/logging imports needed by the inference path. -The BF16 path uses the base isolated environment. FP8 and FP4 streaming -benchmarks opt into the `quant` extra, which builds TransformerEngine from the -pinned git source against the local CUDA/PyTorch stack and explicitly installs -the ONNX runtime packages TransformerEngine imports. `bench.sh` also seeds -`setuptools`, `wheel`, and `pybind11` into `./.venv` before that build because -upstream TransformerEngine does not declare all metadata/build requirements. A -new checkout therefore needs network access for `uv sync`, GitHub access for -the pinned upstream checkout and TransformerEngine source, and a CUDA build +The BF16 path uses the base isolated environment. FP8 and FP4 benchmark rows +opt into the `quant` extra, which builds TransformerEngine from the pinned git +source against the local CUDA/PyTorch stack and explicitly installs the ONNX +runtime packages TransformerEngine imports. Upstream low-precision comparisons +are streaming-only; FlashDreams-only bidirectional FP8/FP4 diagnostic rows use +the same isolated benchmark environment. `bench.sh` also seeds `setuptools`, +`wheel`, and `pybind11` into `./.venv` before that build because upstream +TransformerEngine does not declare all metadata/build requirements. A new +checkout therefore needs network access for `uv sync`, GitHub access for the +pinned upstream checkout and TransformerEngine source, and a CUDA build toolchain for low-precision runs. ## Run parity @@ -101,13 +103,21 @@ BENCH_PRECISIONS=bf16,fp8,fp4 COMPILE_STAGE1=0 NO_REFINER=0 FORCE_CUDNN_SDPA=0`. For bidirectional, the warmup/measured counts mean one live warmup generation and three measured generations inside each measured process. For streaming, those counts remain process-level warmup/measured runs -because steady state is measured from chunks inside each process. -Bidirectional outputs are under `outputs/bench/`: - -- `bench.json` - machine-readable inputs, medians, p90s, memory, and stage - timings. -- `bench.md` - human-readable report. -- `perf.md` - chart-ready data using the same benchmark metric as `bench.md`. +because steady state is measured from chunks inside each process. `bench.sh` +defaults to an upstream checkout under this directory at `./Sana`; set +`SANA_REPO` only when you need to point the harness at a different checkout. + +The default bidirectional sweep writes aggregate files under `outputs/bench/` +and precision-row files under `outputs/bench//`: + +- `outputs/bench/bench.json`, `bench.md`, and `perf.md` - precision sweep + summary files. +- `outputs/bench//bench.json` - machine-readable inputs, medians, + p90s, memory, and stage timings for that precision row. +- `outputs/bench//bench.md` - human-readable report for that + precision row. +- `outputs/bench//perf.md` - chart-ready data using the same + benchmark metric as that precision row's `bench.md`. The bidirectional benchmark metric is steady-state in-process generation latency per generated clip. Warmup generations run on the live model and are @@ -129,9 +139,12 @@ implementation as the series columns, for example: ```markdown | device | official | flashdreams | | --- | ---: | ---: | -| GB202 | 77826.74 | 76938.72 | +| GPU | 77826.74 | 76938.72 | ``` +When copying chart data into the docs tree, use a date-stamped file name such +as `perf-MMDD-bf16.md`. + Bidirectional upstream comparisons are BF16-only because upstream `SANA-WM_bidirectional` does not support FP8/FP4 precision flags. The default bidirectional precision sweep therefore runs BF16 as upstream-vs-FlashDreams, @@ -154,7 +167,7 @@ Set `DEVICE_LABEL` to label chart data with a device name that is not just `GPU`: ```bash -DEVICE_LABEL="RTX PRO 6000 Blackwell" bash bench.sh +DEVICE_LABEL="GPU model name" bash bench.sh ``` Run only the BF16 benchmark used by the `SANA-WM_bidirectional` model card with: From b15c91750fa7eab4110cfcf5e04d3ead9d8b0cff Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 13:04:04 -0700 Subject: [PATCH 59/64] Scripting changes --- integrations/sana/README.md | 9 +- integrations/sana/tests/.gitignore | 3 + .../changes_bidirectional.patch | 0 .../changes_streaming.patch | 0 .../{parity_check/run_common.sh => common.sh} | 38 +-- .../compat/mmcv/__init__.py | 0 .../{parity_check => }/compat/mmcv/runner.py | 0 .../compat/mmcv/utils/__init__.py | 0 .../compat/mmcv/utils/logging.py | 0 .../sana/tests/parity_check/README.md | 277 +++++------------- .../tests/parity_check/run_bidirectional.sh | 38 +-- .../tests/parity_check/run_flashdreams.py | 42 --- .../sana/tests/parity_check/run_streaming.sh | 42 +-- integrations/sana/tests/performance/README.md | 164 +++++++++++ .../{parity_check => performance}/bench.sh | 92 ++---- .../bench_summary.py | 0 .../bench_sweep_summary.py | 0 .../tests/{parity_check => }/pyproject.toml | 14 +- .../run_flashdreams_bidirectional.py | 0 .../run_flashdreams_common.py | 4 +- .../run_flashdreams_streaming.py | 0 ... => test_performance_benchmark_summary.py} | 14 +- .../sana/tests/{parity_check => }/uv.lock | 12 +- 23 files changed, 350 insertions(+), 399 deletions(-) create mode 100644 integrations/sana/tests/.gitignore rename integrations/sana/tests/{parity_check => }/changes_bidirectional.patch (100%) rename integrations/sana/tests/{parity_check => }/changes_streaming.patch (100%) rename integrations/sana/tests/{parity_check/run_common.sh => common.sh} (71%) rename integrations/sana/tests/{parity_check => }/compat/mmcv/__init__.py (100%) rename integrations/sana/tests/{parity_check => }/compat/mmcv/runner.py (100%) rename integrations/sana/tests/{parity_check => }/compat/mmcv/utils/__init__.py (100%) rename integrations/sana/tests/{parity_check => }/compat/mmcv/utils/logging.py (100%) delete mode 100644 integrations/sana/tests/parity_check/run_flashdreams.py create mode 100644 integrations/sana/tests/performance/README.md rename integrations/sana/tests/{parity_check => performance}/bench.sh (89%) rename integrations/sana/tests/{parity_check => performance}/bench_summary.py (100%) rename integrations/sana/tests/{parity_check => performance}/bench_sweep_summary.py (100%) rename integrations/sana/tests/{parity_check => }/pyproject.toml (78%) rename integrations/sana/tests/{parity_check => }/run_flashdreams_bidirectional.py (100%) rename integrations/sana/tests/{parity_check => }/run_flashdreams_common.py (98%) rename integrations/sana/tests/{parity_check => }/run_flashdreams_streaming.py (100%) rename integrations/sana/tests/{test_parity_benchmark_summary.py => test_performance_benchmark_summary.py} (98%) rename integrations/sana/tests/{parity_check => }/uv.lock (99%) diff --git a/integrations/sana/README.md b/integrations/sana/README.md index 897de8a61..a6df36528 100644 --- a/integrations/sana/README.md +++ b/integrations/sana/README.md @@ -168,10 +168,11 @@ uv run flashdreams-run sana-wm-bidirectional \ ## Development checks -Contributor parity and benchmark workflows live in -[`tests/parity_check/`](tests/parity_check/README.md). That harness compares -the FlashDreams runners against the pinned upstream Sana checkout under matched -inputs. +Contributor correctness parity checks live in +[`tests/parity_check/`](tests/parity_check/README.md). Performance benchmarks +live separately in [`tests/performance/`](tests/performance/README.md). Both +workflows compare the FlashDreams runners against the pinned upstream Sana +checkout under matched inputs. CPU-safe tests cover import, config boundaries, action parsing, intrinsics, camera conditioning, Stage-1 checkpoint schema, Stage-1 CPU forward shape, VAE diff --git a/integrations/sana/tests/.gitignore b/integrations/sana/tests/.gitignore new file mode 100644 index 000000000..291374499 --- /dev/null +++ b/integrations/sana/tests/.gitignore @@ -0,0 +1,3 @@ +/Sana/ +/.venv/ +/PanGPA.log diff --git a/integrations/sana/tests/parity_check/changes_bidirectional.patch b/integrations/sana/tests/changes_bidirectional.patch similarity index 100% rename from integrations/sana/tests/parity_check/changes_bidirectional.patch rename to integrations/sana/tests/changes_bidirectional.patch diff --git a/integrations/sana/tests/parity_check/changes_streaming.patch b/integrations/sana/tests/changes_streaming.patch similarity index 100% rename from integrations/sana/tests/parity_check/changes_streaming.patch rename to integrations/sana/tests/changes_streaming.patch diff --git a/integrations/sana/tests/parity_check/run_common.sh b/integrations/sana/tests/common.sh similarity index 71% rename from integrations/sana/tests/parity_check/run_common.sh rename to integrations/sana/tests/common.sh index 808d36750..df11e5168 100644 --- a/integrations/sana/tests/parity_check/run_common.sh +++ b/integrations/sana/tests/common.sh @@ -14,26 +14,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -PARITY_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PARITY_REPO_ROOT="$(cd "${PARITY_SCRIPT_DIR}/../../../.." && pwd)" -PARITY_REPO_URL="https://github.com/NVlabs/Sana.git" -PARITY_PIN_COMMIT="6298508" +SANA_TEST_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SANA_TEST_REPO_ROOT="$(cd "${SANA_TEST_DIR}/../../.." && pwd)" +SANA_TEST_REPO_URL="https://github.com/NVlabs/Sana.git" +SANA_TEST_PIN_COMMIT="6298508" -parity_abspath() { +sana_test_abspath() { case "$1" in /*) printf '%s\n' "$1" ;; *) printf '%s/%s\n' "${PWD}" "$1" ;; esac } -parity_is_true() { +sana_test_is_true() { case "${1,,}" in 1|true|yes|on) return 0 ;; *) return 1 ;; esac } -parity_apply_patch_once() { +sana_test_apply_patch_once() { local patch_file="$1" if git apply --reverse --check "${patch_file}" >/dev/null 2>&1; then echo "[setup] patch already applied, skipping ${patch_file}" @@ -56,12 +56,12 @@ parity_apply_patch_once() { fi } -parity_prepare_sana_repo() { +sana_test_prepare_sana_repo() { local sana_repo="$1" shift if [[ ! -d "${sana_repo}/.git" ]]; then - echo "[setup] cloning ${PARITY_REPO_URL} -> ${sana_repo}" - git clone "${PARITY_REPO_URL}" "${sana_repo}" + echo "[setup] cloning ${SANA_TEST_REPO_URL} -> ${sana_repo}" + git clone "${SANA_TEST_REPO_URL}" "${sana_repo}" else echo "[setup] repo already present at ${sana_repo}, skipping clone" fi @@ -69,27 +69,27 @@ parity_prepare_sana_repo() { cd "${sana_repo}" local current_commit current_commit="$(git rev-parse --short HEAD)" - if [[ "${current_commit}" != "${PARITY_PIN_COMMIT}" ]]; then - echo "[setup] checking out pinned commit ${PARITY_PIN_COMMIT}" - git checkout "${PARITY_PIN_COMMIT}" + if [[ "${current_commit}" != "${SANA_TEST_PIN_COMMIT}" ]]; then + echo "[setup] checking out pinned commit ${SANA_TEST_PIN_COMMIT}" + git checkout "${SANA_TEST_PIN_COMMIT}" else - echo "[setup] already at pinned commit ${PARITY_PIN_COMMIT}, skipping checkout" + echo "[setup] already at pinned commit ${SANA_TEST_PIN_COMMIT}, skipping checkout" fi local patch_file for patch_file in "$@"; do - parity_apply_patch_once "${patch_file}" + sana_test_apply_patch_once "${patch_file}" done } -parity_sync_venv() { +sana_test_sync_venv() { echo "[setup] ensuring Python deps via uv sync (isolated venv)" - ( cd "${PARITY_SCRIPT_DIR}" && uv sync ) + ( cd "${SANA_TEST_DIR}" && uv sync ) } -parity_upstream_pythonpath() { +sana_test_upstream_pythonpath() { local sana_repo="$1" - local upstream_pythonpath="${PARITY_SCRIPT_DIR}/compat:${sana_repo}" + local upstream_pythonpath="${SANA_TEST_DIR}/compat:${sana_repo}" if [[ -n "${PYTHONPATH:-}" ]]; then upstream_pythonpath="${upstream_pythonpath}:${PYTHONPATH}" fi diff --git a/integrations/sana/tests/parity_check/compat/mmcv/__init__.py b/integrations/sana/tests/compat/mmcv/__init__.py similarity index 100% rename from integrations/sana/tests/parity_check/compat/mmcv/__init__.py rename to integrations/sana/tests/compat/mmcv/__init__.py diff --git a/integrations/sana/tests/parity_check/compat/mmcv/runner.py b/integrations/sana/tests/compat/mmcv/runner.py similarity index 100% rename from integrations/sana/tests/parity_check/compat/mmcv/runner.py rename to integrations/sana/tests/compat/mmcv/runner.py diff --git a/integrations/sana/tests/parity_check/compat/mmcv/utils/__init__.py b/integrations/sana/tests/compat/mmcv/utils/__init__.py similarity index 100% rename from integrations/sana/tests/parity_check/compat/mmcv/utils/__init__.py rename to integrations/sana/tests/compat/mmcv/utils/__init__.py diff --git a/integrations/sana/tests/parity_check/compat/mmcv/utils/logging.py b/integrations/sana/tests/compat/mmcv/utils/logging.py similarity index 100% rename from integrations/sana/tests/parity_check/compat/mmcv/utils/logging.py rename to integrations/sana/tests/compat/mmcv/utils/logging.py diff --git a/integrations/sana/tests/parity_check/README.md b/integrations/sana/tests/parity_check/README.md index 4de3af450..86c886f23 100644 --- a/integrations/sana/tests/parity_check/README.md +++ b/integrations/sana/tests/parity_check/README.md @@ -3,38 +3,32 @@ SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# SANA-WM_bidirectional and SANA-WM_streaming parity and benchmark harness - -This harness compares upstream -[`NVlabs/Sana`](https://github.com/NVlabs/Sana) `SANA-WM_bidirectional` and -`SANA-WM_streaming` against the matching in-tree FlashDreams integrations on -matched demo inputs, seeds, resolution, and precision settings. `run.sh` -dispatches to separate bidirectional and streaming parity runners. By default, -`bench.sh` runs both benchmark comparisons; set -`SANA_WM_VARIANT=bidirectional` or `SANA_WM_VARIANT=streaming` to run only one -variant. - -All dependencies live in this directory's isolated `./.venv`. `run.sh` and -`bench.sh` run `uv sync` for that environment, reuse or clone the upstream -checkout, pin it to `6298508`, and apply instrumentation patches idempotently. -`changes_bidirectional.patch` covers the bidirectional entrypoint. -`changes_streaming.patch` covers the streaming entrypoint metadata. The patches -do not change generation algorithms. A small `compat/mmcv` shim covers the -registry/logging imports needed by the inference path. - -The BF16 path uses the base isolated environment. FP8 and FP4 benchmark rows -opt into the `quant` extra, which builds TransformerEngine from the pinned git -source against the local CUDA/PyTorch stack and explicitly installs the ONNX -runtime packages TransformerEngine imports. Upstream low-precision comparisons -are streaming-only; FlashDreams-only bidirectional FP8/FP4 diagnostic rows use -the same isolated benchmark environment. `bench.sh` also seeds `setuptools`, -`wheel`, and `pybind11` into `./.venv` before that build because upstream -TransformerEngine does not declare all metadata/build requirements. A new -checkout therefore needs network access for `uv sync`, GitHub access for the -pinned upstream checkout and TransformerEngine source, and a CUDA build -toolchain for low-precision runs. - -## Run parity +# SANA-WM Correctness Parity + +This directory contains correctness checks for the FlashDreams +`SANA-WM_bidirectional` and `SANA-WM_streaming` integrations. The checks compare +against the pinned upstream [`NVlabs/Sana`](https://github.com/NVlabs/Sana) +entrypoints on matched demo inputs, seeds, resolution, and precision settings. + +Do not use `run.sh` timings as performance data. Performance benchmarks live in +[`../performance/`](../performance/README.md). + +Shared harness files live one level up in `integrations/sana/tests/`: + +- `common.sh` - upstream checkout, patch, and venv setup helpers. +- `pyproject.toml` and `uv.lock` - isolated `../.venv` dependency environment. +- `changes_bidirectional.patch` and `changes_streaming.patch` - upstream Sana + instrumentation patches. +- `compat/` - small upstream import compatibility shims. +- `run_flashdreams_*.py` - FlashDreams entrypoint wrappers shared with the + performance harness. + +`run.sh` runs `uv sync` from `integrations/sana/tests/`, reuses or clones the +upstream checkout at `../Sana`, pins it to `6298508`, and applies patches +idempotently. The patches add instrumentation and do not change generation +algorithms. + +## Run Checks ```bash cd integrations/sana/tests/parity_check @@ -42,26 +36,55 @@ bash run.sh ``` `run.sh` defaults to `SANA_WM_VARIANT=bidirectional`. Set -`SANA_WM_VARIANT=streaming` to run the streaming parity and continuity path: +`SANA_WM_VARIANT=streaming` to run the streaming frame-parity and continuity +path: ```bash SANA_WM_VARIANT=streaming bash run.sh ``` -Defaults: +Shared defaults: -- `SANA_REPO=$HOME/dev/Sana` +- `SANA_REPO=../Sana` - upstream pin `6298508` -- demo input `asset/sana_wm/demo_0.*` -- `NUM_FRAMES=121` -- `SEED=42` -- `NO_REFINER=1` to isolate Stage-1 plus SANA VAE decode +- demo input `${SANA_REPO}/asset/sana_wm/demo_0.*` - `STAGE1_PRECISION=bf16` - `REFINER_PRECISION=$STAGE1_PRECISION` - `QUANT_BACKEND=auto` -- `FORCE_CUDNN_SDPA=0` -Outputs are written under `outputs/parity/`: +Bidirectional defaults: + +- `OUTPUT_DIR=outputs/parity` +- `NUM_FRAMES=121` +- `FPS=16` +- `STEP=60` +- `CFG_SCALE=5.0` +- `SEED=42` +- `NO_REFINER=1` to isolate Stage-1 plus SANA VAE decode + +Streaming defaults: + +- `OUTPUT_DIR=outputs/parity/streaming` +- `NUM_FRAMES=241` +- `FPS=16` +- `CFG_SCALE=1.0` +- `FLOW_SHIFT=8.0` +- `SEED=42` +- `NO_REFINER=0`; `NO_REFINER=1` is rejected for streaming parity +- `STREAMING_ACTION=w-80,dw-40,w-80,aw-40` +- `TRANSLATION_SPEED=0.025` +- `ROTATION_SPEED_DEG=0.6` +- `STREAMING_DENOISING_STEP_LIST=1000,960,889,727,0` +- `STREAMING_NUM_FRAME_PER_BLOCK=3` +- `STREAMING_NUM_CACHED_BLOCKS=2` +- `STREAMING_REFINER_BLOCK_SIZE=3` +- `STREAMING_REFINER_KV_MAX_FRAMES=11` +- `STREAMING_SINK_SIZE=1` +- `STREAMING_NO_COMPILE=1` +- `STREAMING_OUTPUT_MODE=mp4` +- `STREAMING_SAMPLE_FRAME_STRIDE=1` + +Bidirectional outputs are written under `outputs/parity/`: - `upstream/frames.npy` - `upstream/stats.json` @@ -69,7 +92,7 @@ Outputs are written under `outputs/parity/`: - `flashdreams/stats.json` - `parity.json` -Streaming parity writes to `outputs/parity/streaming/` by default: +Streaming outputs are written under `outputs/parity/streaming/`: - `upstream/frames.npz` - `upstream/stats.json` @@ -78,166 +101,12 @@ Streaming parity writes to `outputs/parity/streaming/` by default: - `parity.json` - `continuity.json` -Set `NO_REFINER=0` to compare the full Stage-1 + LTX-2 refiner path. Set -`COMPILE_STAGE1=1` to wrap each Stage-1 DiT with `torch.compile`; this is opt-in -because the pinned upstream Sana stack can fail during TorchInductor Triton -compilation on current PyTorch/Triton builds. - -Streaming parity always uses the full upstream streaming stack; `NO_REFINER=1` -is rejected because upstream streaming has no no-refiner entrypoint. It saves -every decoded frame and checks both frame parity and chunk-boundary continuity. -`STREAMING_NO_COMPILE=1` is the streaming parity default to keep quality checks -debug-friendly; set `STREAMING_NO_COMPILE=0` when explicitly checking compiled -streaming behavior. - -## Run SANA-WM benchmarks - -```bash -cd integrations/sana/tests/parity_check -bash bench.sh -``` - -Benchmark defaults to both sibling variants: -`SANA_WM_VARIANT=both BENCH_SIDE=both WARMUP_RUNS=1 MEASURED_RUNS=3 -BENCH_PRECISIONS=bf16,fp8,fp4 COMPILE_STAGE1=0 NO_REFINER=0 -FORCE_CUDNN_SDPA=0`. For bidirectional, the warmup/measured counts mean one -live warmup generation and three measured generations inside each measured -process. For streaming, those counts remain process-level warmup/measured runs -because steady state is measured from chunks inside each process. `bench.sh` -defaults to an upstream checkout under this directory at `./Sana`; set -`SANA_REPO` only when you need to point the harness at a different checkout. - -The default bidirectional sweep writes aggregate files under `outputs/bench/` -and precision-row files under `outputs/bench//`: - -- `outputs/bench/bench.json`, `bench.md`, and `perf.md` - precision sweep - summary files. -- `outputs/bench//bench.json` - machine-readable inputs, medians, - p90s, memory, and stage timings for that precision row. -- `outputs/bench//bench.md` - human-readable report for that - precision row. -- `outputs/bench//perf.md` - chart-ready data using the same - benchmark metric as that precision row's `bench.md`. - -The bidirectional benchmark metric is steady-state in-process generation -latency per generated clip. Warmup generations run on the live model and are -excluded from the headline metric. Model construction, checkpoint loading, -video writing, and frame dumps are outside the timing boundary. -`SANA-WM_bidirectional` renders each requested clip in one generation pass, -rather than as independently timed frames. With the default `NO_REFINER=0`, the -timed work covers conditioning, Stage-1 DiT, LTX-2 refiner, and SANA VAE decode. -Set `NO_REFINER=1` only for a diagnostic Stage-1 plus SANA VAE decode -benchmark. - -`bench.md` also reports Stage-1 DiT, conditioning/encode, VAE decode, optional -refiner, memory, and frame-normalized diagnostic breakdowns. Those rows explain -the benchmark result; they are not a second benchmark metric. - -Each model-card chart file should have GPU/device as the first column and -implementation as the series columns, for example: - -```markdown -| device | official | flashdreams | -| --- | ---: | ---: | -| GPU | 77826.74 | 76938.72 | -``` - -When copying chart data into the docs tree, use a date-stamped file name such -as `perf-MMDD-bf16.md`. - -Bidirectional upstream comparisons are BF16-only because upstream -`SANA-WM_bidirectional` does not support FP8/FP4 precision flags. The default -bidirectional precision sweep therefore runs BF16 as upstream-vs-FlashDreams, -then records FP8 and FP4 as FlashDreams-only diagnostic rows. Use -`outputs/bench/bf16/perf.md` from the BF16 bidirectional row as the model-card -chart data. - -Set `FORCE_CUDNN_SDPA=1` only for backend-isolation probes. It is not the -default `SANA-WM_bidirectional` benchmark setting. - -On shared or mixed-GPU hosts, set `CUDA_VISIBLE_DEVICES` to the benchmark GPU's -UUID before running the harness. This keeps PyTorch's NVML and CUDA runtime -device enumeration consistent without changing the benchmark backend policy: - -```bash -CUDA_VISIBLE_DEVICES=GPU-... bash bench.sh -``` - -Set `DEVICE_LABEL` to label chart data with a device name that is not just -`GPU`: - -```bash -DEVICE_LABEL="GPU model name" bash bench.sh -``` - -Run only the BF16 benchmark used by the `SANA-WM_bidirectional` model card with: - -```bash -SANA_WM_VARIANT=bidirectional BENCH_PRECISIONS=bf16 bash bench.sh -``` - -The scripts do not set allocator overrides or GPU wait loops. If GPU contention -matters in your environment, handle it outside the harness. - -## Run SANA-WM_streaming benchmark - -Use `SANA_WM_VARIANT=streaming` to compare only upstream `SANA-WM_streaming` -with the FlashDreams `sana-wm-streaming` runner: - -```bash -cd integrations/sana/tests/parity_check -SANA_WM_VARIANT=streaming BENCH_SIDE=both bash bench.sh -``` - -Streaming defaults use the upstream streaming entrypoint -`inference_video_scripts/wm/inference_sana_wm_streaming.py`, the -`Efficient-Large-Model/SANA-WM_streaming` weights, `NUM_FRAMES=241`, -`CFG_SCALE=1.0`, `NO_REFINER=0`, and -`STREAMING_ACTION=w-80,dw-40,w-80,aw-40`. Set `STREAMING_ACTION=` to use -`CAMERA_PATH` instead. Outputs are precision-aware: - -- `outputs/bench/streaming//upstream/run_/stats.json` -- `outputs/bench/streaming//upstream/run_/command.txt` -- `outputs/bench/streaming//flashdreams/run_/stats.json` -- `outputs/bench/streaming//flashdreams/run_/command.txt` -- `outputs/bench/streaming//bench.json` -- `outputs/bench/streaming//bench.md` -- `outputs/bench/streaming//perf.md` - -The streaming headline metric is steady-state generation milliseconds per -produced chunk. Warmup runs and the first decoded chunk are excluded; full-clip -wall time remains in `bench.json` and `bench.md` as supporting data. - -These streaming benchmark rows measure latency only. Run the streaming parity -and continuity checks before making quality claims about a FlashDreams -streaming change. - -By default, upstream compiles the streaming refiner transformer. The benchmark -passes the equivalent `torch.compile` wrapper to FlashDreams. Set -`STREAMING_NO_COMPILE=1` to disable refiner compile on both sides. - -Run the comparison precision sweep in order with: - -```bash -SANA_WM_VARIANT=streaming BENCH_SIDE=both bash bench.sh -``` - -Omit `SANA_WM_VARIANT` to regenerate both model-card datasets in one command. -The default run uses the BF16/FP8/FP4 precision sweep for streaming and for -FlashDreams-only bidirectional diagnostics, while the official bidirectional -comparison stays BF16-only: - -```bash -bash bench.sh -``` +For bidirectional parity, set `NO_REFINER=0` to compare the full Stage-1 plus +LTX-2 refiner path. Set `COMPILE_STAGE1=1` only when explicitly checking the +compiled Stage-1 path. -FP8 and FP4 are passed through the precision flags supported by each measured -side. For upstream this means streaming only; for FlashDreams this also includes -bidirectional diagnostic rows. Those precision runs sync the isolated venv with -the `quant` extra before launch. `bench.sh` lets Transformer Engine use its -upstream CUDA-version-aware default architecture set unless `NVTE_CUDA_ARCHS` is -set explicitly. The script records that setting in the isolated venv and -rebuilds Transformer Engine once when it changes, so stale extension wheels from -an earlier architecture setting are not reused. If Transformer Engine, hardware, -or upstream code rejects a precision, earlier precision outputs remain in place -and the failing command is recorded in the corresponding `command.txt`. +Streaming parity always uses the full upstream streaming stack. It saves every +decoded frame and checks both frame parity and chunk-boundary continuity. For +this correctness workflow only, `STREAMING_NO_COMPILE=1` is the default to keep +parity failures easier to debug. Do not carry that setting into performance +runs unless intentionally measuring no-compile streaming behavior. diff --git a/integrations/sana/tests/parity_check/run_bidirectional.sh b/integrations/sana/tests/parity_check/run_bidirectional.sh index 67faf566d..2efdfe896 100644 --- a/integrations/sana/tests/parity_check/run_bidirectional.sh +++ b/integrations/sana/tests/parity_check/run_bidirectional.sh @@ -20,18 +20,20 @@ set -euo pipefail -source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/run_common.sh" +PARITY_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SANA_TEST_DIR="$(cd "${PARITY_SCRIPT_DIR}/.." && pwd)" +source "${SANA_TEST_DIR}/common.sh" -SANA_REPO="$(parity_abspath "${SANA_REPO:-${HOME}/dev/Sana}")" -PATCH_FILE="${PARITY_SCRIPT_DIR}/changes_bidirectional.patch" +SANA_REPO="$(sana_test_abspath "${SANA_REPO:-${SANA_TEST_DIR}/Sana}")" +PATCH_FILE="${SANA_TEST_DIR}/changes_bidirectional.patch" -OUTPUT_DIR="$(parity_abspath "${OUTPUT_DIR:-${PARITY_SCRIPT_DIR}/outputs/parity}")" +OUTPUT_DIR="$(sana_test_abspath "${OUTPUT_DIR:-${PARITY_SCRIPT_DIR}/outputs/parity}")" UPSTREAM_OUT="${OUTPUT_DIR}/upstream" FLASHDREAMS_OUT="${OUTPUT_DIR}/flashdreams" -IMAGE_PATH="$(parity_abspath "${IMAGE_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.png}")" -PROMPT_PATH="$(parity_abspath "${PROMPT_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.txt}")" -CAMERA_PATH="$(parity_abspath "${CAMERA_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_pose.npy}")" -INTRINSICS_PATH="$(parity_abspath "${INTRINSICS_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_intrinsics.npy}")" +IMAGE_PATH="$(sana_test_abspath "${IMAGE_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.png}")" +PROMPT_PATH="$(sana_test_abspath "${PROMPT_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.txt}")" +CAMERA_PATH="$(sana_test_abspath "${CAMERA_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_pose.npy}")" +INTRINSICS_PATH="$(sana_test_abspath "${INTRINSICS_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_intrinsics.npy}")" NUM_FRAMES="${NUM_FRAMES:-121}" FPS="${FPS:-16}" STEP="${STEP:-60}" @@ -44,9 +46,9 @@ STAGE1_PRECISION="${STAGE1_PRECISION:-bf16}" REFINER_PRECISION="${REFINER_PRECISION:-${STAGE1_PRECISION}}" QUANT_BACKEND="${QUANT_BACKEND:-auto}" -parity_prepare_sana_repo "${SANA_REPO}" "${PATCH_FILE}" -parity_sync_venv -UPSTREAM_PYTHONPATH="$(parity_upstream_pythonpath "${SANA_REPO}")" +sana_test_prepare_sana_repo "${SANA_REPO}" "${PATCH_FILE}" +sana_test_sync_venv +UPSTREAM_PYTHONPATH="$(sana_test_upstream_pythonpath "${SANA_REPO}")" mkdir -p "${UPSTREAM_OUT}" "${FLASHDREAMS_OUT}" @@ -57,19 +59,19 @@ FLASHDREAMS_STATS="${FLASHDREAMS_OUT}/stats.json" UPSTREAM_REFINER_ARGS=() FLASHDREAMS_REFINER_ARGS=() -if parity_is_true "${NO_REFINER}"; then +if sana_test_is_true "${NO_REFINER}"; then UPSTREAM_REFINER_ARGS+=(--no_refiner) FLASHDREAMS_REFINER_ARGS+=(--no-refiner) fi UPSTREAM_BACKEND_ARGS=() FLASHDREAMS_BACKEND_ARGS=() -if parity_is_true "${FORCE_CUDNN_SDPA}"; then +if sana_test_is_true "${FORCE_CUDNN_SDPA}"; then UPSTREAM_BACKEND_ARGS+=(--force_cudnn_sdpa) FLASHDREAMS_BACKEND_ARGS+=(--force-cudnn-sdpa) fi UPSTREAM_COMPILE_ARGS=() FLASHDREAMS_COMPILE_ARGS=() -if parity_is_true "${COMPILE_STAGE1}"; then +if sana_test_is_true "${COMPILE_STAGE1}"; then UPSTREAM_COMPILE_ARGS+=(--compile_stage1) FLASHDREAMS_COMPILE_ARGS+=(--compile-stage1) fi @@ -84,7 +86,7 @@ FLASHDREAMS_PRECISION_ARGS=( ) echo "[run] upstream SANA-WM bidirectional -> ${UPSTREAM_OUT}" -( cd "${PARITY_SCRIPT_DIR}" && \ +( cd "${SANA_TEST_DIR}" && \ PYTHONPATH="${UPSTREAM_PYTHONPATH}" \ uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm.py" \ --image "${IMAGE_PATH}" \ @@ -107,8 +109,8 @@ echo "[run] upstream SANA-WM bidirectional -> ${UPSTREAM_OUT}" "${UPSTREAM_REFINER_ARGS[@]}" ) echo "[run] FlashDreams SANA-WM bidirectional -> ${FLASHDREAMS_OUT}" -( cd "${PARITY_SCRIPT_DIR}" && \ - uv run python "${PARITY_SCRIPT_DIR}/run_flashdreams_bidirectional.py" \ +( cd "${SANA_TEST_DIR}" && \ + uv run python "${SANA_TEST_DIR}/run_flashdreams_bidirectional.py" \ --image-path "${IMAGE_PATH}" \ --prompt-path "${PROMPT_PATH}" \ --camera-path "${CAMERA_PATH}" \ @@ -128,7 +130,7 @@ echo "[run] FlashDreams SANA-WM bidirectional -> ${FLASHDREAMS_OUT}" "${FLASHDREAMS_REFINER_ARGS[@]}" ) echo "[diff] summarising parity -> ${OUTPUT_DIR}/parity.json" -( cd "${PARITY_SCRIPT_DIR}" && \ +( cd "${SANA_TEST_DIR}" && \ uv run python "${PARITY_SCRIPT_DIR}/diff_parity.py" \ --upstream "${UPSTREAM_FRAMES}" \ --flashdreams "${FLASHDREAMS_FRAMES}" \ diff --git a/integrations/sana/tests/parity_check/run_flashdreams.py b/integrations/sana/tests/parity_check/run_flashdreams.py deleted file mode 100644 index 2901ceb7a..000000000 --- a/integrations/sana/tests/parity_check/run_flashdreams.py +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Compatibility dispatcher for FlashDreams SANA-WM benchmark helpers.""" - -from __future__ import annotations - -import argparse - - -def main(argv: list[str] | None = None) -> None: - parser = argparse.ArgumentParser(add_help=False) - parser.add_argument( - "--variant", - choices=["bidirectional", "streaming"], - default="bidirectional", - ) - args, rest = parser.parse_known_args(argv) - if args.variant == "streaming": - from run_flashdreams_streaming import main as run_streaming - - run_streaming(rest) - else: - from run_flashdreams_bidirectional import main as run_bidirectional - - run_bidirectional(rest) - - -if __name__ == "__main__": - main() diff --git a/integrations/sana/tests/parity_check/run_streaming.sh b/integrations/sana/tests/parity_check/run_streaming.sh index 42910e18d..2465c2f3e 100644 --- a/integrations/sana/tests/parity_check/run_streaming.sh +++ b/integrations/sana/tests/parity_check/run_streaming.sh @@ -20,18 +20,20 @@ set -euo pipefail -source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/run_common.sh" +PARITY_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SANA_TEST_DIR="$(cd "${PARITY_SCRIPT_DIR}/.." && pwd)" +source "${SANA_TEST_DIR}/common.sh" -SANA_REPO="$(parity_abspath "${SANA_REPO:-${HOME}/dev/Sana}")" -PATCH_FILE="${PARITY_SCRIPT_DIR}/changes_streaming.patch" +SANA_REPO="$(sana_test_abspath "${SANA_REPO:-${SANA_TEST_DIR}/Sana}")" +PATCH_FILE="${SANA_TEST_DIR}/changes_streaming.patch" -OUTPUT_DIR="$(parity_abspath "${OUTPUT_DIR:-${PARITY_SCRIPT_DIR}/outputs/parity/streaming}")" +OUTPUT_DIR="$(sana_test_abspath "${OUTPUT_DIR:-${PARITY_SCRIPT_DIR}/outputs/parity/streaming}")" UPSTREAM_OUT="${OUTPUT_DIR}/upstream" FLASHDREAMS_OUT="${OUTPUT_DIR}/flashdreams" -IMAGE_PATH="$(parity_abspath "${IMAGE_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.png}")" -PROMPT_PATH="$(parity_abspath "${PROMPT_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.txt}")" -CAMERA_PATH="$(parity_abspath "${CAMERA_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_pose.npy}")" -INTRINSICS_PATH="$(parity_abspath "${INTRINSICS_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_intrinsics.npy}")" +IMAGE_PATH="$(sana_test_abspath "${IMAGE_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.png}")" +PROMPT_PATH="$(sana_test_abspath "${PROMPT_PATH:-${SANA_REPO}/asset/sana_wm/demo_0.txt}")" +CAMERA_PATH="$(sana_test_abspath "${CAMERA_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_pose.npy}")" +INTRINSICS_PATH="$(sana_test_abspath "${INTRINSICS_PATH:-${SANA_REPO}/asset/sana_wm/demo_0_intrinsics.npy}")" NUM_FRAMES="${NUM_FRAMES:-241}" FPS="${FPS:-16}" @@ -60,7 +62,7 @@ STREAMING_ENCODER="${STREAMING_ENCODER:-${SANA_WM_STREAMING_MP4_ENCODER:-libx264 STREAMING_OUTPUT_MODE="${STREAMING_OUTPUT_MODE:-mp4}" STREAMING_SAMPLE_FRAME_STRIDE="${STREAMING_SAMPLE_FRAME_STRIDE:-1}" -if parity_is_true "${NO_REFINER}"; then +if sana_test_is_true "${NO_REFINER}"; then echo "[run] ERROR: streaming parity uses the full upstream streaming stack; NO_REFINER=1 is not supported." >&2 exit 1 fi @@ -73,14 +75,14 @@ if [[ "${STREAMING_SAMPLE_FRAME_STRIDE}" != "1" ]]; then exit 1 fi -parity_prepare_sana_repo "${SANA_REPO}" "${PATCH_FILE}" +sana_test_prepare_sana_repo "${SANA_REPO}" "${PATCH_FILE}" if [[ ! -f "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm_streaming.py" ]]; then echo "[run] ERROR: pinned Sana checkout has no SANA-WM streaming entrypoint." >&2 echo " Expected: inference_video_scripts/wm/inference_sana_wm_streaming.py" >&2 exit 1 fi -parity_sync_venv -UPSTREAM_PYTHONPATH="$(parity_upstream_pythonpath "${SANA_REPO}")" +sana_test_sync_venv +UPSTREAM_PYTHONPATH="$(sana_test_upstream_pythonpath "${SANA_REPO}")" mkdir -p "${UPSTREAM_OUT}" "${FLASHDREAMS_OUT}" @@ -106,20 +108,20 @@ if [[ -n "${STREAMING_ACTION}" ]]; then fi UPSTREAM_MODE_ARGS=(--output_mode "${STREAMING_OUTPUT_MODE}") FLASHDREAMS_MODE_ARGS=(--output-mode "${STREAMING_OUTPUT_MODE}") -if parity_is_true "${STREAMING_NO_COMPILE}"; then +if sana_test_is_true "${STREAMING_NO_COMPILE}"; then UPSTREAM_MODE_ARGS+=(--no_compile) fi FLASHDREAMS_COMPILE_ARGS=() -if ! parity_is_true "${STREAMING_NO_COMPILE}"; then +if ! sana_test_is_true "${STREAMING_NO_COMPILE}"; then FLASHDREAMS_COMPILE_ARGS+=(--compile-streaming-refiner) fi FLASHDREAMS_BACKEND_ARGS=() -if parity_is_true "${FORCE_CUDNN_SDPA}"; then +if sana_test_is_true "${FORCE_CUDNN_SDPA}"; then FLASHDREAMS_BACKEND_ARGS+=(--force-cudnn-sdpa) fi echo "[run] upstream SANA-WM streaming -> ${UPSTREAM_OUT}" -( cd "${PARITY_SCRIPT_DIR}" && \ +( cd "${SANA_TEST_DIR}" && \ PYTHONPATH="${UPSTREAM_PYTHONPATH}" \ uv run python "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm_streaming.py" \ --image "${IMAGE_PATH}" \ @@ -150,8 +152,8 @@ echo "[run] upstream SANA-WM streaming -> ${UPSTREAM_OUT}" "${UPSTREAM_MODE_ARGS[@]}" ) echo "[run] FlashDreams SANA-WM streaming -> ${FLASHDREAMS_OUT}" -( cd "${PARITY_SCRIPT_DIR}" && \ - uv run python "${PARITY_SCRIPT_DIR}/run_flashdreams_streaming.py" \ +( cd "${SANA_TEST_DIR}" && \ + uv run python "${SANA_TEST_DIR}/run_flashdreams_streaming.py" \ --image-path "${IMAGE_PATH}" \ --prompt-path "${PROMPT_PATH}" \ "${FLASHDREAMS_CAMERA_ARGS[@]}" \ @@ -178,14 +180,14 @@ echo "[run] FlashDreams SANA-WM streaming -> ${FLASHDREAMS_OUT}" "${FLASHDREAMS_BACKEND_ARGS[@]}" ) echo "[diff] summarising streaming frame parity -> ${OUTPUT_DIR}/parity.json" -( cd "${PARITY_SCRIPT_DIR}" && \ +( cd "${SANA_TEST_DIR}" && \ uv run python "${PARITY_SCRIPT_DIR}/diff_parity.py" \ --upstream "${UPSTREAM_FRAMES}" \ --flashdreams "${FLASHDREAMS_FRAMES}" \ --output "${OUTPUT_DIR}/parity.json" ) echo "[diff] summarising streaming continuity -> ${OUTPUT_DIR}/continuity.json" -( cd "${PARITY_SCRIPT_DIR}" && \ +( cd "${SANA_TEST_DIR}" && \ uv run python "${PARITY_SCRIPT_DIR}/streaming_continuity.py" \ --upstream "${UPSTREAM_FRAMES}" \ --flashdreams "${FLASHDREAMS_FRAMES}" \ diff --git a/integrations/sana/tests/performance/README.md b/integrations/sana/tests/performance/README.md new file mode 100644 index 000000000..03811023c --- /dev/null +++ b/integrations/sana/tests/performance/README.md @@ -0,0 +1,164 @@ + + +# SANA-WM Performance Benchmarks + +This directory contains timing benchmarks for the FlashDreams +`SANA-WM_bidirectional` and `SANA-WM_streaming` integrations against the pinned +upstream [`NVlabs/Sana`](https://github.com/NVlabs/Sana) entrypoints. + +Do not use these latency rows as quality evidence without a matching +correctness pass from [`../parity_check/`](../parity_check/README.md). + +Shared harness files live one level up in `integrations/sana/tests/`: + +- `common.sh` - upstream checkout, patch, and venv setup helpers. +- `pyproject.toml` and `uv.lock` - isolated `../.venv` dependency environment. +- `changes_bidirectional.patch` and `changes_streaming.patch` - upstream Sana + instrumentation patches. +- `compat/` - small upstream import compatibility shims. +- `run_flashdreams_*.py` - FlashDreams entrypoint wrappers shared with parity. + +`bench.sh` runs `uv sync` from `integrations/sana/tests/`, reuses or clones the +upstream checkout at `../Sana`, pins it to `6298508`, and applies patches +idempotently. The patches add instrumentation and do not change generation +algorithms. + +The BF16 path uses the base isolated environment. FP8 and FP4 rows opt into the +`quant` extra, which builds TransformerEngine from the pinned git source +against the local CUDA/PyTorch stack. A new checkout needs network access for +`uv sync`, GitHub access for the pinned upstream checkout and TransformerEngine +source, and a CUDA build toolchain for low-precision runs. + +## Run Benchmarks + +```bash +cd integrations/sana/tests/performance +bash bench.sh +``` + +`bench.sh` defaults to both sibling variants: +`SANA_WM_VARIANT=both BENCH_SIDE=both WARMUP_RUNS=1 MEASURED_RUNS=3 +BENCH_PRECISIONS=bf16,fp8,fp4 NO_REFINER=0`. + +For bidirectional, the warmup/measured counts mean one live warmup generation +and three measured generations inside each measured process. For streaming, +those counts are process-level warmup/measured runs because steady state is +measured from chunks inside each process. Streaming performance benchmarks +compile the refiner on both sides by default; leave `STREAMING_NO_COMPILE` +unset unless intentionally measuring no-compile behavior. + +Default outputs are written under `outputs/bench/` in this directory. + +Bidirectional precision-row outputs: + +- `outputs/bench/bench.json`, `bench.md`, and `perf.md` - precision sweep + summary files. +- `outputs/bench//bench.json` - machine-readable inputs, medians, + p90s, memory, and stage timings for that precision row. +- `outputs/bench//bench.md` - human-readable report. +- `outputs/bench//perf.md` - chart-ready data. + +Streaming precision-row outputs: + +- `outputs/bench/streaming//upstream/run_/stats.json` +- `outputs/bench/streaming//upstream/run_/command.txt` +- `outputs/bench/streaming//flashdreams/run_/stats.json` +- `outputs/bench/streaming//flashdreams/run_/command.txt` +- `outputs/bench/streaming//bench.json` +- `outputs/bench/streaming//bench.md` +- `outputs/bench/streaming//perf.md` + +## Metrics + +The bidirectional headline metric is steady-state in-process generation latency +per generated clip. Warmup generations run on the live model and are excluded +from the headline metric. Model construction, checkpoint loading, video +writing, and frame dumps are outside the timing boundary. With the default +`NO_REFINER=0`, the timed work covers conditioning, Stage-1 DiT, LTX-2 refiner, +and SANA VAE decode. Set `NO_REFINER=1` only for a diagnostic Stage-1 plus SANA +VAE decode benchmark. + +The streaming headline metric is steady-state generation milliseconds per +produced chunk. Warmup runs and the first decoded chunk are excluded; full-clip +wall time remains in `bench.json` and `bench.md` as supporting data. + +`bench.md` also reports stage timings, memory, and frame-normalized diagnostic +breakdowns. Those rows explain the benchmark result; they are not second +headline metrics. + +## Model-Card Data + +Each model-card chart file should have GPU/device as the first column and +implementation as the series columns, for example: + +```markdown +| device | official | flashdreams | +| --- | ---: | ---: | +| GPU | 77826.74 | 76938.72 | +``` + +When copying chart data into the docs tree, use a date-stamped file name such +as `perf-MMDD-bf16.md`. + +Bidirectional upstream comparisons are BF16-only because upstream +`SANA-WM_bidirectional` does not support FP8/FP4 precision flags. The default +bidirectional precision sweep therefore runs BF16 as upstream-vs-FlashDreams, +then records FP8 and FP4 as FlashDreams-only diagnostic rows. Use +`outputs/bench/bf16/perf.md` from the BF16 bidirectional row as the model-card +chart data. + +Run only the BF16 benchmark used by the `SANA-WM_bidirectional` model card with: + +```bash +SANA_WM_VARIANT=bidirectional BENCH_PRECISIONS=bf16 bash bench.sh +``` + +Run only the streaming comparison precision sweep with: + +```bash +SANA_WM_VARIANT=streaming BENCH_SIDE=both bash bench.sh +``` + +Omit `SANA_WM_VARIANT` to regenerate both model-card datasets in one command. +The default run uses the BF16/FP8/FP4 precision sweep for streaming and for +FlashDreams-only bidirectional diagnostics, while the official bidirectional +comparison stays BF16-only: + +```bash +bash bench.sh +``` + +Set `DEVICE_LABEL` to label chart data with a device name that is not just +`GPU`: + +```bash +DEVICE_LABEL="GPU model name" bash bench.sh +``` + +On shared or mixed-GPU hosts, set `CUDA_VISIBLE_DEVICES` to the benchmark GPU's +UUID before running the harness. This keeps PyTorch's NVML and CUDA runtime +device enumeration consistent without changing the benchmark backend policy: + +```bash +CUDA_VISIBLE_DEVICES=GPU-... bash bench.sh +``` + +Set `FORCE_CUDNN_SDPA=1` only for backend-isolation probes. It is not a default +benchmark setting. + +FP8 and FP4 are passed through the precision flags supported by each measured +side. For upstream this means streaming only; for FlashDreams this also includes +bidirectional diagnostic rows. Those precision runs sync the isolated venv with +the `quant` extra before launch. `bench.sh` lets TransformerEngine use its +upstream CUDA-version-aware default architecture set unless `NVTE_CUDA_ARCHS` is +set explicitly. The script records that setting in the isolated venv and +rebuilds TransformerEngine once when it changes, so stale extension wheels from +an earlier architecture setting are not reused. If TransformerEngine, hardware, +or upstream code rejects a precision, earlier precision outputs remain in place +and the failing command is recorded in the corresponding `command.txt`. + +The scripts do not set allocator overrides or GPU wait loops. If GPU contention +matters in your environment, handle it outside the harness. diff --git a/integrations/sana/tests/parity_check/bench.sh b/integrations/sana/tests/performance/bench.sh similarity index 89% rename from integrations/sana/tests/parity_check/bench.sh rename to integrations/sana/tests/performance/bench.sh index 6274e829c..ca1ec292f 100644 --- a/integrations/sana/tests/parity_check/bench.sh +++ b/integrations/sana/tests/performance/bench.sh @@ -21,20 +21,16 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" +SANA_TEST_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +source "${SANA_TEST_DIR}/common.sh" +REPO_ROOT="${SANA_TEST_REPO_ROOT}" _abspath() { - case "$1" in - /*) printf '%s\n' "$1" ;; - *) printf '%s/%s\n' "${PWD}" "$1" ;; - esac + sana_test_abspath "$1" } _is_true() { - case "${1,,}" in - 1|true|yes|on) return 0 ;; - *) return 1 ;; - esac + sana_test_is_true "$1" } _write_command() { @@ -45,34 +41,9 @@ _write_command() { printf "\n" >> "${path}" } -_apply_patch_once() { - local patch_file="$1" - if git apply --reverse --check "${patch_file}" >/dev/null 2>&1; then - echo "[setup] patch already applied, skipping ${patch_file}" - elif git apply --check "${patch_file}" >/dev/null 2>&1; then - echo "[setup] applying ${patch_file}" - git apply "${patch_file}" - else - echo "[setup] refreshing stale patched targets for ${patch_file}" - while IFS=$'\t' read -r _added _deleted path; do - [[ -n "${path}" ]] || continue - git checkout -- "${path}" - done < <(git apply --numstat "${patch_file}") - if git apply --check "${patch_file}" >/dev/null 2>&1; then - echo "[setup] applying ${patch_file}" - git apply "${patch_file}" - else - echo "[setup] ERROR: ${patch_file} neither cleanly applies nor is already applied." >&2 - exit 1 - fi - fi -} - -SANA_REPO="$(_abspath "${SANA_REPO:-${SCRIPT_DIR}/Sana}")" -PATCH_FILE="${SCRIPT_DIR}/changes_bidirectional.patch" -STREAMING_PATCH_FILE="${SCRIPT_DIR}/changes_streaming.patch" -REPO_URL="https://github.com/NVlabs/Sana.git" -PIN_COMMIT="6298508" +SANA_REPO="$(_abspath "${SANA_REPO:-${SANA_TEST_DIR}/Sana}")" +PATCH_FILE="${SANA_TEST_DIR}/changes_bidirectional.patch" +STREAMING_PATCH_FILE="${SANA_TEST_DIR}/changes_streaming.patch" SANA_WM_VARIANT="${SANA_WM_VARIANT:-both}" BENCH_SIDE="${BENCH_SIDE:-both}" @@ -294,7 +265,7 @@ if [[ -n "${BENCH_PRECISIONS}" ]]; then exit 1 fi echo "[bench] aggregating precision sweep -> ${OUTPUT_DIR}/bench.md" - ( cd "${SCRIPT_DIR}" && \ + ( cd "${SANA_TEST_DIR}" && \ uv run python "${SCRIPT_DIR}/bench_sweep_summary.py" \ "${SWEEP_ITEMS[@]}" \ --output-json "${OUTPUT_DIR}/bench.json" \ @@ -306,37 +277,21 @@ if [[ -n "${BENCH_PRECISIONS}" ]]; then exit 0 fi -if [[ ! -d "${SANA_REPO}/.git" ]]; then - echo "[setup] cloning ${REPO_URL} -> ${SANA_REPO}" - git clone "${REPO_URL}" "${SANA_REPO}" -else - echo "[setup] repo already present at ${SANA_REPO}, skipping clone" -fi - -cd "${SANA_REPO}" -CURRENT_COMMIT="$(git rev-parse --short HEAD)" -if [[ "${CURRENT_COMMIT}" != "${PIN_COMMIT}" ]]; then - echo "[setup] checking out pinned commit ${PIN_COMMIT}" - git checkout "${PIN_COMMIT}" -else - echo "[setup] already at pinned commit ${PIN_COMMIT}, skipping checkout" -fi - -_apply_patch_once "${PATCH_FILE}" +sana_test_prepare_sana_repo "${SANA_REPO}" "${PATCH_FILE}" if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then if [[ ! -f "${SANA_REPO}/inference_video_scripts/wm/inference_sana_wm_streaming.py" ]]; then echo "[setup] ERROR: pinned Sana checkout has no SANA-WM streaming entrypoint." >&2 echo " Expected: inference_video_scripts/wm/inference_sana_wm_streaming.py" >&2 exit 1 fi - _apply_patch_once "${STREAMING_PATCH_FILE}" + sana_test_apply_patch_once "${STREAMING_PATCH_FILE}" fi UPSTREAM_COMMIT="$(git rev-parse HEAD)" FLASHDREAMS_COMMIT="$(git -C "${REPO_ROOT}" rev-parse HEAD 2>/dev/null || printf "unknown")" UV_SYNC_ARGS=(uv sync) UV_SYNC_ENV=() -NVTE_ARCH_MARKER="${SCRIPT_DIR}/.venv/.flashdreams_nvte_cuda_archs" +NVTE_ARCH_MARKER="${SANA_TEST_DIR}/.venv/.flashdreams_nvte_cuda_archs" NVTE_MARK_ARCH=0 if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; then UV_SYNC_ARGS+=(--extra quant) @@ -345,8 +300,8 @@ if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; th NVTE_MARK_ARCH=1 NVTE_ARCH_SETTING="${NVTE_CUDA_ARCHS:-default}" NVTE_INSTALLED=0 - if [[ -x "${SCRIPT_DIR}/.venv/bin/python" ]]; then - if ( cd "${SCRIPT_DIR}" && .venv/bin/python - <<'PY' ) + if [[ -x "${SANA_TEST_DIR}/.venv/bin/python" ]]; then + if ( cd "${SANA_TEST_DIR}" && .venv/bin/python - <<'PY' ) import importlib.util raise SystemExit(0 if importlib.util.find_spec("transformer_engine") else 1) @@ -362,19 +317,16 @@ PY fi echo "[setup] TransformerEngine build env: NVTE_FRAMEWORK=${NVTE_FRAMEWORK} NVTE_CUDA_ARCHS=${NVTE_CUDA_ARCHS:-default} NVTE_WITH_NCCL_EP=${NVTE_WITH_NCCL_EP}" echo "[setup] seeding TransformerEngine build tools in isolated venv" - ( cd "${SCRIPT_DIR}" && \ + ( cd "${SANA_TEST_DIR}" && \ uv venv --allow-existing --python "3.12" .venv >/dev/null && \ uv pip install --python .venv/bin/python setuptools wheel pybind11 ) fi echo "[setup] ensuring Python deps via ${UV_SYNC_ARGS[*]} (isolated venv)" -( cd "${SCRIPT_DIR}" && "${UV_SYNC_ENV[@]}" "${UV_SYNC_ARGS[@]}" ) +( cd "${SANA_TEST_DIR}" && "${UV_SYNC_ENV[@]}" "${UV_SYNC_ARGS[@]}" ) if [[ "${NVTE_MARK_ARCH}" == "1" ]]; then printf "%s\n" "${NVTE_ARCH_SETTING}" > "${NVTE_ARCH_MARKER}" fi -UPSTREAM_PYTHONPATH="${SCRIPT_DIR}/compat:${SANA_REPO}" -if [[ -n "${PYTHONPATH:-}" ]]; then - UPSTREAM_PYTHONPATH="${UPSTREAM_PYTHONPATH}:${PYTHONPATH}" -fi +UPSTREAM_PYTHONPATH="$(sana_test_upstream_pythonpath "${SANA_REPO}")" if [[ "${BENCH_SIDE}" != "flashdreams" ]]; then mkdir -p "${UPSTREAM_ROOT}" @@ -516,7 +468,7 @@ _run_upstream_once() { echo "[bench] upstream ${SANA_WM_VARIANT} run ${i}/${TOTAL_RUNS}" _write_command "${upstream_out}/command.txt" "${upstream_cmd[@]}" - ( cd "${SCRIPT_DIR}" && "${upstream_cmd[@]}" ) + ( cd "${SANA_TEST_DIR}" && "${upstream_cmd[@]}" ) } _run_flashdreams_once() { @@ -528,7 +480,7 @@ _run_flashdreams_once() { if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then flashdreams_cmd=( - uv run python "${SCRIPT_DIR}/run_flashdreams_streaming.py" + uv run python "${SANA_TEST_DIR}/run_flashdreams_streaming.py" --image-path "${IMAGE_PATH}" --prompt-path "${PROMPT_PATH}" "${FLASHDREAMS_STREAMING_CAMERA_ARGS[@]}" @@ -555,7 +507,7 @@ _run_flashdreams_once() { ) else flashdreams_cmd=( - uv run python "${SCRIPT_DIR}/run_flashdreams_bidirectional.py" + uv run python "${SANA_TEST_DIR}/run_flashdreams_bidirectional.py" --image-path "${IMAGE_PATH}" --prompt-path "${PROMPT_PATH}" --camera-path "${CAMERA_PATH}" @@ -579,7 +531,7 @@ _run_flashdreams_once() { echo "[bench] FlashDreams ${SANA_WM_VARIANT} run ${i}/${TOTAL_RUNS}" _write_command "${flashdreams_out}/command.txt" "${flashdreams_cmd[@]}" - ( cd "${SCRIPT_DIR}" && "${flashdreams_cmd[@]}" ) + ( cd "${SANA_TEST_DIR}" && "${flashdreams_cmd[@]}" ) } if [[ "${BENCH_SIDE}" != "flashdreams" ]]; then @@ -652,7 +604,7 @@ if [[ "${BENCH_SIDE}" != "upstream" ]]; then fi echo "[bench] summarising -> ${SUMMARY_MD}" -( cd "${SCRIPT_DIR}" && "${SUMMARY_ARGS[@]}" ) +( cd "${SANA_TEST_DIR}" && "${SUMMARY_ARGS[@]}" ) echo "[bench] done." echo " summary: ${SUMMARY_MD}" diff --git a/integrations/sana/tests/parity_check/bench_summary.py b/integrations/sana/tests/performance/bench_summary.py similarity index 100% rename from integrations/sana/tests/parity_check/bench_summary.py rename to integrations/sana/tests/performance/bench_summary.py diff --git a/integrations/sana/tests/parity_check/bench_sweep_summary.py b/integrations/sana/tests/performance/bench_sweep_summary.py similarity index 100% rename from integrations/sana/tests/parity_check/bench_sweep_summary.py rename to integrations/sana/tests/performance/bench_sweep_summary.py diff --git a/integrations/sana/tests/parity_check/pyproject.toml b/integrations/sana/tests/pyproject.toml similarity index 78% rename from integrations/sana/tests/parity_check/pyproject.toml rename to integrations/sana/tests/pyproject.toml index b853466a5..7873bd8a0 100644 --- a/integrations/sana/tests/parity_check/pyproject.toml +++ b/integrations/sana/tests/pyproject.toml @@ -1,7 +1,7 @@ [project] -name = "sana-wm-parity-check" +name = "sana-wm-test-harness" version = "0.0.0" -description = "Isolated venv for the SANA-WM upstream parity and benchmark harness." +description = "Isolated venv for the SANA-WM upstream correctness and performance harness." requires-python = ">=3.12,<3.13" dependencies = [ "flashdreams", @@ -36,9 +36,9 @@ dependencies = [ [project.optional-dependencies] # Transformer Engine is only needed for the FP8/FP4 quant backends. It is gated -# behind an opt-in extra so the default `uv sync` (BF16 benchmarking) does not -# try to build/import it. The git source avoids split-wheel ABI mismatches on -# bleeding-edge torch/CUDA stacks while keeping BF16 setup lightweight. +# behind an opt-in extra so the default BF16 checks do not try to build/import +# it. The git source avoids split-wheel ABI mismatches on bleeding-edge +# torch/CUDA stacks while keeping BF16 setup lightweight. quant = [ "ml-dtypes>=0.5", "nvdlfw-inspect>=0.2", @@ -50,8 +50,8 @@ quant = [ ] [tool.uv.sources] -flashdreams = { path = "../../../../flashdreams", editable = true } -flashdreams-sana-wm = { path = "../..", editable = true } +flashdreams = { path = "../../../flashdreams", editable = true } +flashdreams-sana-wm = { path = "..", editable = true } transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", tag = "v2.17" } [tool.uv] diff --git a/integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py b/integrations/sana/tests/run_flashdreams_bidirectional.py similarity index 100% rename from integrations/sana/tests/parity_check/run_flashdreams_bidirectional.py rename to integrations/sana/tests/run_flashdreams_bidirectional.py diff --git a/integrations/sana/tests/parity_check/run_flashdreams_common.py b/integrations/sana/tests/run_flashdreams_common.py similarity index 98% rename from integrations/sana/tests/parity_check/run_flashdreams_common.py rename to integrations/sana/tests/run_flashdreams_common.py index 41608ca07..9fd43c10c 100644 --- a/integrations/sana/tests/parity_check/run_flashdreams_common.py +++ b/integrations/sana/tests/run_flashdreams_common.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared helpers for FlashDreams SANA-WM benchmark runners.""" +"""Shared helpers for FlashDreams SANA-WM test runners.""" from __future__ import annotations @@ -44,7 +44,7 @@ def apply_backend_defaults() -> None: - """Apply the stack-matched CUDA defaults used by the benchmark harness.""" + """Apply the stack-matched CUDA defaults used by the test harness.""" torch.backends.cudnn.benchmark = True torch.set_float32_matmul_precision("high") if torch.cuda.is_available(): diff --git a/integrations/sana/tests/parity_check/run_flashdreams_streaming.py b/integrations/sana/tests/run_flashdreams_streaming.py similarity index 100% rename from integrations/sana/tests/parity_check/run_flashdreams_streaming.py rename to integrations/sana/tests/run_flashdreams_streaming.py diff --git a/integrations/sana/tests/test_parity_benchmark_summary.py b/integrations/sana/tests/test_performance_benchmark_summary.py similarity index 98% rename from integrations/sana/tests/test_parity_benchmark_summary.py rename to integrations/sana/tests/test_performance_benchmark_summary.py index 5aaa17285..cf07e0f06 100644 --- a/integrations/sana/tests/test_parity_benchmark_summary.py +++ b/integrations/sana/tests/test_performance_benchmark_summary.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""CPU-safe tests for the SANA-WM parity benchmark summary.""" +"""CPU-safe tests for the SANA-WM performance benchmark summary.""" from __future__ import annotations @@ -30,7 +30,7 @@ def _load_bench_summary() -> ModuleType: - path = Path("integrations/sana/tests/parity_check/bench_summary.py") + path = Path("integrations/sana/tests/performance/bench_summary.py") spec = importlib.util.spec_from_file_location("sana_wm_bench_summary", path) assert spec is not None assert spec.loader is not None @@ -40,7 +40,7 @@ def _load_bench_summary() -> ModuleType: def _load_bench_sweep_summary() -> ModuleType: - path = Path("integrations/sana/tests/parity_check/bench_sweep_summary.py") + path = Path("integrations/sana/tests/performance/bench_sweep_summary.py") spec = importlib.util.spec_from_file_location("sana_wm_bench_sweep_summary", path) assert spec is not None assert spec.loader is not None @@ -481,7 +481,7 @@ def test_bidirectional_bench_script_rejects_low_precision_dry_run() -> None: } ) result = subprocess.run( - ["bash", "integrations/sana/tests/parity_check/bench.sh"], + ["bash", "integrations/sana/tests/performance/bench.sh"], check=False, cwd=Path.cwd(), env=env, @@ -590,7 +590,7 @@ def test_streaming_upstream_summary_uses_steady_state_ms_per_chunk(tmp_path: Pat "--refiner-precision", "bf16", "--device-label", - "GB202", + "Test GPU", "--upstream-commit", "6298508", "--output-mode", @@ -715,7 +715,7 @@ def test_streaming_comparison_summary_writes_chart_data(tmp_path: Path) -> None: "--refiner-precision", "bf16", "--device-label", - "GB202", + "Test GPU", "--output-mode", "mp4", "--denoising-step-list", @@ -740,7 +740,7 @@ def test_streaming_comparison_summary_writes_chart_data(tmp_path: Path) -> None: "\n" "| device | official | flashdreams |\n" "| --- | ---: | ---: |\n" - "| GB202 | 2000.00 | 1000.00 |\n" + "| Test GPU | 2000.00 | 1000.00 |\n" ) summary = json.loads(out_json.read_text(encoding="utf-8")) assert summary["bench_side"] == "both" diff --git a/integrations/sana/tests/parity_check/uv.lock b/integrations/sana/tests/uv.lock similarity index 99% rename from integrations/sana/tests/parity_check/uv.lock rename to integrations/sana/tests/uv.lock index 4fc781db9..9b36f6e9b 100644 --- a/integrations/sana/tests/parity_check/uv.lock +++ b/integrations/sana/tests/uv.lock @@ -309,7 +309,7 @@ wheels = [ [[package]] name = "flashdreams" -source = { editable = "../../../../flashdreams" } +source = { editable = "../../../flashdreams" } dependencies = [ { name = "boto3" }, { name = "botocore" }, @@ -383,7 +383,7 @@ cuda13 = [ [[package]] name = "flashdreams-sana-wm" version = "0.1.0" -source = { editable = "../../" } +source = { editable = "../" } dependencies = [ { name = "accelerate" }, { name = "diffusers" }, @@ -401,7 +401,7 @@ dependencies = [ requires-dist = [ { name = "accelerate", specifier = ">=1.0" }, { name = "diffusers", specifier = ">=0.36" }, - { name = "flashdreams", editable = "../../../../flashdreams" }, + { name = "flashdreams", editable = "../../../flashdreams" }, { name = "imageio", extras = ["ffmpeg"], specifier = ">=2.31" }, { name = "pillow", specifier = ">=10" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, @@ -1232,7 +1232,7 @@ wheels = [ ] [[package]] -name = "sana-wm-parity-check" +name = "sana-wm-test-harness" version = "0.0.0" source = { virtual = "." } dependencies = [ @@ -1284,8 +1284,8 @@ requires-dist = [ { name = "diffusers", specifier = ">=0.37" }, { name = "einops", specifier = ">=0.7" }, { name = "flash-linear-attention", specifier = ">=0.4.2" }, - { name = "flashdreams", editable = "../../../../flashdreams" }, - { name = "flashdreams-sana-wm", editable = "../../" }, + { name = "flashdreams", editable = "../../../flashdreams" }, + { name = "flashdreams-sana-wm", editable = "../" }, { name = "ftfy", specifier = ">=6.0" }, { name = "huggingface-hub", specifier = ">=0.36" }, { name = "imageio", extras = ["ffmpeg"], specifier = ">=2.31" }, From 56a35c6319559bd72ff1761a36fd9c5233d8172e Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 13:53:40 -0700 Subject: [PATCH 60/64] Add AVIF to model card --- docs/source/_static/custom.css | 5 +++++ .../sana_wm/sana-wm-bidirectional.avif | Bin 0 -> 366041 bytes .../model_clips/sana_wm/sana-wm-streaming.avif | Bin 0 -> 447942 bytes docs/source/models/index.rst | 8 ++++++++ docs/source/models/sana_wm_bidirectional.rst | 6 ++++++ docs/source/models/sana_wm_streaming.rst | 6 ++++++ 6 files changed, 25 insertions(+) create mode 100644 docs/source/_static/model_clips/sana_wm/sana-wm-bidirectional.avif create mode 100644 docs/source/_static/model_clips/sana_wm/sana-wm-streaming.avif diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index 1f90c92ce..9d2491270 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -977,6 +977,11 @@ div.admonition.note > .admonition-title::after { background: #101214; } +.model-hero-media { + width: 100%; + margin: 10px auto 14px; +} + .model-video-placeholder { display: flex; align-items: center; diff --git a/docs/source/_static/model_clips/sana_wm/sana-wm-bidirectional.avif b/docs/source/_static/model_clips/sana_wm/sana-wm-bidirectional.avif new file mode 100644 index 0000000000000000000000000000000000000000..796ffc9825ef98d879b8ccb8ea417339e4152d1e GIT binary patch literal 366041 zcmb69W0Yh~)GZ2^%`V$Ex@=or?6Qq6ciFaW+qP}nwq3XS>3+|5zH`r?8#(r#bFQ@_ za>a_+F>+)k0002Bk^OHQT}M+pfUoukzb+$7J0k{DJ8L#eQzHgTQ(Yrzeg=Ur_oJnu zz3!j5uSR2HU}5{00sz>U>f8Sh|AWe=28PyRx>g1jhX2LD*_hf}{ucoFb;mWeu-5;h zga7~lzg~at0ANx806c$^AU8F&GWz5F&x83T41xYBT}OHbL2C=^|Hd>kwXwDNa!8um z=v#jwvZ3Am|MW^T&jwZ*5^~rElR3 z0QBGM%Y&+qydDJuUWaNb{p`WFx4^jBk){zZ_rzX%uj7r_+1QYHfckXHNWFZ_#;|KdTF z|LSiu|NM&o`~m;;@4rY`{ZId?{#Tw*|Lh=he>K|nKmY4rgwg${WB;Q1zj=aW|J9KH z@&^w3tKt7?pzFUH9QrQ;3H?Pp_J4ZdFGBpw6BzTaMvnZ8z~z4tn(Z%wMEpe^+^^d1 z{^ejgUmW5uhoku7ga7Pb2=Xsqf{m}^&HtZ$`YYzo`Smq#gs(b+O8>JjeZ`Oc<>u~R z_CF8*R}5J4mwo;L1sU--3$|2uBK=%4-jb!>x1##aF-l+cM%r4iwK5bu`JS2EY`TQtEZLUhh za0>r0r(t4s`kB;5K2s6?cdA>MkPZizAvOx_hXKY-d$CqkVltUiA%SKS1@rlSFcP%< zwCNj^>skPjEY{8AWB+g)S$x~qrC(j?=fP+}B66_u#r>J6{VZG&Q=^LA^Owph)Q!=6 zBeu)345z%SPaC?`k(?Ie%SRv@zdU1OA`BQU3Ob@ESoECpne4ykRXK}F=qW03?*v-h zilpWKq?7#|00Py`l&`_kqB&-5VPzgt5Iu`7_O@%#GE^#A!QEJ8IYU@1lUlRxg(`&A zepDtlqFDwNWxa2p=E9OVZaH9;EIn=&cE(-5mHl8(RtAky%$MtX@JK{)({=q_lF99G zTTfM?Hty~Iz1fhPO2UCT=5yI%N7|>hO{gsO2g4iEGHVN8v@;n2)6Bi_fDjF6`>*B? zWQfZ|6Y6-(j?Hq9gHX3Paqvdx&WM`_CY4PW%3ZXMo4XSG4t~+4pTGbUt9F`bVm}wh zTII+X7AcSqSHb$wp1L8IJ2D|m&HCzYmz+wVdJr1)W^G%sve?j?c`crk1?cJNK3s4_ z%e{{cv-r)oi9FwG3b2meq{v`AHqZbi#y#7+b#K$SLf;#j$V^3EaxQJVs6V7#;I_q) z+;7zGa+Vv@(%D?}W6nhN;F)&qkbBbVJDNAIt5+!~Ziy!LrZ&ws-!!6w)Dy+MvQ%6O zI9KA}>Z;!YNt1fJQUC_ofphS1W|p%lH3GYLx2(|U8^EzOrkH$yX=#ewiQlx$?D47F zMlH~M(cV(w1xOYR?kJOstG?kP z=zntd>0dp%)6p@ESG<(RQyNM3hyklfg^=H3MMg5r!@DYPD=pA9nyqtxcXA!Ty z+Qlw?TM`D`(Jk+f)T|h)EIxYLzebd9w5jaKqvhYK*?n0$c!f294EsZDzBAf3yj_%< zAB?p7h_6o;Rs}gs!oZ|vS64nPRP9LUr0iD=WK*F%-kuRR+=81O4&$ure6Xg+rV+E{ z8Pn)NRK0nMVM6ErDp}}S_w~67_(vU_8#L4ArH*sn_W(Yj(6KdE#p$;#rj{hcl*s!d32L|KLh zP|*ONdaTv5yOW`9sKv0-ccG4BU})So6c}UCP>(fyE)Cv(43o7a27<&`gQ@m8;^hgR zlmcvlh2b>Pvgouu6-F_J>hp@@X=nf;S<{4xP!U~UbjMPJAn0AW`4SM&Ql9TN!qENr zcgO3z#G!acnIjDi_*$AjXQ*#iBv`)r2m4L|VCZ7>YbBJz{jlP0qH;#T>uD12E>tBcKmMHxv$U$V$t2{1VO;@Pd_YCFgSXjzSQ4{89^7hYE;Ut z-U%F_+u#S)Z{^!S$>pqNZ0nK~;kR|@Eu!tKqp5fA#qIE0fe=qGj32RGD2M9Kwb}5X zpYWuNHLRcg)aBrkiLQ$VBY3Yt1WoF%K#BoSyG1y9xIdN`Ix{rTeeR%j^qPFjX4-{dTiK=N%}3KmX|LZ${#7Q!Nrk_kUGrywqY83xngNZuzxY)>i-erM8l9=p`IT8jOCBjo`l zJioyE#2g)k-V4#eTCYKJYc1NI!&98;q*L{^Z@bmYd3CcCFsue|SE{p>N>licW8Y*cB(8Y$v1278~*OkW2rS^ZGcnxg;*=F~E^?6@}bc zbiHkYm>Gg`{)-S!hUw|f&`~R-KfJ#jtR0FJOw-|K$}M^f*%`p?045W; zLQ;XKt7zHq$6BEE7Ot6`CGfhCu|y$#3)5H4q#fhKG}mHeA1y3F32LQ=#c#vo1BipR z1H&9TZeE1Wq;0gX+Yd?A-~b_ZSO#o0@`4FK&Ng+(?IuG=;OkMQqM5BPML6dR2Cr{V zeVvjTXnEs^Fp*7z`<`y%wFD9Mdyv0@>xnh)aQ1!ih(lTQ4RWZ6jQSxUxLZHT~nswPH1;I$J5j}H*mHdeH62DgjHG?MHTqtc?${60SN@$h~_@FpUp@a82lU)cO;k zKw5)JUVk!*E}dVBsKDRS8zvO1ct#7~_5tvq4ep~N!w0ojubo^&v(W?j2{_UnP_yDL z98W*>ed?D7+h2N*N<-a$m&vJu;6!PNc}5W@WzB*bwpHvkOq+gK!w(z?nO5!;7qqzy znR*0mJLlIOv;M67oV+Dy=EtMnsqGliwO_%9fuZfjtsXR!u|TW}u#a-CBm|uAI=Urm z^fPm19-dwnN)PtrdNot387#P^y4esP&igUqK(x73TFGXI7g=;7qp6eSLF!Von43C( zUX(+lo7R&@%#)YIE7a;9K=~HiQg~ z=#Wc>4eeU<@Ivz!Qm*ZV>v zN?TF)15g<&(L$ZQe2I#Yr~)e~#m&^>33@r;C}I=y>x9zuK66XapxNbM#We2`&sow0 z?nz@MLCVOO>z`nm7X{Z2*8K*bRT zOPga@F)DwCJ6BQ^;k$NP_Z%vHo+%#x>jJi#{4MjM2ZZJMHz($h!y;l7kT0yCNU1pl zD)U|^z#KODR?KpQ73z)Gpopje%CYe)h8;P?&w|qo;xk7|x>(Vb=sIuw@%s(WgW!!= z4eiOKiO)7tbwrSsAShiy^7h=Df&|6%MdHxh$C^FAae2{WJt+>M);k)=$Q(7f>e|eLdAw8+ z>Gw5t+~Q2d5RniQ?_^%|YiAMSZyIbO=#MIKwTs3tEHy$mkagcj0jvicSgkr1#=iN7t^aejTbA+A1Ir0UAhdL1$jj}!Qk;(+TG7=Q97Jbbp3}Z~|nvERb$6S0~hTOiIxQq|C9?gVS z+9!ba2-!rw;Df8!eLsguw|SNH13=l09T#OQ!tsMR;kEarYnLk<73^tVTWQ!3fXv;er>B zoL%IMQJAOWH&j8aw`32{id>)8<1<7Ni-g>x6_c_EB+N(p!3jXY{Fa*s2AoN9`U7s_-qR8!IwX+}Zw>Wv?c}#ad!aWzLn7-C@E*N(E->E8MVem+Vct2$#7n8m`9un@R9r^uojSWb`;-1>g|K7)L@_gYp#V(&&n2s3~F}j_!ge){mob0 zzktoV0~5D?8O$bX;CN$~&IK+AC+JFPUd2EybIGvkx_r~&ciMFhEpNBD`#E+hDI*;) z6wAbG>`8g{%cP@3J?lkW+nP5g5CLgceJX|BbW3L%@GTrtf50`R zWRyWDyy(3&4O23qlFmWQ*aSk%gOS3QT)J_>n3=Tei1Gso3q*v?c-THYir?T8pHr#2 zWHLeE09xszwF(~uqa3l4YKWIH))q`W-kA$=taPriWCUaV%s6igA}ry^nIHIFA#d#^ zkXMuSAoH4)G_HD20k=dCAhasX%z$+~YB~$upO9NaG=Tw$A&O9*;|)fn36t?MndDlI z4QFCvK1f$Bq#E>k**U2ivO&1AB{G-{19TzdSZt)LUNNTc3O7`$BV#au(na9It9n2s zK)C*87zwR?W{cXXuGo@0PqjCZ@%@?pyL7}G6po|JRiZZ@U2oC%%3`1d= zCyZ`LcHyv)K3D78bob0MGnqwoMCMfO6u9SG_n_bYZru^niXBE7^(9d@*uG#cgc5MB z^G)VVE`G(~fd*M2=p9@!EBIwXkym_@@p1jaQ80v0U9Y>?Bx?u{`6V0k2+wUG@3QK! zL6^gKrLix$6vLdx&VhA7WZC|UTA0V=zaYs)Q-0uG9Z1Tdv31_SOLrnw9I{MTW^%(F z&v=^JjtC-HhCr9TvN{U4WvOfcvhD{4MDzt{*E!@-hGR+46nIS5g@aw>0tF!_<_+#w z1{{QCXiAnSRlh9J{Y0DoZe@2zQNO$d*V@Md3Y2?EZ`~`SWPURa6wp$mre~S-9pJm| z2TY;|s!n8+a31xVj784NNB750xRxT5Fd{LUIQX1jI7Xjp%q?I5Wo~gR zR30Aho72FpmhX^fVpKi0PE8B>2RzEC4(goOCo)W zA3FX`zCa~(SEzQjvx@usHqju-V@9V;M%b^3%?th*UyQyCodeyI2?PJN}(sx_w+2`x`I@%of!veTe zp&((u6v*kS(WGn+EKWr$4hp*L)8ODI=p0ktWQKakiKx#-ll6aQveW=J26~)ZbS&kj z9IPNHsnAD1#$-fbV$g3+ko4DS1576dQL%u%ZYW&9ltY}@Vo0sVq&f^wRw^tUpFDfE z18(>uf%JxpZyd0JP>!FzC)!=ZUi_i%`Hz8T)oU6t|bzCdtogeADYaf^$rh?hA4 z10yde50)gc*Fd$#+V4T(T2<{Ot_pdgVFN}yt@|9;kRx{}(p^NzRrnvowDZ{vnk}eO zwcTa;a=)E*$EgFc?MYKaR^v84{N!Q9aXo4X8LOyyy(^MRK}KWxEEj|5h!QNyA%kg? zB7kM%JLK>vGfLuOxTL}Ptzd1K=r%ro@)9C~Xev~|D17JJ=U`@?(X@Q&gHY?L3BR`A zIhrCBVN{S-J&T!FXr2nC)^BCa+pcU+B6>JirHsPyR?sm>B`rB_!~C+5F}QYl(LoUF zJgU)CxEBzpO78RzNvu`+6DbO?9OsEx|!51H<;rSj0w0UIVs7+Ii zWT2|U2C`#>l)w`I`$FKdIpI*72+-{nAHO6q>~ihWG#QjWVPp2%z1EMqi^il${*vp( z56O5-T7cFh@3;DPq=bO2cKTBFD2-TG=o8^eDd2>(ghEyrNAU6mWKsmHJ1E(ALk1-H zD`0S=!9^SMEL~_ihRWj_1g=+X&_|p+g0A0f(7(iH!}QOJQI@hw=>r!yfq%%6s}r(R zxXLu#o!hC7{~$OoN{?a~>1*jSSGLis#kc_DWlu9rXk@z;Z^CP|Hk4KJ5?xBsIfqvC z`gKbTh&#a12w+PDVo{dM-jXBdyTkbPzk5Ex& zl6m~Ff)sfo%(&+bYIaD67?TxA4TTx9&M!i$q1)!K^~-g8;&@>w!}NP|h%JZn_e$)Z zO({5lr~oF+ri$1oKjdRLs-Fc#S9xkLljUldDD%iqEW&g#M#AbJ7c z`p4B|F$&idE;#nj)$PS6AY-lu@+6tl;rhe&;#ZQVft$cTfY*}4uy!?>I zVcKcIqcjVkeW2bSKkvMD$w$FCTLHSR6`x54u1vmWpV`s~7QN7Ue^NH4YB?w1(muUwx4$ss z15b)p*1j6%gWhzPZm2}#Y&pqKD+frQLy_%z&}0V<%7yX?4lZ_KwF#r0p46Nh%1TD2 z3x6Sc#Be2C2}nd5tS}i~tK44pj7l$@9`SM+L9SOcypC-(_TW)Ux={hm#7mKBYvB+D zE`jo$?I-*Tal2h;-bQnbM3Xed3nM{T7M|dm)bH^`*Nf9B%k>OTZg=nT<1+H zKR6US>)8rigdj#)hUxP zv8_8KI)NO^IS=`DT&#RDN*S^g5_SaKE;-;4XZkIrjMHjY1}G)GR1qR^msTZjS!c<_ zVw_A*9rGG=tT#-D1xtDnZVcPel9m&$e&wH86m zufFe}1o;FU+>iuEClxM(P#HfT%2!wu6E}&CnNA-zt7zp+zrM87lK1bh%v(L_41&05 zK)mCBG51it3)wuJ_>?PuAHjX~Osymz&RCxwAS5cHzO?Jc3zsyGd9Ay!^+EfQr}m^M zUkOX%gqu$0%AuqQJ5O|XW1F#;bA=i_}iY8*GmT#SrpF5PSBu?!DTN~YPoHxy~n8c9E(*xu>f zb;N87W^UO_s!9Oj$>sIC;xiH(-7V>E^07sy;h71SoQ9Vqa-$N;S$)G)%<$@EN&FvR zXD=w|CTZbNE zOM5?1So{|&Jy~TXfS%U&>4`#i?$1_)SgWB-djbM6FNF$BylV(Xxka#-xpaisNM_2H zRwx%9(%o?Q!!sVzq|Z^577f3rfSOG1N;{+!UxK;?D@3x|XT0uk=|$q@&%DL-N{}DP zkp?QqTOeM6CfeGyBqFDwxVcgyc0~Nj-+$Xj5Taujudm(NR2%{p+@QM=9+nkTZQhDY z9)ZInmtSzw(m|dAeC0FU`YcH`B7W~VU?>}SKQwVc_eku)CE6IJuTI*y3HqKElt%-V zl}HqFwuM@U;vG-Zy!T7>3D4(%S~wrQNw4RE&Esd~Z6nXeZ$EI;N2fyERiyKMozm&LyhlCT3byV(t%u%p9cmR`!mBUQwxtIFTtV z_Mv!Uv2c}-xKhEz?`EW_dL)KKgO`sgznjNo^H2M3=TUa`o2V?tgL6CWZzxZovinJAY%|T916cbn{LV< z#l7C958gja!b%n?gw(euO0GGfnH6roDyJ9;~(S&InS<5KT0378pN1HQRy&q$c5KLsFqZkrZXx#R-?l$uIGsKE++WSsSCR zf(zmBxIA>b#EEoA#1tJR2Z4qg9gUGt%ZM13C$R6GwCBs(vSuJxbD+JXr!7|}C>q}y z!$9^?dz1s7PUamzP~>^JHr`w*Dxi+7u-_sE^!V;fUl2GX#o-6LkDR%|*h)*+ndRQg zB9fGA2yNH4t&0bKyoGGp2^o)8H598su*u?^VVfyMh3?Wa3q#{IK$u!N8vNRTfrxT& zf2{nykgx}Gl${%B^56~LY@t%gbw>qJyxWozr`>q$f$`oIEPbsSVk%IFzky(%ao@jO zn-!VpiAsvD4q5Pm&&eq{xVw8C-sd=;$F27M2yZhQ9No;etZCHE;&Q;kB#44NNnILwGuma~P zTVwwTk^6I<=Eu~MH2YO?ht{NZy$J}tEM9vq>M&-(xBH#)mXC)qlooyN3lGVtrHY5stR|T)v2#H@7~ViZ zky7Ktm2M@Hh;wODVt|Uxp``4OGnkliL*gPNg%IM&<|b|h&N$jSV_kR4_u%A_&b%it z@rT+huRo?_!!SZae&;+`Xju&Fkt4KLK;uG%)&)`|i;``nNJNw9xyk*712>ZqWg!by^pB#H0+g>unPQ7yc2m|}54#^3pTsidw1=V*m&~uAU z7D#l1F3Vq-tAwE`JQ?vLm9?n9#(I2C%m1CzRNb1VT6tz5f*D)y-G#sDkN7Q5+jhnF zn5XQ{@3J_($Q`C{!iPynxERX(q@hkdFF_@fsHp5Xkkcc4Aa1;9{U=R2FlXMfY3p}P z0x5_>XYkLURX9f?+&VQL@b$1<7SP#=q5JqA;D)#Bnz4&vs^P`DwMu7!F7>*ofL`L` zeD`X5m$A$yIvP@ctxfRk!f8OmTRtHVhu#VWt#~dI&TfAoV3lO{3Hysy7*?wOT&bC> z`M%FBn-9HJB>VOybV4{afk~l|K*D#i)*@lfzghD165S3NET#FLg7OJdyPy z%|eyC9ztLpdPG__*mvVO%+TPl2y>2%DH(Z>F%aXPXbwn+mMkPOnYdmMZH$J)t5|?EdUb(pWYF2fNa5ncIhK?$<2eZQVu) zVyQhTFY!6tO;8Nnk|pyYqURnCSFx}H;DW(#R9L@yH&(+c70&{|qQl$Au3VqQMd_)g zVBA5-#OI_!SI$5i;%qJ}X50ISNG7Rgv7)Nmu22?L{Ux@|TXv)og&|`M9pBZQ4(kgH zBn8D^_uLet_IX^DEZ=+$Udsx(4W7{43itqrc`DY^KsUB3vNkTNAu^bUBY}~;3A=(&jTO1E zqsYS3G=<~{D7)5g<$AQASk0EJIy`cgM;?UZ~!XHUWp& zTSrSjs5!|gn)nIZKwu5fs^_I|^p2<{d$qq)kXeD0|991d#X*4!aNU^NNfiyyFP&@Q zsAVUp)!hUX_lq;VmmhTC?GWq_wm{0eu)mCFZ9 zqzcF^J{p%)Q0GOp^Kevvx`{O^zPap$c@@y=3xl;q?`cEv1m5HZx8~{v+~m_6$=@O2 zj}EWiw!Nl0)s=$YeBXD^>XIUkm6M-T((C;Gj=dFi#Kd{}y|&!+=XU+`D)L2HoI#w5 zme9hL-)&Ndft(90KBNm(Q(g#2y@p7FrMd7O;KCVNg^>y-4 zHq-6XkLWp}akg5dbS?@T7oj+W^5-*{!XOgYZ#S)_^>_CT8lGy}J#RRKB~h#8B9v9u zIzM<6Iykb(9x#cJ;j|Vuh07f(mR!!c7bw(zFP)f2ot0L2i_e77UAXImhC1{jp*Q-< z)KW14Pwb8UnzZ|LJyU)z>}ou8FaaK-itT_a;|ME=8}`e;$1pV|1A(frtNv#Fw0uf& zK%n_3Z3EmjSq=uTa913RTX2-}Kmn*{s-pq{5%SENhq5!(h#RoAt*zJ`o>wEN%chBf z8yh(9<>X;{Xo*6FOn%AlsAU+IlS#0lB{B8rs85>EKME|cQ%E*DdK~WMX%ALc_Ir+K zlP${(47fo#u%XA&vdRT@m9aQe1Eqm*0Pdatw& zPf@k*wmw87Jm?`U^LzE}&@OX?Bxk=j8|rH0q{VR!C}vV<2nP*VlA$1rJ!K8vrlwi8 z#0rX_U}qVEgQg`C40W=N|=szR%RGwprA6zPfBE_867gexrAr(5`)!Z zZiYE5O`0c^G-)Wg1ba+sZC#efb`X${iQ1otMU*#;!M~>5f$H@jHK6V4I(&dK1zbS& zkH2*%C)`K>s2XHRwd*b+%m2Mmhz>;G_Bp-^Q2k4h*#rULIk%rzFM|@4W8&iwad`k4 zgK!(0QYE1Hlr^`6)5u&}pcC+79k`}8q^YeMz(3_g z;YyWxhZ(h_5)y0Z0QwD;J;LRt`7pHE&htU_(l%vMx}khRkXU-|Q*R{kELn%9PCK_B zMk;_dWF^xIxQnz8Z4k8_89nD=PBErHZS!UN@t_+^ z*THg`_xHer|9%5OnqLzOcMFx-d#aAr(F z2%%oB>U)F~7sJr~QQFP>xc3nU&=W$V%vHbBJLVLk?>L=n0R?{98VZtv=mUCJJ2F)Q zVZ5Xj0#5>Ex)nHq136B0I^glHt!l{u=!yDG{^1J5l>gSAKol}xQjkTypK5*>y7Y|BR%m!ISWa&H zJ=@cEDi_=Z{pxc#RU7o|kRHjRhgN6>PY|Q$<-j%fK=rpG?Q#{-nMRlt*+Y03}99}kbP1%0*2*JC|A zxXs*sh@7#t0?5|Z!v;_0n1=64sOe#VLar(z$Etx2K>F({h7AeL1`R7?D7XZ{x-1PImE5|fdn#L)Xm5i@&W;=^G|8d@MkF=X1(ZG>EVm~7%0MD!ed7n7<5aQ z0f_*9-abZT`(ap{n%Dr`VmUqnL{HO0eiKIi&%bz#%Ql^AFR59YA6;Nr_7yEY=yfKycAQm?(=miNoTR^<{_dCzg$-(;^-Bb6qQTN* zK9MgyP<#6NSJ+)~^>sw?shF&>8BN>aVe$K+1u-ta{~P&6xsQzSR^C&xmIVK^4rA+W zfHl+9a^;lScITZOldevSc``mC90mg`$(hVm7E*bPfBcd(;FQ*G-M7<$bEYmNS~PzC z(-pGlL`Gt*8yG$8K?FuZ!?cgjGMre)OMB;G&Vb9BQMxljt1RN5;DpUlA`si568;~- z(B<_@6&5EcXq)sOPvI;km$I)bx$hmzE)6?xtU;IgX9$v^A<}jRcu_eNJ11N{kq9&c zA`6}lG_RR@=Q3aar>`kJWzdSa2tfV@el4nZJdDKlMpe4_q%`L$lca1nW)s>rOm`sv z5-6*aDTYNbQ3n4H;K49~zKNnUmBfopjIpUqS{N@+L+ZRW))=;Wg5 z^m=W>TR`@vQcd@EF&7fJAqBIO0Ug@ob;0PLkXxP^sH`eu#RSTiPraWsSN4NuMwio7 zPLdA|V$Z;WgfMPPOw}AV#zq;D-P3%Pns|6m6T7kKiTkO%6S+4A8cNd7mw}$eNV^;en=y~MR(EKtqchdfCSR?}DbS(Ms zC1gwPgyluo1=xm)z7@A zOwus2OTx=;^^3M`2r6UcLY5`6K0%50^r7cab%OEInliV1RP znqy|Szzd^Js86UzLMxuY632kTkA>!EcTNBrY9AtgaU;KXu7a~6ldPDp*I|GHPV-^I zxw9FneW`d9uTN9MCq;e{jT&T zqC_v-J6csl8&uA3YCeeGG0Af%Y0`Z|zQr>C+jh^nN} zf^!?`t>h~!*Zuq!wRZTBh$^XMb*lDblh)tH+%N30;N6}-jHy)oNohgS^BDI(c4fLDBk!~saE z2~kXXgE!mR*~|bIGoJ+yPnL0z(Z)m2G%8YhWx4q*+JZ|s9Irg zLMRQ#Q~wNQFxyt>C*C*~Krm_-heyI1HxBHE; z6J{7ziKvNxhkm`To>n!v2n137SMA491~E{RTn8?w)ST-~K(Ng0put+dK2zfIe8Wqd z6r#!Wvc$OhFgL~(Ah^@3z8-W>Z_oE5%e|XHxmN1- zwB}yNc4qzag=?LJ)OfjklX&m&@U(BCse&hQ&_c@o(J@7u%}>VnL1`Qvt>Yxm(+%dn zIp*Xwo``_yA&Jmd;(+zVB6x@M%Jafd3wj*X2-A-MRNGbyDHMyLmFR|}J$VTEb)KWN zct^lDa)xrNa;6+{VO<2ZP88d>W6&6Hh#oWGWqZA~+EW=hxC#%~v@739QkQWM0!rfH z#PnYl5Om%B!Rw}oa4v|TLQ!&o?c#38zFT5Gz!`|aiXtP z`WXk0bza2EUpabLcDyWjKF1+EA)C!-7xjQ{bmcBU1G19?QUENnKvK&`zp5onuq+zW zdUj>?+~5_GcRrLFj{-4)`0?V{S5P}4L}`&SKK#%!2dC@lQ^If5U--u3=0<6I6o7#~ zzE?tbsi{o`uI!^qTgX3~(_T|F-ze>O-+pi0ZE&Iqw{bNxMQ3RfcIFyzfGqmhP!Sx( zpKSQM`p)TE{)ugfa!I=!8vR{e_BLJJbQD^h#kdVZUiefLHj5x4KcyCT5_kphfp-G+ zk`ej2Hw~xcu%&o_xn)pf*uM~_%IeX4{zcAw?f0-4QO@k4GFT!+?>rkA7D5n*t!&}| zQ!O^Q4TlVRcmh?ZSgb0vfg2RnLP_tQncwbN&AL?+<+_LQgS)0 z#NEs}XRV@OupFH;rP_1kX94TG;n|O4*_y*@I~g{fP8L#p@;2d(Yl{)`ahwU125!LU zrh?R?^xjGDeFGqgGNK5!_`%A0eT%b_xyfwvw~G?THebtwSY0ANo!>9WmtwA9XKMw6 zT&!rspD?q~b^+l(b0J8_flhrvt-ff;1lCcGvyC3-u=M&I<{95yZh(-WoKZA(7|3I7&uh9qH z&-A2_4MUC~dv<$$+W@Q~mjA%b&vgb3??=~eS z;c%)>MAZ)S@R<0Nu=h-A>fUL&h2KuMpra5Xewc1VlGD(r*LFQ<6g(c~{WtRlA6UpN zgp<2SZ~O0F-qOU`R_#09djp;V(y6asF9!3lJurHWQ{vO!SQXP6$#v$vU9b5MzVXUv z!uDETd6vTWqBOoH^TUZyz;u)4fFnNTL2uOZh>l-H;x5**P3WtT3F2lp2k0Q^>nu52lkrGsi24_#<13&EGoAfhyEa2O~&3sYpy*V%k43IEc3fAir_vx*r2A9MDD0Pfojn3 zjrZC=B40C7x<*_9(!%iY^Atf+s@q5>XB` z+Hlv#ZT_hPDGm~;oPB3&Lq_od*OQ;l~Xr-W^`rq3Z~rFAIS!{a`BL-i4a-#jHsy{PYzyRs9U%qcB}htwdi%z>w-VrSX~lx`GLm#ghT#b50;&9Y8rrS8Kj)W8)S*Oah} z_5lOu@WP!n$H@fBBk+YgA#BH>nTQFw}L_6$+lBRZHE{W&$#-nwVk5jiM>7SC; z8bQXN$p0|y?p+d6t#m_if3HwLrD@=-TSPL!tv@Xj{>@j!cXAQyn&W+&nV~6>sS1)D z!TCf~)iDlA$C_>qnmGF-T{5A!>5K%*zc#gv2Fop2%03Bk*g z91kFXD`I*bl5;e=wahKI=vnq;HK&Jnd^wtR z4=YDzYD$Dc1SGND;VhHV&EgmG_z-8V&ApvF!8Kqgp~G;Vcg7?8JqNNK8_+zj@OXi_ z6{`)9WWE7Rbl<>^L8BMeHjK1|94%C4yYj9qv_NsZkXDf_gz(ML`H3@yIW|#6{!GNh zCfQ|kgjW020Kb4j7d00Q_Ri0Gpo;#yhAUT8kj_31r_OJP4@s1>akQtxbz?2GISe0s zl-;`gE%XtQYU~Ybs%eXomG=+0Hn98BdAgE^?~}!~r^y%8NSyW4VgA*{8kSz#zOy!f zZ+%sCe&SRex?wqDghc@PY~6uW6M3*nOlPa8<{-SC>&9grYE<7sJ?$McYo3@ykzNlc zrX!zMkJvWDjhKaFL#c$0hNT`1PTaW4FzGG2Y$OTg3Nw9hvclYk@s2mE4d)ON=wW;V zDf9}dm6BcGwps&(UEj@%*v(dgV593S$RguGeo4t!NcrbkMP*kE?!qTD2pPu6!Cr$4-d zV*QY>){WJLEs+%D9Jb#4dB-&;_GHPn<{o|TbRJ2m$4D+*xoCqesPn^XFtYh>8AD<2 z9{c10bhMA6TOUM$Vh%p<^Yf(ZeBMvq_RWHuDse!lrg_^$2VhL!!d;^t#wHBv=LY}; zavDa`ioT?<449!T0B#D$q{CNkej7E)CfBkd<(}?d7(JEne&4y?SUCJF2e2wtOxCuV zPlepo)d1sgGQV6Q5O5#>01=_MFoFjlL=~1;5J6o)0JT2J&N2mgf*^rol^wj4Hq&U) z)tEFJ1g{ZIHEfA-x=tCzhlMm6Py6cn3!H=r(zr$2R?Wtcq{fC99C&K%1U zY1UbK01U+1E;{Gn{Rq7K*BHC_cl6r#)u0W$)DY%?6-S{0`U&F8^%W?zxXjyiI-3qe zfTfwl?zgHASLWV}w>L`%Q7j)jZ>n1frx&rc-p09$$9NdQ3CO=Ot-?W?lSoAN*niuQ zeU$_H$GoUwplTe-Glm1luxH!5W840P6EQ&}zm!vM1oD${wlW>9!+ARu^U6ro$Cyq&S3n&V_g#&sa*yu8w#(_G2R4iX*8?CLx!L`PY@A{rr z*%cqiV3Rr<^L`)WR{5RJdFhF}vof({u%9WBcbQ_^wFsPNo}?df)aUJB!4G9Z7eh!rgjkiSisjh3Htt-WEdwVmq9H>gRED8Y~WzLQF+DC1Zf z&fgWnI|iHQ9Zb>DYzJ=2K;4Z7uC2&sg?NkhL0c71z6aDKr&?y;XaqY35^r_SbJWKr zPo`oIsm)Ouz`h$_WP!oTxA_EaWL;lRUn|y8X6;A4`k1|d1wk(=fX`xfo+A2JH+ctC z<9!sN`Sgm=xQ7bIU3@}Rmp#)L{Nyhaiq%~1mQq{@9%jt*NI^j=t7WOPoVa01wL7@% zV&<^{?9n{AKh?#!EWM?O0N>*8XH`3dMd@qt1w7&Yd<#UgZ0luVuu87ie;fMb*wW79_TC%T;XJ`tg2fGQ$i0~v%YTz% zD(dPvQCdzVz|?r#j=}0uoGGU$dD3D0WRfQ+TMAMm0qwGDT21>&mt|j)dWnIB&2mwu6?n)2sg1%(pNxpJ3x~riagf!WNo^e(( z6cNq$YXuY4qwHLHx1t&B4kBrIds7=XbvNkqULjBZa>BPz+Kg%jX#nUivc*^p}+ z=AF>vFk;pmf0My6+232lOlQbeE$n;ymMz{V*ex(i3AGE2_MD^}a}Qx))LpAZ7-_dN z=FvMmXKT#e1!-O!Se?9Hrw&Gc2VdW~R+E6{Zqex15JEkte`Kp2POe{I{=70OsnZOO z9U6P#tS43008GjmJ9Q4*>BwycJ5ptjq`6KM!DBz$w%FNCYv*!GB0>>!mD(`)^T1Q3 zA#I^=3+LK<0Vw9&J~Sk#XU8ZZmamEv)kAGhD=t@*ubM_0Gauun*MJZC>n4e2LR{!P zfNl;(c};9u(=ohpZYBIA?n+p(bvp;Zc;3ao=5q6e&W*ao3SNIq_K)?Qcp04g4Rn&7h3gABzmMN?&AhJ$ z0eTU-CowP7EF-;!v#fu4)U0%YP^2&Y6(UxJYsg%+b5>JH^~0s#UMu7NK}tZt=4;aHk$1NEcn(H6!!n!6= zr&$l=&1(TMKo!W{WA&;fayLH^<(AB+*7$K_ zb^oKdCmpfH>Bu+TtWhT@P?#LFamxT%^g?@S<&Nm&*W8xiSDh9QAT0R$qWa#XGUzOt zNxqy3*Z(Nvx$GzIjfzUN$+Fp6XFImLULm|7FB}WhXS#3e)UFv$ju$C+ zjxFXw_nfI@DS|jvFSg8th7;DGFRS5!)Gp~4M((m7dt!);U7i>s=hO=} zI00u@0)Dk5X{AT2y~~BhSYpMOz4?=NssG9}oFqQrm5PecgaK6KDGhnbADL|IiYW)$ zLr9Fp9|oc>7;F4a38B--L-fd7qyp2aGyNK1NRbWJZ;n8EAl@8nqCH90ybe|@eR4jU zFl$ql7YO;A5#<_&(p%v^j^+0gU~W%aA!RwxG+$ugi2k-_OSXUjpLy*bo~C*Xi{lZt zMu0o&*R*rgwn=P&_^-w&1iT;dG(+lsITUEl!v2xXYvP>uo1@;j_e6*yzczSHp%di* zdMXMNL73}%2zIn!57Ri*p*82^q{hhMc5J}fr=1U83injPP;_kZ`6RW78;6$u1y3R0 zO0Q$z3k5Ton4V-&lerF-zgs10tyS=#?7IZf8L%l08^@IJZw_Ha9b?NEhT~WK0Hk9P zC!MF(hmnbFUl-U*B-39x!@+K5pUACHrRjiS|6L3R0DB%w;r8h7hp||$@bo%_Q5So5 zstML{!(J`hUi9K+qPJ2031>OmSK(MDYUio7SAwf~h!uPqKCP@!|ai$1Wg;f9ljGVj~-hzIiVi67X9`T9F7b%&Z zwn2B%QltFRAg$u$OsnV~<>sl6pf zJ#)_?mXVL?4!;M=n+O^$+nuJiX&J zN8r+~nVz}GfGKIUR6)A6lKlqGprb-3-wNVgwkvZYD^rkO%UXO?-Y5dNNsjIcrm51c%5>tlXHQf638`1P87oKdX-(C-?dP32Lmh1NYG3i(L>xB!jI*y_VR7LZ_4v^n<$1uG%+6ATg z7e7)?>HwRw)t`~YLq9}uG$vIJm}m|W2uY!IUix&GJ#uG3N-&1cUR-R$O*K;U5_+%w z$!NYvI&EgYp-G=Rh%m$&Z)k{tzn(RYkSI))=%tS({2+H!QGJ27OUU|S^MU4@Cix3F zGavv}BCn~qwX_{UoLFoKxoIQ(SgtrFOK!*5I5GOJfDKhG8PG%JYP+PAU9Sh9hFE6l z1$mWft;eJFlpEhkEZ%(2@CPV1E4YMAUy;&bWg&28iKWTMvs&wb4O9b=wNd4LB5s2zC zP9ejg_QO&$<^Rr2qbWYw<4Ey+CvPY0ij1weluy|k(Y9(gp_z!5KtyGH5zG}{l+x}2 z8MMS*jAYw)p|&?&6TFX4^7|KEOk}yZC8fyg$x^T))-kABlTElPxlHwh8$(kY{EH$P z$AdLG(O7TH>99Hs_pIcNqy1Nf%mteBCv4Az6`Au6;BFau)F3`>AMjc$DVqGmQ^W@A zTx>SjQ$#}MpyZvJ;~07a-_7Qttmca`jjFNx`A0eSHL@cD`+_Ql>k##j%!aXZVxbR2 z$`y3q2W>lt+Dn|j=lV=hBvmna|BylP^FR36;E;ATZS{^T)Pk(YsS>2x-;L12agzFfbHE=4FRB-?Z4VA%-Nih`p7t_rynFP+n2+I z>r*AZR!QYv%Hg?X%X^==@G`r`J|Kc#io1C}ptDD~@oLYj87nfaRTlKE&J0m#1iFSj zv%v<%4g=vh{1*fBs_-8f`OQN?+6`ImCLHE;dBZSSD;;ezOgeP$6p69nRDvsjdJBPC zP9>uXX%Y%pXw6tW>1L4{g#$1kbE*N?GCMxNj&Vl*fZN&DNK8*0;4RYjIhPc5l>KKs z53=R~RkT|-2LR)aiTMu#>g3s61LjB-?#wj1@Wh_kq+fg@1G&~8b`iO!FRov9{^lqd z^@ah4l~BlZRMMghESYEMRi%|m=FwshdZCbC_2xf(BfxJGV!U*%IhzGX_G(AU6vslN zVV`!M^I-H{cK4ym*LlqB$At#M>o)|$2|YbV`POhWJ;h$+-l({ney zy=SM8bL@n4=9!*H@D-#=m3b3lmYwb8qdLLVWgWW~4D}*YT?A zS|vT#^1ag-HF%bF8H(Gm(qT^p%c?ka?^Pq4IVJq>JRVw7InHRtrl|>}s!L^|d97In zZzJyNq)LX($FNWCsL}3g^%yI0#fkLgFxATa5|gQSKH`MGNF^*EQ%nMFeGiqmvHsG=%k6P zIGBoLoc7VK9ojpAT6`qtsJs!NB5G*DWEQXBTi+UmD#o?0)XXu|cEnqUz(ap$l`)XT z$&z*#RpnWN+p`^mo?VZVE$E2%b*am_3k1k-R?!{sgc;|+$EH`$w(b*DeHcLc4jfwm zAL+&V;P;vFZtvT)0OAfTOtHWQ$R?zzI`#a0uhndstvbqb4PAsK$hLq8!ArV>u6@%!#cZpMu3*9-Fzh5Xi(*~RHKW!w0X3P%Jxap_(h zzcJW==U0-V{GGj^Le$?k>3Mzg*n65&G_JOWPqMtUy~8Qo#PiFtk>n$|>1mts_gy;b zVujujKg{Mjj@eE^Tqfj3oWXf|OeY?vIS81t+=MI@KixA8s5~^q{FRnc^+{!vYSWT? zO0Zh>^3mRsvZ^)1Lw6-lWjbMw_HqCIGJlS0L>ZgXY&Z1dy%?XD{WTk zpA@CDUBDdBG+V^zG~P!9g5v@1tyDR!Pmnmdeq{ittL1&Ut8?J&`#v?IffY&2wdl+E z?+UD>-j~qlv=%=spbg*>*gz8hG7Fgjg}nay;PHKEAbF2WvP)ugI@=(U9j*`Ifefej zhFEyj0w4iEmw9Ap+jDQ+YXo3BC9mmBq|I z#bN5QsZ0w&%Z&Wk;*FAt#9R#gio*XDK$(EA2UJPS1l; z`Hr_(D;wRc+e>BwtFao2{HB9h#sAOn%rKwggFz0 z`m4MWwi1;evOO+}29(KLlZ>mU+?uasq~ck9e@@TpP-}8l;BX3$innJHTp_B3p(&0 zNA?R5-)=ujDHc`IqC^_Ud$z75qid7sYZ~BEqsbdH~)-sDiyj(Y#JeE?kEu-6ChDMEqqMbu0b*^11yz~EV5;o6YZ0-G|xz<`|eOUFf&RYJvH1}KXlc%-vb#NWpEh0vcnN(<# z_^s#ML=HdiE^P4Wo$`XTj6#8*#y+7ln=cIEq@m@=p9z3h#QHiyX+|OQZm*UAmf5P= zb}rH(JVGg!p`|||C5FlT4Gk10nzPR+dhAzN~SeraXzGgH>z5dW+#f_IKA zT&F@HDPLbF_V zC^KDHk0Q@ELp%<|z?LNxhzYRun) z*h(!&mw%fmmh1;2=bc5a^Uq!rvMDl-S--npXE8K*EkoNo-5f(W2pvA9`Y+rH)u>zs zk)^t%S?u%{pE(T^R(u7SEg!C6HMjTid)UK#`mwZhQ2b6p;}5X_1Z*C>&|7fWO%8mo zN7PQuuzN`1aqBpBjzWM(YDcm2WZM4fHirP%ND|+(5JF93= zU}v7hqU7|?0Ff|VaFA2$Za#0&*E<58+2=>r=s4m&!!fLyAnjtFTjtEcFW}nHPI9TcF9d_PLWbxvx|6yf$8)KPo*Ntw#(zzNAb`Dv!aMKDZ~X4P-)TcxENWcsWXUa7 zf+6gcMBi!NBKgZs1H_6tn7+3K^EC7UVeSmfUs)#;=^(|g)_}ZgO>uBI+oQvnYdlWI zOcG5u@r9J}NXo7QBK(3?Q-rC*PTqFZKmTmW=?UuYPKR3#BQun zf=M>(V|=SOKp^?YI8QcCI-?_%qAQ9`sBwso28DrzXn)oNy3=mq!UmwRqghtlF_zzT zeN8w#BhKE5JQ=VC6%lZdn45@A-eh2hn8ji+9L9L>w6~|Rxh1o;`|~q8&yZ!9b|=1+ zt--{UW1jYl@q;bKtJH|gSt|9yd`1t5_9x}Cx z0`Q~t#ieBx_3A`rspeZImM&ck$rtv}WZC!Cg>CZjnp-_tBU(8Rqi*)g%=`w)mJRBH zH-M)}pm2~>sQ+?}dui*_=b18_YVfhcEH^V@UwHD0=^|RzD$H=I-iyjQ_I8uitiJmb zwc~Z0H#Cx-(%BD`gw?R?6vJ35zH`J7H_mw5zk0{JS1%}T8!m}!6h(+;Eh1k+8Bn2C zDcAU(%dHPpCe_pXN(B<+!0zxK(Ym%0#2Vz(lKWOVh(n2+?#oG`CTj&lCk*|E5D)@e+7Nzd17jMfe znKEmAf1W>`#Y$*xdwXlJq|dfevJ-Q$Wurc-);@-*PXE8$ih#6JQ}Kq~4&y+9@ffc%s`=Ck`hRe=UfyQHKp96($A1hf`^v-nm=VCT>P5OSr1rlORFEO5t|T4H`wykp?V8!q2d znY)P~?Y4;%p;&N+aUXZ^#upPhw?&(}3^aDFoEv;uqU2+sQx*v` zf-jD{M$Ap^a}vWDZ&p#!(ON+g6DCgJQxG(?aHmp=cAYZG<#eKT9o@X{XC}A`x==OV)UzGn|Oe zI1+u=LHmruvc)n_$VvU#BpjXKaOiRBw+? z;H=_PJNKyd)iV3g1R!61*q22rtXj&9ThzzZmnki9_7M2fFfa8x#K_+mt4L z;S}H+itrNJ+{q0AX4T|HdF*+>U$c8e(<*&Y0>xCss*#n4MVxDMds^pXH(Etikw3@C ze3D$ja{?%Z*?zaX2PHY$FAg=~kc|sNfJm$0k%(pMo764Dp^B;=jSy3@0zSulT>mS! zsZh4Xc%oTnecZ11;@$#F6EV>__>@H~be?m=Tt>d;zku>wrcYU{(SxuJ zc2-Yfa)r)P&OGjYb?6^3@K=gP28{0$&~cQJAOhKWVt;X{_0$nBQPfD-hDIy5f|mV5 z$WU%f<4(=y?GU+h*Z%uEbZ%DHpC1yaL`f$RrKsV+WQS{eG6E$Q0v$RV%@-pxBw@0d zfmW~4rMyB5*qBLw$i(Ep=zNYE<}(Ei7`1+(_Y&_GH~=Bq;9y#;^W!0V!zz8u<$^h2 zU{)*_*IEJ*H~_-Gk2_@h8g6~@n{BD8^M)3c%`E^R2*N&)rnoPt6+#T z;`ao^Hw4q)xBF`&aQ4#&j00N7Ta!Zp3xx2igukt5c}~J#BcS>0wuF5wsOSz)yTJ{E zaNH1#X)!~cCop{SbuN!l_l`->q?z{c3OYpkec2c?2pWJIcI!;S22dg78A3eTYM0F& z?6-Ybt1*{d4%p0@5K14;YlXAcN`nO$4x0~kUS0CLZ5w0 z>f(GYCw`zW0(B8bckOA$xRH&9rQU^97b4vrGa_K_WkHSgKkcB!{WfzbL$H}0smSfK z!TxTFrGXGplQ`4RqP)4HgEIan_SUUG6h8mnQ}gl}I<>*sBhO;2+REb>!2p39LK-PC zQeN?c5p9lbz z`FWh>YUJ9Ezjnf)(oN&@I$=rYi}D=L&~advj2sBVhH}w%_;?=uWW8A0cwJD}|4&eq5z&2?E}8Xm~Iz zDbKXfv@G>qrI{8*W$sJP%p>HDZ!Qo3e>9p3_3Yl8cK+TJ|8=51NL6|l)Y^03Z^fN0 zPns7x#;du0oASp}Y@)xuhbePy0%SQB>F*9!NgWqfY>(k@f6M@(D^N>47uN0QXzVn{ z0+i#cL#vF$d$zDABMlGV1x!b|>{$K-G^_b%Z3W&dd7&Q0VYr z8pa5q9p>{(RriprhMWQI;P)w~=Hx#kG0|yljX^Nx36-XFndyaNrPT7=Mspge$gh$* zqlO$Mnz@$@8_Pd%wO}^6MiycVq>Ym60GwR-*uj2f$1i*Af||%L@_1sX2>%6)LJLbX z7rj6zh$rsz3tEc;-9f zip~8kCP#5EZFoBH+?XC^f!xG~j^Hcpp?g~FU8^^_ZAmGQqBtz=Nzf}c2 z)$3DJU}*9uY&J+(eT)h-E^t7IN=kn92lN7C$uj}+pxk_rz|JyKe^xpUp}5<4RGSmB zti%d*9l1*s^&?3391p_E>${myzC)v*PxhZI4W>JuWM~+RK1e)Qq~hK((1&D|Zew-e zY<|q@hB7f)pV5xB*7DsaQYw$~H=AT%Qoz#g4F;hxaJM2iBmt>w1esHNJ`5tS@CsDL=`n z=@RAfS;&H&(y|MA&E5EMvzYvEIDf*XoKfiQEF0*$!PRylaSPQIw3$3s)!#I-Kr#)F z8R3gwbQR*RGM*>o83M^9PJwB&ujnf4!sIqGpbr1e>XTc{#QqYHaPFhu*|iizmZ%z` zN9SKRsd|I_XLqH0qd!{sb1wQTvse{+fV~-Vk*Zzw3xlxrh=V+zK$l>TlxWIylK)66 z_Jk2eq4i3rN1|`k`Eu60jfpjiN#%y#a9OP~;*Dk)dZ4P|EAggmu^?`W^>T-nvGli4 zup$;REIqr+nLT@6oD)Z67#Qz~KlOXDen^qWg0v}c|GnEZ?4j)IUP~_^F#XjbZ3>E_ z>Li+2sOMr}cXH0DlED6m3*)Naj1$`|dAC|DupA-vcY*>7dO?5X1#FPmcfugcw0eCoty#88Vg8jgv)a3keg2PnKQ>|> zvOMK=WS4)9X4S|j;oMd_7~=lhD%>HUqTqnWx_jz&;Yi*rtka_k?~38Hb@N}(p;BsZ z%bBX@R)mY>LKFDm&>_@}$o?(g^?+gS| zZi;ODqQG_S{2D%9gm5Oiz&Nn87gH)DwGRqwTosB(k$ixoe}zEBu3$}tCr9&$GNV5g zEyatq-ChFvo6B7(nmgs&UY_Quz>YmLM#p1XK$pCxqG zo5$l)M!(dgjqa3Q8U51LQnH;$1tK;t$V87lZ|lv|ga5sV_Fo3qz+Obis-8iz=@zXflA zo$008s+sb&NBLwgv?G|m7X&heL3*6Wri(&mWaGkyzCA)$cI6_tUL=4Nij*|h`8HXoIgAuF?c1!< zFbBdOWek1AniLUBZ)4}Y$}q#r`_7GGxxNSS2L&d*PjdIdr;`pg9t4sy#`_N*-q z!f`_z1FBc_mDYEH5hok&dxSIV*qm`-UEURcy!SKDps1F5vSIwT!EkrWJ^er@#Ge>) zDhuY0iWTm#BBVCq$ieX801Hejnl)GESVAVPl*mS8o8rhNC-P{5q_&jM%r0N+wmyTp z>aVbom=eIwcpitGeMY;%QH5QZK%``vXu*eUu%WiIYn|2O7$TSdVq>1M0Hhnsna?9} zRUVtAI$0qw%l>!LZ*Kr2b!OehokNj(wspygR@~JyT6xzJrJ=X}PA4@;z98#>Z-Atp z;%MYW!0?mww zqE-Wz$e*e!ZM6ZpKo%+)@=8F)g;&CR%Viz?iwmh|QV}qFZ0OE_{ z2~-+bbaNHWz!QY9VN2?3CDS*PW98QZwm$II#pf@C`R@s-sqk~OCW)}Xm6^=(Jg zx;-XE5JlN(MxFUw^OzCzFIv+3XUy1yGn9Z()6g2uc2zYNgSPk0%%l|}ROQ4;L!U-T zqMk*0OPt7dj{0*ypZ>miA}dYVI1S*LR6{|4*V)w-P6H2d{OnE}2YsQNy@6V5@|1`^ zN9m1|3XQ@J!(yLJj`R+yAr^ZP?*`?K$mVA+z%fu;dBy;PlFlZ#DGkT$&iOf83&z)f z2LU^9M6C>=C=th1l89W@BRBB@EQN=8qv0dfQDw2{%$5xuyiUt}&}y>BmtMYVyT!vE z@3eu?#cI5WDx9~Cid`B0?HDvsRK}X|B`IMX9Uhqjc_efVTronYhEkoWAb!G41#$R+ zG)VI5ZY5rpWCD-lli$`~oZnFB-Cs;}+^fL;*4P{QI1gMtw z{&dhkS8Y>D>dqMnXOA^?rC*3@fC+Q95`%1RZuw;XYYpkltz0s@9ow!N1hO)mL@5Yx zKmY(yq4+R@2OvTaR$1Tx&F3Nm>NHEzp3L{PY->=(blz~^{=6F^g1)7dN^&qaJyOSt z@n1md{X8djSCpT{fg4<=U~<=htCV_6P79z-J@yc#%=%RQ(~z6R5$(wo!{KSpp3*Vr zw9hHgR+h4@-frt&jL0@htu~(aDLN~+4niX=tA35$LyXAJSK?lyP=eUnyOk!t$2p6 zu=m9By4r36{K!a~0JCB{ugD7U$7}$vaqtZHbv=RKz#oKItCCB-iB;`nEwTZbt~z0S z@%byh^yEYAC{8i&qTlhC&{P&#wR~Bt@Ry`&L|1=ygZA0z9uz678DA%tbpBjFn%es! z`e$?)7xQw(8e2fPHmFkv<@^*}Efti^>q2^?`g$`hyaa?8PRpvfh~#i?GDKJywI#H* z>>R=&C}!fcxo?66Lz^N{96e@bl$+_bYCk5xK;cxb=%suxG=$95M7C=1tCd`sv5grR zo>>Mbf2tTl%^j+p3Hzk&bkZPb3NSnKK+Y~D-0oz-p2N)2;-cT?g)RYQ)x-z?g|O3I9sFlV1K+_+kuaP=vF6adQ^2RW7St62sQ$p~GQ?hQ6cE-57VW@0{RpBu0Bhf+ zhvWxj#t!AmU3W=u1WC$OT6+|oVYyl1DVEJN`-&C8Ym_)<1(Z%o=gCEhKC{Qft>D8^E;fX`;;&8ViMmx!|c0={~EeaWD!c<&A z)6x(Cn{BFD(@7W*^^nU%%kS$+MPE;-UHth?OBDEFyfq>%7=;?z3Lt8*)Ug|Yf0uGd zl_qwEIeaCH-0zQ({$2Xn7J_7}!nIuglu#vA+@D`NxFqHA1@0mnO2^B44YXx0n1Rcl zg}EFhdtTYY)p=HgmvJv!EV->p7dX(X(9T6e0Z_4&ihY{f3iOUSPva#Qybf()kD z&b0H!%&?1uc4$wc56wKlWwQDK{{#@73qj(+cs>iXj(PUY3aBRv?w4#hkLM(`^@(0S z&7=zYIfXUSBAn*WY~Ja4c3y{oJ?PjF6PvI;c{p+GAUMJgK^R7UuSYL{>OFPaXwioK z@g2@UWnz)W0Re88HMXMc#5lOIl|4S|$aC^(?-bk_XIU@cizwor z&7ANEPnPJ}_UrjnvM7qANn!?*vh$6Hk%T?X>4xj8zCK{Y7HPb@y?w0f?MCDSC>RtE(_ux_|Rh&aL zvjXoQMM>z~F3vb(v!3rQDJv&IkWM`oe34`j!FnV&8pANm!%JX=&sy}z$8$89xscdT z@4k69FthE@1wKUB6$7CX)?)1iT( zcmlicw$$6BJta0CM&3w;ciT^>tI8XT^r%Wi_DRDFMFBQt%llB;WhPXqR6s4bznwMLCYmcJlQQiX zjnEy)BG!w!N{_9O8yhuRHkAS97f-#wJqERZtQ7(G`s2f4CAP0P3?dytCg0vTc)3)KN{b@r2AY`;rgrmCG z*Vc}z_aUxYjpCq5C);$@TJx}{abj9>5}UI5U65#32J@%a9fZRkO%ATOh|zRwrzODs z9&#(uQpCKxdW1|wpySo-zy3Sx9oP|qbEX}_6v2)!^O45DH;g{6V)SR_iJ`z#eJo21 zqr|D!%43L12if7v1d-4CjHV)1g`R8dSe!h3uS#_?rZgGUi^Z{t>PD7ionp$hr^nTo zA1N}CZWG=hpZy1GvJgRIk>SHA31h+8Mp}ro*i@7dXhpnOhv47fABx`&>+|dHrRjY! zKx*p8&+uxXFi5t#hN7lvM#s=e34h-HVLaju)U0}Y$fg$pBau50ca&z6TmWf;8hkkK z&>q~Mt#ex;gvcyoB~+HTQm5CbdXa|V^{6aW?7tNHW3VTzP5)}5O@OT!V~Q>oJ~S~T z6Gd8nVy{T1mQ4z1YojUcBZ4nd;Nw|DVua%~j zS8$Ig(sQog*ZZjY4~bv=Z%b-3ZGh_u@Z1HrAdmyfFKx<;9Yp(xfdgN6t|#={$a1;2 z2utBdHpwJ18-_qNHW5oD%MTGNK^;cy8Ier>i_VrP#;I4c@45M=nhr!r>lXkN9vkda zD_j-ldHw06#0*bRYTX^PpBoolDP`z>%?*hV>)^P`#}wSe$KLpgmlNvA6ubB1(#1Srj69nbHTQbaUgV5v zfULX(F}UbF%`*2VnR#LBWyZ&f)(k{& zvgv!5r20}id7!aD$;x4+$)uAGr*46#t9YSpc8MdIB6@f4&=nrpZKfE5t2!IDOA8z{ zMYlhwbwbMK%23W7yR3w)52FotY$ra|jw;t z(3+BlTP(-O9++j!s?S&OFO4E7&@#7zL_p_o-!sATc;6sFiJ!551)1}IikE80l8A}2p8Lo8w#ze0)4~bX}Cb( zfyUL)6!ZLGfId#WUs7ty8XM9m$}8GV_ZOx`1cw+h z4Rv5C9wqjlo|MVwse!z38fBtbWDSaBo_AT3qqCje47ie-J=Bu7hrp*Gt}7hO&9CvI z3~7nhlcm1t<4Vjchw5nhn2ka2c-RP9H z{Z23}_Ai|y(tq9IY7k{nAAM0972em&TI}Q$gYzBwJUbZ`1IZ9Pb;is`VGf;M2PpaK zjmJ>{5a6Rr2BO9M3yW##Fm1Z$A)JDfckthlkx5)wmunhvf}PtaFt3C#W*`#1EEk+% zHjSEmVL;(u9dnmA@(MnR-zhioht~0**=9YEH2>h6p5|A_3%q60Uz-2>Do1ER)%m+8 zPW9TxHz6Z~W8~(MoMnFC90A0dLIVy&@mIsm)fgyO6z$4cD`7<&sIm{NGDylBk?n{r z&m)}cZvq=lK2l2Z9$9tSoi_NSkd8NPmP9-~qj7<#tnx_X_dj2C)kXl{dxMDSLAK$F z%C(+prz2J6y+dbmNh4c(`&=F4ash*#hj4fSGbX-=WuPtvy6Ofx3IVZso?i7 z?su_D;8VEXvTwFR!HKu2>CZQm>H9z*F=NvoJEDyfcBgVTn}mrKqO8qlC{)w+PF7L#w!pr8pu^)63C=VLT>Up2*%Humm=;eBqN?d5x6n(-+3)MTbbI z@K9QJU$4#yGSoLNN5*Fyl;K5|T^!Vn7}%XxeAxi62&~;pFz%#}?YRj1XlQcK9Jw}47a6FTwt{X@I$Gd|;WZfA*=B$Z(!S+%=i}_-yz9{!C zoLmvA5RtS$B6YV=RrFGO@BW0k0q`!jnlknp*$)2Ic%0pFzX_9_ zI$09(_-UQ#F;aq9hNFm^Vu_6tGY89ZX`IWymLNY=zP!hLTD%{J!FOrB-{7L^(p#5b zfJmVv(HSH9HMSSgaO9uU0QLH@gaHH+wOy0@wp*E%8F4&(coVdT%uso(?9{*j2@+MJ;W;rh!0KQy#wP?^TryN92%kx!wFi1=(m z1>B;vBS^%ynON|fC{5FTVwhONgFB(tm#oXRVn^Epa@Nuv?jjn{ld?Sxgi{boj(o?< zP@yba5B`vXfd*0>6!3>=OG`VHV#bus9$<}@0mV|^_|}F$W#M=k06IX$zZ;D>G4dZ4 zqT2gn3K=wa(*koKuO?vuhv@4Meh%T$Js?kG6kqUsDU!J;e`Jq9zSDeM2_O+of$K&x z2|`2{J;$%)dxk262LD+ncX?NzkbjDxX}7yJvLQHK!igph+*o>DaQaWSjTCL+3LL75k?W2b7atyXN?0#xaQ zATQ>v(Dt7Ae7KM)5uQ7?C2~4|2Z3qD>rcmIKRLxMvx+Fkx9XrCir({u1N*DekuyS+dx#ii>owe#q0S!R3 zY={R)vT=gnM~0He{NGzEyI;SI!>J0Ko{K%_g#_)a9~aIC}b=~j?Cxyh~d zs(~F(iCh~HQet-%!IVG~C(?fx3iJnxT_uGw0k8&qA)~KdT1T!+>Izg!z&4f$iE><;+|z<4w*fJ8_&k_%Sp3s1QPF+VEpSUH%Y zUx~0FTdV!^>88`OYwH8t0P4jy=kQIqUid8Q&rjD5S2MbLq&mq|WQ5R-#EbD}Kw;%N z&<}Ccd9xSKna_>V1ApK_>N_DD%f1VP{mv3Ud4hZxldUFLeZ+Wx@Q(*gJ2e_4_{%y) zH%GyAaDnn)>T+zn{9h77Ek5d+&4K0>j(V&O?lXIkxx9_#wZ>8a5?fwQUDg(4_*>_P z&3!CBP?t_X#CNH&PbZ*WF_zsaJp&~|;YGuJ#8^*zt1s+&j3-CMRRUve1dM$m=SVX>gVkk(EZ_4M6qcU6qDm0 zm^1tw2DL%FAm+lb9)wzOp2-=Y2wS%bfV2YvI;+6}HVLytC27}+qFP;|7vfmdtIZi( zH<3QA{|SXMGllYoAmcGDv=BSSF5<&&)wqfMJ+88<_4ua9_e4~9 z1Nbn4D|Q7n$b>vOGe=o%w$JJD-J6E#iTyD7%fCvB57_$K%vsE(4`mI9YluD#{9H!C zz4o=hL;NaxqeY6t^-IuVTOJHj9zsm0ymg$Aib|#i!&ZWvoiG{dp6RI@LmVf7Sp$^A z?}JIpUK-hbdCq;Sa51ax)MD2+&)*rYNEWXvCOv=rI2uY864an{I8zx~k!N@UALFpGu(dIZGM%p{UKgyA*J$YZ?%c zKDUx2hfIwf`&1bCiAR}S#X8q|%S+(_N}Vhsxz z`DEn2i`Y(dKAQ$f$pt89jl?h7?{4BG@v6dtT2gp%y;|EZ72BHB!O0`**O|0~^L zj}-{ADjfs*FjJd5%L5o|J?Mu;g@@n{3x##1t%><(@(3Y-J&e$NB}s}J?Uq5fti-{G zijI`+$`=5$&SaBnQ}YE45NqP6#jN^q;2qMHoaM*cW6Bz|Z{#;~5Tmri5)YzDZa;yj z38Yt|lmKL&833rshn5Ps<-S2q-(<-mvj;qo`v6eQ+W4MU$O%zsd;6E!=@d$0S2s74 zjma=Cw=do9(%Kw_^yTzLyDNrPS3~y>(`PR!QH#;HQ50a;Qy{P^uSo>kJR!MAiq_OhXJs%B2~H z{5U+(GCS&uKp^c4u+Z~xBoLt+?mSo27yS_e--Oz{_R9mQ@6o{b`2M~6mOg2Ompr@O zUfYeFvoaa^M1fF)GMWCbr>a>g>z*YrIvoNxHY$F;u)mp`sbB-GWeU3c(o_ez2n&dK zRffAg5Te%{LB-3nO1C3I2eBZL|J2D}oDMXu$i`NEv&Nf%+D1OkKr2^0Q}ZG#C4clz z;g(klA?a=w30L-mA>wZnPx*m2X{3dHix@8{^zm=M3sRrHZkbh~{+wMk%r8cSKwM%` z0$*KGq-+P|IdeO+Eu$D7wwu(f3mXEu(>GUAFQsb)k{xw&5%z;xe|g4P`_pBG5X0GB z5hDw!dnBPJAdV>AOmG)m7>#o2u~hxIG+_IN zh3|eqB-polk7eXN{-d z?m;n0?kvGJ?y#8Zr+j^fDzH;pA`kF6E{#B=l;ub&8%nbzy~=_AT|Jw?>}02{Ha!nT zGK951X*`W?`IBD@Mv?RnzRidht`Q2lR759!+X-TFQSaTQ&^A$*nDUZV6%A-nC8*yvJwEL8KaNXQ)|2ruJ*c3t zjJ73O2!=w&=(yW)JWj^b5r~dbS7DNq^bp00$I{aM)2Tl-wWIs&f=L(x94p%l!S^c4 zKQ{+x-We1Mgi^3i(8^cXy$qP4G6$^I@7r%X#wHD~G2h5VzR!2(&gRly3pge`?kZl=iC-o9O#ND5Uro+((J6ijI43wwu|>isW@@+WiOJ0!OE}-2M;l85%V@Tfp16ZI zlWUEAFl`Z-Ks$n2aWZHqhLL~+0p-sH%7bHh+{kVuq>cCvec@G!d&(o|P3XbKV?+~=`oPp?iayq!t88pceXRr{H-YE%S=?Z&N39(rU9}uGFY6skR>fqGvRGT_poEw^onq9!}HTWYsEqvcH3bRB@`mfI9kL1<)qF zVYYzuj=Pke4VixHW25cIV#ILV*~4k00CbncDIx+5a@3w{e?vxNtA|t{u)^({vKUxV zg^XG*`oO!^NjXfm0#GarTRwAE#%ZHyhOK7Gm@k$9Z#U31l!82ROCyjDL=K?@@jqkQ zOsu?&Sp~)ec-4qf2j#Y6@LPc)s{?lNshnq>ye-#`*PqqvUrH67LE9K$0LbeF*?R-H znPf@r=t2qae7T2V_5Egvp*jG zPT{ZWs~xJ7D_a80+d*_odIWp6Mi1yJ%<<8F*4KYBI8jO3{_NOI-=C@S7S-4nf8c>1g*v2np7X-^cTIZSj8M<%5Vg8JKiIMRaQ#i3#O zb0DzOqc8+vL!H?|mfLFG!wzI=boiQG;Z;stAOY@tA*u7m2<$?_k1rm zaJ13j$+qk|cJfJO4%O1X7~s=j6;mUvosF8OnZ6B#^GJe!7s2F*PamW7A{Q8q>EKQs zX&0a!3A1e{;0**IUr%iPe~(lF1&Siup!++9(y%O`_as5NIqNr5-I4Hd2yYEK1ezJs z2|EW1f>d*aj?cns(3ZPyA{x4$!wcCfyGrM$P$0lwBA720ylSo#oj$9b=*CjWU4arp zT7C_5%0U`zK}pXPz7}`4^TJaILL(UAI4a9wXH+4#pmogB359Yl6i6`eSOu|J6SDQv z<>KXjjZ;|cbUGxa+!ZnI`N+;$y+tkFpK)toEbfP5_tB0*kCDbLh+3ozhXz5O zj+u+pCTG7-ynO5TUy*$=Unz<|_Ul~vRHO0NN7p4nBY-ZrBO7JL#$ZUm6+#@zBk0rQ z^7XP315gIBGf<=OI%<`GM9tWm*C*8pMTiE=^oNn=%J&4Z`xnvQDB`6l0+uNQwMpqL z=v-CmfN1Zvz!K{b2A96sIIg$(4q`u(i*h_SNf2S=fVrBwsYoFZm9#i1nbVpIxv$CX zMT6OWfPaHD?T3Kb2vzul5k2s;L+K2NdJE2t&0m~+t8H2MV$hGArJoiM)e(Q4firdP zbX|JovbIv86_E-9F$vYMTK=+x+QGhfg5RqgE8L(h*ebMYXC|wb1==zT(5ek>3>Re6 z@tCVAy>FN^qlScUI%Y5yDC&RA=Q;=|OZ0zM4)UD=chpWXAVxjMXQ@h*nO=YYJo|)b z;ww#H6hbr9+W}7`kZ`lfa_PjD^)b+`c_CdN?qle#rR5y@jvopGL407_AN&XQczS#m z&#H}Q*kB{YD9))=bHrBXF{Tm3b!llFG>+COmq5lhpjlg))2lWQm+-$8)!P{fYu7Oo z+sYk$vsMPpgNU8oskicm@}_r&p6u+h+2E1rrn%nZI3!9YqpV+^h6iebq zH-z>hn^#YjiOO`7dXG{jeJ}H(TRz9{up#{xj^1{NjULOoVL8NgQ14^dBo$4)W__QZvbEY1B-{s|q{>+PaJN;J7`1*h*47at zcFN3*Wo;2WKD%S~YN+I*a02F)=D%S#fqGG_QJBi_VUk6B*ZLP1a%r z4PR0cwIv9rvW0~qOO1jT+xCxWa~ZJOj7^!yVg@UsTPZR(u7{MA1$UPK%&g_y%QYh~^MH%#UW=%w&K|nH+ zD=7p(fB+y-qL?s(2Ovxk7FYmR1>gYOU3tK|Q=>w-$FxeuskafeZfbN+fSEl!+N0u{ z9QnFK`z{5FWp6ivxHeYIN%`6Kuv@_`Fg;IGJm2AHZWqcUgjkp+FoRWT7tH#ECg5)I0?(_*ryQ@>LRZQzmhW4EV+OhE7GFh^-{B>%;a za&_V?vX%vT4Ygd~6N2~?^KU8cr=yR0RMKKCH#Zp~_X2i^4V!XCfFyAB3FfOF2))(I zBhENT(vTZ>1OPt>Bg;VB>s7-~veT2Q@|IJh-AGu(60bes(aJ8L`$P-0BIT406c53! z38Q5!#dmWMoRF-&T^1<%X!;cNW!B;-hr})ok{J@6cVbg$UYJNvrF8DA@6a0QEE%K? z-%=&qm5@Fx!(r7WIJM{FT?yHiNG9F<1;ClP>mx3ev$Xw_rr@bEg8-47^;@w;01oTEWrbx17HvIMR2 z7-vlpp1VW?%{y$irt=31*#6wxB|~%xY)h*eG zf#w3t)5jE&?-I2QPcue|!?S4Qw}HJQ#Y~{lz%bzx7i)S zCFSY}hsUKqcKNvBh7`5s^pvz^Cb5L^LT0vGjiq8YHkB-&_?)PpHyJPhBbR@ZuoR7kjOj_R3)H`?K=k+mj_CWF71!BG00mCXcg9QK1rt!P?SViC?u`-S6^IMph`~Z>FOu592=lzj_DCczOAw6dO zx8lZ(v=uT$bC42BN6{G%P_1Ff^V#7wzOvH*4;AKyam9$@xhbbnQHvq3?qbDI6Rgr$ z*%)uGfBW>nIIK9jhtX6|Xr6^%5Nm)OoHrfaa+X5KD$C(Gok~kiF?8TNlUC@VO(hos zVcZeb?<%}xaa4<_5^_l~kVU+UsqoPS*E@8o>^N#xNvm2l84T-;FDOPkoY(YnT1jEj zd}L(FqwuEb1W-}x;qcQ+p>gFJ&PnhtDG6kXBsFSSU>&V@gJ$Et${$W8Y2J)7&o3zS z%CGa?zn@0udj+9Q;Bk~^4d|lLLQ9`kB}W^;+sJZkz0lYYsl7|8Z5dC2N{6kim?eTm!rBO1&HISkcsFCrL6mvO3ofUY(zZdffp`P-x@x zw@X`J)4W?7_)f%evc>3}ho=>>`_lx}I9;kXJ^3cab%B?NT?fxsNp%(}wW^B@l{ak$ zvb;QR+6Ib$c(ch^lcA9ADl8KpdJkdpd#CdlC0KDpLOnDn9A)&y*pTzN@MR z=^~Yjf+C!+za9%nR3s+NGSkkwO{__PI4ZDe*!3LL=SM2Gdc`s|895$Tnim8Xol=+J zS>E7{37@TgKL$?`qK!JH3BajgQQ}L2iKzFiTxPEkoI561tGcTI4f-X(_c(}6@d(w; zynmqGTMYvvAiBe4Dm79jS6_WwUlZ$19<9i$@-Fq{AlaeO}nrL z_(ec@0Z>^Xo`^vYRXGAEZ6*h1_)Yt~LQmm#OJ<9YR`yi|3AXT4K@@$g>taYe0CmTF z;qft8=EY_wt$1!*IsN6agJH!DjZM%om+(+gOcavktupppDIbmv0H3N(LD56c8|)(- zYNvq(rLH#)c$d4v>^~Y%&wzQvOuTBhW{eN z!yM~k+06L3$QwD*!B)x(wO9`SIe9F;rcvk^fM9Ob8mknGA%*|8&wI;QfhRb&Xn&D6 zur58P|MC9zKP7ak3e(Nbck{E>T)X%@d4u?Y7^ zFXucScu9Cp+1#V0QS(hL^jYM>$SlN{_EuS)aN{RZH93zrxM=iCZcfyrH7;7Uw3L2C zKwA+0Z-4K_2TI%;8%{4b!WvkR{v;*;75(4X_1UW--Mt-pED+KtS!L_~Y*Mu6?(0lU zIQnL6bfg}^TJ6SRr%Go7#Tn_OHzT>wvORmSVwxL%&&q%FlCsYc7f{2E6IAe!)jxo? z6YI|(bYlw-<6py$F6(fUZmyAd?*l=9=k|b1HP?t-u|z^AtL?KeBty`WPW$f;Xw*Iq zehxZfFaiJlbt()&exh|@me_e=Cm|q|MV8S_9iC0433x4l;FXqElktIkD+&uLa zFToEIcHkw5p8~8NI^gR7MNAO}OYgp6ZufM3V7(KYPP-}8!T?~NM$+H%PBLQf7j|t* zSwBg<0_mD)8!O7dLPLJ2}m&B z!d9pM+;AXUr!l{dL>7GP4B*}9bbfM(G*J5%SA}Ri4t01G-iar@t8Y+3as$M(jXl^C zU}D-pmHx6RqD?benDW~Am@rJhIu92(Wyy2?NiLjjQio_$`t{SmxkeI|D-Wr7i#zW~ zMA<@55wHmpBi$K$Ky>%aC)A?3snEs~8NiNKDo7(=f{}!xW+7Pj(@+%ON974MWJCgU z?z`;84i8@u9!YuUTs-H^q|#wx(^?o1MWzi>Q>F#7?;TNMY@(hWGEifv%pVNL+tdua zWX&Rw?ajBWvOIPv@xNq~j%&f+2n@nXt+tk|3cMO?)MxV7p-x8VP?uoc>XtsJj@qK6 za=SP^!Q{+fjJwHQ>S}M7pS8m5X>{=;df~z0rPtI-p(WYP%;!Eg^*gy7Bj_n?2_#DE z>+Q}brx(`V7n#4|9n*RDV4Uw{Gh&*7Ow#LuOJ4}J5*rgAHWVbkcNGe|YvQ$E$9Frx zlDjo|z(WOE)&UVe53K`9Cl!@O-hDG=w}|l3`z0}1LG!O< za)OEc>aKFWy>|<;+hLmr6wJ+3g`kzcwvBkcIi?+x0{IvmqWU2~xZurazH*s0D2TS- zR2WjoEG5LR$e5y)_wtSv`GC6A>Z7XzlDPY-fe5L|MAh}yuxHvsM5&}aW986<9*~iy zn#?FL1$b8<`Py;}9m zgVLO^Su%bgzl+J%G5`Gfy~LdkVWSolTBSSLv7?GQ> zolB3DVI^wmFij*S-VC;O^__Ca0U~2X#2pVJ(+yfn=P(!<2EwYZmEEv38q>SaQR(5m zN0AeOhd%+KUp^&7G#6lG{4LUsR+N1sn)X~g(Zwdxl4;^|j9916US1$P>l1U_?d^0` zw%Yrg<+rmATB>nDBSeNp&LHEzzCzDQc}@1EMM~!JP_ox?zFGe!>g4%Ri<%OnfwWAj zwmIi*e=57J@|n%LZMZ(H6(*|A0IL7(d$=Zx?~~MztWC(cImiD9o4z9@>1SA>KaOcA z$7#a#8M!Y)rPv=vbrrl&kCz=~FEu}<;g2^o9s|2KS3;Rg-{q^4_(lC!#cLdl3Xc1D z_q&9}VQx;$#yo5yHrHOp3Xv5Nfd=P{Pi^>OC)0*ueaS9jVI2BnA4-gq9!HX1L1(uB zy<=08j>T3Eg5rkcM81Zc#UMZBY|AoC-DWA1n3g1H8mB<$-%@bE&Zms~LXaE2{K83pGh%xVvWUAl5T#VAC61jWt`9Ic>9C!q4U`m|NN352?6hGPfN$ zOXc+EWw({wBqLtNtp5jR&kXmj+DkX2FbpyJ$P2L4lp3X$^Mtu0I)$X zs|@LNp%Y!Bbny6A8}fc7%HTOmmol19wlqCnWlUopXgUPHU9r@@G*uXH)(Fc zC@?MpP7{q+q?#s(Z25oqJ)+sC{0N8+O&LhapE;g|7Ep&Ah6LFJ9--8XE`M?itNdQ< zCfoKLSkqpy{nlP*LtSBu_VwH$$T(8R4qh*)g)dot|bBHts;r;KUnkYh+aj4)TL z_~4_G4f<{llpmQN0T|sKU>csX6;H(%4!$3kmFU8Nne@`n{v=_+2|W|Iaeie zKn){#E10;Ee}0OIXoQbeTAc1Pt$mqW89S?1v1j_;K6OIYn;V?c)ySkJ>Q}&U9l!}+ zZ$?~6NuQ>f>n=?jINiuVq1%E$#YxQBLu9E{M0C6s9>%8ATqF94HVq8tf={y*7R3uX zcKp$x!i-r;kHyZ6V+guuMu?MaT|$KiISKk~XPK9{Lx`mg6v)Q-@AZuD8*~cHJ7E#@ zrO4+gK=>{I_(^oYb5=S%P((eSw3^SmN<^l%tBI^RbW*iW-EjfS^e-p zYRzHD6R=~ug?5iI27VED5Kmh79li8qR6;LXJ5y{WD-UI!&walnMY&f2=|7@jPEj$G zvQjIADkFC2v+!=p$OFTgMK;qdIa}?cADPpf{?o2B1|XO?O|Huw#Bvujc{Ze9KBDXc zFzpy=(|d_+pnh0%ww4S<+kA@8wh2y@JTl>0&rpTM4ir077D@)S57)=M%qBL{BbQcD z$O*++ph91+8J}MdhU1y@n_-A4@_kxl26cs(ZQ=4vf=+Ztz?WQKG;wf&(B*5EcdC0QSmMubfa=ZJ z8r!YG=+2nP1f7l%IwbPa%R-T#K0oI7ZB;6q7$Bv`T*O5#xPpx(;b)&+r@WSC;oH66kL0A#Ad7XnLOYi})7X*fU;I;2B&jXHLo$z2vJrJ29&bad z_s{t>bZA9Vwam)Dk*OU54f3f@7UsD~^7%RAq^m7A5%Jv*M)XG76k$Yliyl;^v?DR?bec#&UXh;|blQh)hJu!-9mHxL<{08vA^G_2{PB zL=FcXM=R1KZUEcVhr)tsqb(KlkDU<>jR7x6veJNp?e|4yxf#Zvpf@~EyXqEGE*Fc` znvbnO-5?BzcrW#txZyZOl9MN2XKxyxTTx_rr+I#%c;{8RD8u7ojGd|;Adt$TFX7gN zdS%!HF1f_yD*w$5-X{sCi}#L`UyKIG|C{+n(21z|C_~*fI_~wOp82iK^E&C@b+}|F zkEl#v+|CI+p>aI)Z2}Bqg8i}9jB{Y9S$O8R)EF7C(-+NF@S?Nyf}!C`(4MnbJW@FL zu>uga9qeVsy8H<@m1L~+D$@nt*lPvjHPN%{Xc1ZS>@vwR*Z^Wa54%zV3(wb*O{%Vx-g>$5{_VTP zl@rv9`?4zKYsDYRXJ@L24PuaM!XTkBm^%;LmK6EzP)0uF1i}Y(#AVc$%e9#Y0L`ZF zxS`BP8NB&>Q&Bk7J~4vt61?K{7=fuK#scdzPPFO=wRR2aVgj{HnDh&0Urm)Z2p<4$ zun7ap1~JYTE}47PyUN7Tzp@MWalQjd%b3^;)i2IEI& zx+yBs-e_5Lak6E*Af@gtLColk2+G4k{4g|gx@6#85BIo0R21F%zZS14{4a#HGC(07 z?CQ(hh2nRNw-?-3`4v1t?k{t1WB*T6qG^naDB^frxAYU-(iqy<^aiHz5rBVS&9OHs zbcRKrmlWNRSQhnp#@&Fm1=yj!PjCM)iOK)TQz$)AS#Nnqi~k&|+%(%~qSoGQr4XRu z&4f}UP&vWO%XTCQ)=;cK2ZA>)#1@z5$M zU~Axp2aSn*B}^lqqrgeD^ppyXLx`v&PI#?x$91d_b4f~`?dEpYs?I5TS)w3|g8YNL zn6+?D95UV?=8n=AO1_YS5bo5Gs*R#6=34eg>kj}O`=#t$kEXqtfoCm+541gJlx1jk zx;xHvCT^x-D+qX=!?yh)$*(X8eW(|WR)puDnFb39I1C@g;#VZRYTog=tV=3t5K) z_k>t2fou=MpFBwp<+|Ezl?rCM*v^gK@yRLiwV~tVj6_pCoL?RaSN+`mCMRFZB4jbg zUOF;{*ldQOUi32o%h<8f=KVJ(%~^qh3lzES4{`JZcrhaMityU52mi2l_Y0O(pew3; zz`pRf^CET7fU}N%wBQaM%jnDDufi7BQvrVCz}%ZhPEL&WZ=gFS5OC5d%%0d=`Rc6e zx)qR45WOFiONg(RVS~-6{efGiWJ6TqWE`EU7g+vL5aTG?0i%yr36MUpCEzJFS5KO; zZt$v7)ei-4;m?#>m^`D?a!%f^IM;};xl zdr)u(b*hVjFv1x3un&2tLuk@1wEkt8ZgJ|(>gXVmcM_8M7Jo7r8zUBTSA#xi-Y49@ zUCJJDW7Mz-MC!H3sQ(h9eLnDgeqSDX@Nb4~YblYk>^~h^HYXfnQ-sLpmPMZlrmI5w zG%LdE_De?Ry=9aQQ-knlhI^A*EJbb6JmgL(^Oe-WkX8G zo|g$C$Cfv63){UduA7!3KYcqYHWdC2muq(7z4){0At< zHuXDB$1?Q>68IggIIIGQ5Kq9pZv4B`-OVtbY{rnnMj6!e<0d{?^2IDX@*3RBq)exP z^ru!#%cL@g2TtQ{b6$>dx3+p#usYev%{|&2U+C1!3LJ)@E3D7*jsfQ@*mcuEJU>OK zHa@dK;t&tKPFI{ln>fES1bqrk66>HH7PkcMUvR`9#GowCBDf#&;Lxi-mr^CNp8N?& zu+$snQBk7n$^~O%dRa`c5?;`w9fbBwUv5J^wHc2x-mA4cCHU{^6k}N*#JMgo#OG&^ z^dpL{NnK%9qs1yQS#Ld&~Z^NbKsY9hW^8$dJY*A(WtdN6LswyhA}6W@~gqBV0%JWtK{Q>55- zmHk3F`JM=EzWXGuwK@v^CU^arW|>HPBTGJ-(|btM9>$B^kJ;E-!-HdGQU^|t^f4^0PVeyAe1*lXapdEefZl7>zwxEnBn{A@_Vb1m|hBw@P}}1~Ll$N=(&|CVj9#dhV}bTy-tF zwZE5a2@_?3Rgkyz0V>ugpu<<3d&O48e zHk<|=eNvI|WF z3-j)0??ThBqcRl~O?JfcZMD>5-o;ub?HGwA$1wgk8N^QDGw}gWU8{NaWoP#Q5Xu2j zVS1GZSxECcM>-oCcURH~#HTEJ4{G#3+`rGKrt4A|3%WRHoKouGHmu+04%yE+89(-Q zHxUAopgm)bpMAc;hE*GtR+-(k_p%~kd!T-gYT0R{JV#BHRVCA6yOaUdPO}}+=V6}u zuGj=A^v7^c^bM-ut+U%4 zSPjNw58uoK*@c3vGO`gV0YU%>Mp2_kFvABRNDy8Cbc=(;yDfFykX6T9iS0l8AX+hC ztY-9Rt&nmh7|Z;I>)zPE6k@mP3^?;f>sLt#wnRExxjVc-iuQx1d;hPNu8qR_D?8Z6 zPn_}1_SO1Zb10!tH@LMbv*$ig35k;lp6rHcu44V`!_BsMTI-S(4*=+*#QpkMWNX_R zHOS`d#UXMe^EDBQ)&WAy&7r@=m%SqR;y>`OT9iHS``JejUIy{fT?$Jx2z=4 zUew$0eB8RQ6zktBjIeQEZw=KEBKmuE{7OoIF+ZH6i|73uUMW|PJ8epMh$qe(y=4wb zE40Osra&UuiMP3$B)bSGSii2B9)3lwy#73xx%ek)q)rnRyZX3{Xb0@pqp8;06C5f# z^QY;BR!j;DJaL?tMtC;zbjr4L zB)Hr_8GXRY$bF%?fIS2SxYKnQzXz#o)Os9zXMjc4PQcrq$!9vNNF8RiD$%YT@`Ki0 z0``{J5d@z5g&?4)|Ew5kDZS{vuwNcV7?ulAbxoOtdO7fR!$tlJe$kOFl=zr>iczy= zzle!I=a@j7cGP9I35zOya>>fIrUOrrgopdYZ;w^H6?Riy>r8VVX9pu}uzIzdw8 zx0l|fY!S`~-)HoER384Kj25e-2m-f|wMNK6PV0qaG9H#PW39=vM1{)1l!f0PZVL9lVDR}JX) zy4CKGCw6N|=dyc(*poP1PbGL;Wr@Xg!Z+e?WNZ83MQ5ZjE3$K9R1RNIwNWU290%;$`BNeqCkKJ^?_zFmBs2*0)>9B zUBCDdbMPGbVO&>AD|iC{Crgs^)?gjSJI4T-z)^(TQl;~u6F}ks@X7}w)n85VO@WBe zZ#666@mNCVZ)$qrM(=jh{~}QT?16*@UAI!zau(=Ijpp|YZKUE{<}Cg+k9-|YprbN# z5>f<@5&yEcC~)mcz?q&0LUF*QrtlX!4PVDthwobW6C4$Es5~Oz=i5AA65aP)OPH>1 z^xVUh0>OD|-Ml&Elaf2dp=BfsIgiObol5Lqr$Dn7`D;e#?&b)y-18EJU}CSREh1-Fcu7(i*v@NIJWykD?2_cSUxcdLj! zbzXn%LZ@*3O{0c*r(W>k29Fy9!|C&R7v$Ro`N^$kBX$;Q9jYi9SvCiII!au&0c~|C zzc$4vaO_nt;7zmGD~o<0W|N<(G;QMhp`&$h19wE*XXTfWiJx9*ep{5+R?$)qVq66v zL;vie;54AzTc9*L_^dV-#g-U$=0a<8z*-M4KR0y_4bKBaAEL~pc61Wm@;iJ}3hjhO z%zsVf$otL*f&Ux*R?H>YE`mK(Al=$FVz$^v$uP zWp^gFEfz$&Z!&hEh=~dFzf*K5fE+nlAS~dZAbxMCzyf=vkl4hM$&S2>55@BwU`jio zD$8o`iITs?@A`>Xq$qW@bDAJ#o?UU=2Z7pT#WR7ss*jGWG(wqcrX(H1 zU-DwE-;RFiLVsk?Unu7lzk9{@RW2(5kTy!ECG@WPhA4S@g8q%oyIE)==J90D{Wngz zb8^NcVy0GZQuLpXNDU9!mZ?tiiM1Q~;kgbH;P4}esw{Pbh>5lXmTPxSr|71dZ~8H& z!E686=71|;C_PY9)KUdPsHClCvZu#Z4@5IpYIOGG(*V*3yx0QW1RmHiX6V}99q#32 zw3(TqU@^^zCTmsaa`_n%>2NEVKM7&o1?iJ|svnBC-|^L@dK`OiB3|VY_fn#9l)!A| zjpgSL*jz|j@Ood_#o8NgP5S|=oCs+AVr`@^x9DeQ6&>t;Y^N662$48e!s1+?1>G#^ zu~qX7EpXf3DA}$>V#Vr20CiQ2p>52c%w~ar7Vsua7Ur$+gaO38pLU7F_;34j*uzKZ!L@H=Q9(RcuflzUFhq;V(Br_9k)xU!FU6VbJC_awKgX zJ;iW5h>97>yI3z(YoJz45h2wNF$FH(1OR5bO+iAED3IuRxq&@oaA%MROMFvK zPvrt7sCV0BNYYF-H*PaN6A%noZQ0*XF;~nv=}RRqC`XqWpTWtnA4B=Ue`V&4uLxT< zB?C)hU;yN+v%|bs!(}@NCI4EM2nh=EVhn=y{~YScVejpEY5-;N-W*h_*D{?ZL=jSG zxk?!jI;MLCDq0-j-6nW3<;f}<5Q_1}bd?D~hVc3Q0Lo6GOhM&)`nUyZ3*15tvE!-W zu&$lfW&O1J!gXT!hX}?9I(~%LvO?)Xj>}8a>tu=-j+~QoG+^c?$gmv4``Mrx6waZq zW0c#^=#@z+@DX@2&SzCjrWBp(1kFXt<2=B>ch-JQSJqV_I}R@?0ycb{jT`^wKV_Mb zA}u3Znx~@Bnp5`XAl&R|)Anpek}^*qNct($W$4*1n~D&Pz$P18I=rPmq8`wV0C|Mg zFm7itsr1ELH;)9UIwGcby=XBT8nr$FE2fI(Bsx|k-9-L$+N`! z+giW&;^Ag1@1R8?X-P5hW4}A(IPTkngr4w!PNV#hAf|=B19d|Db(ZvnQ8xqcpoT)1 zn%&YsW6pQO_DrV|7w!Nh7M6{8!cN#RfxgLJQeNs2u6Dn44gBGQT7-f%&@|Qn>fZcL z&Z2uOrF23}Avt$qBm;HVJ5ffi@!TMR;wZ_Rt#`o?_^A#hPj(UpW39#7T!~c2hna>_ zO$yjCyhsw#m71MSF=%7OHC!365+UdG3j7GzV}`K5Q{CU$oDH{V=s%<)CZ9g%-O6_= zVsk75(AU<|aQ}vZv4E!H?S$yh|9CZ2I>UJTJm`R4miYs*Bz~rs_o1JM`3(kO_C=oS?3PsX6F<}0}*RZTlpYBShjSketDwFkF8Uq|_Z@!-zZSlXm} zd&EaY6@1_llrBaa+F8@kGVONcFmcVX%ll0;yb~U2P(tJJ1t1-&d?4E)X|@+yD&$z; zU#{!hqY&=^Lg!`X{wpNM#q0J7DLTXUy(jpf4mTPdUVwl2i%7!W)y(F|R*v zD!Zg6!o@iDRC0ua@FxtLR4&pnbFEVR&~)XI52BP`RGWIiOh|fbFPo+K@&Didh^gTN zb`GyG7DgHd!0jx!Wb6*{xx%Ni0s)YZ@nzmCJx(9=?Nxh#Yl0DiD`|;88Fo1IXMJ_! ziy9GEH5GF_^NyuTj+)3Ha8=lH(qWBjkTa(s>Z!_@aF2121HkyV0dM zGXn&`Ol&-Ea8?`}M)Bjx{E_Gy0oXF+2r>gO3y_SXM}QDOF$9(c-~iL$se7~Ycz8R! zMwT}4^*Yno5kRRc#f1MfHd6zpL0$R^bP{xx5oleCxZF&M@N^8E$$@KH#Vn+-3Ei^W za^22wW}QOIE9PAY3dY2=E_D~S0h<#9bxP7TR`6{$Rf0?2Yg|AehC-~>#s*mWo%{T> z&yDrHE0jn$*2R+0LebJ&whncY>K9f&w4b5|4>?3??5{@qsP>nbps>@){p}{UNGnPO zg6t!3IcxEJ)LRl10V4er#%xe=Hzc%N&Fd2iVx{;b#1Xa7>H(%W#O}|p;18m34_T7~ z*v8@=|B(qv>|7u8yw35f@Td7+MYG(%=&qMzVyrwQ+Wr=Ep zyN1gX=GX~oD~Gn1v9(P|?Qt4`Mv%9NKXer#JR-Mw;nI+j&8N14%iLbd$qdM$F#YB&a@aC1-b#sX!|e23&b3HVtNE>^Sa1l_`n+wBbWb_Sh{^k zeRcdjrTxtSbh!0;%pM{6^T8N|xG8|lCbcoSP#m>3bm7B8|H2nF&yezzAXTKv>+g&j zLDjrRY}I|jps0LMCoKHyd}rg<&R;N4x>u>G`2T=3T`he$GcMm^m`1TsdtMvl{2-Gb zEd-$gj(Wy~8QwBJ1DW9BOG>pg2;pw8v@%Ac`SNY6nx-w4@co2lGnF?#hr?p5(;f^> zBX(kHafG$P-(QatWiW!xN_0!WA<6jgWR)@wG$?j+{^felUO*{zGFyT+($c3;e3zQT zfnDy7Ws1qv_Hvb!6_G3oGSn{(hZy!=V(H*jBknXS6{x%m=yrCq}I{RsMFWV8g#}!u(HxUau^-s+9MO1KRQ2h{7`^TDI ziC%Rr@|0=wDOboF?k;7s+Id4TEdS|f(RKwJgq2HQes#1_%R7H2{7{P#iz&bU>?aZXRHq<3LW9V;`3>_N>E)=5ycpmV38*5ess$de}TGZyfBw5~rjZw?Xp9CXzJ_GxA!6cHM z!P|!~IC4Roud^+VbY5{&8?M zajW`v4E`Bg-ZcxERHN+EeC-=oOy zmfItGOF_ZtThG6*QeP2XMfC^Y4Pa^5@a+Cb^1`Fk0ot*k9vYu-sl49PkhK@C^^2QP zuF4ef|5p%pq!2*gaz@G=(}m3kd-8@~BaDcF&gLSTE22>Y{J1*lBk-tYd+xJ-3rZ=( zh>~j4jw0=GYhury#tsIJVP~M_DcZB3an7(3y@Ek?d8oQWD5} zKLlhUp-atq?-Vt09#q`0wzvs#l7dqU!%^KZrs~C}`H30*yU+kXoW32-8a`? z?8KhIN8r*yWPOPIl}XN7&Qg&|3P`oF7tp?*=0eS|Xry!IEoaeGUHr)QQ9-sERAL+h zVSJk=E0@-2c_G<{WNX9qR_#iMD!#sactLZ1*fLrm_pLShZt9dk;xg~Rsf7A1tqI&d zQTCbZyX2mkiPVr_4+F zKAbJw(WA8n6Z5{0*kvxVc*t`Qm430c#+5`Pb^uXL(eM8RC!w}4jtHxgtrN4$w~pdD zd7pE;_|mEmTPI^wDI`*q>q;51$>b{!1BW+Q)Npq_(>f_a2BgEmwn0xAmnUbe6OjX? zC8c1g|8yqDu-}#bpRBY!0eexxxY(tJBeN|R5lyYF6HT)ixbzOuenvEIY(%gZKU@vY!ryibe0|z$6j-CPzQ_MDQL_*Bp_?} zPsS^$S}$cMkE*pUvFcdCy!AK{rVOK)sIcv}7BKZ->PVFcpgd_*`~`CEFdqtg+e7Vw zlbdGV3e;3~3UUQIaiiuU9aK&OwV)a=Am`j+Plac+?^c390$FFF`LPQw7kn}T1nH(9 zIQ<8{c2AUH^pKL!UMahr5Xd{k%Niz+TYDBe!gAa+=!ONi4lvqRPz+YyIE@_5<`{GU; zMY8}aTX24Vu@qb6_rkMNRDEI@PXWx3-f(v1YNMMoaxrB{JdcA2GxCI#x0%r5>w4?| ze?)Totdw-W4j`G><@$=86|x|V4z4f^Ta|^8r?ayemi@S02EEK@;B!y*>(k&SY%moi z=*z@%1zR{zIJ&lER)y}Ch;6<=GcNsnds4`KvLzD%J5awdH(}w<225{QC^=rBI%)IU zFu5-M5zN(jYLw24V%*ed9}G`BePS^1CT1AaL!17#1YdcevH|(f>R2*tz*9i-kMT`N^k;LEsOu?qi>Y(*L%LE4yJis zn*8pxG=EN9K${Kjgth|MemwPfp(irJ%Jpsx2s&&A5(;-DDoIrDiMVA&pls|H>bxyv^V3BMd9buV0y(3J^iq2-w5NOXtin zL$&C5k}!8`mkSF|g+Xr}I^I9)QHt%S`%b_Jo_ccEn2NTonqE_qH_V`m>m!sE>rW%R zjQ!EC{&4_UYMe%>CaN5@C@aJVcqmEhv8@AsG#kQZyCun|VsVrVz6R%A-9BI}fHKGjG6f_F=**)>fDk})1eO8d0O=8z z2>6bkx>~U@D3OVPJ1!2Xf|k>HvHRu#|HF_{%&V>P8dga?86;9alR~~mlINCe&o&lm zNrSL{oDZI=dEV*~J6830P}H0z=?ux=ai1|eUt-Y(_b0{`36mI|;$&Fo)9i1*g1*-U zD`E7U^vG*`L@iq%>+Huua>&G7e2eH&P#5l%h>V~UD}~v_Lq?EmD-hdTuPvfGc}LBB zim^Dj6bq-^qIg>2RT!KtjzjaX!3;LX&-y0~n6 z)a0z_tx~+*Emn>oof+wV7$l1%;}blo7_c9KVPqq%?1i5a8r#&uH%43-u{5hOT*5Qc zDASlJlh*)p;a7lGljbV@qWKeJ=GH>qL6;LN zdElr*;IxIfs0l^vDoU@+PgTszD8&T7gwEe1%&^Q#o(}vOH?lc}vUwwg#-L#~`Kj}5 zc!|nUo^IVzdys1FqpJF*A&Tm4Xz$+wbo<031u#^h3pO@Ukz82uoz1fz!ZWNRCF2j> zTkhJ~(~pAG6ybfA3v_dA>IG+;$(al)4E1GAke?Bg}VJ* z0c0w5^)i8#&9rV~54&`906FSN1ByY+Roh$E%hbc8XsOu)kvHpP$hSH&fL$umz-tRO z7x3>*tOok45s2f7%BH9)A5S{+V@v4&G*!O%y!Rzg)2=A7EGN%8}hh~6cjXIt9*B= zGpSS6P_=}Jw`&bIh@$lOLW}!}WyOr_Z;ySOLY3~?Kzx@rO*?qe`pHJFz&xC9x59^G z&5hPdB`{cyddG1M5FYla>VtE+Qm68{tbk*_?nWRUjS6VOQQ;W3z-V|^1Jnb{h{zjs4X z2QMYYIO=~(NbVC7**>&~;=44&e_Nlsx`sfdpdvtN;kBL($7|kao~2~0gNh|fv7>ex z2mg{et%5ciCHc;h4xh7ch7>QWucrI9SXnPhFNv!`AHB79{Hx8-d7R4@eaLHMlZ3YW z-)?S&$Z3EGPm|*VmLmA&O)=je9LH+V{8HU>XsK}j=DE{20Jtq$O#2X&d#UV{u8tEy zf7j6^$)NnW&sWJLssLEz=VTCg{W=Hl6NU9W9$JC%oI5CS3??^gPI2PJj0h_cGRD0#8g1TLXE3xC$${8-EYF+=GR$8yWT>vGZ+UH zbaQ<%yZiPH$v8$i>NHxB!fU%G+PuH*Ep=*0yRxFm;Y{U&e=62ig>P|rLgi2d-Mzsi z=rQFH*$eY-723md0ps`^@Ph!5Z+56uGa{sfT;l|D8$dOqdu)bLlEC^)4*&@8UNe#3 zy5Y%5P|b;jp8Y$mMNG+>GNqF;3k;u>Y7Gw&!zzT@WQKZ}^eb>LBlR#BX zRWvofWgwui?l9PE?7c6=0nd3H>Jhl&WcqFPl@+VR$92kMlS4G)fkEvx?yDTx9Qp0j zjkAVP_|Uyzm1aaq+{tG%Eh;eaIq9;!z9p%s(R=U0$?L(-A+ zq>Jf(nEH?+w}c)1qd9(I8Sjzdp8A&kKDrPo*7OJQYP)?B9O?O}$#Kwhv~*%=$)}S%U~qtI zb$t5TsZcfn!!|eEgt_|mHr_;va%g` z^0t@1DpbL^!tCO9>w`Pz&Dbp&ZnsiotWSV+dG2c9xKLqvKx95q)1KrN!#DI7DM0Q$ zRxKpKPImSmbyI3UZ<{KCIG$T-V?!3A0C^O%_4j@+RZw4ha-z}-WaNF6SSQ&F>9%AH zsf{}!)(d9Wn&p$E2h2zNFR!E7OMjER{(W_J$;Jf^DKVLf{(jz;9wIjz5=@yUK z2ADQ@1E;B{Q$*Vb@utiJvNkvx0k|^%7AXcH1NbUYqTn#Y0}x687Fpl``<0BvtNyg? z0F$4+!#4WIqnpnpTt~%Zy)Ub*rX|EZmF&6M+v)|f%z$<(@Bw$a^x}Q_H~P@cS~)yAA_J9m+6$QN)5rwzgCUc!*!nt46U6(eAr78!)c? z9BX}!!?_B+2Es7f)KUiF=g|m-@rhoz^2(yOT}SOC%u5DhkJf+Xk;X@QU}5&_6UM}K zCHU5ngZNlP9(u&nk2y;w0>?Sb5>Sdbh2*Y^HZ zcH)4A#$>lSPx(AOch)Oc06Y^ifwrt)#IkSTof#*elKF(!q*M@4o30;#^O}~Q`PqIG z^Sc$ml-tkO-e7Mml2i-;E}<1ICTl)VX+_e?FfigHOmvl8p^XsKJ)D|lMS%5_AHF2z z&VUCQk8uZP{J)e9|C)2a=h>sPY`v4_G1U()DJH!CZ`szpDOF=(7F`Dd zr%H+*fpA)6e++J`JP4f+-@KpmLVy6YiC&FfU{v5q*A5n9?$}f9M=U`W?9VH$x0Yv> z7EF}?S6`oZ%d<&lTBNjUUjM5ARyo${1Lm%PpKF#gXrF@D=Ys z?YtIqov1fV1?>2)On;CKE1PVdm}@~|=;80Da;l;o#;M^;!4VZ8ftfjs2xFIIHixso zV3F~D?UiGxHAif!_w%2C*0i3LHB%>S@)$jLc{?aC7835q+wFn`Z$11LvRI2_##o3K zh`aKJ{K(-y&7OL%{IkX9XwyfSlkL7eh1tc#bU_`+7-v=WXaqrn=Iv|)QblnYDR2E^ zrHkHlcni}62A6RjD&2)F;+bbP)BrOpi4&7qM3ZCYP%z5D>qQIYN&Z_m8cRi3hVk)} z+IKJ$3tr5~$gQa|*9B*c7*sX#ycIKLDgO_q;tJi5@ZBoW`aZ2>4Z_l>ycKG7$oo(I zb%iB9V5_2lNiO7_L|q8E0xSiK%TIIwht6vKJZZ3pE=kqFm7qQkwm5OZCI zOj=@+b3+u=#+_UNE+m;X2+?8J#YkM5GuiL3FP)An-s0!`3pB%V$^@GKSWcXkRgr{6 zZPB8o4)$m~LMNcZMah37gpu}w38U3@mw5gnTU<`d%e*~4z2+;J>q!^2i5pd=RS7!| zaikJmk`^xN^q%N#j~gI@$rc7_0?zTQb*Y@-J$xt4pOtL7-F>l_FDa4xc&$d3}! zJeq_O@p&-o=NbCQxgFD%1CPA#OZ~tsLS;B&u19=gb<$M^s_=GIiDS=_GT}%@oi3hI z!A8ZG=RN-SPq?G&P}7(!qeNMAFE#>hc35iu`^dV~VX0%p3;r(cX4gWyz2B|&GVJo6 z1_pE`ve{Vyf$u|R0bWg>hfbCXDAvJ;M!HHg%6X|=(23prt>T6iDD-6_RVHmx;&si! zbZZch2D?iDlTuge?vUd-`~YirnS##yZ7c4nLW}aQZ6Eyx>7J_GDb6rosnD49M0~ry zroz5PW4r-$%CuksneWfG;$hn%(#!Lc%DMclhs#Cdd4$T$eak{MLCH4k$mWXWx@hlIGYEPErf?DJYaE{auc{c zHtn6gqoer$AVHWqwfughx8nx^%>or7l$L?=_id>_Vtsw}&x1}C!Yf~G-q_H4+3Z3T zro3Z0=xR4p%J~xQbs@Qhb!_uS6dSpi_1ES?FMPHs5vmntmyC7gZJnLt#|+fS`a`*^ zcHJ>gNjAg8CirtWyiZ3)5c^xn4k7hchy)Ia9FzCE_vVMr;A*_{vp`u#j3`V@e)!ZV z>)U%`?Y7?G(+`*ZDoZ)m@rn;;E$u>gW6HyzUbCvzEB&}iA)-VCjs9Qeu+`cAR4L3@ zOruM{s5MYS_a`MHLA+o^M(VQ|AJ<-3^s`XEsDw3=aqsgR$Yk8?Vf8Jc&fy=jk#=#K zf^>QqPY!O1q6E;NiMVeG8N7Wc+K$YX&}s5jt9gQZmklDBv$2#r?KCuORWj=@!q zJv0g86rkGN1AY-pUZ~d0SS(4y7~E3Ab>W(60fGmNQ26(((Lgt8B#w8*U(7yQLKs!g z%@76}(}QFOUpy0P^$)UaUo9TCAAwl8Q6}^TKO}9~MbV&C3g5}vuy*T3l^TZsgN?$X zipJjA?)DATT!_;JlKX`Zvb{osT4^=bzo?*+z&1vUQ2-+?QftA7n?huXgySl~zn%WM zHMlItWi16n;6P)JA%VlBSUC7HB)Vbl@yO#ihYip@I>uyhaGIb%5h{9}Ul@Uozk81i z0HwL(ByVy3VZr!uUq31HA0}{!z~}u1ly0y3j>mh8B-P@>m@=_LB1uTGv?n+V;)Y=| z4iydrVCnaRdrF{nsRV#WFT6hG*kRL|jcNdL&t~`EwS6^ikZT^Zuj{h$?mD~K#D=et z)Vlv-Jhl~+$O%p^pMHLLnz2qtu|9Kn;)Ft1+UQll42hX?l)B2r#>HXow~pV%=dU-) z+R=Y;CQsp~V673u<1mK5xzkv{t%=pjeWMM8o0k^cy`ob7%w79IwVhJgD_B&>Um6=KPHf&E~n_j zDclm$ox(Flh^x<}_;Z*@1wsS(N>QUwF$4!7N)T3A-~i9|w{qbH31F zv$l+FMGu0f9CU}}ONh1ywv^-hRAXZnZS^|1P^`LzgPIA^7^9$vU2zwtiypMJ3vLw= zB5}{I#9&GR&84WfA>)7xCx zGWg^}s7eravGErL@JB=g#m#im$77mT(G9h@C^EoLlcjBixnpQGfhg!)KegSm4-+J#xhD1mv?if>+kvXl!Ql5^~xS`Oln-ukB38ma%0Qpn%-1ad% zV}B0gk>1~(l-fClul(qEZh?Y$ll#b&Q0jYlUgo8$5<$7bO^Uo`7ZE1)%Df@nmdb!! z+(%Mz&Rzq~jMY-q%9HkjwdBf?USENe`ti+gecNhX;=-=EX7U%i^U%p=@L){uyL&h3 z!7!?)caXte083?Ra5-%mdUw4IZfL6Pbh<5<=Oisdu^Cpz{z~oLej4^KX#aIqkU~#cH-myKPSb0Wry}tyS8m zV{gx1(w<9d-Lkg5yh9~B*M_L=$BSHWvkG2V+Z&F85*K6uGQNo-kK-h8iz<8LwMsf} z6DY=$u`F4igm-m#bsXPmv=1!xwBtq z<35*?sZzEk12N+Cl+qOh1ofK9X#={wZ?&%3bsT)&|M_zO0G>8Xe%6 zd0ALOoWVzcfR3KBESX_rRR7@sTj=5qb5BU>peqYwP-Mwgd7eM~#<*d!yPq9uqnXFm z;y3~Q_5cIRUW(5-(4Vv5Qj>%~qVsV5uV&tVq$aI%ojZ+DJh0+jiGRRue#bnai=X!{ zGX3Wsft7++XspW|ro0E?piT9LAp{^F9PwyNWl4CLuES~sj*~qw2!}Ut9`8TKXD*_< zBBmPDx6c=VpYq?tRo^&ssXbX|R<(ZEj8l{d7i@j2R9?h{L+`CS2_AGogxCTSFhBSB zOhKwUZVSLPTJM-=I%sp<_V3Fv3vo82|7cBAFVa#z)T)W=qoR{bK&aaO;Xr8ClHuplrG@SS*tIVdXssbdZkuIxWvP*}hNd-KiP%0O7 z35*byhaW8jt+m!0Y$rcmq;Q3Zq{od0joPWD#NYjS z^ak)dd952H+VK;2vmNtx+{R4wF`theg<^X7k=}(| z*1FDqW6rCoCPKZ}70t0SoHP`*wod^%w|URb*Xb+yugcqvVSwi7fC9H6`cw#Un{_G} z;WHxCSLpth7Bz7FshX)Sj*(jGSsdBae3Ek+&j5{a1aRx*gYhpC%J~C04+z`^9-yRS9r)k|dk*XTV`f?0m&{ zT1j>Lqh(pNHCGpq9kh^}trZe9Yb_FnO`}|J^Yg&U$xg2~RzN*5lBggM;*A#Wzlieg z5npkpG!ZVQK&j{~h}~nY?^N65x%RZ_9E4-H5lty9aKM4j+w2}sXEnBu)|5y|^NoBP zD55t*Jlfo_zPvAGfMoN41>4a&?C=jH4i5@}aGhXeT-HWW zT!s&zx1U=KgB^32$(@)FLPnLEJQ$Z#FB4B^uJ}y1STPPRr{MaUF5P59kh6^3c5uvY z_b}(%xJl}MX%#POZ=$in((iM@4kc}eL4>P(wp7cdd`g`!m^W48wl-$2X>I}@IzD`a$;(tMV6 zy4j*FE{+AOHAoHN#Fo2b{HgFy#cYi0+%SclYSqWmSIlhZB~ST)+W@g*D3Uzs=Z zhrEIeYHE8L;72sQu@e^t~9GbQ$w1Z*Y|0* z2vKT&+?{$VEpt?+4D|NL-0it`HumL=x zd~^O4i5v43m}!3kJ%}iWX=g-^1Ql~kcib@4x6u)s1^kWGCBH91MMqy?6>~N&39k%A z|FDHo3wvPxA#E6TvbgHvzFh9PCQ$hyNV9`LDu;_1+MXOVeQ0_CGerBs!ou#^$(Pt8 z4m+Y^m|Mk(J_@fu$$^b&rqKA$o_6U1e8c8XKY($P#DQ~F#Yr;M2=f>_r+1T>AWZEN zz|!+W9%UHFtko3@r#PD9V`=J^bF^fx_pRAW!LWe7RI_w!mtC9^ZZG6%h-`IWcs;Fq zIYo~=Y0;?6c%o^OBDp}(^pe?R(`Uo99(%f#{~!d6rwWroLh+S4ErV5a1bF2ob&jDF zL94~A21z4p46r`I7OQrQt@h)n3Oq=$EbHe_g4q7OuHJ%cPH8z zKqEq5lGzTy`}`8xI=!vdKdQVuZUj$!Y4kX=>WKL1tJblPgd`rTH_h0 z({`Xvk}2|qBj5t?FLbT&vg04qKmSO@y4DsWRFQInXJGwZfnp&bRTZa3QAGzpq$tei zF$~%drc>jgy6pkh@U(IH3@P6*gmbtXfBAvA^8<<1dt?)fWgQK&LdXh|T`0Zxy^mD^s))FuDsIcGdmpRCczd)6XS zCy#RNKR0}^vQsM*==mdp666BQezarNHlp(sEAbfhK*v@>6yck)EbRG{5!dtljhfzC@e;_)NCRmG`|!)uh(a~i{u^hQTd@;VK4id0d$4j! zziR&oLGcuw8HZFFt%ts4G6QLb2jvfxDPRClHrL|sZs>c%2tvCqoBi;?_*@xEoEmmd zPOamnAY-0C;u)r+Zyvhgk3!=T`3)?0D>#F1b8f0}2g`D$uvddsi%7^dlRdQ# zAg%L^u3c`vZco$mI)+#J%Ttp6_2YzkfRWNguP2j5$sV!I=YznUv)w>3CCt$}3_(?g zNRb@*Z)jI9*fX_|+q}VKK_zul3rF~&R}L|=XDN}kmugil6+bHSs|0dhES+q}!S9V< zutQK$qQlfIQ^sq^xCr=wkmIuzm?T@S;Jn=}C8+TnRdpy*t20+Ni|UGJ=pFA_wWCf= z_yxCT*z9quBPY60p7$?Fg_D>d7TNmOpAL4JALi6DgPu@VD2<)TkF zjSiewq<)P(9aIPL|%d3cyyJ;_i$c@FxmV--yQ{Qv2THQcP4e(E+0+ z_<({_tVg|!o=%d$8Ue^M&J8ICAcXWRQKLXH1P35W5LQ{>0Le}| zI+5GL-dl1YtsKKh6HSsBO1^`&cQbyRCfJb6!+3UYroW~pHII}f;Z8nH(t1!rWfx$O zG)|Dt_!(6T+K!XTZ)kD~0@1{J!@g!gWA6Ic4x7CF^i^fcgQQNq&V=C(?$Z*nT^*V+&{&5 zaaQGr5Q4GN0|O57;xjBwM>7C%VMOXo^3r$v`CaEzo0kT@(ao##;_q2ItF}zd=_`G^ z0&M4Dv%(M6GM8Af7$>+pKtRFEf^MmmQe!=Ww@tX$)JG}E$_+;|$7M=U2%wetSy4?%dWW%qVm^{VmzFjSjl; zVr!PlJdx~2)G9yv0-0J59Wg8_%5=$lAucLmeyAw)XX0?T*7&$zvRGkprUGLmWu9bv zd1P=<$tQFCLm4h1k%;XjKYgcs#aoS_^I6TuW?Yi3*O=Hm*~sl~@<~o!D0?l2Wsr*f z7!&Tfi-N1y;7Eb@{`)7qa#VQrAe1|nl!Ghg5Q(PJ>ezCpYcl%}CdUuRN^O#zmPH7H zvKy{f=fn0yJsg&eb2zmdUKLK$wCYjwh#HnCyj*vEsdqs#Q?Li3dOB8DhY>-Kk3mU< zqkFII9#oVvt3N6ysW69nRuU8}CZYLsiC&FkI+{X*5beyvYb)QLvq?MdZ50V@k9S z&m+<>|J47fQ{{?u!YGFr??LP>6Ud(}LHHBwwag?F2l_~E>m2)&^?P55gDcPNH0S*f zFpYS5%{BoF2Yx8{==RHWCPnQBN;fvOzwa;6lnH=)`Bp2W+MWNO_;?`fr0dp-hEO<6 zEnf)r5@WzbkR11T_B&U39^d4oHOWiGB|;rr6vH)ho+QqLJaf{vGl$x;c6rq*E;%|O zX-A`h?l1R0P(-&G0n5fSdns`HKG1xOZ-MvSL+mgG)u#4A2x(O}=ECV`6KOu#b)JQP zF6Zq}jO_-a6_)@?V@N@!HUMXU16pAmBE7@m&>7*6S!+N1JF+>Qsw=S4$TfqYt;CpKp9={;zYHWhmj-_>G-|<2=hwky zjh)#a0@Jg+)7a_m(83yj_(h=Pv(Hv!yJYV%aavEG3{X@VRTe_Ykw9-4fiXPztj8m8 zrEgM1?%_&^{K{D_&Q*Sz+ga01=V@WTr58aiuYppN{`=~1tQF9i%Jm&1vr=BiOZ;1# zb1TM5C-A&@OYyqeSd6wRBL^T(wt@qo6UPZBm~?a{W33-j=ULz|ekDFTSA(fr+aIk^V{J1+XQ` zX4LS!@B0GwQ6jb%a4zMHRl;LNKr6Uh?Z4p!Idm=b=4pX!Ym6o&6F4s|s955kB%jp{4R+ivNDpYyiUQn6H1q3%wx_p69)3gWP^ zKRk}q^q$jon?LwV|B)-by>PR%;SDDlwyS$ac_52vKuS}r+&zj~ z(U5%5GQ9{g4JZldXro7f5I`~nmRaBc?dmkvG>Ij_Pk~SX+Cd8jd(6RsHN;d3-*b86 ztGx~Oc}Q!_r9H^L1NjoC4wYBq6}2Ho77`GSO!N!b7eN`a2%HrVz>fd*qKUd+RMZ!5 zD*0Co$DfI*f|UnR{;T2|c72NP*nfS5SG|3HF0_67}YQ{v|v-ns5YPtrYj6TEGJ)v-alV%8zWr_~`)v zbm=UVWV+h5KwP*W9g7i3*5NGZRu}{sW;VRj|IBM)CPuuth~G{EbHLAcQ>pO{a<@%r zeJx?fn2py!JUIoh{?OWsRIsJtMU&4g%QQ1i$eMhVt`pDEFi@RtoF!WT4&(*-<+m*Q z_oVy&CQjxCSy4n2!5B>tknTYCON+QW(vh%p_8atOa-KDFXUc z1e01Sv2S@5)aqE{T#19W0fWQwZhlL(Z zzvfJ8A(cZC?2=P#K)Be)MJx=dYh2Om2WCT5(p0I$)8O@t#jh$-*=JzLZUz&fBMU9p zoM#X(Fss^IWHbJ+uTL`+QFBhDnfDLMh>?lo-EaPrt0ySo$A25jH#uYQG@JsQ0LAVr zgEKJV$`;u`^lONuVLwLA|ErEC1ID+cU4cIuX$f_Y^OXY%D@1`O4D!2doZU#B4+c}2 z5n*jz3Hy=C%5N62rsY%j|~zn>1S>DWJ1`fmT|L8sDwnFyH7$T5Q7n?ELAO(_>%LZs;~tp_HGdn zV#qs`8F4IrUeEsrDvZmEJ?ms;=1=mauH9xY(s?+;1ljXY-$7y z8UeU6_y{r&BnO~eqep-cKr#fDS>OQb`IQA1$aZ&>F)IfO$L_@Ou^aQembk+DSQtIP zRK%tg!nF=*n3$Km`5J!$mj`5Zd*(sB!gc;sEEOOSC8A253-735!KiJ4-)BDoV5GgM zONx$P#+ns}3CuN0mgYa}1HgFl;7N)~&0i+_X`+5gV;_2U-3{xF|AuQzjFI0aElXhp zpR0o*p6hPKsQ4w&vPJuZ3+#&lYPo^S4VQz z5UJuOdW&Y!y;CiPMHP3S-~;31rfnImi-c$@2|`pQY?mFr-)%a%SmG#{_*XzG;Ly<7 zPLsjb2jj0FJdDZP>!s*UUy<$gjw#9uv&>)LHG`T(c0rtj71^r1Gf57s#5IUK#P677 z2F5HEO9<<<0NlaTaYhf@<%b;eSHGtGtlETa!8X67n(NBW>{aaKq>DwNhqESp@n~RG zxn3la7T4(#N{h@QJDCznLsbCvg*ERwad}U`MeSw+S?u5cpspQi05;Iu(m&5Wxav z_F3j8o=o*jJ_PbZ|545%@HN}6_b|sIefne74q4kGybNx&z8<57THGklx3|sv%#W{7 z%1iwLJ@b(Ifs=a)9!T5abbMyLrjsvaGRW=+KaFOij7O$lQiaTGS2>aKaPe(ScN%Lj z;I*TRh&nz2Fk5d05)ir;j@>_Jt1OVbPv2zn@r3H4)!*Y9Cs9R!asdHxgFAq z0y=(5RtPKE*eIg_ctD50Hjj3(ClMj(Sg{fw;`Cu81#;KUR^leWXa&gTg(7S+w`eA* z1~&9(cTVp!q#-;S0jM(gBq0}xCQR#*U61>gWIT_(izfKQ_I1VGIX z6@$QAfQNlD%fu*_=}&3?&{2^cOOW*juM@P9=5@^3fZx4c20LI6H-&bC!}GR)CyIq} zb6H9lA$eAus=A%E?rpM+6ft*37=gG@-nHKeNS{7wo~lsQeAR0B4#y@jYe%z=P9-%6 z;aJI^ddDpT`&wh}ienJh$k!Wh6vTOYSj-U}&US{pt@3eB=L4X&8Z^dN6E}LL@n>Vt zNN}H6^$p;@CcHCQb95?c;k`CHf;4)#UeJ9~ssucXubvmG#Ua(FIfPZpl90a!1#SXH z#+&L^LaxV|r!*}q>2-)<-w(l!vW<-Zf5h2iXwe~3=kYhe%u!D zrtxpV=!i6;FBov}i~f5P9AE7D2^~amI^|gaGO2O<@S9rNg$l}U0R1DP>3~?c?DHO& z#=o$E4eG`?Uuu(J!SV$;mLXcA_R)K?=9AJEa`3vz8w2nzcAB2fEzU4N8W+(ShCN)w zX(fa_Ds<~V$3lQC9Dua#s9BrED0JI4B;KiANG2w_%K#S*i0}ePNsi~grVC;GKPzHGvfX=^KPSN`06&ye zERjzs`yjuRxXDv$2X}~m;&r%EB9L81qyCrkh?Gk*+S=QL}p<)+j_S{Be6^xogdWV?`EvRhWQZd(Vs34@HJA!cfSCC6kbX&sNfB8Dh zv%I$1msZQ*zBvorPu*?lO>Fu0QGaCag=E9MuX%Qb-h6FKjs4_sdAs zlU#jO@Rq(L*#FJ>DpC0qiXQJ&#DZUlcg+}+0`RtD zas5pnr0*u>=sXk_?HFU(A&Yr5u^Z8s&alo95Y+z}D)%LVR5R?b(AZ~QD_I)~*y4NwxttsrRWwnt>dMeStl75xZw^K*0!O{R=OXaZ3SSe0;nW zz@MoZEWeMbz;ECQcbkN!KhE{1d(F{8J@{Ku(e^bCtrh!-l?WvT>&|8%pAYf3%HfOU z@a_=~%)&V$4UwD`usZ~Fy!NMKQ?q305ZC*K7uH2aHqPF8I=auPuI$9|Mn(`8RqKTMc|~oylwR zjBLSObu{srh^noZE%?9nE-bjE=M+DC8E(s%%L}e(OqRddH&Eyu&X;IiNLrAY+H! z1DtN7`zvb9lZcj3DR@KOxF75y#(L`E>%&ujP#FmOpWKJO*lUcpKyzU%DY(A%{_;;J z@|ixL8%qiFG#{~YT!jtO0#VQn7=I_lHpld%J0D{ni1dq|Y+j2?pOnd3sq1YYIB zO=e9j`)_l|M7dYVj&ScLc%Hj%);X&o);6QjwIKAt@eP7;E!oLXdCYo;wN{;z{TL%r zIs8}?cA*ypp^Y!SK?3mvfjxKU#ik~el{dahQ&mX`ZOD9i#00LGrA@gnf*j1emU>C08=f0{?iK5J8nwp=Y@ z2zfrz<%s#ahTP5{Wzs=29&Sl2m2=EAa@H`eV7I_BKdAqod|av1u+FI=~@hkcxo*q+rFTyb%Esi$WC3|$(?pX320aB5Ha8Un~UfbQEI~1 z_tjXA2)Mk!wFJwS{%uyg<3EM59X~16Z2;R7$9DD&$=x>@HSfnaS{P z@dI8vHn;9by`&+IyetIJv1!;dGrXRs@?$WSf|6s_e`o+#=>h6h!PC%$AB9*esq9sF zEW>ap&QZer`xUE>ECi^j073umpkZ10DcZPZ+vQW&N)=>iuB z&7mnD;QR zj1}E1Ma`6GS?oXmf5y8AHb=%a$ShooLa0uSK)UM8rg789QCTNX0Pl8;;~jWnh2rdIbVl z8G+B0+1CA;ABWR?UmgX`oY0~Mfl_XKg%;8>0R=gtYlODo|Vj1$ZAKuHNt3hVi zJftDf!K*W7;u1YOhrUb7Cc9S+ymW*ujh+$t$tCK9c54H~geeXarSO1zePf(GB+JIz zEdh9M_fA>ONoDTxBb-=-Gi`S%{B}8+c2}c6#;qtAzZRDfS>@_r3L?nFAN3Gj_Fy+; zWlG#v)Gj%`VYrkSVtfbOd_`Pn(;MgNTbK9&Wgvez-dXO<5MAy%>BRN(m3+Dkbd98R z6>d~~f>2KHg^aYNDl7W$^;U_QvXwOcq2m>W|9CAj;0cM<4@Kn0hD17a4Twq*m;BN@ zrJS@k6isr;P>m0Y912E|EJ0B^P$r=o!4B@melq{KU zbU%(4xe_nBxdi}!U{-Rb`g^)Siq_vAaSvv}!w@^HzRvr-$uHhymvD3h(-r33MzYQ( zhute#1*P6S&8yomC>&VQb!NMN;z+O79g9W(9|vE~Asz&io0nG~%xSw&0Q;>Z@=bV0 zt|iaztjk3!~s zCr}OM0|%0gRqpaEoS+TIV>(x}NjKZ&x_typEVRiMezM*7u0Uy(&h?9=JZd0um&Sy( zocR<%mOCaY zpDBqC!W??oM;osG&Zd`jL=O-jcHa`z@;1>vc1-j*!4P8&l2oKXf+Ml)*uf17|6C#H z-M`dqzqaSGRQbJEMvYuYbSyN)JR--XQBh~eB%V0hoa+Vp9Fq~~-QI{CvXc$w{8dGZ zaxhx_3iOzjpg$&4e;1`_6(LT6Vb*#`j!*n+-+mZb#tqoVk-qcP5ctl5>tTOh0n%*g zydws}VFhe&R8nDLk|f;Xd`DsQ(+^WnhOBd&J+g^QVuk-l)9Cpc#Lk6v3{9mo8DkH# zku~hfmJ%WlSnUl6deiC$-C*)uL^mts;sJ+wyh{t{-JVW$0(AGt%MC}Y-3{TqZx#Vk zMoyr+hZoThFJ7di<1oUqmWi}3e_bF2Z6j356su?v%0zj^gEhw>&h-NXTN9BJz#rU~ z^ndS<-1Pc0@ZnpgXUJdI4!iwz`YYKKGq-)Bi|d93=d#sx!3J@tHwWlnf$jWr>_H%v zk*$;zlg|N4a|xhB{PU{09gr8x|6>_4^5 zN_0ED@S6%LOOQ-3aFI<~UwJRgIdoEr#B*O*IlunWC%~aB+|0Fsu7-XbUHt>iufyaD zLnx}z0l-ArL2M`(QU8;-Dh;GQ3bw7jCzx#s(5r~IGwmQ%aukx=D3njwryo!VWpS5Ch(MtOcxLQ{|~vz+BOK$ojU9n7PP+fIQOh)HY>hw-1|!ydO* zsU!~_qotx0n*H1~FD&+Fk~y?=QvM}2m_5p}9n*n^lNMnHNIJ=yk3i;Z0MwDC3|u|U ze%vh{SwSkLi%U{zM(AbNb`oUCw~4oO+wiL|Yar~#eP$@Ny)};}B0mj@5LYASZY>1I zsI9xAzE1xb(G=R*OS&-*_B;Pdp>#p}*>H@~t4_7Iq%q*)8kS=u_>o%8{_w_JD1hKr z?_s%|ND$gMB`3JE2gxK0?OQAESujb-gzu+XzNKM z?m52-yqtL7^l4rp;;Fk(+>k~73i2Yxt55;T>z=cKRsmj0Nai-i$V*(OoATJ4$hm@4 zZl@Pk60@~oIEx_!N(6D+s~Q&-KB{r|+-gJsMqQ^ zUT4(^`@0DP(`fQ!k*j&BSgILnWUn(5%%T5i`;Xwe0o|ciIFvf~zh8Y4)yW~Oy<@Qc z;Mn``ne;z&39H)&mhI5dT!tPBZJkHpsl4Ul?MWYUvu_6^{OPm3Hg_?(*pgR8{zcoXm)Ms zfVlS)%FYw+3K&Ypam&bNq&Xr*)iZeH&8`cuFU71#OcK(|WaaMe;LSvg}U7 zd#lRqdNXJha4{f~;`mdn!Drg;*YuZkkOU3_347hSlAA^nqzZUZJ_C;WdFv2XY zR?{#5 zaUPf1JU7SqMn33~SsYHfQK%5dxhl?*Xma8x;#J8;c}F1r`goHw@}UjyEwJO(cW491 ziUx(Z-P7?#cLcn__+bx389d~HPMR`xAB>Ykhz#jPT9pLx3d{ZiIFGz4v_h+7O|zd* zck)eLKM96}bDc7fU%XI+C3#Hjgras8zSuP!Ls;J~$KOJ^19AeYQR;Y&VfJ=jT09CzpP>aqg4r4U|g~LdN0DQ!ed)ZR;5g7N3 zjho2$l718Oh@16Zzfy6eU zX+Vals&9Ug-?SR1u>#$Aexk(TukU^-V-i>1@oA(OwRl(vm#!Y#sJy^b19}VTJRDOl7Tc$U@wPi%1?N}gJ*SIiYexZvvk&?gJ%b&!$ z+E5enzb|;J7qa3kbZe3FayRf7vNxfnu^!qT%_{;~iT`&3Z22aF4RmZc%J?)R_BuLq7Q~h8B}oPV97m z*hVqEBBufHEC>AiP+)F!=UAhZr&T(g*{5Unsh2~&y#mfz3(+||>A*zT6!DHueD)Xw z$bBc@&499zHNKi`Uw@~QEX@;wimy2t3?a!&gqB<)bqF<(>;50^mpMl!D;j9~T3Y8#n5;8ODE_87#^#2jUcC8abb3zj~%6zkKmR%GZ%|6 zfJt>hhEG&&3(YNEIYE%^pOrl5E_v#Bha(g8q57e;}kn@h}OhaLv9lFX09jL#1HEG*I8p@oY2$ z<4;ZrFIo6InJ&o$d(KyhsVSC{NagW-uH8Frp*VI(V|%ym(!uKgMUe0ya8$=p(!98_ zP&5RqE6W)%L3w~ZiCkKouiZJnth*1BId*xK=*eUx{hYW18{#qBq`@QIq0Ba1PG zlFagxz8kmz-s<(6s|Nl#&oFkc-BY=(3XNHaSP>ph)@0jLRfzd9HlMB^|Aj(~4!lae z@Qr5`p}2}kO0Rw0fl8Lat6(Ai9eB;;B{LahG|xNNE>J;`tb3@WiXlp_(!ulpPvkzoZ{x!M z5JxLD>ES}Ox-lDmmY3_b#D%mpdE`FF16*AAY2fvVV7DN{&2m9T=K+nheAEIdX>L9Y zLpA14aX$cubZK#?MFT^SO6hL|Z8PaUtbtzY0W*v;pmAr?{Ko z5XVA9DPxpg9+&U?WdH1-lS3BGm}%vTz3UZi+PXNIuxIw=!=l6&4>Oi5bQTQ3fi26+ zhe4t=Re4a&c+QMd9lp^3ZggZO1=*`?)OkZszLEA$-SmJMf&u_j-qAl)c;rztrS!cbJ~9 zP8bqpe(>3xgu_XVxY!LY6T`|B=s5s-rWu4#JEb7UNX<(UL0=4WpCqo?s6 zU^Qgp1)wP)Cy-7%(T-2H{FJft9f~az9a+Q-(*Vabeh+0g>K?ar&1YI`xSPw5_L-9# zfcSM?&S;5a2{<7+3I~Ud*@~atmLCzUsX0$}K`xk)9e$B8e)y>_!9FxZpfw9sKqx!%aPtdR=7T8t=|wJl7&p1${*+MU~=nF zP}wTb%KM>wB~WGFWj7;lmTlmY(ugL+Vfy{|{RXjW+IdYkBuIw`;hOBR!q8)J+$;&| zQH@oW2>Q7m&en6QbM`sdR1v9ri|$`vsB@;}hhVFyb#C&IZhVe?(rw=5E!EOQ4evl^+Eca+c-R11Om zs&}*dp(7kRt_pX+pvYF5u&g@KiYy-n3JP!P>{y1-v{{#BYE-{x&E5RZralhH*Q|2@ zB#3}0gDro7q7%NtsF%afT+45SD&ZwM{t#4`YZPSD?n3p65f(>uzinbXf9u5if_>-! z{`c!QMLLX|dv!xh+HILZ@-B`{VMr4LTvO73-D%w$-gs^jN4I@P;HCt!TC#z!^l_@q zvrdMWA+dKvkl31Ny@yMgf{67hrG?ct%ZuQQxUHr6*)IxEOy*ETI2O|u9Pr>pe@?xX z&RvohdEBV+s{=U#dnEeNX86t(hECYfU~0Ax2ee%EeGp^eC7M-peh@1E%^t5m4X_yi ziq8n!5OI*4qS&R0dXk70d9+%+akfRCLqPIy)qQ9^BKvsqe3wpXS{Oa6dRAF|6-2kL z@?l@22M!?#w#=jbJ1WcRZo4vR$gisAhlW%r{WchdzgJh4hr&F5pjXWf<-@i|c0n!At*aKZS23 z$or_I?ad;1vm^;PlopiAmlyR^BDBCthWT#t{wnXWT1(?Ze-BuaPplKQU9*b72W*9g zb74qe0WCP~L+WTqb@Nwd$P3;_9NoK^1?O7pYoAk0&sj;W8_~0n?)kz=!_9MryED=k z_tJ^W<^PVXWfiH}nZi1~-h0Q^Q^I31G7&0&LE4T%>*I&~S%S7t(>*5+Qd~diB9%{xj>iwr{bO{S8b4g>qOoL2;RBTN2nM+z0mr8~2 zR^zo8E6}fj<|;N;1pFCI>er_M#$I5YDqC#8K1^~o4H+Zm3dnvk*I-J`~-WY%HegCS6j7z>y}}w9dLKcjWX!|JCy? zkC1kE{*17^pnPMj_c!BsJESfe{jsBWB!CBF!N?I6w*ro`C1sYe%$cl}07g&ATeq?Z z>7f55!^KDfz|s1T^y35i*Dlrm7CIF;d&5)@8HLshe5pReZ&U46>bAPaONFV;xU@=P z0{~!`pKZzM*wvmLBkMa)V&Xyz|CVh@m?A&MKW%!yR-Xy=dN!Sdg;CP0Y*@h|WRcK0 zj0Bn6FHFKr0+NY0g9ts#@WZ{dze4pfg{2;i^zR`IHhU1Oc{&X0)W%;6HKh@D_+Fw& zstvov22vZCyqI%vq8_{_gKHw*F?uqPO8YVg@8F3Q%6@vSpYy}6s0YE<>|N|a6xi-J zBC+15*Z(JCdyM|!K>PkG%1_&t)g^(@a@{8SV{4F`m5ouSeM5Ex1D2Dk*tz3x*H@10 zL5}b*fpG_wo`y9wSL?u0RwZQK{sq8{SstpgnB-ICxWmLnm0DPRc9oQA7&vtJOgID0 z#xlVPG7%vKSkR+KfDk~k1eRIg06jvsp9ENbqdJIrx<7`cj+(l1Ml*g3>Wxwq3f_Q& zCDUy0z_0=6C@x74?lOH)bG_p{4T8h>azh5+qJIy;Yyf3|mf&bmG!u3DeC7)1MxM7j ze~9dB1#KQvGRI^MA!3G!iu@u%z3Cz6f zM^M>gx_EN@ie8sn?y$qJdemPy3d^J(@${Vks8BMqpKpcsD4YR9IO!_$_-T?GMc8m) zb&nZhZ2nI4;>36cNTL%Z72xmB1(DUfBEG`1%?8z%I2_7%P;{Dz>!y~*{}8!~{I)2M zSau_lQtJtygs}}LxEF-wwk4)V9cwKUDw3M|gDtd{n$meAkaYe}^z2=uAuVxTNM?17 zK$Ca!5cRyF!?j5IS(n*uTh#S;{*9~GjO=0|gF(5EC^J7omwi~x;4feuQap~cEqgq< zzkgU>>`QH)(VXSDa%22`Ki9^m@I`|ApMg*o`ije7>X)ftnlP2rrm7%GZ{A&7h9_d8Pmqgn>+lF1YJqY!8L` z$ng`I~UQ1PH8f{BuyFL@{nPMy2IKYyt&t$CG-_{ zde!hIFpak?UL{#wh9(o9Pj%rpgiIr=b`qUs)q&yBf9^t6t``f8f41 zJs*KqLc^OPP~3thE9d&-58MKM!huZXZIW1h7GVPZ;qEo#k(Jm(+T9g;JD9s1;}O<= zA&GB{j-T;Lak*0MVtr>Zu0gf&OtC3gTLqG=u|tktg^X(mC5)9T2B`h}S!gXP!(^iV z+%Es7`9@iEGJg(I+U>;i1CFb9z^UXB-nW^U)Mx@7GiIe7U~@fmm3VPCNP1T$2o(%T zL|X5CXUEBiM%FRgeQ5-1bWAs6;H@Kr9Zg9SnD5RH2K6du5vWi6e8wA<9!*DB2q&Xp z>nR%`K24*35;N7{ob6yieLXhqEkg|YJ(EaNqbIPE=*X$1#txcPfdS@H3K#^9>1gTe zYMlj1(>hiy4+Q4rmOg_jzH1M$#XWF613;=SnkA?3xqnST?1b%og9VL)9V{FQJH4I; zy^q4Ol=KMX6aVWUl9onkCu}M0J!$k{kj?7w1n*6ZVwWm`Gn3l2`PmNTRVOcCQlO!+ zLs_>J=GdqHKr7yT%uX;YJN7gGlfk250#gw)Cb4xa99~wA&pQIlPER zF5)P&XKX}8HEUAtLh{LkoDWhldg$98#2NwUGNA`D6C@1Kl%q$06i{*mmRXFRMJ zd+Clg&(c<0lTVtVk0`;+R^1^A?MARzCZu|Wtf)&f5MA1i>g#cVf0F2KsiPpb%o?zT zgemxx#c+5&@RkNRXoQV2(9k2{=+PKD{HEX)hXc&Ncd}7xqqPq>pKua>T^exNPPYeL zTvzwk#AE7MF8_J)+RZU`guN|e3n=329QYPdBc99+h9 zT?$gyhKWpltWad%r^zO4w~fw?w6gcrc??-w%LhYd`-ddxB-}SKu2In$-yM751aMph z9yXMQg`!deKwEPut^)wK!^D!XFGKZjv`exZcN4AlH>()D$T(QcXeF|qbf_8$&pV(*O6!-k}yx4p=4*nv64@$gx1>D%uKXdNsOuj(M>kvcGu*@p@qte=#NQf0*WP}M#dj^qKjxMO?RZi?KCzi=C&WJz-@eKF z7DB~o=H(S#&Kr1Eu6NY+1(^+@m;=R%`^ z=fY6pvp^v%@H6JGDeB1bb*4k?F6yM)3Xc12Eb9svZU4uTJRUUgZFOV0e*k=Nn?DV; zNcC~g0Z1@^q|C-;MJ6hgFD<_gU+!@{|95F+@)&YG4~JNhj)iFJ2AyJh=>ti5Ly$#A z`gtqZ_yv{)Lc(V+CAQ`EM^McAKM)h)49&au6mMj^;nY3a0OG9|vdlG0Yi#T30t|H$ z6nmw36}NxF(WEHj%MHMDdCRR0BLAdmT;dwZ)d7cQ824>DN8NblahpnX%-%h`I<%)! zZ(<9EHLZKj(RzK_2T`e%vN@zhOFi3`LF`$aUv4X;k@7sB2 zuVGAx@6#yLdkKs@`=7a&p$^|VS!|lk=0F}-%WYyn`svRC2FtT8W&cN3ZU+hI%!RYQ z!h3eQ)2bS>zge2xLa042mRZd#+h@#W8Ue^M+YBiSAVj!!QKK+0^T!ZM5LQ{>0Q&g< z)ebJ&Sl%l7dSB6Qm%n%Dre+R-xjVIw#B4OJt38dXBQ(#KY!&_nPG{;Ht2_<~Am;|9 z`iKH(6v={3Mn^0w22%!aKX$IdDoiTxcoeENo_Ld7W}Pjjsy-%j*}1pblNibWhzSnT zX_Z$+&jpl5zAO>)RWU-MH<<;Cf3~+BM|{UcD15)Iz5e}TO)pqt?G=luJj>!FPkzi& z;dBFbl^T&{DC2Pc1ptCZcxhQitFmZ$P7iPCf48ji+`H1k5GB2OnpESudV{lEDM> zS|JTq1M#}@f@QI%?cb{pO+GeL@cB7JFFwn=uR{wU7|wxP1#wj@M^?&))ywrc>s*BFi;?r=6StWy zq(xCI`T*h^$qr?Uid&qi0V6i`Iz8WuSJ(kr_7Z zH1q^j8Q<}_{hl(yMZO9glny$JQ?FY5^pt3zdHi8G7riq4v*S_zkYQ3RiC?O|WntuW z1>zf|!UrseVq#{^D9T068z?6cthdS>Sfj=2L(UA1Vn>80#AWBZ3fQY-PE-&Ph>7V{ zIvE)?yO5@0QEW1k9&X0>E0q|z1&n-h0$p^lKj>xKff55b=kL;Ny=fJv5#Cx>ET|rv zL?)BiL@}=$J?l@Z-3Tr8^_TNL4E1xM9bSv@hX*=2u_0w*;y|@A|%*G zD%t^Hq0Br?IM|TP7#C*gPL8Xa)U*hudMV$hhB-im>4(oP@sG1ao^G>fYg#kbwX9cm zu~}FOT)t$-9gwA*$r;p#9kCdE%{F$6#)&CyHTardYk0493I!hB`dlq`A;F1f zfnIQ1Nd7NGO|XGV9|(Xt3SO&fkt5E8gZ?`D1|pW`W*Vwj&dk9)i?0(~@q8&STJ6;9 zWy(&J;Qs71fR=GMfjF?@5ozq#E+}k)kEos@lHJ15L<%nyLLnQ#z18-3QFCF;hN0Qh zGg2W@T>@44yKF+3G$#n(hK!xmZ*?bITFRC{FBEw*EHh!6yi1IcN+sY&vAIwf;W)tr z^Gw!kpDr;Ka_Gw#)r23(y`EXM!JH0;99o!TUPFm0gb+NiO4q{XhX{LUaItYVE%eSsHxnUVN-$s>uu)gOoTu$WoDgL{az~*jvHkZ`_6APW! z__THkf0|&8BMUV5drZW9tdeTKIS+}aS_>#;->}`wATmiQ0+Pg!y%^+rVBf^* zOO3uqFxG-;`y&@RC^Wcy-;%7;^2~17u_T0kN0%YuiW!zzV>GwLMrpcrHNP*2cz#2H z(c$)SWoO3BU*_efCbUeN3R2u`sR*&XgL&^3H>7A}QXAm<1|8?9f|G^Uzd8RWb@wU* zXXlNRGA$nuYFLgx+qPN|*dXZVt$#VED`u6mWiIq=2=GVP)bD20mzn7#ATpT)N0Cvax*K0I7ny8WqZShvdY6Y?Yf|kQ4lv)GKXIndF*s8c zg1GBVwAkR3)v_^*Z~w{lfL)4M&&(?(AMd5vGaPzLCT3`80kZ=h6HsB1KcngP?I|vG z=v>4>+X`x`^`hL81KJ{Nv39Li$NUqWTh@1e$;E2co^=y?euQ9(u+lghAItQ+g}03j zQA@IAuQ*Pc$}~(2@M004m8KN4>9A0o?E0 zA=zmsK<@Y)^kYE(`l=la#P(_}VVX^W1VCDj@wPMnptVQWNm3ZeoZoDv7UyK^dJ*o4 zD}nF}VS>=zZy<2Zn@K@l9NBfc!Dp+ebV!hB9RVp1;Fsd*g)lJmp-TT3 zY>m*KB1#K_aa_<;XDKp|C{K2~)=d)w`Z`d|CV|WRq%QbdHM3yH|B8WHx;%cY(bp0J z2XVy&iknlX7O(>ES5w`L4$(FL_5aa&@Wd{I5jv^$7JmNl8RKp zG_?`Bz*U5}h={c?SH0h=VCv|Wwy;yv0?TALkW*h1z$8w#MaAAGk)E*tN^nb0L{AkC zeEZ6JWwLF;2(;opCurXZ%sUIp+fv^b0p6(qVkC>3Tv>}fj9qHL=zy>g%6volJosC0 zcfc?|b7OLK2G$zA*!l;5FSi4v^+B>wcw!+Jd;D;+s=axjWyux#b)em+Z5{_BxwRYY^$-sC3o2Zgi{!@S_<*l{jV{~Kmu8?m{++!CW%V=c znW4;nv*-UkHUOn*Z|de&=SwZgcxA+6{S#Ija_tC1)L zo=SM(u=Mdiz)>yJR~+GHa|)Tc=}tEnf2-IAaYt|_)Km>z2z23h#Fm;e1F5nrE{e$u zt6#Qtjk@LZ93ok=_d)qRCZSK<$OCm8#Z^~akc5@ln$dpRB}H!)KrN4(9tpj+Pt$`H z$xB;s)9L9V)_N@1CpGS_h)PtQLx!5w+XfdzcKeKL3ZC_~VNryK@Cs{Rrk5)EQpNS~ zOsa?`jBGVm1bYkWZMd7xL`uOjOKiN>Pz3=gMP6O`d|;E|4+Iy8MY)~Q3Cws>jr;h?UgJod*3yz6x5}VrZrKXgPot0LkCfVs zj`=2U=l$$qp?5+I1F9`R+CCnuJet${jc|gmzM?G#@uSfpU?+$=2DN2JsWK}8o35Mv z!)ShoE96j*il8!O>bEbwIU-TBr@rzF=G>aq=fp@uS}6Qv74*9vJDNhng?g4pl8nVW zCKKnwd>)Usm$)Uk=q#uxS`F~dPHYh)Dvr2{oQr8E9DwGT8z6{5L?{JG`(5Y38!HLp zJcp~A_{`3aNM3;bvg2}n^=*ovL;qcO?4ZO3$no3zBdig1?q&PdRR)gyfMU8#)4 zs%oZ<#px7{Z48)uDVsy^#ieix>Dz!CH)sTm{5-y*lGXx~2JVM5QZQhxM9Frk+9)s3 zLD8_0+EUYv1;)IF3kX@72d6RL8nvu`dL>74m&+eHyxs>QYIudcK|Hv}O}SpaS!ktn ziQ6fvF`uRX+V7lG76g)5jd}7W5|2L39mSs48I*>3sVt69I~GoDPb$gPB{?|4dYQoM z$oEo5ZY=-l6U|LR{sWkMa%kV&w77qm7duL@Y=a%EpU*qaD?^`=uL8YNa+thpeur0M zClyUO{MU3~Rpxf@Ic@gT+Nw^_Tk^%L3AulH7gGg|^YVIgc#?has0+c|dCZR*0Ud8~ zs}n2(eQj$Dd;ekV^ZKgwK(nhtrPEX+N?|!M6>&{sX6C_AaKg5GFi!I#4_yt~FD0jo zkE1A?I6xOAF-G6WzcWG8Hp;g51g0|17%VR~p`^4NuYA$Tq)l1~Cp(T64ICS@=)|-~ zA0`bG@Gl^_)wsdx2<5Rv*P%qVL@;OO7KoSeeil21;SG9uxd8n(yL4*Bz(`Kb1L~QS z6-%Mo>Q7u60r)cM7by-R2go>4qX;mv#}G^qS6yX*1>gXhv7eVp$(DEX2$PGNp)27P zpjDbyT4L3kdB6UG3^+G8{E$dF+!B6o!$?^c&?W%n-7 zC<`G{$S5wYHBQVdNp*J9dK7Axs0!%C!p2eskhXW#tFi(A^52k=%~qYe@y=>y-#^$v z9_7Z0?>t2RCcB5++Tt>(h33Gn$n~y7Vh2(#zL6JZIlYO%CKJqAYR_u6Qp|kaL(SE% z^4o=NMS(*Lfs|w&t5Tk#a{AZm3YF)BP$~PN?wNtT`x96%@*83{Y`%aG8-1aEq~i z6?W_|h>haPD!k zdLm5D_f>7C_DXgelUAM4+M@N7D>3Gvu|GRD(ALhXOEFC%wMI-xo*6igUkyr>#36Lx z99%}z69(^3CAhL2yh@j@S27ZO>C=T zKp^l<4Z9#<<*L*cu89;Zlj{Wngtvb%i0Q1PW<_6bPEy9cgVBI+{{Z_oi9;y}QiT0?JA ziXBr}>j3ZZi{w0Rubk6?bmy~DQJV<*)=3$`^;USF1(KF3cMbas&K4R?Uiu6V@fFD% ztd$d}Hh*(0Y`a^Lm^CiK+zO{osuK#CYiM8O`nya%%C5)PmxeLKB1#INi=5}uB14G5 z^e7d6uBla!pE2a-$;7CBjI~7T9pQ3S+q(|Yd1DIjs~l?JShI^{4oDrNYlvIx;(yYR zED~;d-ddw&8XQ!?-y#J680F1^nR6nf9*+%ciI&aWMqY3~|N}`~NTNLdVmkS*4LxX%4004wJz=9cFl0L@7tQ z%DWD{>V5vq=G1)JEIIzTw^5qzu~qZugmi35L`i`Cuo=w3JNJ`q!|%jwvaUSfKxZ9s zJh{b?`=P_>6cJ+=BzRkBU~WBR8g}Z;ck+z;p=d5rjx_LI&J(m2%6=x^Mss%CQXfKt zi``)xCXk+#*d%ho7FVh|Ktr6CwT1Nh3rDVE+Hr^Z>y{o~)v7vAQ3H4>{{Tu%)f-(K zq9_s~_xT4^Whg)@!BsD1E2%;PKIV=yzr@ptx96#FrUQ!KGdoN51U@J;noG`=Vb#ya zt|_x2L6jX?SThvb7WX^jdni>Ij7Pw`c}^$i9Sr^!;_={^47SmDkH>}gSzWxGLeItl(jAyU8hW@E{eLAN(_idu3%Vx#?u zhX=nU$G7*xacwDs8&V#%E!03HJt3Hvga#MYU=eQ!FIlb#(HGZ|K6I#Ubkyl6)Xd#L zU$Jtpur#XlwWr2pCU*2Dlma$-_)dh5D!Z{GUq>~YiCd06#M=kfMQ0H7)&Q21E+T4; z@%l@a357I=LD79Ki+anH`kboo(1Yl8h?StggPYK&&ba~6WdVl@G9A@h+6=vPZ}nvj z5#jk=k-njrUS(ieI4=WgFFc&js*k_sR0kJ5C*Ni&#S52QJ7bT7ya2m<$U zu!e8sX1~+Q!N8**I-5bVN*l>zb4=#+s0S^}Y8>NRzs39B-nyogaA~K6^C+jM^3vR+ zxDzBJ+SzKGaJ?(;YJi>2BD_l0c{oaa%tWH-7u?Ra!jx>;ZS%EwKoHe`zmI7!BjK0F zyk$vfMtc#7t(q4GUmm#B!5Ju_=a>Vt_-%<0JFBZ?G*As!{p6-;#XPyOAGhXLKE}5e zD6Lgi7Xx<)0Ki?msy2ZfS^&5xyf1b4eovmoRTCsM=f=2SFicX_Q7sjx)N6YksnAC~ z{*Has&g#H7rw1_(=#05WKB9W|BsCy4mQx+|AQ?b8U4lA_CTwru40o`XHd>botY$%4 z3a2%}?tcxLh~agASrs-q<~gxu=Bd4vO082$-*m{?%^tytapcvcY6LJ+t@BO-%OMbP zEobz}-(!vw=~e!JpDO@JN0y4Px3v~XGh2v{qw-@>;AQ4h$JeC15%Sx?cIr39Ai@=n65Sbc#smm-r@0Rojf z^Xj%|8u6Af)uvsKk8I@P%j>TQp^LiEqSey#T9_VqN9aM0lWal!aJc}*`0kPR`-I%# zh3%b7(Vdbv7LoB@Zf6riIQOw-MWbk}&bPAIC-RhYW&y`%QYM~?)MW(3KLTs%$sls2 z8{Z42k%)os8d{4wI<4>T_05}z)(G(K5d=cqm0XeY^-_G3coEB3a=oRVu0@zW!btAr zi%H9LK7khB{^Z)ZO=Rrr;yHhZ9KNKG&D439Hl$}x9NHVcHnPhjGKe50+!cT7h~-Qis|iIPCoH9Lsg0M(lG^20-#QarP?m ziSmLT{#ZNFsl~s4uHJ}Y-|)K7DJgpq4WAm;M|S6``arMb}7oWB0!lQ{yecXp#<7eQ13H~6-z=5*zGLi)?^tR%}RTEDrmCAvymyN+3H zLNy|XvcDB?HEP|pCVAIJT;mQ6J%NfnmE4(kh>GV>`K`sTCnq=~b5Pa>IT@$9hXJ0M zRRDNcFEMuM+B}@JYfUzjwD4nV?NZDjZ_q-vX}9l(1AFVOAKa43{5c92$xUB4KOwXr zWnqvA2q`K=-UO=TI<%O`GjnVw8+Y6_!qp^fH>y3k-=(88Of*M~FnA-cXA8a3f*h!e z&BuA)M+*;HSL|TWg9eTLW(J9ycOlHXOF=kW?eQCiNoxqMH~%akqz=+f!2~~~QmO)UlXD;Eboat|6MIY_&B4KDU zf)XhWK?lfIQKMinG{+!F5MBW2b%f5NHSMNm?>8cRw;JiI98}ir{Jf&~nu;_lfby{d z=&umHW4G)92VZ359P*NYHq5WgVJOsZ}Q?8G)i zfFlxk87`c(vIeD@V4r6dEYT&`!ti))bFhC;G?}Nu_h@LWYXh8LR7GiqB*RfHH%mF2 zH)CdZUi$ca?fcBUm(M8fpotvbCmgvF0?Z6FAQD7aiemx@!5axCd!$uo87W*)^(5L^ zDtc9o#*CJ&4@MVmVtoFE+6xt%6xu+~ELfT5{n zerT<^wY~@$1nP|WEw*VJ^Be^Djh|eACG@-%b1LB>9|&-{{goz7c}LqpOxqj1_)2Go zkwL0)l-;+n={UgHA&p9ihhDv!+~cAF%7iRdNM10G$-Dup0wQT5O`rSUbe`W01rSBI#Dj-f&!mGJw_rF3!;dx#rvSuR>{-m8$th^lQXLid8 zGN2-oI4#yle+;FOna+5~P28??J1Or@WSZ zFSX?m74)+r1nkqY9}c)D3VCulhRmOyhrDqIFi!2H@bzdVFC+y3ROz3XbwM1SL&jwG z`i+5qaQ&l0$6A>_zCvp$+a|9DoNIWi)O2L+5ndF0&PPhfh`%(dgrb8aMHOxAvS8z=C3f^kxmqLbWz%xGuh&FXbF{a~nlA0QJ<_tyjsm+olO046 zO;PScEi8K#_nm{W#V8(?f!hyo1E1gCygzk_>}&lXPVs~bdAp%$cw7XFGnvC)Fc!OphxXkZe?b8yLyU)G-oN6md((c zn^-PV>gcp*6oRzEkZ^Bs{F(9ZmxfZOi2?lduKtmQpHR)+|Eo@19|ZGDK72cG@p;Ki zl3ZzI1FzZj8=Axsq2qyAKaVK*PAa1HGwPQn0HoSeg%r^EtXgKV#BAKdnxRn&{RO7B z`O_~P8(CenWo!`R0b|C&1@c*spFZbBhFZ|>-W)bkrvhi2aOTzV!-YGyL>{CCn8Gfv z5`{?2{=~YCN1vF!L&&Yi4AbMZKOAUWF5p}zPa{eQRq>1T$szPo4gU%*4R};gq_Sg&O!}1C4-7A{v#|!@F-=d(u*)WKNVTGa3JEfk{99l@UQ%ICq*6 z7{JD0F1dXsIKW|x_LDadahB>R>agVI5Ki(^I#WfVn7=i(eCF>A>{eYZ3sadfA4t+4AFcqRcG&EaUq08XPV?Gb#-ik8=?Sl5?LB7F-4pW*r3Z4mIZh0)*X z_|T1WQgLld`1N|RFf(w#l<>KmRGOdWQ*dKp5HcmBl}~W*VFT!6VpJc%)Y5TXv5~yn ziq;v;-9m?#3RgM`;_Q;-A>m!UdGi#mK@@WrrYoJDosnG@HqCMf`p9r1Qq3hu?gkB8 z_>0E&WXbxKVPBBh+>UPWTf3Q{y2#pELKQJCViNnzd7(Ed@$oyqX$VCH=S?LpOk!}K zpI1Y5!xhetFB23_xxIwNIZ@9xKUJpB-rJfHW!`U7Qs?>3wvqxjMCwMl-t*aw7!ZCL z&>^%9?6ck$TF5!Q_7}fwO^t&x-ZN`R0A-fwbdjKVn)jybe9&bI8icVhBuB7seBiAyydD{?q4?h}v?S*xK)%c`6=!Jk(o)IipmCHYb6M=e3@{oRtFOz7qO zvG2~nUYRZnzr#kDQ%ekoT2yZdR=xBc%B`3=Fvlx=}V<3 z{{zUCEqpM@+wvR$m+Ma#;QJ|se4cSJp#EBXFlT~_uFZcNN(m}V+a6t{gO%Li{}Znd zM!+k~UT6ITJ*p}5s-MmVq+!~MN+^MD!P45qHT_7-*aI&)ZFjch1f z&c<%m)qybT8lCwWpe{Ntd5#L24KXnUGWFwo%))L@6=jmHe~ntQo5g;RrJli=UMDm4 ztu@yPJLZ0wHZNz7_ZOUBGj}Nn0^ZUg6Q5qc1%XOl<^MFGI| z5+@`|kUalD9EP{gO-xCUZS4hulRsWAT*IFUzEAW#8b*D^EP`&HUl^&k7vRO8QN)GKFn&eEb5qDo96+C`VlmOp zP1}lj`mT&hW4A&ZAQZN29*+7iFrjZ;>nnfDsNgxKvVr_89rV*!w0W*G0n>6n z7@rbwSkS_RkbAd~kycoOq(K_767!V>CS<^CIq%x>mNPn8KiC#jGD1s5UYO-vwnfv0#^nsp@zMvr{|RR-X?Q3SFd2MlKMS2sWY)T;D#!70 zpUGv696&*4@iIm;J-TZ-KcbWspn?GSX8I@S?j}_e2T%DMJ3L&bbBgKAR4B3RE_u{C zIVjyQ;9A8fqg{5_^2Z^h={mGmd}-Zv%N=~4!$1|I>Gi6Mp2$Rl#x8GV4fj3_@dSL) z6kO}oFL>tB=Ys^?OI(F$HRY~Q_Ygo|kx#V{iZCgcS;TXkMb?nnOL=-(7m4KAl}VL) zJznfYn7{lE4OB_UkD(4WCC&uBg2lO`U);$+tx(cJ?DM{$_`7)#G@mFK()>m6vcW7n zxZhm0WtyEVPqSKFre8Z(Kls^LM5|!#)zyiCN0q)3pzVSQ8>ox^*J zN%^73cLA@4ARFd`&;u-;<(#&t_KBy2ZmzqS0a zx*ij?vEsr;&cxGScywqA82y4U_v0^!1N9U`P324#6`&kWyiPcL#??qOk{~0&fg!@S z#hzHNKTdZRT4Q7f@N}91CPOMA+3wlE1r0$T7lS*NJveBWvoY;zSkNa~qlKp8NoU54*6%3WZ0qGLF$4ob+dC-29hj z5A+YWX9l!QNSxVcu;1wI=Q^R$wnXbs6o)tqA~om$n5 zo=8)*qLZRpz<8~BOwRFc&d`gN*PHC!>^6fM2;F{xFV8_I!eY3hb^w5_RW3VJWHc#l zw|6Y5GXc1BoEzEQmH;zC)Mu`0pbkh|7r9@~SdVq&J*-sEChTd97gm(F!|z7^?iK~S zj-C__(7-fD=rz z1eRIg0QfO(q9g_a8z-1I?m#hp>nFtXw@?pFgLru^Xfjt1hoPX@66!1R$R45@EwKY= zojIS*eY=%uk_EphHYlF8zEkj-|JGx1JK!M&N}_EvGtVP6pJSSp=v{M)l_m@tg1md^ zpA80VZMyYHuIr*|cXa;JYt1&)1G{1^LTF;h{Fmy79PZ@p?nn2}@ZU6l3^E#|b%~NH zJ1GafJAJim!b&A%OkPHWKw#!iU|+7oyBr^EyE6PE8F9vZ$d(_So^BA2-3L6<{S2}<=S1aePtmV zexVeAb0IxWx~rJZ_+loNLPq<1(E)hq^Wrg!=?M`GEz9`L+8v(Ut~o%w*pdi22! zdrs1(U|+{0N+!2^h2}saXS1e(&;?sP>cR&Kb_!WWRs^bde;`G<5v^8p33vcQQzmDw zeL6oigDJ%Xh(UzJ95I>(&#Shc5449~a?B1@{}IKqRgQJ-;Qn+_9%rqW&~V!uTyU`Y za#H#}T@34oihx5=f<%C@JN8~m?-sWL?h|Ao#RWuAZraL==JPCncc?Np+9-&ddAZ2Y zOpJzDRJH&q{N4Sv(OSRm0&G{UW-6oX5nyJyQwJzO`Q zMG#H*RB84{Z2sy-lMo=kcr7SrD;{c75(SUYU+Cl{(lLp{ZwmWshnuDlVGez*&ghMS zi3^1T708J-iBwSkdakmWUI*%hKr=M4I5T`-x6&5j*;T<|mq)5Z7Yp++d{a=yOgm@W zCO|OM+PFR971`Pc_EBy8zjl;kMc-_I1)Q2%CuaVn+iFVa^-)50d zc!P8T_)ifl_ZCJR3ffK3+=02U8n2WPo5ai-cC&wGL;0qsOoz%v zzk%R-E8>6>Z|jBvhGf(6AAF!?veDOBsD&;FBLB|N|Eh`4G>aziR+R}Vg(^Z)L;2u9 zmOR8@skT0|!d=+Z1JhlQf?2vAe!T&U8d^Pok$WUd44_Qu@CKUu}a`;{!L-O zh!(h)YqkF~U}}Vnx6!&(>{EAQ*r5MbFk;PU$CL|1^qN-hyX->BJ^+HiuvE@owC;&< z5>41Xu=?P}qONKn|4$B%uERJvd>EXRVkck$g^&OjC}uZp@*;#ifEHPFnmZEO>4fz3 zhvB|Nk(4G+gD@vN?+$~~CA$DQd>R1N-SvcP#3HCl%^F%YJv=))bTFNaHy9XQM&Z%1 zpQ0tG9Ei)Z^uT!uHRWU(XBz8|)pz+=@RlUM?%9L-U>X6~GPx2d4T~q5kFrTj zp-vcbkzhoc;xFSWJ^BMjnC3u491S_YlYk?cK*5qecSd4s+6=lN8Zn-MDZLy`Iq8vU zU-r3x@?`WHw*Cdo3nI2F^<#^}s#p!&Lu8B&O z@LSX~a!*^;C;3El0yT7p-fQ-yDq#guua+dnAL-s0&StM1iit~6Fh?i`?QEz-Z?lL$ zFV<#DQZy-rwSzjNdEz^Q)tcP zRU^I9WQhs?^C0FBF=Zz(Bn-wFd{DE6P!&)19|V;#w{NW&yzQpJe02IJ@MHczhEHIh z3S2c;)crCSzqeiwPk)*OulSP#gKbUib&g!6l@zU*JjA7AH8Fvm++O>6Bn;OvgE>B` z9p?op3ETFGEZ3rsyyXc@4%E*UdZ|>fmLVwhZ#l}RO|fWEWV$Rts0SM zGiM{~BYns{$6l!(V2To5A^%DuBP&X?=1rmIq0sV7#(XYJSQxhvHYL4pNo{NW7vb_3 z^JRaZJJ!HPY}ES~Z?^{;j*}r>B74f+940h>L(B3c59YwQUT zYo$G1CT|=*N4e^G{|AY*9yP9J*M)v(=6N%N4>L}1l#TRk1mOa$IQRssv>&-IZYM3P zLMcvLhF~?pT76Di#yUT@&TG?ZhRy?m=3YPg?6b+%6kGQGnsn_nDM9X527^%f#YrWcQga`j792>?@8(0sEkt2TR8iyP zG>L|g-9D~F0k^&@Z*U8}McX-v7b1MB)9y}Z^I2SySCH%RcKpIkKf zv%zf6hYw+UC!l7_YH_Nk3?BlF`f!MzAEOF1#mdigqLHK2+?bf|F(Cs4>-yGi_UKyI z1)PxIYQGGeC!^+~4m9_&cKiko9$nhVWVYCXsCNF>*NAjtRMocrrfa|n-VW@aPv{hD8|fzR5~EX2N?{z7^1f}lTkU5 zbTBskb8fITj-0-8tTB#at=0LKm4G5Wi{0Q)z4?Xm;;-IaVywailrZ7~HL?X*e_W7F3?f#1myo7i=OkrfD+`Qj@x6a*(t*q7lPM3P z;tP`Q(ff@cMu!uX9UE(>AwUT=U3(6nq9>ik-p3$<*T`i~LN)|}n>|ykBU7J|AZ!fY zC6ciwDq2qZdUD~Ea7ott2u{-o-|Sz$w(ZzxD^Vf0Q`P50;VykG+l>#Nftle4@b^b< z42FN2RS^m|%6Qo$e-soQnC+8HXSC25J&z=_-NQ=9nWAWD@6y6Ev5DyjmeCDJX-;tb0z|JPvFrqOs!SS83~;DPd6}osK;Qj3vjew z0?@tcC*4_6E>Yj+QS)a=rxt1%Y|XRWhk>tvw=c%r!;D5u0Q3E3HQa~<#2(e$au3EVB01g9T}v@~r)>!s;)E70+@ zZ6wf{Mxwr3_dEkNBLh;B>S#wQh|axU2z|^$0S7lNz@@iSp?Q8*Sd-?SQlbIHBp*EbO>TS~(3F;bY-%W@TZ6LEU4`WgGe;s7-p z5O%NR!azWODez22dB7wC#pULQE~^T*@oyyQqO?1ujJ$JpyP7}%39XkRz9BqvIC5!!ZjwQZ{_v4)O2}0f+q|<{L#*QB+&!b_)JKj(Ox_QPB3gr!HI(k#}p7A{cJcuEKwmx$^bdfPt4a_jirTEX&pJ#ILBlr1v;ls4+dZrs@vc;N3&S`~$Jmj;pO5c`WYbk+t|l z`B_U_$I|$Nx5RJB75`FsqlE?x%yl#-v@(IZc9*HWE$fjgW*T| z$U}c8Sj^DAm=rlRau_tbN-SRLBP5^~0_kugNXce?uUfl;3V-ZQ^b4r+hTCEFtY5AC zK?;U@^Dkr`Tt&?31O`^aBQD2h^t?5MQK#KkYyU=pS!DRFlzZu(C^VA&VBuDc-WT^h z84t2Ez@CUjeFMA`}@aAgUf_Ya3!|oDKwgS8HKhsHt z4nNSjmI3N&xfwkuIg* zq794*0OX}stAc^em@Xv7Nzxl8Dyl!~Yc0?%!j5yE!yB^G-J0P<6$4#q8B0pAE*=f@ zloGI>L}vg|y>zT2a{Qz?&Qg}URqP)1!7_fg{(g=qE_1}#Jt-DmRj$eIIqabe6+-%2s7WG$?(Kh{zIOA~fMC0pjtS_PdC2Or2ZjNXK41sPN)rq?sbLH(rH7oLZoo~X- zMQwkjWK$=yQf|s)j0d21YH5eQ%+eyultg9lRRy+(?(QHhOvttRCp?<2#sK7t>H^I? zt1qSn-+VpczHmKolnQ6QPu`VHZ&HfjRgG5oN3F#6usl*= zX^xSEsE?jYM`66HTn1NDdygY19VeOZ(7?F!WHB5#n+ewA{JR}f9RsaK0$P5G5L7oeuGPgP%fzesQm%uf7k-g63I0XyR78V ztQpq^LCX1I{S!?-UJSy5h*ZBeI#Aboa1PZ#Nd);$`@`<_jwC#<75~5;GvlIa^5I1j zhac=dA`Tp^zLdgIIhF#s90-7UN7vK*LodM3I-TkjY*QTw^y)&@jS^U{o{ZpDxRLK$ z&Q6&z^q}$qEmD?iv4>N4L{J-WmVr)RemfS9Hvm8y0pu4}rJb{lGhbI+y0^7)%1+|C zJf*l_5hj8t69o_;{;v*9)*7;DL~TdBu#Cb4ld2MJG5TR$ENnSwalrc|*G+8pkk z_=-nP;<+k1|5Rog0q8Q%3^E@f4d_s#M(__jas+`{-~i!2v8SbKs@y|@FO9C*?7{!z zK^bKz^INKr;iA9e;GK%8i2!93U<%#i>IoqFu4YXqVl4!(%|e9*ap9*Er#BZpGn#6| z1u{=cO+W?jN)MpzJ}X)Ctc3!e#Xy!I=Bq9sbHTtG7uA~w5k?0l=oEPHn@2ssrz~@) zljQg!?W`O)$)WMp{J$ksLDFQxO>%DVs~qVL^ZawH2QO%V$EnNmSrqQUi%;d9o4c8m zE}7D>m*04X;2|H?Ci1)Z%{je&sNv>sR~!{LRT0p+N?sE4XL6|+cRoJ3&zf9kByKnt z!z+SAHt9yH8|DD57h;BG3SusmMLCua^5*;-IUTMx2dQME>=2eaMoZ^tvcP~&*c`4$ z4=^Zdg~Wrn;(NO&)~%~QS%J(zAQ)|6v5Y9Aa9(hS;SIf!P{nc>$6nS53%(}vJlxM9 z4?>qTJnwCoFpHWdcdy(%;F$#V9E(Q>Ktsu}r5;*!_bS;h^WB+1XK!^pE6P{RwfvY1 zihu1V-FYtJvh8Rsba{?q_9E6!`OwM+-1so(cca*-HcGbog;=%uhhqU^F6BW&va_CS z_dPQ#jK_wkE{pNHeKy~rXEIxT0NkU`-?{YYzQro9oF=-LH_W6o7B$G3`{`nQH{)HXd$NA0tzVm-_{9$}VA z3B(UEYL$=p<*_PQeFMS@slFvYEzZw84$kz;fGm#%;A!j?GppDEmnG5tZ}wry6Do-Z zi3gT}=uLXa#eaHT|CQ~&5%|_4A?8YPCNp43o?-3!Y`&J8g7URB0M){l^Sj?SkMsjJO~H+XcGh6SUz|Q7<^|Gq z7D0uOal!FiT5PN*gTkoO99-HxnQuu{P@tE3{H(F+X(i~K0}((u(#+H1>XYh`j$%4b(Yd7JYCsry1Ro0_Bc*U+lQ zE0GCZ+JEx!V`4ul0%29{=Yj1{D$a_>wlF^(|UxBBB&-M5p$0IeNY8ieX z_cSkC^G}C_C`fcMiR1QmFhm{g%dYjfHR$7Czhlzs?KmS~s4b=7^|TLuv0l|K`asIy zI|$k>o=^;2K>k5Zm`01@F@zzRDZ01L?B&4|j^-3pf%`>+{A)Rmi4gAS#@AV-9Wn^@ zEwM8vd^uPd7sGJr<22TK`-Sos>5uYO>F0|-HTGqzNF^WS`{-L$XWJSB2^Gj`Hkb#H z4@;~$Fzy9Mn~DcsxppizhAUqE5-)$fUz)8&$sb)T6_-W6C)^L8{|rhKHnPH2SHY*s ztheeaVK)m;sWhk=N3Rsal~Ai}R?mK|-KVD;BbL@zlp57o#r3)oLnqCa+uMi$+n#X_ zrKpM%(bFz8tXi$B9(z_ixk@hx~ z`Fnl(F?-M&0~F>U_*!iEAUy7c6h%za4#6ue>5R;lKTmS~T?Sg3n%^A2;ST;C`wueF z`{4Sa{1bGD%7AlH065e388Gy2(O89r{xzCUjgyZ2-I3L|YW!C|VVxX$>I3OD>OuhgC%jB_wi|UPu5%te7Hg!qJ~#GBnn}>~+>Aa}q;L)|aAh4n z4qMVT8?mi@nG=dCXD}GUxu~Eqb5B$uVPOo9sv6H8lCg?{)zPWZuTW|%@6;J?hGGy$ z+|SC_#}DwxN|sS8MXP9AsRql2nfNaeJNIts*0kW;E{inq9h$!?$752W352m3mq#^) zx>`?BXsMc1X>#f!=^r)x4?K*(GNb}BAOHursG~=K4?M91fCWGR=L&XAsJ$6puGXnR zK3uq53M^C>o*1%r!@JNB#lNkXD;1ix2PWIsDk`VlZQ;|lRerP8?&;7MF1XcURLuv< z*4HL_G1oKs|Ld90NQBQn(+C(*488U*+l;0Rf~Agf2k*>Ez{|5T?*@Tqt;E;W!Q;lX zYoO`klSxYKy9<7goR})sew~$=l<&bTk5W0^6nPVu7 z{sw=`{hU)L*Q_cL{r{txFA1rL{hq949U{+NxBfua*ZGabwQkP@|Z`n=UCJF$3I+k)p6Lvd1t=5LQ`00PQVTqbV;+mglWm$n2A;Ulxo_Dgxtsr=O?k z)>Oc#O)_H8h+b--A+Eg?Y?!mUG|(pDbxM|^Db5|cYMqF$!^M;JYOFM~T5hIz6dyLi zSZQg`Is{OxZ;mYA;tS_)9J&TYxN8)%?L9pjRCjhtrUkD1>XwR-4#P+}dy2h>{xGQv6F275|5U$Zd1vtXkz&Iq)*dMl++SIMfq$NS#sy?Y>HwTv14Y7tq&( z+=oeT6Ew8SDoJ$K00~IYa5Ip}J6gUjd z6D3gxM=I!BF+&T_@xd+gaRsK+nBKkz*cl`+bTmt(6D^qHc)Rqbq_GMS zhbwiD?~ZXG2LTcv247%ELMWB~0IeUaa{=OaDtUy?IF}bm;2E^Ud~BLiVSxQJNdeah z;rk9oMUQPzlWiVp_J~--!7Ae-_i5p^rH0(N-Zx#}KHO0Gc-PUP76FE>kQ5Mxxv;>knuTgUwN|e zUUli3RS=y4%MRd3O=bObMHRr+(DGi9xBY<{nK62=o;ASMTkSBhZVL02PyOyJgGeIk z{}EZ2z;qoXv%a&iM^z+!W;QND$Fg4Ycs`<%D~Yoqecn^ci?(xWd1n9Po=8lY`E~I2 z58GK3_o1%Qn~wM@_lio+Cdd9R`}sK@Os$f$vx}kfw|6dtkF-XYUHU!-lQn=^;1aKK zXgg8gD3JMChhn++$qD80bNtNca6yB9L$wjTeL%dBT(H5rMU z`pOua-MiLhecQYjIye$ks9X(K6;T1^1xF_IY79!a!)=h$c1f&$8@c00{yk&>M4xG# z_Cw2IzPIyd&4K;kWf5h$yLvB72}ad0H7?P$Hr!GgO!u=|z>M+o>6Q4-jGh#{%n@cu z;*S&ThG@dKh*i6}+(rN~ZSh!bSip6I#^A7BpA#0B>P4D2^qWix;#e2)4|!CB>MRje z#QIa!)9~UvpkOvNj3Gtls8QArC3XtTGXQPTO$MZva@^!gDEm!+LePEHKFIb8{N6%Y2B) zU^SXcd2Fw;=!F>I1U>}+J?$nfd&M2T-e>zC<%FUwfDk@Em6fM+4hlR*bbr{g1|vt( zLY;I4>oFJhDHH$B!041OL82de6q(RTPXWS6?bvmj*ESX-i%<`-o(0})9-eIJfUSge z!idoQdvG9!*6cnqwv-T!-@OD@tFk4wwTc1Q&;$^9Tj#M|0lyQ`=0|B@V75NZe}ZO| zNwYdiifG?B$v&q3HuR?|D-yTS2IrHC(eig?xY^vlcTMZ9ntsPz2>eg zALGv+&^V&bjYegF3E&9F>y1HvwJIknMlF&?2E_;XvU@WmDM+URl8JFJGZ&*JF;Ul{kLYxAfQtj5e7uw;s09m(E8w>ByRl zd&a6uGyg6LKc-?fxx*JgMP?50(wY}@#T0}L~d4iYdjo0bri~0jm3ArO|`!@m0nlxO#Ck2lxWA^Sa*B( zhKnB%Y!X`oTbP1W6O|wgO6Nky5qWE4(4Z=A1(~$ScA_7W*sQQh=SAtj1iI*wtf~>0 zPkO(+2J8c~wp6=Q@=8ah*Fov`|H*qewEw!xi=vXz6Sa{9#mQk9*%{yHIS*#JT95$w z@{MMLxpf;;D}ywTq=e!q=U&)_mH%93y@xORd1yNVUjyD~;~x`c|Eo?ok%d!#q--MV zLsoU6?yksa(leW|*V>PmRsIWma~hKZv0pN#z!PfGcbW1r5tOL^$(0HX72)AG)M0Zn zr`RF7Ry|hWt`EwZE*9^hfk>r^yHu;&7t1`WMt^<5d%b{8%}5uoAeH8~X2@gUV=`Y? zmAzyBOMxuE7MZM(GA}f+PB@%-0`bm!2({&Ut@5pa$M^wJ08gG&KS&z~#=meE_Qr!L z1^UXQcOYRQk(gCw64(7+if!zx*^@^MtkJ;+$V%cngSeVyeYtVD;b)peQC6t&_)T(#&G+LLr<#R_aAxd;ftb_+*zjp&g_N|f(#oa4v;}qfY zDbW2X&jOWHaBOZz4R_puj>ZjzSAagewJ6h+IZY6UcLf z8#3RmKNeeqxMM8azv^ZPa!F1}3CZ@Q`hi99^=XoFhwYLAI%wQcHyc#f&S0ZVc3rz| zc=4UCsGYiwMZZ23g|IXO{gm0C2qNH7S*k3Hlz0r5NCbl|*5V|t)debLp*HVoZ z3*+>H!rL1(+8tfSIq%?JUkvY_21!ZJ95$Qz($ymoVI>$Mvt1*KqK*tSxK>Y_-k1w! zN*^_7aRi0bmi;}lXOEE&eW@eKkXL#5Vf||al z*hye6m2+`k3$PXEUD$%zo)S~F=7g{*-0^>R^*?$uKudQ0mLvWI|I_)4E2>~qD=*~$ z35*(t(kH$_KGd?{c7uFHiYU~0Fh|HxE~9jEykUXPCt$z^$$246FQPziaom;DjQN_2 z?Ff$Wy_LP1ARzCLvV}eJ<6CcB8TYW}TfnTob{hZvtTd1Rq>~>@>(kpxedaaEH5TH% zfchr>?nLi%_gPrufBpakfw4SRMMjm?#9f|U8#!;$kDTV9{*Wm`4{lspQn%>X_cNpO zyv9MeqX7;w8gta+aN)A4e9?pD4 zP}iEEFIw?hJK4YUzVpCgwti1EIDl$v1cr+URpAvt+COie8lDJ39Fy~4 z2&{dypb*s3)HHlOLV(F~)9cYZGcM{nqVgxW|&M;#h%`7t^I~$9X8-{De6_Vm#QvYJN-A<|1c-JO)lkjlqJpR1@Os!xfN_mNB zLA8eIAJP-$$;08+N6cx|c*%JP57%BrYBY&x3aRg2pX+hA32=AGd@S385jmN$x6Rvi zx^hf5i9om2PWo()d26ig06FISa+!X}=xGqN)y+UAcSwLFGq64@*OQTQ-d{n}Pdb@Q zJT>Nl(QcsrE%4l*h~tlrdgh9#uN0iY_z^!r*yh;c2Is!xTUZb7tO z7rm}Zoj6C*1aum=2V~>1aYJ&}F$usRn_l=Wx`P^kAU66#`!H0-65*P-3h!-G9yvsd zET6Q2+i9@fss{8@Ki|KfD4c>i_u2$GCa2N>(H=()D#hAUc<~P{*tosoKL{HTVA2U19F&V*tUYGkYo&*^@Y04I_+uklPR#~&SnT2Cb2<4+n%{n)YxdE zFZBiK8AFM{qUakE2RsObD8762Pos+ocyg`;A>2Bya=6M;cZf#1C%Un()`-dZVdVK{ zbrC5O$GKV={M}<&5`pIQ5V!p7T^)pw)GSNc9# z?y)|nRPVolD`k`((o&Susm5b%@IzakHB=0DYx({>QYjIVv*H{Gz*S+R%Dsvwl_Lkj z^Fc58P*6w@esV4oWI7sq7hTNHJ_yRxKvsw63O2cXm6BxIFh)v2nt?yncINKMnFg3$ zZnLA?T$-+J4=t$=g<8HZ@nHaTtT-SEVC{of6hs-$?HbfJ8FYx&@=pFgkhE?k# zT`CiRk!E^=$PYB{wUeeNhXRej%nsM-mdCETKDdU)xFO~B+R>HL)K445xk0zEkKR+V z3s27+aD=`5su5XV8~zb{hn#~Sz{I27Go01UgGc&ar99qdcaX5P;2qM;{=n&9M7GGE zzsE?Mf5oqHxNAg5ogwhXDrziEkOF3x3?))daRbHb>m}c_((iA>AprVsyfWf8n;AdC z{Au&AxHz2oxS7Kri*EW0f{FJi-RdYiaCQV^P0J;q_YQOa{xD}?wRz7H6IcH<>8Lyu;R5vhJMEDLES( zi!`GSSg0N>&SRv`kU9wC6=Maxle8JsuEDHECqz9dE%qnfA=ZschO-tKtg^V*`8<5N$gZ+<9R;_IU$S|#msCF3zVPBG~nSefM^hiBI_M{}f0%y^DY9wdfN;KK6IUyztP~$qa#jXki z%w_;cZ#A$ylGG-uyIk=*ghu(Wb2G)7la2^s@D=Lv4v(FcssrQcj-KatOpfv|Oy#Np z6x$jzxINn3z}~$yQO<10m>!wyQaK1t8sM8MJ0;PH^IHs`*&rM-$86<&TaE6r(hY6P zob+awEBi9>(5(bY;!EHE?QvK}9Bs5WB*q*A%&Np+Q(XmJLjAp{{6hrsY(TN+ahc^9 z#{AxM&8qB|Q$P`B?<>i`kWe{xM7&26cQ@cF^RWJS$my4M5+9$%988ci{Dh*8ilW zE?>H#txz=W*J^N9MK6IVm$l-N{776peuev>dLd&A1;Q~fG=>O;mH$WteF*gYI(kSL-`h8ytiKb0JA+2UTaBifOi;3F*i63-}ooVJsa6Nw<3O{ zl;u&@``E4kj22WWH!;l*7wRTnAC#C)U!)0NjVia?w9FGEj_Qj>jF$q0}=kTJ4Ft z$>x@b>Xqbk63dlitDsQA?=A($*|;i^M|U_stL%%6fP-n7mY$aMl+PmsW2TZNv2Kc( zZqDT`4l6nkpj;6>whikgH-Xp#Fiz)~-K<1(jSkkq{NSQ|Wkhyu*x3fm@?|;KqFC(+ z!&z-z1XhgZ7!y4LgY_Kb)A^zuX@hjr!f2Z4CK@3roC&KS6&QWqKNsWNj+{qd#REM7^cTU2?OyJ2Fc-#b9jlE~U`iJ;5@! zEGZZeL)?oIqR=rk#~?}&R$1Tx_5=q1tdFmieNHz$T2xx5BoidqOLLI1kU{QKLJO=|QY5t>#Kb zo`*vR#*vBS(b;IlH$&@8DS0C)s2lS@gU=wzd-lHD!_}R}pKduU=XkALeK!W8(0O3N7TsHi z4fdz?q8_mRn^dRP9RF79}%3MB<#+YxZI;L@HQ=#tC zUA#HA#F|;8W>c)|+bcf&UU;n>^Z|@Eb{BUbvJ-uV|FU0>IF?Px6>Y_@P8m9=8_x?E zYQYr~E(6&-lWYifXuH`?M6bag&*DeFn0^(#9-1;~b9Cg5mzR)1D^FS!Af;(P3 zR@MmpZ7`5&`QIr89@qmX079V{`rM1kUYT#m zFaU%b^tJJF^lP(^(GleF?NXx@sg?sG6b)$o5T6vfI+bzet---Y%!*rmG#P^H}+&+=nv_K@yVeqnxrR639 zx~DWbXU79O8R4|y`Xnix{q_rRm$-??lu-)D^A9tKL${W`?XyiApJn?^#`$@74GD9> zUm!=;bUDqY7Z||BXLc|~E-aSCFivU6(SAMI==6I1z`k;fG_AD>606^cz^NqRfTGn` z0$*|#!K<@)nZK)#=kW;@F(F%BUVL8@DN4N_zFI&1!84e@tlFF3!r@&eC zGyT%33QYKD7;e&K8{Jo@OFozwg6IG9VWB#O=&#|Tq zvx5#B%ZTc~2>*TM{3_%%?!1ZH70?JK-jtEP#8l8RXJWEi{z(L6iZroXlE@gT?tEUVkYD!|)0O^nrHrR_G zT8jyvWIyxkCQ?-{`HKlftr_`3Z=I$w7r*?mw2nnvP|PKfg!uW_)YAxh-`9R=DZC^k zvNCDK>}$SMq!ae}>0L0_AbkoZYi{4h%`gn?tRB-eG=VkfL1ho#2)65&;L+%W3;@ZC!c}r0OarfEHJ{NE z$Su_NV$atJpIJQ?PL~<3W%@r==2}zYiP0;?G=t*0)VY!j)vPkJ`pMb)rKi~AH4rkO&HKZkBL?xT?~#`#L67O92`hR!XC>`kZ_>Py!BmuB&7(oZ54N$SQO4B9;Vk zY6lqY>c~9D{sd<8^rj;fFL^DWljtdCSK3c4j0^oTm!773T3cL$QTbY+At*zz9UB&0 zfYFa9e-VB78McQj`eu%G$Oh&TR>5d1$k^S{r#ugHu>~BXdl~_4BiZ?n*lhTsw;;*{SBWXhD)!Td}}k znikmrK&-c>rbXFDC{E&&=yuxE>~eUI_7BlJ@w!xx0{ZkHpE5TjWWZdr&L8+XoKQ~k zz|=m7R+haN7-8K_X`~f6P>z4W4(f`fn?Yo^>A~7=H$NM$-%EMVOV+=_gX%9Pz=yCh zmIh?uO=oy@h4~1X09~h_2B?(n%zfYwVR5AcYx$$bjSXTA(6QY0Bu1a`hB+0*`(Dft ze2PY+O=lF%)sNyq;Y89lxA~kToX*!$&$Ik7!KUstaUb;~HZ9*diiL-~A(<9O{J#m3 z-%k;x7;(&O7ni&RoCD|2i!3G%Jw=_O`}vE68h#J8X$Uzq4v7_#`U)JI7GlMMt(HFn z$jI%e_t^Tnt7qW-Gh8O60^I*?n{xky%?0uM8-x&@QhU6mLSaR4jv=KHC-j=2X-BS| z(6+ONvLI0d*~ZU6*K%fc*aB>rvjT?9uN7nG@wxp2da>xOh=$cX=1lZ?(D2LV)$#i+ zrAtH&1%$P}>ITvn4}a7&hZ<0QHjtvmY?ixV!rTIOY+wNjjiprs*MOMTy42C_L=$scNi@F zIx<=^MiD8)%QRvs`27zFYTem`HDQjRXAeHTOS9*17^9j1e^_qni5my=x#yH+VsT;i zg6A&;S}2!lu#Pvg=Mwj$Ia2~iOf@2FrRcHg?J2##vk?NyC#dEyxAoi^3dXq?a$Qs~ z(r6_DnVeW8Avd2Ky(1PzFZV!4b=@Ni4L5yXdQFMj*kx|rms_~%1j_V?;uD5)0zF^n zZ#R?k1syZ|?w)n@vMSZL0y~FYxPp|i7o^K0e6>PL`G@Q@oDUObf z&vhpRnlGoyGdjNw#Qg5U9Wo=GX=9SWW1Jp`NBY5Ly})wYZSE$xS9#^ExM`A087n=L z)TEYC#idXB&tZV$UL-ihuugDKTR`Q7vs6lYje)M5%>Ik`M=0;s{mWk;th6xH-`SV= zR-z}ORU1VALIneQ3;OfGzlltX4(rx@SD(1?=o+A`s+z5h?Z`%Tx5z~W(+czV;gc zK~F#&&`?auNFvu7;zMJfFM6_`puaPaMdfuFf8cka+=n%%j%h`cQehs*nf;2H5~A9} zQS=b5xT2JzI1duk4u}i%rkq+Rl;NE%zDQ8?Gtrt0gh<&X%Wf@UM&*1FA50W|A$)SV zcVa9DZ*()OvIDj~O=UYfpXc36N#FVi;d^C=VGk|?*0-jBbQ&NX_FY@vH(9PScWm&t zA0?xmE3GTSJL6nYJoh3G*YNv2D;fjr3QvMlhP)r&;orSeTfH#Fr;k0`yrT8G$S?Cp zIrjP5Z2)(z-=)BwxKzR>5TO}q$goio%!uPAMX4kf*FAk+!RW>Bl?^FIG^j@$Bc`+Q zmmyAjP8Gr1dijW-I>7%hQ;#nF2A5;|yq?drfK${{v@Bmq@G2S9>t*{fVrY}PZ(sre zW4OcUQqkY5Z(gS^^)ilf*f~S|%Q9tjx{+~-6I(7M@5>Jf7mAB;#N)0LgU7vhI3LLB zsD+2FrtZa|wdbKZsJ+$V!{?!HJc@tWP*Ca@VY+lqIX^6iYciPZGd%H~<6#2OvD@NaOCo#srnO?moczm^Rq+6$G2qW? zbIrcNqiVebr_lQUE|HV>mOV_+dwe{F6h?~KC{+FUI84;#XbLg)-w?7rk;k12WqYd*`i<-Ixoib>5)w^iGJBTf`-sFHod z(NvW;O(OiqslMx2$;^$U7iQJ_MG-uQuJG$mVp<<6+GauHPp@u;ZS4d5_OT>X$wnoQ ztrRmjD|YVM%~h!D5Z)o&e+)U3VKP~O55~52g&4+TP0+A(GFx|4iDce>##`z*{-Uz@l(0RZT3@^;623Tt5UHG>`37$jr<35#3>h2nSI2Bd zz0gXFWsW3?@B2NmQ-45|42{mf#0aW0@xQO1OlRJ&6{!l&nA>!;&Zm$Q zm>fJkghRHG(E(_aUX*aA?DMXJ6|IM^6@Y{t+1MB3r6sKnkS9x+Saq2`9srls@JN0r z)7(0EuMnSHozhX(tWeQPXS!~HsxZ$#!eJxJZj~y@3y-!%!O_0EOS~P zyI+j;_yJvVkoSk`Mk)}^UBhGWE9k3)i|{cq-ua(YM++?{q=46QjfP%}2}u^kT3AE` z0J%K<8hVF(87716Kxv`{S8jI-Jh;tKxM_yB)r3ySjt(TdKb!;Vn={JpeVlb&-aruiuG{VJ6am#3i3xUS1pc8j-84D)Mq zdHTx{i_=JZ?QREGu6p!<4qnVh1;{|+&{bK+s5G9y*5j>>9vk9HvnIzOc*Dxte{7q$ z!bpAz#MLF?D?3-KPHKHl!F&JX+UGgFR@P6H7E%M+4SEfB*a@e76~8-py_EY^?6w17 zWMcuFLVkYyZ!`(0ad}5cr7(CWeBNE0Y|bxM0TFPnI0dxv=Hlw&fskfj_=WCDKbh#J zwR~aaU5;SEhJ8%EFZm5*L`GhEqMW8aD4r!bK{eyJr@VEO?efjwm$8 z-jfQU&ghRh{6d3fp$&UTgW2){c=(NELCgw1!J5TYRiupuOWNgv-@*xv{UR|X5iQM< z7FIUq@#2SH`~9f27GptsBYB;+apB(UR3uvJ0i377wBLO0E1H=NV=~c%nR>QXox1C< z!u*#RBR4QeCtveHMfc3<%y^39gMv&Q3A7EXy&tU;*Ex}Ed%fRBFlufmBT%@}k}hxm z2U$=u&~I<*3UA8x0>cW&fFN%|zeBpnh%$^1laN?LOB(jm%@S~X<-=*+nm~Qtiof2-*yL<9tm`L} zv+jSOMg-DG$aLq<9@qd<;z2(-*&r(f!S@Ji9{<;D)lSe0A0yH*vDEn?gHHH&`A6^{j=ADzIG|oZ$ui3c zaskO!qSc~tn?{JM7E3i*G$vLW>-|J^ue%nx#1VQ< zjjE8C{&HluGUevH)Yy((rnlZ*;dZnhOR5Fp*p-_A9nxIz`4M2H_glP@nUjI_u(16&2e4jyGav!sG>_XitJp1Jne@{kKYFuW z56dx%)$g?Yu#!(4+W-ozLy5)CU3(+P?VDy4!RNWx-XhVgAoH{@LOm(X(V+85S}HRQ zcYJ9}rXu|2ZiMFB(DuFR29}rA5YC;fAd=vM(3+Zj}8N8`ezp5)j7lWBqZJJ=D_VFHz8JQsyR!Pp-sukj`m?eo=b3pog96>DP05E zY}mC5V&G)y>F~?wM0s2?-)Uhy>$#)JI+tGp$i| z&#XX1bRw=A%JWwLIt!n}H^33kCmYb@n*KZc4RUh-`u6`6@5 z2lQF#z<-V6Eme*(6tXxEIow&oWVd2+B8xgEJrK6iVYA8sD=UE0DPh-Pa2d5hhjO(k zO`~5jU*Fg1*0M=2lHeg77P)i(mn;MLgszB8erYZUU8DOYN&Uo?P8k$g2Yax8FI5i1| zeXGYZE_YAxB{XrQnK5$h&#I?13HG z9ZMPCj?}*k*{V9Wc`Q=eUchyZLiqlr(%!AK0P~9cIms{yrmAazvPUj|G~#IkbbF#F z^1i3|?@b_BuQam_Et?LRWMPqmpEm%`me6RG=|Kh@o_T{*%w_Cfu?rWz_2*z=9yu(B zF0Tp_isyA3Re+a!Bn?XvBUY21x|1K{ax(@05gcgQ5_t+tFvW6ZI)C?9p3W%mL-JX9 zE#QloLl_g0Lt574wc1@k3`YEq04(lpU56iC^wrdC3?$?lv5!sW2vH&8bST+JE7_lk z``GK2*GOz3$SK|c$`DI#r!)%$WwD+e5-~u->!iy*n81R#ljo$3J;EFpjsD1@8=)>S zq$IiFYMG0Y13+(st}V-JtiKP#)WfkM*oxysJ`GJYo=Oz^82?RCDjWR6>O_? zbgR~Y^5e56b@kD_;KNX1Q7I@*;%bg=f%QhuUG3G-4VmswUSJD^zlgfyuH@L-!r!{b0WAb&+QOclY*omjLp^Qs8|uj^+)Ek0K{kizJ=`uUcoRT1Uguc zcEpC|D*#@ghbVL0k#pz*nixBRQKsZ8yKF&fvNHxpVB${2k=;4f83O0BnoYLE#GMDz$=*18p?*h%OYxAl|xJ0eW97=WO(flWQGK9;LjM zzDZzfqhMV5p1eEQ3>5P8xlH4o$-~j`(vec2aDG7b?9oI>Uj|XVv-tY&jj&iRLP9zT zC@fp1m<&&--$?yGWQJw}AGuO3b}*E)c+KTjy(4&m95G^;_8K+VCLpp5LI->~;Vb$d zUXhV(&P8_JDD(gDRqwb`*I|9T+ArQLHFrIp9;uCJbzDeSywqVM;? zChQW%SeV`_&=pO+YaTceQAYSfI8 z30!TpYGX+5`yYN*(wYX#f-{k}y49*dqk*HnFz>P~3z|v=8(+VEP7g8^Sd2Tw<##(& zRbiybc*t-=76STu4SRmoQ8S~fUt_~>7ehX46@>EnnGgrDkO0S`LjIg*eoaEG$t3**SlG{gk@(&0q3MBh9SNWqYt*9I37#W^#E zwhLedVj+$fg>ka*yH~i{P6Kto`%j_GMTEd;j92aifQ4J;B~t12KYL!`lMi zL0pJM{Z^U~eS)#)A~xQFV9zkNKFXdz4++$F0vh$E#o3h_F-U(?$`HwJ#fzt4NzuT8 zqL)#u!6ohhQ6;7XE3D6=#NY8X9=$2x01$8=5khxJbzHi9y-w^1kO5O~-r1|wL_fYZ z)U|L+pU7*Iztx)@hWszYc^!6KPw;k0tHE8atwjYq1jPpBF5& z`_P}um6A)IIm*kX@Z$6G%sd)EvH_5g1ScK16l6~TA67WXlS%QMgIvUb34~Ko({hg~ zY!!{VVHXkQSep0cI!>*d6TbV{Km9VUx4qxwv~_(g4>Y39I=CS7WC5zP4M0-6wR~)( z+gyC%XnYlMRUkn$dvd@24dB#5*ih4L4xwO=cOpy6rqrTq0ShClmuhE>G6-qIj~{G5 z$TI?~$B%@7MX9*xW~aeta|>J^Rc6yy$*275d9`|?Yw=3VkOosash?3;D<>zd| z{Mn~5GCHC=HCsaBe5I#*ar%=JudpfU?WY$+Io)T1_l}U~0kTf$o(}ax{xdW&Bx%-r z%gC~;lH=92oa1Vlk{fIlQ3t^NrHAYDGFysp<&zP+s$ltveP4`@Fuq~vEK0TGKkbEv z9P*R(k_Jh%a-b9^0SP{uwG3!f0@%&MBZ^EA)hl+dTYK^~I~&Gmkasd?HyzzB6tpxk zFM2yP^ZNgq*v8+tVh3}7MJ%>9sSir8%r_;J0RXy6i%==$l3n9S{G#)xeFIL$d*Y^g zU6^qqck`bbZ_OMt{GNwsbtVeSChZ}Fg$$nVmI@=q>UskQ87CZ@1@1}QvyYff#-wex z)?l1U9^k8rM=Mw>D^OsvAzIsn_0a ze{{QIIuzqW9S87@*jnCB+@3$ZR|fuX25(<(6FnZ(j4w;lO|rria?700ftjVrm{_}b z!Uq(F5*x+APPbW+#Ja@_aM-GNU`V5q8->%+~4WmJt~hV zAO7uu*d4gi3la^F(Q6<*G<%BOY@%a>87VNFj1VuF&#iaE!Uq_BNn-*Evf0j)fLf zKuqr2m&D`+hO(YF*ckl&(YFIW#&NG)4j}OjBk6wu_@30AG2nhL=0|zi4+2+Go-sYtdu)@w`M0oh^Y;tCJydT;QM&Wfp?#72Z}yS zjWu(&awUH~ruRg8GNBVG5=aBw(GjCaF*L^@N)T3A-~gLnzqAh%dJ~(I1qK02zN(p* zIW;DtVHg~T#t*+y^a^pWR=bGv8rb~tP|j+k?((Xy4l~AP@@H3M&AqwL}xsr`5(&%CUA)B z@2VK5N48eTPYbD+Ss`g^doArzLg1_&7}6T*k5+gpE7Q(8r~5vIt5Ag`coLuyL$ z71+lip@W-?@!P|&^VklN#e%6lz-@LMcVkwgU5)#(=nS;*sMJhyQv(&u`2c{s_aag^ zwuW6`&eZbvX8|~6$-S${)^YQ4@vNcV3ZP2+n^Tn~`<$Lf-{0YqfwLGC_DXHWCSocu z3p{y#a@FlgOyzB!i75CRfz!X1-Wiv56*GRM&ilGb*!;>b@oeGmE?D?C8?KNp&=9P7 zCvr1fU)S#NqmiyZv-_vKOp^fv=*Cr0ljkEh!A)Me7FrNcj>1AW;xQSHs3;?=@V&ps zL`ys#HBvO}BFL$!UfJr?=*6p=Ei)~Wj+mhkZd(#(u;>4{omGt{ej^AsfDUR0+sK69oJcbfTHc1Y zj)W?skoI19B}>+pBTbv6&sMP=+VB(d=^O|h3#Oh6H=%($#SyLvdw0AwE+(`4`UjM-aTXf) z5K(m|@LYnVUw^Uv(th5=GujezTzUq?*dK#Sxc=C*zM6;OQD+pI00Bue!g*FBV(l)L zy032O!(J{5RfDK_f6?=|?xbeel~SYyMC$|zXpYo2k6I3t^fWW$Rfp=D1{LRHV>^ArDO#?$ zjjQTBTOpG9tn7Sae-D8P!NKN9Ihf4-P%66yvxP${B4bGhvEdiWU$ z<_ABg|B1$A`hcfgSG+R-mq`RwZq*uUKtAc6ks zy_3W;xM?B0?J7a6Be4%F?UN<#FE6QIEPQXPRD;`k-fPHw`DPa)G>PlAH0u?&5#p9f z)*d78?Q2bt03#oBe734{ytR2^Gg8UIrr_ec*2$r(D@L&1D`hp}J zN$Zh8amL>QVl0b|yt5Q_ZQ{^z>XIUJLVVzd>PmklX3`Y4Ch4G`gVF1vDbe-HfTb=j zRy+|NMXEtY1iu~K!J&|Mh|c21j76M+BN|ZldMGVz2a`vJKunxU8*;$Ody40jG39{Z^zDW|>Q9h?5LkWH5eH zXY7dlnz*b3YoBYg)V(AV=64=xzl)Rhfh=`l}8VN+%fpUc%jIyA?HyoCAE)w=d# z(ld+ibM4Ogncb=OFqBC(ceQOl_;(Orh=3aEzePL`w#c8QSMWiGP>OITE7=>qz1x6# z&ORWc29s8U)3@M@J)@fXN1i~k%DH9jdo^kKH?chZX+FUYn*6_!*(#y@b1l;!j#!Yd zP_8e9-^Ix9Gqv>(B)%$+3?&x$r8uZqA$VJ9NH92%RCupXN#OwDTuyq& zO#g%~aq&#NAu&cd)R?0c(nrefnB7OEm2|rE+g2V$7x|+*C!Q0LAFF~kR<)3mFJI8mnx7$(eMA~l;SeO ziwFY!tJRrWl_oh!1u;gz2jKRZ@%AM>CCgDJ0I*tZ`>O_wT-rgW>Lmi{9rdRiGZD1UtHy^aH1XKgc*-4yeN95e z^{S5*>qcDF%T{%z8@~u0=nXgjQ)aaMW<-FrgT_bcA>W`vdf_M()H3PDRte*3z7^Kf z-lKYTdB?m#dmeVdE2`_VxT)YvV{{URcPk{xpOI2;i*W3|i!j+Z15KR7B+>p)qV@Vm z3?I%h7~s$Q1vI-QL7<(Td*~=^59C%S;_()+P~HCnHX@Lw?%C??l!f=R1%;)Sl*-LX z!o`T!X$cVB+DZflk(b(l-A9O1pD7)KkaK+Y)oa8J`A#>mGSmqv5kv#P`w^qS38pxL zK!Wf9%_IY&L6i{!A851`7xb-7 zGQ)PB1ZXYV=pvOZ)m*@tYc|Q7$i2&hCFdek_ptDh{Z+kk9P=91j#h@7r>d|~!Gv^# zOX`ALEB7`ix#4o!Z@_|9e&Wxw<~b>mhi-(t&El&U{ueFkqiVcl?QqI9=l!$W7|`r+ z=`#S|>S+}5o##DcEnjY>GbMEGMjhDh9@TPp{TsE$;||Y!TiRJ{`-CJ+jc|>bNKilR zs51`emyXMqtJSvMI?MmXPOlt)h@+9O_6jZlQn7mER;Uw$=RF6ID~91CCmq$7>p>$1Z5ey9JyPMGFL*VtR9r$XwdhX?Zl_|;6eu6olVcC;TL?qmy6FCqIqCoMy|*KA z2F1H*ie_uVE&Ks%r`?JtMsEuXS;SU8&r!V-o+{0ipDXuYFI>)Dx)nX`s@ckRYv61aNHvG{Sug=rCt*%DfN);Hm8unc{rtGQDq7I#%4IxJewn9Av0dE$<$pk47 z&z5=sGcNpAbc+y-=(y^M#!CgsyrGfYEJ)q&E%*T9U)=af4X;u>_>w>n+cE2Fi~n0W z&Ahd4&Q0Td!680(+m;ISP(c10iH>#IiUhSY1d=gzrasRm+A~YRA0Zaj5U+8xGPN;K zg^hJndbtN{jJ00L-}i^f^x}vOj+OCo{5r6>tmvEz&mH}?I|q#R)C6t$H{^-P>i84k z9)c8cTZ@p}vDLfq6E19t2jaDWTueo}wo>p}^lHN>t7jpND&kEObY)m?j%9}6xl-F1 zn7Vr|&W4DT>JEP~^$gNYKa_?3+!Arc0SpLCa~-uyB{Op)G=$!h`KE)O%&F9A-y{|Z zg%^hx&Mjw&duQUx{Hd(B%NXtuw!th`Ot}}}abG0WxwUbYPw|wfKgW}l>=%gohNmHg zRvR=h^9VUJCHs#xpq{V7Z(>LWmM;)w|6f2ypWjLlw5CkyOQ{r6h1v0pq9RH4%e5~z z&m>3uJVPP%I6-ExL&keEBPi?OQRks8fK z!$+=>KGdiqD)XRmhWTenmK2ei$J_g}F!*cU48VWxxsoy9WZYNXHL<#`DAApNBub0x zHCI(AIz?&P$(-7uB>pZFcMcHB#;QF&EOR=xPdM21zD5+(J$Kbapn!$|yt?f6sT-Ch zAwYV1RfcXwqIs-$0xOli0t{4$vr~-dW7Uprx6LUhBur0bLa__i^6Q*0#~1v|(m&=&Cm%9;hSq-pTo{wF26JbJ_~(DkTlScaNO9VeTjD5LuxE1{)$3kHZ>FF4K z-d8mv?7Udwb#wsFr{Av0&YIrI>qNPlpzND5akzyT&W)%Wnb0K>CF%RtV;3AZ@;pC$ zCBr{pm9bC|*xoHzG_i@4#YfCh`70*89a0;JXX_~2(Y+@@&_daaCXr>fCe$>CU;AC{ zZ-)Pvtg}J=6BA@RCID3kQ{Goy+2_Vw^rt5`)&pcyZ^zGq4@|t9kg}Jwn{SeN1ggIk zry!@UTWks(GL#-&-pA=H{8_&;8+nx@lqnaCEijPU{VyLn8Jtq~pIPO&R3%rgCrs-N zzsOVcilxU*+254tSPnfA0SMNLTejrZyEj&^jTq+kCnH1N!+=yne|p!2WPGu0ug!&& z7fHG8)c>bdqB7W|tsCy~w~b^Kky1MI&j_1*Z#uM)K)IcjrKx4s#YFDIn+#p>}E zudi14Cf8U&iKrfbiS6RK7D}@SHn{GHwmnTLi;|z(h0PSmp3(F`pA-KX3nx zXaD3iaxQ_e!)sHBN{^VAXVwaHMRGw_=i4Krh!e1EL%{ZxdU>1z8Bm_`>XZrUCA{tI z$DcjJPw;v6iYj03Oku78TD8du^TmFZeu)N?co%KGtaDIe24YXADq#6O} zGNcJIBOweK_9I7t6HIXgmRaBc&vz!O7)H+%CxO(rC04%ge%sugD{}?OVLyB zcpGZ&3Brghy3r{83~6h|O553%;`g=$nWwTb#eMU1)Txp|bKyFfXiJogGTfVL@)h64 z5sGg$X-RbK4o3lSj>yZJ5Ow`8t`Ur}^4lLIi-drd-Tnt|d-LUUW4Q5&yVzWJxSOV0 zmLzlEM7n3rl;k-H#`)rkqF(`qS9*GMy{+A8{FsAUa2!}2cVxM#A9Fvy0Im(nlvh{n zm{rx2ipRP`Y1?=H!bAW8#-;E}mvb=lr4(-L#bsM3G48f9dy9OZz&8oaRAMVRAyCx0 zzcx)$o58A#BFf2NX~bmGyhF_lk)N?*WqDG_*?My!0+AE!p22kVRp1_70)`CruAY>+o)sL+Tcz#h zm6fD-K|EhJKcy731KQMTkX|U}sPTXjuC)t#og_97claPR-X8#3fYJ?MP#3bKcs)e% zYCBQEA&71pxvb2HQBl!f5ui(zKfGWg+6C>nUZ>HM3L-Sjf=a)`XXWHnmaiA!06+R2 zdr}8aQ)bEbyAfUhb^S19mF=}P{mJ7e;POF39h?sOn~l*#k$M2mhp^{MzZeG_4Gh`a zU!cMuge5r4D1k;E-G6A=_2er)}jv^aZ>M+Wdg^w;UdBTM|x?KmK!Uyn38UUBI0OyDe>nI;-^ zgi2;KHXahe=f*PL<%>3rTzdeFKy$xU=874YU+Hs2l>5_3pfjL({ti`5>AhMpe4Nos zgNX6&I*vH#F=}5PEeMYsq*fEU`bM`Fa$p|Gim6H&kY4C`tKCxRSXXL{(wMI@uZinR zS}$8^lR{iC&_E#>>7B7@YBSFe4GF^W^?v$1&jLx^qxS;DaB~HF04&nzvhIoajcmYkyl-}_f0Az|Vhf|w$@7{L28qq6erz>4CzZi99nz^JM` z+azLVZNsOBYUsVDn%ih1&2=t%fv;t1B}7$AH+(ZP+-52NzyGuh;j!HQsm8IDKQ%p> zmp+Xc6oI&uPYQrqgDZa|FN;Lb7>ikWGMXdlg>^r~>moLl`zAy`kcnXyg!mF-+(I?W>CIf^ zl6u4#?!h^oGMfqeXvy26@zz)3-5g*Don92u==*!QOJN5p1I45hL0O-9#xhpr}!m{@~4BF}FQ{ z8Ugq+rwb_)AQyPi5u?BfrZ|E?g75(Q*H;H%gd??E0DMW2x?6l@BuuPMc_mOj0#-0E zx6jk&uVQTS9e>}x1RJYX06xm0YS-toy|2l`V>@4yFM1ThER0~2@u;jKZhKsWI@mTU zTf0#V+<Yx+s5OS)We9N!_^{@ze92oVH%nhM%9p_x_7I33;|INiG*BS}-ep0JTTR z-py}8ML=a>CiVbElVgh}^s>$YDqhxOyS5Yohp)sniH~OP9KA0Sy#QjOUsAEMOIyb- zC!!pkVhA*+w8~NP*a+Fw3G8HgHb8&(qA1;NECdjb>y5^YkA?7k#3_pkx#V+{7N_zX`dj@xXU@rS!LQTEK#!l}e+2TAwP^@cLK7ett4M7*v? zC%vImK#{VmHqj&di9y$i{onlv^*!0cACpFt4*5l;889XU6-&|!O{%x^Z=jpNGx4G8 za;`^?6Iz!UXBG|^((i_dcVZ@ylzrX#KT+j@5F&tBXwYms|x7lWp;}+(W}Fb zm39*jM41jFVD}CVf1K)}4Jc53SI!dMC~>3##rk#8_qj-+X)yKZc%Ig>C`P|2g&dbC zwP13z-bFc0FCQC-YG`(S>P3hErsx}cH`?Bhs-jK?la8yuyQ2esgM6j*afLvOT`$zK ztNo5?8xKrg_uXrm)*Jb?fX`;ky?#R{VW8^}GN%9uM*&4b2+T(=iT4Biut)?107)y} z%~7_+mq5~8v#hF%YRUeLIc_B4f(Z4?&?e%7^vp%VjFJth)s6LQZ61O@ln@N9`ii=> zOP@bIpQBuS<-3kiM|>v`2EDc-xfKdAFuseJKY2G{$w0*B;O0NK__l#mAK7TIqIk#q zXh$Awt-paEr9O8(Zh=ypt3=&;maAg~#A3~pT9G?=JDI?t)y|SGsJq-z`lG`B0A1CE zYZoQL;J9a2eG1oq?tHu;x=btt{&^2K<2 R;G?9X#>IAg6;1|l$jSy-y8;jJ8fyXrLfzzfbT!NSItmI{h$#|Od(z%IRhy2O^Hpo z`2l|Qm_9Co5y@iHC(VHF{NdIsdcxPzJ{2oEV>7jUX{6hp4brysKNcRc3VbMHOZ`f% z95QH;;v8L`olL+6I48}5-LCVc+CBr^xYEBLa7n48s;1JtCBQ9AY684uKv zkogp_#Abc~mKGho>J1+>S(0f?Lh&{BS^?*Tz1Oql@M>r0qb&579Wc^JkVKr2Gs-jAbXbE>Fm5LX8USy=Upb&LPfLz zm{Pegt_Dh#XY}rfH6UwxPe>5#=gESHFRlbuf z**%W|qr>uc0CrPQjUL)60Npa1s=eYK1oXcEf%FBYlmW+a62%{mLqRY|%9OF!v@x;%YU|mXv#m;YPnIxG5?zVat9>UF z>LqvOBg5L5Z#BKLBbtM8EM`R!o%U^xGJ{2xgeticN%NtD0URGAK~RMsscjmS8=ipt zK?L;YfI!d&{iC<`m#xW62o0gdC}cgawo>6f1&G_Kcn&lSLbg9ZE7l zS)|7XrM}!|mtVgArNSszHuPAsmVJ?rPNb&_;CRD_v8`tTJ@~MjM^;T^Boe6xoWI5) zuaJ@=>~2@63?QQ(9;FCIp4eRZI$wBpjNISzG>vd>mF0`fDHBX594>yDzksHxTMw#d zh-*WnD{~G$rdY}IGbp6aAsM8-&WUcyYs;X-`Zxr#yv2TzAlhPi;_;rsC3}Hmcr+dm z+G=;i+e#*-wo1FxA0q}qs3YlWle|Dix2J?^ZdCG9x8h8X2+1#R$Z^1ZhsUIPvHPBj z*wBFe5lHWAbtul=iu;X>@gjuS%NW%J+45REc=0m|&)9?y6SjdG8P9#DxGsKFRWy0) zA%{TBdGT@kH@`6S&MIyZk_exf%&$%e)I6d7ME?9Bja>l@&qdjI#GME|sX=Z2G?| zH&L`=^TA-hQM81S!Szgt4Y|)hU;tvNma#VwYgypQ08TW~Jz#dU9E9Bp#@@a|>P%1o z+Ta**!tapvWZJ9lSF8!nvec!_8~tLcEN{sFnPChLKn&aG1!qEtD~cvYJ;W?{nTvCC zz{Dj6*e;{hYpc_0Jw2F)dcpa5?JdFNRWSy36muQDoO;cpwf;N3JW#+VaHnTT%yl-Q zB`~FS$*ehO=~^HN6JdWF-vu!@9#u&;?P;EV9HkyeKI#%Ou7su$ib#s5OS0BO*i}EO zJs5mT{dOL=c!h?l#QS8ujeC5zumZlgA^g*bhe~y+t#4v*if?)L%7Kj+J${+OM0?T?SQxWunMc%95zNy z5@d#VMNUnFc5yi?5OdJ;DOmj~nj<~T+*Y3k2k|B>7ET%gh%(a(GAAJcxvnEefD=ry z1eRIg0RGfltYB#7aUSsG0V2|}mNt{rG|jV2_>}2dYiRYG6N;p!q>cZwrp7tb1>S`L z0fv{0vZ*Wa7T3Po2)RrH;~^sKhi}Xt^-chFQyB(CDT{pp*(5Z*WhZ`*_@4vEfiLjA zc%Ib)ivVPbj?50u5laxv-|ZUGAj@?3x2CDsxFx#$gQxg=Bw1UK`Q@AD&Cnw`u9^Ls zO`1nBWY2eYP-;_&%U?>327ZZ{Xin+)Q6GHe`6oO1b~);D4xp_~fTRby&?;COMeQ#O zHPkY{6`tNG6kt>^JWOqWBJ8I%^yudE()o)L1Fw(TuIIL{I*F8bSznmoB>X#v+!#k& z?42H;K;Vh0+#1LCcU5eD-i@b3f-RD@(BV5k%+$RZjH(saIko~;)^t{zYipL0`sCV_ zhrO>o6QL}YgKXIWUZ}OtY7shB{_|lVHS-M4OS!Tp6{KzmZ}YsFE`T28Ba*$_K@+L9 znPhy+DOrHvTl(LtcAMU}`#!W5y9nHhGm8m$og5*TW2Wt(mYt2F<(fn?KJRC6-Wl_r zo6f7MzA4<$A4l`9VzAwD>l?`RUzs2qSw-*g^0_R1_!iEVkTyzl8lrR+uYfd5?byQK zOgH1||E;?d*l(^PjctDZ9DzAB2)97|uf?5r(ubE(QHxl-Afl_*5m~b01B%^gps<+@ zEV&!z*NhywW*gaMW{fjPet*Piqsb4ar3;1!6u1($2gY9^??!U_G^!|get^n9w=?Yr z#KX~GxREHFY+l_22Hd_Pdb9jbpDEGl}YmK@Itiz7NLLCGW9}h`&V^3{|cX4ZLXZZ8gh?n(~O^4c)S56e4~%W=lZTR&^5sNuXM+b}_OMtNyg zN-*-sxyDg4wvpBTSTR^PJi0TNGEkDyk=_EmL4!DmhD`sKdK;2yE<38abgQ{TiJ$M< z3ze)3&8_q@(8aS)oQGGjx*rlmvNe$m7}6kUi8App+|UM{kD)tW0Dm9Y3~{6WXDJ_8 z_XSU5JKXUi+zfhG)s2lPhgmpxeO0!0MqOc(wvF_F=>s^|(&f8Vk*cj|uZOfAhtb-VSvuj?(P9LxhC} zf327#{%(}9*0={~;Hos%+rxMAn6V2iv&w{OTM|AzyDiE###T-S3dXra(ld)|C5w4@ zl^eS@n;;y=+L74j{{<*+SS@ZIf%7j)I1mDHBtCGtFgSiv?a(?dERK$E6{(q+7e-CW zCx1bID$lul8KWcv_!t<-aW-W*b3)KE!hIItP_dk5!Y4umM?PeL-5=8>d}q5mA3*ou zF-zLfLHvFdVs$oZ|n`i?1R2+er5rP@#n zzl#5$5$tolYsF+F2@1IA&ox9S(FE2^fHk(z)FrQf8)gjC;n{q;4Hy-2@4-sUdeTzLwu;ACVX{$@|)sj@lqZtg~j@@_YP%mV6R!G6FL z!1So}1z)~45Zz~g@Ph52$Xzj!Aa=Ox*X?LMnIwj)@w#D{+jOhP44J-pf&YUO9!eLy z8;4X*x&Yz${u*Xs5Y{EkhtemcRN!+y>2>@~0ofxUk*#sYsa*m66W$pyH+p2SYS}?REz*@oxTIAuaDz0wKu{=( zKhDwOS3tH<8W}fmMERp!?v*u9@LCg_Kh+aD|zA2G&f zveTaYqVWMuKzgPNv;}kv9(zcwIVEsO62}h3qgLfj03W9hup(~H{0}ugM)!2guMZaT zP`&wI`j>BgrSJi3>Yh|E$m zz(I0NQi2U?v$DF6hy$FCHyR18hNm%q?-(NLjDWlP4pXcLg`l7ik1n48aX^G+DGW2S z^r5*}V=03*>_z!2_!Fh4{V%I}YxS=iMTdw@aJg=8779NZDSAv63#2w^=F`iu>bsVId!unRi%wrre+xs=L?c zOu=q0TkQTT-FSd+r0lwxzhZ{*=*Ljn43@B+?b6`T#r&0by|KN_<(&;lD|kurmd~*< z@4{w{%w*suk=$3Y0=@p(40PJb(}lpRs9s%9cyqx0pT=@gPeVF1HXk4qHiZySx_aYo zf|@k!MI{(?_FxHkT{EQjc6kt=M`}so`ftXI&e}wen9Oho&`D3i;;bpL|cHlc}R(hWczmOehED#WFr;(xM zSSWnLAl=rWrwHif=Q-&m-qV>&j9H@&Kgq1?11i}p=Rp*(&)RN zfbbE0c;bv89%WpYjk7!7N&pa*x;IG6*iIJy*f29?%R3}*=cc=9(B!(95dZh)tp>!| z={+-8)Bk4LKoqO3io5=wXKQ6x=L-})22f@Y!)wMK8;HYPH~uLCJj^`DE7&W>xS`|x zVY{0Oh+=clc~|bu{a5E?{wdhgzP#@ye8K&fBUXJTkH@>}51SZ&M(rDCI$=*+MTYFF zCScy6 zuHpcnSGi1K|BrDa76kb*)wEf0qm%flb@EMuLbPM+&4h9=RHSH6OMwjJl7q!3<0^{@ zLtHa(O5!B)3vt7NMPpz)nWP9Nv@)mtlW|BfnYt=jAEtQCgOcmuw)3Xpb(7)ERon3g zV@L?(eva_M!p|mdp}=3;Su*glqE91%I&i#nY{B|dt?V;c3!bKh$;h5cH6?{?l4Ee1l0zE$zJAO zJYIEi(A^`{`+TFxUU0m(YnZSsL-9BK%r~KTYnQ@?_F5I{7N}L8D$nKr95S2>DHTKo zVEqxJzzL=~fw?2zM|)#~gB8`66l0eHHC;-_0-=372zM2PB)54?=g@p|Dr zkGsEH*zvv541xbwZsV|nG$O9l)XI1M_b;B`Pm^?|GyyC04 zjQQS(B%!c(E)Udw@;Ej_0!#Xra6Hzj5JOsN$WGjQ4M5;)M;vF!DeUo_pzY=?CUZ9{ zfcz$94RYf?2<|fvIdln}zO3AmJwBi@k35Xi_z*lt0yXp-I{*6EJ(uEGYEyaIPNky& zA}x#vL5id6)UvaV_~M5A!vls1iQgeVecqBhf@I0OyIC;v zX<94*+LR=p}uF%7~;j|fboOW4Ewg_+Px3Opb z&m$xG<69Ym1ut-eOKd}l9kt!?ZXvG$5;CaE`S6s*B~nC}4BIN9l8t)|8822b@Fs3B zPcoUGYExiq(G_Nj&XI4;;_n1B6wms}3&=(x&h~+#>C+dW!%bIWUyR=Ee3(r(AId)#1E=Z-sC-(Ug23x zkWJ3+L6(F=d~dj7*`{Dp%&r&Qnd2g+C^Mb;<+*)hk2s14VU!g@He1y*v1;mQp+NC+ z-2I{^y}?LwML?w`_m{-akvqlI3>S806?90%P;}g8_^kS>vl2MR-mmKbv3g419HZZB z&;Du?U&x@-y|i9t(OQ!lAcKFVg!ayN(WXd|4N-N{bqPHgdBEm1W4U}{$m&Oq6X9S; zvtrK5tOFw+S7S_Z%mv~)ukCN1%wvu9TABMNT{id$S!OALL2 zLvgT;{nWn`Cjh;PBi9G*`HEeM(X!t`G)wM^bhdFdn3j4-4;ka*13$tOKRtE| zNVgYEq|h$#+Oe|LFOcdv7fx`Wj`F+KVS zt!2J9)Clno7OjEyI5-v%s1x=5j{p>i#?5l}BCDA~^m5a9_^-2sWwX#~jwT7KZ-5o6 zGpR<^p+5TdIY)kG;~WsNpDwiS+I=p&_ff9zW2Zc7pIb_bBdd-RL{XFIn= zdh2nLA#yN+h;8L0F2b`DZP*;Kkr-Mw#dVWr19wy+uwzLbOmiMELS@Dolj`Ue-mdl9(~=Tj|PleyGrT5!VOYi221eoH}x8WR6#F z%?hZj`sQ(e3y|G0Qpd7pMC<&qDjop3=qQ`4`os5adXnIZ2^w`n(k~sAlGAJJP|FI8 zbl5yCHU@(|a<2i5fkA@$_emNc$DB$0@_gznRgWnfEY84R)2sY@sHhkS1+HDs;K+G( z{cL`=UX0kklGnoYxQFrzCQP}F`$6MCS9rXrGRX)sDIhAr-XlkV6HK!Nfmz@H<+?+;W;W3&m^+;ApK3wDFN?^AaSc@xu&p?D zYmml#D9-^fgj|^4eAUM;Q8Z%bISeht)TeWEHt|AMh7Qh-0b)|*pEhmvd- zGwnSD)kWa0a0hm*kX1I7cgEX~uj|I~@oy=IbyO-53Ua5z(3~q%=ao?FWDYwvX>&mRgiBQcuE*Dj}}*;b9iBL{@#_QXwtm9yY>dUwyj0CPpi#GYm1FmVfiKtMw#>+pj$ z79$BxZ@&Fduy8qB3fIWV2GDmN;pCJ?Sc;(f1YYHTTO{;}JTrzl9%IhdMSNG0l-F|u~H1hIdMoTE_`6mBctDW`;Hf^T^YW` z-m2d-#~r#A4~x-P9eFg@-l3!Y*7`RoF1}KDagaPyEF`b(3vBqsip~QV?PsN#6<&IR zzca6#<=>&TTAxSa{zZ8XHF?OY8^%8Ig(1K?-PLIvd$cy^7vhPManb$BOY>AOl-o)3Up$&fG{*v}vt>Ss1t^W^ zJVF3Ll7r4Y8>0gxJNb$-`qa>1OC}un>T#N=O(xZ|Iy40K`ZEO&Cs&2n`%`O(emPNi zv4-aa#KQgmQw13o+Vh=@Ni|{|iIl#q7~$kdXpV=64FovUzQ~GB&RNA91F)t05;)Qn zhWyrd57EauoY0NY;ff4et9r_>Q^Q~uaYSA=Z8+lcMDilNw4u!PkigJ z<|sev;VYD<9B-9pG`V3UhNHM>^#_NaeQpz-DN9^rLt+H(%@jlwhC98LLW062sH-Vi z#-$G@WTaIOyakm9j6TAw3E-fe^V*&g?AKNIv01k8yEnLXIySBlgGs;A62UYHSQbLKj>unc@(VeP!D)~X^wWuQydf0XcN%_^>< zE4sgl1&ZZkI*XNj=3%=(iP(1njOV}{ZeLx6_h+vt>htO#8Ug4s(FigtAq*+*BS(M} zOmPH(S>OQBTq<2;^L!Y2UTH=Jx9-Fg6P*mJy?TcKNg?XQO2|*Yk#9bYwlAac8i#@l zfSTeQ;!|$mE+ACuy+pa7(o3|imDI_KD)rN(*_F?1X|4n>G=rN%45zk)qNdMzULn-% zx!NKpY3TeiFZS!kMo_1SBs-DwqN?AV$efy(ATyof1&VZin~2DRr@H_tpsfmBpVnK- zIFeWrVaeq$hQ9}=u^Q2R*Y48fyAl|i)7_`Hp6Q`{fbbm#V}O-)Ytq#ZgA^9;^slAy zDIfdv!P7*84TIA`BiGG^nx`YK%zwq%?hgx_n9EL4nweu03s2B+c=dv>?W#evG+dH< zCC1QQ)hSQ5qLn=fQprd1U7hB|R{DQeEc4u3;{&C5hjXyrQdpJmrw-95kE>zot9VB# z-NooOg>*C}=#JFGv&rb1*+N044(eJ7!nAS}!n(0}6ik@Xp2*IVnaX~w;{p)7LtFF4 z0`&-d-+bg);7qB3o~ZOnJ^j_x9YBop1L1pAXe<& zjEC~R4pjS}c+8MKU(DINSYcA!-#n5y2kn!EgEw+Ni-P@{#_~~DufCi9fp#pvX+317 znAc2*p9D!O!H}Z0NvDOsXS2+|rds?|{j@Q6jboZQOo%RiqdPRWbB-B`vztkuc(-zA zU5B$paTxPYkT)}~Xy5#ir>3O#^8=I)syN5x* zBv()rQ~s~d_Fr``%TH9jQk%)$2G6ubeo?$Awb1Skrn%M)1#YMjGEfstp zG~T#L!VQG8N@kdvX3;401GENibu;`}Pmnv3JD1~VlmtRo`H6k_>2-;DCIY0Sf z_?sB-t6ymDd(p&alF4l?kk9Y>*!$Q!qI8QlS_5srs0Wnwu4H8o3@sl9NRC@t0T86u zX1BHVJtCIT@5Bi0eAFvtmC(1Exgpm`O5byy8&REq5s@A*=72rK_| zcG0{-$y9*~e0hjJ#3luK^D(imAYwkkxN=^{Z zVk2N00r)cD3Mm&N2Y8_oqreHKIRZd}@BsW`gzt@H@6l63uTk)8xjW#AyfsyH|1QIm zT?8c?*FJ>X+os|p3seO{gAC-&C06FNRK3jxJc&%!__UF*x(k|uMp2x3{oL3`9~N+9 zIGw?ftvT|oVK4LM={fd;4%Y2w2Hm5?G28#xWCSw$9$-h1LET3s4Dc5^M!d(oOfXr{z~i#WUc`GdxO?wWs1lvt`?krv zZO~6@DlgKFT?5<}iC3Xqv4Gm}zp)TLth@1>;Gm6FpB@H|@PAIfOOaOUyd^hvOv?_v5AtlJ+iwcnR|5p`0dXr5Dy-O129(=LQqSwKGG|>C zm(t-FA%eK#^2CZUMYaaFOL2rveAU)yvf)*{XC;1B@gp_58R+#9fmB%LMK%vIOo1t z=%2WCnL7IX{vG=m{&XIi=PV=fw9JeN@w9)>_24a%-!`&)ye*>I1RqocJc5KLuJaw~ z1qjv*W|s|jv()A`sQ`~?S}_OC%2WlvM!r^@f(P^zENS9H+gxAY(( zg3o`gJqk!(HB*duV{qICbJq4`_Zpfv-nDXLWz(F|v)Nn1HcHM34_ha#K#Zx4*D1diegr3(J`WF_XqHPq`QsZFdST$YxFAgMuux241A$&Tr{E5tWa)KPwQ!Tl0 zS)hhg!(n9mnX|S3G1;-2@~~ip6Np^{Y=#au+fszqEeHSFuo;y>d!>P_dDNoWOfHHN z@OoV7MhQ{z6R(8#0LQR-!m$gsV96B>=M3G!i_(@cK)a2@2kV655V7u*bdbbML?_DQ zh%a-*KA|B1=Yf`;y4qK5WqP>*by?@*?AnMUBlxVpB05EkN?AAk8C77??Ej1%2k10G zEe98K0yXJapHB2);wIFY-TKH+v!=D$=<8+rh#B3YP^2zl`3tZWiM82Jn|ej57v_uV zFTEVr!IIx)7H&fZpk7sCqDA zO+wZ*T7u`^6Scn#Y~FV0pH=(jAvQ9`yZo)h!r<4t zzABxjjE}Z0tljE#m@%M*SdZ|$uFxEDMPo}~Dzlzdy`~Dxp-niN`8P8m(qg2Hy#chx z9EF(Ul=`Y4f&ppfa+VJ}do_{c6wTt=*0k?LJ|ke(X`ZtYpB|QWSU=m2S}7(rKpc9i zv?N7UUCtc-bYvKRxH`y|T2h#;1OVfZY3rS3w7@v>j;S)=$ij6k8~$TJdBK0WL;5T6 zxw|8yn+3+q-##xNBE625wA)HLwd^g`uvTLn=@IU#nzGl*uYxL^)0diMxCb1d8kNN5 zXGVMt+AQ~EBZY3q{$xt*mF0(Vi(`y?4pYM2L?KDm)&9@dxRb9a4P=WFk{R>;2U+R0 zcW0frq=*pJN4#bPq=aVgYR0kI%hGsZfh3Cm3UlmW?LV>w>k6~BmIMD9`= zIsc4r?mltvWT!}vZE;e-KITLE&-^mNC-iLI$2U3lYncATX=|T5qP_8qCgbg<+ni)= z^`JbWysf+ogQBBpiOG1o3x>NesXg9c%3-c7yuz?7Z}20f+%JjtrmPTeg)gc0sP;Roy%z2WX~v%kt*_k9Rj7AtpO z?LE+z48w+e&EO4EOO^!%Gj7kC=u~QY-*Y98O%G%2dh4t;&>h{n@>YO}EFilrh45(1 zbZR#l0iukmSQP=Ln+IjzIoXRB-riwtR2I=9ilZ$*-|>l+hAxzrFI6N>h0EE zsg5uUqAuOBEIqK%qhc9os$sYWcMTU8BctQ-5%1;sQ)_$IdJtk@W`S5F-xQ=H%#e2!n0!5TeO(uSY3cax;?ZMgtKVc_L#%{EkvVX{kSD! z695BDLOhNj6x(6`pIC7*AN80(V{YPreo_|73l4$y&XoH9&$U6m?Q3*0B&Y>rYYGtF zkRcE0A0fmA2v@zK1@5ufZzSIKz|P$CgwgUL^J1Z=tKu|Y9TfqGcqXz=K9&kQGy|wK z=2?g~2-T~;C=nxr?nC?e7*c-j&j*@0MFeSmjKLVx6G?dWsoWP$;YCn)H^ zEx;@lBcH0?b{BeFwmfI6d8@RE>IN8Qrz!vQgx_$tewphKX|=mtqN>fdPj$M|EjUg8 zmvi~@ju50e(8~hUfAH-hNHxz4o?Yo7f@`*y`EzQA zb-UH5Dr03`FO5S{JgS0VUEg2T5bbQ0i;LvokBee!uh7sOjp5!#qm9y$M*DsY&<}Yi z9=0X61g5HmPjFg}{6gxz4t(IJGXsoO-}p$g)$@x>$*tjHly;R}@MCJQcPV}`iy^5} z9u(9A(D2pCBdw%7g|OQ0B0|3{lTwZx#sj7A1zAHyYT`PRsjugc@XXNrI(cKbGq0U? zW$3wYcDMfMj>@~$g8ISV#E&LetRntxM9m7DgY zSXb;p8tiOlxfLY6wg+s};ycWt;R1q7)&a%h)gnDsE+Y8%lxk5ruCH3we90Mc|4isr zXDc*{icqxM_ykG0&qoj?wR0tuI^%`?XeI28u&)PL$%+Ke`|8mN1Fq7_*V-3|e`l<% z`xz-5AcEA75u;!+G|wPP6_*9z0AnIZz?5S|_ghK_Z9)Qr`Du;qN(7dh ziq&I;5#zcXX_%plhRLq6K=SC{l}|j>49yUqv`!fIF(5PP=$CD2l;frm_8Lf!O7{FG z7;GtZ3tkWwsT+s3n=r)up}g9Daa|V?qWTyw%#N|>9g(W}G*;a%ho=xu<0V{C<^t0= zb&o|``9XgVZpiE@Y$pD@d%E+ltRArkPMm1Si!NMd@EzbCb*xmm@XBS7<=eix!Lz%% zZdX&KUhjdAJO(WfGyMGAxtp6?LJQYx0W*521~x7^Q`x|JWSgQ+q6fv~e50PW2B{Hc ztq2FvE!#r9WhF2gFg$P4;XI?_^ z1xzFgor-G8Wt&r{-{=E?w=s9Kghu9KwaOf*p5auhM|?DJ3=s4LgC`Q{d%slBp~TK` zwMiL?`Qzjipk9fIP9#?DdGVJX%J{VIUmar$I!umhR-#vB+7^eXzaUYUOLI55M%4BJ zcpEU2=VwCHcs@|5;j=~FJ&L4|Rq z;Jw_xFKiPoMQ-7OT8()1FlG%O?7|Wc9O7ODe#6sN`dJ}rXk0_9xUL3>_juh z?n)qZb-{W0wZ5MMQA<|)eSORI9&DHqdtalV^&hkf;b+eF7GtAq6m(~YB>V;t+~B{b zUX2Qga7EXqRV&c?w{*lm1TlYX9ytx!)|bDn=X{e9@NM|CyCxSSa4p5C_#L}oe4-<< zxzW&JdzIM{qLe&3wbCR8f@hCU^wZVfDfmE=laj6ElIipUO3=Ng8Sv1DjrF4n@uThz zsW|XziznDmp+Z!$ACb4_yT#R6+yv3+25NqH+UkWwtTbys)A6wXGL(ITXi$J~Zgc@# zvKT38#poiX@gJq?;9me%uKl}aUi_G)WUDXZ%L?x)i{}`E-A)y4$zwp|4VK$QBW*gU z+s?RIDPJ}SX}nJzcbwhSw(n*x?~j~lm=&sBDNnJ(xAMuF@alMy<9>l_Jw zD=DbI&oA|%FWkzum9b3F$>l(LS(iuru?w5%!Os&hhKh=mTR*5b@wQ8w=170@Pz)&b zhL{a+D2_2{SL*P<^qbixydd1pOvi8o6hilX1UqeHt%PMCpGb>C@mVXrcB|O2NbLrPpzIl>tG2Pu|k& z@E{6~F?0Jw0afua_kxv=(aDp)1#CQXTb_^e*R{PkibKLtS@WH(ejPqXMOrZP)YV{P z*fXNyjwp}k@0ei+e8Q6Yt`-eH#lhRM6<70mwxF@hn~9$%?p>Bgh^@O)pY9=k_bAGk+*_9qs3kF!b1z3P(;>!~2e@ zh=3m2qysXA7~uQ{7VAmQvFyf}#Zs&Hu?L*IH|El|7cAZj{4C?Pu~oBqdWb3^Nr{8F zOW&^n{yzjSJr0bX+jCSvvk0K#NWPkGqZz7+Kmmv`Kav2Yr;rb;iTO3L#qV0_tma#v z$r7`KABvnVSZcZjrtIy)`gW?wg;&G=mM z48{zkrvKY^x}32Dp04Uj0{z)cx6FAdsJje!#ufAEF2M3i{#K?<(RBVoF1<#q+*h;5 zn4Cu*|M!k}+-AN1Fix9CuS-l zsKrM9?J6Ii5r{F2`KytJ>aiupyzAekZj$CY)*Gk!=!Vhr->>ximfgI% zdl49(=w`FreE=P!vb+)B>?crmXTdWCYWV>2=vOx+T>}HxVSxL>`!}lpXXA9u=u5qi zt)kC-`MBAFz}D8THsAxO53Zu)JpJ>l$OUhaz!$4I)9=^`7$M@qr}pcIHc; zn3`{|*50}Dm3|tq5t{&VhxhDjjgntCyAf@xs!E8TAHf*TOr*wbFOLK*6oVP((~;5h z^Q!N9?@e?A9hNZ#wEmn_$^{VM=5PL&keyu{hLjk}lLc{%?VuI$kS?N=HWhC?UHxoM z5uGX*2c}0%*q?pj@1)T7>4!}p0&8zSY3Jy5*S@v+OuLeFthrvvVERW|;cr|qr9XRY z7rd|fZXl2Uace(-T^Q!*)46HV{1SmlChLPav*(ibpU8BS59L~N;RX1Ck8-_*2{D#h zIT+oY*2}bfXeP@+FPh2f92vabiYn?N*;DLAtN`8-{X`bEMRi=scAIxmsox#MIVKTj zYP!zatPSALH{3nZ;W5K1Y&Oso(OOfx`cH#JDy9d7?y`hyUSVUAS=33GEz8M%zRW83 z*E@@15KN(2s)N3mUDDMJVHJ2n;^gJXCT%YrZ0ndUc#K}wrv?|BOo7{R$q4g+Gv&tO z__Xh69@mH;;-{GB0ird8s&+1odY1$(K-l#d)wfWN`ztfTcuQgw{M&zdN4|(P!IzEl znO!w5FVW!09e|+ZX^8LFDO5Uhf%l$$=d$L#4LG%Nzaflj;+AOeT+(h#td$8ES0OvZ zJ`VbOY#AIjs5%T&8TsyKAInH+3IcTc+HYSQ(yFP0FH=&PI@}@N1b#J2CZ969{It%M;%< zDXtE~+#t!@Nbw?9dn%R`O98q8fkucy2SIcq#G7KuxbWYB8oJ> zh6h0=TGW=1(Gqlx#yjn%`{D1E-@fL5o8haW>KsplzJBXioqPjJbomXn#O_?Qg4S(^c*~0nPvD7k*)Bv$?e6f)DB|BW&gHgy-Lhgdh zE%#_O`|lr7*H%NEhVq4=Pm&)@N{t^)U}3E1$}Kvo5_dpm$`Xm6OLQ(X#~ zRe*YAlEF!Q)+OrrfXuEnMp*ZQ8N?08U*wlYsC_8Ce`>$3mIbLB8~d_+VN5I)QBr&H zhE>+Qs=bS#@6{FQ8d#KdekelNhyh`Ev%>lFtgP$}$}iSoQK|xKdjV;Te^T<-Pzn@b z2uit!?ucM1?+>d0Za|U0{)K%Y0WF)7hsetzc+he#rBlS{@AfaB(8CA!py!})!=}LQ zYKs1W%vqzPhXU*s4BfML$}z|(l{A&lKJUbLg8B=zm)|_y z$N~0B`Ujo-+q7&_ZQRY7;7no2m|V(Cxbmu64$jv2QD7X*w!+34i^>`~nXBO9-VHu= zW(h50>ZTcgu3O{AQ=>R=1;fKTy-w7HMGWg`U3IBv3tfliCHsSe;9^=f&Ot49ObCiV zj*GRaA@=O&@Kds)%^tkG&Rh%>I$n1h{-Q9l1@u#Ru%)*$bnwYwPR#2Qiu$WsF$m@7 zmCO|;?!T7t{#TpiP%cO{!)?q33_x;kVlo3gAq#r{(F96%Xf&#uNKbcDQD0TmNnzQ` z=gdNdI%-&6Zp8?9PT7CeX>sD`R-S){hX6}~bx5#xC3(vd=5&f)d+%Xj3KVvt)SaU|AQ&Ne z|JTo+Z{c#cP#A-ei)8hj|K6j$a0`D5krsWdsN|&h7k0kS^l-!}OZh07v3=<96GuqK zhLc~l6_J3WiK**g%HsM}>|dsfv8VGElA6s}3>i`P4RNy^Z8;~;ly%P% z?^XH2f?aJ;B?|Q`C1Eh8_hHGMJ$3UxyX!Z zX^Vi>-~#TkB(O4+7AYD?0@Se)qc|}%&oE37S6yI01>gWJmu`6`kA6h;t)ZK!8)9p@ zb2akk2pi3PV44J%ik4L3x!}%#e4#;FfwP|_FMPp?s?IwY7$Aekq@oa{r5~%Ozc~uG z^VbXK^=vi=JZLh3F(QI4=Q=QSrqsZielYrmn=j$){)z+1_N9DjO!n(MO_9+h1Mz5i z6i-eE%UO0SvM7tH=H&~WnAHzj{H(-Rd(j~JC1iF|!KNpjsEOUcyn{D8E}cAP)1SjW;DUy=xb`Ej%Sj6(ceQHY<;*PJL(Jwo2TszB*}B6{g&#wt z7-02?>Kn=>ZbB(-R5sH?3i$`sgJKg@eHXms&sN_71&;ExyBvBW_KyuNK(5+v(ajTy zgbRd4%tD!*<{{1Hky+;(KWwn&YzE_jWH&>MXkC=|H4!Eu^QgUdbt2-8JENSPf~~vL z7Bo_Xv?3z|LKqHnGeApvG+}X+s?Ic*?az9~5laeyF(vZ( zh6Jx9QXhg~ZfhZ}`V3oiHe#OOJ$qEo1Sly|c~w4X5WVQ6_Tw^TLS4pJp{f}^osl@RePK9UnPJ%3+;Og$aPaS(|tLRgN! z31b5a(w~$8M#|>QttjKsLU zRV7T8sS)0FXd7td!3CIdJ~v>}OW4cMX{!1JTh_NTb-70;3;r(!UsmOr9$}nW9Rb6& zL-0_X%&W_@aqnexG$eHZQlx)PkQWjsR`f~s z94@lPPf{^}Ee4f3k5;_Iz>^Co#I*;HsX*T3i*8 z^zkC?%d)Vf+C(9=%_G6}al~gdfWRgy6LBaCXkeKnK&pZX&@mP?EDiG{E%)O}lBV^L zX7#Uh{}kVyizV+uw7RY8Tz~^A3K{t#0?a|dgz!*Y25veBWR9^S&8HZgL;jb?$OuR+FY2J@iv#GBsG)(^7sd$5K+nvTl01x z>s_@7f5ZM!28sX}5Wvhmv@5Uta2O}k1DW#;6kg=;wJ-W7oq_K`A%-nt@fqX6_~gr? zsn(r|R~JsIT|Jwdhwn3d=u$z~fa^{Ty6 ztFGCdY?DRg5Pf>$4t}pFd`M^A6e?qjOaBl#9pw4>gV*?an>_(T80tyv&yvCQu1^u( zv2jg5piTw{kQv$1oFh}k_ICPPk!xNKXeX?r*64L)Z&_{xaoomx?oh=ni~`fcv+zj^ zEEHS@SLM=`@loSyR9<;54awd-UZ{^-6lb6y?a;&LM6zmm=}%Z(#72U2+JxXx>__6NOKQ^xtat!j3}2MKcuLZoI7Z0eXBDdJ~k8t1fo zq5S`EB@n9jA@3|F<68`7jwYpNTX^Z*i<^gBLf1Y^(gA=q?4r+bT??I&_AGe~V~(Nf zLsXN+iI4>%2mmdNukkJ*i2MpP|6&Y)p(oNr%ZcbfqD=x<#_YU`=Dpr6y@OeMk$wns zm@&R#y2(lf9Au6^m`fczwU-C%1?GSkFz_KY!WJhZ;R?xs@m)>ep-M_=nJZFk;hhzz zt9Nu5FpfJPHA#rrGYpGk@NNP|!Le%{2qhoc!(#6w#-1N&iy6)3+zY(d06U~(cWVTx zYL6re60)l}xqvIBp_GgU2CbrLxaTp!bmf@$y2}a$MXDQ@RW$FZq?yxFf(_jnJgst4VOMm-A3`Sec5k3jPHm2@@!ogErf zPHs~;IPn~*^cs~@=15H#YyS6ZY?2VD@y4Y(dVcLh7U7y8Gq7u-jT|n<6V*Hp%WaY$ zblxq7wd_qvHUSVLdA00T{TZmsogyRu?=lc+RSBC7+OadC`*=zYx`d0;0%|`o)!GbR zd%_0g%gtE1DyTAHmm-G-E`%9*jMK+2C2ZEla04*)c=Uq9WzAeHS*Y8|#u~+q;kj}l z>ka)Gx}TL4YV^5$Rcg~foAzR~t1N1J%Lw*1U$&_KEUHj!imG?Dr4NJko2GZJj4GJ$ zv>%**48#(nbnC??pQ!8r@WGC!nqC09-zq6?0@#NA^lt%_bLZ3#zd7aYp@S>3Cu@<9 zN{FUo1;hUHaEviQ*EMBJkS4UEh9=kTaqH+JN`K@!v@fmJVKKI>KczWw<-g$0nWAU8 zpkxCurx_Hr9EtijwuqW?h4*P2j=B({FPv)N^2TG5Q(Aus|0kCv?7XRb>A;4R(@XGY zr%DwM9;Ym`8KwA@6EfpvX#GF8h$c|>@T>})f^=LHR}>#2Q^kyb=kC>z;|05q+HAxZ zj?aF2k!uT{lY%vdsClL*7j&bLPo-K&R@*7&ydFA?6@cw>E z*=>J<*_<>Rq{&3ef8ckSHf7dR7db{bXn6sbZk5<@l)N2D?Ug#f?F;KF96*sJW+K9H=9N;;=d5=d*OM{s1ro!o^zyDfu+U<^d$1_{u>)xyl*|U zlj=_Rb45=+{P7RRUKr+X69u?+GsBtvxx|cg(Q#saDfcgDn#(oS<4NU^TpKR$1Ofo) zF|%djsN`tlmTO3Sfr7!(nR=l0*>*zw&Lj@`aSN5+c8s|Xp?S#5Qk#q!chAw6<_Lq; zda9S5^9@4T?a*o%4(qHi_wvf4w+_88fj@3NTX8(#HE{zv1mOI17(IOV>gkuULQ;U# zfO|jyuMRMbX>bW|?5oqMIDE+~#8=KkZ0_XeWW;?RNQ3vhx1EI128Xpx$}cV{myE12 zUeI!3<0$$wXCo#GT3+ecu@tXX4z!@H&y=&{-bgT`l@i$S`FXb>{ylI+y zIs~JTXVe~Ek4_6UA8(1Gsp1p54~`(vm)cWRDkEy?;;HazZVH=c(;er7MO8}z$^PE& zg9YctGZ*p23fX8RJ5t>suYw)cB6*juCJ}Jn?^$*vHoi^i%XW;Dbn#@0%Zc0pgT_&O zd%CAle*lq)O)@hTQ_M1*x$H6Q*ZNB1-w}UL?fCzuqd<^-3g8=^>D5A}Y0Z=RH8X#% zii0%nFtZ*ll+(WfZk&$R^+}Yc1_2Ms9Y_kc*l?xhTjVd6LNg)p9WbE|Ikyhmg+bK3 z#BnN*RtsM_zSqEIjq0qW`v9UJvk|BzEu5+zSr7?x-$Hjb zy3xT(lvwP?xT=XiF5DC}s>_@pHCwMeujj*a9E2xvl9}PEcXg~~`&(W4yA)3u#}sMT zr5aRudk~^PSbmM&WJHCV+|02}N^^wGQmy7!gagdcogSwQfUY_quZT<-FtU0*P;hp+ z@UM@k>%tHgRM62z0qxTuL*G;eXRVLA*@P|iHh9v4rpl#Vf5(*Y2U#Lc{d|4K=82!s zB*m~_fmCrZZ`=Vi!@-j;Rd5Yv6E{;u`y~#$DsK`mg4(YD)DeVl{`!FaKUry(NbkfS zSubR@)+#j8_34+Mg^oq1V^FYoV#6F2Nq)aA6ooXNJ5ZJw< zKMkZ!7C{fqNZ!kUjyd<}Wq=Jo5O_d6f)4|LtYND+Tol3Fv4xwa^cuomiddEc*7%;=xc3L*W=EhXaMI%Vn)JS$ZVXd^+PCVfm+cQv*Ap zj}r{^6N_056-`I8;ejOXu?WI&Xz%29sTyO)I82z__$!H_Py z7<5s*)X98Pclx-abQUW3P5M3o$|B`t6u9tU_8Dmrn2rK_(yLzM$s-iE7sAJ@E>aMy zXWm@>N661Ud+gqQ?21b4VBw7kYkIF(R*ZXK%08C2Eljwg%=tWUr2J2)?dDX93@BuW zG&?Q0Szt!E)~@Q#`cYK>d4#g<&7hI`O2?mvq;5q_IJU--3XUf^C&t57-} zr$8E07vb)_T@fF6oQ}1w)cA7?NYw(cuZ&%=Uf$r@HcbAYJ}it~=!yVwxus0*TNmi4 zN(D>kmd=#n^b;>}hVfq^CJ;Fvp)fN12{JJx1*sk*M}QMdas-xHzyR_sL*ajEP(t~e z9OvDDT7j%qGbyeeYE5bwI~;x>_c5Q(o(dUnYB(iO(VEdNwFDYAlf7H9bSYDk2@s9L z**H=APhH0quOkRU#BbpM*wzPawrF=D`iX#ShS>Z9)<>NK^A`mZyyaoj_Z$EggsG)C z=5pu#Af`ta<2WEHds>nQ`4j`BFMorc9jI~hZ&Pyo+?KY`#ZXaX^j-A#-}1)2mwbKb_2mSH&$2g7Cls|pD>G4cDU?&xK(@xR6yIE|w)#vLE6QtikJFeGmfdp0YuhRt&Nf(b{p#>*K#;`s{!`5tP-P)s&U zoQyg-^Af6_s^iMk97PP9}NiNcv?FitRudLx$O1&vCeF*pemR(A40G5 z#SU6Y&Pw2zDTRM<6{yIgD20G19K8(Gl^1sE^m11t^7N5Y-&FpkR58V|af_!_9Vg4R zI409lnU%<3H%2g5vWI}s3+DgXA_MF*M(YmuZ5l46IjJjQ#c ze|!oNfh;}d$AfcV8BK}(>gl3T?310lpI`=TMEa8OF{S5638`5hqc>gYPch_nbbe8` z@W%*Tv@8}}K9~-p9RJRmARTM4~3y>Fl2D6Z89HlOMGWts4#bhrFT_Hroe z!+cG}E1#!X^mz&-C-g}5u59sN&Bi{O%K>h~@?}l~J#Nkh9ca&eI@VCB%8I?o;4o9Q zr~gXUn^+{LQ(*F@jB|x6c!&hJ*)bYiz`AQMvZArVoz5Mh_o2?3flLjXiC*XRzGf$8 zg4Uo!pDW2TD0zzen|ruO;wT%4*C3b|ox~^NoeGRHGi z_|QCZ%;4+o)O*@c6VB#)FWi<%D!)ZlY}Mf_+K#5SGS@62HY9XaoCEM_gKQ_I-P_N% z1%5T^SoCK!N+fLFX9YHQKkUl_Y+1JC+X)e@xWb)64vJImr^=tk%E;HnrA?LNXvr#E zU660qw?XtwMi|E7SzJ=~5R~T9L~!{fu_uaDaI#~A#?Mm^DWcRJyTWSxf!Whzy|AOvaSD&ci|&57{R4~zXjji)S%%kpk3Ss}>E(rKA|OrSYfL)kBi9nn zlB??8A}e{Y1J8!2)jGc^zoo$PF;<(p-dXR;?i3$qm%uiS8Eze)7R48$;y0dY(QT5m zVxdexK+33iRhjkfXw7Ev!XBmTZecnx)H?UtH-Kex$%0^Wc}D>bEalcRs;N89kvuF_ zu3GZatV+)bvETopmgHAGjup7C)=>r)>S_Qq4l{c9heL0%;V?BdP-sx2KYposh*bsZ znnPBOBpuqYxAGpk{~r(V;BFf22w035i#6Fn1&^Fj|Go@nG9Z=&r<*x88@esa)>^F7W>7 zz2XzU_Qbm_xJ1ode0Udp7d7Nky(+TGd{{wnIdc@P&0p5s;p&`X)P{9tkIKU-3JfSo zkS_q}6=E$TzbIz2_-aDOsIU6-@NEkV&`1U5rb!(2%X#bWdj0|Qz73%4Qnj_j2F+Uy z8~CN|=jt||t*onPH>BGj0_FT0B)ww$0>r-{J&q9bJ2cN7JbyuyJz~?UBDyCCGNp2% zaYEIh)>-bL8Ugq+`3or&Q!fu8H~ksgdE&Gb$Rm?sf~kP;IMR6oSe(21f!0B>+*?IIJG-zJqI~yklFEa)W}+ve5IsX&H~Q%4?S! zyS`$Z5pFwMM7$&8dJVRD)*au-q3N(@>|cB~VOX8jO@eLbW>1A4*$PD{SBw!zqaT)% z_OB;8X&mU7yWg+S3h}Pk15emr*Tj83{F8fA4D3 z$Wp#-nIi?n5UHS!|2o!kne)dbYC{q$ny5Eevtde3FS z&m^_p6;LuLKj?bowa;2Evhv9cOA5!@j*@A2_?mwG9g-FWaXV5&Cf@*m&0zG^C=Ye* z1(_|AI4n}Vqo$_~moeYrO>7Aa!o-BLkcQU!GB77MAoKtE@TtRGHzt?#8kL;`! z^1W-P$-Lr#H>C!kAw{vz=fMoN7TxPiB*0IbU?b3B7|v`dEp=fgMrmI!K4uin`<#~d zYK44N_jWb<@8kit%uCi})Q?m%@y9l5P8~gJpBnfE+G)*fE1>BU3;W$Fza~nE<(Frk zq67di`_|EGzJs&J$T&mx$uL+QSOrB88_gOVY88lvan%5o?7qAs?7pt>hKCg1j{)Xw zHE2-l16fn-$|L-XLa=UDpGi(WoSa>-pYaHIsJhd*A`V3&cQ%jGNj~jZ48g^ron!6wBUP?8$9%M ziG}E_@asD@Rc1)EmEdGt&+!JwOkAAeliyf#8jw9mfM$}XAg9fZibXb)hQ+;BZHMR@ zF(g%g1Nreq_j1F0<~@}Y>xCsX`GtiY?FgHU-Uv<#B}V}RPO*7r76g^Q|G`;Itx0-R zXVv(;+z@mHS19VmI9#7a3tc1MsJUnXS4TzCW~1Ful;wPjnX%oT*hj7roC_PlPy{;r z4AitQY*UP0Dqi9^FUJY87=28)7SrK>;dSk+ey4|aD9ZZme0~ATC1hy>8O&x@OrYu)#4RuGsjy!g_^b z?AHd}?UvL+v&UN512A~gb;-mjQq1`gbb{hn&0VvbI^mAar|XpA(cQjdkhDy?crA?Vbnp(Amy?X`OUn^P8Bu1KnuqI8+}#ag2uXF} z7lh677s1{RqX#wzFN#x1yz5u!$Dj;m938u3CL?8Bvkd4 zCzhUzZXa#h2>k6)ZP}X?5mkGab*ws(5{}HN*m8hpAiUP?c>>*y%%loRl&y-<&IEM! zi@pUL-S8(UOE|$Fr)5JEP2NVkcVs~fozIM#GGeO%18e1gWnZ2=svD{CV%+w`&Nqgo zWYbCVblmQFbuiz>fbabrp8qT>>lWZaG-WzlAT}B48#&XbYdDY*gX6mbPFJnC;w?dO zhfdI7W}i z1;7C5jszzM<1hhnGZr!p~ z^{&eNmkX#JY$ccPXB2cZ9Pz$Dp$VjRiff5a>~&|*E=FRWMLD^4V#gRruP#Xaq5Q|9 zXRk~erai+oV*^+ivuEPMi418infJ2b{r~fd`Q2oLQ5`jml0xxkT$Z9EDIQD0O^Tt< z$%WO@`4+;?=TCyBJaUyq@c!<#Mtx!mtEvDN=S3MR`bsSw@a%FbB)hV&)tlSLUYxs6 z6nx}%?_TgA*j2Y13)-&#xDc9~~UoP1zu7^}=GSMZ8Kq zIRUmZu8!Wfa)@6W(MA$21g#HOOh9jo&SI2B;xwQBhxMKHgO=#5JU`B@V*G4PKu~k( zTcL0cJBPjk(=EvuYlFtBnGmfp(A!bRDCTECWu-Err$nf=X!4Y1({dKye0NvY4JD+% zkW8Tj93c#Am1PrHLF0O5a+9-_1Q49|B=C4c^qNDRV%36)91AJ&#C(yw$6#m0o@$Q) z=aC$;F3!uh^Lj}BxoT;ilIr9kw;6>T0(@^7v2_Mj=h^cLD@t7&&Gm3K+Xe&NsQ0^f zW~g6h*m5zis;I$naN1~faE5s@%!!9R+oaUNz1}a23gB=SQaqZK7hJnCxl8QQk@{XL z^8%kREf2<#pd!Jak18>O%|tM5!`d`ClI9^k2nOJ}4!5 z3`ALh=@ByLxh@qtNtpy~cs)%IBhAUXRdkkTV(`d|WilDw1zQ4$p4vtLy-$>_-P6d5hU)0_O05v0313S_o z@4ZPlGuL0lRfJk-aI`H6;nMUR!OoI<;@%NY+fY68_DbdKAxKNek(Pt(Frm)ASHQv zBS(M|NOA<01;7B=VZkfWrxyr&E z7~nXH+IZ8IfVo&)dYio(?s(Myy2i2KTNNtI%q^j8!t2B}FilxjC@97ZxR!YtqjSUY zlt)cry&!uuVklht(yFT++#;tPqG$KbObkxY6}k3A(b~@DS@{ zz1Kg=N-^Z8A@HgUJir*D0RiE*kI8p*DEVD-idrKFuS0-ZMZs+*W{$w=wIflhvGPfLp zIP>u53B%`r1kl6tkNDsRa`+;pIpBOW0I(mZ^tEnItQaW*sv2$%N%in`yFf%$vfE{P zG4YI%N*l>%YGV^;M`!(C#TR~S(q6If#LjQoeJ5jASw#RtEi;A$#qQd+2$1xIChvX} zszp(~R_w(vOxlAR()Zy`W>0xF|!Lo^WR3mrUb#F-0m8-GQUF)Z0>{2B*aox0ke|VJ2mnv^TW$-|TuJJ1^ zfxQf@{7R_EFs~Z8eC94jQN3;;KSi}JYi_I#y6Li=FOHtd0i_1*HzIe11vQ*k9gv+G zcUJEpxwl&F0t9&?!h`TH$K$+xV}R@LE1;c3|LSv|8sA<$2jGl{^Mlje6Knuqr~;ec zIq?KS&yf@M#ReD29dxjsn#*m`sab)nb71d}Md8c;k>*f{L$FT}4N?Po=zjx@A&RSc zXjjbqCl5NW87WT{(PZuf7hYPw!3~EF(8b+$ja;AQZ5qWWWeOu6zl$`0H|6uXz&JjO z{8tdMW57j$!7e-O$6`dMaPI#?ui>DCF`2>-?W{8x_m{aF9abZjgcbW}{k8e1p`;2` zFo*SW(4xQC{-t!puY^tVJ~)~ZHz)mOztgw$a&5@?8@%1QL#oxqO3Pw$r!$x>AK$+XyJ`_K@a8TTgNw2#kM~wpG;3(= zB*k8K3$b28Fo<=V#P7q~8~xOO=d96ke&)&E#U(GG`{E{gEP7dthHY1MKVdqNsLX-I z7^!t^g(DU+WL2(UFtV`1BhU8_X$r#CxX(&b^n9gU0dtl@eWu3WTSM+zTnXGjfy3mJ zaT;&lj{M5^i@(WW=(Ol@Z8IM&^vD*u;{8{-?*%-2F-@oeg8019QVP*4y=P>mf=H4* zBcR5|>Q;cU+@H(ELBv4Co%4COJY-W8^HEt!JCjH!n51SRxd~#IQs;sHn~^t0mmM=6 z;Z-H>eaw!CMu$+j$yu1s=$W7CU||>Z6;JO5L;!#q0k|^46Db}f0|?&{qku6alQ2tQ zS6$!$U-#rB=%o{1c^!>G(z-?-OX@m>(dYx?Z%=*oMQa_$smf4<^#OrZH~Lp=2466^ z)U|E-Ao0C33MLC(Tt*Mt)}NiZityGlmK$uE8bM`2PJ`e3+U-lWOnJtvEu zfz02xTc%WzzaXes6XifI#4gH%aZ^rrE=yX(NUC_MPXv!a$DWebB|u>0-$vF{$8jT% z26ziBYjAPV7nJ(<_tmg1_tx#r7WU3>mEC_=dY9S*xhuvcl;g`rn7@GC%eoDQ<=QXs zt?KS}NLw*Zec+Mw0Q>SSYO>`3i+ED8G|mc-B_eNYa^*T!`mSxm*Q)!XM7xNlZ?nF9 zxJ&E>eBPV)% zQn8gN3Aa4GNl3s>HYZf(Xw#v!Ou{6Zs%R{>{iptYrABmSpiR~Nh&07C4~Qxj>uWYv z>Q*6LxYD%$SYh~8DOWxfneB;U5Jc~T8dX6x)E-{2&+cGG&ch)vK46Ay=`n_$m;;)0 z)5?F@zS#t;O)T)MLqD)>z2QrdZVYRh2HP@=*7J&xW{W2TipE?+>cj3qmJ{KrI?Ow< zWV{zXIA;e0j%=AP_x*t9)75g==ZMo5Ypk3@34k z2AH!JGML{lFz?+Uu`7nQI3!cB2KdlU%8Fl9KX%gksG^cX4SfonBSC!*Q%}fO;6jkl z*cwxh?Lgbnta3N`b7J2=t8KCZ--oh)90f-ZG1-WqrX~?!F$iWSt|S$*%v13&=0vpy z`$TVdvAMc(=DC(}^GV92+)Dt=PzV#1tc5c2gLf*2*iL5>1zW352A0xu!~s}lCI_nP zqv7{W42H&;nc$5SOsyBeMP$rslSxao%01%v)~2%rfiJEgEb%GerB34_eAzZjhzB?M z6CH&Knlw7(E$m`nKAq@~k3n%$5P4KfieqBawI=N8>gq}kAN+guZPp1w#|RS)VZtN( zQQTxv00Y|Nx-fn;1_3f%UoJ7`VZfj|OE>Z9j-E?Aa}ON7RQg$5RlXL~fVk~(?hldw zhIXROME4BcMt=F%1o0Gk-8L@T(WnnLK(@*9PTWV@rhiX&IJ*Yop~3tao7)7GdBoY* z+Cn7hWmj9Q{mki&mDi0r9IzXM$p%r)y&1I7S{!^)d(WNuMz_{a10kcds6bYzxHA2B z{>%o?^ct)Vk<`=Bg{VV(1QsFdg%BW$%4U7;M^$vcwoDHeqi0xTLSC+RJSke*Im4`3 z86+2T4RJ1HGcM0Dg!c2wdUTVs z;7#S@P8qZ8pO!RzFQ%k#ewH&_7^J=Tm>fjJ>;~AaUD(>(S*ck6{6r{t_L+S>VIL;O zi<4&=pBM2BXEaufr5%eY$FP(bxUBUa8|5MMj?N@mJ^npI`wkj1dC(K;VzNQd+TKHZ z?l4~Sk8TA}WD0JYAx$Le){XhKk1xX^M(-s0qZ+8^SCG&)1VVW?UPm80_b83|gUR$iPK|8IlF6l#| zAXJ4)?~^aJv4B&cqpR#~bXg~FFqgrZr z-$N;ziN}$ZQe_1Ra=&&PN*zRf&P$vu3S-@>@1R+0i-yOIK1{Oxfcx3Pd)K568S~<=xS!NAn zx%Vy2FMIz>GGjE(r6ZS=r~O_dGY_L#DT2gR{=A78+?25ZZx$GFvJjurS4f~KLAvN{ zKBoHDP*7rNg>!Y>DP2J2kq}qx(be4h^a{m_uyASeAY}x*@&_PFSSc1VS>8g*C#+II zR=Zfb2^)OwFMV~=XS6bqKxD1=i+x8&qzh2>@IO&#+EuF_E#|BG(sNA{A^~boZRHTb z+OU(-P11@@T9$uv5G0XY`%rt7XZLFov@oW^wZ_GhFb=_avvsN`O>fYssVThpN?qFn znlzqcv|wu;f68Kn zFIP}9WotO!X2dznnm+rBu&4tU0vRPN&_kTGxu%d59&fM!8mK+|DkNVTQAUPm1=oN7w^muC~vhQ6Cz?7@i;o`Xh zO6FgIJsVoGw_0|#o#O!lNL%i`y?crsd9QD@nioxi4mB7EnKBLJFFBwdE!mdRdSQA= zPND6uY!>28fx4TqaGHn?+blj+RP!h$QrQdzwS-IJOu6R^-EEg1hJ1;K2f%63OggUx z67erxJUN0Ir=U0ac*VkROlM@lS{D}8r%;kNtIEYvFX18JkKLmrUh`MYi~Z}BcqLC4 zxz}BuBeqRe+9Hkf)d2$NqB4BP^r4&k!_9>Nq*5S+j28-6%%-&d@JK_8N7+@<=VG$; zydve%;GX9g_+;#DV2u}SWAQf{t1(@$!tL^%__!_z{sjfc-t3{DbokHeW4+FsGKM{N z3%E}p&x=j9T9JCBmh_=1iBmxbJ_Bn=X9!o5^~w21yS?+4maD)r>JKR$NCOD#5u?Bf zBp8A~tndKh-bX`oJ8!hntp8=(#Ty&nvM&bFimbyJx~sCfUrlksuTS8e;EySJdQOLRpa)ePhxuwy0rr~FlR=QEd*YligqN2CSWZx(h zrnc)*43G#$&0B(9?=7TW!ErQ$(wOD~k!svDSjje%HA~;cHA8nYiMkMa`-h<( zQ4F$t88MY(t6o>IMdo24)qgQs zG~R^kpB!cZfaZ76-(F6ImWZDvSR;o~UR6al){<}xIZ>(C605I}f>4ZL!nn9N1v-fV zN;3|OI~C+eu+SGB8Db=FhjLKLW+vucn;XeEYeqim^W<{O*+AlTD^+>7;|TjXU$T`{ z^x3~;;|{GAd&ym2`TP3}2D+S-1wA=pWGopiy4WU|=)>gYPS7dO6-*>hzW0ZpVj5|a zdmSAYvbk;xK?Ji;=tuu-mVR3PU1exMR|hZf9Xr^jg}u>Z_HtomAfy`^Aj&w?nQls5=;mrALVXYDi|}$WH4d~ zUIqWetM90Y7+=lFQOT~`@q$O+A5vI|z!PELPkx3ZJObD%e~V+u`#;_RVLsBj1*A{-P^7 zO&w@=G+g))-wfWPq>rlA?9mIwDX4)wBH@vdoiyL-ofU<7Ntsn0`m6``ua9$t<$)v) zalsjok|*TPXN)T zXJrX5GyU;OTk_0I&?hR1jSC(WTIj=xS23#8m42tG4x0~c>#9t8eQ#OyNp85iohbd# zIE+LmtPecnK~g54*05qrXa+)U=o0cu3|10eU@6S&=mg0u#FJ_`FLSB2CRB1tzAWYb z40;p}7_=yaceWD$22$@(tuxh-LADjt<&Ada>5x5gc8{UPbzwjBVXli_b1#hNQy&av z!1akuM0oF?7bi1#O2EOy*Upo~C2~(aYI)e4h>pBM&++ulGi?>%3Aj|Qwyb*JgAi&D zu^RRwo-GMJ`QEcHtJYd&ZYCQZ;dtT$hm`$A9zM5F+q%F)bU`jqM?dYZvKfNGgI4p8mZhNB6*7{0RCuox@9=87RFp}hY| z%ynyI5k23X*-29=7o0X2N2n@lDdff}tJlDFED1tSk5eL)XC{~&1$SdB?kmv0%o!A) z3Nn#0;xRw)Mubs(Cyw|m>qdh(7;*V@t!NEroae8cavnI=oT~`&5rZAV*F_H#_W5fb z>Yi8|V?>7d)@MoN^+FH03mlvECsSJ znXYuTMia7mPv_09W}7}`8%tKtn{Rc_IIuYAUk!8NQ-vz z!D6FA4h04(mA@~GZ9)^fuY*^PLCaG#b@<`b})Z|dUdWJM8#3BJ- z53+i~2&ID5DFIDbcL%%LJFL#@K>MjlsZ^2gEDexi;^lj%~c))G+d zf((_b5A|Mz(XR?FbDe{XnNyX~qUOUL<-QUekh*7HL2oKmM?FDXv|7Rm?80cH>0J~k zb}1<+cr*NhK0v>)2#NA~Daqn(8QYQ@U%huEqLRvAcNoTVrJX03O2M{^j@*DNv9zSxudO88L}-Gpl#X}DFim)QAHgY4%V=w z!}ZPG@M8@vhRXB{J(<(Y%i`WgmRhKtX$Uxq&=!kU;rhoKJ zx+xTy9+9B+gLnj23RximYfPeOFa^Lk{CYymwqRwK;^;S#@Pw;J5aBmmUB1~I2ZhL! za+-8mnkcd17UM=^d(F;Rj}}S-7vgskI^BeWFLmz&sUMayp<4R86Q(Y9RRnVGHU68Y zZvg)-J==+re3@ey)1?+&oN)f^c)U98+l(!BrxOyYn)`KYXj8xYH~)P>-VuiQdHue z8eRLd5ciY{4&iy6Gk_9dD3CU_g0;GxV`W_?`y*8#F2fF&Xy+jthT$F4$TB>8 zfBrf100wsZ;M{P zmoVkz4yr;zQ}h?D$n^iw+tk;qJo&PC;1{|6ch0)Y0U$!P4?9sylXtCeVd+FZJFAZI zDEwx(unr6qEs99u2XqL1%E>$leN{ajAfG~VkrFEL4Cdjd(`2*y_1h42fUx{ceXZfK zAxz=%@l_bf1o07IJRQ#j)F4f@(z^Y8j0Z@I+F@&Qz!`$P(`EdK(`9!1BOXeh3sXPV zxN`~4(j}+b?aVKHn{?{aXQI&8WRuJq$}(9zVD{;32KStgf(-O1OvNEbA?rcqQ@D<{ zgv{`DaOppM4BaI^qB%PNH`W>UII>i}ErK@THpc{uecyna!u{@m(En2%3!AzdB?f_W z!vzgCybA~Y%X|RDP{(gwHJ?DsGC1Vzm9CnaC|!?0gztAWD43Lh#M%ZuO~rfbc3;g0Ru zKK|YnYeffj4T_NhdDIBgQR9%5#oh;=DZPcK5*8$eaPFpkrVP4LcsL+?q)YguWkPwB zb`&!}aZrP))Q{1Awc@KJ9M8lSg_X?`T4mSR7_s&@Ma#DSFlY|Z7cHp^JCGJ4ER~Wc z9Nw?2=jcyWFOUNWsalEdW1ZVuis>2=Fhc-hK%KuT!!|JtcmUso-h`e;F%4Xr=#?T& z12OS#ChfNUum(GugZ|dx%ufy|Qd!%BRfuO?m%ev$RrJVGhu`z_Kc{w!qPP(vd1PiA z?OQf(WBM zg#3sTmr0O6-k!7)Cep?gV4Jv9t-jAnI=eJbb!&R_Wlwz$Gl?T3So)_nKxVLPY164C zbBR{?xtbMD_5b6FiuO08^>V0+B#B4i@>$?s2u^CYL^;IF2G(8NnVz%b$3XmV`FFEDDw1lvmuBv%M}4njAapynBc<1MbTY~<#MYgJQo>qE`RROJ3$ zn>+I+31Zy1A?AXF&bOdQXXRwot5f-jokQ!c>OgMuhXF1P=S*?S?m9-m$H`c_><*Tk ztYUA733XW^eYZ$ocJSWHVQeQ9U6e0S55>BAEg?``-|d4AO|*l(E{=SdFmiHdr22_a z62vzJDZHP1l-MwQ3U2=}LDn>i2Za3}^bA8pG{F(#K_o+)j|AjoJmwAftTTG0k;wZO z+k5XH1q6-eK}Q-ZR3{b}>+704rrT=?gyMDD(zs<#KU9_smS3t6KHt8b#kApmT0zc5fr>IADWTD89z` zdA4pn_voCgdAH~v61;^X+?btkF|Dbob8T2KR|vc5Hk9wN2zCH8-L%eBp~J#Xw9UgNV87Q))7TrIIVd z)}9#3fbJ)Yo>i0oLVg*T8Ugq+_z5yQAdgwXBS(M|NHhe2S-=1`(dDhS2e=gRBCdA< zg|6RW#%g~GPZ!9&2?Cg(v#LOYpsni$52qPv49PS^rb5DX{i4Hz*#a`rb-alRNN3qz zF`)Fw_eCKs0s{c!;U`c9fpK=!a{HrV!~#x-xHKqzD5dU`MLPMLBgNLUJN)s4;00wk zerB`VL9DxV^bHwBs1U9n8wBSV(A2s-K$L75gY zmP`Bm^Ux;@Jm4}3OoI&eQP-q1L=uAanC9*g|2qE>F|1A%sQ~FT(5<)4d)A9C!Np3*{bujh972eb9mk5=YFZx3uMGn zaib=k@lMab`F(1`24F%AeRYnSQB&)S4H(~B`RsF^NxkXzF-1GYLXTJ%Q2T#;e8mp! za~i_=LrvF-Yn}ZSfVcGpFF?hMUe&vuEL{C`nCE+8#28&AcckoB zK1c02=-r++#d!+K&k=i)YLpfIHaxtO|96S^W^F=m4!is8zw9$4Oz&WuvcMt`pr;SvL&%6W9EAAAHB96?Lis=8f=`u&~7v318r2#lvhNU}u*4tRMfz@)rFM~tgNj38X@ zb|-tSgMu7buVB*iSox6MAmJhz&|s}WUh@3N9m#Vf2gIfRplYMX#Ms1IUe+S|Z0wH9 zr=$^P6;LJ?ciDc2Z_hLe*Lz!-TF4Z)W_@R3nCuMxUAx8GvxvmvH@3Thf{exPo%pxb zE{~!{tLMS|>KzNQzi!+X%?k!VY9*XbMavP;+%~bn zSwN~j>PkVHyjN+6m$9>+Vat?{iM|oqfJ=MXLHk?2>N@~FiP`Ncu+hA+dih~G4U5AH z#ER-|c@-|2f=(vzKo>XU$>u-oZaiYIybZ|4fO-tYium3RWN{i1UDLTaCByizc%jn5UZlf@!~XG#{k~M>G{$ zDU#iAUapyUHaic3fsB>W`4E;zokPH_gS?>-mL|wJ( ze>iY82NdXbolAVR9jMq4{CpRt%16&VYUl>~#o*D&*-lByVDmFrl8`zMVEhmWr2tr) zXVE_$P()4)O#V|8r*Kz#z<^;Ren%T#qL*Kiq-2N&Ad0^B4Ukz&TFSnHX58{E&Ca0} zU^sTN|D}(g(jRa<@8)4^Gn<~GL-wQmgVI3nAz4v-{8I^0kBn%S7J&usejpkF=rW-W zDIXvfT-On!un8nNf1ky2E@jFL~~}XWx|fk8#MnZ;6WwSJK}MT`QFXI zbzvssMzc$$J~eo6FS5A}te6tGg2Xy*|KO2<-(D+`YL%2icMKQ&bvV0FMxiCcCR1(3 z(hxq_Advf<+6Q!ke80u&A|nj;sCL()v|C$z=;GiU-AaFz=&;Q+s3Pc$)@%AE2d0kZ z+4KzIsBrAm_2t^1Zfo?d)Bm6Zv0)T-C}#YA;Q&ujKAWpZgb%>Z@yB(iiqFLXWU|=$mjBoOKxE{MniM^hM1cb| zvW!XT7-^LZf;JMSZrn2rP+78BVcD67T3~78A^lyHAKQv^TUL(Pyc-v}@^WlwW0E`~ z66^5k);VWHyyaTt=Urc@bx?!%+@Y#lt!YTt+q?o|*x*BhPe5}Jwez!4Ywzmks*gIv zwIQNQr2VOk z0ZaKi8M&z~(?m5%A+MG*t4#SS+C&mETC>}(B`-leg0_Wi+vFbIu}pwuY!fae^G*A&hP(>7ECvuAQYN66 zDGWBDg`(Q#Sj!dBp(Im8c(h@sM+J1(BP!BOe^Ogfg{N{i60osESwB{+lLWErRuOpK z;O9E}*uM;7E40J*3`nv!OG9}{KYO&|96&!}hEUKSRYNto>Q27yy8HzpEjJ_Isz7Dc zO;=6#d`rMQ(bF<%3H4pNPi`Ge!p#An8S9unzMS2CF3^2~tTtm1N0lkIK!Dj4%mon) z3>Vvvrb`OfN&NDT*lSQD0F#=&U+4sO;GZnk4QVL``XA$C4nG9AdXbC)+K<+UF&CtU zHfTBu4p2&rmA+2A1saNmg2{5!qS{nvlI45?s+*nsKS>7I`+oZ+=5Vp?&n@d5#YJ+D zzmJtt2&2(E2z!cVpm2Y1wMrErFL6Ul9GB_7tfYNS!CviqwAZEer9=(TC`Il(t;(OP zAJL-+gBY*hbneI#nf28%UjGe?di9luJx~)EtrO?Oclmh_83^#6#C{aZE>c7Abm?qV zuKMai3^0E+D=w>XQHo6fQzMzUKk(X`Z9No;_Zxf)KeWs#`mS=h+ZRtP-fP!O4AU;( zq}5Uy;@q@xe8+&Gk3+x=$UKuRaQ113-q4=yIHc~DEdC{i*g9rE_nNA{51GI^d^73F ziw{Egs$ZM0vEUJ1g;{`d;4(~KCIW7#`5uiX6|1^@yQE6%8)XlC!Mo49EBCP2?*=5R zQ2^vtQ52)J-Is20jF3V$(~}n(_s#1RbL*G!yud^Rv92BEFr)OV5HNP zv7F`di%>v;?*dW3j6!uq(R2~8R@`l7Q7C4R(AOI;#*;YlibWpeL+cATt_X8Br2|mVII+Y;uq{#&cti_;`u~PmGEx1 z^y}X5bsmIy+guUopr{W*$A}m--=qHoG-55(8<+}abHDXoQGagfEzI_PbE6JAi1713 zct>t&1cI-?{APBC2wXzv?BrTqFc*dh30Y!R!V;tI`hzDy(>gRC`JGNl!?EsK7D{p* zFt39hO4f{QiJJO^EO5j4F_2u;pIQYw<~ z&si$b*A@Zv7w1b>%y#9tN>Yxy1v%wS>GU7dNn%0KOB2VPo%XJ3`5)Z|8a#8nVpiKp zn%akU-HE62pd;`Zy*>05E)s6pf1%L`5`jK3SUkUsMZTB?L&!uZ8@CywMdg|+*)rWB z(zgD~mWLaNaGiV-;x!QNVzuD1cVqE<7Vt!A;-*x2t!$#9q!wp9+hICt+DIou)dLxp z{F20XzsRbXqMaM#gbwpNQAE0FGP*cL*&qpN>u|ERuJ|JTOKn(y| z&ZP_bs_c8qcAL()9u`GI$(&!$a-s%cB>!xxCy{p)mn-mTPkJzg5Mq9hDHj+BpdvTlpF z$#b{{Wg!gz?;+q;1U@a}*>*6vuQriUBKl6mf&uh8E3i)IOLU8r{aLfniSUfp{Q$*& zCG(M_Mo-_b#t>72blFPtnI$sfrwodXeZjK8til^Z9=X11!jKy?BLG(1w2Z>no@%5P z%GJw2j)S;By5|EPGB$`(O?(#vu@FBjMLac{Q-Nc`iTWg_uk}G+te%rT@WIJ^c5oy$ zJJO7pU_<+*x2d6IbzqrJvz2Ps#WsFpsN-P%t!7=FX9uxZYP9w2HpICjEANV2H}vhm zKHsR8GuY#Vd&EyUWt`Q+$a1bAFsTv1>@m-B1D3M_XB&Dr{1Sj?AtKh-BfYvNlLNae zs-SxCVqupu04}HR`Sx;YwLo-|hz$ZXwtrnu+t?=!%0}y;{jN!JhoKrSPuM9Ui=Gp* zpRu4QE^qs8Hs>MKe8uq@9#>h<>1^F=2*<_N-B|m-WwcH6F`nwTXhCIA6H4}kyu(tH z>JP!E#M4$k&fDuWO_do=ciy=~l~MOdgNcKX7jmrWCI1XF$>_(Pfkjs%;p{#+-u2NN zSV8Z`6Igl0*ak$4O4~BE9+e8fqoVN18PRNx zn`BzZtv0kuX9l$GR6~|*1sJFha~zz!+8~=|+eHT>$DMe$ zg+$p-h4#(H)q%fj3E;biGk(KHy^QQT-oyl#t4dxV$p^4wcIAQQF|L!H08iXBqF#BWInQ#%c_9~EI z9pHwv?usdKZyW$Jhj$@bQ57<_G(8zFCL=+b@`sgtFU!mQ2F$Wu$aFcxB1{SZUWR0w zy^9Z#$`r)7;{k$jHZ^tD%Ot6~5?)uJIEO&;%K_VZf96%#hcJ0cxX2-i54n^gFO^l( zosm804KMng!@13v<4TUyk<_L8W8$iJM#*g~{_4vRuQjBL@hy$;R|h*CSA9M1n&~LA z(Y1BfVSaPFLwO!M_Ov#?dd}M~r4p!$ zq#lLi-Z-VMHA%d6P%wp3CU5B|@u@=Jift`kQ@ecdVNm&*lt|zXT1Cz=8yf$s6I~uM7b8TW-C9!S*UVOM}5vW4P4pPF*bHY8jc&D^P#=K6T3`>GVaVwuF=`bv&;b zi9-Hm;AfXVg;!V!Vp29E<6HlCJ0iFfm;vUoIt?cM-H9?+1n-bcsbSZ#LsBD%>(jAu;n`L7oyK-@y#e_%B70pq3Qe8pQUj|sW*=|j@mND1y zKT9NRpE*fJELf)7qOM|$ksrkWM~x@^5+qkh+}VUb`!U&x>X5hIm=RCA4K|;NIg&T? z=B#00jTKkwcgVoqociQ%^=VECk^N>YKLMbu3I}?_;0Rk;wJ9+9sw!B>lCh*7a^{w^ zmn2B%>0Bm`eybJbGgwq8<1^rpD^m*?)OM>R5-@Hm>s2G2GIj1cpe&jqcnVMf3N)Nx%LR z_NL0m$r#R0Rh|bRu-ShI;$D)6-QDi*&^1vN?p;DmOb_Px4$;Xnnvu@WEn6-WJ*;Oaegqs z4qB50k}$yYOI?--9Dc#l#9I~>(4x(26RnDrx7oI=ZBXPiTWfxxGy8%sEt)!TJ@ywW zd#TDd!KZS=L3x(Madn1KANb;FZ}DIpd>2cc)BsdTCmQ2Z;oYr8+*s?r=`UtoU7s@5Vw_f@>v~RkyP9 z)XEf=2ehVt-(U7@raJ3&*ZLU+(cHdKS*BW9n`pc}@ftKS`aefgyVz(K0jxw!uNgy$9J6_cJqe(gf%2fzWF zPS;R|Ykbe^y_QxNk9@CseOkj<7XAg+*xtZi8|MD1QfOsR{5wy5REBECREtruVYK)z#wPIarX4tVT$UwDfyNVXx#Fwe z6$A_jCq?`|cHR_A)A14i?6-?KLllaW+AMRFL5)9ZE?@!>3jcw#01K&z(b!RCN%P}q zoCc27Ra4)|HI86L|F%q?q1iZXO>Cs}G6X}+JvD$6-mK2R&GY^03G)b2gl6@G9fQIY(e=G&HCUz9DxGYAt_v8I)EhJwLibYs7p z$J@F3Q8dlUXk&d_!nlI)PLc>HR~EDyT`i;R!G-zrE#GRH|2f8LgwyaAQqz0Iy2>&V z%nbL-GzV6rZr|t}I~VQJcGVH=_5Fnn{)d|IG?;+_BZ@a~u^{!Q`i~DH6m!wPNB${- zVN9uJ9}c<%yOiJHWJ!ngcf6!877<9`aAzquT2bgsY|koNVl#=ZRr$NvkDNn^O$^j0 z_&*1{F_(nR)Sk%S2$BnPvDUmtyt!gDlZ&^eQk^*B~38!sGM z0uAF<%yXwuytb9KHwbNM)`tC^s2K_TQ@x}Kz5Xz)Lnj+M$s+%>{-&O)MWQ$vuOG3o zb!@~Nx*PQ_X*RdD$1Jv9MjaazpaqRdM*trHSb|Ny?Q!#vNdp!@3BEvEKes8VdHNc6 zmTq{YElrpEBs@k17xvYkKF3`dkTCIh{T9Sm6mYrFBf|L(m%B3-ncu^f#QB80>$$|d zQ;sez!xoR+C4j*vi!C3O;iAE$M^eW|!=ru57g#^htwP~cnIjr%34rke&d zzpw_F2ZGGS?h26J1%Ok0s?Jzj-Cd6VMF1AmQwO;`&kkNxG@lpyrwmgUBbIyoJF!UH zDVU4G8Tot!Z{D5lcI^rti*b^gG1%J~K+9Z` zY+%l|uo*$Zt0fIBUkR^yG+|;!wE&W-RvGT&;VP65GVUr3755njI*u973i z=0R@A%E~JLxt@UN|6xV~J|3>Vhmi}g9FoLKD5vz82zeG&wZp!5U<6Gcs4bC9NAkWW z-X)K>M;La70=OQo`=zi(|N3f=jxyS>4mVj%)!35-1i2^u3iVF}gOl))3T@Sa%g;NQ zZtXXhD;`$D>;1b$NeH2G%S8VqPZjI=DkeM&*uQT^P%~Opj1SeoYtI|!r6U26$Kkl# zyDcrkkruWttxfJYlN#tDc7wdsImHv)6(9dTy@h+MnnKx|>2vU(E z0$IXz3=$G^=Sfx-4@@yu-w3q5Z1t?_U4sAwH>gpGfYr`WEm3h?()g9*|n%kEjlzw(`dW|R#X z~t4HUN~-qLn)KQZJ57EE1HXN$6iDYu+$)$ST_+$?A}zR`Yv)a zsi|KW8u-`2G4HvaL?8+00&_3F^Eu+v4oFpA zXBgf9tsok}cx4;+SeOUthSxGg-F2<3DT8zHSLrMks{UyG|9Y8~cK#ucmf>kQk6{(my%?q=M_%$X3`D z*07_F8>2R0nWmQ=&6f7bGE`^EakMIwlL{tksFYhUgm*KgsKV5aj{8xZ2WC*k+Q}v7p4^B-;d}+!O4?KwRqN6}7Ua-UX?&F_(jbF$s^zNWc+~?t4xy(fMy`E%8>4q-=~<9l z_;yyi3reyoT%@LiIdQ(9rnMf5qCg(3zce|i)HzblGgXCG3+Y0Xdo$Rgd(JRhuXdh* z39xf`_kkU8XoQD%j*pPiTcu6OG}?xivoad`W|}nX2zem6b4&n03t|g*EMQJ&wYNr; z(#H`yTZ+Ebx}@s883Jn$rw4%=8^Sy*j|C&A-`d%Hkn^qHp?_ruvxMJl6&%hV3!>y9 zfCy1je&&nUVt!u!VRbEJ(tdgu9GVpjMghj~=_PBsIDq56r|5Il-;6v~1_x*~=nvHH zPbs$~{DX}%ywETV&86sCpN|}Ml0Rv{-K*+cnDuRITW?Z6)kA?t4E9oPFFe|TsJZ;+ zwjI&T@Gj#x;@$7TOVhGBq8t9N;-rZ>$<6rWe%BYs5vM!xT-e+bi(FHAS^J{?ym09$ zh4XvnYMIP_HyN!w94iJ_ihGyqnv?`MsTxJP78lfvXCqE5<7(7-jdA~&(D6FVaZ5-Z3<{C%Qe zVkH^MF{7+zf;T{t z9^z$E6&O|+l(TD-zIVw&dJv|^eqp%$cvJKqD5$(m_%lhxCN>qftkTXA3{ixMMzyW~ z<#A>TS>(PI?yC*Fx@I#LC|(=wy9~qAi<(U~fik~8r@#*oTkEFF!uC8En=nCX_9Hvt zX9>K|;bPIMPGA}eet0JI6e$1?$jK(l5K|i5j2FyR;Y|vejP6n-N;)?f{6PEgrrDW? zK^aaQ(bL&XuIMHJIEI64ip)IEW#1J?$(6!QwVE0eJwqvak*$LAcsz$?i0GiFeMpvZ zPi!&)I07h&@QLt!%SD2q{2WmMl8H6L`s%tppy6Zq_}zge?w7mPEZ3%8kI|%yUp%Vq zSy6w18rl}pcK^)FKP1g`V9ArQh@;RQm16@X^ovzRV%IMgu+;ZqpOH;L2 zrJ!hZYq`vKpc@N~A! zj^v0)kNID-y6C{n-`=KW<8R%V0JX`a18Kk-gOB5BwiJNXrMVK$3PHihA>$o>Dr17T zPj`&o>KHk@TA>m0?zrsPuw8CG#9$0(QR*{o@nIB+)0>{dBJX4!CpINx;g4h^e zY?heszr~Lf9}|YD7IAlAmnmjpsQGTX^ z#syl`(MSll5fw>v9ogU|O6R>csG`#iu8`Bn%MiJ0C<=zv)`v(x0WFPbO^zCYdhOhL zMO`STRsy)$VP%G4FX!-vn`KKdz9B2iXFo(HU@;;rtrHCG(l_w*r0QSuzSU#WjM|t0 zs+UPLillBcjzU=2LA(-cG9(!84OK9qAVNrKnEvJ)IN+5i>*YQ23wABL;52@s|M5jFz-iIn*C z##|>-OcRaw0?%R_A$P=tdm}HZWYe}ZJTzJ8_ew=S({I)rQ2WR5^P+t_yhESUp#BA5 zNXhSFI{H6z>}odKJOQT)Ou-ono|y~>bc&H4#pMBL?u(FbM~_YaPVH)?aM1?OH87JW z6oI)WKubT=l^7~VZz=+ap7;KuJY4_RH%D4o#Z0}eV8($mx>*N|a=34Q_j-0(A%#FO z2uYFx8-Nf+zTebCEsfNf~Yo3M7a+mfbz0DVtx$)%|q6S_4J9VAx_w06O7pN?LL zQm;1$+2%kF$-sNx;53;9yjU@q(-IaUZoOB1VO*T$v8Cyo^$q}%9h-+ovv|Gq1p<WonwA-QtR+T>D#?#X zBO+Q9PX7tPP3^cQ&;|v$BlCVehCL+SL%zIG`*D1e?sN7IDzHTEBiw2@;M{fo?}8KD zI>d6tiwg^?qm*0ENZ+64#P&O^OL8$|H8oSGHrMGFb2=3gsWpXDQa+F)zxJW`s#2&_ zcxZnFLp>*|=RVmvN30NWCOD`YNi{Qpa%Z84Cr^16aNv+&bGhy31xA^Oj?T!vht?#3 z9M_zRVc(G5tw!Gcgynh~Z^5lZ_21t~G-6zFd^p1^4gZaO*Fy}?y$;E~aXm8=kQq+O zcMu~HfzQF{i@u$J+J7fggCRXe|qMh*pDJ&OV zv~Gd@ZYtfyMueW^GR3vl3xScl1+Kx6g*2t4X(fJe&g;NOTobd>)rU(lnvZzE4kkpD z{Kkvk$fD313{IdyPQ0fn@aZ~K0CiwwOwqu*CMjXPe2Z6bp5-CbtfahTSKl1r5XQJ^ zD$af>Z?O_nT5z_Z1}Eei#xjdJox{ZLwmwBXeEsP_0v*;gmpXU;E9 zX<5j>8tXHRoPO&>-MZ`GK8Z3@&eT=A7y|WD_CzuO(X&Jm+K--;ZN$L5?ql8XE4mdK zFu68O(3#>cWRlL4={5U`ri$DRvS(v4TIoomP`TDUuwc~$9>GuBoT5tP4VsnJ3%zre z(+{w9*a5xq0@(EQ6eAQ)9#4|Al$Tqy<`KMu4s8gErPEB+X3~qzO~mtPQHZlF{ob!R zihUbrN2B$XnD9Rf-`%IRk7MAL7blKpjB zO~Mw}|C-Nb#2N4-m>()P$*_YRT*7`yYGyPB>99kjWh9bZ0}xcDm@iL0Xc!@nIqD3`RDfKmd@l6F&t@+-# z&CGNnY0FiS>f`#4zvq1yLIY+{y+cRdP-4g=^e&S$7O#fH<8!wl^XL{!fF> zc@M_Ft|N*q$W9-Zo}&qegFsm_l36lCK!=|z&s@o8wxOmqYOMNaw_tkfEUzoW!lO=6 z|MRtnvR#+O!))>XzUq=6s4HNt5=h5BdxfPC)T(>Zq-at|y=bisTSQTqBR(Y0&>9cQ zYOJ&+gNmf5^Lu&*e_{>iEj{ndb&tFc-QCk;-Rsi`skNMCx31WQt;aZF)tP);WiSH! z;A6B<<`_@g>uJ**+U@4WGcxOG|BB4+nlrBc<)!!n&Q|K0OpxIkAQ(nNi~IaeD3nL? zH#xk{1GQ?3jhR;VVF*^=hk1w(lK?peg_uPOsFv@Brd?u&)o! zk9qC@kIy>jE~jHJE@-Wu+^nm^Nff2>K|pY?yls+GEDyiK^e_Asv4s`|u2*=?Rz($# zk$f}u781^;QP+4BZpt#Y?&@a37PM8HS<=O!8-Vy~Zxu!Dr{}M;?(VkT-)c>jqG)|k zeB?~=AtWv}eAt7}-Oj@wP_MBDmr6j!$LSUwI)L3fi7DR}Rx5zmQhj%PvB*@-Io(x<9RO>*9c^{HJ$OBlih z<>o!pi+qvLR{5qUt^a@!<+Q-z;ZP4YJ(EM(U&oh@~{ z#uZzspNEegs=t+thCv!ZeXw1$ZWq?jgOzeQ`_#BvN+osg;Mwx^M$+d$0LvN zKpro-ZVi0g1))2pck!$#BI^GYR!EJ)Y3vQAK$(cq$vQD>F-Be@%tU$S>oXbulA(-zaW?y@9VG z7N>dzcjCFqc@h%LQMaHxR{3zYR6T*&T<+;w8!q*+{aX}$))YVM_`}Ur4{_~QBCk*| z`I5`vJRpl4&q@A3U=A~Xk=^{_)S?D$#>_14S~G==W8n1m)quxhTuE=k89`s=`DUx1 zHr|t6=YMhKeb-87@*6NlXCh^fqc#;>$LsleT}nLW6}K!n+P~hn^b(yqwAtZH15|}T z8CEuuUsmMe%vpRXPa-|lmTpFChKRE^nQAtX+GIG6SJrvq%Zu1xjy3>+Z^PH}8l7ST zzQmh&2~FJQcGrat_D5#NMj-1IW!$F)sUJ{yfp;e!a@}U5F$qh6j>a(Oj{4(_fw1yeqAv6tW^TMImZsI-u$mLp0tQggsLUgay8YK8OCRHI# zaE6bPgR6*;(CFAnqdx6c)M!x@>;J|(blTDsz?F+p4T4%mii=HmH(z>&8n}jIF5`{t zNGX01DkneO9ydc$R;(Ga1=f!37dFan+F94Yxc-lm21`;W^6gQVQ<*R*)$NhxC zq*U?!2JD4cd62WeGS)0s-2Qg$wtXfa_~z~AlCUxb4-jR|XSGw^LB*&!DI7C#DK z#0~^{ZPaHRIYbajS{K+?L|*royC;ik@)SdRTh&1|0!ibp%~v^-70p=DM zrKTQe$m9C2zDdt!D@Jtb?Eu(2Ysi?G?5_`m^2r-1&-X11;4+skDJT#{4js{=z%e9; zAWRTeU3J!3zyRW?D+E4Q=nYG{d_#^1H57tg>h3b`;Gj-{l!{Vc?DTs4 z!Pn%$WqH~Sd;_@Hag?EP4SZ+j46HLT^p(eV*Af1zPLau*BXnLV%<}S-c5}VgzXhVF z5iVnOgR{sGO#2gMfj`l+2I~9b+lHSwD*u(P(s;@t+nRqFf6iCdExn77W7AnVv)1~_M$?mA zYYXkKq}8-({5X!1O$_uMf|_QGrmUqz+k_SvGMBY`{fCTC!C_p3VapVImyqGM&A9T> zqnx9~cpcka6~G`B;@2QB4Pd0~j=vC^b|gQx~%@umNk@doiL2 zB%}F=e@WTK`-$yacUW5$-sm$mdW$1-|hHmd)VegE8w8I2Vfw3e+hN zesx!wY4}*`rNDA;mpM(~%7>!EXbx?n=Pv9R zIovVRm~C?eC`yK-x4E=aeT&YzjEO@| z8fGwK!u&{&2RAc*?JTHuiUYy9r)B3wFU@_^GkZS=H;}Ao5x+231Y*?Ig6WbgF>L^^ z-CCx~O#le+KO!(2U(h^`di0UTJ>Bg=6e$W>aONW%Tw2q@=Jc1^EM^W;)VT|-;M0Ty zR!+(U6YE~7=-yhZAjWT=F91_gW&Jep*;2&bg-H~?pN`4Ci1L)#oBw3Wf<4&@_<|Eu zX6@@yEHN`Dmb{4N5EK7LmkXUCX6GttFxe|Ff9z+nUo%g2qWd5&$v|ES@1RO?AM|~5 zxfa|6&?0#)?qhwp`U@>c-;Rtn75>`gtsEsgWX`NZh&u}8p^rFz+y#QaUBPKNs&gKfcnb^lI`}G^PkScI0q1D zb|ZISGS{_2Xl0BX>(bzpSB9cOuSV}s-Y9>v!JX5`RiH@#6%NYyIv8*TmnfrCk?aU> zJplQqv67MUX&t}U3*?5Q2V;A-p8=8}#+HN{Ug}j^ogJpR#P1?ZYMeOw>X$B(lNBMox!|JSn;#Rttor;KWDNox9@$&+h&e!lde&bgrr{JL7Ks=w!Zw8OUp;_Wtjj)kTp&r?$+W2O66VYCB*FNAxeVy~xzUQz+ zJO@;0{k#Eo@rGFt7Oi4pN(E_0)xFj8+in0y5l6DmhQu8P@7qY9yYch{PDQ?B8nc9^ zF%urf`DiFx`mf|Mp-lL&>Fh=w@=817`v;wiF7j+92b~~K6_5Q-iEn|pnyJJH=il>; z^&P`gIoUS-?J1YK>&@#u>-_5zI71bM;@XkOWA8TH6%jzOEO+zYc@o#g)UOs7V3urm zNjs*L##vxhufQx+pYW4KS|G)ltw>!o2uc-%sW=^WeHFmS!L5ia0}pH5+SzpzFtDg`JG#ND z9fcs|NA1Sc2ePojb(D7+^5m__>Hv5iptkq0UA4; z%s$4`M=MHt-p%8Q;eT!m0?`6DUdk%tNJKv6B2wNa0{j|rfP)S675 z5=J2Qh2-W}0e1s35CRM~T+n=M8VF7A+*PR64z)huo4X_O$t2gXo@~Yd_DbvDgG8G8 zXR*QM`%-~3Ti!hYkKM- zk~emGf*c@aG&jg>UQ)>GXDd4$Z4pZ4^2nxai!%=AVI5=N2aKul|I6MlCH?ilZyAPS zGL2Mz7RD0qOahMEk1n5{LW{3(jJ-_J;@@jo&H!UToWHN#Nl*J5T4A&cfE_vmHJ|BC zKOgoLM(ZFoY_HawxxF7msi#w+Urr+)w$dTJA?x|BZ0?mW#ZH)`JMwE-R8KJTWubQ( zteikmF|#x1Uws!$*&!qgI6n}1$#}biBwp8kXf#XMr@LCx3iR`b2QL?Gt;>#*ZBP=m zcf@vC67M;agoYY~k!~M^iV`#x-LAz4G()|btK=)dc1{!=QF)Mq3MN*jQjGuWpgp_|T7^vQ&9Sf$o(^n&MMeZ__X(31v>{qgc9v;<< zEnImR^q?A}2|C6`J{0kj;u@C)Wd#GT&kR7HAK-Ty+3=sSuf;AL_VOuJ&fRBHet`em z@dFC0yw3Numb(1n2N{8h*FUXZqZ$fvV$dGNRlsrF=R4Ohd3iGHrKJ55xPDIvwL4>{ zQV4L+5T(IlB?A@RnB=eEyHdFpAa`xM7(OpVXu_{twSs{a4Pq5fd;!!$Ar<>h%v}V#4bv) zUmuaTX`KAsK&M{5qA<8Yd%bNee{lDUDyB6NzrE|ZoCSS#lBlM-&o~A3nJYb^q?OIX z7$y+j_8;z<#@_r*b8LHr|1g{#cZ45Wl&2E(&Ut|ov5;2)*n458P`fI5zJjIaMl{76 zynp&(8Gke?5KAi@5&yT{+{N~(K;VU=a3w4@jgv&hTK5c>k$^C61S1Ej$b)DF6GJ~3 z_|^6zLf6pxXA7vl<8OS@Zvod5Cule z!V1^HZTcU_d|||ZH12$HL;96#h+O^ywJTDTr3U^;oa{zYWl`4Qa&6{FJ@}cxf|0pI z?U|!YNk3{fO@;!D7DSRg9zar=L}n&nK;~9advL-2p$)e2q-D=* zQG#bK_C;yD9s0qi0nTYRDq9RGxJWABkD0<23;=pMPP>LG*TFx*eK2Ax^663#E{1*< z>Byf+eRscwk-uQRBzmUXC@7V#A@mA4Rclwc2HX6HGa_c|!27cwpvyp*m<#_Pg__Gm z0PxAQH+3~fhc?&|Zzu=Bhy|>o#NNs~l6#ffRnTh}Rx(~AJGn+=Y+=&unR5K?uW{l# zd1->_Ijq;o+b?UjdP(ES%Q$5sU1>R-r;Ox5UgP>5C^4hV&~b+e%xA~utD0r9Qre!j zJ4S9;^?l7wP3OgmZ+Ss!$(|yZ*tT1!S-h5Ph6UhO`Cbym5p5scar*7ro6FA0a_dn+KfjO|nc5*v;y95%JQuL)=$_`jJpvs; z_(8+`U+(UcZf#2s26}fAdmUj~sh+)PaZ;=6(CnL9Zo-P1)2`X`mHpjj73`TN%7cC z%1n}k-8?iJq3;WKV6X+A{Hca*TPgJ~H+S;a{aBG&0lOD-?sIPx)VZ9JxWN!t($ve+ zAH;I5=-+HmauhOi5R|;UhYZ;n9_rkgp?U0zsCwyEWlXH5nh~2+V~HY4A-%cOGPs@0 zes|c+WD^YbjW=@C*}cCjV-ML9t`nO*;rKJxiwZxMXZ7vR*6%=0YQsg}bbidj=|p6* z%_X=Cu(#9uAamX4RKuht*v?s4!Ep+~uiXJ-Ry#hv14vCCtYcfo zTCix_R6}@txIvZu__bS*C5k^TzCo`>_%@p+9KG)r~ndC8fX%ffHhnfMA@Z9zZ5lxqXV$Mn5Z?!F5J zQ!eM`X;e;0XWZNXAPu`xbUX9NjmePxr!F+}n*Zx^Jx0cSY(buvX}l5;3UfEOM*=PO z%}()EJKJd!?J3>Gu)vfYcBL>9J*9S@FBrSa7Sng-&~LV^Gp;v-uncf zMJjB@T(jNfNnj;Tt>Ul`9Hn4VTU$JN`JUEPHn;xpNnAL@^A`>OEcdeu5qf}KW0wcm zz#D%msK&b{eul=T&d`1xYbKX^NjL7H)9yK4^Wa(*C-bjD;uqqsza!@3^hk)0&HLF| zF`qh-YgA^;Qz91Qwr3(kngL_2U?3#+P;qToA;+tP|Bun;?mWcNonc!Ufo(51SHCH% zad|Ll8``qAZo9$dokN+Ga0om8@j}V!crmp*W}D5qfKiV!SFp_gIDB*EZ0e)aM3(O1QF@ zENPbQ{^AUxQ;!wC^<^5@ZynPag@XYLBG=tX0obATg3H= zBj?|_ybJ$EE}r!%e{L7JdsIjFd!*55!}3rz-5UIFyApTn4@bc_0l{U3b9>B6)UP=c zeBtcQt#!jW)tLDuvBkKfg{O{bk=mOjV|z}XjDu`hF!H^TMBDl-Dq$RI1~Y(MLzZ4+ z{Sm_UF7%HHrIduraZSn*F}Deiz>GhfXFC+fhX;*kAEzK2Vo89Obq*=C`)<_MP*wAE zuI!jPhhe2#%VBWe~ebZfK9`WyI-$nU7h%REXLH9Tht)1!leA$X%t zbSTaOcG^phEQVktGq^E2`EarC_NwW-Wx5rQUzq$T)M;v);jG*wu4FrDRnVaqx5Dcy zj6CfCI;AN3=Yiplhc7ocx#}t-s`IB0lSw{YxfT8(j;{tCJ`C%50fOhN<=vkw?*heQ7Fw|D zlMbV?!JovG^hT}1<_7=a+=4He8^?boVJQKdW>~k4`)sZ^2>BOIF5l8gK`kaiD5n4&NMLG0i}+yq2u7ovDrYA#XQzW z7XtF^fZ+QR{c`iv-tw=IyRjhl3W;`U>@%*j;IciKKY3_oJm`?J+WCRzF+lQgp0_1% z?YEn_k!V?8xFGb@nlyW^Vcys-1eB8p9;F%0zn<#`UajB~0xY)2xpw-NypG^DvKZ23 zj%5f>!^o;E=4$@AClZ#L);u4WQGO>c@jY1VdsBF}ih0e4gsWCgdtr#QtRqJZ+6(dd zmNW-P9XoQUKRT9)Gykw5s3S5MeLivHyH-A(YznZcw8D#f*a2lpK`3Xr-91YSOmm%> ze@ANJq*m;?_tQVPbpD%hkB4O@5+rw1cs`2jXF3!N03Pn#~fJzCmkp>{1U2jc%tLx^RvDt3jzTkLT&` zW40ee4myR8N7&IMP1#R6Iek^ei?@nxYPcW?b4&}Y9ExsD zVQbVkdD^p>8cJI$m?8#&ppF(iv1Ob}KDb%9;L z0MezojsW@zWe{ZpX`uk=X4raHUdYW1paLJIn!9e98t#LcEG`7fBGIHMr~rnA)IR2< zg!N7>Nwq@LSU5z%NW#0qb{1Rc{jUWo%#N|c)*a!+!B z3sJB99~)oNY{5tr9U&L7U+SoSlFBW#kLeH@f%SvObLt?$RR=%W!{4mo$_S|KWXErh z7vBuiD3Mkyd?O?Sm7TPvQUi~aBle{W+kWJExQ+gdSI`O1&UNP3{4BpXCK0G#0w!VG z&%w=6lw@p>xw~|*qc4Okng(Bc+!6ra-3Z9;ml38Yd4CuSf+(wiA%J*37}kDC%$vRJ zl1MTX_5}Y2wnRXaqq)dBOVeNd&8L^blA1)Cjny4^vb5sU+W3&kZ{?1nRSV)FPVcHu z!SL3Zrmp4{BTpG!9iFDw7dedx!CeSfsI2iVBzo`BQaLoA6(!a62h%AV#38+1;!|a^ zIAWFN*9Cf6u<6tQ0yaR|pmRCXNKXHRJL_P+YHUVTMofW((6eGWi}Ve&HLn38I5$Q1^&1)Kx8YSJQV9tRsNdx0*OP$;x` z##`_rR-@Uo5%(iosraNasE1VrQp5o8)1lP&6pAUk>8oiEfjP!;Q%)J#V;sX|)zIy& zhg|L|HlV|uee0R(caWQ8Z_c40Q7{I9vDuxrwOr=f?S}kKhAd6Dw#tYipvoPewF`Go z5kxRch$?TjYnGa7ya`A@7!B8}YW7OVxGU*dDfO`#k_o{wS2iC19c z?{jTfq(BVna3#RC`Gw?OfQn-RQMMA3UOpgZzE1Ifac1@@R_LNkV;!wiSJFKi1#2*bcrU zHexm6YeF(6G5u?(ly-iZ>OsbNN5N4h+W(^(lf9;2FbS^5)@p1n>W-^NY!|hf{P(hz z2b~Q&PfhOq{j_?+Pr?h#r{m*2;FLyfH#NskZz|Zw5Lr;h@5WeR9zGpwC&-$oL#n ztcdmitDmV>0fncT_L1h!2ZzoriLH-86l7`!-v&bl5>8o&!uBxpvV+hn`}vYMv1>Vx zpE=XEG^m09{yp*9nyc)W4gZYg-@lVO12kLaccyP?n>XS_*Cex8z+%XCT&iK`CZr{d(&0l(3*2sC+@K|7P{qP>depx_>Vh$@^9{TPNRik4`iO;_%eN z;SxP%*wDep3iEs7iMZ7Qh@c8J@w@ORY^^n34l7u8M@5_NsV&3d!X>zev+lhi9Ay(% z`tm!`CkN)lwl5Tz#zo32=T>8py1dj|48cD=s#@yR|IvWa8{6SE3B&m3Zb1Sd_r6S* zLu8e;;<|Nkm-OCQ2~+AOV(vFUn=gy>-rDRYyo;j43i=aS0hXw~7V25(Qc)@4&C}F2W>UfcCMKA-%QzNb1pW||LbwPxSTfZL7J1 zy;Mm@&1;U27&pOM9@a^^S0p@MEvaly4IaJ`zX=6oJo*CyUtWIFK=;e?_0LC+vaE7Z zjUm_`H^gBn4{5l|61AxTGh#(kX$pl5W1#EawrUE}T6BOq3Y*HydgM7Vxzv*kflBDZ z@&R)${)rmHUgThFx_PPJaMtENdhu`r_dO#S&j%W}}i)iIH8Vt$N zU1B;AQ^aj%P#~GlN%i1xTaUiwyp_!am@h8@bj&+7vnGOC&X)t0C^sLrZGTU@+uf+N zb0+qWMvsqyWt!(Y?ne6ejOHA##v)1ub1n4!CvL-e#?XTFiukhwJ$qyR+c$B}BbjaT zMFI}xg<=1kV)>|a66q+jA_l$w9ck{`?-wDQOkKfeFhAp=4JTm*WLpsdIag|{$$BIWE49|54U!2Pa1aXhUW zy7~f_gdC5(RZHw%PgU(i3Sd>}S&3{?p&k3)@KeiftT;9YRX_umjoC|XsEK(q(CFob zJBz&V>Vn|jA&FlwW!DbcuJuPGk+YTkaQ_;cfshw>yW)i&Z{$KLLN@wKlz`GFqm3 zva8rWctT?yrTkMnFI>ofQR;1WwMo{?sRMd-VU>ewq5lK~ttSO9QrxSY-rMzgC4!yv zieM9JPdv3)spBYbu&$9?LM$t_9LEQ)Ko@|T<4iqzHJHdDZjq;uDSz%|{(8(R!UMXm zek4m+Yh6OcHA6|u@~d~h-{CpnZ>`0Pau$hyRfJBTRiVj!CxqM{qL=Y;8>DB-=EGH3q!D-c|Mdcs50 zB5O%MabFDN2L2j#FuwH0Czm3aNK6t#WqwOEg0>@Zn)YA)N8>zZ$_?M<)G>l`)u1}Y z>SItn-KBE-2_Y-mZ1&SJR>fUa>}US|Z6?^xoc2bAkiw)0C(kWH2hNzoyC6mP{pDD z-s|aDu=s`t6bHt5HDFYx)>$#toKuq(Qcn`U6q_#3tF*VluCi)k2+jZ z*x2y-%;A+NLGL)z{0$%Ch{adE9K zG0W{9P*&KJ2MgvS=+)oQ{kVG#X9F+ZDyIhKjtL!QcH4NK_Mgp? zaO_63n)e`Tl@QiA9o0dtPCndU?!>hGKs&UO+by z-#?h%JYrw!Va72T9XOwA>cTDjqm984QYBo6mw7TV`6tN_1T4hJm)O9Ox@dVoH*^%L z`#b~!?sUg76&}7&%^{aj(gw<5!<=_d37O6P$D@DP=y|dDII#P&gMffzVWx^Z_*+er z*7Gg}B7RixY^<)|*z)4=?o+ds8yD;(m+kQ|X}wXq$jk-ySj8J`Xn(!o29!F(C#IQt z##zgyozQ8m%D0Q~er;V_e?aLF(PjsXflX{FRLE(*A9zX@l)^r|wHyrXu>!D9t`Rz& zeE2ippMg=VPG|v8S|&q#^FL7_ik;VNCN^%l?pJ^mAaQetn2E{II!qI}2Gb$2*XxVk z8*v#+juiEuTt;`ljkOv@USi=X`Tk9q&rJOE&Js!pcOXNCxl3a*LbNO#5fXT-T6W3d z(LWoA^Orxctm}6mGw^v{30;iReE8-445*~Iu6vXpqIQYZc+!(t-(aq$j*QY->)79k z-AzLUdksL&U`nhiM?d@dFC8qhPa6-^49koZl= zSg)Fn=EXmBl^O0p@yoWnwG_pS>+KGmlt=$2Un*`q98DZ$aIoD7|7bS+EO==)z+NHI za$MC|W#lK>sC3Ag9TU<`;w}>jo{5TrAEYA~BnvV^Wl+)9IQe96$Q$gD5vsp8k+=w3 zwi4h_b6u~(=nZ2AcWDyEG}^)ZbSz@;vJ|kT4-bMIuR9B;D$FPk+WdWx8*+v~$W(R| zIN4mD4*L=*hS`6FqsQjM34q7y!@^f1P~S}qJppn+EnKZ#uz*wfbHNYINNc#X=(-}X8-XZ0TNP(>t>Pf-%`R>KG<}pA zx*xqeHG3v8^^{w8d+I`z`h?XYqyVdq_C#x`xUbbovYk7XxG8M)a$ zw@DvSUT{WMWDr6;@Ob4zv8jxAyx+?AfM3rp`LOt|*!JEB#N!D1iTT4D;!0g>)n&;E znq9zT`Qq8oV-QoKFycGh5XG(|0u=l$^dE4~vV3D};2LjO@ap+vrOsC8_Q=nF|#-tAMr-E+hfW2jG$X6sK5ZFu)Rl-J>?*43cn9bY64gtARR6FfyMMDI!3H4r9@yP%xs0AWL9Z zUBCd?4lw-&Lg})Pw8>Kbr+7OBPCeGlIB#^z0B-&h+D&^mO4fVMA3~&cD#r4*|NB_L z>G>8*$Y<=q^T1Xqw^Kje5R2Syv71$oNU_f|$MpzAqSR68L=pZ}=C3NhWO+ZQtKLz3 zENs=;;HLdvP3~H}OORno5#Jan44L63 z|J19xqlWKv+n-$LQ`T8>b>fhOqAb+p*4w9v6ou5uWukp%eL~Cp3C;=#EbA;DC!6zE za~CKX+svZRk&p|v_^*P=yty=;;_sYPXT-)U0uL@J zq4Pj?1LQRa2kfRzgE{{8HSTl+N@HELGo}F?QT8QXsGl;bL0wD4mG3HcX82xGMD}er zz92vnu4KCxB71SRjx&bBOnh-z7vlEkw|=*5PV8r=-Ll>%>Wgq(i_ zYCb0>Jc}(blNM1~ZW1hyeen=-b}@o%4$fr7Wiw&C`66-buC;s4;5EP0X~^;1Huo{<<`eXcZ|G>zTvK!H zXJlJ_4RHi@>>K?tFS9=!PDR+&G5LP9v*<)PO1<0l7>Z>2ut4)T7-!?6I3--_iUfxu zGtpdn{Z6l#XFVi|rUw{{&Q>C_D+&QYj+H+pFtMb>fH`WRUXynp-XI)b+36mfljZ{3 zX+D>kCGdwre?3j&7Y*i+8tT3;n29i0!g5s1lPu_a|!NP@?`p3gW*@47Xx^1iNH%uQ{jIxYW;iJTn1UPZ{jaMLPV7 zj(M@-2{=1y<`ls2JV~SE9zXk=-|Kk6^%oGZ&So)H*tG({LaU|e0^eu?g*e|fZG>fe zQ1f;j!9bJfeh0O2aqhak^6nP4nz{qu?=cnR`6vVR{8yD9*-~_i*gU?+jNk1FUZVaK zl8uJLL(X3H9^rEX$QCz`ewE8B{PLzWs`PSTnc4)isD)~EJ~g~+1EIuR~3sE zK*cthW`zQ>*K)+0U6*f{mq~?ys9Ud%9FA60H5d;tn)xd>%gr~QUX$s{IgQH8Xc6)K zJoym746Y(&Im{_Y`E({>jK_{ucw>&B1C%=oLojb{u6;k_cIM<<+3-@Z|6rR~C4MD6 zJAZsWxw|SiO8H-ka(w8NJ+>htd~Bu%5^IV=YG;6n*1yKn*A;0X7o>9N#xFw3f*GXv znb)KO0KHzgFGgF^?*{(;+`l=237@^hmc(o1I<_IKl8!#|>2|F{m)PGEVJa$m3624m zdiUS~_G1l#2nKwa)BH%v7g2EHFe$$*@CBX(qGbVgye60m?f$gPX4G=%)>5UC=`gz? z2`}F})@ql67X^Lf{X-N9qkS(IHwKS+7fTN3M+jyl;@s z7w~{h`QNXkSer{VdaFfS66WW^0dfuwu?|)*^z(1D4D#F{RvOdCxbl0f5ov)cYt>m| zGm!GGw{j8#gV6Y@TMLA=Jk!6B82P3ZYtViU>QHAe?Gh$4pBYk2(bpuL;Be6%p}-bGmXX6_c;nH&6DEVu*vd&+v7?dwd!Ey zn6KYu+0PDzeEc5d_yi6_AbQJC#&Jc6w~?Dds05HyPRD5VMD-&c>yCu=-Dq%@`@Xe) z%LesmRWX`RnfvP*jVnpf9-NgV%l^gPoslI~)Xu<%D-F5hNrXdNMoTPRyh@XbljAvD zb%o~VyN3`Z22rl^y}S57nQC7B=b5MZ`k`V>b58n|(fa%=D2${1D%be?twl6vS=puO zXn&e9d~gxo#j6t2@@b21x{g6Z-L2~q@PntK7`R~2IF#f;!~BjRpd^v8BXjLzxx?&;_yPz@i z>2%9~=K>%#i)<3(2LpQtVOg8_7E#-VR;DdCG-a{3kUm4=n!IYUG)p}j-+)0d!KY?U)DRc2rc;2qqj_xK%#4i z(WJ?&Ka+D>E&%d3?dD!_Hk0B%Tqp%R%g9+n!zckfc_v^7nHLP#v<)9Ad# zjf5;DAO~ZlopQQyHR7oCV*X&)Qa!+R8ga0yI#SP~Y-+``7PrjOIPHbj3P|{DZyTEj zj44#blTyLU=#eH;0o{FX3OarccHAqusJnw96#@mEsgT+_ANO86er9W~{(DO!yV5l! zqSXmbsWP;htd`yp(SE6{d@q!CaKBUa&2&xR$kb*0VMA(O;-iC?vhDOTqCi;p-K1(HghlLtY8ld(_K9TgAG@x4G!L z8M4~+kGJMFvnls&^gJuG`@tmapN3(-m2d8HqV19wtn9-NNwUme$j2RP6~c)|oYpW^ zT#-6(B7z}0Dm<26VH@=>@fL^p3D;FYmGe-ux$B{9gw3d`;s2qg?n;4gPfQtk+iL@&p|R|DEp?4NGsZ+Sm}1b#csfJm5*b84`^<~)b(UgYhSxh=!o ze%_|#`P|^TN5z+}C%yh_;65Q~n!D+OVPl=gTee#2Z+*n0)z>y$R{x@>TWcT8YzB!C zq+nLU=TLV^boq_8VRjmZ+pp1yGHIz9=Bd?oOfr#GkCOG2vP^9iyf3wAZ4F0i{To;5 zm?7MZb7$C1rc{O)QaT~smU4x?d$OTAqAgrcOI)EQXVKDQoMEfB-5;t|Y%EH0BnW2mh!B$C;MKtzUznKBBgjVEESqM*8wt~f%3Dt2S?6}1L><3X7t`2d{F|i#yiMaI zzsa^f#Kf8S;vFWAh8Ln{@mxh0A-1|<1mlGH6NA}EG`bu%RFMf{iXfk1aai%1mi1O! zJKE>~9G-rhk40TDzo2>KkmL#MYdIZwqXG4Y_5zF+7W*o`G#<>%OTYbie0RA!nJ%pI*4co&&ErUmrKQL#(x3oiEC1HJi~LHpf(O<}CjL5LH`Y*o?FELmV!^ zr;t-%a)Hk;&$?Sp(1e{V{!G#!_oRN!15c_iu3*UR*eK*bWxEUdUG35+u_Ft$#&1N_ zvVcC=8@j#=zTvusszy6&PCI)P>%Nd(;@q$0TDLIqI@&3ipB9=3s|yD`K1 zLO3c-PBspcR4i zB_T;AcNF(kwxBZR6Qc5KvmYR}9m>iamdL?wN&V?YPRx}M<-|C*-Tx$gwsGcH2CBJm0zDF%D#hqI=M__^C8SVOkEN8NzuHU)Bi|#EO zwu-nYz!u-&N4Ru?6Zc{9%^(05tm(sQi+f!L=VA#rm zV#q$sea;mgU!h!1XGW$D->@N2m^P4J0zxjHQ*D|6_e8F(zgHJ+g84OdQsg-iV?8kJ zMKFN3$J^e1#1WUVSVTKzd zG}@QMfOoQJ#+{C&Bc>r%4Aa<*Jh?4>2KKjG+VE@}*Jc;`V5;?thDT%zulUDltwl=a zn9sZHrtx6LpsRnvbF4ZVWRRSBLyWtiC%XMq;78EgWX!Rfbo~7yU*%rlu23STWw%mj3P21W z2s|``ee9pi0aI9tq6!x%Y1Rp*Jt8?zDyUJTdpqM~aC^^8!qm1rC-A1e2qdrQl$#|? z67Q>uP;@ZiDiQv?5gK4iyO8^TcDsC~4;MKIJ?I)=hcTF70a2OaWFAghO?YS{MkS9M zhG3IiaM8}#C`+)uPxhhp=i_uTmc}{kh#*xAjf#9G;IaqvBWoFC-b~qTZDRAiC3}6F zN}Ky|Ii&>)95U<(GC=@JbPuCPfD%Y@1eOKB0Q>*HZX-c42bUjQqEM+(cIVxvwcia} z@B&bO1StM4u|v1vfFHwV%Jws}<$7@{TuSad%EUg&fkCGQ?XUPMZDSfTB^NFC@}G`~8n|Jq zX75vft2`x$JG>M-n-NP+uP2AxBAKNRnx9EcIZwKv!(@zhJuC9)e&Y_$*EyXmAhUT( zhK5p^BbogALhR^-0MO#(z_1f6Pc2}YKOde-Ut1M2Un-89JT|&d#M4?FmlH`J21(fD*n}d_0W3%Z~CI zq!PzA6Lrv*)kRUPo|G~6HX-R=y;lZ9CFkng(f3!RJ_)t3P#eVHK6Zp*cN zl2&A8g=Vb$itkKY-1~O=!W~O@QD?U3hdI9`HD$s3m>@#Ag?~eA*0lm9V_A`qH(D28 zQLo^gHk0PjAfnmSwlc<7umx*|K~9&gmX&4?^fm+;JaepA+0DGC@YWd1?R5J9eu42@ zo6daPneJj)DG}371$4jrJs%#}DNWSF+)tv7_qPVWfYM|7KINUzMiI);Ovb9Z=n}X~ z6KVT77djJ0@#v1rEkcPV&>1y?^=w{geD*{l5Z5=Hgw$KDhGIkgW~SfF^C}nTWR*qr zZ1DVjfUrBgdnzal7W3OSb?ajavSEYhz=3g-UQqNWh((K43@AZuX??MCPmxQAh%33z z_^_4kE0yS!O(OGyO56WN+>NM>&Ht30 zL8H^saq-(9<|gw6$t$QRM8D=Mn9Wg%v(i9z`*hl+a4p;P8`E01m(H@Dq5qMqa97_R zEX+hLDf3B3BG8MVHzUL9ccCY7=(0=5?$)$`<879h-7BEq0>~R-aD#VOsmH{JvBtL0 zUxH2)wIL)%o3VwxF6ZBeyBTw}A>sXti=~Q*V;t*Bj?F^m;E9jpy<)3QDWRtlbCSyb zF||V21!mCCUMR6SSFA&x-^h~l!884lYk%_az8hLkvPuBL$fnT_ry}lJo&3JCNk(FH z8Uctho(VETBmiuCqep-eNOJ^%S-=47mSSqA6v;D(=?XY{qt`FCR@Jm|$F2DTXfx4B zgZTm^9BZ3N00lt$8)>sL(4LN7iU(wL$~=5*h%Hi~J3)XvM-8f5HAuc4z**J_A(o4J zGy!<0IV*fbmJrwm_tOln&BX%$Z9wVV1~weL)q>L59+fo_s5JlGAc(*q!HK`wn`YZnXb!__0>Ljls6{0Vd_>(eZJ$Nv1~PLojoa*K>bF2D5C7n z)cF-Wg8e@KwH@cOQs?jz)AQFW(_f&Q4gfy@r23Ef>rOcHye$f%G`jYIh9xQKRQOJT z@~SuP=&ApzbxD8$PKKXpH-#^SnCAV@jEyV9;>Gt@yL0HtX~T>xH=TkHAiO8+QUG0o7hi@W2BKGIGg1KZ_c?41RgH7()495Wik%}!bd5d?ml_;d2 z`EAdV!HiO9)*T3bzmCVg5xf&}_~m+F;Y2cYQx`A6BB)don;ApQmOH}ypYEN0D{5{I z@!m8$8=Niqw2Q>oVEDdSQ`Yge07%5vp!*c7>rk<6qPgw{))nM~X?+dci0?7IuD}=R zR0IvECdny}J=Zpr{CSgF7BS2(O!D-U=wgD03lFR*jz>MPI8QOtNfCA3ru@74#w((d z`qaUO@nL7%Sq6YTnYx-*NQg%geF0)kZXVO}p0&L1I$1=$0%-G{?vYn%*YB8NmHh9Z zkKWY*`Z_d&i2cscYYetZRyv&nQudNu8t4sisaU zu7DzX`5}nv(jR+(t8z$Xz7j1+QL6PF7fY%&wG>J2BN?5|jk&;iC?sY{i zNMgY{Y(&QLK(z9trP0swd6kS;Sxe8{wxQ$iJT~wIc^V0Ns56+hr8Rtml1V&ECmB0 zUw`9WP30iMad>9~X5R5W3e}%?&fbG^r@DU$gE`c;#+=iSqs%Ei7Jz{`8UeU6lnW^% zAS0q<(WAf#BpL!hg5Utf?O1Vo#WCezOrP(A(K;`)i$vO#N09MPP4wR3Vi%e4N5yi< zY1?oXyAa5u%ZSK+75HTY=_GPhTmyuRYjK+#@Jgjs88PjE>KH7N81DBiToQ+=!ujOp zHID3OSrZ@}l#@k+7n;1)d-m;4aQ)@4{Dn4zT65tsrld}oT*fN6Ncqb8w!TCHvXf8Vx>WFwK&fxoBY$TwPC z;;{x!V^#IZztzu)6C(>v?QKy?l|NHKEjH))Q#BZqU&-(1L(nn@WjFCWf*Px$Olb}j z#hA`}Njc=5C-Zd2Bl^!{8LCV-*pK75cub{k(;iHwf>mteNBDT+550W|jO&AAc|uT_ z(=$rc3k*IA;5VZ@)kg)IROfx8UfBdq+PVC3QCvUq-LgiSJY!b9LgWv;?Jr#*XEm@0 zppI9Hsw7nEZE8(-R~=&21SxlmZKul_P8AXN*&swv%-J{PWcPRs5VHb#V#o#t~GR!BI1Q0Xnu-HnKBOB>sf#5h@75>Zib z4R2lOopB(#l3tZht%xxc%#0Zg3v!wOeg!;Mm5Z%8TTd%@TZHVk8!~ydOshEWEz%WB z)c$RrxqiE_)Ci*fAcc)hB|C6mS``Eu`C9fgq4L;2&p1-G!ZG)lPJ-=@Dp&Yua^1nC zIz;fvgwD^538cr&qsq&P3H5h*0iQ%GzG*;BFa2ZllGMAgysq~q#c8G?onOZ?z@pAP z0ktqAVH57QCx)*9V$9ks_hDoDWeRqaY|wsnz@hk*zhi2@x1qnf=af#AtZ6|IXH*P9 z79}V#SW4@UL%#S8xyts>Q*~u}OqIV|-bf@qJZy-zs>hqZJ7WL65Xx@+?B0sgJ*tVV zj85Ssg22B|LBB{(x2(AVG?53OCW@k)DU^go(M|Gc(anu_2^awahb@A?nz{@3sYOY_G*8Ag@pOF==*GaC2tOEL<@^x|V%SOhuhr7NuVKe## z-!R5L0O~o~`r=RjcBoAE!io+m5rbLH!%Nh3Ws0ZBV0Y4l1klNFJUr?X8ZzXPXRzfS zk+a#x@HB)YoNAY{g79r)oKlzf533p`a)vFs@jHpD15D7F!DIfsT2m{iU}wIC_SNV8 z`+jW0%kG|6!=!S3bZQO>4(zWpgjaoxp41$FzZ@cmigUsp_`aryeoMw*!`A?EVK?6q2<)4x zMMHV9Q%gsP{;XYC6lPRn8Zxs`k9Wd1XvB*msn=Rz0lgNNC+-}tNx0i=?`^q&KOrMo zujsZgLx5xT)q3wa0!N)NZ-g;C7(g=M2Qoz8_tiCyHU;cJEk_W&AwJC-W1#9|= zzWC19_WGlO-9TOJnsuRM&-#Q{$5ZP_4N_t-^0~PaHZO?{jy+{(|EZb4N~$8c#U>hk zoTogsq9MepIxTdLt&c_Kz*?|v%bXNS7hkl5=c;x6kgOkCATnwQ0vQc)RFUfQjmYmh zF?!V9gGDV{L3SEb0a(x8jDLPs?z&YvoN{HS4aFcRa$^|tcQ7?gp=jxwnc4-K?pP`r zUD*moXwu>1*9tlzmUjP>>vn*EPLDx#jA~fy0{zjTXb>Th0-L;_SNv{|bu|auPBQH|sAl#0``?(j%`^4o$QZ_j^!dKKf z`*-CX2{epXs8Ri)-s9wra_|*`LlGc_)rW9yb=)A;BA?^ascwTDq!1qLoW zs-XzlO(8Y(+Z*kr2PjZ@YG#NV7mwDecU#Kpj0b{xgskyk2u*C6Er5Ct8}VaAX6L>5 z6ni95Lb-DOiw@zmJBJ3xLwUH#_iLXwe$<{h^+pRw>;SAlQ@=$3B>sH4%t2_S7d_X% zw&Lds`8GKeK{?bvz!w009T!8EeriU4lEQ2kAbDHidiaM*`mQDQ6`B!;1L5dhzQD4d zZf7S*P>2{X9X(KPR%6I54o~{d7cl`g%vg@OXYJu1{i3o1BLQBgr-+@|l&}5F=W|JA zZf|J*+_$hbxSPd1rW}LtF@)o@vD~K5-$@@sQ;2BV<4r9X7+ePv=%sHTyTuH$6rt#; zR-%_z274jEkn$F^X&W4V@V>41Dr6Gifp3=KziFrjem2@a!TUJrTo;g@Tbc;LPDP6UCiPH8ZC zJ)ssmt8!}Gq#wKsLxsbs$GoeToX(>I{}4_k}k^n zGaa;nxVg}Apv~2$EZm7j#dsNOG)Qdf%CY^@iZWqYg$bcxfS9IB5 zuBpFaRulo5-$%8VlO13D=mi+hHedUd-hQq}c@QHv_FNhPm@HV0npC^6=$6N$R-U9_DGSGIwA{o^wgN z9o=~T6U|8+LTi;(irvj`S#`VC9t99z5!(g!Mj_mQ*bNI{puoj5ohymdcx!HEwIqfw zrDsvg=YE`!3vk#gSPR?M2v6u$aWG1(k~}sUx?1mXUlwh^yV!*iBrdDZcy5NOJOIAFFTGskqC8vFXK8!}-5u=g6O)mXr~s4zxi|qDorugmRxZY3 zu$LXBOs2vONd5ey@DW(*P%7e4-l3X${#v1cDqIg1?d;#4*K0yP6&P?Vt6t-3p25|a z5&R^cIvjAX6tChc_6msG2K`(QYl@=}0t4y=3eFx+a~bYzXp}OEqbci%HT9}Korwvi zf5cqkGfLb+@RE{Rk}-;y%`sHg{N~SrZPE{kiH|}2zx`xK;cZ7|M_iJiV1RRFy>orG zE*1hCb6#UtvP9)A{SVcMWZS|j)Uc9<|MSw>wCYe$2;BBNW98aK6Ms7lDHRy?OEAy5 zG0=IX=W7c!8*5)qFlF2Y^JkQw-Cg6x(5f@vuZb4lZp3Gi!->75{F!X96=(uPuPeFm zs!oJE*THXQlC=hm*W1>YB%JL*3WXj~HD>Ah?*D(>Wr)rY=XSL2e4BHg#$1`0HBT*O zcPb4nL{CY_ch1bQru-b6IJ0dn1(m+?D!X9DA@4gOJnM3TNFa+PxT@`|g;_`Mh!hW2 zKEMku@>*Q?*TLoC1C0WCpOIuZ`&+5xe}W$Gn0-wqRb|N~&)9;kdXFAO5V4*6Uw0}@ zW7|a5;A2(pzW9kH7xkOH`nyLA?a0^EbvnpSm#ZCl-?oE1!6Ap6Gh+iI4?tjh?R%n) zfwBEWz#0MQGXD-KCLo5UA`yu-(4DVFlpuQL5Kr`IbZTB# z|HKA#YDcAh8>PEam*VMmeT&SsKL92A-eyYaPIMqbGWpV4&IFT#n8ZN_<@0@l^Q665 zH4jS=$0b7Xe&OGin<+r22 zmq**Jv{T3V!A%)><;wT)SwxJRx1e_KSK^M57Wj~Mi2>sFuSeU(zQ`<)6Zy7)?MUiA zaC__>m}8FQ>=+!rMe0Y>VwR~+2+`Fm3>_EJR>?E~PB%MRe-vEXS`TOXD7OPXrSbJ; zoooiwD0`nF(OOt5gw;*Bpa7Q@4hSRp+@?TL6}lE*O9f7!4me`JspPwtiGw%pgcEPp zUM@ajKigjR>nqxcI?D%~I&oV<{=lXUue@O7r-qFT#8hl8Acj{FnHfnyt<;u#Z6U}b zmXiCgICE`KV@l%U7BQ@EXJt`E&gl|H%Homn*%l>_K-!B8c%$1Tf{Nhtmnuz}2zx7&Lii>eN zi>@CUfz^(i^y^}3K=x*gBpp(+AK)7Wktn9@4ph+yYD51Bq!bU5t3A-nUYbN%ALfcW zN~P)^@g9}jB_#0SWE9#>sPN)5%>X7s7j_^k6y{XbL5w3fj>)$g0(mGQ>0tjO$=ua$ z69ivzBGmmABPSW3{AOpq2M_wG2bONrC8h86R#Hr?a6GyXh3llI<_mg&ULuU7y}>M6 z|2&52Wg?r4YxC4T+JL#uylauok9(uJUM~E;4;cU|PbQD9wY{g3|9vZ2`n^=57)=6C zLSu3h%7VH=?5DOIb>5gH7Qe!R$k=VC3mrjUg&}C?e+U>K5uPk*YfYu%?IiqylR&ox zocz*+h7PnfQGOixT2Lg4Hx^EH99sl3hHN!P{hrf+E=>S!3F^^va~`dLltv{Gnt%bbe%_`i9BfB=kuO` zg*eb`vr+HnOnJ@NVM7#AV-F{W9=ofWPo}c=Hfy>iSVv%+!v4mj0 zL7RivgzDLj1kI-m*Sk#lrT$8iea@c$X$i_Ry@w?xhb)NrOJ;b}E&eA1W*CoH&1IN;(I0}RA9_p=yMcab5jJe7WgU9}lh0R`gnL%s$ z5T9T|Lh?SM)L<9J?4ywpD28K!0ITMHYl|ueRX#a8A zxcBgthlA7nJA6Laf0;}Z+fsJ5G5E%<0SK|0`?dwqoUTc^L6rPvxg%YIJ~Y>^W95XY z8pWrL#yysAkZZE2yvz+xR<@X@8*oMDLG6^pd4T%RyFTy7M-yY8)GwHj3GUyx*U6J+ zv{J_Jf&H{yQ*g5qEM;3fR6c2XJ zl@a_D#xQoM0H&`^+c)RWZ=-U?va7fT@Ln{1vjW1EVB8LYxFPE~20tM}e<9uu#xS^Y z`n;<`5X^!Qi$tNq<`;TrNEZIUNq?y9g9c#xxhYoUO~ zO8NN>TcLMKE{%4#9L-0iURGu~L}93w?c&XJ1+tybq|);A zP^4}?T4WZh@vafl|Z`X&?Kadm)^okK!DPr|;Jfn{|6 zdtAcNgi69pgO0%b$Z1t~opk`}= z)w+w>$awt2?S^i$&$LHNK2wJ5Q=^;A6rY`DfUlX1J6Um4b)fLq?dlmCOnu8l;=XM= zVLuRjU&qKSR}zR^vMCfcED*iSXtL0W47o$^?+9o)mqFH@Lc5=Ft);77w-zY0Ku1XY zG(>c1qm*Mk#-Hzw%x%!72udoPmGkqCoL%ilM^J|NgALLlSDi;1SO2zt&mPVii9dIPbEX6CNdr|7axzMqOJ@Rwmn z$eR7l>gU@%9EAh+4^LW({}8jJ^4_tW9cm7a%@hA8>9MTyu5x}?r=n&YN#L(9X(jx3 z?e1u=BBEg(+9u! zd-OwSe-xXQdEdjx9MH120{DgNktRxm-Y4Q9y$1HvIJQkJJ8?Z$d)Ql4Q8Ux^{3vB6 zZ-w~#T0@M-BHH2ejW&VTc1*tTGg>nGq>OOBOV1NfnxjougLpXS9@~aKRv%0%+d|wY z0_c+6t;bxq=XLoh=X}Q3an4(EG*&JSS}m(qAAd6^0Ke@*&lKS;^|ORLqHQ`ae|xQj zK1wm&bhWAC=HvQ^?xT%%@Y-bFlMyHulDKVK)heAhJswF$p88igadSwtlvHluqy8L1 z>0gOGa@^wP)L9f|h?qyrD=^q$B@yt0Oo=pk(bw+?^`IUyv+K*acF<0Eo%(0a`V}Eq z8s7Vlj-^YYBwp)5Ti4Bg5Y+j=ZwTqc7;|t=Q~LIgwA5CCHq$V15c6J z0bBkpA^}++?EmF0Cay9i?6Q6au2K$lzB7y@GC?ibi=dv{Z{W2Gp@@&UB{B*;sfw zGWs@UH>?KOWZZSr&wsEn;vHqB|Ar0*!YTl|l|}Qrw|Ya#A*l!^ooIjUuggxikuuax zrlOg2nPIIi%aaf{i6{ym6b^&hfsJ%iJSDYvw@q+r%4rYe9}S#Z*Pi9Z>l*LjQ(P+b zg^QMAb!O59LNg%Kcw|viFE*71UtS*AZCmD!E#u2jM}D@7xxzg#5N__zh8=JKUyv2J zV08FYMO^1Dn$DrE-i5!I)aANcI_7>73tq+Yy8w8_uo@fKN%?(W^ogjT6h%j8csVvr zZkirL?4D=i3|32>dsyl8TJn$lUa58TK$!UO7UwtF7N-_0V?h~5(O}jIwLoL!8J%kl zVcfAs8qZo)a-#yvX=8K3y`xC{^5YIOg-T5YJ|TwPn&nyP;0pa^EQu7<_n>CP5yH9i z|EqZ!7AII|q3OEdCa{0TS4~gNA@Q#&73V~H8xzFu5IvDnO8p*72f{cT)`m7pvmtYa zkYO3rEL-Zw2{Zu(&+@($v4E6nP19UmyeyUz^P24i;SQ-39_zYWWif+>vw%3>iC;-a z&DJSK+gM31K~dExfR;vlL4q>~FfW|QG zWjuy~iX{f-@t=@#auR~3|C9fx86OqH3`a_NDYYYWpt_R0svBMPx#F;JM5?x51*|i5 z*7I`z$eZ%x2JAdT^Bxwl^cZ#ap8kPunH8ha3Fc`b8SjbrM4cgNssxjNW{V#c#vCgz zuX@N^-qvCcr`pV1={Y9qWqXOqZgjrIa<-6JQjp`o>OT0>ZOU(acF&3OjkMk*DDaLb zNDCddzifPHfCl#Iy*2ewuz(r?h%)X4GD{=?ba$gifDSlu1c6z=0N+$3c>dMhk!LO1 z1eyQhF{z4x?ZzJq6vqkp!7gu@e4(MDpO$CLUcO{}kkb|e<93Ve7%J98n(y}=_wm>}0 zYyjmy0)eDQrtaoGdo2fp6Uha29noB503_pex=C{ie_`8d#;_Mkl`dFfI}h(1olodF zt>wOv5q@5pg4JEnZZHp%oXJtCDE(;s(zq|v|Dx!&4cVM!~s-ZIL6A4f$Ki2Keid+`#C4A@C7Ut2N znMnfeiy+)1nKC;dB_F(w*OOh>&>`NyO0tMy{eX!c>94zA-S*sB)3T>gi;S{PKr|0O zj|-S8K)SP-VwqIE{S^Npo^ICWgvadm8RytwgSP1TnmvfnNl+uaan)Kn+R@lKry4OE zsLdknWkvwbm4GeoCY0-K3S1*0wO?(SJ!4$z=)%`h){23=41_BVTeTsUR@-X*;wR!&NQNdyh*wwJCK8}DYZ+vWqJ({PH z6bx3JJe}5tL(|sAo$VErv;^~5;=M6p13_2+)v|QAj+@yhzHKvpPM43JRiY+ zi}SDF4n|pf!1yeYi_(N+MLuR9k%iAfn{NcjJlKzq29jP30k0uJOM_5mw(^--Gc&oL z5RAOmtu)}%W|6?`&do(BB}o!CPaM=fx^h=www8zANVu{o0cQQG6os}xH9nw zDJLWYq9f6xzy}-{f&|R{;?*R4dvVEX?LqRq!E%_;fPs zk#0=Gms6vK=yOlzWo_9sFe%&88t6khPKFVy#4tPkEF()#%pYu_t`PWV7|y?uj0EG2 z;_W>~y;aOjZ!vMc%i>JE@O#XcB8RVS{>qJhUWN)J<&a0(C$3DPecg9h1DNnGsdfv~ zyO>cc(P4CfFP5$xfz@)cLp{SI6`{+9rC^NhLhFV-FAhukgN&NZT^3QMt51Q#Qls>l zCIXYPMid}PK4ktvm=Z|J6uQ8eCf4~OCl{tyt0iCvlm=!BOsEr-Gr$GczaE*zszQ*t zCTg46QCl;KAiBdaoEPg93ZS66=#AibUvbC;Hv=I*ZrFA_6U~;8`8#B7FY-Q zwfOcAw3IvX=Pi~#&BLnFX{sV*HMaWlkN`3%{hq@qWd-}ilJVE(JV zKk1c!ufkl_QPLyB0PREDCx zV+YDlxKaH2M_QGYLxm(C7O7>NZPmP4>ns2im#58H2w-w`K9Uw_A9xN9N82g621i6J zOWUsX5W<5JuCt1x9K}I3xy;T zxi{H=9L(<9%&LnOBedH)wgvKh!2SQ5eMn8kKDtGn&tdAh7Z|B;XMF6e>Fx*@*BRA5 zhdXHux6KZlE46Mc)dr9x+^Zd|NI%i-Q>oe`S#wY&Q1AbV}wy~*56s3A4+Ib0?I4Re1P6=sB!-y ztk>WOSY64bALH^^LF0(t8cORvo^l3Nl(7JBAQybqcbO0iLNGpiEX>yUr+}P2r*=bo z1pqC)EQTDi9zqK+I=^@t7ncErxoVzusWRaVzzXfD;=WE=a21SAR3fqhBsbQ{97lq^MYJ39^@sZccSG;~v@HQpDEdzFA9nn;AAbVD_dLlgH76oA zg4GYErF{5cU3_F23CWwiCfB_k-w!gt8lh_v{0DfF&hBL^c-{ZZ!kd#bF?1$WjYotW z5SCNFy=7G{No#xnKNYs7EEQmgobMLB*8pP!eLZW84Zw^kK1y*hC6K-aonch z*)QJJHFBx?^rZJnKc~u^Al;`6*S7lMuyY_jQR*i8GthUeePT60+`WfM@iLSJGEE>y zM9ZT`fCd<`1eOKB0O<1V@0}%Kj5S0dm|+T;RI3){vbX|yKTZcY&Of65$erC`PwRgt zK02(V=0HnO<{JFhG!o97%3qUw z(fT$B+DnsLbr4-LLLv{q)M-H`RUIH@9JmAmRJoPlAEiU{YQDHjb*iE>Mw@)Rw35TQ1YWt zhxi{}Uw2F~Ieeez2NKjL&Nt?Y;R2CMF4NnWO|NX*N+`N{Z+f`oB&`X*9QJcEJi=Mq zTChQY!%LD4) z)XDcmVd-dC)$(=@U57`&g7hfR0$k92Mq2n7KINBQb=?1WYg;aZb7H+8YcLfL5LUn& zkX>tC$ne3Dq_d9G*(sJ!WHaL{8uQ2*XepByn)K+ZsM&OD$5^W0VQShW0jM&~94Ra! z0vZ?5qbM;XhagH2R$0IR;puYuCbsjHaGE7=?}ly!K4k9~XGz`;7)MpU`XB@KD_vGq zZSxbEAu0H9mcW%~c~r(jU?Tdk`A8V^w$k@N6DP6I&T*`3h0al8yVwJ_B-JcBEQYFR zXL2|)ZQd&9FouCSmOz*)hP@PlmaRZLlb6z{Kx4TdLa3j=`~(vx{tyI;<8oLxpA2id zb0E$5rL%g(q_Ta}Gct;&y&xQZEWl_ zFNlVCuoeqUjzVZk;zuYXX$49vMaCKye4TlsCh6Rfv9XsyaV z$r&+Pd;PmSogYs^muxRyQ{?+H%pELV?Yxa1*V+lGc=~8fU2Jx6S|Y+MHb8@&$L_dn zm&$n;BD>>4=LSlT93rk7i~G#;HzefI zuU-(02tyI$8wYcj6e1 zjFZE(0x>C@L_+=@dekXG9$+R=$k zYKc9>(mLFNv)*GGT}LLobH`xW5R}P1Uz{!v#`wK2g{&CM#pGU%VzQ;Z#3R+gy49F z_X@sdpaVZ-VIF4u<$~Ldt+XQ^LA$UxJE1SNgp^AWJiEx)5+ab9~)+06Wf%aXU4lN=0SP0iOhGRBy*-ey2VN8S@FnNmALs=jvpN*SiYCyodi=7kl zGWK?G_=c~;^Q4p(oRg$8KIN{&+;?r`yJx}Lfz=(D&b!Vek!pjr zetYkjRX&=;gvxWx+(Ex@8R%#e0yCUpX2~BuDh%$o_8>&*jvvI$)``9mcWCs7i%kcp zIw<)|CDYhvks@&}Wbt-HC!B*yfU4W%XPkB5PjYYfkjgCfE)iMY!=U z0NLrN3_?yI2K8b3`BrbI3Fav3)b1D}iu?$o3U}EZkgSEm+YJugcWl?&`{wBhTrqBt z5Zho)%f`D5Wu+=29o}dPVg|*ezFRc}e&B-KRis7@1|4)kp8E{4Q=;{Nz_(@x5Ps=# z1w|7a7l-BgiiP%CU=kWmrxB+Si*u?Y$xefl-gK6L!~>+gXfDG!cV~#mD%vm})_;}> zQQut>J7e-aac|i(>x9Krd$e9hjzOdG*)kvND<;KxJ$t=a$}~NkHRDo&c4Z0dt3c>~ zGq4V{aAWTqhj16vu6{K>jwkU31$83c-GuqzXTC983(R-r|3BbF!X9ri$^|l@9){DP^6Hmhf1{%tY(v>fW?@ zEGxCIL3WkaBdLRa8FE2M=mlba`<-c@q782R#^fmUH4ByE3zG7Y5vf?s3VvEJ7Be)x zD(7gWy{Z0|A6(HtHv);R%!s7m-Ffit4Xi_7Rsle!E${IsKc)x>Lu4dj+JESppwQM9 zrvemL!-u;`Uq>Z3IDX41X7{+2PeLU{CrcxFVl{WUj;ZX~XFFU4^K-K;YXSb-3eT5; z9IRTtFC+9oXYR?VHw1-;Ap50e0fErM?OR3@e`*K-7GZ7DBbP_w$S(AIw~P6F44X|$ z!1@GBV2!euht}5HCx8^d#LH@hTN!(5^7*0g+EoZV^}4)~RS+pT-{cczh%n@o-94`*BN^8vd#ui316OVBY zo0}6(PI6@umAI(4M6K+^w0IQ~0>ACzs4EQ{`jJ6|DEhz|$1Z$G3B=qs%2k+yn^r)l z2u})6PbVs_E*#cWIcHNRgjiEEDThgsK$8%Vkp2Sd%~&8jw=~xL?77A8BmdJvm*mN5 zjUM;;A6nbfWP4rYYVxDkACb^fg~G`WBx(x8b zv6hK&$Ql2D4IE#Bj?EwUcp9{sFyV{&x?Hvxk^+1U2jrk?R$InmRkcXdiO*GW2*f7hn$_ds#0%F3RpJPHfi;VgCNeMfv`qjYH60DkfY3RK^fzER_!$#uk z;DCG>A9?Gaf*5+xz9mq`+9(yd);+hzaa$DukQ!2a2g&l#0V@xb2@(*16e^+T2L(CU zOHcedBM)}GxNFDKQ*2h@&)@|{=^jEuNNkOlz4}8!lK^Dzdj`MyakANyvE83VFYexQpIE743Zy!umzL<)p3r?g*nDlp7c|jN8 z*8saKdX##rZ$}~N4@Hte&(@CBI-FCh+h@dp|NSF3%BtX}T?Yf4{M1cH=EH2mw4rzh zjit^g8~RNeO>?W}&Y)!tR|kmzuCew_NJ1!is`A6e5)Vi+jA#0+ry$rGW!#AjX} z02kF)ME%p%GY%Hm!Du~0{{&39_@qd*Q`Hl+1-k10Q=Y%Q!$ygqusLRNj9~wS1~=AY z+GpkYim(O3Rs{u3{D67Wk4KA>0%U7TA@0>w>K07etf|9!mBxudZ>9~9gnnb!wRkmpKx)sm%`|sfVAscm6J!5u6EP(Qb?2d9Ee^X=Bt< zq*LEddx&i#P|mLlsdX<9@kGZumNs z;HdeWQB?`)n8!|Sb*i@>3}4YE%=);hyZGXmRq%U^j*Ie*Kb%2wK>15fH1 z5@{%x`_c}*q5+?8yQ+){{HS$C+_6;Xga3AVw7b|Hnp4tE6`#%e4h5A^N(8Hys8X-M z_WMANPTepey{ed@i)Zqtp$k0vOX)A`-a8wiI*F+fdP!|swSH?jSpGUj>MSkahA-I& zGRkgeEf_C50xEQ0Wb`2m=b^Q-W~9d*z&x{J1_AwPZtk5&8vkL0+KRG!4}l4=57n-ov7{Ts8 zV?(bKd5B(tmxp<>ERNuuW%kSTHo@D(`B*f80wTAi@M1zVC(MlKE=)Ogkqqf#TChF? zeQbB^&Ck)?-9C&2iwvc3%GF&Nm>Z!5+JThA83<2YLVYRQ@B3(9I8BJZM+h=eRlGu#(Ayf)m*0ra+e4Gh zfrabMX0D844PQLRTDg3iYH(A$S@W~_*hUG4*RfE6LQVc~P8ElJMkX0m-}5&56OdN4 z8QC3BlsGw)?TdsrpIl_LAE4Mkw#CWV+O}A?)e5SIfvMu>{srf?Y~5D4p;x|RI^NS6 ze{u(f_t(XH3?|y=(}_(`Qi0r)3no^yB-3Pz9!o~2yycIi{{cPC6YM6F+-4x2oh}C$ zg|ZK3D-A$1;~j`1Js$fenb?sh)r)C0%nHIRTv9vWJLxotqp=8 zPL_)aZtmQ!J*DOa0+?(lgKY}o+>3dOmdg&ngbAR4ENp)Sw13J-!JX2S1dvPEx2{3+ z+TDu&mPMK|x$GLhYph@ra73?#(Wo}O*1=m^Qb8DQy_Rk21$$;031<7Z5FIUWv#h^k zCRK!T);hj4!q>u1vk_=iJ<2@7?e7+q_NhnMj0+`n%yHPlLEYKuDagq4q$sY4Wh2ij z6ob-X8D1-MRi)E{A&M`whdX}2_7i=-0*ItSeU=j^aAJ?91eOUNm*an#SBV-Ye-AW@&dq z%oHY}K(0kl%B;~m=$Zd%whj8cT}pHM7~3Yhyj25-GL$pt02(nkM%z#KZB)M`ak0vR z@v_%4-*+B5FVhm8iY3xV{{CajQW#pD+4W!s>Kd%yQYqip=tf9hk6vb3^RVUy5tRPX zV%*aaraQUm*hfMPO9cI&BS)R`Ko~>|M<&%8BX5)ok*eFG-tgZp#u+g!7324-9<1U2 z2ZuNJfs{qo2e1a2a@aMsh;WoVji+n9I$a-@NT!Wrnb|9+Ci`8pcDGK zeKXeHGxexI)@QCqxR6$F77U`p(K=plP&ng1{K-hi%wKXky|1W01~zRX2ZNA4m1617 zOJa%c!CRT>{tM)+8mOqu)kmt=byH>rK_96*MU`6i^%&h}XuBx5hI;CFTmN`?*)0mu z%(;xE)Iwc=+Nw#?}DO&4{Q_GFxy8>XLn=3go^BWa0~oy1 zc52sT)6UL=-|a;Tcosj8o2GB&z|zj`$~&D3lfGR)n50|(Ke@$R*>F$^9Etku5gx(S zY(tat1)Yp%s=UV3ryDnncg+`UqUZEDIj!_6`c-iQhGDltGN~(;U>vy-`>a_Egsg|P z7(#1s)}JX!luxa;Ga`{(LVRrH(+X}*c!np&1eeY_+(nhG2grEoTKuC0I3>oROEJB; zlpJk?bc@#Uo8JH}5?A{6r(AlU>1W@{OJO9J{xxe)=Cl$`FwS=)fcD6@ zZ4q}!JW`f6xXpYINy$Ff&{M{uxWhkst8)07jy60>}ofDj;uMklgqURQ%mjh|k zeyekKV<#Sxk@CU(NW8v@hM_BzoIc}9m2>K;mLJ}=y>H}%G)}Am9zws)fcLfk)igr> zSgcBhH)C~u^wR+eK#tSKU@Y8*{ox-zP;^;ltCO46|FjKm)=93E^B@chZsCWQz%HZo zj;X#RD*tWWwZlE0RT8SANEmrLE#Ya;MJF=3S6GhpciA-@vp^N4e7k92v?Ot_W05;U ziZ?yfV2zmb`F8nT14#L^s6OS}}vH4R^gcd9**Es9@u9IFx zisK*G8pC)WRZHzVlD0f#S&o8=CK$g!6kL#kdSc~5kr+6|9-urjBdr>qj|uC4wl_L) zhHLt?B?V)_wEoIsLeN>b}bq>B(10!IMyg{GG5 zE0(&OI4RQW><~|7YNl6yQIB!H-$bsf*&s(IL+^9wAEwZ$5Nxr}<^32-5m}QdIOpH` zKJU*(sgR=%n^m(Yn%M48Hx-OJolWAasyQ*EC}EMtI5cBk5|*F^%Bna3ak>tI1LURO z)m9z7~xy1+J8GCn9t-rD@RgY{m4C;tqI)T&bJ*q53ji z+)-vn*iPn}Zvrh7>PUJIvqIsCr#pqXIGcV3=dQ`nvw=%rt49ImRQJ*x)iNR;we34P z#|9_gax2>GAe~JY5|?G3tLNPN7y74B+7|wE@G|iVDJeiA8Z*(OzzZxn0#Jgw&H%@t zmmc&{;GgFI@WD*qMWZ2q^lMzlM5^)D3OTKfk(G_<@=F?p6<+b_WYA~o4`4SRkM8u> zI2!D{7(jX)$ZB=}iYQTdliTpK zVZnbZ9$#GggcNP3lD2hbVghkS(8T|}5~=;998%to29(*G$;JqbeEZfK0sAFFRMI*! z4|eI!_i8tWY5+2OHCxn}#0(MIh+O=2>Um#lB25j|Sn-&G1g1n37UDAqh2&|Z~#H`0*Gj{^GpBG5P} ztgBf>@~?NS8-L-80Wi$kb3v#rgP^FGV+fS7%CX03ZYkywIgm-QKmpilCsK9mck2~q zOo_yv9M!_>bB}-!sGKfLJrveD2hTe0@vu=!;T3P91}e%GKT9OKt?*^9EdXiTkxxPm z%?G$datamY%zxJx6_q=pg`lOs(lR~sG3Mqm68(4{&HbcCB5upyw<2_2F#zx$ZSab9 zBc1ywfx4RA#1tS0#MSqIubV5AivjMP94ru-Y^#m)+{IsRmIspp4YzehdxveFyoF@r z5`8yG5ssYJClcLTVy2~eu?w+%Xtas!y|K@Gszlsg`(iB;qz z%V%SL>!h6mrU8+#30Pzon)ALTmt1J3-U@Vg!a1)(SIBN%r~v*6`iDqE7orn2u+7dC z$YuiTfwo;`{r6rHXHX;(#jv8^Q)!c9^=^y-b1w<>3tC*anNtO_;f>>`B=SVr&RH%) zyx&o~Br|ogkHVc}7Jt@gL1nZJ& z#FBGRgwX(f>Of`-!d2*07*HYp9|i>3Dq|o>%qckB+jpb;%y{yz9;S4UzAFkm5&HmS z^9^eN%>OLyklSJlu(=A6+#x08D0aF337tUd4a_9h1=EzKai9Te_&6TZM6NdTC=QiK z^Th8~e&p0mN%5I?1MR>!cEG5uah{~-dL5PBwXirc_hx}@bq?a<9eGQxbVSh5P4CZg zbxSHC>jt3cF1I%iaQE03Ww)P>1C2GvUL>6Lgx-90yZI@E>~^+uw#Jfh1%yUk(=03- zNM!e{0vd^|WS6o?f>%B&?1X-pWtmS{nuYCXd-&BRjgTX`ien_+t$cd> zN7y^FZ;g-Jz!waoUHXr!AdT&zMEk%}+T5kR&2la4h&%ndbqRmmjIgRrotU~IRKUZ; zt|Tu|e>J*J;&0x`#xsjkG#A=}-*agtaa)bt0bKC?4|-;)fBfO#ZrO#hN-glmWm@PZ z&&`ClTP%U`TuQMxeZB*dhSBzC%MHZR`M0M%(lcSV@}4{DLU1#?buPK5F*nWHEeVEH zWHnLmE^$-|r<_b7W`(0u$Us@@kus|VGEpc5G%uq^fCn6N1c6z=0N^}pC8Nyg5ShdV zeQRjGFvpqx52~}2_a+3Tp;=wMtmPm^P7RI!U~`>+p}iz$$C<5Y%l>&l?VcfjwV@v! zlrYNR53~=c{Uv@zy`|fW z8t1A4J|n>?D-NZp09o_R$?5(PJh1wMJK8H)o|BDDr9DjVzB%rET74VS*L9*4Z$yO& zqkI?x{1rnQ_CPs!-Uj3ZOB1(*15#-PBRedd&G#PD%Bsyr5dPgJX1|Y37U1$hpkvy` z%eSog+t~K_dN42)BQ~1dcL*t{hbjYfG@w5k_Q6+-`{XNn`Ke zoiFOEt&iI6N<6Ko3pP4P&hGhK|iIaZ22v8c-VPFjP?2!o4y#2x-_E_>iRwb@=T`k z$2sq02#0=Os;gWNh{f{G96yPgLyAgIY(;f|;YM6^N-{vmziQoaweqtf+LX9M=xy=@Ve9jzQ10hltw1u|10Ba~yKM}P|~a|D4| zzyRXQQxO+7)?>DgN$79rDV@cymJ>zv%Taq7_>Awr2K8HE5NR@*ydAx*IOf}w;V*2V0okvlKoxApy zUlxQOQpuvibDk6(odNr1*>z8Dm6~-ls7F>!mU$Ihtm8!iOtGqbiX&`~xEZMH&Vx^pt_BdW}2wVY&>A|X*qt#Beqc$aY>6X!b9FF68C}Pc9 zm^D+bDsrff3zs=_{gTxfhOhnk>4l5I<-=>u_l?1ISy*FHFn8#Q_ht{r_YgX8l+8bh z4-iquC7AsGJaLQJx>c9Zix*qUy};~R6y*zei&R5G3K=o6?Q#B{L7O}>EZd%H&4T_n zp>OwhbPcp^cle+JV?F6Vc^CKr>?j74O{m{f$B|3s=^7d`^H63NpC1 z((KnjE&{-z3P}iA5121>jWAUwu4-+obl22-Tg$rOk@V0a4du`@^*nqN%cCW{`}@`p zkLKOO|0d>&=Xj0EF6OW+m?5ElFgU$=7}b{GKa>DOSec8#Z!@A`s& z1=Q!;?1E^#4k@U^kIfl|y9Fz?a2f%)GLH%=DJy(>SEGU5M@UmbbrSaR$l!pS_w!j8kPfE+JCe#%#U33MWN;A z`$*V3C4j&mc4Q`)p~AI(6qPQAMCk>jrvH)MPb~xv8pYdE&P(HQA9S!nqfH6ZLrTgA zw!$=P9H7q;cv1_T6tSi4Z44_1)=FzI9VFuTHTXB~rdzj72EQuLo{>2I5ez`iTutm0$T&*y{KS$b@RGrd^^ zQY-RllsdaH1ch7}!S-y(A{6+)7YbaS$9>>_wjg8_f z?Zj&+QrNYB)aY7P+uUiwU7VL9WOFZ`VOE%vh!IK2=`K&MsCa)aAs-24>Z1`6%uR}) zdg)Zsi>MpbiqpYHzel=>i+VHMcVKDhkR~?9bJhKOtfj8y!zA<&{hXoMa!)=#EL`4= zPJC1~*Uwo!>6v(GFJ?#xAu`w+ey+~gT|lcc{A^LKOicWlYF`Uw%hs-L_y#h?@F^XQ zb&i?Z_yc+-l7y!=0yHOH%}@7rb~ZSng`Z4bQdRP*mGV^a0xT1j!e;8S9U_Vy-%=J*C9qUToNg6B9{bb@p6Frk>lUGMWBpVu<=`MEfqD@s}B62nze7L=Op|lXy#26Zf z5oY8AyWshi8`5ERF!vhjjbEVZ#YiqfT+Z7{4zB@#TIl7I=p3?W=ehWBEEEg1f)Z8gDVk|9lXWXTsuv7l z8jXK5QC0(9A6Q@v(_l;u*MersL%#QP$|d*k*nbBm238(U+1Zq2DCo9fCp?u}cOl8!X_~ zQXkz}46^?D)#Z)GNT;|p2W+oR!2z`|ykT}s)42{myf*&0wUP`7s(`EKISlSSRFRp3 z$P3ThuaX~+Cnz~_9Ud*R-nb3Vo3RAa#gm9(u^>*%Zn~qR{yX_d#0;F1cbUkpQDdL7 zH-RczNMyVRz3WNB`~&w~T5S|$zl?y!YgFllFlmSV27h*5L?8)MW0?q(>-q`2$M$5E z;-2Fi8pC;#we=kB$H=6pc;fHNryG@n65^q8bm9$p6wO;;mY~(>K$p(Oka3fR zsm*WsrfzhUblx;)a#%xmXTM1#8QWk{dqxDUr0rt#Ft(IUwC+JTE?($D__ zv6Bn63>CxE1 z)l{Mm^`tuZyZ%B0&V4wJr{Cg+fKCSOe~=`-J@^AGHG+s6?<6&0y#t%gzZ3VHOrJIp z+ty-nHoa zX2mZZ@mt~&Jt52e0^{MFP?}t?(3odOav8Ssdlu-({o9s#M9JHTn$~ zx-4pgX`pi0(@b1e>MHac)}EZW0x@eC&WOO$Lh;2r{QATT)q^a9`dakN67s}qUpU?7 z)j$U&Z3Pr#pv`@tq(de{mBqq_(A_dT6}p13&=%$2d$SS)!@k#bK;)&PBKc9&8Wtw) z=kbbjw{p>%PhuQFH-mw)rfO+dJuG4SWO*DwfM+vc5|QOpzz_5iMjmItygo5Vc-Rb$wSuUZZ*8z`DZN%3$}&bT?P`FH2?QjWNhR(gmKp*8kFX5 zImiyCgfHW!)axwh&Wn`I#dJ?SDV&c&Jk@Yc?RkJXt%|oGeUPgf?V`3$MZDlO<$ht> z{ZK;%_s6wOyN^mj*IdAVr*`qep-T9C8GK1;7CB-x6t? zQy6leOramW%O}ipt8MD!ajU>Rpl27cXpq(+3@)u@chQKL0u4<%Ag@vocp<&^b-rsn zC5d}_F#|~3_72D(!Ge@;?h&~p=v7?OKB>RIRKs=D2N51QeDPdd=8R>izTd6{F-Z^5 znK5&Y_e{D1O0|QQhqLd*E`R;bzuCm2)T@eoP&*3$rIK-V$PPD6y0WZwHz7U#zwjR(LB z0uAGk0j~(R$W5~i+Q3AzHa9@17)2Tct16Vw=Sqq0NAW5-K`7&Xb*$UXRh6oFXX}Aw z!kmHpGE|MWB>n`yMUGCb+r5+`YnMPne<*=RxgJTa_F7!L9`cKXFZ48$_y@5Y$@gH| zRq2P4s)w~Q%mCeVuDq*dvbtAs@jJk1H|H_pePz-YA+^%Qel(jV`uT(8=Bugh*exT6nfnYYZrOYFW_ z7YNkA!W-_vkP=M@W*Ssc7S;XSsNQK%p_cmTFX{W2^Spjuv{C33h8h9bGWZiIE+7>u zXVIf*F*L^@OJ#utzyP_Bi#|ALK*xC~tyke8lf@U6NW2DaL45%+YI<3L*@2$vra4iF z>F?FV3%Gg$=p?f6w(ibDjj&5Y9_J(~0ES0OP+V|@9gnzmc>Ugmrm6bg(q#}pvzFR8 z+yR2D?*p~`@0&VGr^%I7TQKHaK_`>WzTF*|CfH}*)tH#ap21MNasX4^nXZy+`G@Zs z_=0yw9tTnpAQ3G>m_BBYLiBqgedWzMXN^ywzWGH;5cmxwOY%YR->}qtKP9U8yIhk; zpH2L=HmD;Zi8%mA7~T*_zIFmoa)+yKMaghiF5wu#ODWTdlneDqn-AdIQ9kj!2~2%# zun@Ib(iWvqx|i9X(RWrS} z!RTQ&{=h&&(E9v#^OJ8Df8DK<5R|4xp-&#d)IoK6ItyhMhcNad7oHe7^=ZCQfoc>^O2E))3v&Hb06!&)-5O37*pQkA zGNEbxq~o>KP^%9tY!0v9qa)^3~*+o?EwzC~adp)TH=gWp@@z{YXmVsJM+)rt8p^6v0h2?NKXrcnqE|_OS2#L6hkEL1 z2DI*_159)MyB<#MyJO6)b#obJl%CC6%sa!dVS*gWS{7$nQE64@2?DdTn(Lku%XD0E z-(*%k>8H^hRc&%SnMTC6#GR-QCb#3Y7EotRkOoG#m*}1w5kCs@YF6M*hT}DS%6WBa zNRds<4;wa8H(yZ4YFm2nPJ6Eu*SQngG4U*`nPMDvFJ=JC@G!pJ3y zY8DY8rfOg!%-nitT(@6KDDFU8Q8shZGSBfxLVYoA18s=ZT6!1+g6!HDlDMUkW%8jB zC3(M1VwF!Q>9h_A-;m5GcxQx?hFc96mAPa}@2DC5PvklSNC^9rHKT_oRvv4Ypz9r= z2g!he{){d-PS)M?iHNy-UzK~A(Kh#_F#9U-qx0R43Sw&&ARIwnZlBCwj-NT3H9+E+ zaT^*N2LkTgX)V69k1a1Of*?;kLAIXgz{i?_-4QM|5)XtAc&b}@pJ;L=hfQen`#+qU zdhQG1kK~7)GgJs1FuyJ2a>1~&Az46vMPcT+XDTsj-FL2kXB+vL2p7#41m1?aEbEcU zCq$!G#f70s&0_|>(Hb4MAj?4~QM}~zEPEa+Vn9@@NZ$62u>RwXtT#T1e28Hewr|g_ zt&wr{DR^*>0hH>U^xu$`YvsM=h&bonj^bOltXtPXq%m7l3@5>j`L3~388QUhN)3`? zdWCj~kW~_^WAb5w^m=d%n}`T(U9qzeJ7umcc1y)a((hsV@G**Ud>cr70ph8={)N-1 z1)*otPi<~z;)nIe&-Ylmv&wlfFmYRLB|4N`wpsy`6)HZ}4nwdeM;&+KIA*ZQTrU-l zMyK&wWxT>Y-+VCQo>GMm_{iuAA?$Y$-0`FQOrUk_9ZBI>Y@k!X{U0@-^=-0!1|ZY) zOh!8Edfm2RLzjB2EnG#fD9+?Za}XqGOmyv^n_aLT=L-lBDn*ZrwqhBP?po%FDs9i& zXoVLjIKw38RL5ZIdkA*@#%JGbSF>-jAaL*y7X3u71=|R{Xlu(o?ry>vT3$6zKKw}j zRFy{Mn~HBUagw#5`psb4aNdJXMR&g+Q5r#AToyMT{;z)LZSY{CT#_+-UU0Qoih0VU zF~EgfZ9HMUF|ahc+g~413V~45rcRD|Yo^mdSNbCzd3K;0^-{EH@p=vtsJe3SOqae@ zOj?};$t(XjY1LA)o0s4Y3=!(382t)XLro5#4m!31A9_o6Q> zNaBMiWL7jwxXb|(b#ltUWyJqHR0$Vdj2Lo>E&JsKxLth{(=m2@*GhZ#8J7MGI`wJ$ zCyT6#b|a4rImy9ETu zg%k02&|Nd5U=^OjVhnE2^INPV+SiEpxO33{F!0P0N+a|(%1SbwS{wi=)wj# zp5z=WAN`MlCjvWcp4Ut^$T+4Qh3Sp{we&B@NTjl98x6c|ZA}+7iKw9wzBx$W*Ri?K z%LJ9;5?akLSB)1NlcYncl@JuwR};72M7#oV)Ze3I6UyG!R*0htofay2yzh;!eJe;;WI88v85Q{@ z?2dvDnR-J9Buc*8SptpfnM`;2eM+WcyAZ}eVZ=i$E?tdF5wF&5!cKr zB3!}M4g(RZOL(A;T*l&qp{SI?!5S8COFt{=N(ZRCYc1Bso9IAo%c z{ru)&iJ;$5M7|5`$9<$qHa%q+viWuvGp#F;49Z9!bFS4_&CtbbBE=LLVh}Q|?ne{pY~^2yg7c1 zPDRi?86vV&lo|k%m)_tQ|0y-8Ynr8ksE(-}@66O9V5C1&!v6#(j)0Td6($?i)8?xdJ{hGv= z9qlPFqVQHBCa;PH)}dyI@4u6G0QGlzAw0_eU|68LftLUvFOG+(Vi*-B)u9g$jWPLV z8stiw|1%%uuG#YZKEUpqAmucvwB#<{z5Y3-G-{`e2&Zg1NY=_PHAvmqk(5LEa$Owz z`$k4uFD?Hd7O>WjSzQOgspquL$we~%^o~sjw0%j!R^5EQl;b#dILY^X;6Gz-;&7rM zqqmkMuiO&+wxUIKv-Ht((3@)9nDqr|L+Y!re%|eNQFo2Jz|jOUf7o>N+#P%l@mPQlK*DF3AB0mQ;jeHc!Xv#wKiu_GigM=mwRGSUG=nsM40k`+{_#Aw zd>+uIb!|0M?ci@Vs#^Xs=4d5d3W6`82W!&5sYYVCKig&J-M5udGSm1Q%r+{*lf);7 zI!<%jEsSBInoVQSw2yMF?i!^ss7T}8G`;hA{^cr1MU)ZN6RU4^uzg-Pr_mW8of?hJ{%76iIck6jUHDI&*#Bp&7 zpF)R=P)>1F$Jo$4qOi5&2Rf;s`Y5t?>abE7&EaTm@SWex27U1v44HQSSA3$iA8y28 zR%{5E=p9XD5?3mF7C;w{%@r@LPaIDCIF^BI($&)+!M7Ct1{4^oOO0IO24YKuLAg!N zwz3xyNUFk*obzS%L3U)Bn@x~GlmgSdPaF^Kpm6ALec}gLb0ji*`wPoufA01F?YNXb z{26J=arZZsJK{dRgD0SdvWhs;g=j9|KUE7@cbrhveA^y!dD#}QGt$D*6E>D-;T^_A z-!@4aZUHmt04Y6esdozG$z;n&{-WCO_Ue#v*i|KAEdTzf7R!+!Ca8EilscRPv;Ddx z^FS)@#7Z8(P4|@7?#6H_n+m{GHN?OzS}k#T%5+#;aU`40wXUKZV1?0qj65X_VV3h! zgyDfC#A3&FgeXw)6VW}iuVKP#QW2W9q?WV;A1iX_ct@$kx89oJVt=r3(zy&mTC4P%(Gb0Vq4Lu#OdsQRfWExT zA?P4u43C2M-Sdd5O$3Frs4pP#@80!~__<49syCH1DHhp0AX*ii4Am?Iq7%<7V6m?v z4KDc=W6e@H>6eB~l{;=BwTJTYbF^i+SLrqrRA*Z933_fIGVBL3SttrrW1~la4?Hsj zmIc57*@)e2oFHVnrYYfFu(N==5JWNWqir`W0o^8^d4h~aMSD5t7U9WhIN^$lOREK__-Q5`3}ga|VFs3K5_3 zM(A-+KLVl`aweZK_`!z2)h?`DEzouWobU$VRD-Idj9MGL02_>04c~%+8O+v?rOPcc3A=0Dc}> zSHlpfz5;sK5(8elgI-2Yk@rBFO6lQC9!)hMSX;iWQrts!8mt#Id3F1x!Z7uT9O{>8 z%uNmCWU_F7n83d`I*ImT4J_JUtxVZf17*0$f&B;UL%XJ;gb-X zO|91_u<>Q#akpk6liKvwiAmy|xKhApd8LZ!NcP-}Xqf7RiEYl5TBcNSZMZQ6G1OY?;8YE^Y2jV`^Eegz2xOvcO zO(5F%1%^cyr@{%=r@r%V9>PsSsZ|!&Y(4kX!G6H77twZ%DvNs#C~wyvcL*rAP1Og1 zszg8_kIz~b_67?8Ir`KBgJaJfWS{#KFP6E#USY78*PwF)jL9uxE<*KFnK$$OUg!kw zhjqULVBi3zgeT02}~)oB)NM({(~c&@kHsooVJH-!!t0k&jk^j zJ92PEm8cR!G*IM{28IS4Vv<|KP{Nb}C`G(UpXd8t5c`mGhk&3-l>`Q@+_~G+!0>r> zY}a0JcV7Y1>AUQIwrVu5Wc3zz(ljSyOCaIk&><5-I>R(4L2S8rym*>RrAG#L3f|4Y?CpJVf zQN?!&fjD>hw+kQkzX(z>MyBiC&n`P$DyNqoFLh`2?v5>W`#MqbOayx?(cbywx*!N& z#aAdA0k|@%2QphAhpY#qM}P|~as-wIzyQ`-y~QT|kP@5ak-Tn#ejnFWWEOKgqiHjG zE5p@f4mMYXNM$y{j?rXkwI~!X#HZhvV>Jv&UG(!(pE55DnNbh)9K25aobW-=Kb zM8uM?cctlI9a9u61pHLueR|9I5{1U^6V2p_J)AYDntE;`d}!(iu>q1}-VDYno(U5w zqrg89OOxCd)L#@7{LXbz6dP=JYG&lY>C&u}Hh2zGG3p%Jdc~!@oSKi?jwc}5_w-nQ zNN8QsY3cbW+2{dMvjNZH#o@K)=80b&&l`bZy|XXG&meAR*Xb#^H{Vc`LQD2_ic&*K zm*q2ODU;f94(P5Ov7DLO_urMS@^HR&$>+Mf+4i^+PhrvRao4>h`C}CGnL9{Pov~e^ zjvX~P`fUgpq}V9Bt`lZOzeF;F_yx_Je?fIh`#!RJR9{Vd2^+^Y15$xlIn}Iw}@v)gU{ODM-}= z`7O}o#%@TiaY-kbR#R*XT?l3{g&>J9I)PE&^sxl9SiYn2xOeZs<-ok1IKnZIEit&WihEL$Gqr zoA(w)(#xi<3AqSgs|y(J($w23s_Y?s94-KI_+=_6n{nMxbXM4&Yw-fS(99Oa9v*I{ zDoYur;5cTWpViJZY(PKQ#>RL}bVdjccCA7A1lF0Gn)PIpQO|tfNY!gyP`xF6gsaN) zOPxL1+&kXo194`Q`YIhfwrnc4=`FqmFmZ_)E|{Bxm6hs*==CWJ+mQI+oY$Sw&(22_ zT#Ej;rw(!6KMq_$%nj=RqDfVvQUsjL?36qu>ArKIoT)pfp-r{-h4iy zQtm<8(c`KH;DnnqcOly+uty50QvTc*rUaSjX#1`=A>e3Lv@f$sY>xt96x{9&9z%Hi9cRW1%K-wrE%zW`}wsorC{3?l-c%OYy zt@d@JE$j{xOoaTq^DWEGBYJsATHYO0;Lz&2EmLDZq?4trpt8FWSn6^ zOkRxh3jJ(ZZZR)P2X>1yU9Q~~E_lEFp0x2#v#}#X$J+H>dXt4J6~oYRREUJ6@)YRM zCWc(6Uqev(2q1BcrT5Bnj3eMli>hcl9Y=2*?WiEk(UXqK@S4V-7>1mjFpq)kc%*)2 zN;1h(LI%7Q$5DiBDp)vY&}uPbmoq9>u3cV~zPy!KT1c6!gY)Ced~7D&e`PveC8Bo& zpdg)PA0kmQOku7UE+MN?!agWNpm9-M1w0o-Cf%BQ)WcrlDgC84z6~>tGy>E4J<%CA}vO*NLogrbB;q^TDVCMelno_VX zn|<)2TSWS?@N;StVyGFzQ`{?*`Z7I8cz&q1F2Ju-?i_0J_7)ujjfLmEhl_Gp^E)r1 zvd=$CVD5l$tH~8s;%>PE-}>&7@&9hv@^N&iw#6lYXP4E({4b^eEi;*Vq5R9I^1?&J z0`Y@*FnnUN^%o%`{11~h3v%a5U^&RjRTstUn&|VYYANJvI?1AN!hX%Zu@Po41{J%^ z6%110w4NgXcS3i$1vmX^IuX#OV*gS=DvL=G;^(pF@3DTR#zvHpFug6FwN-XYDyBVj z04D5FET{hWv8gY2YW%TzX{Ov=7j;5twmtRwHS^Id(C`ef5#Pam(6tr8G~(cBmAab6 za(J%qoNAN@K)mGY%soMun_&Q13VN~?XtLb>8RtU-MQXq`E^&>0Tl@CG1BBK$OmHro zBSQwZTwf|gO>Z64um01Kf@9gG$X560HX)wCG=jo5Cb_Kjuzw9TJP&RJHu{7)gMp~{ zEGb(5$0&7hrC-_&Hy-KPTDLRgFW!YKOQ6^5rzt6nWltP39{=Oc#9U0wq}>x_seT$6krRF8Gfr-0ZFi8J#?i| z_B9bMO9SN2UgdtmvYD)Duu1!%llIdy5zHj+&1Hm?(-&pWv7T7tm&=`AgEMlQJ0`7( zkxBSOYeP=uYfVG){xUrC_9@&zZ~dKqEjZ}w+75z45LB}u@kuPRryn>*lXX`Wzqo^P zaJJe1{s7c=$fqEKboX=n(C}Jg@#?h9-NOoyWs96c=67-?Y%-=$kF zQ9_?Y;XwmgEwzqROh{l-MUx{!$K@_L24u<{vs)lg8F!f;0y2GmGhr7q!t9^QLdlrO zq6`Wc81@2QoYJ&R1reU-m9eJ3@*%|HGqlV9u;n3GGh+D=b7Vd5I4kuXo@j{QEkEAX@@QsHjdv zQbJEw04#e*Rz`!8SD!f%z9niSzgCLdx^riHaVR$tQMRHDK=s0Bo2u43ED)iLT!roC zO&WK(e5k=qda*`esX^OeW-KFyGT>>P?Zeba0nn%icnZB zE>*?TztPJl4j}RG&jK_xsli9|o(e93<`KasgMg>kP4sxt-`EA_IicFstCL)2JLK(w z`}iFs6Q=>KPbILZA6MV-yatrNTxtG2w)bSAV~S)5b#`GO`YwmAf}M<}MK z<>F+$79&7RUSf85ntkbkvFE;am;5M&pn>j@#a0A^c^H~`M0w)e(vK$G4%4F?8n%~i zJt*e!pdqT7}mDZw~p~0pwOGUG0T1 zV+G>LKiFH=mj=|c-0^?%UBdY^|Mq~l5r8}hvJBDLMJhh$VjfaF4K(m~4_W3F- zQ%@04f=bGvq&m?Wf~uXhWtdTcq_ifL4bqFB&ICOaNQ`$RO8Eqkx}*1;9Z#y7YYd_I zTMQz>4-(Ez$8J8xV}Xz|(`jgtnq~Xq*pzj%+A0HmvDxSET%~{=lcre@kHc%*$GHrM zD(Cmm>KOgJwQCuhem8h251X}VrVUK=m$g_^AzVDg)z!YLC8Isa{RMF~2R`h-Lbhe z?4ae!X~w##QC3$E+s4GH&|oq?gr+6zsRd#~~=hsU8L}Xc73wccycAz^Kw2m#96Y3D1sQ5=th(|9`UI z9E1mmSvX)WH;kjoe2cK-e&6B-s++ufJ1`mnm@=F`Ef=(eViPAz19*;mesIUa)=-mHUB$E$*a_|AP?`q;8H;m#8DXg z(Kx|w#RkZjWOpKezE`qSwdFS;^#ZC+C~H~by0pUZkbrt8x`F(Js6AhP#!KxUE%zEs z2T6>zW8?JGq(dZ|lzS0$pH2jzQQNLTJl)C;;10DBupWsRzGJ(6@ofy#0%yNe51}nH z)*@@DPT!A19kW%*R**D|6wqT?+dSjIayhFDnzg@XeC*NVH`Z8DIw+Mtv!4Rn<*t2A zR@9{n;xx$HrLb)|v`qFg@6f_cl^SY*d|aWo$r+FnT(o)-f?UIf^S%mMTIUn>Zg&UR zJZqL~vJhU~VmGV7(vA~dABxQ(hyGlCG?FEN^XHMPL$P=397e71c@zxH_eI`s)xJDg z?`P1%MYzcHz)?b+{8@!eM7=L+-(aQ$OcYQ8-Ei;{ofzM|?z>P{WzqD&jTY(|ueu%; zgVoWrbm&-C@g|Qxy}}zj`Ra>K%qYgW1U|p}iR$Y5Eb{e6{=ZrF4~te2Rq#%>1K2=_ zEg0~;@y7)AgrRQ9j4&!D;l=8cK99Y;aCf*0_5f-}7xlinp^y3n6j*z}vhRpToA3Uj zWLK8+mXapwzgA7Js0P4df93<2`mxuHDH`JU6YKk$=Yf&U`ZUoaFnxX6iy<*9RIy#s zgz4K>LsEGa7{6ZYXaz)+g(j-c>O$ZRsL6n#Ly+)?I1^+AY8?z`;A@%pXQX<+2e`Ai z;-iV!BU2ek7WJrUBPPybtXUYe{($-{aMtcn(#QsSOubU^l&YF(J}p=$NX$cbPn`@P zSu3ZSmn4<7562TsIfJ~01%nEj-%3wFP>Gu3Y*6SYuw*DJxp_&$U+Mp=0&1vdVpUZ9 zR!&S>MtYUuy}or>idNw>1)}&V1A2gCpx_}(eg9tGWf_HM)J1HkY%yt+oCR7WO*rQw z^kbgu{936&W7`l-HeQd})l*5-<#($hrk=+v`008vy9-Sry1@WA^yb{EvJJ+2*7d*k zM^A3BsB~3~?%u!vP#1b!;x7Z?i^lb{Uz}gd>_BzZQ%f~mhOt3CEl;YO$1E1MJ;qM;|N>h>Edii2A7|zK$ymC#sC7?byP}CBg{S)=607XA(o_yNYen}P2%6< zb8PDo-L+)SQR*`)M5{SBTFT@b@oXDK&tNOjRA6~sHQ?^-cKbr2WhE3jR4unbh?ND` ztcWkqSMHQlYq5n(ttTr84}D(D0aE`SJz-H+6^wp8&p-k@&5|!_8pXww6H81i3gPw^ zrF09h0yji1(>R-g-u42!pIQ3ntgM3T27CS=ET8yyt_=Mo5avR-1_~o8X>#nHzf~=& zX?YYK_*{-t9JnEzrcT~(NCy`&zM12`9e`f~#KX`HP^r!<4sr0yp6(7T^XljrxPzY3 z{PxF~_!O6+eLeBmp6v+pi~q{XwBaV@azw36FEnR&%-8Al)D!n>MVHrrWaCJjtiymB z0r)cXNhv%qhBv^GqPQ@!%P>u4fdzHeT|fZS>?jDOup`OZ;2oGL44CK?--pE5R_~~3 zXC65%?%z>`_T~(;?EAStw?W4|tFG$BvO41%28Oz)y>#k%e7*Kh!gixSsX? zhr1>e^_n~vMM~HL4t930;3PDM#s0v{YFdH?DQv<;V=@K(N0~WssMa1(x=g6tD{FU{fxW6 za>gz)8LqrV&@vFfFcz%dOKsDGsRfTK~b6R+h0G{!E>sPWv%eqHOZGY;~~NHZjaloY;>_%nHr ztVzp;%7OYKS5JW;M6C-vaYr*W;Y*Cirla|X9rv16Kb!U&4l~#Rj4*isati1e2E01D z<1)9-OnCgYj~NQRW2-Iq(G_MD_{U7UG((qw)`SjTSRxHB6^|I%fMMF?lc?F$1)-%>RZnHK`E_L3A-T5^`r)I zDj+=UOc?o@ig*cdk@!cb-&?Yrh8KxeD)OaP&a^iyy8ilS1@OvQUOqhb91J<^J(k^s zlBXhAPfM+3FhaPwX`DiCKJQc!${ie+)E{s~ zpIKUM$qZK2>hGrtNrpzIdMRTI&TJkJED1de^Nz#`BLh#EaCfN zi2K_JGralc470oAaCOCicIwclsF7BDtNNZ=Fe?-(>%$L^<$1u9C5sIA!1GZ6%bC-~ z(Gd0`MR#-z@|>kZUL!5$!1MKw!SujPp4A!`$i!S>hJu+pGv0Cqu&A!JjF3*m<8-(2a>SUi0?U9x+ZxpEh}r| zX|g&bIB9N}!W6qD<0C}JI^R2lyx}cOj0_JrV@a4ICY*oJy|I$=*-F_rxsJgWAZ1r+ z+C;dUpi+-Oh4>H4(K1z9!6y*;n|}I!h}FKTarAH$d$4Ql?isQ%O)Kc&O z|E}HBJxnjVZSwk(S`XRw@(3-r60eK;$$RBBA;?w{fjQT%^)_z)sg)BJSEr_y8m5~0 zBCJ1B2!UoRTIMIST0Yz4l9sjXh9X4A*}h|*G#>G9m7Gaz8>zvhis{n4kIt#KXt(St z67s4KmKr9$W|Aw%sXAaJxgwwP&K9;n<_*}`FUf{?V3W<{5*~sQ#am&q9?1y}NCKkl zOHv4~MSH6_4_UxPn!peEM)!J(U&{^NRVaL;Hw``w-y8~$_{r5?SlTA!-Jp-mZQ<2ot{(KLBbw(BJh_auqA-zkfEp0UoDN_D(t4X z2yadhk@HIXhPSv6B)d1FbH?~^P3#KNKH^JYkVLG>@mIDx%+W*fZ=*7KzIc$QiF$tM zvQ$;p0KHO$6rGBUh7&O9^Bt((6p24pscLYbXR<}jWLcz#LnM~ZUQb+ZbmuS8eBs~J zLg&VQ24xUa&*H`v5ACVuyi*{GN@!;D;I!Ya*OZ#(eA&xdJ+56(45)#+rRTuGns8nX zRF`dwfoO>Z6c>S>Zs%$xf8g1NwZNgRCf4Pdm#8d6s;rw`V*mi+TlieOhhe|XK485+ z(zBjb9It=G9udhWO@d1QPPMBw5}5#vd?Kaea12?w;jw!ou#?$jhkgN50=ov>u6g_S zbD}?ok>F(44j%HDk@zfV*08*ZM?T1FZ`_<#s1y^(q}W+-(3NwUy?~;bo{8a>jXy)i z8p(v)&QLdAXwpn42Ln;$Qje@#_g7!a)IvP)hR(9EOHgWCm7vZo3VIDbJYc81 z@EUE+0@}4fY@W<9$zC)MZK;^6D=K8?4s}@t+#FtyUnmg*9T_Ybq>h6yZiR0PzK*1& zj>Ox!%}b6Mj{=QvUg5-HRT3Gjo%r_1V>ay=7H+JDNySC*RX1*aq3Q)S8nf>5#HCyo zDop`{Lq%)X$B*lo6_ntzpWO+=j-Jz{H8cW9D`Vdq!g|wARbC^0iegM8*{WrT|0X?c z=MdC?2h%;BC?E~iT+rO&JgpR#D>cbnvA)5SYnm%j9bt#ob_)c%Po@4GeKR4YvT6Fw>wydQ`+s_iy6$GjjGldpUkO4onsU8w76tX>lcNoQLK<;i*c*?^WzLYV$!r z-JrKc_8S6y^tQhU-X9b;G07*^ZiI?8UMoof@Xja2Mj_t#;O%Z0J&+e;(loLv=#@|K z`f_AoMH_^id@8!uriv*h2+Gu{O;^>LI8%LNvH2R4(2dGar9 z-SJ2o021SN$n2?E1NZ~?CfQAH)+dz#rNaV#tWw?uv-+EJK^@{5Ta-bn{@}eL(KS7k za&g6>tLf82-;D*|)9Z~0)O}aXDA+C__G1e0mmQ`&GOkM|v%4T4pLk=zJk%cn%f;hI zr9t8*x5rzfB*vKt~b3SAx=Og+zz6h1+F((rnJsK-*Nrm;mRyF1%Vr;1N%t@4=*KYBlDB1J!F>@+EUW(!9 zfXl1(!umDnCI(L9f@3%U9c{whPLKwZ1pRCfexzt)CTlWhGB*rW z&%aW7OK{%$%2w__kgy)p1{ax4`cT7glb59&4$ZAOyM!Sb@Su|^^rj|$ifLDP3+*=6 zPT!Y3+`XLg6b3Q6&+QDtFb&qG@#UjMMCAZEK*qm*^OndL_N#SO3#8Ep2iy%AKI9q> zYRy)*=u~2j4sUckxfog``EPv*%C$X*oL$)8u~VcP%!i_2LxRs8fe5nG46sz}CmsFpIVy7% zG`Jz%#5l@Q3d%Js7l8^^+m;+lP>;vi!DZDlnR#a|#yj4yP`D7k`FZr*c(LAYaA0sW zn+?_#ED7eU$s&|mv&YX1PcB3(&hN0mT0kOt!d2^o1PQXj5<&QrV*WMkYyQ!%#d^ovS&T?60SaX-mqWcw|MA zGAypA8r68$0E;slWfI{mUHXa9UvJe<+rGhsHe26|F%<2}>AMeCGZV#xS(!bBQ`U7- zQZ5ZPkXu7M8RhMa8VX$@`1JaEEb=6ec$Q8`SZ+9eHO+kV7cW7;NL07B>7wfGmZM|b zl&a-hK4K*bFCS;0w@w%&GB-$&E?Yb$pupY}J1E61af+u=yI*^V?~lP^f#Ftb;QN zBy)yZx@`s~qB)Mb4PpT}qU{@b+;~IWZ)^$4nlUpD>YZ@6GXAdJegk)o*m#4gqKMro$wd%s8MnuH z5nX6eS-Za|JZ;-b%>i4By$_onxT*Qg`PAd7T#Hi=fKl0rvp5*0;?I0?hTcXLR~- zMa&N6+!{5kOrHs$JFuR+dfS@}bJfbQ2TU{x%M^?QH#o|>Aw3oUn(MM7nHh0mn#am~ zRCbof%rKb5Wzit6u}UkK8$9B+yR#jbfmJHrP86>ft#vXWGc zO>|LMw7_g?7J+Gb>tpK+s+uf>6vt0v?PSO7XcBbH zf14<}dU1$Y-YkpW)qceo&NWXLku)1wx)@4EOFKivVi_-s^>Fv<7gg}osr|Z3;hTZj zq}2x#d1lv4US`|!aKVxF#|?XsGusKF`7J-?gft>vv;8!2*1+c!{HkYJr!kraQiu0G zR5g|$M^*w%l1MlRtWr{`WpoFBNKGIx-nFK<6bCBEsN8E&B zYM9z~)L|%g68mQ;V|f`?w_F>I5wi*v9)eu8P>3hV6Q=K@LU;*f&S#1OMvIcu{;9b|S>2 zOdy4$UAvtvO|Yvk#Rk`EMYfCVH`>eBQx8SMGjy<)Q;w3qm4F3{fG2IKmnY61P$;uN zt5i5W|3$oAn()18?!(3FN5U+(7qcw7cZkzX0i$cPM>&qUvP}qwqd{Cdf@jir6C;`I zaN;QeJLKCxP9`B%{@MtYxlk;$LVHr!|F2g@Ar~=S$dd9NPA|_BKS`0sT2y_vWVWOs zHp2e@{F73wY6R?sc?i%TO7|=gi9LIn@Fs~r+~FCv+(AaM|6rK&Ah%pkPcu<6%5(M3 zHKb9g*c`&MritZ2MPm^)O9g^sKB)OibG)oNwjJlb^}BO~dnDSyb~)Em1WX%k+j`8B$vhl_!Bt@h0Z1r6tHpC5zcBa zOxsxl`X}UAn}1PG*RK*syX9;5>+`0DR9ROo+DW2z@%GbkNis7HOk^aC?@dn| zrAWQS?NL_%(yliM+N z6IX3;EuIwE$)w9`_uCDqr7C;_b`Fbr<(;;6(o?L~!vHT& zYd1X#n8pl(?cDH)Klui)f zvi?Yk1oz;X_@3%Nh2Yo5O^u{?<^f*G?kYLG`4f`PI^SbJyKhQ#7py;+le2@phunE` z%wxS2S%4lUF3Sm_O<~EBytWG32a;(!v@PkmB7WsFt%wSJg90Rr9sal;68Kb$j&l}J z4Z!&Q@!kL6&!2$=$~ExqMaHsP$nfPaKt|;XkPK?U-)D5>pfBcI_H6g)A&x+rsM7Hq zQ}+2KrMV?ZZlF0#@UR~)nB9S^y2JgZztv^{xrfHG03Xa(H0kdc-i(7)-s*Gn$MCqH zetj7pLdcO?!u_16prBml^d+fGK#SNGbi zP{MHkMJD@%Igxsr5v8*J4I%WALffFIu#1Pr2r+{64CwvUu8UxiK|YO^E(ytUl6Hnw z_FNCfp~CJKO1XkI4=~JvP>h~PJWv7L6cA6zFV|^G$W*>W7CNpre7#-AB37HBFq7Kew1B>~N-Ji%7Y7gf|+nsk4tT+a2qtFlpBm;KytgoXi>Qj6b zhWo)4Fo+?1mJSVG&7P(Y-H8ZjubkJ`zNk|2a0wEa~kzro#M^GraI zn6raz_PMswUATc@nX+E%hw2_AX8Cuh*%twaZ+5}%xpm!At>c~Rk6^JkhiyA-%Jgrq zJM0ZR&T<5&+!~!WtQJv73${9(4|c}WjZsndxP-c~drgk3?xsakG;i)pE^m=}MG+03q*F_F(} zU{i(<@ppXz*`@fCjhFlumA_?az{I)xrmfUKvSSgx`$N7)^HzVEAP9&%&3g9d*4^qS z)gGxSg#eA)mfw=b=Q07+0Fw z4@)rN>C*EqCKA)B-}Dxp82J~gzO6lou+d^-)KS_>(e44s;zWo#g=60CG6crgzw5YN z0DNZk(-kUiTzSB8suH6EVOoCnYU>M&@K`}j$mE!Fo?OPE1q0KkUB+0n2P$2l4+p8O zfn)SFR-L9p{TZa`I!$Fw2LQ%5kd2W;(zzBCJoFcMm!QtZ=-YJ|CS}2~b2qK7Sl-se zd$G>1dX(2lXggQvxy%aS7^n&mWh47zQz9dlb*U}+ubN{c^kAY2lSl;xRv!6=Q2XOK za_RF%sIL_2rlW7px!>pJ4kONE9^~vCri_J0k*WOao;R8P(R>5DnZIBb`0R=|fE1vA zHCX1!mD9g%IGv3!Y`WHH3j2jYRXc}? zPQSV(^}do*6EplLe0MzZ-PM`*=`6#uS(p0LdT=}UCzynv+%eszujv0xhk2X&>*`WP z|CV_Yj)1wlVp5?uCq()e0l8k3#MA)9&-szSG^Ve0s6nyJJ4rm|>D;D6Vsu)h`~vJr znqROV>*E9JlFGOirEJEw*4U(;~$Xgm)D? z?PPUzeDr+aaX)nAR2W*j-(ae}FGr%sfA_7>SZ@hS0Tc$% zuE6^gg}W<}hysIP#^2zYvJJ&O>mRse(S=VTgsLG4N6%8rOKmvrq#ekPIEt z*{(r&_zLAhSQoHq^IA-Bsvh=*kN)U!?)qMDS?o8jAUk;5WWVkoA;f{-!U_RdQ**Ha zW{q2GQE2Op0CHxmqowApJOeNyM@c-uyBFLx{PbcvB)N%&e8;apu}v7FHsG&@(Qz;O zP79IjT#C#ponQoT5U|z8Td1*6i*W!XBf5r#FbcT}qs=rM9l3L_3~ zLzXdEnDk#59+4{}(4D(WPfiE;3!cF%#toJSnT$uk!Y_6|+ByHnN(JiMwDc;>xa0AFPb}&P{Q;a-DwMS?}0v>r>@tF1IjKX%Kj(?lyPA!uRkg ztp_Yvs$5iXLRAA}@!5d=NP;7WxWhiBD0?*}UUTUlNY_Hm(|En`=I8LbjH?j##3EEg z4+UtAHA03H_;LJ202iPodQR5;C^bWxYxP0?PHX6nX%4t%i>>kqJ`ZTz$Zqm7(QuG-jwV0@D}~tZY3l|rvqMT8 zy7EOYo^HirKg%2T;R<`5sjMj3Cbt9m94HrXRIlvG-y&b2?g*tIJb;oRRm$&hw@(Ou zQGSzXBR2=Nn!(9UM#;;+|9Xgj%%qE!%I$P3D8SQ?c@#s8N`n?rC z1uFhfj5eHk*^dS8pOz*=xh_2qqdDGBP-I72lY*H@c3L8eD;++YsqOB;OkFV8$BaVJ zYLKlP*{_YC&U=63h#|ESfn}?h2;$5S1YiZH8e!gH^6rQeP@V&4qgvhJE{0i2uhe3t zcQr5%bo?L)4O>n>17|A3CBX8gEQgj6ISDm#>?KH>QC6MW(6;aGs(ujp4i8g}0PsIv3f zCMQtD1SQF(d!i;PI&tJ3z$I@2R;k2}xKzSaxg}h(_}^=f#}UGnlEOdxuvlNfp%n?j z>;zxgyN02yJE9R*Xi|3XIIgbi$2^c*2OgF{bVI7T4@Mz!La0fD1TgO4zlV@sCX6R% z+~tc}akw5mZMcxG={NBbq&D%Py(rC4 ze*3hJf|MiYj^qq~=qfO}6yRk)#?MQ8D;dLt=B?Hs`y!cxc_IC7Vux~EW}5H9?ml(& zb&H$JCeovM9Eu7-fn5)7lf%My2r8&}{8G&=M)>g>{(Zjv3yb~jkF*LRU@k zT!<~c^24HoYzw!@FjPk;G)cwlx_qcS&7e4&H7|cc_KJ$JXT>`!O(UWQzAhSJZYu!z z+_P9)k5ANyX4E(fnWR?%@4j~&_B1q{JNfzFAHY)k`2>O^R#_7Li^a;{zEzS&K7K|( zsSJVwS;UWb=)C`Q7OX<%x8E)Lvlh*7NZXCu8)2|>#%ZLHPZo5FtS@)PV^lHWB>HV# z9+}_I0mO_dC;hc3h$)tYa-nxZu!HkOpzGUJUwxHn4h{NwgnRS$tUVl? zPULt#CZ;=qZRsd(!UC2~4M*(~1!@ZIHtYedU|B3Jx%`*@6fORJ93G1~KZnKidC*1b zY6+k-RBn!xPu<*H%^ff?xR%tvrYGFfF<;_Di;7O|iIYOp;XF$^%z+lX#^>1cf^rgf z*0S)-%1^v8WHS)3Adl=(q%l$LAoN$E`LvtVUnXqly6oeRtkY&^o-fgpptYIrRoR0A zoGIa`F6Q{vXUwt^*+KQ5pHRT0p|NIR18Q3`M)?nZ@S&7PZ9zezg{-iP>1?U+qSY27 zd^c(;SA-75#7jOE3C%s8|Sg-=VYPo9$?S7^AkU7my zuW|yMCmtLO>)ikn&ixpOqKR!YG1=*GWB=7>aT%1mHax8-fm^#NiRF@IUSLQjo+uAb znF)ynWw7{(+i}j1U&G|Ud$`MAr8bYXvO}{*9I>F2941op9wbtLc#VS53vc_hI?0e% zMS_P!A=rA|4x4ukPLW6r9D+6Itg_GAcuuuSrcHK28m#W2vvA+&!6zI3%B?MA#dg3l zK;8NN4-#F*%Dm@3aYjR$@rvfF@5A9zwiGC-grwePmef?eNlwI1(~9k@2tZ;OT7088 zzl`UE=K^mf)B$UZoe&{ExchQ25%Hd`$(WuK;4S?RKBRLMgP$UT{{#07yoOfFWJ`z! zfx`AiB&gH)7U2CD{tgHmRP&BbbvzEK!JPxZ<{?WdTT>+|;5>dOiFvb_QVbca$;N@r zUI(=>xS3$rhB6O_5q0Vb#~gw8y85PV&~D z+cD@QH3d($;NEiTVdvfIJ@%w7TJz!71r519V4$(J!{^#)V|s*HBqPc!A1b<5Fn*Iv ziCYfaVnDuvb$!~w%IT??nT6M1u(6OI3!TfhSrzYjljc@trpMxtQ^=J;L*n16cZPG; zX9}ohc(zFYNkrwYtUClMsm>lPJKqz6u4K;;utikIUVLbNl}_pLx5||LP!CKctpC#I zIdQ-nTy^#;UytURUsR!*EKmVHt=xcrv$ggRj?6hnq9@{{%R=5WIrGyJH4J%epFi=Q zF8~ev&R|6R8j((A(#h%41?0*hMJ=hQ)*!)pCgWBS1)eK_!yW3)ZbR-FOYabEa{(T6vh8K5XY%Il-{0S-?=b&yh9RYBeU%UVF-%#uY z*uR|&f1oNV2sB*KOmN1gG(x=A-w|$st0N#TUQASeB(Q%^js+On+Oa5OAWc!Ch z=YXM!v5SFaxn3CCez}Phah+886Rn8mcsv!Q^e1o3yCJtG%7z(M7=D_#tCSoIE1Jhf z?T@O#EXjBBg4MeiaIh=QxlCNZEW$K~!nK$5=Uklcbl}h;+ym1}hss~3YEFyo0jBm> znA=Pyr`)oH@(d?G%o<4!D321F#F2^M+qo4vn^n??lFv)PbRX9QF^J#@e3y5WyIY`} zZyr~ws3?{Ds6=FrL+UA7$yTTROyK5jCsX>#?iRGi@8>J6urJ|;Hp!x4ZI31ku;QHc zzEcCc-mox-mFXle+X)8nZ9m45mk zo+f&o*pf&@`R*A0Pv8t`B<)f#;*&tr_Hy{x$oV|f(VQ++!;IH{AN!PvLD+HI*Z|+s z*{z-)SVs+YOPcUdS@xS4`85JpMS6soK|b0k$dAGUX&(JSbBx?kJ1t}ygJtAnaDP%! zkJZpvn!n;2txPmuC-P-^W&ZCAiHKPjl%!qbGO+aKKOPXe%r%>MF5cB|v&_tq) zSa{Xwcfkq5`NUntPrmXLx7z@_HV^#8%)6CLxTT1JKE1EQhEo+h7xGT-S zV(}VdfolIdI&p6jcY=Xunm%{!6d3;YP!MDfIX@O?4LMG*fM=%C+FJWJ_@SSj1AUpL zeq3F+uG^haZF{x!9sMI8qg z?56?9jFVHU_wwxjY^+%1TQ8q)%%^b)2-5yiMxjOh8Gn4UM$og1J^{2*cZaV{&k+zdbWWGCwpAP^jWF6WhvCxNCA9U<35 zu^@JyV*G82uCt?Rl3tpxuA#PP#9vQ`u$b_RjplIU42+A*SjQ(%?DEX(Okg7D$N6-~ z6R;83i;~Ez@P%5is8#t`z`o`6FuTVfb%)el@lr+g7!N}^Y?xy_IaF3_)NK`5S&NY& zF0OMq>1AcwWJmDJe&$eF&;%&Ow6dCI&8=S=biwka%r73D5WojVf|JT3BILh+B^kfljY#p*Masrfr;l~@oNE(HpQLU z$5nSoCy3oW)L^OucvhRnHzc+Ry7$|{=B{^%r?`6BFxo=&a|F&Q`6)g3|9b2fDrFyj zpgZsy;TB-C4`r_*UtZhnB>JbXAfL4qjcoadOxLm29ni4eD(c5cUIgI zA=A+CGx!xumph$u6W54`^rk}4Wg9EVTK?L9!p*-a81HrqM-=_0D7x#-NTIwgh^BN4 zsmK@<6>9v;Lp#!599CwMx*!L<ljv%d1S;gO*&41(*C^c0y_i511^J;&BcEsI0PtxV% zy~EsjOjoS&QSml-9Fn8>#4mdsOp=|GqF3Z3YT`&EqMa>cHa^&Mb*j6TJhHL((oG7R+IR~ z3Y%7e9Ikud^XLfW`;3?nu(48x{F_IO&spN%x4d9T)IR* zIdCgxMPp7G^OC#^$Ce`Fi9oBm{)1-WNjZnkDa>=;N6vGP1@mKhk$c4yF>#cFVJNqo~R1PA{JXVA}_HKk*#wL6tQ!ubfPY zmacaf2>g8X+lH-`03H(^rpUttwLv+-x;APyyvuOCRUrR2 zF9+%!Q&rXnnvKoWO9xucd!(wFo#Wh7^pFYN(Aq&ocZA%T*r6he7bFf;J*262FA$-} zEr_?Qj1LQkq!XM^&eAsnmap(Bb<1Rq8WRSuXBqlOKgar3GA?lNedBi>bQEdC|88yL z#@Z%BJXo4K8mmqa=cNQr0aq13dga z#~|#^;=F$)g!bJ36=Q=MY8uf4R1R-v4sTvcvOVPscR91)Vk$0?s=GtY{(1xt=abZ} z-?yYFt{FVT8mEJpv`eGXi$aNBA))Sa&k8vUGU=zYbLGk+uvE95)%(;0z^rfi)#QyK zWicCxx=4P7!%}DT7Y3t}KrwcT%|%zRIP?$9jQ~lQTNZD*00LX-@&W6b16p!WsE%O6 zOV)-i64@c+sfoL3o9{5!1}~2PaYu5j^s`^{);{ihmY<{fg!Y~vmhfZIOacfx_^HUH z7L3u>J7q;cAx+3^RTC7j9h*~j9Hu+3#MXnAWQs2T5`n4o@7QTST+Z!rasaXxg}iHx z>2*&$3mE&M&rLq~e{Qyzp`C^v->B#xBELMS;L&?hX;Ff$N-LkEz0}}oq>Zd9Ji`^!SWyMZ9;m{bzl>9EAe7ORww5)qq z_AvLlOpg1H1RoP11(dAH%60<>bR7M%?pWT^-Jf zQD19x-!ADJ((ZhCpmSNa#rmSYv9_+HJ<0l-XTUb@QdchV`F4EYvlP=K%VmsRw_3{q zC8TvsY*4vvhXX9M+;6xL*5guGT5ietEQ0miA?18M%}eY<5ET((n!0pd==O9+MS$4= z)_ayc-Y;4m7*-w|M&XA4$e4;MlLx-(Al+EuX*{>p9B+dI)oz6cPo3kt2cbEcv6l%~ z6s1dsl@K=ZzR;^$5#D$KWX;O%l$_L8SyT=C4-(p(|Fo)1dpo?F$*`CRO8Y|)AU(Z# zh}Xw-7-a&}r^r3HdGWeL%x!=eWqG4^SoeRGEK!Q+kcw4SuGI0fB}tS8Ro*XxWZ|Bm zt5sfQk&HFwDFHp7U2v`n%AaIor)4B=&dYyz?)}4vPPv9G(QX|`2yTaIV66cPTuOjv z9;%m&$n;saDRX=vXI+;@N0Z9*%;TxF;%1SXG#Q3jN=`ZK)wKM&QDQjr`C3boKuhxLm%Lqroxpk9##R0@;30*8_5A(%#w)=5$klI~e zigsbUy`jV*%xPj(8aZGBo)cXGmhJGdPC(@^B#r*qG(EqyidAMPFhtu|mC0}X$#U2! z49WaZ@-8GkXc-jo_)|w>Q&|1dEAKs4QF-On^QWCi^9Mkf?dFZNQ8D^yVZ&-lX|^w+ zq6L`DoDt83%SKXH-LEOnj*H<7j0nTd*8+n7eUuj~oDz5mD6ru$oeDU*|4QM1A#N6n43*d}Mjn z4owpEd@3VSd?Ua@s)So3f*7?Blh2ZiEq0{kMGtCpQ(=cY`halKO}S>z`JWM4{xu^S zD*EdtxGI*#pac%XUj*o=c;iF+qF#pK4mU4C-pN3fF`x_0NJZZKo4N`6ay1GzTf{{p zWF$o?US~M|RY($t^=gVx8?N0?Zh@Y=+X1s-46oprhxvUvvjbs4G+rw9HnOv*rTsjh zW${MsB{&?%t7P7O)(`Un@LKS^`wMp=Q-@BKKmO|j+@BR(+|H&q zk5&OXf)CO9`z=xn8yxRadEzc|cM-Um&Q)UGXl6x8c|3mBlv()E&3P-M1$%Y+TfEjg z5_<7W9KjLQY>)5=Y%@ZH%+a??_zt&0PUc- z*k$3nf{eEcGm>0Ggwkw(1a+0n(OBMU`Zgea<%n}oST|<9ZF*NTA`Uc@#~{s&%y%E4 zHEJWZ(_h?KYpoHF=-i$BG!DHIonh(|FeEzSu1xT6^dvH8FvDbx$gk$5DYdP9-LoVm zIH(Pp5FWv#7qhE+_22%ph1${Bnv36*LSKTc(EwrZ2G_4u+aT&&?QFW4{ZVBmJ(jE` z*-HIl9G0im7w@c4Q=3!hlF>!>g%k{|?O+xpqc63&9A(g&noAbgq{%<HA1V4 zIeI<0Co7u5*{k@yg{_=3)1vidlV}xsfp=t*nNq%Qk3>9mZ6=4es<$_=YQz6@NWn^=V@6-gD}@=4SG^GJ8VbO#SKEq89w(@XFA@Uw*pV0?n`Q9CMCX z*j0i{3iem9-r_%@4or$(z=PG5>8CoY6y zV{gG@8609;8Deumk<uv5ev~ARhi7 z8{smCTqz#T$SW#c62Xu|#-~u^8o%d)NiWtgB{o$?3Sr-@i=MWdJ-id|W1iBCt_s6DQ;3A-|CUdf<>@9Z@cb+ zMJdU@W7l;JvU`taAD_eE^aR?%!t8horxE_4e=O}tqYUN?nGe2B?7e4v#M09O1FfzQ z0JVA;xWgYg+&cPgp#27C6Nbxx-vn}b$oVo~3XYDryG<+m@UuwLq-@Y~^k)HhLd2f^ zl~MtrtHijWl$OqEV`Yx}Y#+nd5Y^bP8t-1SldwB~qie+D92x^P2O?P7QKI2f`W6p3J<|&n3I27&F3jzW;w{wFS|BGGPed&HYtz8K zqM{;OgUS7K<=GYV5ySmo`ig{-y3u$gIDnJDJw}`*+wmfV8Fm7@;)ix=p~`7y%lmL9 z{zec(QJ&G>^HwBnK)Dpiv&I>8I>%XV5!ML-H5Z0EVxaj-Q~>v2CoMGFGxhg%$Xn@6 zK&&g?)JuAbY9-lp_^GA2R*7xRD$+9|=nR+Nc;<4fw!Zihe-5U@j^vdtjXDwlN7#8b z&HuNn$IBue6S&u3e2Vu%v%Ti=Se+1C#Guk>;zfqrX*MKxwV2kMafGqL#W9v4@8+Hf z?uscspak_QkJJsK9MZL$4N^By?;mbix}A(5HQM;;h$(ysrU$N4W2rDWtnb45V3s2D zF5=1lOBZ+l(FMQ5Aei0jyj5{Rotl8UZ0~2VU#QONz1n^`*=U1R74J}^Xaq%dy;hU+ zED}q8QN(v33f8=k{!!A$eI20aNO7*B)1^3ZmlSRy;H;p(7LPO<#hE7!AJ|+p&1N_7 zyD*>V@<2In3wqP<>vr_bUBOd6#T`gmP?$i@-f!6~d6lFU4rYN9(N2#8a31L#zv|^P z;5E!Lz@}uV;uY?XQoc0cjr9dh0SEJX?D}TjRhc&C?=KZRz$BzC)Kd0c26b1pz*2XB zO!TEIv;r_-B-n$vWpt%eCib9+TSd@c<;|S7A3GIb z)>LvUiGB?F@DGKye}U9-Pat=}8n&A?@<3F%=PR!$)$~_baM7Wl>o|bSpJ6#?bPi}e1T1qyl*PR94b?lwQHBj@?^CscOfEh zt>sLkBB4kqXWRNTTmk*%>G7|0@kg(HjcroW}QD zLdv1^*J4c@5?Q9MZ*;FF^~kK>(bb5mGH-QFY$WTT|1^ZW+V`c%6h}&v>%}>Dz(hs% zW&sCp^iQ5m6xZIq$3%ZtAp#cMBq~4bk<#)2Nm68}P}$OQ9O)~`a0Zv$Oh!nd_gT<4 z*6CTPEOIA@kEE-ifH&_GnPZ#lKG1_96j0x`ih=|SAB7}qYx@!0#vK9Qzr%h!Da??1 zM4}RVaJ%QZmDo37v6yl{8_bNp*$fuLa89?QeO}jvcs_cTWK5zdJq5+Vrv%UTC*e!b z+`o@7#@ZJ;H)UBg&|fo!F|7oglo}1qFG&&v-ASmof=cnKKKO+&{*ggSHMO$w-m2l= zBt#RKT8&s`$cb}qBQrb`Uwp!nPV4SR1gy#BYq--6hA%+$hA-W9$glyN8;=0q8-QOn zz0e@tR7+f^pP^9tc0ZBDAYO2SWL>Ao)!O=o`b*Lq{X7{Dqnlx%$)TV0@PVIZObJy0 z??$}S4{6t9&_tRLK_&45lV2E*XJrw)0}d^niKIXIV{YX3FzQwJ!K>3ug0MSMQ0|+{CCyqOqzGup3m5iL0x4_DMO5n3~D&q8fC7C7!@sP#ly9LU7H5`n>tiv-^#{rf#XW85&?LxG+V2<9}(m zH&&GK3;2Ge8zMsL`WnF(iW^OJ#utzyS1R5{%!{C3AKX zaa%`Y(;4)W;W;dR99q>R;aSiotpGlpZ7m+r{hiWLgm0R&w~tGi=ad#p)6jO_o4uc8 zeOQFqA6{_@_SXRu10Ju@z9YQ=(nC~# zZs=D*u|tXmgJ}!w#)<9H+Y&Mg#6oFUNB{8*HN$7-=-Vg&1X8A-wH7W(_+SIBAmDr% z5-;46L%yR)Qsb^#{dJkCf_~fOZtz-kH1|%U-Ld~hRt9)OC(^Xl^m}E@#)+mXL1M(} z__|R(-hUWsFRRSps`CLR1xN=F9^xAPq^)OJ>=33OGEyxvk5kir>7=r%idz{V*n&ov zX96``ukYQhdpx@E%UrjTDnte7VPVg`G$fpdCEiO;WXDo zbNZ@1DY>=VCJBAG@=i+7*OI+az-&weWI2%~XxaKkmk%6}w8jS`$tDD;tao-*Lhu&{ znfkEWXf~oPRJF<5f39RJij-COO@f;F4RHUhkB zNk9_RU8>?>th`7tiGbjbpxKU1w~pv1SoIb&qXfmePmpnL0c|2!oTpgi4z}YBC2HI_ zOb>F^Qp#wGmy(uNzkhiFn-GRI5?z*Lj8ZfEO|70A7jdAuT^n+?>7&ZYjtT{bJ{Jjw|)u2!M4^q{^b;Pe;n-fw&5xNOTk9!8Y*G6fy<>_U=09 zXCzD=ACg-q^QkAzm4(O}7hjSub%qxVo{g{4a@P_9ZZEg7n9*|WN^J@8@@%B z-pUWxJB!WdRTzcjxyMN2#9uJzOK(z z6sa-u*+o$!ReUhllOq-gVpqvivOiK$OTdeGjcjo7cD9q<2zWQ58j2jSrIG@JXK)3; z4M#+^Ui+c4C=4N?cfAy(9@9V}M^q5ABe?c(^JQq!58l(aB4~5x@N|5uQqWKmD&5Bq zphHQ16edbi8Hh+;{tS%KTq+|6B$(b28>VqK#|bU5!p0|wPfI0D!qtuKT}TY}mej%@ zgLrH(aFfxRa2$AaH>~IT#cRQisr6xH@c)XYE#|vg;VD>h@_TkHB);rS9VWUTtpS>BypxKOT+4tR!b zBO(*l5lME=D0bJV&QMPiuvc5CQB2?1IN1wQN6iAZZC(}$qnkt_p<|6Jy~b#@?irw z@hXMd3(ofU=1C45>mARP5c%e-#+c#pe*xkf$>H&{qdAtvu4}#+`re2l_^K|0JK>bv z(PmrPcz$3LbhBu$k7N~`C%)#(&M`qGiJq(M`)0AFbRaEh-z3|8B3E*Br{(tTI+LO< zMJqZ>zFir}GnM@D8C|ePtTDNw9ZgHcnS{5kiu5iG+!A?SNiNk9G>2S;1e@2ghKTFx ztV{ z2DzI=G|}L>c83L_qxMsF#oR#D+ruHT_vK|XL{&pia~%2zn!n7(sydBg4^+H)Y~g<= zaMC@41_=pPH}9WbipO1ZrLkcTpT?GaLBQurR`;1PrC^>1isU`DiN5lj1Xcl|j!$J3f>!Lo z039)O?3iYaOW-$;bleRGau*}<&%`n%f?~PdrG84#>!Ao zaj+~k&mz&hPK7Cn9|1J^Bq12f$M|mWdi%k9R|0eQQWryNJkysz?@ogBta0D2quG3@ zL&{mD4K~H_>iPB~pFDe3S_Kc|TW?JO7Ce#CR^%L>+#dH-l#s~G18oI`5gd&9VT(}s z0Mr0gs%#2Cn?E1ACfhVk9erqtCF&72p=V2g!h8?va9 zcAVfA`ixD;x%Y_0+tlewhc$o0q?2J|k5vzf@*z-t(k>mr+ge)*b0-P{I#Z&QiR!7t zv(Q7SkJPaoyo&NwlfZ~2$gT@uozzGHKkkyXg&~q~-E;_66w#e455}zq%Z|)M9gdJ2 z@02Gs4Z0E?<9nUIq*4WQ(ipoQi^|k_8Iy$Z#83!dh|dNx?X1q$5ebiltss=P9&m3F z;$^oP5wq8y(-@(?Q8}wYr-iKhSB!G>Jro#GAUyx(GQ;seLZO`6sF0D!^e6q2cn{mX z37+r9rcoC{85v<X5cW* z7U3?GX`>dDifu-M!M#t@vXhRmx&l5*R?gh&{Xd)Xx?(bg!Y+$~mY%Pt-+Z+tecps_ zy#-zxFtP$aqZ%Eab_Hc16D{jeQ(@dFg_Wz*>oND~3PTFcx}n?w@>+R=($3Da-n*>o zfnwrwsWbA^g~h_=&&sLsq(2j82r$#zXux1vfgZ?=hb)95 zj9cx>I{3piRnu|Mg&nto$v&g(zvj(Gj7xrk7DFc=FNY)d?dI|tw@~6a?m12L+O0?> zm^KaU8_+@)Q7QlzO%{R?ZZSMAkpRpc8kGfRKR}KInpNr?5jsX)2w^gf=+&e&rNy5&PY$y4%Cnu23y=QsC?jb{z>5A>U9MfLds)0&3gg5%PCCv(iHOI z!OoNgm%V{qVj>#p9jaJMsA*lH^0FK;M1&*k`_tC2N_}%(8~G!l&1bv;BRShec#2xe zYe#LZqEBw=0>V=n4uL6bY97XRwl$YEl$K%y_Gq}tl&qd^&e5JX%|8iQd0QK`MaeWB zL3g!JOwd9fuq}a8ayruHpsPiE1t$)53pcT6F3zVoRBxUuWEH~F>1R^Ko`yLH)KB1EuOVuMe-e$)5 zYz8RR_O_^HSDOkiK{<4mWA`l~fdjhm623H0xtT07`dNM+{wv-%gfdUbBa`)^Kwvcy zCC4#Y1>^oY9=@=GU3O0W6=$xfW)anhXCJ1In2PF@U-DIAde;HM?clLh&XzB~qxiMj z7%6U3_gVv-LLP6bWSUlhJ(AX!_Jnzs$i42g#;Js_Egn$$uEe-JegSN+mfc_U3@}5q#m^V$(*Q*5!c^B%udB@5>+V z1hu3aT$z#&gymkUmztDZ;6=B-WAD)&xB?d}02U}=tpKWB{za>oFHT+8NQB%yCIeF* zP?~a)Y0u^7jQGfbs@&JfA!KSxEZw0OTo6?25*_&=eQWP%?(IlQ)!yo9rWfIPr{N}j1CTYu=AgU&vk-Gy3cbvO5-u0%$0VK& z97ywjvcpogLs9NR1cJp)W!*|MCyZwOmvRO<9~X+BasId<^FjVa=+WE^dM#5uC1Sgn7N&U;OpdjDSQDG zt@QI(neXX${q51p){!{Y?Bjk}=z_Al6#;$h6_$u!yk`k7f+kq&$-?t@4Lx2kH>M&m zUSg7Bk?kgxuM0!QK)Bk11H~<&KL%@j6?aQkbNAD#qKL&lwuKc0oxXr1wYlgW1i4rl zK`YiNn3I)ra$7gayj_Ny43zd#QXlVm{R+1B90x%Oa}bQ%5_hyIb|@L+5w}B{q=rU-l7Dwykkp zm*d{#z@9r}ZEDQH_BCOy;|nfkWK+ad!Lw2>fwe}6_}(qAW_+aqd?vpOraHaU?X^#% zjH(gGb%iLxxvRSPR7YsY9${=Qhn^z72-fcd<$nk_l7wjv6gVMdVI*^nvU2oaF9z zLs?78B4(%W-U(>tg|aw~-j5eU4UN}XYtzm3~pBQgHCogJ^$j|0QXHf`t@NXttv0XP5C zf=$-@q7Oh2@_-&q_jgWBpXK$%LW2%{RVy3~b`=>0s`1@WFx%Gm8aRK~uyAA)9B39SGC9E9=+DHpy7Rg0*h__{##6X4C zYfG#X7T%VCLZpDf!n2ymK!(qEqTTprv=!&nZkX3PS&rJexG)O9sA`;5^kEm5W(AG0 zug8_SQQKWBkb1;Vx-lBZE}n%_d%6~_EUqk+e`4F=6sSupHuZD6^2OwjCCoZE*q5~S z=HJq)Ox`U7Se6QV9s#VlvilE$KsUeyBg4?#Af`5PM-5{%mzp|o)(d!`kNFBjkFMP| z?4Ni^!^8Mk`Z2{_?b{PnZ*e5*jA9u-Ai3eOyePQ(F}A@7)2j30 zAQcGWFi$b;UxiPg{6bEw?kmiOHx~5ch2m*W859zyVjCBm%gLHL>IfMUAQkj$>!&IA zN#lsE=nnV^w!+r<`|*S-GL!PGkoY=XI=8o7SJ{EVbN%1ZClidn7!As`jHYUAUd7g3 zy-wY*n0CuOs|&f&QKhU0emn&Ms=E9$1bPWrRn5)W#{25nyHagYfGi9(6)sjG2@!aJ z?SBr77%6xMPaWaH$j%}Wj5xWG2xfb{|pq*Qk0bnae_&BL(EUp+8-wK&ap&ze|uG&clq(Yw{ zJ}Ki2RHCUlmIr@7Zi*SfM*Ogld8nlkzRQ+1uwmj5xLahJY7ndoE_17fIaY6~55Lr5 z`dhU}&lM=S`QVqy`MJy!5?r9&2PanJ0T~rHITF$8gcZOdU|H+^LrPH-Xf#?^i;lyo=;ip z6Co%@j1u*gIh`6GrV_U)8tC<0Z5pKY#do~2br*Gjw!P>RaD}+)HqbBs7Voe6 ztro{FiGV5-h3+UwlGxJ-<_UzK_mEE``0y-+tE@f9#+kM?ZBwA9o5T~Uw`(bwr5>)U z%On;(6xEAI558CQlkIT0EtO!9(2VU5^Y-=n)4Dbo3Qw&VXu>>JmrAz+U`{xS6ocB9 z-_Bthv;egkKC{9y)$LLI6K5i=a7ZlpO7H%D*M-cE`QVIVX}0mmBXtH0UV55AJ$Dv7 zA;JZTZ2GxAUXS))`$XSIm1#<)sQG}I4Jvh`J(DZAhf@eyCChs}mBvazO(62m?HdW&Hco5fygx!VXOP1TbeQpeiEssk=BM|$-@yZv)1d_y>A~jAL(|Vj^mXgH zF%<;k%R-twZY%}rnlfL4EL`xwZrsJvjpRc;+u4|`T#TM_;Jk!75>ZCTKwd6Rfk}vV z6(5+e8e9r@OS6gI@zYBxXQ!meF0n~0`O~1bPDdp|yN81@ zZ-7@4q<Rx8gG2{C$ zY9#ACvF2xvRGUaR_ zZ!Ht-PQ!1Xspde62PCd5z4{YMfyS7yA{Q1~pO+&WgS0|V)g5;->Pw);P?R0Xw+pZz!PcQh;Jh5ua{y(NU4Wa(` zra+Tgms%p*L^pugl|DC?XU+eBL{`}}ZQ?&~A^sM`WptSNei}o!2B|%8BO}k73ZIi$}87Rv*})3Ytf7>27W`fhquR!0%YE0?KpEmSJeRMCCQtILtMUx1M}j?UhCL-_0|)15hIseQiWk;#~8 z5Q1UzC`m5pq+Zar1P(A2M1Fzggj-^Lg@=SE3B2Mzak4Fos@g2iuVVHv!jBabVcAf6T zfKSuDI+Q4u3d+UyXz>0_3g~3X^k?mu_u1x27lC+>=GPD6)L8uPV5dBYjb#W|iYcPA z9oeHki4%o9;*-LT)P9eiyk$N%mr$yPV0*tY*v7Ge4jD&ITJe9Nm(4uHx}7#PFmJP- zeb{BgaLs4A!AgtEN;6l-(wl-cW1=>5$F+d0vqPVUos6q}q+oS{BxE_lVc7dWbl?<* zZ>)m(9&KptY?*U%Z*x^r{Q7@@9a;t@gdcyV&*ya>8-&UXYvRXJbKN;~!9^!(y``xj zkj}!%Lqa?DKN}AsMW3gn1|6=H%v%|KqJ^C^cvD+ki=ciGLEr1hT${NLrdh#flETZoW_lrqRl_*;-|&AM1etFefl24m(LmnF17O*JPM zb2~`>WVMiAC^VBrRj=)P5@^0FG$Nkhv{*YaKG;(#E1mOP2s8zbar(*`X6?~H&1?cr zVI}b`ntvdwydqy&HA|YipH5ehbqfqyL-2(yzNr;@zASvDufVZ}laih4Kui6BASEQX zdj!KD$Njg|BL1uX3W!9kBNmmlo+6RQfK)(ksVl297e$4?>GplUh}hkS?5Ef$nDL@S zWI{uM%x(lp;!^qYl_zM=Yh*-HYVp(RZLztZ-Zoo#-#h~SQv5(D?F^vhRue8&?)9-n zGv09^8iK&Cg_|L{7@Mbw&E9=lV)z2JK~gyYvR)=Rx1bw*xZPe#9#q#Q%3|$j@V?X^ zIUWl4)I(=tyrnr#pXY#DHOIGOzJ3L`q1_|+gvuBM0RpT(`gmFcDk43zHN2UN&c=t8EF+~< zyAXOZ=1oE40?npb`k!8yR|Pk ztb(jSDTOl@mppU8WyUQi1x{ZGVBpJYJ54F8#G$W08&T@b<)v7vc67;Qa)r;e;n0KN z?~c`U*Dn9gLky0NEa1p0361GXaeuHPP+FQ~Lccp^v8thC>|| z=)v6LcBHDsT>S4(PZ(z5?n5-hT;L}~&?9?S@;;8Hx@2Tht; z$;rq%)WJC-0eQeXA3Gj_o%e>c)?oy(P?XFZd?PdK9{&J70)&9rb4`0=idRa=S*_m$ zDWl`&JNe4A7gjPcq&{2*A)}_Vdxk55`p|~e^Z~KVoJwWS1w3)4US!<=ZcgZPK*11E zYJ5fftlmmEWt}vb#G&g2xU0MBeRMM53o>E|1E-;*M}QJYF$A_)5L^JdvtnalCW+Lc zxin&yW$vl6ao7$OP1=tiUrTZ(NL&SaTug<&#!7ORjs@pMY*$j`UR~be##(q_?$Y)-dfh<9-TPO ze1x&7ZG0-atM^?dC!ZiZ2(-MR=q~9~(iu(cj7dF}!AH)FhAmWNOe z!;ZNk?O1cy;~oB-26!vRs16(cmEySE*9(xhj%2MSGzC1SOL3~&$u!8kp@vtNKTr5f ztNqmunEZzay%~=>{QT!hYz0LY7sWXwvTzK;DtDHnC@;HJxDyuw*K0W+$6$UdKeOwW z$A&Z8EG*9bby+#T6Oe!GyV9=qy7Q1Su?N(s``Bu#=tF_b54!-G$54U^)6{LR;U8c)5~ zBUX!%WdQ-Vrk-i8h~0aaC6h1#v{1)z6$Q~9y4?S4(W3)nHlqASjnFX z0?g~>&~l2tG0n$vj21z(*8R0@I0Y<_2gY2Pv9h9T1_Wc|7r6X$A>{9H)O7fCt}e@Oc?K=DFya5X0=SUykdJ>I{(Fuq54S}O+hCIpe%R>3t-NOU z!4oiUc(asO+8~}TWq{kCtUXP|1RB7qU$%EKm>9vaGF&*Z-f-q5N1#6U=!nuy`^c5p z`(Eta(MyUg7gzm^fxmh@+Fj@LW{3QsN$k7-OmteS8_n+VL3n^};2Mv7_G$_{HHW-%u~VID}6ecA+`$Ka?Zgci)fuuxjMp$JDVz}vrk(ylJDwm#-gHOf7KdIl zw6R8eHg9O|32ope!3KAhV$pDGKUY>zDfegi*JNxpDXK659Zb z(#h}1;R+(r`Pf}>K;w_8!AhvFZ8L~t%!$o{-3L1C z*4P9%NtNpaU;syC!R0g_jIcD*?(WfJQ*y^Px^O)7gE&b5XXMoP^(`6~WN*|i`5NmUe6WW-3oLF zubo0Fjw6|?{Pm7BNs;=)b5m7ERA&o73TP`wBnx%H3%d|Y3wDON6CK6({}g0;sG65P z&$Mm=o1@sM5akqHW4gZatpi>r7(QJ?)FMTl$1Accx*e8|D&V6nzlgxSPZ!41&tAt( zYK^S;@#t6qpKe<2hz95}`c}5AhF}wi5CAF+vF<|y>}#0`67q7>ee*%7lrCm89>X!r zw*aQ8XC#v-t%4hUm=9z3ySXDIJL?#89biJ{{(}&&{ix&Sc8XQM^waK*;Ys#7eaQRumQ4C$B}_fD7QjRlcj>tcc`o2);QC#F1}~sO*tev}bOFwi*H0GTjU^WFSkA5u-5kk;97auLoo0AM#1Ij^rq9 zbzPw+`hxuiq}AHPJvx$Wkw}U;sMIo@j9H4#tL=9cz3NV2nGQ*25aIdQba-|cY#`Oo zDa=}mE_LsgFcfP>Uq(kL3cM;d&pC?|ru?w#Ml2JQksNbf1W4_qsoCzHkKgUune>qW zk+Z%uTw@rw<@65|*TqcRd;R+Ndg%*gZk3A^ z2pgaRf&(N~cjb&RW^s!WAx=cYNs1Ft=#?VC-r(%*(v{B7GJ?2)X*-UX2$pX;TdFKL zQa_+{vy2p5A0F4Zuxdiw2Fy@mA(|B`EkY;gXV4^nFa6~#8nOYXsIzlN!#3aOrGf!JioXM2cmo<>MXcwz3B3Go**SNv2U z33F?TVw?ds>bXq)qBmxwDj$XTqLVj%i6E;^0U(8)jgJA35xB6TU+DCxNni2 zLdNtOh}0H-S(*_<9LX81KT<^zoNQo62hTw=Vbp^r7RXwl(jl~mhhP_P(qrkOQi0pn=J zz(K#2DA+3zf5yhpbSC2n+~A2kA)0ERj?VJyZQ)NJaLg!R5Uh3!vUoZdBrvE+BIH7c z_vvts0L3kW7Cv*F*au~Z5a5}X4Y4Y_n`y~F5o;v%#&9cxKROlyQi#EcX&s`yetCi- z_>J%lZis#o$5^vPRQbgipN^t<JJW!7{3Aa&|qY9Wno z^pT*X_Gy~@=i=w(!n8}XbP=`_8nu>V8>I9PN<$+#c$e?+S@JcvGKJMR_H%)<9;Ix} zFiL*OK`t$cd9Rw?+5*7!SJBRwXKX#`dUu6%FZMO%zH#Wo0$cWprVM`K)Y zcX6yY5Y1TqE$5Q_e)MS_u$wImbPy_87sjvXo;*|Yrv8Tn4TSGBV37#qGSy!2pBeo^ zdBongrQzv^!@vZLUEBUk?NrlyesQV{7t6?iV2464a5jT1)(&cc+L8qW8_Y`kpB~kp1s(9K6ctheE+sG3X39lSm};)Me3g0osxj8hDbK-ZiDmYNEW;B!a$w4jlWk39c{3%b{BS=;d01am+e zC>xi%U|ckC?kbkgu9xKM2lmOY62jzelenw0uzl-Hvhi#2uWS2jq8%CQZ{R`j9O;rP z4|iA&ZGKC5FUdpmLFnBIz_I&W@RE9p;W%GH;S=m@Cy=kQxaIBa7$LTF z#3MVTU6GD9hIO=p@6ny!7DZqxJgvOR^$OGWOV@ zyLMrq3hcB%9!*;Hibw*$Dg#0K80omn*t@9rzJ<{kBJT zTFF&6%9Gfj<;T)k5rr!ST}XLOjg;McNU4}vt1!;?&;x}Iz{)eRdBjE$sgMBYaW z-<&CpMLlP^5V&u|q;FcG>vD0-F|)YoOk&+p*}&QtT232lLV(wzts?s~c!!?oMzmvr zR~?&P&7|UwQJV2ts3l2Rk{QUb>1_aMOTzS~pkCjrRsRd62DVFMJzt63@alDB*9@WQ zcOt?g+c5tw0nZXldZw$S>!`pT(g{+d946`GEQ4*TH#yobFE9(uTu-Jy+7d8zX30H0 zBY`6;_7*f>wvRpD(U+qo-qN{5&8*@!?@Psf11QeNBC2_-jFy9k>&vyY!oGn4B#83C)9)?v|$FgC^t3izzk zWlYMK;L?>>BU)MA2(h0ZU&y*rnA_o9;+7psvTCx3Zrgrx- z=ySZas03hi6>}K4Q#Nb-m2z{uR%y;%OmLCP>*@ymSYvs{O)%^x<ENl>vK1$UkkZ zP4qi~G=AMNKe8QqiZB6H1g`99To#~@_m)I(*F!T^85>SkzaQ;r?fX-RR6>nqwZYS4 z>HKidp9|fyQj3dHtAbcs0=N`gOBQ%4%MB=gU7pX1n(Kgfx@)5{t%2YM*W`n*K2YjI z7Cz-!AjrC}D`YFhzhEuANZtS}Hfgs8T7mS%Awb5aHuWPq^0dML6&qSy>>mdb4GoP7 z0^Ti`QxV?;M$1Jc?;dsO9l|br&sa_be~O5jd^FXAP7oQryI#TR5L=^L-LP@TtgSnp6!0ubCjI?JJ7m08paK|+B+{R;?k+}(uw5yeA*5jk`1lz5*CpIlsv8AFnFNhYf{tW)2}P6=nQUMUGr33L`HC=5m!@B&KmobkZs z?a|F2w-GQ5u|_tn!`1J>ci#qUgFmuisN?UWcjK|mSPLUJK1lqU7{Q0I@t%k20VlM= zulVoW{pdBIWJ8ab<*PJ8d86^d-T&&@Tdkb27O2Qjf;w>h+DZROT!`@#TY|`4?8|Rs|o3=`Ae##jaT ziKYN-x2WtSaKkWd6-F6ifI7>Vo{@0;6P`nHYCUk92iyB*NY1E@;uYdCQ7#M5fV^>t ztO(Jk2b%kvnGuSD(y&YZ?}`r`8Ls;8c051Q0QXe&R0hQ-%Tm=_%H`nK0A-MZ&K*+e z5X#aAfhvuM_52%bp*%R)?8*PAs%jxTt(KL zU_)rk!n?CcS3aqep|!5`K~`Ai?Ok|)iYeEZ2FK*c# zi}*QT1Rotz`3A9~4xV&1?57npx->hS_gbAysJLmYx1)yR3u^tu>!%1b15pJv1ksio zFd!zSZP^@13jf<{?_w@*QNNw9{PRVdkASiwNT!dhmBu(0k@F4NqQ z&SwJ3at|H=T70gC(euy0t)@V*%xy(Pdj4UsWNImPOGADL z^lpGqC-Jv(%$^H-6>a=3aj-r*jWrjY!Wgc4pX1Vfto+0~h>24Tvhn>94+CkwQ;iI) zx*dkrN7P-a{6H!es&()QDR(81bdG2RzSXJp489oIBpSu*U4SngTZCu=`u-wb!r2?B z#Mpmdg@W+`390TnkZ>X^H0F0GwmacB2F*8wD5XStUOk7#5y; z_iJz-G-a@v-$MU{Ws5Jj4%N5XWF>|M((_@e2}^sdfweHGKgH*zP;HC_LXBfNf|+(k zR*HiJlG%}`rb$`CfkO2_VqC;H97%6WUE0d<=RbGMYFYAnY@w#=L{}YTFkFxN$cp@$ z#zVfeQMv+3#j`+svxx{!xies0jwm_f~F&6f@p+VZSg ztNF<&NCZQfpTbI!@Wa8OWI9rozb!Hf?99)B>_m7@G~=Hk*hzgs!28hrr5pQhDI4~Hnbuu{BZgqzX^7PrX|dN-7XDtl{NegrerZuGQvE!dYFkd&?;Pp= z)(_RCluUz-!*_w&URUW@(CeEI^ikM$1Mz->fR^Uc*Ng(4Dw*~GrCc)`^Ja@QPJ}Lg zflq)OT-ae;jj&xOCZUqer=_6ctkexQeS;e*zVdq^nqC(}4nd8Uv+GvL4jo&)-0=H! zAB`rqcmL|ABZ~gyu_***7oHJX6Q4kE0!A?&Rs{a@w;k{23E{-FxvWuJ_IFJiw8ek z8~=d?{^FgmGbl%J@_B`!L|dPC?EJuy;KIvOSqKqVuUFG+BFYNbk1 zszr6M!O1#&uI^l)y^6->v(Ja3uahM@@LnzQ$EhF)4P^SwR7ay&8Nq-z2AfZOua|hv z+OMV;oP7txcob^L}!?E~UyO}JYz&xSfYP-C{|IZd}X#2wKXzD}&3qja6hZZ}A`mHvQaO1Kg zbap+^hFi?EHu7ii$DWT}z-o(y!E(?AK^=>{_8CV`Yae;>F%(CgE*}UkFs=dTEWQ`L zX?z7!Qblnha-KCV%ckbHK*oR02B_u)w-x%9FCrdjD)Rf#+lUi_X+b!dJ91&>x||Tt zQxr3IgI?86+fVO!y`{K&~El5Yp)yhgZ9ztua` zmEp5Wwz5**CuGby5j(FQ2W8^ZQUlJV`wxoZ#4zQbUJTs{%pNLI2T{N2sDo@fSE*`B z=9Esc-^JvTR+o&mzD^>rCNCOSSFJ5RbF#GOsqR7ipxsGH!Fo!hB#rXJDrL<2$&Uuv zrNWKpLFdN-yR-R{@l2e`hh!yl)BX=|kL6;+ecc>a4#S&baiI~2CCATjPx*6rELBLc zWU;VJYxsyFApz9UB)>s%i3Ve4CD?xl zoFrN*_Xe)Y8Z|0Cd^=5ac)r!M)xKIq;+1w}ZcuE3gLKQ>ayOy(fEOj(x@<388WoH7 zuj+{LVJU_HB~H8^DfIoNIaME9J}$R`qxydQb0-XC%-e=4G@TAuIn3F~x00P1i|W~J9A>_*C}w7lrZZGX z)}o^ZzL62g)an&D7SyNFJ*8n8WvOHzl>Qc{Uum`WX@cnMTk$yFvnf?mE}8m`j078N zmN{aQTzbG9E2V|U=E^YTaKMz^%`xCCLOJf%U@13P1!0qfvknM?ujba2BHuqe_Aw9O zYBLQly2LJQ#T}Uk-Yo86?d!{+&3KMEEiH(wQaNf~d_378pPeyHObj_hT70&%VP|Qp zbrcE}W_}p)MoHM+IvA4`;@qdjq91a3YVRyvM?W_;^e^;sqTOj06T5_RR5a3s+SGEF za?H+jzgIcwn-m~_d}rW@!1PW!(58X6 zvpc@X)y#E^<+osnQ$w{{iLh652O>7Pf^y1(Xk1Q?yjqv(WPn?Ja+86WXk&lxWC?u?{se zJ_f5q)1VNK186a!TJEU=N6dxjTAPeM*37z<;SHD+kSXEe?hNyQ*3j8_xKz00`JPP= zkos><({BGIR39h5Uw2`~tnSRNVEY8JyG+G}7LCB3)*eTH`MrV1<&G?XgN+MA?ip9w zHi>l2B>fkMUjn35n&?MeY0L`iy0U0k>@YA87i{#irs+ZF0RhWfm214O#xVAYex)PY z!^BTYYZuw7n8a(&mZMpXn$~B`)H?^m#g_J9Fd5E$!}pa_;7OR5)k9iiDp>ESQNGZC zoGM<5on=_os0`*q`!&piOTQyd$j4rp13?3DVi3}w!O8dIl!IPHsZxRBT2{rwrPn=$+>V(0WO)`obMF}C zM+#bq*|#zXa)N}0@yX&MC86Qokx%!bI=A$ek|GK1JmfqqDZ-hy;G*2jAoNvc8f{j! zPbC0f@WfKL;yNAXj2s9=8d23%`|PjTxQ9fzgN}QQv2{=e6Gt7k5kZ6E(Oxcbz^C}U zeuQhub`#Fn-Oasfkb3(Jk8wVhwZ?CwUV`)_zOWE=l9Jcu_q>H8NVAbB8* zGMHY5d@NjKu|?R)Cl~8l|eFruNX__mD5?)AvJ-X$gAiB+{IOy@9DW+ zh;f$4VVuZ~v*-E41<$dBr4E!}GQiPKFAO_sS<5Kg^>i!bBcV=&s)sUG>$tE}%Waev1q9=AruwQXUx8QeILgVMx^_+fw zx+DfkI~!)trZ0O?VORYk{ouv^bkuk;blwlE4sp3}1~%l9)jX2If3xX9;C2W5@y4`P z7i@NqC!tx)s+Fm{BRpiqtQQtgzA})$eK$1BH@{oNlzE#{6Y>J?fSbz)emQGcJy+L?W0fvt`GRGKj|+06dhYH=}FiZGvk(Aq0xo zGh62dUoLBK^r8U^t%jDp;E|9v5&77suX!qK_hTCPq%g9@UDDqJ#s*ok&`^#V0MQoT zU3sCKn^H>Eok2CrX~;^xVNMFM;diI0JuB@v+{l=|YC@K!9_X&Y3}vU|Mp-=G^3X-t z2Ur?90k$kpTu-Ip4A~9S8&ezCYu0KZ(P0$b2+DNW7)V6uFr4aYDO>$aQ&e2LXD(=g0w z01E{{5S>0=(N|WRfolgB0NP`=;$Qe_zpa$3!eCSxY!)Gj`9afl+K*uc+W84mRgH5T z{MnBhF83sI6*j}b{J(dQJ!xF1_pb?xTz+7(j zH>~8iRZ!0I_;}IMKbU1xI4u8!=LeEf?fN{k7SI*0kx3T%tF)sGbn*O&)rNUs4U9g5 zKPM~h4HtxX=?J%=`gX>vvw(S!>g5v7UvQXdO2Fz11Rh(7{Is@&{A)6uO9pxJjO=I( zmZPn%t_|3l-TP}E$et_S1tniG$JV2IS@}b@3(f{3Ooz2Msi=Nj zGX~LUDW~&P`2Y@1jEXzXM`V=h+<2LQz@AO8T7n@uCC*zUcMV(*sY9lo9WWm&{Sw+C zOwIZ<%u_;Bf>mwncho_UEDj$I}RZL8_+z3Gv=o2sJqDEeH?57^;u8T+VdW9Kqq1|S*yV4Y- zl(f9hwz-dfkC_LUQ`5UZV6QJ3c9rCd-Kv7e1S2JFK+*TzY3~rHM*8z@ujRxnF8B39 zudTE<`#8I=DaYAw=0HW2yw*V?ASNvP&p2!;U0hYvzi?rcL>?!FHDQH;qoUeP1qCq1 zm{*&=fn%9+k#m9m74&~14Y+hZhKtO00Y+Q9wCjsho+XMjRh3|cZeKo&q8e-WKtk2*;2_s{cTe*J(6$XZ}+2!Q}2nYWOPz%%2l_UGd0L*k6+B3 z#1WZ73K69YG`SRUY6=B5&WkeZL%qh(C|wr4>_b+&GR94#bL@K-K^(@W57ffy+hsx! zRy0T}&pYUm98%OSHbO8So~qY*l)x{?Ac*|Ncep|(d3{Rf&B{|AG0LtJi7WWo#p9s_ z?ik_kMpW;KOiMjMek@OP6wLBz;2 z{ZXO><<5htw_p2VxSM}+6QLco_e_pVfbtke#?1TBjovs4=K8x3ZdS#3ZH2IhWDysD z5WU`WVwgmS)zYSEozvOS8kz|5k$ID7d9J4gNBwU~kHHfsP0l}#T^|eNcpTk1)lwPP zF>P-f@Bi^))FM~V_-nj1&(aTY%s`fNS6R0*Dls>m1-K2aq#>JP(( zu-#r|bz>*sk>M~_5|E_y`y#yi$WB9QlkjgSeJ~WrXrI89dG5)4RLH>5H7Q`kgd>~t zi}**|u2l3H+4}k$3Jy4MiETAN1UQHi`p~%1}X*!aN`}u^n-8rlB%7Xb?)!2y<|-NXFbZdJHnZ z4}efVjN(%yBruHmdx_OU+lZ-b0D49XJ_QBL`&e;Nm(4x<>>%8YZ4nM84|o}To*7?q zip42lSi860t2%v3fJD)ArhVAda-0eor|L77qFnK&93aHv8PL&qj!C_lpC zv>eB47pVeuA~CHWuE^xR1^SfW+c)CBjs2c`WW>tXy)iSG)<`mgQC?#Yiig*Tnd!pC z*3mJ|3MxLRP_g^jOK;g%f?UYPt9z}5$wOMKtx_n|j+edSoQb@g8JcJlpy0l^TKgy2 z4-7lB)e2KVgX=$6vdSIUGWpfSsL_$j=s1Yoxv+M*9qu|g&<$YFW}x?MWWG>7AaV=1LsuKO&qeI9e(N%<47k?J@_6^1H+AqLmK97x#kk?fWXh=^~{nZ`9Hr3<^*fnSVJDc<_X=@9Yi4lB;Bv zHu3BW=Ld8W9kfVj2GFTjN9+e)mT8ULK1EXBo+W-Mky|R7A4L1ai99OWiE=1*3L8$a z{Pl5_%qa(1Z=z!n1B4_bWNDhOypUf87EUL1AAPOpH&2_s-A((rQus)>55j1A$D{N5 z662<%(^5kQ7uo!I^r#InxgUyo3c~FxjoCiAF1#QF)?%xRL}7Fg_YY|_vg`Cb(vqD8 z2n>LeZ+e%E zpH}A0rD9o*o^HsB4X%kQ00W`b$Pg;=@q@@DLB5bTt7nah%owL>5V1 z7v3A}!1f+QAqhwob0*x)29jtL!zvM2rP=JWWkKG!_tgHIJ^|ut2Vy^?g^!e=Ry+eL z{u;T>N#vviasYWihQDMLBemH{<`_7BXs^(?K^Hd$;d@YUDqri3J(qFfYrsDj;nDY^@S$I!14F{t;Y`qU%rvKlx_Z3 z9ESISM8+BI>6MA2LtPBOvt`lP+_kqpiA!Oz4uCPx>FqJUp}%0_BLQ4wPdPCBS{_M< zyc^&vKR^T@*eGYv!gm_HhHDY~^eRK9H#UBO&eS;EaK|$;GnQq+OzjiXl@|r#R%fpY zh`rrgu8DrYq<^=%ZS-JAOOP^<2{LN{L3dH3M}QGTG6a?dzyQ~JG%Ax^upU|9Gq~dQ z-%vR|ym)S~@V}JKgRFjjn}32oBx^(7L5ELS)!Kg#dVwi?YD%e}*|-#R8qnK^qdD!Y z1X=n@&~c7{G&onOq3J``_>hYEX0*i2qbkZn`~gj-4M|Utz#uc9r&Um;-#Db=OW<%X zcT2>cZSPku2;>6`;hZU zQbdSVjTuq3%KqNC=UDwa`vg^>tzWzGQ;>0V3rJ%d$W8GS>7jD9(Qd2l0^3>CYiG_PNo6 z)gNCjaI&opMRE`1hJW*UY|5+pjXutfmBv7PqTe0=^{<0)Q|MD|PW*HR(tW-vl=69v zEb;1YUh$$Nh@pQ;PHDzbJK+n8wu_sQ=-7)ALkfYm}s~sArDlsS)oz$oj<;zcF1_oFo7U zmceuUMge&34!Tzn_ai6g=@|21#C=;}S8r|9|KV&*vV|!*Dq_8u6=iAi#P^m6$2t>m(?fJq5rPFDrZ;hPUDqil_hg{k1Mm;IL|NM5WHmcObN8mr=%Mh^sXF z8saSNV}k$d{l!!|VBQ^6aCFB)t4!lb#o(-4^r^w-Lv|scht@VL-`R9>ON@G84#*6? zHGuui8rr19SlsA`(&lk9K@aT==vatULU24JQJJ?ex+XQ5v1#bx$09omSo5bn;Ce91 z9vjwdYmGQAR>>oaN7RhNd3E`=sN_W^R$V# z2r@>#1Vct>3CK?Y{jMSP{LJrw#D>V)4KPRgU;PTEzD4cSDon16>C)kTQOE^P1 z1ryK>MpRWMLBOx=BW>C0Hf6aacgTJ1$E1Nq`;CopZ;Sv(9&&Fx@UgX{fu9?YY1KP> z-cI|MUej_P@>X+*`&xTMK(hz>_NJ}UT&B-NbWHr}ygj*tGH3_cxyrCvZk!KvH97;d z8C}rd>xcwq^9^_o9DC2iXdK!a8+BR5GKGuYf9g2<48TYR#2qL*&E~rM*_$jbs+7v} z6$YjcaXKGCAD+1NR^;GaI-a&baYOc zwUp|M^Xxvj1O!m~g)t?dd7^d@k4ru>jRP)7Dfs19?d~SM>C^w z8@-y-If`PDcmIMJZTwwtU-SGN;wcTbtoiBVSAm9dB-MfA*zT7G^$UTB3Ni|SA{13( z+B?w?RpOR>&NFz`moa$LZ(eVd`y`!~?`CA~B8BPbpd0*T5$FOfxb?+tU_L z0L;oc_`-B`J|;#S`<_KHm2BPij`!%Be)po~Ug0SYccx;42M=;OsabbaG;PxtatZ!O#mZgVlp*fYokw10G3GT?dMR`cgEJ&y@HjF zxZOp)TU{N&gJHuFHWVU)JWt-j_B3W6%5#v)Oj#mo?(;MM*EJO%Xohe4=A7`g$UqHAz*!Wu$=q3V1*+{e|0k3Y25>TJ}i&#~D1 zv30Xjh?2_8B(dkMI_l2~B`sI24Wm_p$VHmga`VNnwO_^`S?soZ0g;NI+t3Bun1Jp% zCLNwKB2ey3|6SM5qwdj7A|#dJC63U+cFMLLI4ubH-DWctH8oNcg5sZe<+mHyfr^J2 z(UYypLmMMNH$!)A$|`M;lr&z&{RW*gbRgWelIYn~WvS}t@)$0J zQG5ix?V%qP--M622E!cP0Co9~f*yK-yVzra=H5 zfJ~WhJ*Tuwud9&R8bpc9^^-m5 zon%wkYuvkF3uN^bfZ?e$3dYg!A6wHZPE>KmPM^hx#e;pR>+j4@AE%rcM=$u>urPbs zm8f)|fewDJln61?Y(y3g|MLDG??hkUV=xIZ!-EMv?o1@Ue*U(Qr}ezwgd84c+^53X zoqseg#N_kxm2ieJ>?YWhN<%r&%8x2~t*S%7{LP~pSkPEm?X?S`;H0Wan;io$A$wDI z$?SBKt_=N9+=(z^EA_LaUOquKfb5;7x$qYy-POP#9nU1?YviEzO*dT-(W@q3wPRve#D zMI)S?RG7i_?*>_wqlB)%IR`ldZBG6r7{lEmixlaNik*&bYY6mR}JuHTx2{Tr0Nl}QXdZ+li(OU-wJ-JLx{(LrN$r~bnpwLMuOlS zIP-IevzI&ZG$6`pX-cz`PghcrPw15L)EpMfVunGD5Zp4m2*zp(MXd>YhjLt!Fs-^v zmPZ;=!fz7Q1~cBMwwz{j#6@$ZsPw8Ew`vVRJt?2MQ5u$Fdq^jYc4b@&#g zR`$pn4sbIDX_A2UZBHN}f9VRk!}GjHs{77g&%%@yGinGC8Yd4lpO~87I9D0mq2g?2~cxiqDO5+>(UyWq3NoFnizQ{!RbqE6AQh6)`|gj1RNjMg#L?obm9HX9&wvZI zJ20_zm?aQ`&Q8L=3d6}MN$`tgc=tGz9rSnNm?&!L2$^?dVF!>V?b-806#souUrGBu zG>k(y_)HYnRBE(>{UC}-<^M3^Cj;G2>Z|Y(b&`#-`+GT5w0Or2f#0rf!WQ|%2{wk( z>j~1e2oEme3tYNQ9-~;X0*DO|o5E_6FFnt*U!pEtcJ^0Y`95jszP^^x!1F|E*h!v% zmN8kKD^RrrmH>obL(DoT-BJ#+UxC1`jM~hv*{SW4skFj&@I<9Y_=}Q%8CzRu{ zq?OBEIP-OpB0`-n6Gr=>z%%i4>f?p0;O0gFGnU?I1F*o%K^rv}fU zW*Wim0&?K$ur0LCOdVY+{DU0+klJ(zvfs?Rf{GMM|2JX7a7l7zSY?#?FdPW-pKW^RGXAI1Xk*B2 z3BM}Fwsmk{ECWr@?$oK>IKo(MYODbw|DFnLVZ#yZcgc~T+X|gzSy;4M zFOF>{Ixp@8S+m~!Bvzdxx!gG?m9-mN!)aPP(PVT!!WF|A7ux;fqL#yJj((0%;^9x3 z_D@SFZ3eQe!FYA}k`co-&;OJ!J*9% zP*1~qR_qQu^GeuL(pr?ZMu!lzX~W-){fg2u)nFfB^k1gD6oV)Ejzw5HgN4+ zmn$87eAsU|y72t@9GpO4kv9CFL1=9fAM&%e%l*z~t<)_)7v-hB(K`}>z3QaG(bxG! zCYZ?M3_}@7Y=)wEv8;DbMhKT-gM`fhZ$$j7+!HQr&$VwLTvUGX8B>EWBLX)t zYn$~y${%l3Oxl2lNG};W$!NGz>i5}r!<(rFSj8OJFi0;)lLO}0Vzo%OCRt}@d*u2L zYr1yR3xLxIDnzWf5NCY9SUUK|G%v!}I_kh_GbJSqG+D{!MEJX^FDm5OcqMr}$gr7* z)>qT+1@t6cPakj>Q8u`9B7Y^sH;DWnn`#qFf4oO=0`JB{^K&{1toL4%@ z^l}O+GL!-FK-0Zi0KT#@T~@OBuqFJ@hGFXN+k)7@tP7cjfzj48e8uO)`uB4@0Ktr+ z1OW#9*Q-yhjDW!&#JgT9&DVoqLawce6)TFY(&ld8bpsC6BP0B=D`e}Yi3u|J8k*xw zFFCuf%=EVNIZWJXkRA&%=G-rLY<0~Z;oaY?|<_e?;Z zIoX6i>0zawu23;mL_0;v4GP>Yj8 zIc8AVj~K>$K$1?1wxQ|F8t$ug`kd!I-+K*?QlZgRw+WI$E#WWLT36?D%a3LTVs)m% zFD%i+?rCM~3Uw4N^c>bxmv|>dg z4ov~W;ad%t5B6s8#lh4K0w2C+5#*rc^d>xSqUAP_Jdp)4_Is&Q%22>tGnP(-*^s&A zvr#<<8YR4!jEIg@$D21#zu?x_=>VNc-z3bDRwEk(6s^Bz#xWb0(`$o575l3%GWeeW zHtlw2N|m-)rG@;^>t5=*_@Tw6C0@1scMh_qcx)$b9}}@;i06?(eyeYJ$LltqVYD#4d8Gel9nd%GC(jwQfjUw3UHTtI3|jQ6sbViHt21^0awDm=q+d$8j8yWM z@AHxiPK%h9Qpehj6~4?dum14lnJR4n^J83eM--zZZnNo@{o@$PP2Csm2lGXhwCxR| z19`V5`?9*McNzBD?Ehd1G+ROphs|T$5CtMj*B3@hlN^yEAU}Z7XieICve(Q1-yT59Y zwQN<|GosmcuPK&W5UrRu4;kxDYiGj?RJHBhbFZ0Nh*wrHMoSXK!!q>~DL29bBk0ki zzzV8)f=dG60MVbH1h@&)f*A65G{lFYcEGfdkW{n{AOg))c+AvgElXJup_FtRW`@*^ z(Xe+~yh-LvBnP*LL=;fL1z&$|c&<8w&@<-eG!oLt7GpR;81culxV|M$U2c$G1)k0w zf;^F&=t#VScgbY!eKtwIpSg8)lfe+|Q6=Vh`L1k%n!BQAn~iN4(zJIdMEi)h$gi#r z^F$(Hlf=G3I_>zdQgvtEaSox)1P}#FL+2I566G61Z-#BsjwzzLw2F14IGQbt4LJ~| zf$Gz`3f|Ci6^GG6_=knZV4kk6!mBHj(=d6|h;KMdOe)H*Fdx`V$SSl^oZj=oW#pGY zQ$KXjdaQ{6B#JK{l2WRPT0&)5B+SAKhCyVU6+~IUrMoty`0D1x{&7Rd!Hdb=YqDMc zj@h@Aw61xc^e{fzUm=S;VomZWU9Wq}(U~_@w8!jANBi!zt9Y8nNf~`H8fAe{o3X~$S2u3MYLVhmz$XjQ z@r+yfCbD_YW0tG=QY)p{L?ZD`L5$PfO+p8XfuS4FA8s>n|En2Re7SeTwNEkH+w;%A z+=$huj7-C)QkMY>4qu;Hc}jj#7BmmOsUfbt<_Bfd?z|{zI3od8^2I3v9%E{c#Vl`m z9Fc2iWfbVp!mV!0Mow{LB8%>dxgrASrpz_Q;0n!JNPE>KWHE=>Rj3k*8}|{t9M{K$ ziW{^$jrltkT*PvAomg`DY#zrTqM`+H9M!Y3IbYA`ad|3v{O6b8W8v)Ys@Pykv`p5u zdq4PyQMzN%V{xDOYlTo8T~{r)Erf6M^@9hAG@v_UtQMj%e<$yZM6_XT#|9n?Q}^Y+ zYiRcPAnqd$?X*+X!HB~kV72kcvxksO>zNQOxQ)W1I-pj)pDDq2G$X2nUjBRnR{}mk zA_?VIy|W|_$UJxqKpV746{7ERO$e&2G~1k(;aj_}?o*cj{>5ow{vM0C<_&ay3@X>} zS?^{irVG;V@?xbC``Q57R*8kh0^BM`9oeiwqQlN>peNJ}fX)Q$MlGi-UIr!ccLOni zC5N5|v6n<}$dfeT?YQ4nx}dn{PTF%3(qZ=%OMRHPvWbQ`&AC%`^n%-#=}U-{){np8 z#2E81MCECqEUmfnp{*=XK@svqXbh&B_=(KvqAT=VPvCP+S!E<2n6N#fD*PV{Kekvk zH<6%!04|I)9k&&eWVF~zqt*jUPKt8vs-$9qY)RsyE+j*%TaJEH|zDJ4x$uUM#g*&l|CU)G|9S zY=QBO1bKuSXkek@=!Gxa_JlTNy_sw5=NCJ88LA`Aue%-zKJqrc0-7<5-`aDwfUL@_zKg@LEjI zyZeW7KtJJs=ah!FG&Vq6Vkum|v3$WADI0p9P|(J|aS`nlk= zyGL17=DFuT2KxcJyGMfSn6?zDOi59kXXg=9XNMaUdYn{2 zj`Ccbpxyn3s4!55cx9H5tW70di&AYw8bD zUb*THk5=c%BwOfxP^=jsw?t|rh%VB%1JOQio~TWnAqCl`ybaHK>fx0vpyjGa)n&u* zw~lrA{?F#)BGN?|a4{1uTpv`Hw?Cl+9PfEk-Y{w_h8Fa^wz!H0ZU7v{!Wg9XremC< zgUm--3b(vhg{kJ_*pp7>dPSVHPOqkJgu>eJ^3^)u_)Ryxf(;+s`+EF$3^QrwSW}BO z|JhWQQ_()rnPs8%=oW~=!@7R=!tk+#YsvJWpd+Ynog9JP+clj&Ji<_ILu?d2ov~;A zDiMkVG?Mm|+i(NSZj{whosjXZDc|qg`}rc;Wv6mwVqGn3?n1rxJw)TBA5Me-vh=Uw z5Lq+R`unm#g}9#jo6zX`z!Nc>1XKvLcOg#o&-tLz#LvY?@Qa45-+=^v`1g&qpz|g<9eJ%9^Fd#W+y*22Rr}15x?IO5uz84kFk%x*E3bCKC1A;>@g6*@@Ipo{ zETVWyP(3 z9Den^1l}@+&v6khP2GU{QE!hiae z#-Ae)ktEAp>%i3)Ep5%o#$eSnl3)YJC1sLVeEp^T$==YZ%8Ocp9wS4C>);Bm2(;37 zdyTVx$~w3EgdTy8SpD*K&}-R&EEJs5zB0t!u4>d?Y-e0FnvQjtd?=8=69c z0*t}c6`Gvpf>wc>ODcP}i2biIH$+I$@OgbtCGKVZb8L11Be?L<8V@f%a)?Ls5{D?c zv}|kJK!xvtWdD$ZM#H|7$s3)Gi;@}MB*y7JJpTwhIe0YMXg%wpA5II0#klRm459>& zi>kCJg8SZjzL9~b6bP^OYCWl%TGza}2Dfn=( zZ4{i6@U^+bX6L=!L7uCwDe%`0bTKhR;})%}k;1=+Y8ophPSvz0wK<#(fHJ-eGHoCv zN3o+vfDl0Q1eRHV0Mjz$oyhK#8QshrS7A{@k_-?1kmhhZO&+I7VaMgRQpaoh>G9ix zNp5>qA-q(3mw-({OL43CXeF&H<~}9D&=ii_Gp3x#u7964q@@V1(U5HENz+kJvG9L+ zD>E^q7rnQFSX?FGW1Yw$#m<~#$?K&|!OZ8sD=J-INGFv2OlOAt|D!+Un_sg#0O$F# zNkf|~cR{Yjx5zjW7I%+(&d!L%q|zcJec$<}9||*zt0=wyw%luJJ(V%LnOi9Vqr2== zO(WP1>f~Xiy=-S=ndQO1EL)RFL0+;@0v9(jqCa7xXNe!c*}TU=>=FRrlq;|aW_PwP zI9=m@2Qhn#{hpku)u)o@Z=>jjq%L#+{e5XSLf3U`>&HD>!>X+WobWRY5qblVnn25U z0AsSCXC&tEi2$7c)6skkit#u%5CeuG_2k7U`k2bY{-*0G3R)${|1VvINpGM?CmLO)?vnXPFy&PK^@6xHB? z2ue}Gh&gv<<$etf(7bbMT0HSa0=ABvq=)-^53yLEAa)O$2%Y7b*G_2z=Xsa&_=BpS zb#OvXi^Zz{vGQ{vl>|Pw-*k6LDt!OObg1m^mX9CTgqZ5OPD5GiH&jLw)08 zjVW-I>GFGkkRj}1N4yP*QyQXXxHcO0)36VAvys5LY^@d#5Ki*z;q?|r<;}b`Jh2s> zp&j}5MY^~{n%xOwVGO)OGF7ufPi5)LZFtv8R7o5w9%PvP4cQ3)W2h0hAgfD|bO-0b z{z{dRt?bK}Nv=KA!ELI^aCc(52lFM~4oG`B+WuEtevT5#bA#?TKpmlcsSQKDIC!$=WekJ;{ILLh0V?-U1xzTil>;G zu9`A1Ngq5NcHNi|{1DPw(XDwZ9(ud<>I>C$!og6g(8XvZV0l;s?eQYVktBJEK5r@5 zWzChS->bOJLeAVh3Q&><{DZJYlr823@ubXvoL6KToeG5cNlkxVCxNK*;yloLKXePF&YCP zBNbR;hebqljK_UsOZNSjEbs7LlT^NA_KA7J&VZ=3Loko4BBK1Y!HAyd!_@_e3*>C| zBnsCRw;IQ;dlcwQf*1pM=i@oD-;Edgm+s%*Jj`oak@}#`n{D)_8*q;_fBN;JsfAdCWXKWP<}F>HvM?@XVQglq7?tf~-OUNe7RzV5Hi z86tocURvrun2sldLEKWaYsq%JcG<+Nqv1vv{gMlO(%U8{K3z~wcJiL0zvcSW`UAT1^Pa8`G zPJDD+-m1#gR;66Fr;3jMcY=r!`ij+qh>a-XphDKiCFqHX0Qgc_O2z3b)jv=j4VyRa zLTtDAUrD3<=&TIsAnkUQu6BCpfO%h+nq^C#h4HQbMA$vPA8H3hTy(9%QnD$d_2L6d zPDZS>-^SctO8A$tARR#zg#t?1T1wC^H#nt1aXFiTUSRR&=WeSVEg;fGy_7QLDN_i z$IHQj8vD8Te_HlSU=(6U_5_Fs=r&;qDO3G>w*&;>8jW?aAQ2)V$}bh&YNAC zML08fwl*l$Trx})_;2${9Mn=wk`ST`2vr8;5CHz9_0{kt&wZcBWKaPrlX+&0Nnz7{ z65gCO{ zbe~!Git>nqXd;oDlsIxW5S9z`o(v-EO|^XIj$>FE-vYxJsErLbdaY!FD}CFHx~~;^ z6SJW~f^50+@e)4p&lP5T3|&v>T;eup&C77tK&o!{?r zuVpAD{%`KGor8RMix|ac8!v%j;%yT#KsuhPieS{m=k8%LlR%2}G@ZhTUcu8*b0U8G z=NF3V7$D546F_LsaYlI+tNFH6CmsaP^WUa!hAg@PdP8}Fg4#sF+zYI`X7wO4mq^lc zs}G7YGJL!Fkbcsw!=7_dWO6OaYxj+L^dPk9B2LU(K^KiP&;;@2JMedsop*1bl$3!( zfx}*7`@hl>9TTnUDM|?6n-#-_DTJmF3%%-zNXG)$1ir9PF^JH65=Ox8xdVUN@=p^%lFlGlCA0c2 zpVkgR;ld07pQM~my~k)us{X|}>-$Q<0bz$1q^_$%Lq@IjQe|hZW-v(>H$<-v@vse{ zeaULt^w|c#T+9vr0fOm^i#H*K+&}C8_M}W$^>NL5EOnj%RI21?Uj;-=Rfm%Bd`&Ep zyf>L<3EcRJZ^DQyQ||SmJGNwT)G1pupsko~m z0}-jeoL@ZzQ^C?pEbiY44m@xKyf>DBrmpo;;o=Z%yvl2?t>{1ev z^y=!oxPEVt<3`|fSdk`oWm=Ifv54B_mWIfr>p8t!Vs79iHWZ7uzWvJ9Zc0XinEG1v z-?erQcT_-&Ib8nj*F0$|43l%M+&;U8P@$N{n!k&xtfVyZ@j>&?SG$ynP%K4ZxcNhQ z3E_u}vKXF9nn{Sf8bvUBbF|IHOK3-P+$cNPO7bIMF#M2isiM;M=((+nT{+)*ev~)= zkmN0gaSujL+>@l?SZG-_q#(Nl#?_@+TLeJM6uBdJvJR$G{mBSr>B{Wq?6Z|t`U1xf zNyMG4IfYo2EF#k2`FLC(4E_i`6eD{#W?^#k>j2k{Xr?WKjso9E>5{D~1Tr_#!PbAG zJMMw52>Ed_LM^d-hl}2&p{uo6btk{m zsSy%(ELfoQ#vMb0IV^C1TFzX~{6Qa~M<#aV^xM%x9;@j4JU4(E0oXFm4Ki>bibv6- zM}QDOG6c6+5L^K3AqQJ8sBdB^R)@m=#vc0g7^z+nYMvDa{i~9q7@wr0B+7L%*S)rV zC?dy3}|jvduA3J;tb-f2`&$GW;y9-{RA%dqaIr^$R=zEU#SFy7`-xW#z~^ zjJwqJj>A_&IOuew%yy8nam?E;iU?)Dr?lxg?-+b9^s8o*b*;M5lW}oEu#yHP+fO!) zxc@wV>3rzWC~w`HWDuV^uo40w>e4DO=pi%qwF!)f4=Ft;QWR?rfxG=J9VQ~pjN>;m zrdUrnP~uo&cyZ^v$*1l!ru(fl+$@-@lq+M-y8|L5owe|*60;_y2TF+Gb^d#U?RHaT z{g9EU!r8`QdrT7ZQJ}|ii*sg^>%iO<7|e0!U#+Nqr?go`e3AmYH^777Uus`P3kiM7 z<r({%@TM{tC>`f}T*4UPF=%n=`s77{Vgu=XEHzPQ1Yeuu{K3Me6oC3}oH( zlni_fI`aq5gP?aI%Gj$s(O5HE%w!eMk$Qqer5HOMqKAKtVt5Wrr>Dpyz4Lg>Af^W@(Pc=a_?ZW>!67%by7vTi~^!f zfzHN8u3gi_5GJ8R!){-;gH9Gs`K2_}TNfLrEUw62V*xuaf7*>9nb9l(yON3z!ao3gR(H#rD<^kXxslCLh zrR`GSzQ>KldT=4C4>yMVPY(w^oPs1S|NQ?)pE(a~U_mr~{bRB$v}qR-!+6l5@xq`U*>^fbYZs>JB{+-CwnJFSu*+zWV6QHbN zaUsdxIEf)+*bc`s(D58I#tg@_>O1y7Ne3&dIFNH{^^>DqkI=Lz0Q%2QJ+%E(ApqdJ zm?1CK-nE3$BcqdxSAxO-%551(roKnX$f14YW7bGS*Hs*rmo}#@k5?QSrD^sAs(+DINjcGCS@9B}xWmEghWeX$Z$c zfT3U<3;s;z6qRIP7Y`znag{VeC5q+yc*lj0W3fIRARHI3GC68)Ju7pQJ+AoMHXcSzkeS->!;g zvhlsI2uaz{oJE5X+X^#N|Jy(%LoS zzzGkcvVD>V&$S>=mAvV;ZY-6&I`t5lptfCZ?Dz6@i)7ZIRC8>}7$#$^!$u80&YwOR zOlA2gscDUHfduC(-Zz}2=}!jh!#RY;zadca=emc| z0uNP;hyelAbb@frnThO3*hchUHIo!y;UE!?H($Imc?kDmpu(mqnsgamakPD1(h}Dm zd#RV5bAGmuuy_BTa%6-`CS>D}emS>~8gn@DSZ(_X)zWs}3nm3X7myCn<|wU$SvO;p zSPEiNAdUJXz*JZUtxwb8X!zNar{^KvHH~#~rBP!5mmXC zvtj(iqHjTP=g~D!Y9@r$uy~{Q-b`Y7ejL|7(fF$&gGz!v976utIBJl(Zre4n@U7)X$dAiwG0VHVQG8`Q1QAl?+P+;03}b+qep-cKyn0@S-=3{N`@Om{@O^(wg_Pb z!)f)Mxs*$4DHHs|K-=pH+(5>f$`MIw4NW2|4y-vr=#nT)ro9OKDk}5gGm^KszpdM_ z8<{nqpvvpgNa9y{(AL$N4Ujxi8N#|r<9_aw5jqmWZ3o9_1WEo;S zKc|4iHqT1a7lFTB4X|{*1{<^2;y6gb*eusgP2YP-RO&yL;*LRRfe>3d#-L?rA6f1q z5RK({5m_%+Fs~25ilK^bAQ`%)InRod7l6y`fLvy+FHWd~I0&W#v#zN~7<3_r>%w+8 zC>3cNZ%IfFfHRIjCVsi?DU1YLzc{VjcJ8@bv_SJGDD}g0U^|mOt^%`D^lM+{Zz!aX=OK%n(sv5?d$#bTy~~^a0|A656i|y;SjY582AwF?%wlrPouej-A5eiW~*iU z4u~iYL5W2`FuKUdqQUhH_BjRT7fcm=azZZm;oRy#5Sbtd_3O2^;twQH7;^*xyA@_C zPNj!w@ppsje%`+Je_b`Op+d!GcVG{BmwGOe)g9Ib%Zk(fC{g7us7@rl!SZ{k?bD^Fv-}|7L~c#hah&g8GkXG+ z<{u~k!e;bY8Hk)|tXn~Gms;sQX#QCeUwSH{aaee{kHv2C&N!8csm6uaN_VIsUL0Bu za2VWYmLHZNbhiXD+HuD)e~()xwl*h;-_-$!-mYHv3GX?6(t4o*z4WZ8tqC#L3e)T- zAbTf&0_1E_8oZB_SS)}7#52?{sL}FU$E8`tHy7X+;5q^bji3U>KG(5_mIkGeN=Ey= zb4kxgGG?j4OeCmU>;bSX6o(*VdJF@v9@JF`B;lE%1u26?`GI}=JY|njLOCQ?7v2Le z^)=WVS+$PDRlEd6D#)hw3_ny%k@yyL}{c3NBSXFL88Yj39kEuJToH7XK8f>rs$> zZ4bp}1X$2yfEoelGLaN1I?yHy%F(013Me^(TPz4J0Q`6}Q8(D2QfVAAEN40E%dHA) zb!m3^@y|mQAsk)c1P#0Wn4=wsatcURP-e6g`qq=Kc9}3@fg$8IVG!f=)99@YU? zW8`;)#sCemiMhEGok5hCCb5ZXJOlV}w$DAsp{dwEvmOi%=itl>_yYn@F^!eb$>;jf zZhfM~G5KNuq?%?iNkB?AYe?s?dJU_eB>rIBv6wp8D|ojS}a0ed20llo;~VET_QfI<0>G>=ip6h|6c{(rGcjWb}%afFWH0 zY4e=;`PIM=+_(Ualmbe8j-VtQ5L%s7d{Ebez%>@eiaQiVZ+Kw^%1eQ=)5xU-sX_@QSq}GP#^W{~Vnz zfzrkh`owt?bG*d(9Wmm44I!PfYIgR#?#E89=H#Yd8atmP8im13w?kmxHijvIbTwi1 zkIm@VI_B(-^7D)#$BY3C4TSunO*N}!I=5sd!rviP5J;x7oS)z+V&YpoxpXSCWszWh z96C@47eT4sO|2A5Dwg8_xja+N4`9~vdT3=5DJ_W;INk>|9#sHO+i$hfG=?Hz#vZeL zI8s}U#S2X~Uo=%YACVuoHJ=Inq1u4T*qm=yq_V2yyo6c{E%6y+&!fw6JD4P4T_7?L z2ZeS{z0+?fTF_47mZTeTs8DCU+OA2B3?T^&>H*zPxv2u1#0XsDB_gO<)EqO^l^nX+ zL+kwm=-(`GZBphky+OEh3OJ~8+pTpk5ygz#4md7klGnSt9pxD`B9#cX*b3e~*oq=6 zkox|WW2{eA--c{ATJXV1P=D-Gcx?q5@j`C+m4i1$&?we(j#sh8HS?3l!DCm~Bv0=# z8qIVK?$~JAMx@VOJfAdyUJ4m{bAET8A54aZs>yj_)LY9h$${t@gKW8p0BIY}JYA5& zob_oMCOYgmWX{?ayg$0VZy>Dx>kErLbYJLiRDR66aW|A47O}p2#6vDl?uhLMU6AR= zzLfiJmL$M=aB;Yi8|w94?cOQkUe^{Y0(#MAn4jlP_l3e6X7l`#KL?#j1*@X23OI)I zBaoB4%CyzU7$SiVPMF?ituc_1Bnqp74eK&PiO}3;D<1Vv<~tk(72_nb#9lZZcBdV^97BN2EX*Q1 z=h%swDZ202@+KvD>F=Z<_7BCy72KIZ^*8n93;#Hb`yqU|rvMwbUW2$INkE8e^LSz? z6afhtsrn7mXa-VKc0VD8aKWcquRih({g9$8K=mWWzh3dXK;0l(MaLoR?w&EEJ2>Fn z#DrTaNgIv8gu7wqG$cF{_npkP1)6bpCqXmrB`#7z&%kDI8 z_D@Z{2HOr;d{K=dFM0{n+kQ^&kzVOhFnNmnsR&3JXI#IlNk3p5PQF~!mf1Lz2CigI z;urK;kdoRUfC2G%x2U07q9!LgCdOJ{!nY9N7;A!{yi7PRu*5CsPQvxkUSv3pVA&a= zlF=WPQIvL1@Kv(3zgNa_d7j&N&PKRs6CvK#^~ucxt+r(52pv*W2v!Ui?%gCoUZQfP zj!w=eRT7B}<*WYZbCZ!!kY=+4^4yf`4s9Dk zLX|HfgoT-~2Ypmb^aixJ+?RQ~NV-=vfi7NF&8KY)@$cL$d}8HKtClF$6UPRRe|Hz1 z+#4YqOBY>m=jb39AW8gx4>ULlaXk6jXE(jEGWaL9Gr({FV?dn0m7O0++(HqGg4gX~ zdImI7k8;4OhxiR|0z7n>#tA_agGrKhGu927SMzCxc_Nq&J2oQ1-#|9H4QWlsgw1uJ zpDd8Se+l`_*Jq8- zd422EwhkPGYbBLcl%LC*jqFr-B5L$AFG%ZVgxX7U7owmF+B>s6${#>{eejncQ_)?` zz{lkX-L*;TOT~4EmXwYY?(p1au*6HvDC^Vt7d7+30R7nwmQ@3qxgxAuF-5#mZ87F| ztlBO1vcT;Og@ut}H4vZ(%C!%D-r%I!PgKOinW=3kvbJmNBrbnMAT<6r0Mx+@91~wC zdd;(JF*;Q{Q5Bv(iO(IRCJUgdVu|e}k1>s}2vJ4wpe0gq@DmhQiyfeEZUvs}-= zcYHtr`!30i@!(WZ>V#nPux>l9c2tn1zytqReGnTgF_Yim2neTmsv(t4vUu_m11vyseXvYZLYwva&16lz~rQ z@_RS`Ifuk(v|$2cFwF!PZxs;mV^#;Or-R#!Pyl_jvzZ`_ujU z^;_&cw?xCwYp|NxMICaA*NKeZ>DX58a_36{9AhM^HU2_5WxZ8hI>{Weq+?uK_dyTv z#71Krx_jXELOF$wfpb6P&|^X4dt}FEEURB|f>s?iTn$#-l;BJ0f}>aLv-)S?ccLW&3_L}4eHD^0XE8rIy38{DHz zS0YbgY*y$NYdofVPpo~n zO15Z`UG_WnYdN+JH+L3xi_Kq6VB)|Uzc#W`)p9zvgw=OL9V6--_)XUY`!`d(7Lp`}ZyHpG|7>c57GkBT+P{?GFhA$b9V#XnqJ3#Cj*F=F9D-pGUoRw$*RZ*fS zr8O|P+~6@bdNkxcf$`5yZmHP(Awp(dt(Vx8Qfw%*4Z|9j2i<0n4VZA@v}Q3eun_VN z)nt_s2!1aE7cMy{rwmS1fh03(mOrrQJsw@B%NTt5o4rBzJ$%EkSY|ZYRo#Rz>2#J} zyi;q@R*w_8PIXY_%B%QA$8=%&MZMal=<6|D99ZCfJaIWC6Kf2gp8SlgW-v~|j;99HM_peA+SpebGfV#wZJ=i5ff}mvEtI0jO?rv5EBEw)1=AF^ITq_kW z{B|Q6C+%8WRa@!ACiHPX8ho{A5Mpv7yTXv$&?CvEB2}21CB?h8Px>*9pDdB#0*MSZ zqCHE!6n-GaKfayXpX<}I4Vo~}D!bI;EF5oidNTJ{t$C|#hO^*UTuIiR3*^XCI`0BnWSU8OzDu*5Q-S)-@?*r*$Z}k46GHZq1ds38Z*epuA+=7l949n)!N}0YUkl zR>+!J*$nPZGGqK9;{)kK737Mvi((cDoYebVoM|I{pZ`IHtE4Yqp6D)zc7%W;$po2M zYvn4R+MCvr(oUK2E0MaO;T*%x)cUvq) zam+D3t=UUn^EkR-OA_htX}Rs0gm}_``CmD!{3z56XOtwk>XBLLp&muuM(?;Q9|86> z5Da}hw_P_oS$9G4{`aOMR14+3VOhW!#LA)uU)+3*J~9`fT-|@{7FmVfKYya`{Tyz5 zNlz;E5qA4&#FD(Qx{CqkRgD4;okS4teFBZ#&l-54Ce8_%6ooWo-B*yK%2iavsl&mR zdG7r1`QuPInspwm1!HTM%e_nS4NVHBNL@aw3&dIp6j%e~%SOZ4lZTO0;(%cs$tJlR znaL;mr}Af&gHwkEpb^@5f)fRYntk($JZmg&=IrWyE+F{Ul{@hlP>O)WRf6{2>6dgQ zNB1hP9o|MvmM#K(6o}j)i4s3o(tRleC8?|^_Bzt#E_e?L$+k#wO!zX&3Nmy6i-$3z zM}QMdF$9)bzyQ?LstZ8kBobO5@=mY<*d>S*K5nK$EQ5u{W-dH^ejCL&hFKldp~x61%`r-px^>KREDi0bG2r{F3uDIXmHFwM|jN8=fdnuG`vrWss% zB71AWv{pGo)8f||gCVnt^@4Ws*pAVZ{&1TL2JC`PK%6eMC3Hjt-4m|)ccO8-Gc@)o zZxKY+cCu{GoE+har6H;MC|*Dl{?-yWGUdWD4)J5LFs(%`Q$YEp7&WT3hS4<-C^WSj z>N!h(JH}O?c|s5Hs))DUogN1K*=E!Oa{Q>!vx#RXvZ5158$jZJxyKF816h6?;Y67h z%lFAXH{g-eMNFZ)09&xKzE@@?l|!sz#Xne8Dz~*nB&Dn<$U3EW?z;qMSkEyYD4e7e z(IOf$B67BZ*R@$=fN%3oY!ER&!us=RaZ4%Uu z3h+^FJcO*~5Bk>TG$)JkcMSrgPWbL))}Nr9331RuZQ~B|fVQlqrKaAnu$pd;7d7JYvIWB1%dxjT z;z9klQz*~&9djoBS_)l}=bPYEa$Yk9`y2(=Y{p@u?dG&v25w}bebP*YqxG;l8t9`h zF*9v(;iZ5*CO_^7LlMKNcDV(mAgJJlV-=baXT3V2+Mr6{00S^d^E@m5V5&qOf&p$TReNL)i_(f5FSY?Y#=FrUw?B;?!19ud_26zO| zSuR=--pPdmIt23heLvnoC4gq(Bk(5r{}zv!#sahZk)J3>oq&yKstMPAuQB}<=xpNN zUu)W}DIG@-ZJL01F8*S(zYPt;*f4PxK+=0>P4sm5Neg8cJZmN0G41z!(;;lp$>S!m{eApeTDX* zK5eGl^L{d8(`fU982pNUeRsGk zL73M`LzR0$s)UzQLCk7I&9imcNn8!u^W!kg{!Nf(fb@2+-WTxj-{0rqmBy8FhHa!m zq)aB6uFUFbSPM(v{DXrz)n&f+__hHDMXE+cA2b(y^C9c52f6?m;>`O+8cN@nqXb^L zsX>CSR}ipO*^31%)t?{Ia{2wt{j5|?aqq(R^BN8Kw^I{Q#^Rf>PmbG`*SA0#0hlt? z4Kj8hBCnaFM}QMdF$A_)5L^JuKj^VGB~S~afGgnZ2NRn@H6BNPQr~^t_nEPA_v|`37gpCa+z9zQD*CDRHYdeOG>0pc`5l6q%-U8bN zSg?N=@(Lq{C0OBP1uCCA_m|(Ar>vN}VR)CaTtplLxe_(3Yi!Irs_f*j()pW%F9Gru z$o>$t-=3@Q<~ms zS8WQi3Cy1WK^GhoXq%SQd2{90Dm6v|VyDq%?vq>_g{n~^ue5Q%mQb!BX66;9w7Kj= zZZdHj+WQnZY3VNnJb>)UO3hsS_rl9-v(n@EoXVc%d>fFPKnnH@z_9W?d&JjV{cGB8 z|3E@EMhn7k%;yJeFb(7}>=3URt9yq4&*C5J1EFBJ2#v*Dc-_LyA+c z(Orntp<5+a+?_^mwlh(MSIeR>9*&f3ca(TvAP;lg7X0gTTV-3vpn{ zqFm-yr8gGb6>N?m>VW{#DeH~_6uXXWzl$G(Iz_ZZlPl0(GO21Vvx5Kwu0WfB9$#Eu zX*wDd@DLx3d|bP*f2Fv7YR{aILAzp zZ?Sk7LRswc0S#_0Z9`s~TT%}2=k;`2p>)>AO4;dyZNpD?CjjYUC01Xv)$lH_)r+}; zxczXzL9?Ea21SKlFwn~uM2f?*J>ugmqZam0w)M%^bqJfwB3LMnw&8}kZ`!Sk#$X+Y zm2MuLJtubKE=`xYr)S5Z-`c4FWcT)-{;WhvHECcMUr zprHQ5&jrS%?B&}5uJ1C{k&~2TST_AXj|DKiRp@DJc;N=)^5yM%l3p~bU{jC?#EO~t zrZCn>6+Sv1QbW5g^D;cMxQ_4L56Fg6=#dm1cXz7>MuYkp>`6(tg$oTs&E%v1aldL$ zFnx&L=-7Ow8Fyg~8_O!^_#ZCm9HOq@P;rzkpV-0{@!!fSl-N<+Rg!ddG@C*-%)LZf zbzS&qyn-0jVEe5!1^a2~=RWz(stR&cy1N?3yLFQEnZQB|WSO-;{_ODWkxR?6c6n&< z3}eYFy#z41S{V{zCU&}4!j;+-GN6yMlEljL6T03qBTQX>fqS05Mxx3?XZalGTGy~ z4TSNU5{F>KQbqGCoSmm(Qe-x_8~#2;KK~X5L9Y?kv*NcFVi$4dEV1%>ON74OSKeH6 z1Tz)JWe1x}YuReO9yKfA3=LOnf7wX%u9g4J<%Y#DEWk3!2{LyG09TcxM}QAJaRinG zzyRh!cAh}pwauo!%%Eb~uExCIW{9AAk@LC^wjW$)i}b%X%DH&y%Je^w5`s#c*tn#( zk?)LL*%R=%>Muvh-}~ZTBjV^acW>^x)o94`!hP+z7?KRrfsANfQQM5sB{$D2M=XhE z?`^BOTB`Gos~FkFI`7P+<*Zcza4Ujl8u~T3O2heLgaBIO?BZ>e&e6eDV0B#q_yG>% z=pS=-11dWH$ORz~9I!z2mSO|Ui3&?Ta|pQsw@gC@TFt}*pZ#EM9bJO{msj@l?@;Ri z42nz0tkBE|tk)PdPet0I>uSy}txJwhc;1c@tvm~9u18ZWPU0;Kql~TZ3tJzJkQ9k- zZQ<5aJINc!2*{HIt{zGYyOfYsh?aJ^x1Um)2{EFT$wd7nQK=@E_T?n^<(#FYUQ{U{+Id<1+bJHQIScRU6v4X6v?;Imd04lYpRB;;=U7_RWna2*)Tv3fxu7dNQ)6R-rf58H@7f;Y5;ALxg zU!Q(dmW(ihkV3FQ{FmIG>lt5Q)`qVbRK4LE^cJE}5^n!#F3zI!OTA^m0WUim=sq4D zJL+`dgxzsazgYuf^zlPGa!dtmA_w*o zt>!gExk=b!f^Dr|>nb{n`8~H%za0puf&2@S`(W_96Xg~b7qYcbi!)l$F z8#ElfwQs4v)fz*^V8&P11a&8+0!4+NI$9Mz4xPgM-lqQNKW(I z*6SX6waFS7K%+&jS5#T2>#S!PJ$%N?GtziVu(aPS5!zaLFQ1I1IWJ@BAq@gRntGo5 zdnse!ikc?SBi%H4DsGOBZ*)u3EY$Y@l-AaT2fZOFjb7M&N#?jwq{ky!Y2~}JUDSq- zVl7^$K0N>=KwnA=&o)t)(oBMNg8CesZG6QDZq(A8ShAhlSIQ$1qWMro&}ga>t!k?W z`OTbxmF)l`;^OFMjj_C!r-rLUm{l}6iTNDWs8$ZQoZMza+cD%#O={Psn{o=g4M0a` ztP0pQqcl6xnock+1Eq`gzclo;=fl~U@`OQ^W79iaR-{0SyPqH%ZTyC+v}AG~=FX^- z$Ztn3L5~rA$P{B>x&Z^Ep9Y^~4LjM9)_F#ON;BP20%5f1q32OL8UHk~e_;7?;Iiy)9uNt%8fK@Ju`i7$s%$ zs_r&<31g+FA&}6qki(9rP4Ju)qK^ay&L|9u8qt!h>5>agX39F&aa5f~Y6A_ZgPWrg zP0E#bDRaOD4)kon#XfU$NFimBF1$EY4Kmb!P1L7P}$1Ng8 z)IlCo99@{P*T{J@kKx+@xVO~$eX-n8ITlpC17qc~d=5=`n+3(eqzZCSHB868kB-f5 z1!xbE_h*3d6tcP)?9`aj9MSrjNp%;|`#v1G`Heq?h#46FFk|T3yd+Si<DKJ6X_~o z-zp@YgH437D8@R9ME&6@f=|5tnV}zbl1~r+bmV9|r|trj@HHgdrbp<#yl@o}k$&^D zR_4|U9PLqd=YgsR%=3J{39~)+>;HqY_3*bX5MC1>qgskeSOGus_}l2-{`qoZSyPzG zWxi?E^7%Ll+ezjesjgSOM&D)kC%_IFnFe`dl4Q#Y81`CD4&pt@qL|uV0c*(Tmv8$< zOLW)Z@9jV)$5?GkX}@q9eiuX={=R8bOphJ$XUpRV;$47Lpv}F5w~vNb zc!^W;(CJ>E+Z*(cP1)s zb%_1veBjO`ZBq<*l9(?@g7>N1WZnx1f#^NP0Tp!KojAWp8j1XJV|(P&W+l^{qm7eB zP4w1TnA}zow11mHp`9b$Yn5nq`)cF#Zi~{P3*7v&_d0M7cYATPjdKPG0eRSd&Qj@L zb89}wwGS4d1@(RaRgL#CXe!~UE6AA{5aWFh2_HkOAJaD<_4vcTsirFBkcZWr3&^&1 z1@GWDM*Nm9PQeBt^0p>@4h|Y+Gn>PJ%B~!i@CrTj5n~pE)p-&+*J8FSnV+5*mfw7- zcj$CvbSSaAr)K;`N7h-u~F-I6YK%Yamjjss;Ev8GKUkrtQGq8zW3~5s5$8 z8qNv0{fhi^iwEE{FRBmfBBrhL#I@F0e|m(VtNsIrbNT|qzZG@&swuo%m<58HjZBQF z2oiVI!&@iGx%kKk#-FbUcE2L%soqXzP_;mZG+BLKzr#NiuR2;5=~E`DP-8uC_&r7X zXBC$vGohjrE%ag)8KUv`w$8|9@#4N45h{~DmBTuosK`yM9g|NeZJkzt4K?zM zTeq^?U~ACgvYr8brRVpL%2zc|F2{V$wI3B&nXD9d7diQhZw-34KG7`bT=~*7G)Z{D zS7l!*wSEhs;|(>h|0X}aDTXSWlU_rS@}qsz!*MmI4|)q4!&{5KoA)(+h#jmZcR*

mZwToHaHFSUKj)TW$fwPdU#US_w0~9 zM(HrVvC$tEo>PAC{76~xpnMx&Wmx`QA{}BvAMMm~uYOv7lBq~iJ`D6iIN z9B7y@%|6|IV&N;%{Z9Vp~{ZXl!N5fHg5Yiep6p#kPp0pPl_pUw z_DVkp0hRB{1`x7)g&@pdue40U0(cu3SeGQT@}E>&6iC^(97r;Mt>HYK;O^t^*DdOM zUNf*49R0qc!{@$sz@G|+%{)&$XRvG!0`~OI1H-rOs%$1iorK z-f_MWk~=INV#w#v{eV8KAG&$W^E3AgjF%0AHBydW_4e|eG;emKWc0PZ zYe(0A&gn-LOSc)x6`RydU~f&d1}34UyX^tlT|0nev+gN#bAcnMw=a(cD2kycqZeB% zS{S#gCovW?fN8e+=Xv@BAy(B{JMBk|2qZm4)IaF{ExmNMcCgpcp>>0qRDrs*(0RWE zv8z0d>S=VgQEu8_3W>4OvYM-^blgM^c2aP_c)~18D%OIko;>G5V zO06#u$Rs{EDJ}}=ww}zdSnk&Xsfp{GrnROr==5o5fs)!;if2i+seuOU#F1s(fj6Bb zU4+>WyCu}2!K5|^8|Rqu^paL?b+a5Ld-Qq;+na{XLm@JR2{4!t4H zKGu~)%U+WYI-#p9k{M-OWC#a6#n+$V>sf^nUdJzsan@yUO?d%-{7~WeP6Zs-d_$C^ zuChnud>{}^PG;zSGZ1y7&zaiMd^e|$SAi_UYZipC;%71NimoT_&3584kEE$lC~+33 zq+NG%>z+UGwqqB9{aOUG={W|pJ23E`DF=C!stgxFr9ay>QhEuamG6h6`PHl$kF~w- zw3`QYTry)4G~X8O0yRjoja78Y>ls@;+BkCCt&RX=j2(E^V$Z}6=i*{pakLef!J$Z^ zdg$y)fP+-+WR<5NqKis~hISv)wUuNRxgNVeD!(3PM^5tIwU?$?b%M1}?t>H?DrhdfjKfiH|fpKH9dA*b(q?yj)86 zk4-&k{Q&;t+e;35%`)%p4&x&1h3h5F#&@8736?m=G0iunq0j97A@Flt2$@T`H{^Th z!`!jxHgZ_WM**M!8dap5_VFNTBG&4G@d|Oln~YxDKnnsJX6kY^;>q6JIP50C;J*<> zeTN%gDg!OgoTg1~gSB6j`8h>01++z)gFld6;WUjs?|#|n+av8nm>AW0jKhiCDcXd1r3bGC?KH7APG>=c_0XvGd-C47>K- zQg7jvk~8J2aOTp^nl0bsDL?Vk4gB+cGLLjm@7LzCv1~Pz;)`e+N zWVN;ikgbRz;U)XfgNrz1#P$w6!LguGhmV~6-ACR~1ZE?Q#L2}|J~yFy-cdblzb1(B z1ppIvsrjvr$p(nOEe_Xt9sc}ur-019jz~hCp`lvj&-V=rQcHR)qkye2NHW$vlwYW* zlo<&G*PghYqs7S;#@!vOC2~fV2EHONthLQ;ymdX5vM^@W=zl$==L(iijR`Tp0ue>X z0;1=h!Tu7H>U>+IEUDrl@7S($n0mZl(vmzCnj9&T!se)I-VaNsGj*f8H3_Q>w0Jqw|vUV;nGJBu#X z5f3J&Ux2&+QJ^QMoM_#UfCZ0WAuK!Jz;qm8g8F;!+-QGMQrkdw)d)F1$QfUvW=@Av zbv#5@+f&c`Y~~*qr#jLgecVb%K*Iognk;kJaWPvj zQsn#}L@KwQc%D_2W3Nd8o}kmK3}**C-)tz@u;f`0)}bIe zK0{`RcClqaB-cX_^Vm3LcNPFW{{iYewWPws$dnae{T4`9E5f!6;<|vB*C2;z%j5AQ zFAvfd29WaY^tJK2kYa~dsvJ0GO9KKdDd>^vvKrT&05RM?1z!KCQsMl0&^JUTVhzjA zqx#nSG0;anQF&UbFm=Sg-~%N@FajJLPXFc3I5P(!-wr-)m?s5$r3jRWhbG39oB360 zCD|WoU6*1S3%<;<5|Jw8&hK9~hsxzyv9(I5eyY6>^o zD=#LX+|(&uxK+>$;i=_d8Q!U=Ug+GStaZ%*zdA$3%yEkhB(mR?}6Z0o1)X>uE! znn-6d^$^#aU~dIi+T=*}(JQ9KMMpR8A{Omq)~ZnzL`h1Yw>Bv!4IxKVrr>d7Nk$OP zKD=(`($xDLx;HC#ZdOtd4}+{hKR7BsH4V$xIi$H9W3_2rp5)^YQvq@S&WrFru4@&3 zIjpLfDd~zbqYGBHckoOryg)6*8SN8sFqGsr#%007*Hr)EAAoX^)EAp zOLBl%4saN_sUv5%v-7bZ90psQ2^GriUFk6{cg*7QkrZ1Ixx^QE@hfaMZ=7*3tNBI)&l8HI0nw|$al)Zo`?L$AQ3_qOGyzfSC>icAt zux*L?aldUk$$?w|j)i(A5Bql%GM+L`oE-UTjcoY+qDO2l!WzNKnu@n8DqLc%s-trn zmsnOaDvBfYK)WJy&56PjL0)AYC1&rLh^F2VAql(q@X^m~&!D2gJyx<{x;+wbgCCn$ zdk+#_&)ICpm^SB-fo1^=udI?3E}n-|v@o;I%VrE)*gm?M;5qiZ;ReR2F5_=4+By{T zujU41e}J4M>a^+?8+a2b4jbVy0z=1LAoy$I%_F9jfa>HU+tTjKptZ`-{Jk~<=XYJS zZ5m;5jl9H%2xwnh9kcQe|Gdf2$wI|E@&|S^2t4%CmHVY(Y#S96V^&!JNp({jF;bH1 zw%W8W&EP3}gqUHgJhJ&zQQutJJj4UNgq1N#n~y1obqk8gp6%#Jq9cj)-y9TdC}ZA4 zigY?503n0F>v#{Dvfvz|~C5iI4|`O!RU_(J*8bu9B6(E2|*YW6A&HFHgDu z>0)?IRsr#HP)dvI?t~%U=AG@g)V6C^e|WB{X^YKp?yo8}R*#C7S)pr-p!YI>+;+2M zLidNaDy$dL`N0UI8%cIxUzTJVaKuql`q~6!i}Z)y-o0WC4-Q`TFcGU&@%21{Bg%Y^ zs*hHy{18c9QQ4Dce6au&t1Y_9U)$Ty3ZcH8s5-T);+%t^GQtKhoIw{jM@EkT9(aL? zEVF^E{0d!aXA%M%AY~S7I z`CZ>^ubC#j1_ojcyvc`_wkWI~m>+{e9#NZqLXenYV&odbtB~NukuDwc8nG8M*t@0v zMojz0r7*-bq5nLLZkP`(v?k<-`8ZKYmXp(+I$?u9-tzO2|jSn8!emlbN#yMkVMJJ*7z<_izR1VDZM6^8lC4j-1 z(y2h>bBPM6tivm~sT=%Bt>H3=y?O+r7g+B>!*#vn^~)k5oD|a|d|Kaf zjA3gy2pNaEmte~+$X^JX7TBSRWl1X5u82|{98~^B9sdNUy~Vazv^?FQI(3RWV{wro zUU}Ezmr`ymL^Hxq^nW;Cv*b4l#1N=oDs%Igl46vjL)*0~;MgsC-lh_)^402vU~c+h z6fJ-Id!nTP&3)twF7AXhnj!~4>SX7C{_FvJ&tHTgzR$iV{0A{)G62*>j8P zZ|wwwu~BcQRTs0DxyIPZ>i*t9iwnY!4u*ycvJSpoJpH$MSroYD8H1;Sw`3aM5&+7tA zCn{{MS9zZU*R!YiS@Djf*+8e8ENTM|9 zE5ULHo|j$KIc!oN)|LNaA8*X=s|^Fltd+1#FR00r&0X_3w`Q=8#1KLpDbCaqGR?pQ zdIuL>pLe4892{oF>3!tUoVxDQj#c+F2MTq)qrY;`?Jm2e_*jnjkD#dWcKtFYGP#Nq z@koL^IdE+xWxx?SmO#v)5vV;Jei567Bql!MG7zBg6F-)B`jGg7Wm@OOxYgYN?DDJKfgaNzXw^jOKj=H@$2NFaioBtKoFo ztYZ^YY*XfZ*4wI#h1I40#YvG- z5pV^(Mb!xD?rr6V4vDitHImRG$sMJyRJB=###bCokcVKZU=XpyAmSWa_31;!u-S?I zt+HsO7|c`5_iTZWISc8=%^t_JWC2#zif!vLy=PW#{aPCm99XDHXzy6?q5CJHeMhic z=!RedKqfdb1QF|u0aCmSSmJeej%=UOd3o+rr9aL}!{V=A?Sllgt=*0$&u=^;<@l_& z_@(o(1j?({uX6Y7a?P1Ii9fKKaBMG9_C=%t;F!008_t>{(@Dt@xIHz>CwN96_4mHy;yPD&o3B*ed}@Rjs#v2_^C1h1a~Vs! z&YvFF1APiz0dA!LKPL3wkI+ehSp|_v$u5RFBy3oOOrUYb;yTTIOc6L>$o;B}cyWvy znlfp=lf@}OUPrbmqX>rvKz;}v9we@it$@18%7SCnsw(zD0Cy57J(=`b_QXQQ z0^mIk+dZS4@B~gb4OioL0mPF-u0qiHU)Fhs2K>ryS}#PR%r9R*u>ozA&{q%r{@Xp6 z!hb;X=#J_jA>&3IsUf6r-S#z~4sXaWO3?f1iMrXPvPCUzobUJnXmG%|o_Ep%E_N7< z#ggq=@i;M~1pIo(u)&VjEd+Q8Buz4QmlvExzmENoPk9D1w!h_|T!&F&Avm3G`AVn* zf0-HEdib)L1c~V3o(LNGKLd8wss0^_-1Z|*Zq|2p!9jpD)>rCjrG+bom5s&I^HXvq1sC7YMGq

^ z#{3{+6Sr=o&~bYmyDq((h`U>4)%<0Ujtj-&g-JiS+0!_tda)Nf6F-M4P8iu@G;EcO zrgu!y#!0g5J01!zIvjTZDF&rJm`JE7`r;Le)MLhc{R4E0M8k*XYWw=jS!%&{a)vRl zH##uX^V<3UCb;6MZAd<(0?as6ubf*`WArF#?4KD=*K01_3T|4lvd=fjz4QRt__&e4 znl5FObiext6~!+>%(z{<-cr(t{dycMrSz*U}jcG!=En8$`{UGn4)}X zR(teIDN4WOqtMr3THv@E0f;iID=1V*0L)jTM(|>3kT{yl0t(;&1Hb_Nlsl2A_{m1? z!1e}-?c>onqqN<8QR-%(^zgPQHWs*u%`X)_t2l#aCWmG3t)ii-bC6^ewBa#O^p0uA zzmooVIeP+DLA(DlW#$KU8g94{9C^k}f!z^vK5q9kSxxJbo17W)<>P2f4nT+dj@|PUW@vlf% z>O5cpxiNz}78yvQH6=g6y+wQzR9gJ~g@;~H8&9B)$ASEgshxg!lydX*DNv%#$hg8p zaxQB7@iZoG^J^qT<&f_V_@&0;dl2zu3x~`AHx=j5P>zKeyXQv}ryggpGL$AY@AFWy zNL-y?O&B&0wt)P5JtF{8vKLf2M<`F{jl)-EG}}EvG&mx?rh-?)qX~}Hq<)B4c2t^1 zTQ&``8-Ph2e`fL*kP0zCGhHr2!p(%cuNI1vhI)ek4rPZ0%s3LnRKoWQ2}M#ohLMP$ zFJ^V~g7k2a`&MdOA%f-M*TE)cEQ}^WXSY`86`8(k)!XO|8{}ti`%R9wM95sQ5Xgr5 zehTYK3&sxDH7;tLnbm*A2}`}NHpy@xWgYeTy-dVoIDshG_`E^uNL6_lj^rVHGe`7a zzVefyT%d;y8$wn*C_U;Yb% zZOuK~3~O<`lv6%#@^-?o-}fq!BvpGpGFEYLnACVR!#J{eP+!o{S#P)~P_r!oE~v#; z03ER8$Bu8Vl>up%-uaTWb@e2ruvEwR0ag8R=8#Ld_>M2qII% zsq1$VXli~|XR+v{7L6KAYl^e?^=hc;SahxxduB3uy!%dr$qAsP*8XB*z;!H4{Fb zsZuPB-wnV@@y}zw8hhG@+lP8}$&swk&+-UwPDItwI^6~1#6q56NC`MR6p&S<>{|@? z5n)}S?t%}hN{s@%$*x8)VytUk(J3zN;$_vHewI?zVR-zctW9_bM~VyHR$yiXJeIM5to!vv z!uXH2y#jSu+fZH5H_ZDSh8@GWz;?!nJ%OEy1OZI^%aE>53O0|-|62LsH| zJPO#)kYQ+pk%#o|8Iwx;YpFN82b6GeruYxzlY=Q+Bn!KP)CH`X?w)hp2{m}+#anRy zZyh~_)s-eNR_r-LviO>`v;%W5&)3#ySvV86XEVxtTY+xSc5mW)<=I>$MgNKjF1qKy z8yItqP5=n&K1il1CEE^o`Az}C=27x205& zWP%&tA3(3>1_}w`<>M6QGO;b}Fg2#Pc6fJgmr(Ae7N*I7Yd;4wv3o`=6f^!*D%(vp zy<5fiSS0bntT8zhuO&>>C_cbpx|fGYt2K>lrkSy@xMsW56L{kKH{7rAK|f7koKVU|3#hvuogGTg2_SMnBCli84?b+p7)^ajsW4*ZVSZ4^Xkj!u zu*SPa8h~!lo1M6stoH$6QREm;ilYa>w82M@h)9g2sml@vmhvco2uoyyJd-}{R}Sfd zW~dr1Q4fP{AN5wG6^NLVE87|JsY`z1;noy1hW2Fra?EzM?kmMxK0OCLhyygT_16k@@UeSX0CEs4E^ZGz1+z$=UgA!p8JGf zc4@tdq`_>T6zm&C!;1yNiX5?;HAQ_3sZ4qOyl-5IW?`ZF) z+8d>uo@Ua7`Q&>uhHvpG+ zX57Cs&+TY4qxn*(;vIo_XM&G@3 zsLdJrcUX3`6e6*CDvnW`49d|gTCg2?))(W8L@`r}>3^Ju2)r{d7{M_xE7#1ga#)mS zeB~jZzgY&w9Df(*CvzkOcD^RM{n=9Q-sE%>g)GzxgQ)qcDPRCXiqLuQKILj%M86cQ zaHC{#Vc2noJTbHX3pIjvgbE0nNl}`!1 zT>p0`^s8V30E!7EON!SK&$9}uw3tKxix;iWeS%|*%qWPSfs%4Z57rZS0qH>z+%JRM z8#V_nIQ8M)CZy~^?DTNJ6e{t1ywlH90_{@x-uBNhHQ0+6*@A6zorZnIm({{7wHObH9ng3FCd7{<#^OC^!FV zD9WX2-}y;bwAN5yXGFeGgO_!SkhXB2;uazhXU-k`vSiR)qA6WK zZ^8v%Fk>byC*Hjyo# z4wS%Yr>XtjN;{fpBcQj|4c=QNtRY38il3v~itzlHSjZYk=WPfQ-BV{Nt8*ta6W2QZMzVl^k3QMT_FDk z(B(`cs$F0V;5>N%YRVE<_Vgfj+~b`Js#n9xS@0lEcW%64881R+Pe)+#wY6-QYjLF) zlU34T-Q;dE8K}O&&wYDBho#L7%nlwygMoOp{_+MREnhh=)gUI~yPI{iwlv55N%X*K zPm_}L4N3^-IO0NoaqY4gBZHN;wr`OS|Hx%2al;|6UlXK4;_1HUecUZCkU>c>Ll|JZ$y~2hpRGQ&+x|w_*Ms4sg$n7?&RO8&N|U?c<2?ouI2C~8w0z;hpd8YXZ*>^e!F3plw~C4iffZFz;|CrK~_mvFA}BW z_Keh@tnx{`WMDIZqk=LP$stnls&6DpWrh2794zC(8fWf%M-z3 zzHA5WvNJ4cuN6x?GD06Tj7qyCg3@mMY;}nZKg5(5(+guqKnCp^q|RNZy>aTK5Urzhmrm9I3;t{m-@Mx8SbLsZ})2 z7n})tl1VSq?mWzGg`a$i#Tr1B#J<_ha2v_U2R?%Xx)XTxw>D#5!cR&e%#x#e?uM*5hu;Mb1-n! z7f_5#QaZMrTxVr$!5e8gx5W6}#vUg26f|{k5vKxn-3l}$U8W#29W16vBxLh>jGEw= zR`;~r<{83sb~Qk6XYb60+b@6igmO-*a215RnoRur?vxlJIE}#9eenlhjON+@YPP9> z`Y6YD!;MZB7iupV(ir?lCAdwn5dqhv$K|TLM-b^;-EcsFy&yeh39m|JVpg8KSAkjs z;j=@uWPCEvQ>y)$?0qz5Kst$S>ZOTS5lPv`ZG;@Aqt3=qgD@%E z6x8YyqYw?HvcRW-8jy*DP_z*a6G#5;cNr@KV7w+)FX!KMajF!1?KyBJYiKn-^z+EH z&yUr|o^Of!85hgE-=14yomvN+m-h&#!3a9@Hf6gpcQ?mZ8*6TG!aJNPmnecAJoAzt zx%MptqmTRdO{2%c{i!wgVgiTt=~;`!z>KQ7c;>|v1}`y++yswgEkNpz;m4jZY~$Rz z3<%D2DM+=<$;rs7w5SzsCh;i4GtI;dEVc=SciwpULlYe%fi5ZwG`p-r3m{S(uO7$7 zpAdhwG`CTzQ}2fl@i!s$=DmU1S+P))rJVnNM5b-#Xx;;dGYvk|#e|wRY7HaMh|c^8 z17&?F&l%sMyFC5b@5wc5(KnQq@rf2L<99c>d^-DuZ8n{7Mfx1xd006a4n-ls3WNN+xLe7*V+RTss+h!yFRNzZECd&@UtT&M5Wzk&AT#BQD~wgHit-Jy2|Rn``Q|L8K5S?zY;@eEeQ{|Q zp6qZp)m&vJkeb+G*qKP2z#~kB2s0tkAKj~>gdiduedsxqlAD3XBAXRja<9JF!4`lH zypnda?15yxUiiI~o+C-d1UFUoI<(QKe4{}A@r8`?rVOdNb$=}Z6RqvWLK+^H`747@ zQjMhES+kF5ZH|+MDwPDgN3blV(ugbwJx;gWe|K!d%*!&qP2(Zpg<9{=;`0R^q|P|O zlg}GzC`;T~SQec$kW1Qa``G->=W-w_5QkSYvrQpugSDl4r@{n2t26%??EkV3uV~__ zCn2A<(z=j6#IpF$HgH2#`ax2UfK<->?*wG$-G z8RukoXebE2Z;jYg!4jHvT%h(eaG&Q}))@Q6%lAo-et2J`8(y&C8p2FJ>Axp%EBul<3+ z4kJk&|L32!&78A4w0-t%x!GArVz2qb8K^p)`A(<%X8Nj_CQ7tQ5O}b$#jepttMmb*GdLu zgF7qRf39vGA@beg>Bian;DBj%c>%p4`DE8%IXz9|C#jGW+1~^qx&v3|h;h~d3>a)i zNa1Yl!;Uve%YPr;JGtYiqDL;?hH@*mvNdatIhe>>3==#1`K!MibUmPqm#W;~@z*C`C*Fe)P+Krc%IsMdI z@C|Ww=$5(UyF_XxoaX_V5`NXb9ocw4&b;(P>66LmdzLe#x)dT0=` z#KY%y2)hKypNht2M)&Z0CtlKcAxOY7bT(Ltj#JqBhWFFJgYb%3C+r`RTfUfM z|AASJe6N5U|1+JYa`ZNWaBoJ$Fv#|Yzq(i53RPio_@Re!LsuX7@$wYPsEn$5?mD=} zaZ}ILjgD3OD+J3>Xqba_>9R@{DUScqi2$UcXuXldlRfENTHi(xLpFQSdH`_sQ?k>7X6Tyd8zHd>WTj0MA$%Sfq3+)Yss9q>TZ+uMSw%=^st{qm z;qktMZ0Q;jXcsrSBg>j-E|VDXUy^V8)~-?eM@!W`YGcDw z5w@|oBM`b5^l{iC=ASVzjr+l5^ggQFPlRtcsilgeGVQ?#X=m2NTZvP}taR1*Z=&%t zhAnT3+c^R5n*nW>rXgeEE>f7wxJM!+xfjy7uv&`?ra~9{mc_o6rfHKhzFr-KUnU_Y704AA*i7X3%0RAZ+i9IIDLv{Vh)cxMg89!d)pDkY7 z|C90=+aK+5{w!$66-mfGThb(G-A|!;q^83VG)40*j-XRzROi>RZ1=|7z|qN}9_>JN zAO1qbeq=ME2%VLXfI$ZaZdVA`2B(lpfcInv32lKWwg>4Mm4I6mkhsc6I5$Uz!E)*s zgfN4$eU!XQ5f+3e>|o?9bD?9D;O87JYd^|5C5Q;M3hTD%c^xs~AjX48%M;t|2Bq5q z1wZKB(?RzG9h%h2I?muPcN~(Vknh1!b(3n_((@mwJhx^g_^`u6DV_fM%T-nFjhV}U z`VOWYqlJNP_VmQ)@) zmM;NV1dO-=q2rgl7N^>XQn@=cU(G0KRRJnn4^L%x>RzA>B=(62>D~9ezmjX*^)07# zHRsk8Kb~8CvA{Fl4$rRb39Fi)$hebQ)b`z|Q6(>HRYZ!;R9%yG#Y>Tb*(l{VB@hmg z=;3T*V9%-QdoMVt%0SA$68UkJ2JEGgQ2i!1<7j6~7qp6T+3{;&lG4t2w2Z!&t)6E_ zP~+}VMB(!0#%~r8$bDQnbSHP5lb4aegGU$ANeT83RT8VtbL(;VJ#RC09qz!)9ey~0 z(erWg&zZGmOkgA0p%#A5enBnrUd3@r-8aRP=6pQ|yZ>3lxQD%K+uqThzYJi!^HNc? zTT1m{U0&++hBMz8jZoN*x;Dde8t;Ks%dI^nO$UC(&)ONl5&kIfBnMUc4(3f>4sr`p zPnqeDv9RlUqb|WfPE`7r$`TfZ7zE_3_V5l`i@BXh7OOZ!0xKWgZA@nvlqBOAx_{u9 zAW@{?Z!Msl0aI^No>T;#VETDr|D7&3j-=O8Cv>Cao>US@eXU`GJXVcKDx!jw?->~_ zK+SHtgo$$V*VDdowLXh)ZcD2^;=!hYC@(LXihq@Vl>!hXn|-1xKEPBphUtoKowZXb zQ2+rDW(DYh`cBLepm3Tt-ilrK?a5mJS(6B}QRhw$$H8b!7NO0+suvQ75Az?l$j$UF z_ee%F&YuKV&of{{(0NVPiaakDtn+g#NdZ&MqRArGz8^Gz5R!j;kVHcG^g(4}Q9C(- zU^3K^FL&^e1NvKop2lr?)Zj^Uo8+wcdPv9n-K4foIA_yW{%%ao-_9lyzwhz!Z9S8M z-Vy6%Pa(H(B6JIlRRFbC1E zplD*gZ}oVk(L-odYovUz#kVvku+gG#eGT{X$1;N<)|Bp*tsAydAr$Jg0W3@3r5k=X zY{$9_WurJh$>l3}UY}uduIfkN%27Qhz7pluj_)WBusoc4Bo(*D$}!)nEJhk$3H)z9 za!_(cnm3-u&gPFM&LJ%j1YMy9QbnE7RSiw~58!O=l*3F($S+OmczS}a zC$$ANV)1}J(j}_ipAx$`W~!*TM<;%m0%QJ2UI3o}R=RDB@#vOaz8nH!-*_+``q@O7 z|KB+nn}hEbb{1<2;55Na0co1W6T&_9dFk}>l2u4{X6&>wtWw3cjnKK#gD2jEx;h@F z4I?+!3m=i401)UQSI*4@982_MO-<~rCl>hDQtmSa#` zfQM;p#Esp%!^!si%&b|5>uHpE&#-48k;z z677_kF&g=L4!QCg6w={Tpt4{qx%7bAM1VRAk9+gn+TE<^(V4qJqsqrjpnry^UQMFofvCrytDLSU($)y%iQ40g#dX%gNdC4S<^6DWKG z;)f8qQhGiyr&lCuL42Y;UXDX*r{@j`!}mZ}#e;BSN5Qw5p(;Z+c&G*R>`#M~?V+pi zNzN_|T5&FOM5ol(5?kcWX^164_;B4oW$FEMCy8!t%{Nb{4axU6YE}AA@8wD(;7ezN z*`|0B_Q;rpNUe}jy!J=xB+k$uj! z`j8RJng3?gEtU^@2{PjfFs1<%Igdt<04A9Qi7c~#0R6<@Z7;Sng?7JG1(b^nmuNg3 zZ>{xlYhNCC$A;xc4a6l^DbH3Cuv@|T(hKgrE#y0(h2JpMQ7}t@zFL~pZ{&2(^K>xe z02M_J#>k`>Bnt0n6?H{82JjxwA635BEk*l4kvS+@mYh_3gvQpq?CWffpsS05O1W${ zb$CfGgdU@sLN74934|xb;;wmN4xUblaOZ5eb*O=d)Uge3copKn&HXhIyJc&b!|o)3^@-lho)1C`2cp-|j-sIM z5VO_$U8yNLW^U83G;(#V<(`48oey`_lK*oypD&h49SKv2-+IsFC3c4^96=hB2^uRa z1B(#QDqj2qSo?=HYmnZ zj%bl~E7jEX=b@K^_!p4?Qw1A*=5K1-jQ==~;nG;v8vULEQf+;Jek8knksw{mbOLK8 z9xD0*ux#s-K2fnvksbd(jG3$`zwQ*~*o-`W8q?VB3(-8sJI;Ha{5LRBh?u&ov&E1j zeN><81?m^7-g7DjJUXfWb>ZMdoY|GjYa<_iFV5AKq>Y6JfQ1nlc2de2op_W0=#Kez zOWaM*-9WCaI3?!AI)yj!mRC+eZ3z(!))LD}5)_*D(_`PpV0bf4ODAIUfNRCNj)R++ z;{&$8$>`0@#5oKaSoXFC+`}=<&$o#!c8VWdsVxfyAD8d=(3KCI#t9nyuvVgT?N_-7t*9J{(IAm_^hj;1JJ?Xp9%bqzs)YO#*XRQzy z%g@~SASY?NE(i@GSh{eFFV_;yw)62M%bqDBYA+^^Oj<7m;Ufw!y{VskJEWT6S<|nG zR(QMcd`k7gq$f4YFKHFEXiwAt2KG453x#>V>82y_P%y*xUcnGvKWmvy$|19Uhz6 zgVV~=ok`GSKpAjpWYZdJrnmaF+M>lNJtX8gwaJG9zWjz+aP(^TAG!B<7K{I8&&exs zuyOLI0Xe)Vn&}=4vTUiIs}5I6NEQELSt426=28;)&7)4PI^x!L&Sezy3XVMt4#^q( z#g#zVEDIo2hwM#PoG7$qG~`BM1!Z>+j4xohF$A0RFbRKgd;lZ$GNNyN1Y&JLJ~T7V zULV{k4-gpx%lq0+mIt}h#{t`NQwJ~U8(V++ry^>Nq*7VfJe}UA3(>c3--mtJnCGG} zfx_hlvLZqc7gIwW*z#m1=!?b|9YX&ypIBZ*i2zM5kE40i+3WV&QG3KKN`}8?9A~!dEW29PTjp&p~~7;iCRa z%5s{UpaXE2Z&bF4^^M;5({knE{R3&y6JvDsG$b*GC=7=Ou$5cak`(zbPm4%dxu!vl z2=``xpjc|!$262JBlCkswzprF`|sfH76{7le;;$<8|}lrIZoUJqT3xQ0Ut<_P#DhP ze;dr?6SFx+Yip$KBB!W)cmeto3 zVMcHx*ta>NPjFJjagZYm)Y?!sr=?y`c5Vmd{%OUK-JsH5f-AT#RRT0bdZqq&b9%8h ze3Nu-7G_S=Q?1Vuyzx_18M^p zNUWmq!qwYhV5+P}s~Il(nV18dNPzk9@vACgXLU3eAPht7Px6y11czA3UkP&&b0T(r zHnt64gNC>}U!t+S1RK!C{H9DpPWX=qj<)J>OR24S*faK@<{N;4k~9m{BwcV4ada|HpERR}MD7Z6uI z6d;Z#F!CT!{3N8yxd^ZK;ytDCia6c7m$9tzymEznxXN4~8fL}ti` z_QG`PU*RirjkiTb=z0)%h>o!uk|}=68{ZxJ0_^?X^6SJn#zZ-26EO^Q$DQQEpsx{F zBSssK%TXe%WS|EUzm49H=j*#>Zj4#%A1Ds^i)FQjxvMKgBhL9&`}$S31`5$u1t%*AcnjyIF7}}!j5yyOhKO`Eimy=T*n$+3R~szPi5W?&L?KSirM>~r zp(SHbOh{C!ix_;6>^mlDVK8}Cc`zPG*n#b$iChizkho}0XMXHkxS|T|L6hTy>`n&XtoD8l0nxwCMWZ_<0b{#7FNvhT@W#JTznkL@ zT<3;U7uk3VSjHv$2xeS8me!?zYdZZM6t*>W>a7zemx$l=)gv{!T5cc0(NDLkEvoL_ zpy+OYeFVTHs0zQ5SyyzAg0 zX>xtqFg)0>Qo~c6^AgFB6y_*G(H4g3>Pq(x#yuyR$Tl_F%Z5Jc_fxJYCTY_p0aM zM8R*Wemw)dC8>&&?=XqK;-|2G#)Oa^&y;A72@KIqov<5jz9MVDr?GbF-l1HdJ`f^j z-{`mdMjz_JootLpvSj0rHcZK*^QYFC1Ip6X4F>PjLA)pm109VZPL2Eo068Y|i@1Viu!R+~R1itp(2L0~ev(1(p6<+6it0tuY%r zUrInw*NS`o#uU?&a6#)wgkKQOv>Yb z+>TzdBeD1s#QHI5Is~SOYRZ>ob$W(10VLe^c2n5Typ4PTQyv$~=QruHD3c?4=}8Sx z8UW*OIHsiC&3Wat&?hAYWITW+X&K-uUnkO&o+?xkCsq5a1#{?^`VI0^7f$V0g2)}Kbys) z$SvnK2i_BGVz=3jXix*^zG_mR{L z(z$vX2>s0_Y}?(EPhaQ<(M#e{>s157L&jW&u1Fv`p>Z&t8O0BKZnZZshu?W{$v^`Y zmOE$Cvzf{=$OPE3)qZ5U&WlfAcC}2D#x&CybTRYL*3xU-t^>i2G61esX2$3vyxhiu z%iK0Z>CM!W^(-0pX#kvM>64FdKT#?5ju4T((GzngKZKMmS)bHG;Fdu%2M5}>!T=!F ztj+HfU%|SaPZXhtQO|dX3hSQMjC-e{2D3d!b2bA#WmQ`gI!jSf^QUSxzxqN@+!^$I zo?2ZzUNA{0!?=R$Wv5>QRyZkrNL7$u`?Z``!`ZLY77Se!usG!Xl|a}SPWQt3g4`e| zH^iFI_Lbj6H6&%5hvLVp+Qq%V*0qi)T?kd=##V;i)vQgVxN;dgK^(<~aYMhYj{oRh zEZIq*el@6=hadx(6vi}D&x6};ZwKX+xqh3a&=X4;ScHkcuu}{zl z5J>t1$0FAa528Xh=4;1x@bRof_NfL^cE}tOZYEyO+l#ex3sf)ESYL3+xSTzd7^&cs z+>#Nao6h^%lgob1j0vg+E)(-{-$c)Rp{RVtxR@}>LO}ad2@BqFf88vpUTqk2 zx#8MZ$0u)GGrVeNl>JV_vZXHMz-5C!mkcfanO=@Vv7nl=ny6rkk@#YmUs&oPKQ%kv zI~-bxG&flSlLQ4w>C-o*BU|c)qFe{d*t4S{<@}~kAfH3Si^%Y#o53P3I~MrwXQMVG zhfe147aS!<t)ua_#@0mPUH=zSBv@`Yi~-*rnd2 zlsnLe3+s<~LX-1{sFL$!P9l2JBBEI~8rgAvf#ftATU%PV*)bOQnsPAC6W4O& z-_0TBI52GbwV0CH9@CvRFJh3!9P436=Lte1fw0nnz(|9!y$Yx}Wu2 zbn8kQ^iwb8)u~%8L3*z5Lx_+E(aHIaV9CCkq2N~=;s$^Jb-03Y%gi-eHf6R+gw>g( z!MsZZ;8(P5YRyV$AOej}eRMh-EThUZhnby|JNN|xjE6uYM~g6Pv%j8 zXN_OE1%`QA!*e6?Qrm0~gz_;BTpsY~om&|4!!b0`EJ&U&~VB~-NCGvq8>U8kdTk(5qn4t>-ZX#crCFK6UwDKiC*of}u57k4z0Li-4 zvKRIoO+-1pD9QSYr^ zj8vrkTR+k1`BaarI2x==eq8thlgtWIW8#8h8_d#9!K4f8Yd48CirRgcDv)9CPi6_g zHV>xUzP*-GK3{N=#y*}$1c^B9d7>_$p~=b?A&;bIfq}0=xhF$7z*~_XBK-%1uS{3E zqzyX{d@Q4a)~hk7^m=U9ma_b$xSvXHVhR`%f@)I%HpSYHUg*scxDqan#>CB zH9Lx|c-8qo;({R;zULrz(Pkj|YVgW`V6kjF(El*#$p%XejY<{oH-oSfa%xvZb-Q2( z!bX~tR}k-K9hD52_6aD{ht&4=x|u01fjxKzbOSsWw3f5QFc8tKBY+j|Za--PQ<~un z_bra@<*`F&Og?22*we3vxl=J8eF#H@O;XJf&_23RU^4OuFsecsSVu;W04A9Oi7c~# z0R9Xxem9Y7BwKm>*8k4`H7jJ`ZhTIvw;zB6=;xt6s+NG=Xp7aZ|FM9qDs&5^IvEE` zIHXBo$~=R|d|bcepYoap%W(Sl4|bU6ph7qK$O+{4>;_j#*RgG^eFm`feqK)yQ2rX^ zZc(*%^a!_Bpsb}pCyFqo!TPC_t0Nz>s zqDj<@oA*%0)dFe~K=UJ@J{T#1NjE@?d55h5Qcw#Qlzkc`NM;mDJ}&lwOJrD4&Hf*? zsSBc>TF{cUxzm+>>dA*Swa#oVAizvBUl8dgG69oCG%rG4jbV48-d#i8Ug~`KFV2RE zs;wU@jj~|xW3a+KFzpO&sr(4zH@KO-igx{OKyco72((j;6o)U>tI-Oir@>Yfq@~?| z^;X{?rJ4oOdt{GZ6lnNjEAmOr-cMU%%pkWsWYcS7_a2KbsGo~ks0;$l*2w_>zP;8V z_EcWctd~k0NqW}VH1p{l?rl8--zCozoHs+%qJ9{2S!DcvM?KCbu*p5#;Ebrgy>pM@ z__(4Pg#cQr5F1KBOQ=n59dy8#07QOeD6yf!CHp)Ecp8gjPhv9<6l0m`VU>@Kh)pSA zCI_sTd7TI~CJi|auZ@+a6VdLk7#C2iWa5@#%?tOWbYzjJ02zx1os#NRsix-L9D0?b zDJuUbs#N#DjH)-tnJjklJZPC*p^SN_V`gsyaWJd?dZ%-FFh1%xrNXJx_d`3G7L-se z`mYW*O)8lswEPJ+HA1*)U~uvkF~C0X0M+no7%9S!Dr#}rm8$i!%cXYNvTVPY7 z!-06R_=Lbb4X(2%_ZjGWLl=97Xdy4s`Ceb|IcG$!+}-jg{fuG%l$G}|$}a1pFHKnD zcAgwjM#z2dF2D{d0 zg6k>k4sn?_vV7NHcPG$4B53ISZUkUBo4(CIl7Z!dAs+j1CRa?grRTu!PD(q-jZYM04#oI8LpadgBbzQ9S9zt z%d!U*HwIy2>;1vQwZ09&jvHk(^&5V^khNp$4`0A*N_$fX$k5xs zclp{5V%Av3Qz;@$INdDt^hWKdTawEWk@{LQ{M!+s_K3UhWpRy{#4-5|^^K1&SI0m@=mgFs?vF=&weP04AA-iEIk% zH~{^3AF5?^E_Ai8h{+fdQ8+Y-&h%1G3gooRn4;gb;o@=`D1C{9&7M@t#h35C=-=#G*_?l-KVrIcMO0)zF93)SI;V+x^DbggO^PUzSB`ne!sh3-NY>LB$TSOW0mX^pf&{Z zF$S(Jkq~9|4WDj0-2o_wMqKj{wx5mwy8)gQ&}h|ZRoucgYec=q{;^up{tv%1L4jD@ zb>eCxP`P>B&u0#N4u3wz&HT=I24IID2&HIWW#Ntbp-el*gecmJ+&s)iFK?nCva12L zmdRd*G&}?NrO;JG^TxL`Qh#z2d33S;3X<<#oR<0^#;7<8W_nnC=5;%q^K%3yVw8Mo z@p4V}vBO3RTbi&Sd%d62xnY`P&_A^nr62rVRN{Fc%-lvlfZ+aNk2r?exNx}#g;-dF zZ=52+Fm88exz0j~4vaM6?k*tincEjGbX4lOVYq5wdC)p*5jf09>1}67oY-q$14Plv zu*326oR>`{x}zMxX5*=7q1Y0SAWx&BQ~IZdc&5xWf#J#r>&+Ck%nhzlkGr=ouR)T2e%-?!qwgVx0oH8-j%+BmL14b(#ga z`!tfqyXD)^M`%97g4JjwcAXk`63_29+s}-)M!!}nmMCj7WNfd`t{@L9>)rX;?5v%% z9LzFEAovYp#`7%m`pv0wK38$FCh=`MrUU0wzz6r~0GhHRbbfX(z3GCK%Q5+E-dG|e z`Himn1TG6C5W51d6^dh0ywy4ppN3^wdc{Tq6Da-(caDOpkyP@euEK~MQz@1aza3O! z$cFGHqP8vKGv%O(k18_0EZwXlowkO`XvGxQZe7Rb1ly}2?*iR9;A=OYVFFT?Rdu(6 z&2gk}Din4zC1Ia(qaL`@RxYM@+pWfg(w0>$C#1CL^NhiTR0GC(KM!uv{A53tqMP`V zPHYVg{rwwUB^O9CeR9tmChElIym40+H2pV}mD2V3nnD4L!GAHnOx**Lz!E;WdIB8P ztC9iMLpH-wnO?jS!dYi3g7McjcpJABl3kg3PYOc#`q}tf zIuzX$He`*kZDYgPm*Z$a7^XhAlBe|`gD#Ug!PK8eDU~ipVcg}0>`kjX5z)$96k^n1 zRz^9UFc3=6eI(Tj)&pd_r{at>xSD{x4cp#+T$B9iVxf5 zDBYDgh<3eHFOZhh7FfbzP}e|=)U?z38Z!4LmU}FbvnytO?RZLeg%$QOMw2yaFEi>;ey~GZVH)hb__1e4H$n{OOeUh_SJ$(Lqcw=u-SY z7s;U*XJ+ z<8{&R+otiawmb!j8WRtv7{>2!P}shPr1B)Zl4+AiaTDjh#x7~?-6wIadRn|b@7_l( zoO#dduoET8yqr(o84L)MJtU!lE#V`;p-80jhk?N9H!F)NW!*b^bxgq7j?A^{^lL5y z&Utz8U^QuR3L1(rj%OQ>XXsPuiw?NhLuayGrPx6R)Z z5^5;YWeGbt>{d<`K$0r)bgb|`XW0AHCSM8IKXm{^+20t)Lax_|)1 zkg0y3(>UxaN{HTc?Q@ocI?n1t8*fZH-9F8G_$-X-9W$^B1oxa&e;7+w205T{b-!?- zuvM7z;z=Z0J|DBkQV9iX_95Zt#V}}b0O*Wms4fgAC~!$yqjnDIX*w(<<$}mi?P1KQ zv3$(KA&CeJn}$#<{C;K?yJ^WO2rgFcnoJgGbjw#-YqFeriazLA1*UEf-quR z@SDUyea7{G>)YFRh$q3VW43U{WgB2}5lQ*@Ol&*3wFujxyHx$QZhRTzQwNQxVnA!&txUB!SQzUSIi@5 zpD*wBF8k$xc%nHb_@AEYoCMLf#fPLe_nA0>V7o8aF<)OqOU$bd4mCXYq!}z7PvE7Q zW9=OW4kvF*qoFNT2_I%m*dSZ?TciTkeXNy^1zkp!Xl>%QbpJ11>n}0?a!(yDwuI$w zv4`wx4R;xH!>MPXrUvHwC$UWQf8FAvW*rX$=iknviKCIUrsBwpAfkfYQaKnr%)=k< zb;@~u!MhCPKB{&4&`hU<9e|mWPug(EQko7(c={SkSKD+1fGV zs!i2eTh>zvkJGF5sM|Hhv}Ve1RSa34kP3=}mc=@80@=uuC;!xIe_{sNc^vmejN;AW z^!0hYh=4mNtEm5`s;}wxkO=)r?Y;v&zW1VJpldz*D=WMfYZbXg;V>c(Y|u3fQxG-x zO5;K_ZpaQMZc6LbbgS>_|`hpLHp5J$uga3K*arQ&O~0Yca9!>(dUzh zSzRBpbIV&F<}fy`OSK>o!WxEK2Qpg~Ms?wg#v#v7@1H=#Z-uO%-AhqXsuQ&sVaQfmbD0o zxy+*B1r|=XE05VfEJVZYN(BPWRde^|l4I8`Z5G=ERH!clO$*(n1LVrE>Tg7k19_jx zG`=~DrXtOjee)uU>^s~eQb|_-qNi;YBVLKJbY3f31f76e)RIlT-*3`oJBwWI`Cy%` z`^ESyrWY{7xGI>bvfn|>3uZ+$z+HA{1@&Y2m05+Cbw$*5V=*dAoU z(p~~4inR8CP%Y*Sh((?Xe_;MHPich=t_T>?`(xTcTupF3?p}uVxO5Lxzd_)~Zjora zl<}S-PSBY=M?m%O#cScBDQ$s;-LQn`H8t+N&#Na%r~5uqg>43%z%U22_L)!YmVm02 ztUupP(!l~FNP~y>U_?DiUfRPqEOeFjP}iP-z00=h+E+dk&v)u=j}Oj`Y1yojr~z<- z$^zO2u~@hl3iwlQ&tG8)C)ew^MeGK)@*PlH{)cn(`tSVg8oI6v?KG;On`1p1sTNPY z`Q-4i(a-@+ThCr zvu$4eeyRIfA7KKd0dqvGRf0M7l3Fz-#Ksm^TE0>DN`}Dq8ylhEO&iRt+!oJ`Szts2 zzJ7&4Wjhva{76@5w7Y|s2ynt;3&nF6%}5~&hFihT46Z9er^l7L)ja8Tr0x<;YB{a^ zZo(3uQ-zMvm=$s&OYqf?J+imF7@xHf<-bg{8IWW!B&J~t(t+D%Wkd0JuN|1VOq9`@ zAl=T+QZVk!ht=6`xXO$^uM9fkKDJw>**$5|w1PKWAW}mc>N*_K)-DJ*qt($w>EQqb zu5xv@;ChGv9-rNU=Vb3W5>{`IrG?H(292?AZvhAe04>XM>vx_LB82*AB8K}M7t5`1 z8%XMD#udKV{pKJY)xOIU>msRW@{u$|Q6b%KL%aw70SFb;o%h&RS64Zh5R~+|ctJ%XmzstRLFXfcgH7G7#Vx9j zwr-oH6vzvw4n!Mj%4&LOs+pc+fW6OUa{2yC0%S02uW2aMCg?k19{{HL3wqHQg9Lap zLWxw4<~UX-8rDmI@pSt`vVmh0YVRc$Fl~R#3mitrNoz}O=f2Z(S8H_Vj@Dtu7yWBL zUS3gNm1l=dZn!CfG0M7oCe?vgf_so2$!HNf96BKd3Y^%xy>bA$>kw0XbSW@N*?KoZ z@jjexBw}cdVrZU%a9PrB-M88yZZxTjc&>l-4b+fXs(9c0Pi!7icP#t+Z|X4`&;m&u zDm|cKuukMfIEn}sa3Uj@x~LXF4P+t8z*c5glBONlS$pT(&Bm{gD|A7*CP2%U>rR7F z3Cb@B`mW>@N6WkDX_TJQ%waOyLwc`LKY+!Ii`B(O_EhySe$Kg7Mrt0{8Q?dYS<-MI zSGI%ud;{^sMOC@jeMJ$!R|{K}(`GMn(m_PuqO`_#f{ za`$+{R;;}oF+`NQ>397vb`j}$s3DYsEh{5=QPP{~Tlr%_J<1ET^~0H0r?igT zG`X^RLTgXdNFUbG->?cE%+(_|A?e;l8TE0fN#Kg229N!EU&K+*8JkAx345Wi#aaTd zAXrDXy!x;BmA7j~E7=TxXs_gY{@9-GWx7`KcI-hJ(X@*J#_XBz`eE#Rd;1qwoao3t zEud8)z!y5ijI#+lr@m9ydqMwQZW}`QN1_MLlx&)Xt=9e@p++`4(^1LU<$U?CLnLOO zy=W2q7RY(&kqokGBi+c!eWL{$KE<2PDesU)a}!j6IvfD2nL{Pi%Qp60C#;~RrWOif z`i?$;m)LxDi_4?>qNEKwJFK^;fi59c?%A6+P?b-W=%v{_kVv8v{2U;}Xk`Z59-NQ7hJGQ}_ zm1u!v)1Vdw_%tG1`-QPkcyh;&8OHw7K5$0ty27Oei6ybkMTJ>7H%#qm!8R2ri<_Af zpOhf6hm+;>(AQC8JG{slXFki1fMhbfYA!xrIgHT>0)*^VYH69;b>ojV7M->5wH_yn z&I>4GoIvn`Qp^2_h``n~M6he>8@;+=2_`WJKbT@%z_aDy$L51O1VH!?BJgDtwxJ`* zsKK;!5^882XrB?dRhNCBV4f4b z0;)kY+oJmffm72s6O#$}OGb{3v1y42vHxnr>4cIfZ>55XSd%6)pOIwf_S=e04~j5K z_JaUsbn+24&eX<;VgCIRb3uJ}b7$LQmOWY?JnTv?z^ymuT$Be=1GU3lVZMn4pc78l7w-dUN6!Svy;-Zl&23SUp(65F z*|~yZ?{lN1KN5v_eAaJPfXuy-$;qQjHbP-2fmYRzH1RwJqsyhr@?Pz{Jr|=gYM9vJ z_f{jX%s&vrJn>aOpM06fg*rg~^asWjbZf%u8frwHWF=7Ws4Al%aD+F42ht%`S(F!mIAlxcEvh_09*DbX(S*Hiy>wxpH7OLi`Z>3{eCOd7a{GkYs*m##s!u9umx zHYH|6o|)fiQ;qJW7#8lv+<3_2+SB-FedQ#rW2GNTkS917FsKhFr)3+|qnUGS4}p7; z8z%2v_;n#yjvv&}E3Fjvs%d&BKUKyQD(bRFj1twsto@coYlt+%4Qw5_;el;wpviX- zE&v%x%2~pb630MAEuPg%s0~+~ip&uTRnA?Oxo~%6T(|PHzirq^SkJtlqDUSN}# zEB}v=5~Bt5$}r_|UIiu)=y}vBiD~xCgRFr;bPMqQMv;L+2J8n-RWo{P`EAV)`AtMB z**o5_j!uj1b~b}03&ijN+))ib?woP8{M!IQs0w?fta;fZASk)FKm1r`BbWz%$6JKM934O6-_zE{$;BSn|KcN9CAo_TB5-lx|{WsVcT2lR* zMHd0y5Ae$6V1tA905=AwYfMJ;Fz%U30X~|_!Zo7JCT5hY(x?Ka(FW=q7mtcbvwx4n zt}Vb8!eQz&IJE?`M+%&skTT7p-M66ai~gef#8Zr(eh7PfTQT+=&4;2DoVO*y59{2T zKw+EG5LBH5a*$iohZ_k~G&51A&P(aRBMK7~DlF7j)M?4_nr)fxfJ;P&g>tP!ca>L; zY^iCd#UL`Mp0V-^a2xPcVcyK-0eFUGTh}Zz3vimq>ju1D9zFlhHV3M^u~n}b7W`<9 zJxr2(D_@6N`d+|Y-AhrFyrF~%X@WXEgHir2aoByTm?pL{wyHc|0}%`d_3CV(zBY$; z9rEu{O3p39$`Y>=aMgZ&gk8?&3Qe=o5QF@i=MMAL7Pv_FyHal^2sIR{;ZkxU6no^# zf}gNauk;11X<*0JuaAh-A3<>X>+pc0quUAMOA%QOnQ|3K;-CGh|bYGu|vlJb^fR>3>dOZNSMR!t2D4N!)yU=z2dtnXRJL29Wt&z zVLo#vab!#99up#s!Gor*-Mb`Ij|t?!-PBD@uS&RH&YoKkXq&4BiNs zN6ec9pwQp3FsLWJlyd+nh6wmbf~BiM8U|7r%1{WdWkg%(7X+;?L*xdBIEoJ>s8FPS z8&jcP%AXSc*a@V=S8?IqWHYc%mPDc+vV^Xld|W~)&MP2T8%DA97{dG#JpJBgRnQha zm$J@((*cO;Yq?&nS`XHNQIn)N6KEqXxWdE{`_m z-hwYwiT22Oc-HVAB6%6Np2d8<2#)O`?sUp8XqCHZnN+PB&Lj8n4$hcnAph`|q!un(*s zs>Pl0vKq1|ILCu)sA|NV>G(H$;(0jWKINkeJ#G}5c>64t ziax8l0+BtMSba1)XbZmT0ur{ix30Q;%uc1*Mt? zE;V5^@%o?w_`Lp+)0V53ws?;XT(#MU&JDmqG0LK6%1(8Y5qB9rlv2P8eBw)Cq2DQ z4O7qKZ0%_#l4wU=*uhbAQTa-5JLiBjT z2r)DPXrw7AoLp?Q07yW$zlh-0qV!u{YK6x?^8y-5(fT0nR8X0WD-U!6vP%Ua8JN?L zTcj(=)d=t1rUlpXh@jsb(TreOl%i{z*=E>O&XBN!PrMw^rv#)@j>jY@>7+K-d_(fE zkDdKQ+?&;+0h}sTpWE6JBhySP+eG0JU5UEATlY%d{Ed(?1rMCk=l(W@LT}Sr=S$F(zMDG|h{!3uriPm2N2pnV07w@@_k=qkFLHwVYBRb<-v_r6?0+szwUkW}k>5DJHDN!jh=gRnrLw#W-M8 ztR?5}s+kG<(l;=zWJl7y8htgCFc^Ua4pn?BMdr7BhmPT;^!f7}X2vF19oRVQaj;8LzmU*O{eAp|Or^u{BsPp?5##F$F ziL=hWC_>EZqt54k6bBPm?VeNkl_hMkHnH|_U?t76)iGJ_z~)pFs<#b$ciVSWCDVr= zA!HH-vVu)#h=M9Hq-EPHxwHpRTS(t7yM~W0f+rb!n|c^S1A4+hT4ZJ!CI?m4PswkN zNagFGvlHIQIoLDQ#W9I7X_fou8&-_!H(VG z;$1c%fXsk4_!0P%wl~$2*`1v{uc5`sA;KT}tK1JPefBYtq;dD5cC1;-^el_F+==08 zELf3o2*2`Grf+HcLj<;$yeNd9+!@PuRMVMur=w1zlIdx5Ulg}ztVx_XpGTpxj47eB z=woi#s1(}@9%UoUPg;S^c>vod(Ib+R67_J<4N^}8(Sqm?ykihu@IHm451Q9kOQ7lDYEj2>*nA4F?n z3~AF_=^apiAwP&}pXz7n>UyM@G0Kk#y4^ARDwo}i07*HJ`q?i{ zq4$O8AZ57JYzrVRfQ6=S6)0mz36n~cCM0&^U_4%TZc!=3ZWEww*^gH&x0Z=ge z;f4A!WK=#UD^ZJaQ=s7x`pgS;N;w9OIT%z)zvIY#;&}<;%1mT@RB-7ta#E6}oO} z-6R263{4;$P<~P| zo#00$m9D&01Os^%pdj+QDl4sG54@cxHi--s<_}w^6>#&Y@UAe_ugNu7Q+?T7fwjjW zE6*>F*xaLw=M`kXdOy0i8m&>&A9@1J6%IhgnMcCYh*QPsK_VKoH+N!653~wqsLOta zSg>4J&hlev`v2LAz(y;4_GYK|z)%{_(5`(E=T5jTyF|QECV>mbqxWWIZy%in z&USJ|HlT`SKaF2#wcd$~&ZlH}(3g(S?L;X^Nd18EXnCH9)__@r;af=sfL)pwrCxmB z6?qdfhUTGz{kjZ{(6eUoI`7ZzI+3w|jfT&hhm7??9CX}7OOQ`w&&8{ubsTT*h38@Z z`Mvc26SRI7C&hGJ-esXv^3j*es8f#(&`>%LjMhD!Kq%-~XP!Km?2@T=C0$?bzM`4j zODcY;4;SjW?P0<`>EQPW;ELpYaLpzYxqFfU{m<|KDkP)$VV)&lG{@vz?O58F^&A~i ze$RZHSV6(3>vZpiV#wrxf1Hk^TfekGc)X+JJj7c3q{54f2BbzTO7e!b{j=TcFzWgJ zCp~!q(hbMLuw9I2q4v`RDLMxWPhhPOA%El4b#U+cP6O_6CguT8>@d`8r@Gsg3!n4l zht3DRmpt0OtZ_5wEjn;cEFtv^tc$9M3DSN`J0&D&EmO+PS?Hzb;4fFWv z4CUr;l>>GtG!5%lsw$GAXVU21YvWopJ!bWdYfr3ldw2%!ckb7W;E*}0vo0=Z^c{5X zd$=_yU(a-9q6318;xPm;sP+PUI@_?hk?ffo*5B-X)tpwB7a~p; z6Yc=;;FiT@tiM+EtpALbYYlCC2}7HoVF0#!nhao9bTnu~Z@!-$hd93ZZ+7VQ#Bb)v zmhks~vUUhO#pl>BXG1rZ`5Y)$1)-1Ea|a2EQJh`N%Q{mo4BY})7GR>kQhy5R*9g@P z@L8H6!2-{tk8~mh(*H`+PPi5A;^wllQrAF6_Sa&*lbK>mH1B~j8pJ&%c4JvZ0L!qX zbLmSOY0A`xhwck@u(2ULeia5O$*$$m)sY)X^p#ox#0rq@T2Un1Z61!nBD3mz1) z6xT|&BJuC%=!3e|Vw*$AkEabI$mtN>q#jP8x28Fm9)8bk83#DDgf0`|(Lm-d2>F!; zRLN$Qq<=g5$cQ=*Ex~+9OEzTk`>o`)?+JzQg-3qoj$Z-%iDhqHX%i}rRg z@O(08njN#=fvjVKRMK>o9#~746Fl_7_Fh?=_9Urv7$!m8p2HAuBy-Gcxj8P3la!dS zd*oqjK168wKB-3(n}Ec3<7*N`>Ol<&5mzTXHCd}*qeYI)xFNh?sTB-E<2!r%R!K63 z!FGo8m}h<)c2gI{eFO>$CpO=@8C!`H!n(iEna(Gs10-fh_!qA0QokeR zs+u)}&~`H%Iv5INqT_Qh$o-4e;f})4+S<+p7}v5fbSopamsa5@Sp`YxU64sJmF2B~ zmM*_sr@m;`@l29oQ^L#B8nbap3ZZq*m4P4iR&lqsSNr#f*vHG9@sQdK!=!@qEFFK| z0Jd>Wi{G7-K3m%aYhgxO@vsXBFD(P*u_n??9zmg-AdieyZi?h!?sznAg8eeZLiYQ?z;=TUE?LUg_;hZ!Yo+sZv5*#LE&u)n7STh9B6b z6jVOoYcc%OWnz?+Uu?BdSUIkQfn_m(;>dl(LgEW#f1kt z?_J8>UU0rT&-e6m=_HS`Udba1wjLr^5B`n=r!HS>3lfQ0uYj4UIdBil3W){O&Ut?N zHr7$TK_GO=^P08-73#)07o_U1PRH<{@9tz9_?jgbxSw_>5$X2ztny4ySB_jiV!XSs zWzKL_GhKZNrpSFNF6&vIx|tmp<@rjjlP>cepCy>+a%b}hQuat+h>6vJEJ7=2We;l$sBxu$vyfYi>j#x|5h1V&Z-0| zCj|A3FrntdUGLbYygQ2SMDzLh0kF^*MlpPuizn;eOf!dTV!JVoiBPD`+R_g?5wQle zs6oKx00~0pe!ukKv)a|6QeR8XI*i0$m}xzL`lWT&j9MvOibn}XR7r|*JOz(S`A8kJpk7%_sH z4t1KVF30*XdTI;dVtQX|OyH<1MQC?0l+bALJuWIJvH$Q35nWYFIlwVQ)rJZYGw8>% zI^uFG)ZGhrpYyiN+5`WnHysgVql)X!`>x-Twb={8?f!!QsMTt@FOLXQolHvl1???w z>3WH_t48O^is5?_L&@#C;Y z0Jw{&)GpUs{oiHH{w*<%4B!2vo4dF5y&`wV30)R*vx8nk#|efgY~CYj7o>3NS{hby z7zUduTcO+LL$<=M{|uP@>snTrqyuz<-Yq{*hWq+P6FdIr6#p6eaCX0-u{!c23H!W7cqi`F!{KzWUm|{0-05`|2SOHO7?-7npQx4N zT701W_u&xf&KLm=F#Wyey~%S!u5MhCeQKFLCuVeT_jF4rHf2M_PD*11Mt+9-alYrt z@8busdzl=EdC0u!J&(hjOfyn33j5jxyj{D@6LbNQsm>JA1UtMnd!XG%B9gGP(9aMs zl-OZezOWe*X8OHp9NFxnTiyj%Nve{OU{&^sjd55Xt`8cR&yxWu=(DDB#zITQ4(c+ad<@P&@#>L*oKq=eRUg+5wVWeG{{c_ zbbqpSJ&`nx)}d)52QlK@I`1mR8T11fAs4#LvY< z!-j@RvCzGS>hrJ(t@~#;xCUy!%b1@a82%P9o8r-0W`!NO@uOjVjwTCQ2%wRKAKRwT zFu>>LY33juEZieiR=bK5$aRah`!WdsKg_$B}1&b zHJi~RMVKnz+hXh{PZ9LVM1goWG_##C$Es}RT;9WwZct%!_&S7s9sdFlUN_;-7<7GP z+_UKQRK+C+r_%Fs$NKDQi{g(sQwaP~CYwbOt4=rLh`IasPFQm#$@kX+u;?{y4QIAd z@(LYdKys5)EN>@$}&K^R7`z-^T*E7hXLUVo!G{Z{`fM8sAzl8=xPOt67vLG5g>4*oHoe z4+`p|v&1&qu?KxfV43M|i2EtpVo7NM2SFL$kaGY!oK~fFs-EziDl=|wL90H21Y>vf zs`Tlfb)_|uoc|>+d?d*BiMHGfkgdnC;;72}@1(JIMRu?3P+$9UnP068hW9HG)5k#| z=4;JbzWzEVtiijGItKOl3=_jUt0)^BkbXu5{y}#(nqJ`aiQj6#gGJL=JVMBZ3J@FT~9H(DC?eejLkwyq@S{-3J) z6wJ`Lgk$z(aFzF6UsBN#Xso@v=|9%c2wOV`NchL{n9>Dk)f+tSTeH=)5tmH5!*D2Q zc|f3Z%kUWP1lwQb*S4rN;GyLFx5}MDKx=+kT66;^5jABCfXn{s`> z;?_jUN`X0}P*c1j8`YZW2r3puu;u2(9+k7K=kMgqUXLU#W}`_xq(4DfC&@PiMvR## z#u?m^dq(i6_uLDARsAvFib}ql6C}mIjeM?_@}Vx0mSK{fb{e|iIMq4aCqUdWlRz)7 zUMpur`ug?TY_yGk%?WP}8i{Z+d+?|8g7+Hb6Opsu$$!VlWax3J9e1e^ek?ZQMC}iIYX7^_8XAgmY zM2r@>a1oNn-2{T{f9Wih)I8bUrCil*#a&ZW#l68g$!m<6iReRdmHbM#pWg9Nw7&~L6*`H0&DE~`hNf_L z;38@2fh8$>0dquue&Q%uu`-THDq$b(VGKGFOq+S#H=#h^+@&-?Ka#I)jaJ~2%ne+G zca<>pXScwrc+L3w#z;scVyGH25@T(Tf7)**v>EiWuP?S8T9jEJcAEYdOxKrN6gQu3 z{+z!R(A6ACJ^Z5c9LnRZjZ}9IL@1cgg>SBtjpxy9o+ukNsQz|VW*@j%#OWk2*-N?e z%mE6IK74!Oc=I-cWUy{IV2E(+V*@B}H~mP^pQ_hPd&s=^gVSy=H%z`l>MKni!j(-2%WP;K~8Z(-@|Ow)Bi3bgq}Yb}d06A4~YsYWgI=#O>41&3`ND z)I(wtn)B>n5PN1vHJxn94)?ije7h@@i7Vj5EPgDo4N|b3o}r+0dxK65j=pK{L=X(w4J@PgY^;+ z*KZAS{kHcxHEFB=Jr-_C(t<_+B{Y{-db3!lr(%y3^8lRxvuG@X&jRJwDKxWOdWAmv~wjC8N#w(+9tuexxT zGlvbqc1M`(_kR~ca!QkP_r||y z76x{bO^0@mg}nI*nV@a7Qe_cK!MTmGno`J(Uac{*#&x=e1$G(!?gFbRNv(T=E?2Gg zCKR_rji}W!uoaM9d8vNtGOlrm2kjU;PO{Xnd*y!6e$%_I2Azo)5{ASEIz66ms9K42 zi^f_NcWQsi#OI*Hp$##?@Js$nlrm?LfK64(~iB=Pdod6wMblDdG zgW8Pj@bTGAXZ(@C>w)kV#a}>NgnuZH>D@mbBzWHyH_+`8e8xhp%!RJkR;Y&w``c*? z3=J4&7BbXK7sKVKelmwp1VWa$oa49vI~X1B>k_~`C10I|-Nhh03Jf&zEA82O=avn@ z1*1&R?c;AS3qB4hi(;GpnU@88)SOdoBywU2E`NPqSX1*wY80M_wG5i_pE7xa6r&w$ zM3elF`Z~IJvptM=^<2A25p+N zK3=Jj=ARS*G7uaU{aHqe$1Yo;+2P6q^5Yv@wuEtpyh^G-gfCN93J&@w&#shY09j9jr@^XJN(dblGJBDicA19fxiWW<-T!NuhUZMCTwWvIQ+9e zt&%eqW%+pyG_Tfra*&WMht|aKcf$jguC<(bb_ydpZTx?3uci1TR9jZdO`RF*K_mFs zOVy{(FeC&CMoxJ%wQo!pi7-QISw5<8h~|6Q1}py3FR7TD)ncYudmD3Ii2mPtD z_vpatf(!J9#pMF0Y^&4OE34w?G@df*@&qe!W!;M9S~p)ILxDf%#C8#Z-W|XRosX7= zyLz)lo5?V4d&8acP;go9v?->9kdT=2O~}wiJMuczMid%Orqdw2W1kBbsjm<%|8G6- zwGYuvBfT|9*+Wm&_mhW7wxj_2zXSw zNBPVAqI7AXLFmoJO~QFBxhYG-zN6hU8F!{3I0db2Ts^)8%yC1#N8ryhEH74uo`u5= zPiZ+hAPB96Z?Y62O;*w$AbRz#@L1M_`4{9s>(SE-%2lCoL0+yiq5%|P`;Oz>QK;;Z zC#^O{_!P}d>P`h{7*Xki$W}{U%VWFg=+(^n1o^0_yk8exbt`UhG}A=BlSX| z3i*os&vqFx-H#X-5DA`(q!!yDHdG#@E_uABAkDJM&o(1Yp@v3s@3Bvj#-m`vcl-naSUSw}GffN--D_0K_JcD^arY|iq#pyX z(Im#$O^&>AX9=t09t=&WJkFM?z${Wl8m%J4HK~(Au*#E{h3diGSuHf1Baj4_Xy0B4 zSBhC%e!&W5X(73uoE+o-ZZ^a-Nv*`;(r`R}b#vXsetNvK08%q|>wyIu2nUQ&m)?)R zz*PTdK+ppoju);R>aEb{Y2q-I;Hu{W4QWmG`C5gQi*G^4qczYm)T#7GwLN+0=&6`~ zH$a~m2bYH;ZCRwNQTt!QX>0haj4w(u6r&~BG{9uVE0#mXmP-p((}*RN4STTHknq*- zky{p^TXl0OFY}v*Roctz~)X75}o11p$zZMYlCnHlXCq#;tG zp`fQ`z>EDxT6+F*q+YH9$$(;SdX39v<~~nD#-tn*t!_O;aPGQAN*a7;Z3fAt z0p3TXRyMo)r4*MH%OPS8lGIl8AM*wjsUHHGRCJO()I~6K}tbQmt>B+CbF3iG%09} zJPij}F&qsDlhI$tBpt(N5`tRi70GAK6fTFBbCq8rR<=%9q=K|iV8L`2_b|8CNw~xo zPBp3MfJyM0BqFcqnXBMpW?@g(_i=G}^6Z#Dlg#T@Ajn>MBaNW?NTXr@R#yKnZ5mwD z8~TKnYI?wh9Uan9n=-ww{E$KjD1>Y9uj;81VVvnrppx7P<643?$DV7l?$Gn@*CWro zV-6F2wc=>YB6T>5WJgDn1ZxW=&5R%YS+qG2iN8gVFRQ8Z3Ib ztvk_SKbd|{dCa2txD7A61*#^B;+wKam@X^LP46e^$#XYZm#%MdEsS{btZAt5N44#j z&%-6527mg)i9y@qc*hNhhEUviuPY1^8Ejy~D?GmnQ*r#oBX<%v2pH-wjOaYg>%j77 z9oj%@(K!$k#W|xv=GcJL!N3oVdYrkB2Gi5Pm#`P@+F%HV2AQ7U4*4R=@79}ub z(V2_qL2k9=3}Igq(;RC9ALWicY+_&kyrdZ%nlJK62y;TmQZVwA)cU5K`FOkcr3-@u zt)ZN(bpUTFnvv0`!6!CF+-t(c29CB=w8%>R3OSC)-SSl={mhR;Q=o1pb$XGr!ICq3 z4HdDQ+F{xbN0bioEQ>o?KGQN)$&R%Jv}HPCQ^cu^1)E&UTTnfFkFImk0yXY^uYZi+RO!}tlsfY8PWb?Vy*mrlTm;e%(_<{ zDr_25)|q4>6A9)3i0XX^Ll=}IN~PV|OD9G!`u_JActz0N6@^nH1bdk_QO(oGS{+_p zkGX!H>D+I3IyfPx9J)J`s1RC{CCR-NnB^QbSVSe8%~P||#`wcr$;C9IX}c{ zM+kT}#VfFziB|ECF<)WkZ>xI-K`!!P@a=Is^!M7>(X#@4X{R|AFil1v?iyEro{{Kr zsuW+{wK5($R9JGzMs?6jBX>5>AP?ez?f3!3x&ohoeOY}oI2zS03ld)(Ewwm!!p`V)`Gs==A~*vT~BNU zDvGY=%|H`(RbLkNDl9H$YFE#g72pC2*BX~e+~Rb0-TTJ*;;!g)+1a2bNRP||IxkPo z-Aam%lo`t?Xfjlag$!LVSA_J`oV0pMTkK332e z6b2PC1-Zgs-V9;;AZRmqIlSCucqbA6boYF0n3v4`k>@P(Fv#bUB~PFQqJk&j6W$wV zhGE9(@MWQOiD4Nka8_lmPMeOBAA0p5&wCmWw24}sf2fVS_Ok2LneaI052BiDDgy)w zFo!=E$d^)x@uEo1`5NK;rdYEpbUX2$Jj%Ra;Y~BubfOrs$gOf@%hEe*`%Pl!fY4U- zO;6@){GesKUw{4Cr+8>9(|T+^@J{=6<}gVYz?22y9s)T#3Lybj5^~I4ibISDPidWfV~P0Tk5xB!x<(oF zskkp1V>>M?%G}hVCnu8*tOV1^GN2`PCzo`>XoH`9Mt~BkO$$W)vb~cZkWezoTPS59 zfM1%UMgU@Im{^+20t)LayTAbGvl-f?SFya;A7V#q<;x@TiW1xpitPTR8BJBv2s?YU zBeB$MO8-UO)Ugz&ia9>Kk@y@CY@QI&jKcpm7H0%pX|hZ z-VO%Eat>910}oTzlFpYgU04%e?w&VqdIa7X8;IsNIL1y&G^wT(B9Z1j3QB1 z2>VWl6o2PVFwH=aN9nnf)$@&a#Jas2&Z-RCvnwL=Nr=LKgA(U5YETAt<^Pcb5n#Fc zc+dsuC<+9U6kxYy5#l`5WZU#3<&5kT;d1mlv_SspzHjl z5ZLas0va~@bVMJb+0es`JPH8ZL4LU4%Tn-5@9$KrFl@fHeIrJ_R~IL)i`xA+OpUrh z#c^2iV3gs|bVd|a3&8c0m2vl@1ZMyR5&;7<4>?@f)-PHQ2#Eg&#qzsTd~6VLXfa)I z?_GE7&|`CdiCF}KY;dW-eTlD!Ameh#zBY?%=j#o6K@(( zV&mig<}s4_^T|Gi!FdUarM}*BR^Yz$eORWQ_-AGtV>(<=Z0u}$a`is1zCo-94kHBR zN-sSXgR<;m_{fM8jY9lni?pH3#N8BxvGEmkF+=Sj|3o-y3nk`yOu5pY;XTI|=!qZ6 z!Jv0tFMyfyz2-dR*N}qa%kr*JD=4e|GqSDATY^5kAe-v`t!Le1*Nt5O<|q1KCXM2? zGn{f6^+{w>E**$oGE>ox@;OU&vHD$wbCfoppIIG|f%AjOMz#;^rq-=FhQdAs6rk}R{1eBQzk((P6 z8v@UJQ*fA)DqANkkzbZ%$*&U`H={?I+1k0YpAd(k& z-16E|KEOQ(9ghM^UbK|#odpfYj&62%Y6_L8lHc+a#RWP+&?3?uIlWgWQT z>j*%XMgCE>vnZ*9iK!#+m2!%@dQm%goDX#@G}guo>fHOIUoV>-$0Ed|c_)>%)Mpoe zC8}O#uYbWyAXX{)fq5weG;yg#L3L*p@R=+}iH}hE#v34q_x{MA(pF~_C3bP;$Q^tI zvN4P2Q>o~akT6i@Fo;vIfCTht0Xj&zc-6Z}s2>@f9lV$3A-x+YW@oM&bX~e-<%t+D z1|=0|cj3eb$TWC}`;XCxeS&Vnvbq)0G7V2|_}4t~r*zCfR14I#haE5!nMLN9yLty3 z-H8AKZ@%WJo`U&KWrC(XA+^F;&R3p>5giQS_ZK1^Z|<2I{#lGnbGfIrN{bb!LDHv! zCxIrauSjNrV7xUld;!6mjs~7Dkkor}ol)eLS#mqTKV|ZFbg9hg;w*E*Ye)lKrBp-br|KR^}d`P`9hDlPcColNQ8KwD9FuUN;USy`&4O)^$3gI!4^H11RjJ zpUAgn!&6n{!rWHn;SQl-%U4Xg-`kb7^ZvwyU^i8`IDiIPY?Oz=)D3!Scv%IiEQcGy zYT zWKEFh`dF+&J?i;?yCw=JZ%N0vp)=46)q%>?>_xx-UKgaMPXpHqx4QF?h5HyiFPeXg zG6$vnF2otKqqLy*`SHv1kp7Bm@lFFtSY@=9oyIo0TM^NS!b8DT6!0y`S+u>59W{xi z>EAl6BxV!ny+`Wy*ayvSGdct0$9s(ms1Lu9oXvB^!9TvnYBwEq&6Dk7S4Hgj{dtgX zbRxiLo1KF+N>~6RS~Yjt^S43I#MCV2Ci7}Tt=Xni7J|QGd0Ud2@g^c@)NK#muzLDq z>?qbw5uL(1i1fksUrk$p;mB4ONt?2M4TUD$WZ6N*!v8I^gwA&aU_6=DY#!TgQkpJA zT^jhy0aL(U|DO7;U;r%2hRob+4;D_YcPmy3C%%=WZU;jLZ4=RzE}4&9FWdm0y;WoY zZzfXEl;HNp$h-uakXQ^wpu5u3Hph~?h}iGY1J^H}KN9T2<{gwR(k4wiHT+I*`Bo~2 z%VH9ELc(>-@orA6?<3-3PwxH5G4T(JOy*=v?bitP+U)~%`aUAgE!IuyH` zM&r32TH60I4U~X7vGFCJSypAVo73XfX=PSo+W)#z%F2z@9S+K@fDC}P8EbLP3p>Qj z&J8_NWPOPr1T6X#KgEo330UAdCr$ZKNRYIN0)#^Jylqs{1p2xp3pw5_=hi^j%YfT` zaJXNcEq!Ysg(n<%WBjlvqL~4Zx?}G76{9_6ms)@^E7Z^}f>#?-b}L3&QGuR;z7_@{ z7QvQQB)-@JX_w8@op?DL-M%U7F)F{6gG=OMltj2bxOa)Q_TrLxY;IyKIyvOoGJy4X z*BJF)LpO(oWt$3UytTKcR`(#cpJWA4z((oXVY}_df|n}i?b1|#I}u+4m7Dgz*4-FK zMS5hbB2aU~CqbbDdH&lL!WI;d^%0#=DcliU0-l8D>a}EwUY@$2XqmOK1{#<)Ux4+; zEw_{?a-lc@&n77t3oJ*AFRO7BThn^ALA0ky7ZV!E8m@y?Y2$VGEmJM#EhQW?}HaK@p4eoy`T=_VqzE^Z?nAl-Fk(M{# z3KvxZ!CU$`rSP~2E!y~coN6X9;7Uie($pxBu8`e5ijYj}2(tSZBQQk?BpR$jQuGVv z&wTZHXbH?o?5!^%ITe;e13wx4Fj{^+6-Bcvz`W(SO(=`D7i7XRe+w)3*E^aJ5)Zf+ zk(@!jTUSSM4~S8aS1;Le))^`?cx=a|S3A!tFHtNG>Z9@yzrU9g_G_HI;L*9Ni$u0G zq%pIxGnpbDIx|SJ;((2Ph(AW4;#b8x-HH?JEek%iM>uUhT1%h)kG9^ZlvH$LoZ_yg zqTailFJTr6Z!7gGdyr{6>hW8%P{q*tR3wudhrg#h>wrr$JY&@s3Aka=gS5{j@{mB1 z#cN|1TH9R1)!TzdOdmdEWZn{%#nIWduCx4rjzRBIR7hXjLbHN%iZZzI6nvGxHeIlC zywvM})d_{Wz=CJ$1*Uo>-lwW@L8W$A&(F%?!~H(y>3~)UHy6_JbFv|j8Y^#GsdBtZ z`PIA~@XeZ_V@@$PGKb}5mIKeAv~rij{zs;)Aq+>={i$L_?htW6*3CP}V{nKvRL;uD(gmP7a8Nes0^zoJx41qj zl01&o63ng^&gpYxS)>^wMGymNhnjMMJ2`l}Thv`Bw>~Qky!FHA%pK;3BUi50F0Y&j z%Y+n|!co0-u*{DIST8D%-AJ}o+%fd;4>p?B3wz9$y&vY>X`H~k{PjfjL8r6>ao4TY zHFnaAm`>ZBL@mry=S7p;8J?&x7Nx+Hk2SL1zw%WTmKuv6ny?wXfaZHoZaI=+rYtGi8I@|9 zS_CMHE+is8-M0*|iKC>db)Np)v%#PmS>VE`wC1~n5;a)B#aQy}&)pVbbWv6=IamFG zN9OD=&2gd4157cS`76o<89BT!u79PW4vukdC%i?P^hIYx)9nf?*q5w!AJiq`$Q=jA zr+sG(56q(;7|>am^cfcsJx4q)Zzf^T>vLYr}VwCJ~8!=+qYV zmO7WI?>jCh_)7ar9FvS68Jobbxp9(v5%M_5`X(ufi+3aVdu|wMNm{2)zq+r zT2#N&!O5|y$#a*hA4|p==(}!2*#TkdFolN!{v0?U?R_CnhaY&9k*Q-LEWtKN$NS?H zDSkmV1UUV6uTyTX&pxXk1*E&}v`o1Bjf*{Gsszh#TKw9pYfIPziEq5Y^TA${^`(09 zET~G2^a~F|P({=-%v)f8HgIO2=6&^M9W0Z-b62!eEn3*BCEZ&PiK9wE$CB6ha&_Z0 zwlKA)dU}W>O)@ERN0?#$RhN3HY*6=mbxD;0yu*kM8NMVNs8E%xlDihW;wc`eo&J(1 zW>k(S%ZhsI+14gWJQuw5NzUM^0W0L$137__7R0Hdom`J3IA=l8a(f;Qrm$JYUQ3!E zYhTRMFxmzSy$!qGQ`;Q|&J28WC{@k!dm98T@TC*kSU!`UxdB=I$!WqqMZH--avrn= zK9INHW0~NcMigmEnvgr7>ts}ClgvHoq;@i2sTdkLD|8md^$^&3DYM2+*T2L{X z_~0g8|7X5z*jU&SzCDG*(vl-Qz#|Fl?ck@^%7CFzl1Ne8#I?@eBc zfwP{(S^D7M|LZgs?V@xW@L zZ$+mcu_a*oHKK(WLr8B4`SKpYp8wp-d0gylfqrt29BHiT%_AQuS zECC#;XVy62S z_lyL`ok9bHl~XzloU3v<`H6E$z#VV->a?4G5oL13T|L>qdk|6x0_H{~H|jFx=tz~j zFOG9pI-RDtdorRBqE}*Ngxhto4vo8mGGsE&xZalMFFSz?R(jor0+O2Zj`SWt;Rv`> zejICyc0vPKux`UjXW6ec%BL{h>F9e=e=|hw4tTnu;GoA@&U8gw(Yq_=q;`4_P3n=x z@445*ASOiZCgt5NMJ7*&bQZkcOH3MjgTL%etYrS6hE+=5eeRXF;2NNXsNpiXY9KQ6 zU9Q~VH;v4a`0Y9}lW}6qPUuU7pv@Zi|NUb{h)r)shD=^S*E@YQ*!DgQ-A4t1Fhr1X zjSMCGvuJz3ZkFqu8o;ia5{y&H9R4#(Ka4A#Bym5}yG&l*Prne=0mwNUmzt(oo8O`< zFTRxfb#HOPF8{r&G;CJfYTNQW?=x(fWkkO2D?N=_?j(D*S=i}=TwJcuPL&$q=EfLruooIG%CHcR!B}B$3gX;@tx5y~ zTttkqJOcz6r7d8oeBor>k(~D64s7tyHi47aT%kc~el9xRr+u;7iok&Z+PljTxkNZn z?i%%!obLi;u<_GJHSG3co~i1l&M!3iUXkP>&3<(N`Xo-$y)I&0!Y%`FKHFXWP&*e> z6-~Yvc5SDn8FJQm5gJa&{HG5!+mRn0s6w>|VwlW_C&9X$?M`J+OK#EM5Aa~|-Q~9(*GjHDd-g)JUQ98Mx0(9` z!5SrtQ~-7*O0l6e=$HsOUNB_K5_`Ld&@6a&nXr(<5N#lK7rw``5Mm#~p9rz}5hEO? zX4M!*EU741X8%qZWNKvoJh!aHYGF>;3iai$M?5Y~z*u-GN;ElkzbTRkUUB?3B0rTb zlpxnoh5Ud~7tDfngs*DwR7OPyNCH6q!>-} z){&wEGwl@30tjXof2d|~D6u#K*a6hRhr608 z7gh)!BKD)o4W~WqV-$9 z3ivG1^gH!hK#bRhBM4*Eyo(z+5*cjQLiybjWZt>Xhj zx8SXFbKV|C6?7<|`d^@_TtDkcnlTR&(aL(Elb1Bd|1Q0~To~V65pGvWP|FLa=g3j!-Q9uQ;>K@52oT?Px-tqj(=!5F1b<#10`gdQQ(zlAvutLt5Xkf!+Z3YPETwIU-xK{iKcNB*~ z&*@>c1lZ1sQx-EZw~z^(A(rY#tcO6DKp5k_ngfOrXI_sji0t5ic3C$CmAoU#k~~D@ z$F45;!-pGoI=@vMK(4s5oyIRGy^L!NLx;@_oiqUz2l-W%r8;Ua7odLwBzY0|KPFPS zqMlf3c9s>`(&yes5`47fUi;k2k8S>p5Imd&u69pK$+fNYMNAIW2Z*|VJ2(H^EB5gF zZ-rjt@@4g@d1%S7jZcT8v=Bk8S7$c=FM?%X8qG}J1&}qB|JDs)t%Zo=wb(j>BOnu= z>BqU}&jGs0=qWx_4l%vcdBuT55J`DmcnwvDG=HA$q>Srd76V_5Z6Nxrb^;7PnK;SRr+%uWSd~aiuPC-Y>*S^nlkle>D-TmV7~5N3 zJibVZ>`)-F`=2@5R$(ag8rBZ<*GyW zzx6>*cf-YowtTrU>`pwn1$*%|y9sr#dlA>E?o|t&D5rxy@(5A^`r6`|+#ct%#9nmT zi3!Ln*mm!#rCE1MOHXF~4yhfC>uQ^V1oj$T!M3bZ-bZrpHrq=$oiL54sPSgerIqmT zNO{uw!JW+nI&%fJ##ddb^pd1*yx!;epwdfThA~7jGl$S-eM6f9gweG7WV1~FS-Zfa;_=^-FZ~* z#|)Ybx4ytajVv;!0Tn?r7P-wV?z%Ay%+Tk|oTa}8OV z{{+C=t;Ai0?L(4!+SbqDfl|J?^&Cz*CzICS z*>;_7io224jW46BF&cy;eRa7EK6Cd`~ z5$^Lk9Q{Un@YcKmF15Zs zl&mEk9=HH!$fE7?2q?@84dscT^)V5Orj|z`vcF{t{ylLGW`CCH>6jE1_ zMGI@HQp8rPPE;e9a^x&s!+-zv1;c``aE+^hUS zYwS$8N$vM5N8s~?ye$Efc8+BL*8&kGj)673 zRJG=8zEZQzbd?bNj4f>3{)LG`Uwy2OEm)>4!IJ1=qLiX=BNLX9oDj9I=$lrG*1!0f z2tw49=^TU8OYs}Ajk5@yMrK`+&j_LPkS-nSWCp>MiY|D#SP5PZRE+821iJE9P=CS3 zw1^XHrwka#W1`IjQ4^Rlmfu34cK)%?MbIFz z_=@wa=*QFuA>C!m+`zVELxj9i^cv;$)Q%2oQ3D?U?xXpu=yQ3zQ-e?Er_zVkNu+4zs(-^1YFk4x<>O_g{0X>YQmCd20J>7X|FQ|v zUNBtxUA$4wk&!1rYnd@wcGc@$OJVb5zK5Go{9!qJT7YJ}$po3yy_8HvRcwPBsn!Ow zkj(nSA~>kjc(53PZy6AAcekj>VGD_*aAzdz@0GhpWG@!YkypYm44$3sPo5Ilri87%= z&W+?*A<_o*EkGHnO`_FSlNk%w9B-x2l>Yn0_&e)dVO zX}VO*_3jTMhFUpCE$r~mWAn)h<|b>x)YIwbXbmRqQ=;jJgX8{q`-pSug*rxZcu4Qs z(qLM>Ys*w(xAv<%rvl>=k`$G~0bQt&pEYtV$b&n0bG8shF;*C@wQU5GP_evz(b^1_ z3rsp&Yq{Ee9&b91D#QkpXUGeEeuEki?$w>Ld$guVNr~-YB@5Gl=-E>#3Ejr>coz-n zl-Omyh6)xBBQO((c)|+unZHA)S=5Y^&x(tcpL9HqgU-0bC`FIVsiUj)dn80`Eep4t z%nvL*0`RUo1tx6M-^-s9@8;I_S%u#c!RqF(wLi3pgY8J~Om_0K9upX=D{-O~mms!9 z)}E|z2-g!a`E9JjXCaMRgZ+TiX>q76+W|o``xvr}e01k5WKd_;3w~@mBI81dj?149 zX$+(Ma=>l=_c7l?j$pXUi9K)Lw8+T!No=<9&70k)oYz$_{O7P6Y*&tM`oO9kRpR3i zE_@GcX{G+Ig}nszfi?t6v$cg%{4ym+LaCb1`|`)yS0rv4voa%K1*;Yrf_Hy8{m81M z_6n2E8nrQ+;ZdLWrpv^&3TjCu(~^z;g8UD)=40N#4w`gI*$&gP_!((6K5G}xmD2u^ zP}Ipp;qEw8z=%I9ku=rOV1#yD(;Mf#uu6PyNtzt-BD_T`Mzu=$r%0?xO?~_)za}%I zXTf7TnLms}rke>{h9lXIHcpD~mPR^SYRT!h<7u#YRqk=5=?s1dMlHlE0fe5GiRuB_ z6A)j%D^KThEjCW?QCWJKBB}J;7*pkwzu#OX;Xvp@0~l(&+WLO%DCiQ#=B*z9A_K2r z_0yCN$Xaf)F;E^6Gr9%lxnO<*<*~|0o)fuv_*0%bk+ob2$;-GZ>hu=P9|v+t2G{q~ zlICvdrA&>cd@W_F5)e=|MQ`j!ookAxXZdoA1>q2taGi-c2_a1kircRkR-SuyklPz3 zM6Ua^7IGuxivPkXzp3DM>cR@l3+>)mCs#u(!Bc=JKYyiH!#*#D`C*SRpkaSp6V0Fm z1d9QhYMl#s2wja+ys{?3ZyL4+yMF+X{kbYa5y(xji!hkp-F?SuZ?N-C4pFS>sBiD; z0~eZS(4aqdxlKB3+&}Q#-0Mru-YQQjGk6P(g-${xezPdqde|D5%{bAdVUwqFsr!u1Wkka!v7YQ=Wr-rJXun(bi|AAxNNLL}Lme+9 zB{rB2Ii|_nFV8r>%?2Tvly`yTDS+T4ZcS`~PA2aSl+lROC(yu*s#VzzOCHdM+@B0r zsob@;u)xli!tHh=rN_W=>RE7xlETX;upn<8-Q3cGJt&>B!YIZtp9OM}pbMH+?EF4;&!=VzqJEH!VC;t|>sXO!;OZJ0R~le=L6> zXummAPSJ?>M)JGjfZKT?U4bS-5<-W$;~d)G?i(o@Uxs3i2qf{OzezM9ij6N-9C$A` z-8W$P?LrrS;PZbbLx3YTQ|BWLrjsV-+^XMgQ(5HN9p^HF7x>%e8qpUytSj}et2K$9 zp+VRe9Ua6#Mx~jd7(Eyj=}k5t5+#NC;t|#t1dT1gomynq(4MGSPY9I|B^j5J~|ifpWmA{Pl7ew(1nl!P6IQ~XHA+4 zr2G@O=n|bJ<-cR9FBu?^ z{iXC_9imt=SrRP(pVC0AfVjI5-nfg-OXF;Nx>wu;l!BK(>V&BoF_Q!0xZm982q-Q} zI0PJQbRwwu*DG*Ss@z$Rpl&~}+ik|!NYnWoAb(Y!QvO4dL>PuJN4aHo8BV*%AbUOm zkb8EDhJ$^2ZQG?1|JQJi_)=I0#Bo?W*R0W}Hi`od^-2c-3$K&Uip4zUmqCimRQo9x zK-S*&0ov^pwk&9<2kDBq6O7^r&+q3@q8ILDSnNwKQqnGrmp? zczy-XDgXIHsQLmg<|k}cD@~qp><2il;!?cteDmn(8tmA%ZdO_0QzK8MYrUyQ&eO7< zGZlms1R%%Z(l4UuDy*QPCzpzH`s;q+_QChdNnMPr76)6*Q!m{u-tMkIJ>2z?^DZpu zBq`7A(`7w6#&H2Rglu0HVxce_|3-JwX!;!%M8PhOb-=!s%VoYC{emdgZLjUP43!s4VD9ry$rW!k$_w=qess09Yq8$1W;@LrB+u9Ia8zotuzUq>3i&@Qh!Ys7prHv< zgEwGqs?z2M1wSOg!wWv6?rT6sX2?dL!6;m`JQbl*zJQ`lPjpiY$g}oO#zL4e6@#bd zvtE5uDx*!|L0KxQGt^&&Wc1Q-{!V@=fS|H8_rj7$$UxN{5jSaQb6XyD)=&v{2A<)S ze9P8DM|(09LvgIZK@1zq3g4)f@}{cu2O4i6M)+r{XgQJac9vxU{4|6y-SS0$LZH;S z$ilgZ4A@{~`JM;|rT15sDwYS^7I!A&bexofNdmb)~q|CnSC$!2( ze@QKh6B33PxgR@Kj%>aLAhhAL(nv#P5o`N8>|#Kw-Mv8aRWn@C`5_Jy#^auNSgC0k z<14vnjXXSMP=$Eexvs4@ZE^a|$LCp2+Dq=~Q-*w#$!+lAS`P{0>>r)Tm7i6U}=ymmyjQd6p4Ww7W*xpePvE-FN z`$xdfgA()?T!-6Djj)0>5d5irv@6+L&(=G!Y2GFjO~FJt;9z@)}nJ5+9Z2L9+$UgS})rku)#lTC|6 zZ1Qkrn6%cU*hXCLIoTfYlY$f>i=uOExSxZZJ~LcNV4kQJK8$DbEx`8Yneu>?VU>Wt ze#3&LGQ2_;$%{LVvnuUhVA-LAK|m!Wq_mi5cInjHnai!CHnPMR)c6Lb9?@ej%$fVe zc5%dU{NH01MeW>CA(@dHxCXC|l#^(h7DNaRm5|a8h)vj7qf}5;g=Nyzhd~-GvF}pE zs^BW%joZ|X`T_pVpq)SAkOPlr0F(8iVI@47tau+&xc|;p&DK4jz7rU5bLJ5OQ=Kzo zQt%2d{a9bHrJAF0K6-Rm#+MG1FY3F{bDs-X+qflQ&T^$Bk@4=Kq%tqArXxKwbEbK` z+?`QwdG!&OYiPUhmHjBsG>rnFugick%Q+}shyY(wqekFjX`pzP%PtGR0QlcXPjs+@ z7q^OoEmU)7qb+(JGp4$U@IR!ODT~EvZM$@BwFiz&^!u!&6+n&ZLy@!Ky>4}t=_MVI z1f@%bw>B{TQctHVQ4oI1#|{-e?R$1?i>Mtx0ogy$h7*cFz<`#gAe7@K$ZKBO$bMJS zcJebBbb=pNJT;)U%tsgd@C>kYqY*n#kuP+&f3I*2y>rG^ekZ_JCtYhpmH16JM8gE| zly}kQFW?WRJhuXa;|Zeu!}C2wr`E`$0pOZ5yWOWScZ7g2qpp3l%9|vu(L+@1(W@v*ujwG7yM9B1`yLyAKohsA$VXZi6qty^aubiG9ddKKpi~qK9 zKuTmkQ}&v--1^;%l~_`%@75L^xH+Ew3y!ULXi>1+N3EWOZ-z`(+)chi!=f0_REyAx zu+;BRU5o`NE>Sxyo5M6Ng1#7{*OUyU{YBCRcq*b-($w&95%PD!F<=~ou{MiX6Ww0( zsEg9U!CtPJCOVogwJk62?Q(Gny=w!~yzSVQv;`Oe6r?x)hU4pjD)mQzjP? z$UPf7iILmZMU>8_EifGMD9!70%NS;l2m-S-2ClhXl?UwN^gIN;&Y&uj?Nd9LKCBh+ z^ZU|7?yxt(Dr7EPcgomKvozZ*$~9>3u}*rLVzcB_V=n6gv8jM4*3ZxS?%}m|+r62{ z_B(POODH};X#~U+f~Z>DCpaL4J)A}-q8!~>`w}flfvBEqsb*0r)h({oRO?7hVd0Hj085gg!OxI}RT>V*W3{q1dy_L`A$%oEpj*RHtd zBNk8sL?2rp`_PmBKbaEq@R!J%f!$f zt~W)-F-UDE7yR%9Gu6(hYfTMw;j!`Tf%Ruw+2>k)Q;p{Fh^e*)JP!_2 z^#4P6iAh`L#y%00)G~XY5JydD%e;w&t<_eIwB%beXwsuc($HXdyH~ah3R7+!QWXUi zqF#}n>aj9+^}MuvllwERL>EG#@Pqk+OCg8)fdi7pvq3w#wK&{zHR=G;!W&KYmO0h0 zWkx;#udU9H3RL%8!SnNel_~J#wI8SNafDK|qF@5tURNH!r0lX({fXW3@a`yOCHTA} zcbxhUvtH90?rMI)V@bOLKx*CGeLJ>%sUj~V5Kin-KTpEO)LHlF^FY|`r zbVKzls@uH$=sR1&El+^8@P8P{a*R^Vlfbb zF@M04nL}A#UmOumzC7F)N=*eK+VTd^R$MGpP^f0c7$Ey9|EZNO+~lVB*J(8iv^S0^ z!p)01ATdav@DpJTirguiDZ@1hKnbMk>SEqE`EI!wk{;st%;D%7xI`4*5Cds4A?#N9 z93<*6A)|NCLYYp71Wh8fA=$c#9pa=@Tr@BohGg<3*bdJ%U{kBHvHfas{T}^+{cxhw z^eI(0;=I5uaVGWQxAy8z8>ZcU(fMpVH`_S39$XY%{7Lq>|0oG}Qu^5IQcUE>d;>Ca zFsSyyTc4&*Mdi^cn)c?n)$q++YMGl+3=fr4`NHx;eRmKzvLCBQ5QodtieTa*#Y^=O zduQ7yqg!6d%h{dl{zlId)YMXZasp5$+Pi{@gcUBO(;|T1dDzC`rZVuLF7XVKLg5@q zIY9>w`E?gN{*+;QV(D}LR!*j^ZWVq5Eo2j-nxDfzHK!{E@^)eDX-Gu>r6N-dn&H!> zJw2}im;YbO@AxlsJ^n4atm@h97KA39>XX9UG&e}x*kzanHV>gFCNPjqRM#u3CC%a= zS`|m{%3JK-A~D|Yw+ky2i-v1s8*C5RCP(^u0)XROsN)h$K^{cZoLpcu>-oKWSN&oA zpQL;Y{WAn%ML~@5PACFuPBwTYcYIuTP3_RUB9H_^o%dt<@FwAV`Y(Ha+`-SnrFP!}% zrWtW*^f4e?!5J~k(5B!=v)=&b&5y{t2kgpTywy18JsQ}v`>?}b)OGUHd@$IC)tSrE zn_%{p?LQ&doLkoY6u{=Zj~a&I2OC3&an@!L;-b%@_}-m)WG^h7$aXg7A2dn@0JTB1 zrKW}5)~BVN&5Q_|pB7Mf%I?6)omP#}z-x;^`yGK<*y@Q0eXfH%k(S4dq>Dt!YPS=1XmLj>2#UEs5Z*-~i%on3{nj=*O28w@ozSPawajf(MEx?Duks%k)%MV{ z99-KmDZkK@pweOmh^B;ez)8Q6*N3G_%&D?W;RA0I$yz`nCF(HhBNRqS(@p3()rDl& zKsZr)Ty4r0>V~B1KYBa3iQdY1asf75Zy^}_3%k*ihByPjLPG&e{l;>xG=fgQ{s9hT zVD{k}F>Ac88pZU(GvvRmkms=%-BS!^20v zM;RcSJT89J03Bw3_7fE^X2`#nV$W6>EZ&~XcdgGL1-b3&{P~+aW*!W&DweSka!lHr zJ#im;>)i&V{(D4ibAX#MoD#g!-C8UDqGGmssyeb}z&c2o95U!=`Gj(>c$?A?e79kKkj)~F)O z;Z@mh!KNcv7vBkw616VU-N^SP4s_+7(0eX40p`OvAeNM??9DpE)uw{6AwXPTdV>KF zvhzeKYn?V2#kv^Nc2_?o9R#XCY*c273L+e!?rLWC`}rJ28NXhPWF{&CblO+3<9LGa zUHWJ|(m_)rDUKFcRv~18YRq(i!zes(NiNxET2+)II?We;z8n&?V_E(J zA&*hQdsrGXx$uUu4`AGSX=PzN76PeED>H1`bCyAR;JGU@#}$@$xv3?}TJeON#Aciz zO?xZPFHNF5jWL=Wd(h)=p{2Z-X zxY~#I%VGg$A4e8pMK4j%*e|tB202|~v9MbFi0}(78lB(rGshh7K@v_fA+~S=i|n>f z8LDXGm?uAK(_&3_a}N8VhR;v49C;lXb~1j+NQp?U%eE0WIss%`Q8qYZOb++4;b(odpLc{x znykzhsuC7sdm8>bf|<};R5VyQbizaNHw`xV(7G#L13wH^ame=_ngxEiPE7Ajo$G(F zUNIeT%xVyMNfCUjGGeJs(VVb0P>sIRyoiH4d=Q(UV9j=LgX|`Oabv`_W{HHLjFuzi zL$byJ8CO_AbMuBvG7BTMv0B6zcV_9xO*Q{nIRe)qU>!;a6E~`E(KlR3>kL%lEF3y4 z<8E8WM(YvI<6vE1S$>k~8kPq{yi(xTWoH(i2LoD1OtEB%9!`^EO62$60HEDoCF$Pn zDo;+}4dz9Vcoj)e(Bon5)lPsWLmU9&+eK~}1$hZGb(}iFCTjBq?Ow9fG_vj>5y{i- zkrs4*#t0yKM<%P)uHb!5?iJSZ`QUWUz&E`kRvCZ3*a`1)Pp4e7-oVJag`xMaFC|`$ z*|Ok5mCa9Siw?mTFSX>_D!Ve0qDg&O0M=d(8Vl*+aXFuGd;%lRvNzR0z^Zu#SgH*z zSKPwgMOCIo+J5q*dwKSnk-Y&z$+=-AS*8ExV2#@c|J4O?2cAfXILxc-PLxE4i|`fo zj=7d=PL+$J0tgC@CuD4At+TpmlpwU@)=ytGiM@n&5-UiLs_d`O_`@Q8ka*x(>i4Q> z;GvzLz=7l3ouj+%HgK*ahTpBK5f`37M_rk=GlohTG*?{Mk4JZ8m4N``kk|iP(aP7aeC5onR5MW8$1ocVsoVP& zx1#F%Rg-VruzrHG+i0cma3@^{5G@jc4{$wCqyKR};nm6!W41&JO`r2rlS;j{oyWeAw zn1&(v>NHxmml$UipB2eNbO>lc0q5(}{($4Nizkixn#`U>F8(`ay~L25GgiE~yOE3P z&F2H15@sDtzeJ%M@-sMHkvRR9JEYEY0rDA$^8FCaDTz5K`eFaMGVIwY)@-3^LhChR z6}JACGZNk+>4AKGGVX8Q_b_XM={Se-YViqa==nNa^o`BBcquEep8)|1$l>vVEekAqjI9zcqjMFh5yA@w8et(8v)E9l#+ zjSG<(vN2y*BP`tmwq#GU&FhdeBDV$mjp@$ZC6TQqnL-Cksb8 zaY-3cQbFdIVDkM@y_pyL7r5`mp3H&?}oyrSIbmfOJ+#4p%2nBad6?F>lqnyHK8a z4eqvl~99KX*sMoQe^A@RjcMf6u7e<%L?X@Br$}H zwWk)~65VSG)l>)Z^f;_D0AN{5I)nI%U2W~0@%BA)Ue(TbpKxP^wl3Os_U|8XKqD8{p@5+n6#br{unuO&T<2ph4|oc!cnJPzL=7k>9%H*Z};+HsC{h> z`2wI~JwM)k^GSGm<~P7x^i}^RY@js5Qt#Kic|q%s{D>ryDqjyY(YN}+?J^5RcH1c8 zp(RcjlLnR=o7f%OPllDk5IR_BK`4U|Uanf0j=-}8w%DS;tm+IjnytM#=2V%Aa5ACY z%tK#mwBp2TN=t&`y&#@$=J8K<(4z+1Ng}DZRzkexo6b94iua{NH=*HCAy);gnlP%- z1~}(hT6F@dh*Xe(YGcf`HGIIuK^XmodhAj2PG>BAX_4qFO{dzZT@h8{;mtA+LVRHE z{St~Q_0XmDm9GA$O3=oaKm7W*>NJ#=E$ExME z$0H*7XHINCSYP76T(`R~n+hs=8RSN`BpLB*H7ixbp@vGXS&9)7Xb}4RbFe2ge{1d@ zlF@Y2U5qPmO$XlOl)KY7m*ZHD(OcdUTGOfZeow1{ncg6ae9aW4P@>pMM}|~w*MO%} z=F!oicPyscmfV#6%)H*YuL?|GV#}l9Mk}s{eex3WHKFov#}=+z)-9*iq}ohMxIsl! z&=d@=w zvxthfZ*9HZOur#7PuV1b0w)N3=kIuc)ZFlJplR8`%;Xf(`Zlvu)t#P9_2=!sijQ{! zN|9f#62>ws_>{}t>r=`6w<+6IA~`2yyOS|@-Y5Usq$TIaxV6)x&swf$OOkT$+kDuE z6F4LZyvT$?94I)=hRD$pp*;_8@amdKZhenm8* z$@#hZt>xyy>x`_#YcjPsQSSl^(Wt?lzMif zk72a8;^fs3s+`8_e?kmFwTQ8Aw%WM|rz3kd7 ze3$>Gj>_N%4Cn$->e=nV%i~Bq$xN9U0%5}Ac@Z7s`K_U!I6`j0LGq0H1lRhzmZ=qXas(1(y z6N(ndFyZSBi(S`sDk}q(9+_z~CInRxOk^5{487|rE-F~7Bi32;%H0OmO z&p1#U9RBa~8u^lkzV8&kj_PL3MsBS3s;9@=^w0k{yl>)rxnMu%jX8ZHkF%fh=>;ou z&n&01@RiFJP5yX$JoHS`8qm(8W}~rY`M~B*wJ8Vh{ko9V9)TF8`nvA$9d*NqY*^b? z1Bqd`1IlVdj~&+~jarO7Fw`1(W5nhhM+hZzW&ryZz|KPrQ|wz%t&t;4xzWQ{_hagd zCt-)}UH@P&*39Y$0Q(hvNIFgb_*gIq3c1b%bOdp+D4{I)vOZ_0y@W8=AaHQ-Q$ISB zk*DbgcBgRX$FcJCS3pMWayv8V`%v6tY%I^13p6BXQQ-ePpCJlj>k;W^Y$1Y$!apL0R^iweMe|NzdL6Y zyS|4pa)bA|r*on~60A}p2}f8qVK<-FYcR%yfy!wuzqShfS1l8Y4#=Ao7*N+16zS$q z!u6^EnPlipJ!`=ZUs#*&W^>XzRLO5Yzu4Lj-$2H(baF!>&l*5(I$+g~rj{I8K9VyK zg(3l0mIvQCz47&xJ!kcqM(BxWTqlMkMEHR-Y$e-0CvxU_w_l08Li?qn$ko>7IK#pqqBKg>7*Y`9*XEgq+xz z-r|qLCFLc9VUe=At%5XB07mjXslq2w_Hx%zwr&T@!zk5oM@*NVCwbVvt{u;YJ2DUJ z{siV7zI|7{n{%n3T{i-aPf<(fsWl1-k4nY7Jugw_u?z+vnzsTiP{?r+ee{Z^t6UA5 z^LG1;S;DFI01`@x)*v|U>6!yZ=0$Lhm4(M%xqQRhyw?K2xLC%W1v0;mnf1u27;Uk3 zM!npL9ocG=*p9~x$G?5T>t5~r?yd8K6|jg-FQXAJ*!v!mOGqRseLVW586$QedvFJb z6i>KU|B)TSpFt9Ptv3_$R-fq<$N<;)?>K?X+4~r)FH-Zb5vPiQMS4UWO|I^f!f&3? z-(=mvpg_f$g-XHUabtBDo|~4xWw+~zu$W$zE2*E*J5Lkp9d`w9kANX9vv|bl65ov! z)n7rFyAS^Ht~v67o@SZY=kqZ$2t}DhVE;@t%9R?c_8RG zn0*2ZF0T1nEJ_NX*F%D|%=C+#qn7(k1$i1M;^Q?TpN2Vy+O9502t5ToPo3mUu`j;J;hZ4L;Q(3~Ekzx+K^vDj4BPL__$is+&PlRGMzY6(ure2VD#CF%1` zWm?Xp_12TM5fBXirpwYiHd`V!>^CzXR0ZU2uqa@~n&Tx!x-((>&|=I7mm0|8q?@}h zLN35Y2yOP}MrpuGh{Y6~)!3bftOn4&0ssm3yF<3n{7z~@TY2-oQL%^BUMb9FJ)EM4 zufs*ZnhIpeGx@X7vty-IDZ(co9eP=bGW^FIhmlBEMyrhXGWNwdU#|>vwk>wgQz$;v zqYUdw(T&6~&H2wIqcXQ0C|iL5U(%yTfEQfD#Fkm$0Qsa3aIIva)W6_={J42negoHz zOkMuTDNzdnrHSCHw#UK1V}Ng3d*a!00>h|lg_?U(B$tBV8)H$DU$a>5>D-6^L5fYs#R`SNG;CLIW$WE3T>$sHkF0Q z!8pD@`okRSFi$}txodCFZhlW>Vl}|3xG~{8fSN$UnSXfRGtP~4O}9uwQVBOeAT}Id zm}MA1fg@2WA-vW1d!x20?%j+q=UF>mGNJ>e2Ngu1r; zlp+sB(F?{#8q`8D+70jDKO%=+=b&iRFf>2VJ-ux8KyEW1r`wXf8#1s_8E1s63}H}0 zzh=9eZ9h`dmA>|T_`^A3y!GyjR?E*5YMepfEHUEVi#;BWpxzc3+!Oi_U#%x1m9s@v zF-H_7sCbJ~m7g&&t-9h#hy(*tEs|S=7Dur-bEM7?(b0(M%n)+9F^Zpspaho~63P8F z)6J*Mia<8?aPOW*=BCd^EzyyvMa#ZNufODnDt7J+h>Z%mw_4y)i(p91$C(hFE5a*` zw7U;|y7PTS)vK66>x^lr-T+i=UTMBqf_-d3PSZ8Az(u*Q{(kEjX*bWENJL+0jBAFR zb69UX&N#lH{HGW!aseA89cV_#jeN|d`y+^pHadKGqT$iB+MsNh3&k!uAGIGND_)8b z$vlR$?(F6jDX=pj{K~MqnAfdlaATj331OePG=5R+A8Jbo4Kzdej6>8%d~X8%c<0lt zX`F~b(z{*GGF7GvSFCoM4-oF%BSI>a4U&l(}dcRgZI zpi0Ps&KG{wO^#2H1EbLLUy59_>+F`4=ry$kOR1Gw(7mt8xdi(Qp zqY6*-DZ05^rr4Q;Ed205yF29OjaWP#Vaj5K92PZ64`efw{ZcfybPNe957|H_Zp0*2 zHx!xv6DX1>x~!F%UdHjiefvA13kZT7{H^`HmzTvHlq+ylBr@f*Agm?NwE^AOJrYVO zphj=`=J7|D?ycqFvDh8w)fJQm9hj9SUlByQ2j89qyqvt8v+g>Sgdut~L(cRLV5mt9 z*#^2LnpUe+14%j9*E~CSA$Ak#u+zDD;z=DC16TaoJme6}oZX89L&0ZM)y4fbKDjA`{1@AT?bQUZy4;^vKg*ogHs|M3 zGL8o|Kz(TAF(IeuFwH>-}DTc=mgzTbxG%bw-sz5(u4w_HG980J}Y8g$jNKO=pi zZAfVNnEPQflnWd3=8&f#l3WSLNf_Jlb`0n#h%DqBmx{csu#yt7^VbYz5Lc%CwnU;6 zfxbZnWmTld>a3pP2T&Lu9$?Jy4Ts-`V3=X9>K|@$3;4%8$7fAd{eXH>n#noVsw`b+ z?wKzW6*zLLuo(bsVH#~lw?dq8hCkSz37OStk#?7la)9!)cdoG}j-BAg{VjM~VdO#+ zXMRFP=*oVJ*y9Z^+Ga;|2>)VEXUvcv*Xj^wCkyxr00=|9?tC7x07gUq9Q6p+`>?2R zu4V6Yf@^-;YY~ceP`879tSyK{-rMiNf+eA)Cc@^AJhdT&xV(haEIF4?JJ%5***Z~q zgx$HL9`~Mw4Aka{s(6kMv;{Ln>Q?3Nj2xdT4pLz)(pi_Giysc5;x~sVMq-@Pt`yiS zkllwkA&Mgh_%AZJnX8i`NhM~(pyGzQq6gt`lDnzPfHq!!V=A&G^ogg5?sgLybd*x0 zYO|!5fpI9H?Du42DB^yCx3(+-Vxqbpqd$V@TIZo_e_P^FS!wCCCCM74u2?XfTtspX-G-*K{BPA#56Qe*B~Fgw@M~XHyY(up#79oF4b6G zFeC@9;_DKmE9Z6h=y=kvUeMi_J+E@DLc68w>&X~eeZGhSNSB-uYT!%R;@jouWb}!Q zR{SedLutX&2^)@!m}_AyYl!C|)9z)BA$>BGq3W~sGB@9KF2@KG6$a^i+CcM5;dDvl zKp%L3M}NvfI1P5M3`gdc95>$*h9T=N^R6`J7yC(2Y};2wlXG}jjJ)R_)ZHvW2Q@n3dZRo}ISxtm*$D)A)s3R*uTFTk?J8{Zm4RNOKqT_A1bz zz|Du5NsFAdp2|ZGRq9~gRuYM+H0T@%1qtbdo7-ovIIe^FI523;d}I&J+$*MkM_=7_ zAuDI%Qd4L>P5eZSW20KG!#?4^TCWZf_tmOT4pc}==n?Vm2WFG}BbiF3TS?2z+A4yw z(J-?YvIN{?*r;)7@39DTdg?_BhAmnV=J!2d0oUY3`JU@aQpg$oodOXW~L&{ zVEK7!9eLT}Z@~IdYCMMHVzs693CAKO-t2$W$v*?d_{Sq9V!4e2+KE&qVFf zG642W6jev~1)!fO?B?$q*fChGLjxcSA6tx%O9CpX zXj)ZE+CpvqcDe0CV(xeF#u}7*HeqI6R8|6jXk=Vi9oGTnFZ%#?PFp5`vVt*=5tPPL zSS?EGKIwXkWA+Al71g>=@+R!cWyVTW4;8}(9Qb_bGUpSblOb@^sGu-exBPORMqZ8H z4?#&0dv~%Qg{O}ZZK`k&vhoyo1FXNVncg*s zXTV5+rSLk|G$I)#bI;x&S+YnJ5VQ+RqzDUzCS0$ogTFG81Gnb&7}Gqr?{K@7z36%1 zrxM?zUnhDk1ul0X*Cj5~RVfUkt=XU31@Adc4pbJPgVlCQu2-FtOsOKBT*6 z!J(X_R4rYp1iy%S$V9j5QX*Pj%QlN{3O-aHW!$v2<1H2TWA=y29KT60|J&2389&$rh2CwUb>gy!M49 z+;7TPOgp3&hxeVga8m2hK2V{IXaQ}W9onM;4GTv0ONr(9qszgdw)Ec}it85P&pZ*-UgX&lK;A%Z|&YbiSf<%Zk#*YgQJXk)ywyT|&^Vm;DV|W#G z=^h^xMv2UKnPQWyW4Cd?h=ijTMXAUO*?qJrg`S@Fmk$70U-+xRB=$+a{1LI+{XAas z8Gov(=*AA_w;ytJJ<4dh{3;qE1H2>`fQ|9xg49#k6(l~|lR><04G>a*Lu}UAy=kFY8VA* z9K=3UG$+RoOt`R;n}}O=nTl*STtyk>mC|*DInAU>2Ng}IplGl%su3`<0D#W*Mvnj_ zk%fsYvw#5nc>roamA?|mnXEG5WHC2wrGi!OdaiCiALBNLl?ZZKdO?Bp9(9!F^#RH` zDUxIF`hv#FjO@p+hf-GZ5Ct$pk%lDB{kFxkZfvk6iL`A%Z3DQ*4!FhcZi3qysGLv&%v2%<;3gIvr$tZI z?Yt2Oa|xDc;_uJjq@eWdN@1`lOh*Zp^%x5Pv;Iu;eiC)5DuNgx;J02c(0|bvz}+y! z+8=)jLH9C9${DnRW#J&z7Me$a;;E-J_6`P4<&$2TgbhEvzmnr{pDi#isy)Ge?``RX zzKlV=p1v=GQfdY}aGx2^LX_9PGm8>Auc9t}qN_!1imh))238~69RY6X!r!Xr6$~`@ zF6`#tYOQU4iHpZ!??fV-5G8AsWzWm|qUh*J^= zd1AJdKC!jFdS4v}&s=ppJ*<=AO#MIYx$)n7GPb}ZmlpAUDJ8yYwo}Z{4`Zen1E)vT&Y4zB`6aj( zwK~?-lyc{`ckmt><9nF2zg^=-BzShnka^)BDX{Lk!GZhxH9i5+J2lL#6|RS)^KfKT zhbeiOGi)KQ1oM`ju~R|V#~)ux+vdg&Qs?c8u~k8k_~Dz%n_&+JO8rqrmH3gyFG=h7 zK<0{)7w$2P2caWF3J7Q^vrd#coLon^X1RLqi)3P^SO^hGEkq|c9r?OEIL;?g&7GzF z%^r|)>ZvIi{|9AVJh97^>ull)%laTIr{xudu)q8*beRom{^(X~(oJOH0SNR~5F?~HN zKOZ@>KVPjz%Q2u~2mCntcZ0`;4k|V&5zCZ`EtDwSTsxhv{>tlTU5tZ7347J=!} zN(_K`X{_M#R)!dT>iD~>*&=HENB!rPH(g%$7;%`GuNw?GM<$eF2lWKXhX$LKu! zJd#XukaXj%xebNo8Wnkph0YtGb1AD{WkEQJNoJs=XykJ-eoDzR)=*r5lN5F){tK94 zCa^U?=Aaj9It)P{!#1&8RRG#J4aFH}o4}P6?!q>>TzX~Je<=f5Lo48JznogSFR@{n zRkQ5eB8(T%+5#>v8oi~!L+dh-HNdj@v>G>xhBTO5R5|a7jo;;yVp_f!ZEP3#n?`$!czOF4)tLT8psm8S#$c_?R zp@aj!fP-naK;T3`C<)8FXwN4isbZAUw^f;UuA~kYaP*K1BD&MO!&5M zNnH}Yz^M2nZ z<@9-8mZ4M?PL9*U>LCtlO)zn+#=5pgqp7)Q8JNLhRpt8dcs=bX)%r|`6&hIZk|jJ4 z3FC;8!{eF@UjLyc%h<24RCoCk0jqpmv&aGRC>y0-f zROZ;`jXHA$6Y8u7%$556VPNUFH-uzHXrCA1X@l2!^pl=vD&*`)DPb=$dwZ>m7>_q| z(xQ8dvl5QrA()Kp71{}nHcUM-og)Nq6C*YS@r-!0%JG&b{T_b5b;SB(3?T>KR=DCD zb;7>qN2`fvE}Y7BVFyY5f+sD_2|PiYrg7S74iJHA_&B&f31xPIV1F&Py?FfWnZkfMtvaY-aEb-=qnAM~0(JizEi`LF)ktCOd&hhk#>g(Bg zL}2=wj}Rfnlzn({R|JI1`%%5-XGzFzm_VMV`ExVk3bCp-^tGQNLVrGWWzs0W9zm0S z7q4UYJOEGcr0W5+DA;`qWe%&%uqAnAHC$hpP5qmXyL9xJMPqXS{qliQA{*kqE0uHS ziQ`FeqJk}Lbj)Ni@kYZreb9F-`n>P9>IW%6-^ipcZPfacVgvF^I>Z)MekLTSp}a8)X5FogTX`N`(pFb1AOC$ zc)%M=fI2z`ZkjwE)wwN`*Gja&VhN7t_aMvmV~$=Nj^f^}4cq57StBfDD>D!X@~>mu ziA}lD<7ry}e@#kpIq8%WeeXsioxyAN>LXHr`976*wC%TUe=>r=6hp>1*5u^+x#W(7 z?|wQ$_zVwF+U_PZ7os->0E(&Q@)I|5$;G=6P%QZ7{4?n{C>5T>==d*?g)~&!Nt0r!wg>dz9VGx6W&Fyle12w`g?m*C*L+mavr*HW+uGr_jgq9=RWssf5c0+KU zl-fe7+=e1of^hPhy7xT^R$vX$m~+TDc-C8WiRKmA#6!xujhn)lU>LoMynzY~5DT`Q zh?L}rRQ)E+;pK^+!I;ra9~n3tWr&XrTIiRJRb`x$0XEBX%9t*~qr}175OM}<18Gu2 znGM0|MKzuT(mip!nN~)M*c!rR$tC7bZ(W~#~!&5H^wvya<3`8~0Xk`q9Fu8fS?ybv)? zKW9t-`Lxb>3t*fZRjyXZKGaKr8qaZ@S;_z8KG(PbMrn5N$HQH##as{CvZ2A`X3+vV ziA6qMeU7`#)75W;8psZN-bDn z%JdmTcf0Qja_&FT5PEdfWk$Ge< zC@J#i`6kD`U1&fLg`xm9I^aMU`3v%cQ^|`cesgZ)hxrhLxQ5rD;M)F(i@^LILR4{N z)Dm@iq?mY1H>N*7Bq|6vFSWwdr*PJ%=2UXl%9PJJZ;3(hFpbvcAa>2yL=Kj`Zora> zQ_uhNemY{@t^-B8la9dFEfMqMxxi@Xf=#1ji>+ zz*b4f2axOWzQch5+Qc~S*t+`0tq_L)_KqmcMBTT7qJ`YQfi5FM8x1q*2D*WCF zv$F_#H;(ArxZ&nMj_A|f*gZ8=hc?U#uWoQjQqwyfz9vl!M!2{CQrZUY;ZX8GA;1gX zyg(Hi>E_*r`c}Zm2|7myRU?n2oI!H@&dvZiNe26J+hJexM21(O6p$9W##_woiuo&@ z{G^;19?C~rQG8|wn|N|$gC*vf281;aoHox7W(nDZO6EbPR3I2jBVK*dZzJK}%OL*3mu8vyZ{W}dD*l8Br3U&Hp-?USGn5gM043%Qjc<^ z$#&ikwZda&j{qc* z0f{WLfB@IYzt;5}QnXR;tNMn5xjTJsNlbF|-}3kNH36I$0dz~Bg3Zk5C!`Ak1SsbK z#`%IsAJnhrqD->vDMp~E|C-YW9agkbQ*vEZNh(tE#-n;+T6QTRvn`KT8hLSHYT?oMus|;aSh~`P4Fj^ZF%D%>0EnoIqN*7y zyKO5eb-!tvZ3*-jkCC(*?gbJy!*Mc%@&bJpJG*UP!X;7o2o^jz^Dl^dU?jSnYoL)b zU2{STU$;Bk2%AULm(%arE{xDH_*1N8I1pS;uSMsT0+WDA)-lx?6@r8xkVoiu4Iol2 zR&7tExT3Aq_Wr2kHi)%9oSb&%*S2Aw*at>&=aa6nvqjP+S7Jm;r)|y}%5ZDUs$tG+ zgX+9%rV1Th)Lygh*qRzjmNXD{`t*rpRPn8~u#;7)WL`w-Z#U2RB-ubRo`R{?6N<;^ zZYG4;p;7u2vyCTfM920?bQ!7`=fyMFey-Xb@BG%LyhN3d4@Yg5ZT_^O*AkWiTlwL8 zz3ugW9tkVFqfA=ddHa^V(udKHU%SbcQ;`RmVo1(G04mG0DQDhBtjH$K-AvDfUmTFO zfF_|_mOF+AWfNGRSPNDjCWTO8@c`x?=iD(}6mr~_LROf{I{ zw9aJ0NM%IHcut1!PyB0#!vkSP?ZWG6sQplH98Xasi$j6~nQb@lPxfB?5(q{4mmWt? zH{`U%WbSXTgezw!^FN0UNVyj3N$w0MPDjT@5Q@jBwHY?0gb$Bd>-yZ%O3)@S><;(& z+cwLAFg`wWzO-r!tR9g)Js=;s*Z#I^Fe5=m;^TD2zhpVGm*5g6*C+c0NMV=T2VQwg zPj5o?5!FBHZP8jMHllrUsK-O_xXAjTQiU6fUz%k!f!`@=Idgj?Ejzd9tigG%A>NDJ zB~jV>EHbT+4$gCDkDUVU8EK-r=?DUZoU9i{d`=$c%T8AO;5+k~hrW6$m}ctD-MV9! zf7hbIJc*x=OlI0#EKmLFPiT%85(8zynz`lP$oA-qq#BeeMibU^4hKxmCkkv?)6W^t z&=lLjHt-R*`&|?UtZ%9Un1qRW;0yk~roGmHhvT;>VOH%!cQpdq_$?K8<^An)R&C|+ z^+b7L=ys*xy8i-h)qs@k?XX5VKa6D(hbmr*D!bJptR}*F)Ksn~qKCaiB5QChD~4_W z-MxpD2|23UjIY=Q8xU9KvM>0gGvtnW`}L4Jxjr^!j}y$x^5xe;E46T{8}K+A<(U z#k)BUxTk#xhGDV-B33_DSg5&sBHr^xxOpW1i0JQmxAs>GlBT&iBwyqwr_aT>3tkXG z)6S%q#d#gXKmAHTWkA**fJr-N{cNfI#Em;m8oAtyRQ{q?7e*+Ao7#!qU$5$T zx9{c7VTeN~e5s`VtRM+>=5fJbw^tmq)Ptfh&H|7^aGE)qAdI2MV4BFx zqRXBLsz?p7e8FD|MqC6t|1UehaYHB_fX~7C=VKrdE;Bz}%A}$;PTtT;DV|f?-uv4M z5#XrW@7~R|oww>UH-7~^6rg$mVIzc6DdGM+E|OJNw7Wf!PC0zP@lTw{Dw^JPhLgzO znML`B{=oB#mDuCPm&%yhs{uW>INd+YcCEwniOre$Xqaa18Q_!~FzduM!2UzgIo2_D zqGtlYqZy2o!GmviQ9NS5QfH#RBx8m|0bvg<@>gwrPD3Q@9imqMqJuC4v--Gn`wQ;M zir))vagSQQ*bi@fGDa~soOMvjT=X%9z&P-OfbzXPNz$0&nA)YyW>NG5`V_P_3(j{3^60Er{UJ0-j1$3(lu@79l z@VnXNx)I(0<2S!rUiYhJlsDBy< zIR4Uv8d-Pn7sr($$bJ-s)1Mc4=|v?=(^21#=w)JC-7R#Q${f!E;RRQG?p^R%@Ku*py%pmJ{cYAqn)J@Ni>zB6TE`#i|`aTX);qIA-q))F4#JoVWk3H=F% zd%Ot^KR0;c^Nt7Ui-as&z|#8k81Xa;O5%DMGnzVXfJFvPX%!Qe!23))Vv=^O6YLYavZd^}!|rH~;r1Z-du;E&;dA*1-CwI6H}}+| z&r&WW^YrcFNyxiG#Z_I!gk+zKtJ>&3!+S~-6M8sTSnKuU8U{@iJ&rqjTIj&r>(l z1d~b$Ew9crUpetM&YC{u%j$EX_UAHU=x4fA8Te2R--pwF#}XZBLcR(?KDRV};TRcd zIv%a2-gyQ>T&Fq;s z7+)Fz=rV*SC}H6Y4^pE?5Ms*^c#_LJ0G#PE?~e}#t5GQ0FUTyJ2d;O>FnKI~$W@^& zlf~wjBk$@@?2NUmX zv3Q;7sWlSMmwW=(ykxI)>PY?Vq+1cE!B+ts&vidS!mcxW#snSICG4I#E4 zb}P_}!}eLLX_+I`Y<SSwmXwu`_Et< zO0v@Voojhd%8rLOO+=5$+|f+XFL0Mk$<3dqAoO+j$!9uhSxZ3*9oA+_w)@Y8v4$UPiV(NXDSe zpWFdP<19|*C{jTnJsVw%b-)vBIWM||>0V5UhvcoL>v*BRu;N4nMM zTg8lrCg)tYJx$r2=}t6Cu`|YjA(RqD^I*7VBCJMPLYKo?7UYzu`hN94@(p02`(7KO zENkqU>}$oQ*(4$Z$8r7xAxCwIJFBxIaH6@0;j0U#I6v6lnYfHh*)= z>sj${L7<^(;zwD)OZs8*0(nS<@0OaFx(Z%uZ_;GctPnM@S4%68?J0|luMi_#dw$FC z*^5Qxi^uz?%_Uc@{+PWrHuU=56e<8~)-|4?AEQsKYwU`@%VA`yzCoOdm)U+}UPgqk z6}%bC&cS;3y}0~W9)E_DdK<0xAaJZGSPOxDp^+hAlCr}HD{3TyIsZ(%@=0ex1aoWJ(>~~4pWnI*GljKq^#uwcd7mf56Y?W4u?B1tg!!(H z8a04GedQdkxOQ4i@>tV+P&&OHnEQd*o^%V0X_ZJS6o>}Vz@XmLeC7M1g8eQ@qK%bM zs!WwyD>YwTV|u+~Tf_N&4y)&&gOJ>dUZ$T@?*(i;UOUU)O0JD>^B7J8o$f*T>7m4oOSVA#V_*dC@;Z>7s@x0ok--h1fQtY(DP+1NMC}@H_&^+e6 z@o6y)gI6Zg=vcv5b4m%l>cF+U?W8yaHwu_h1dA#TEOrr}{x zUa|+fL17hQyKw)D_axM;S*FA0k}QI-!9EU;c6oT!W9q?;^X0tFG1w(vh7&MYv5oy| zIgVzPUp2fHIC+wkF|(DE22zha);S2cNB71S9wT#2_xYqRLQ$d7SmUV^1K?QqidA8; z$cyU^(Ft-d$BaAVrYqJJAdfbp3if&jeW4MDXt14w@y3G%WKhoW3(2@UPxDVm-A7w1-?Pjt|1X0duSGm!<4jdFNUgH*m}5|CtrRkZ|28}0px|S-L%@ytEag2;#-`c7>g)GK$s{whd(3sLF%LqiCrF_^ z1eMxtOsR1Dn>cAms@Kt~KaqU4)>wQA_n3vjmwy$q z35H1)Zp_{O{?uPaFsY0rGZ@>gwqM0T$)Wh8L3tG<;#L>mF{&8@!VyCD}DFXuWZqNeL!>0~PRUwZ!=d{P&?Vz(~J zD$k~?20{1Cv1?0z2FV!W81XB4bZBLy7aFf(`v5?T#SdsOB@<(2=mZ|c5uGHd#Oy7o~E-n~<44B~Xml#Bb}`J7O(W*wl0clm;4AGgfjq%Sd%` z<-ElRzN_)@H|Wz8;(O2b2YO#T{G}&pPN3w+_$)l^08f3B>zocoT_@2b=u<1J`FWla z^)WA#TM&PE`~1RiD-nnA1oqRSpUp3el*g2?e)f!lP<9idCf#BE8usVoxo?m7LAoA6BLw$3MN0YKRkEiBL8 zS@HN4SXrOtRPsWmZ0~Qc{gVXZ-OgCH|-#NLOG?g65N zazLYj-m=Qkl^%i|k8aXu_GyKsTaFdpRxF1i1gm{FT=?lChU~A06;Qd}E$LFSL40}@ zmw7{2UIS1NU+y_S@0PvWUOc14Lpk&|C+Vm$6(<%Gqrnyh%aJ$%v4@g`bEj+_gd8r? zk-QKKs8{t`^&l#`^eT7X!~tdbrg`_KT17iyEkJeLj<wVn3&8PrIYd2L2np|f6#Nof(IUt%V5$-a zH~keE7aqm#1hIncw;y1v1)MVKm|9+e9<7A;AP(V!J_ulztuEHdBMzqJU8*`Q%aPUa zQ8bW=AKtTWLL5{BWN-(?lZ3$=_ZG$#w-Fl(bm}px!3VURSL^uBA|Fx$yu)WU?K70r(i!bI; z(GyY&{;$hd=Y&85o1HEgdlgkco^Pzl1Ykj$+1WZQ!(f`^KfZLR3m>HZj-ZteY61mP zSTQ1D3HkidhzdIzqCsNM3OxX&E{M@wbgOkHq91H$>eeKUm8qL%cR(qpGPJz$quda6 zLq2fH*~r$E220K2DSCDzR96;bKeLA9y*dU3tJkp9!4CO5q;H(QO|7## zQBMGn!I-!>Iod{$xxW!ZdbjiDPNg^{G&aP+^%{;W9OUKCB;=*YLHNs9P{7J#dBVSX7FX+EzDC6LhIgQID@(yyC>-{m3X zYxx(?k3NC_7b{ig^5{|wR?Mfb=$l_$;wbR`&(9?-SY_%8APpX-vf*EU^#QZ=#{bxi0nJdbq|1opv zBng4+<C43o+BC6Y75viL-7z;hUBdn=4nCJN^=SxK?`bXg- z0M`NzRCd!7ic{%t?V`7?WUuyGmEjC4!+PKQ*2ni%I;1aq-L1T}MhjD3c1?no*z0kJylx}i=(2t!CRaNio`H37h7 z>H7CY@-J{qAcqMT7?9F2pxxtOt47iHl48F)Xqt0H0(`~Hw25oDTPhWY$$WocD?y4zW; za|IZ!K(73&H>+o$jQk0l^fAocp<$nhzHF*Wx;Wwy38Rv8!M+rUoPT+h3o+Z?sycIqrYjb~q zJ+K>*2*BIv;SG0RGm6;frA%XqPqEimOHH4beUCqT3N|90&9|&9?pM4u0|Q*A|?>d zP>j6aC|x6J`%7Kqc*T~3 z!&>3%NnmjPis~IEvxIi1y!3s3!^Qtsc*X|9b1<Yg&hAJr@!ppPp(uepPz*KwO( zD#xRDlv3$cqsZe@HE!2&qEqR(cWl#Y6YrE;8UP`H3*>i?GO!Y+^7Wr%5AOUMR>54x zBDqkka1RmZc$#3M1~X>@H%f{3aw0Y7*4-YG^-XbIHgOj-vJYM;51cxmp#52%K9~ma zPmK<{MVzNDslcXqSfs(k9{JZgeq##sIak8-DHZU*>g-%Ys2q+YLWc6c6$M5Yn0_v! zd3$aoxdD4JXhF$kbMWeyBmTfqcJNa2?H&pCZx`G6I3JnlzcrP#U`|5W33pBZ)9@Rg z8Ie%z>H`W$@4ivl6c=Kh%Fu-?JKs{ee_MCl=_&a-(31mYvfq6#-fC_98gkiZ`w11ps zm%_iT5|^>yHg|^GE1jO&Jg+U!8-PaKuOFT48r_HWzLNxZJR&DWS`{67?|%Ugf<%~{ zK-6k=hWI4Hbyvssv}fBLMp5_YMv$NcABzGk8nhQSkVPDfe4aXalZ`jMh%5%m7hchA zS?5W!45agV!(1FtGW82EzCf5BvI+)eg}Km@@%tzXJk(o@28Cmut2_km_!7GA}|U{(Q68gAXzc4xTjjo0B8gy3B}^N z{(&bpIgoc&*d`Tj`c`cNUGvOfVq-se^|rlfnER`>8@qPzNMyD&h`0xdODlqrGF}^X z(2Am5eTz4vhrt4JgsgYYMRo>C z1Cf1B(XI_lB*V9mvV+dwoHO;U-K{k~WGLiq7QKR)NTpm>g4AHW>Y59=>o+p!ap{-* z_}APQ{}eIAv8j`5>6J!~tspF(xX?rj`Di8Pq1lHW1xnddZ)de}*<>07TB$IjRiBX2 zc!QVOqoG1Nv8BL7#8j|?qBgj_Ee%3v67@srMoF$9xSEyBtZpr{-Q|R&3;bJg3HfeZ zFbgTOJ4BT>%YFVGrS_}PDU(3U&@IcwmPRe7TYox`wl{+=3J6wr{+jTQE+HNvwAF{6 z#Yf>|q7^^YF(W6f9J=LD!GT)&FA_@*oqb^Cg%3PPUt^M;)-T0^tqk$ka9N8oDO#m> zHTnS1%tyaN6xGW8$%YouJ+x)tLE$R*PIj5*yYf{~*vS@n>pm0HkDQ+rC~hE2Ns9d& zx79V+9JI!Li4H!)A{4!5ffzYFZj$FSpZsPCm?-Cm`9)H5x#vaA;zO`sseYvXd9~qw z1ReqF<5bDg8m$E_9<@@Rh2oTOpy~-MD~IM>t+!{42Mi0E1cyhD`?>ddI}2^UVC&MVeBV~9dDB}1 zxD`~6hyd9Kt`|14DM^tXtlbhLWvgblNP%j>p(X`Gp;?TnGcw&H&7G3awdNS1A_ee( zgq(D~e%!-Rn8cbziYR67qAzebb-2np3q%vyw&ZK1wS0MZvF3e#RWzdKhv&3W9bejgCld&4TYKa5~UJ~cp#8OKSkA|#UgM`curcTbXapRr#=>>8Q$Q9l90Maf! z!0@?}o!NYXp(*`#Il^#CD;Vl*3-bXNI`zq5NqDBQi=4Tfaw8z(i*2@C#Se7(R^YG% zMtc-Nd4ta+1)I$eEhw}29I{k%U<%-aQj4iz(M2xiy+x~3kS;Xx0H1rFy`7_4Ogw#a z;JlSJtZDlSnoNyPD)pi?@5{hJIE!ta3?3?c%t4?_ul>%yrNT z6MwhKK|E>Owtm}gwSfMh*G1RbJhC8`_!u%ga2YS{bR!iL{eXk2gxR<0<9Z8erL^CJ`Szmm63}+ zVB@xSTkDfHY8Zu8-@9Iz^ps>AEDg9`IM9)_V8%TEEnOyNNW|%9ci~lo1cSdBq(Y7F z^60jJP>#sbD+CA^>|bx_U@z-fM3ID{k8UEn!O+R5GJy{;zyL=MSyfhB}drHLs?^o zPm*D0z4upqyOhoQz4zgE*{-DuCUOP<7;mDocIidlOXm#D!#0DY=p<$i;wO9NuNIIN zFA5C{#u_;WrKj|-vJRX`&fYq0yB$uQf4L#zErzy5mHfgkQznmI(S0ymxygW7yhul< znL8t`?X1S?FCXOWzQ*hH2)e40p7eP#m}CCN7T{CwHX^VV z)2-qkho$VAI#$Cs|*b=EcR;!^+ z)YI{Wz11v;U4s*%@wRh6>|B4Iame51eT#l4VIA(p`e$8+egIr%uW53<%Gb`+%ooK` zdoEV;%wg-S({N|D_1`(4{t|szs?my_^2T@L3#6$cuI*#PNkqf~cgizkhoW^WdON}i zpKXLM%CQ`mxidJAN5|v2cG~o@wNDEI_)W?bmf!h>h4@}Fv>w(cS>*&AV8c>+Xx1b9 zu$WRuWgy%3ET*&nl45)~>g&Y->)iS%HU73Q+0F~!4MJsFWG^yi)jCe|5}^kdN<1O2 z&y4>Qkplnb#_1}jlk(Deh9Cm1?))aVFQ8qgtZ(IIDjHx^08g}UPqY-s-P>=tFT1#{ zV+h?bq|{at->m-FChp^LR`71EtK^?fC9Fb>p60uqJw5TMr%nKUW&M-5zQ`WC0IDwA zAB)Et!w$!Y`ZFhYu6y$ONfgy@B3@f6Ck7Q}1oWdloMhA+duoP8mC~6`#)F*^Qs8w+ z0+Syou`RkX+Q+$@Xsv9F606phF>p}a7JdKToP`|NjKT{`>l5c||BV+)qDyzaqNvXz zOvhyJiyd?L{0Oc|?Ob6j`-+eR585B}>Yp6w=6so++@53nT!nugK_^$HJNqp7Nscn| zbv0YxwJR0DP{N)Y_b96F6NdniLEJhQxTXicvFajwL@O>)f>ER)Cfeg1X zMX_8TEe2X(6H^C$A>eKP1RQ}BC}s)DGuOcqYF9vEdn0v;=HIT5MZWt4W?OwLHOX2S zaMPhKqx!6o`ja{)Qmeo<%j`!$v>Vo|85Iy-ZnF7g4}lnO)ERC+@VHsO<(edqyLB56 zb7YUO+IrE+#_0x-JOKIY{6XpnQ~i5h&(9|3Vc58@aB}s~R(uW)BjrqiF@AOBdCCl6 z-&^G$7@AH2q?;gm!bJ3$xAb#qvP~&+eCzAI7l5sUoJ(MALa|^FVc-s=L(CBEmVvBH zp2aDJEIm9Ds7p3vDMU^J1#DA@$9XT|Nnz>ZD#W;m3Id2V$;^#zuW zZ0H4FJJjPqRd|Hg4R6Ca!e|_prDubXpmMgmX{jb-n9LUppB{>DBE6$)jR`sZ!4hmA z6jhD6hH1m?oOJPl1Q$_*!7Z-t^r@?H**riEB3a-ue6215x+m|sO}eZf6c&}mxnYR7 z`^pP+wId)93&S5bUk$n0(Q7{sWv-|DU_G~xZf6y#6ODs|g*KpiP z9Cb>ams!k`F$!j^LmYO+)tTD9n;8Qe4G=wY>1M;6&}_qnlr^r6x6g@bS!_-fc)dOV zY`s!@(7QID-iZ^g0~iHdLe)5!5pLHZop~~DUCA}$n%z{nzTLdY>83q8>b1Qk>Fvi8 z3|@SawJh33O%XmIR^NUSV~h2AGoSILp|4=cvj@=17a|}`ReG++Pps4JCjnGo8+e^l z1_Qo&G(JXKFID|89z}TQlWrQ3Zw#e!w@f6RQ0}cBg)6i9M(Lj4>Xoe=K?RQMQRRmn zUU9Lwz~v26c=|o=uU)Vh8DLh)iw@y2u`WI9jV9&b8UeU6;1MWe2nG*Qqep-dM8U+i zSP)(S#*9$rA-u<)LAZ6uwCkkvHnX*^(g$t_aIFrYz>?t0>6J!Wx#=nniklw9f&}6K0g6v!D!!*^Y#!V_K_Tsx3t+G64q1!7F4qL z?7$L+oJ%Eh8Ba`VP*vIxpyjr9{T`}(*oHQpCl3blg&D0eJzgrc|Hr0YKpAtNRU|0oTe#ke&0-$7v&}X+G6k74l5a# zf%$)?bFl3;w_Pb`+3^8@nQ&Wl9vq78o-z|L$9QourtYZsIDLtZjNEw2A? z6EPf16ZymI7l0ENXpd1M3Gqr!oMOhRfT!mZ@BXcH$n>9M=DFLaNU>gP=S?hkRIQ1z zLZcGzIiZlvFAj}2CmY20LjT#T1WsITh6CS@35mOq~bU3ZJP7p&v5pbb2o{`&e1 z`XHj9g(xljP(^-;>gLXql88Z{2Yzvp*{^4`)M@r@b{2JviiPXD|kA>o*(EUy> z8o28EgW|3LF9}NwM#kJAB;5omSj8f+>vORi3Qx5dZ4n*~m4LK1z#(gsjqg)|<#Y}MZ zbV601PElhOwCf8>Se`L!V7IvB(q~m65Nw;qZ{I zaAWQ{Nt_7DS+nIR>l^P8pr@UMfE9@DCF^X&uV^yX=j0yv%nx*vJGkF`L7=p;@xFM? zvS(U4_DTN6d-so-((q1`TbJ(aQ$DLJ5xz`*%=RJq% z;!!KeP8BcN77tIefbS#g(ut`kjQ2pj3oA(qd+@ubCdwZ~7sjuxb<^1H&f6TnLyhru zwD1t0WY&w%c|17WOmzo20O6%~BMN7F@b622Xz8XPEa}&ZZl`rDjiZP;g-_@Jl<1}cRrhE zPO&4^B_}EE;Fm$|yIrueBU+?$KfDjrjEqjJq5bHcRPAaWzPoG$ct?7fM?7s;;MM4K zPPQk5@3fUoGqW|!j&f?#xa2?>0^EM0$j{cf!}Bnon#4V+X;IbGSXI+&9;2#1#DLb1 z_mrO{a9LE~HMGGFub8()D*-P%`tw;dh@&&ehx{3T80eC7W4C=S5Fxjhi9Ziya1)D$ zqT3^te%V(DWA=YkNsNbJuCg447{xD6O)~CRKFC-@3!>vwOSbA&zTLW_#t*DjU9vqC zx?-0Mt~x%|9Mc!@IsUWbtjYk_8uz)(9R_8TK@lOR7=RiHrKcN5NT*L!-yb@Q4zy24N*q=vrDGG&SivM3|T zpu|V#*mJV%3*|RHi1eo&f)e>LtX%*hW2pM7}@fY(B;^;D4(C zEu53EDDfF<#BiQ%8>Q;SDOUdb6dQ0c=$$M8SMiyF@+AKxVkPkE3Wyz64hDleQN>fZ znsw%0JMx@wZv8bcb5_3#8=c=61pmo-^7rN^E!i7h01*Ay!j@B%7tN$TX@O z@hr+778nI~k8xN1E{oWyab|N!w=&uWRpHq&e2nuPUXv?(0U+5Tjg@~r_|~X_(r?A% z+ZO+u@S3_cx%Cc;T5V)(yxVu4>-KUjvx+^)cU|ADNFlm1hzT&nK#Wh7Mvnj@h=GZ0upqbq;MHLY zyFij!MYNHl#cLxm18Yo*hHRHO#geIrRIXx32Uu;B3F|w8y%WDh!i4X`)c#rGAScPk zu7TiKYLgqu(LsRlk0snJaA1V?;)WH;Jk#js2#wY zQI9LJ5x?-uMkEv+HHjFmX^zst*{^8tB4RG^LH+mQ?pZP}={iipYlr;z{2$>t9UIi5 zL}^eC9md5=`^n)();U+H)2cKr4)m?mWj5|%p{Cot=)XRMD3S@(0_6z zugBb)3SX?M1M4x0MIJoQ--~u7nIKImPN^=^4BREx(;oIk7ZmAeKl7?WXIx5b!x~L% z1F>WIlRZgyTQCnq91=${o<@-?m3T4@dFb!?NignkZ>k5qro;Qi85CmFvE1oeqm26* zW?U37u1;nN?iM-R?39oEmlUvX8OgU-QYS zm`9dQ6i_R`OsvLlVlbjs!j;cv`?P19h~Ug!y0(x9M-4V7d%r9NtvLSn0r%V7JbLc! zlhXv_1c_IyM`!g<5^2&L5PDy?Vl*>T4&4)7-vxYH~#@UM5ZIoVUH;TO@Wm-_VE$lgW!}iATvn3YE%}yuU$%H-}gvp>9 z^Wu>Z{=~pNKQF4mHIjlG3HU)$z}7P)K(7Hya8Gbcx_6%$-1$zryF57>;+d&|>T;R~ zsNQ|2QD%^Q6hmim(BBBF#&!768$(OaLYP~U>WXE-!DW;OS|R&}`6OEpcGwyL=rW53 zFvvhyzNJQw01`-ni7X3%0NnxtEWSY%uww>^$>@0E$N+n7u*b01#~smnUI8%j4CeDW z1ym{6WRk4PpizPi`klKJ8(OLl3O(FEG^J#cv_v|{@_mkf8cp7D8G((+Gs@prC^Yi% zuyE{ts-?HOTieqm=Md|UjvU2Fbw!hH0jtmBgm1tktA z-!wRM<^B6#@XxOW4D2Wdl9OM3h{$LSGG}L+Q3S?1gQw`H(&CN9n>^X-%`5=tCNdnY zzPdsO*zO0Gi*d-4TJWfq?^Rg7bkccUsxCw0ksy?Hj5!Xv#!upv%Hu>rwoYX$mXbaW zU-B@0TGcGIm7^y36z0tUHu$4{V8DMJb}IE@uLC0$)p8f0{nL)q8yMSA3mp|YsXK>1;yt^knT`kGxF;SgyGKqls>p1zXRv zx4QV2ws0PWy?Zx)n-lTLh315MKAN^J&xg{9j(mddZhR*SRdi!;IMEr;c9K2r+-8t3 zOi!&p%NjP;dT&2Itu=p2+R)`yGQ9d}Fw@WAtn$}z_m@FMXgX|5`Mqcr(HCX^wb~$?AAZVw`F8t>rvaq8(kBQ&hev8C zodMPRTrEFiG$k+lh+q9gQq9F_d3?Y7ms000KPF$fk!(CpS96^2 z$hnt<&Q)utW^&JIjW`O|kls=WA&++-tg|4|dl3}a+W@0wtRVmCm@m7z&)UfUgeLB1 zb{npSeOkufvl@13jZjik|D&L#{U4GMeE;V(kFE4CEUvW$pSMACou3d2F3Rb^=X<~ZEhw$mCBYY2$t-EFSW-&y$C2@ePoCwt|@ z%h-oNp@ECQo#E!LcmJ(G&=4@XCY-Pu0jM(MI4EmD1UH7GMu1`{n0S`U0t>(Z(Ll65 zA|5u1+Ds?V&QX`Dg32Lo_enk6ob_zkZOrQE(Z1)t4jm!cn#VFmqimJ)gl2jq336_+ zL<2KD%v&%`Ih}99ZomGy^)f&TC|n6bvvuhOP=@lR8wx;Kl<4xx;ov$~tV5f3oD}ct zHn<==Q3yYMu%dJs+lzL19h+LJ+|p8KL$i+1BZgjEufKrU@Jx%~-ssmEF7YFtVrh5& z?jj9vXiz5BK@z%t!9Tvce z$8H6@b2X}8w~f7x+6K8Uq5Opm6veO22Eyx6?6mJcu^$s=tF9`*pKr3P2Z6mO*-&bI zX=z05N9pm;1vQG$-1CXI^}!)EH|SKA0HIRogYs!yJZ{AS&Dml0c%G<}64qK`5zerI5O zs;HN}jOCTU+q(xKeIkGo?z|z2-R5h&cYIXgq^W&rw4w$@YIdO%3*>N)tg4)0-etp` z0wO#|%=Z-|rNL%cdHe259`~!f|EG%A?pt`bae_xLtYuboW0IUCU-R2%0i4AtLmM~k}p0_9}`RK2TP zDy?5}6yISMKW8Nxd_Pk|j4Q!8ABaCt@!>J4@=@!!Mf+08L9cM4@RwZ7DFAMS2R(Vd z_y*0eVe>9?`)w5`jch+meW>+fiuPVbtd8Fh3H{mTRN=ongE9C^E@_j zFCZ&}Uvv`jcPQ^#i)yX6c>`qtd(Oa9b=ynsNyJz&XCZ&Z1^h~n!1#HY27vqV(+s-2 z>GrTHu^xD%!7OwR!aJm zXhrU@pMG#-hS@=ceclVU-Moc(7S=^;#{zQ{!h8___KWqSGI-idtEhz!yMBt78xOj) zsN`K`_7=9?gkGFoB<{(;LVCd#c(TzeKJRDrCT}yg3+fED9b%VvX7_vXbtn;O$^s>wYGZBsO5-8<-NC#!APUpfxs8rL2DiMAE zyziJXi;&-iZF3$R@7T$vt-QLPrCi<@W`i(Ki~U|NEUhCSV0A+A3x>U$E;eB8Mr6f~ z;mZ_Np}@|YF8g>WjJbj?hJY9h9m=kyzuT=Um!owBA2Q)R%30av#mLxmwYr|^a=!?K zwTouvnVYxEY?}fb9{Zud33`Ehp@4#{u4RoNmG|URy9dB5gMl#!M@=Q&f8bwVRXzWI zlIsO31kev=u?c^e+RoP3HBU!2Qp&+^-CNC^I;qswU^3GV5t=?<2HK!!0+MaW$VcZN z+gG9jj13~bGaitDYu=S z;BMgLDkT1w&G6fX8t}{6FNwpt>eYCLV`mKB@ZzVCkQ+q<1Ii#q{RH}<#*A`0OMTwV zFkZ7UASBM{L9lN4UrG~bdKa+_yJ4kb{OushenQPg90$2;dhpjFU#v*Sm!UXmM~TwN z-CNoHR|btE^8A;%8c2H#<~12&X?Y*Bbe^z*8m?8Eh~7hK?DFWIYmfcOeml@3{YbVd z4VePVeT}1WWF*PW@MW!$1Ujt9rrE|5( zv>khWR8B$iD>{UNTw}8KHdSidh|Ak%s0n!YSF#D9$s(>exglE=b#Lya;M&k?SId^1 zXR6@ArFwTZPb|_IhR#de^@F0LIT4!C5=hbHEUSMxdXX~Y-?}3l)fD}F z$@WHn92220ZDlQ2X>KWPvR}trnpPjA4h+3kC4Gpg%KyA0(a8*|OPmr7LzA>lK@~-Q zZg3|x>Un3B7)v)K70o-GHgA4VoDvK4Z<(FKlpX1oP|!EoR9@RJHvnA_9anXw1qbPb ziXk+AG5o)VLGN=`F=d7P(ha$ZSipuYoyGk)n*-)E1B;LnekM5m-$M+dR0vZWd7IjQ zD~SCZWKtwcojdU5YVtkK@`xk?JClCyYS#7zGOtn6RN>{;ZU6bz>LG>ci2dC*65lOz z?ds>2<(>yO|KU?l%`nM-k#7U(cgcpQ538M6NXQxGW=1RD4J_(M!90~nb+hd$atU$u z7)tizs#!6TM{bfpd8n@TsbC>SZuXdx0Z9syHxeY@ni_&2(CyOvSaPQTU-u{{o!$7r z?TgiFqWg=NV)9E>_Nj@c@$AuFNr%ue=^RhI3Qgu{vi~xZB+)=yo#wCv@L0-h`6G5r zi&PeL!$gk?7mPL2Weo!F89$Y4aA%Z4<9{b(Vy@~R*l1Ja+3{VbKG{-f;V&u}m%_oq zz!vwV7imh@$1PYJuL%j|R^bIk&8H3X>`#w8#WgrvO$kBgU_2I`JowMFoJShhy2^Q5 zc=1}6>$CY}6)ouCsKprMoLugu)W2h&X&Q`tKk8$r5 z>T@xJI;aZFZ-J^OorqQS`y>j|N(99Sl^D0(y0ax!TRhHaY|>R6l+MN2WT7#zcRM_M z%px%CB10~u)nl&z=my#_MicgskZUIwbbm(?+IJKoD{rOj)CByyt6V;in}5QjTr_zW z&<9IN!w+ljvKb^h7*2v^UH7>{_S7~QA%*hL6fR;m2+BjIAsz=pn6=wP!@ooweoaRo z-#-P3raEf}U6Y<{JQ!ikkiWiCuXD4;MECMf%b$O3rEMDqP6NsrE;D&869j2c#_iKx zxAzDs&*T1%_Ycs%j>!i~*-1H+&o!|pUam{#(<6_on?oG>n`>Wm zY}yCRBoDx=wepmMF5rPN0?{QujD~OUh6IqgE~~I=xX%2}FK4eeQ0z3&^ky|>x%wS2 zOK(7-vn;QeIV~T9OP31o!-;NTU+WzHL{!}RGWE_L1x2UEfeg?!Q_2BL2>(godOrRo zO<_H^E?cq{h>nNvO>B>m`w#MNKB~Yy(4p!`Ks`;R^ws`I@1jsC9um%B{D>t3d<@TF zrhM~_GznX)gP1NJAxXNA#DKK(q5i zI}qiaDSJh*d%mVcU#(y^-v{chQ=WZGI`0ze{A zpMO6u3Y=y~5pm1P_w*kF|9WL>Q%BNk3A*|pK7#0D1u`fGHo(O*8)i>;k}@QIOlb7+ z#$?je04f+Sas!z#3`Gd>rS#I-tWFVq!UPQLXNLF&(CVG_G6_HIlxsh_BZ^fEw{25i z<-0&s?3W!rrNxRDfeTrqFNMK@WAev+tiVw@*9%z97GEGW85)`ITPOVc0EYn6yLRs2 z(S^uLcH(1gWm(ftNgh#J2qU{b20p1R$ana`AhHP)IH_0d`upNujY$=g`bg?RH~BtV zexWM%PFj~wCE3=+)>(TJ$aQH8V2Rik?PWs!fp<$uvcM!^;TVNz0DX93L`j}KDc+=H zKofl-dN-5v7vv&IW(vmLoY!Bm*A8MYPz=hcx8}`a-7@n_DO2lDQg^QL@V`Y)Q)KHC zFD#eSW!FehO5~^A{g_EkcdBiHOfbY?yH7{a<=zraCd$wV6VI5FoK{%wRD?+ei9Wc0 z?u+W5=R9?J5X^9g@4LU`<;HIuQKg~*;}c{Pxi`{giU?L`on4UNsw<4_yeq3|Ue?5c zBW_M4Z!jG-FFf)5M;IhFvK%lo;#2VKZmNmW*CV5CVO>$2KS??` zp%|`FJW?z6@1r&N%wKfpvTZh&55`%!2kDm&S^7CQw02Zgdx3{LNMSEqu46g@Ek4UH zG<+$;z`&ce-ocnYnuWQuMikDNcS$P)*k1ESv>mJeTygS?t(J98Ei~tWkBMpte0$oz zV~6{ugSpNifGr1)?MdMI%NjeH$R+e)*F)X)Yv*kHl+|jfdz9T{Az50s1<2Fv_WX4# zU;Z;7XaX!MR|zN8r{z+;-b3S3zM`L}={1gys~!2|75U)V8SRQL-a!s6isJGoO232^ zAp!Z>#w(Ta6#5CIf&YN(q8Yd41RLi1JJZH+7TQFB*g-1>gCifiv$l)A zEy!(?=*dw|xFMbA*!6TABk$dVq&lJA+_b(=jg{(%A6HiwX$YjkwXQPbUqK3*WqcKj zD`lkd+M19aL`P}w`o}_0rqD^vy+~vOt(+{e?RbMq+=4b5r3s=_AuUZ!ku;T$_rbg% zH3#Tj7F#0PGh10mc9Q!~r18ywvy1>sW{~0syd-RXymZ=~+`(igebaJ^LYP3=)SZ&a zl$zW=>N6*8(}$uhZYoko-BwRLc58-;xR~SrkpZge%$F7KXsYF%_UEHclL?HhRakg)PU8-LNrnmz}0mZx&RwP+9b_vc)dQ?mUm;a+2fIGNe7yMqcJy!42~rV2C9- zw$*_97g>4XzR-|5)TjzN$bb3*!ZM66$UMMul1TlKD3n>Othk`wOJUmJ3=^D zGHOJ|D}#X?yOg zKda}s*2(kZG?+GVa>toMbe;`lK&oYKO~W~35v-Sy0Tb0<9$~NFC*Fk++NPMJ=Iv;% z&hGY-zU!xE$XDX+v7e`*yBIG`4cwwCxv7jP=EnaNX$|wZhcY#G0B+0JTIy|pD1z4n zaT?KG3U+r3J8D3Rp_myiU7+5nM8vBVSlOEwko}^$Sla}x%8Wj3fF^U{im0L~Qqv6O zS~`X;8jc&N7D3!AM{8oUIK9Ke=WbYwMj5cCt$RE*-rPVtd=h z&!@x&!lK8PltlQYh%ZCYP-3xi1s6sB)t=uBE~amUoyvM{g0eF($Zy-mF-zrv5PSio z+rbZ9Rr>evsDl1u@Y%(k;&vWA?4m>Z@ao1zTbVLDWSlvCM}ze&_JHnZWAfRebdAgo ziifcEzo6w3ZR7mji=gt&OWWw?Z(&-)X?_PAB@5=#P@G8AusiOt`bi8(mvX;gBAq_659S@+Zpxcrj08zw=Ks^WuqOlJz_M`&sq!gQFYsKbRA|0-o~|}hgg#Ih$w!#Ic00k05tEc?fD!TD3Q-- zvIcGVfM{d8q=fT0vuiK3l19yUfCb^%^d9VpAi(pb^Lhfo=lgXMy2F%hem~SXO~Rv!z96zIBaaZ zGp^8j*U9zIlj_7|_V=Ipt@lF0)%?M_uAfmyOLyRZH=4Mf-q&Gkk@u@2>XPRbKU@=* z@-yy4rE+6*G0;hF^A7Cc41N~2bOiraR8;*4X7z{>_mgrYh%4si$iOm8T{n0fCYGPFkbgJ0g|jm8cp+viY~cO!Ry^B1WH4`gBN(=XB@A*< z>|a$h1BKsGo45W!nZ=O&BkP}}re_Y0a&VE2Em^z+@aU~{Tzl;Ng6dI6@2?TW!d_Bf zlt=`%9^GpMmzRl2dZs!~#Qd;}{IgH@Z}K8t(G~I)-IV(QhKHv&L1ra1bNg}$ASyZa zJBhHU#k`Ci@S9^71O!#iSobY$ZqfOK#c3QqYBmFyp+-)%X<8r5Yv2oOPTJ%Um6xR;tetW-p$J z|2Ex9=8fdq0M`p$Fg;3Z%f1M`18(%q28QNS7u|9LauKW=3K7l&oXkQL%gct+7PHJG z2msha`?aja*?-@!h$75Cz&7jp#}Wa`EsRc%TVHIjR= zioqvn->Zd=fF6cS0Ci2ZE>{HwgihLoLLUa5U;^ddD)Igf5EJ-Et7u})NwS-| zUWB+{!L4`X5mB#*b4$~!v9o7TyygAU@NqS!RCPbjhR@4oq*!=|4rMvu(4YH+4&S?3 zSf)z`N|_i(&rLRe`8<-gE1Rm})Z}sB1vZ0BW9nDcCbUvV2@Z$DE;H1|h~^ZgujuRT zlslOMRLHKvzHZ=`YzWr+6khsr*0=)+lqQOCBG^&r3B^*7JFKA(}h{KN6 zu6A=g;<$TaLeRrPQewPf-_oVA`44cgA+)+CS&VHBJloEg|06sB;cn za!4p$X_h-BJutnW2PLdi2d5*2UgjW9dMot&2vWX14~f`pLI&z*kHwKB80+}of351J z2qaUK^s79QrR5Q932Mp%lR}D=n4G1g@scF?gr<0V{|q%*^i4%L0NvRACa)Koiq-Dmb;_;FaewW_;Yp0gZCI@rVWwV;3#PA8 z_8ijd)U`JXiOb2a-K#^|&d;jfO4@)vEX6y)T^Ro3&HmX1iSFOSFjPH?Qzgo8vPBZdhBTH0+?niM`< zAMBQU1VUWBSvDD4Xqr%vJIqjn<))eF#xb9E5ZZo*m)&8~tKV=p-q6Q#EZFfttqYwP z8A*o)6ba8mP1-cM^%dm!qq?U%#>W9^w!h#xs-T!GI7Hp@059noieWH;)P$Q6R`|hC zQ_dGY#FVp5s5oMaKV;qy3JLy7yz~1YJTbM0Bhe=meeU#n?)z0)(fLh&9+8M5c1&X) zKGka3Hl8Lsw~#E4Y>`hg8zeMUjSUPOd!G6bev?nxbXgV0zuT~Hc}+$Zzm*ZY6p~Je zuIi$zhVGS2nqe`J&JQuhe=^7yC}@C0H*%v#fDuH1#I{%v9sty~Z%KO3f%W=&Y*7ee zMYJDEz=>hS6a!pkK7s_)I!`*pjEhbbGQz?oMU!NN+69`Uj?aVQAui?dIC9W`|d+=-*Cauaj zPXsFdMBdxR6!;=F-q_tTeeYxOwU>Hcg132ND_ky@`fCTKS9*5ptX zC0-WoIkL;Hu@_8B14qG;RIOa0G`h#DHZhfiEmKiG5jD-V@VP~)^E*E}y*e{TuL@oI zz1~6|2p^~)%(arElg>q1!*l0Mov_*SEo!6*W$b2!axInL9MgWXA{+ys;i=x$B}>zR z))CJ=b0{?xukSoCs=x-Jkc3(6s}@;AN+aNR@Qz&bFe1mzClNu0nuQLC&&Fnr&R06P zXsI>;Oq4To_V`m$LY241O|$-mUs-NXdkaZgKo#R>)yPXfolE>XdaLwpUvgX=wnwj> zDz}tWSGcZgJz&OUp(e?}Xe?@r$Nxi3I2&?tjilQFL8&tgrXp`8)M|f9ae#1hBbPz$ zhfjupH(-HXR7kk^stxGBrz+}PpDP>gdQ(HSTN);1SR5E~zhrr;Tyl7=pWX~6QxfRf zc*V?JKgCh1zPp8TnT6NUxD|0^0T zd|Z8skgQjLm^Y%=#*a|)v1Q!1^xd}U(UR5)&jA_d7b!9Gk25<{SJPWnXP|mLY_xfI zuU6~N<@^?-ZZ$YySP}-L!Amal*TO5GW8$|78X4w3~cfOTkSBn~K6G>8X-cO7DB+HQ6hD^fDhw)^FIQWgxx zhUa@ud#;uUxB!M$E{ewte(V9~#}GN#Z7V}p2J{JgG?rcbWAWwz``auo{Ae^^%$n5h z2O`e3WJ`*=b<3@l-BPZbR5(}s^uJ=3ycGGXs~K>&)2W|53_>M8 z&uc;+=cz0U*_CiCh+4#Iig<*DnzL#2@DWHMb<`x|QAJ>rQ7?P>Z=1$JO4l40HAc<6 zmuvj{7BivU0n`aM`q4ucKW0&!qB@YeB;mo(Qeli;{vvnC3cw*5F1%K67v(1ava2x) z%s5Swrc}--*v9UJxyO-m!6=DV7eh9?F4DFWTTBwa0ro)t|W&~ zN$HuKUp}l>23GP5{sI!nNr`+Dgqd9|`eI$x0RKym1D=Ngv>0a5klLRIb;duBZ5~FahEr2%&((dzv)f>_82}rIug6JuA-?JmI~en?@>k1qlj_5|BsSPY1D3(^1uT}p)!z>vyq1m`srnI^;wR?jQ=?fwaYOs9#G_XsDaA4P=WM8-| zn>Ooa{H%0lMD-aZdY>lf`-+-Hd6u6PBV`@19AN*PjgGTQhb$kG!E%>OH+&Zws}Yf- z+HfjdDnJU}%6S%Vff9(XpZleJ`Azjy&bn)~LHefvi(WW7X@ zDdtpBB}X)lH4{p_hCm+udu|rq*}XCsKR7fqIau~`XL90jM7ZZY>&K{*G(F4APMuHB zV(R;KHH;7gWhTNDjc`VL(!?j(l;dzDl0n~IIfsb@A(Isw#>X`{!eST>K>`C(iGsD5 zRa@yDRG!la!i=L<$$?}{MXUnu^o9^0v9PLuP#AUu`PaF^HoQcffukGHD%~!iZi8_u zm~S{#$!u!jrkfd*EFCdnT}Z?=Lv%>S1Bx^{h$!oCh!m32A@O{-W_i@zha@e@ag~@x z+SJaJR~5CzYTuWU+Oa^s4n6D9bcu3)w#XQZrez99jI=#+fkj>~r%O2aV!)zqC^AuzS-a zV|~^Ufz!k)Ow$ZNQri8Cz*a{zjU|%YGh^SfZO_9TcWmx-I&yBQ06R!j1!roQZ=@}M z_@a>drakxYS>&0Sa5?~5G+{MtgZOUQK9yZ%XZX=i61%3-IGlrbAn2tlD9ru3Rx_8e z@mgMrX&zpYax5aIR5G2g^7E7PoB=|IZ44V@Wl6eGF$g*_C1Iq)WCXIMenItF`=}iU z7MN70=lib#KSI8nsK7EM})WL;oN0TJlucb1{$nba?9!@E`8uyVc=2 zp_34(HJ$;7NHz|RF)lhPkZRX=oFtY^bi()QfwpHTmon8xKzK+$k851dI?j^51ZAu9 zKod~LntC1~o4?L+Ue~Yp{eC;%s2V%{w~hlcM=j|js5(L??jrLF1z#{<4Wyv!*O?sz z=$#&NSIbr5dpsBC!Sy@QcWw6Me&#a?C%Pzc$8jxPBiBPh9x`$6Epy)+-Mc`xE@=Oo+IC&` zmCLm-;N#`Nw7p^qvIz)+gh}|RQ9i#n&Ub$06@@HjE+uuN0rC~bnHY*wx$esW;b zBSo`)vOdg@X(H;j*L%HkYTdNue@OR`S1?+@aji;54*ej&9M}|>wkRx+L9omc-zJLN z$BrhlB7Nr5nHO?Ri5Sn|0@Em-5Ln?;zWtK2t>QLZBnpgpFeR#l7#PW_>%At{feJN% zxH(wzb#xurSS-dCZf=vT4QcBqb-%06W`7CNqI28i+&D>BD)TeB+vN2zjQSX~P%c(e z51M5<4qmL6kkCrI-fE`eh60T#2|P-%#0fC`fzA*1-9N4$SB-?7NZ(&q(A%u?$lUj6 zld)9)+Vz(fqsZ)>oTT=wV&D5=Lw!vE+Q?Q!8iBjhA0L!eKmDQ=06 z|1k&8#5ItKU%Vgj^YyA@UPpvbGSdh!%uon!Wk!zxB8dZuY_K4>0MB-fHv5B=TimJF zW}!EQ;y6HMzL&Tlx^{=BdGdU)k(E`bic?bJ8b-cC)lEV2Y-GbvXNtUL>o|S z1>%{Rx1e`dfsuNLyFYjzRZM()EwMmgLBLH(bpbP(nuoUf0e`;RY3JQiui$nI-8&s-_=Ft^t`e0 zgp<5+CU&2zFrX`dr0ID#`28GAIChj|{G*QfqZFanj7u(R#qmApGnxQK_Jr~q=Q=|^ zD#+l^iaF}ogL8v4`S#WDVLk#nY0AjF>Liwb0y_j*WBi5_v%jy#VnI58WCND)s3zsqYSF(cXNie3Bb%ssvX44{ASIgwz1x>1TFWz>S=Tpw-^ZTY?{i zc@VNG1$T?IL`IshXd_~7!}VJp>E4UIMYdFei)DqfEJFqNPHpygu@JUY>YTycHgBW* zU2)Fbu=|I15oB+2;>7KEl7MY82C!UuKjF zo3nY4{!5quv z@wz9fv+1fML%Q}FF4cAUFnbOB>`d^%gM|MRpm| zg#q0P0`8meEXM)WJdwBbt1}2%$Ty4wUiq}BYHtu;HxbuT9pRwfbp890S|b);y~CUr zNe&j9!c%5NtA?M-u|dRMw+_4`jGG3phklM7R`1`^QL*iO(8KT>d(9HWW5RiStI01E zY4qhpT~KrRWCuh1FYwL&DTm`kT)InV9$j&N^U{K~kQIIdbKU-bTI8}*{RRfo8hoQZ z6uyWU7Xme>*M+jmQNjd)n(@~c*k;5(8mqo#INrrXwxYq-hA2McXFlon;9TnYrlKBQ zIcW&5b>yNflvDc_1OGoajd=0y$*_5fE&WU7OhN3SW*TJ8N)1>MN4@3eRQuVL@vE`@ zpnX9penYxo45o|7YPK{3wnDq&(HS?`VG@mCEAo9iO46F&a2Q~7VF<)M;2T8#2s!aL zBT>T=bVQ4gf|EcI%pE)m{*uskh~Bv$8gr+K@x#GO(X<;w)IvLmm)U6;Y!V+Ex|1f)*; z#gXC5ok(i1D(TNo46-?xm@Y42Aw)eC+N{7cYCoID>{sB0TC(3i&C)&;iN6OMvf<)T zBmi&+{iEek;J)@2?t|@s;>Za-QP7>0Jo9++^=H@woU{j<%e|Q#AQm&rv~ne408M&+1+j zs&c}mcok7GKO_d%^QG`8vZ9NlhG;W-u zP30o%B23ig3tbY3JI4=^Y*yKZ{<{33tt&*qD~qxGrh2tbaJCW;Fl z5A%c&eJ35*LyhtGWyiVGIV{DtP0^|ep3UBYD%OlRN*!b$?iRxuHD)kU;?xNXxITKK zP-Wixd*#jZIP0;YTYowaL&#n(1`Key&Og>@>$<)9E|zLj#T>(yv;`8d;$d_oNPCT3#Wij%R* zXJB`6u%0j4*z5w&?D%9!3f|{^dW|=|D_1$AkiQygm0KP)?gOXqx)j%dKVTdDyo%T? z{JR(+XJ;vs@}-uy^o+X7;ejo-K`clLcB;omWK5TPK{9~zt-pvNbccsEIdf51q0eae z2e%Q<@bY8JyXjr`eG9vIwRphA2R9dzfxW4QIAw%)V$!CTl20fTI_vX@`Z%=JmFiav zU@b&Z6zUCA3r6ntWQ+}D6=A{h=Pu>I{ zAe7+cAYY`|>+t_O9v*M|LrrbvzR*z(p5Xy;1pwFeX-jz5w59YN618#29+gX zZyJW5K%=L_og)6K{o}5ot*ajziv01aOpn6txX9J)M;0$luL3v&q52xh2?RgXP;Pem zYsob7bRSJYIP}+BEkGIpm@=RWFw#K;ZskUg03wM4i7c~#0Leo`nq)={SA`;MO_}ON z^}d*2uZLNOt104)8=sIlW6d0>B6P62dNI8V=g!47{3Le20gmd(!%?+SIL|*NkKqm_ z#YxvgtGXA@pKxYW$afeLGSO-eYapfCxfx@K7n<0}vsZgbo&SOCQDIrfSQ3}LMW|j& zyHcnJ=0+B_knOSBDw@fenHKnV09yg4OL7(7TNxmk4{t~#UGgqRNq7IC1tm~esro!x+GjLnrVE+t_n>vX8;JJ zF3v9Ya%alvT|zMSaJT!JTlmfOWQfjPML%$|Ku|~B720=e2rG|GNPWoy^ZS;SK2C+^ zy6%lrdFa)05d`k|8YkBbWD|@fX*ZffEE&eT;nXwL<} z=d|7fq$7pMnC5H;6mj5!OduJtIPE?X61leAOe-Tz5Mtc`vI|YR_2L8{pM||EbwPd^aT)%k6N^QMX?2WLNqT*G26hL1LJEf=t@$0p4Nm<{q zBoDyomSx*glN#HRf-iZi^!2Agf@6Rl?sU7myH|ZPS7p;#Y*XqB350N(m#q91>UN#k zQ#>u8%#0umEu*r9tpT>%|FRmkoWwtGY}u{JiNqyjc8I3C4jEJd2jUo7U%TJLn-MMj z7hc<}0*aQVxidn=irpTQ>-M?>Ri#HWZ+2!~$z>mnNw}$D-n#&N+;df}EDyNNJ=KZh zZd%-<4E_gCT_*T_<=L~tYvKQWr7=jC;jb1IURJ5s$U@<7TGDuU)_i;P*ADZ@{0 zb2*7NaWqCDmxk9!KO-83SH+J(t(_!Rq(yt*9cu*Q(vqhm0DY(=TN-G#87r|1kn0Nr zz5D(iq`vciVu7K>0(jSSs*SD)F9CHtCau^3z{5pstumB{MxL&H1D7a=KpeD-73Jr- z#E*~=yZ5B0IUBqRDSNB2op(sdMYX%GLgz1YT|H>`@;&hbY9^(NwgmuZK$yQQw&m%8 z*nnwQ5T4Pc0sU(pow*8e&Tu&DIuqhk;c^GG__QR#_&?0rb~F{Xj5jA2pppb z;R>tH$@qD`j}$~ynd@{#PGFc9WWsFR#};<@Bf&!d`I4g}$0=&J&^|&@p3;bd&{}*@ zCI*;y-<5js9?N&D@wB@Tq-k;@2iS?j9KUvq+|Zc`^e@bnfHL$3Fw_tTj^##=02*k4 zi7c~#0MUm8^GUL17*(oaOltpp>6vAUhX$iaTMuM$1}GlaJsdgDv+>C8diZf{ID>yP zKHplV&d1`J-HFBzL6hM=J@Y@f7jN2N8i2Jb7!#yenOLqr>u@=dhzlqiGiQXOxIrkO z`ywx3ol^I-3!OWHE*PHh*I0lgj6;RCA$%rdI?3!qoHa@=F|G=!57hvHXR;VZS>)?h zohv3aFfQxhi+jnu?j%LHTRR4Hv}So>aja4N3gQDc!aixK;kC=OA+*UJ4qz|nwl+am z$rbMe-rf{|6}Jltm2CcrP`l%5bUUez5;Q7OS%orY>FmZ?hcn@@Lv8e-$!+KvR?tyz z4f*2<<4lyACKG|)K6VYbf`(W2^?<%59K7{ilb8#9!ktKl6jZe6Im$h~L}Clw?mTjF z3r}sP;qY~9Hlr=5i4)b{59qTTh7H6_t*`G_%8T)ub&MF-RH!Q@v9IJIJe6q=%ea>b z64r3I?Ag%sx~NP)s1z?yDab}Ii1hJ3-HEcTsZzLk3Z5F6M)DS{c%ir$H<0TSWb{_7w_sF)!Jv&xOK8w_p#V zgdrq2DmNVhha)OW543N01iMb!7mZ)FCbfKT_kWPGtlEm&@@sxfHAAk+m`*jZ+sbe{ zBEeU1FXTZz`bWk51PZ4}xElh?A3n-R5hEa-Ky5}o;HEPS-2 z9|HE`8K73h#RY0|abv#XBI}HmEm;u?PNnecGRosZa>@8JkkeHb)i?blM&#!-K5in0 zlLfm%{F{=TRccQ4{6;>f2y)4nKsj6ff7c1&Rs~|N`2p2!wI4|piClO|cKmac^YDB- zZ0axJ5B8Om_~r-D{)&1XrtbV)FVbjD*P=yGvY5Dl%~PiT_MBpWBh;a|lKtklNbl5L-c)xix8$bauz{e6 z1_U5qF?dwTR76YPvaU>QG-3f+o27XwOVs!MV(-yw8qZpnsUmKf9$IOuy6tLtfP$?m zu=7H4_-0zlM=JVd!4spJX-!ie$tXB-n^6^? zWj)1gdXA66UpAm5$9iQBMn=3ipJqC+zDO*)8JAKUMj}xr60GxR(0139wBqZ-&n*VS zu5$N_6N+FL7d$=USZ#h0tCmmtMs$vwY1~>qs*eH`{f$TOX{_MmDmqYlKW^vWMBMA| zuUH&0>t;9KG1dXQA>KJ!wB7rl`4D0af>ry~2))>q)s43=pX$GAbf{d2yAn^r6kyn7 z%#^dU+aaCouBxsu*EbHIcQ>Uh-7a`*2ohGuKCmyNZ`MY$e#PRjV)G?pE3N*-ajQCc zb+KB0rOaWj)N zH>%Q*(!tsQAPpu+iXT=hb#yqBPag$`?f+)qRm>lL(e;}SUBaG^9V@~tACmBi67mnB3G)1Jj4V!k+}0xe!g@1>+-#8@>kVXo4h-toMwfl&rb-z zu%)bMOQA`A;8U2I%2AUXCu@XvPnwZx{`KbHZ$^o5-S59lMN+By+zcLTPqL-MA()44 zsOz#qGJs`adAuusxi+RiqoV_!i1_hG$PeLq4ddsW@hM(=MEXCkMo@fEzMo~_s0KN`4BrZjUa!1R_T z4wlqt>=YX|J{Iir9yY|*xiYxMxzNv-|B+XCPaN8oC28$20?cu6 zLE00l?t?BY+cN(A%!`+?0JG0roaFIQ(j87TZ8NQ|REU&*MyYSpJvHgh_Ueux;AWAl zq3CduII8NV=PpUj-TMal^bD3T_$YI)a?dZ+*b?LQ=7x;7$wUTCSEAcGWL)2^jw2GE z^8SnzPpy4=dkp@@L6_!BfUz~^v={;U{VkzCk+MyD#~C>ervvlSpE;TEYS#ihpR5zQ zeCFlbm_YXNT4AMStj~J4_(q>A`br6jWQh4*9aNG$#XT3m-w9V?n>REMym0RdDb@vU z8?Iozn!xsds{Qk?AWt5xoDanY5NVpDgmfvwy`3ff!N<9LWK^!16yg>}rI44msaviL zw!NK88zX@$=4n>y!naM@JFb?gb*GZvI1;WYh4p9L0(ZQvR?&g7`nDv5-;$bFLFVPh zc;mOHq-L8CiDEII4E!P=Jdj|ALJtXVQ7(D#q_i^!mQSL`8*7a8Hrhgx!d=PHmzZs} zwFXj&oS*7LC4E^dwa+6x(k(A#X^g#>>y%BKZ{clnD{3<*Eg=$TYn+4{O)QWME|4}| zTOU_er}h4fWa4@eF!J>`)oIKUf=4-x7eU|QGV=*Tp$9ImIgaJC7rwb57}q;>BLm#` z2wAh?fV+J!UwClM8!3m!s6LzP|3dRQM|1lhvo%MH2*oo~vl|PvoJieFPs$z2VxF>{o-w zC#+mmgdm~IkX&h#e0M3>$iAe$`R2q%%Al6>U%u<2fi5s)HLm1>K)u~`MoEU8yoCRbPLe}H@(!7RLEvL zB<$uTgkoG{JFfz&IQ&Zh0jUZVj(hs zDjMkE^f?nkR&e%fuT8|+`jn48jWk03lIaGwv;-ZUxx3J_vIiYuZ|mIouJ*{&5R+af zI*faPZ#;@ka+jwh&-Hy{YDR5nxfTb0eu-KXe_lxVt|*}-qc8O$`Vg%ql!x8abv3lc zfjzkNuDtO8uA1kcx>oA}X0TCRktIq~qEgfSt&$JxyCg#{I|wq7NDZANOp@F;TBoHp z>b*|Z$qC;+?F`kI(>eFcJi+L&hn_32O~$^b<%I^NiP8^T09C$iBDyU zNS~#AJZM)m$+Qud5p8256Io=hA!KiK9A$kXSXfAd;&BG&Pr&Ky{q=w6*G# zi%GcU&zVyFePgS_^M-P&>IyzGRkGND3@F)y{4RUwl@N7`7#ZjqVf^db*`$3k;>70- z_qwUrCoMe289R%!=FtGd53e! zDPLiokdWz6G)cxGCX-oF7$oGl_5mBc`1QTNns_*o?gWO`6&A% zVPwX^QKc)I;7A~d{$l+Uv6CR>$NOToxslL4>|9y~9nz3!0+op^n%;OXT&*~7p3tE0 zOBor(+cqs1HO{3vwmLz{*?4GYyORv{Lck7zt`{Wv;p$e&#Cii#zMtUjYI!nZETCIr zaMsXpl$z+1HQH(Z2vMQqkB6sA{ka69kss48UWY?Fjky*&zy#Ywc{NHCp;$;+$A1U2 zhrt1K44eBOz<1>~vwZqE;mP5Iw>S44(E_sk55bLoibg40EUZLMYmUZ8yNR*XN<;8= z#oVH-2P+k{UL;eyKl9}8VVfYw!LXYFNc!6f5IjllmQ zHn-$V;uhWMYjdS@oEkbx@ZdAHwYL~W+gpL*j2t|iCHBHC`%UcL0-MNH^36t}2M-85 zU;)M}GlTku&iH3FJ=gKr!<%Kj%JAIuGq$$iYI{cH#$#H@0lfjxj(KdIOXWgNk%7Rj zLXMK=+;&&_t?45dW2y(U=pCJ@CX=M8d=y9j z;^(#Oo*|GFC)U|zaC2AYdR_@wlc#M*0s(EX!%ZUEq(*i~04`9tG;ebH?03n%muxm& zo9As^z{(i5{kqeUWf1t(iQ7+WNVNl_+o{b=;$dJhWJ`^ZuPwC2OF8rdn=Om=2-d=U z`R%cegOZ5;TAtp35_QtoG3mJ~=!A78Xk@3KmT>-43ZCj`vJFm+9g%gmyIFGCJqkQS zt`Sd-aWs%;we&Q`ppB{oY#3avZT!XquL8}_5|N^b5h9sFIr9&HfNrcf()hw~2Q#0g zzt^&Z0(sGpk6IIel{qErVTR76%>Q$Wge#pH!wmnMi~d2(-6E!ZShM2<+i2_4IqdS! zBDe5%sU2*DkEzl3Z+_?*qzd%uuxJ2@xK~j$NOwt`NvP3uC{V(W<3UV*y zSqY(=S3LiiTh1nlcTaM;WHSs+Vf147iH5z>a5AR{FxWtkE+s~f01`<9i7c}K0Nggq z3b=gaUr18AJZrc&2D?G6IO?EL}P>t|RR z()J+~zhFklAUaqr8Jy(T&&elCxN9aC{?V*UK-jmROi9l}z zDqy@fE?j0zV7Z~fDtQVFtNZbcccFk-O>v>!l&2dU#)@8^*n8awTNX4+)6rr3KUsw= zOh~dRodwf}<~!yF2AGu6MB_P^y$K#GcPwTVVa6^v%A_S`1zaJh;thffmBtHdcC2d(fj1T_Ys28;72#RLle0lMFsIRonstUc zI@MFQ>IapIw(|Ra7rDWn(J8Bd5)6?Pd2#noW~HH8fMlAP zB;8OMOJ7R8MS4TRgMBBQE?m^D*E+O!GLcuh%@Fut0!CGpDlMQx$%4b@n74-s;?-mM z#-}*r`ZLOkeK7M5?Ho3xBhqWMGPy7BbDL#SrQtEZS)^sL<9~`xxEq)iLh(*YEb`T! zI4iT1YXg_-rKY9JT2mquhK_oJXBHPhYGL z6B+o9$tQtT9U79dEQx&9|3|0Pt+NSb!TwFhdbqWAW zlLr4dZ%^KBqF|!&$uIN^YQq&B94s~?X7YTcP7Ytq5?&k*V_Bqz63}fwU7fa&X{G<| zDb^D=64DoqK82!!o@ah;5o(eiz77fKXJ0UVy8v5*HP)uYE7 zu;7MSQ89XwCt%9mZHWaZUvr$BKga(}xrUos&X1J(HZeHa5R@X?%b zM1dCVaZ{AByHcnfR5h1Kg;AVawCkKrjZI7gLhPo^C=fgb7zVxdn|*bhmytx6~{tsB?# zfJlWzG+$lfi#vu`@iQ}~*ayYZgcweM<@zk3k0hn31#*P%$-fo3=gOTM&R%Ok8Ucth zs}v}32mp_cqepNNM1jP%SP)(S+Fw{*H{cUr+o;R69W=kqX4y`S0^cwUn~o#WS_;d>=TT7jt$98IKhp=?f%`)ER34E;o2uEDa)y>sZ8 zm~9^wxj+hVC^Q}CU`qON*i?^eN)zjWhJ}M5+L~ADadTl`C5yK_)panGB3r zbsc5JL1A|S0=M<&zA6U6#zj#M%iKRfH(INgMhyeojIBaE{dM}X#}PYo1K$izF;mOFj$?k)O;r!nSU5o$^kcmcKF%;2Aelw)L)4IJrP zi0de1<26rRl}8!%HG%-bJY^XKxJs8B&t3Dx@*u-6llS&C?c#T<5Ab=gYdKK9sZyH zZEy8R0qB$Z8hYIl^qSTT{+&xo1N`DxS>%L2312mSw;u^@#+|bNVc}BqQiS(NFe^!E zAvpdKk4&Y9a%?NiNKH8g;Kb!-zf0oG6ak@J@a~n>ixG+-GK@;)wh%KWS?l?~q;+(H zc!gR!xQZ1oUMdz1gl6a&DrjCc@>;f-hnGP(#YI39i?>LPjuo#hXCH*Cp+U`VQP}M2 zfV%BXn+sEDRJK6vPIj`8VQhRZ1ZKii<6>4=MRmfgn^EP^Q7n zPjfuVm+7wN+_S`3$5F4}5K_jD!_{-7(8Cl1D-lh`58%MWIzy>pm^!e!M=pOlHo0-c zkKXNX6d|Kfc!x46HpOLcjD-Nt5;ck<7Jn>uav**!_?IfU%i$S9-q(8%9WKlsFnMqHhrP(eKd=U6v%06GjWtb9L4DhQ*Zp>+v9aS!nU zbX0E3t|Xqn3f9NshTwfPr>U{mH0^&~<4+B^+7u1z$IDwY^r^Pj#4iK}J0|%P`5r<_ zsNht^@s9sdDe3O(@8tTOM}{eSX)!7Y{*hY9Vj-E^82tN9g1z#+dWQa(+UBWJ1D0M2 zu$DmCQv$1_G~!19lLVDlc{b>JsWc{iQW-RlS7KM>ciw>I5x4?mD%)5wmD-W1Z)yA7 zVa0+%oMd+OC!a?C(r*Y{LwLo-Hw+3a99(a)T&GlGHhZ+0_aU;wJG<}A0rUR4a!!1& zh7R_pWB4>|acMUEKZioTa*cbsTG(ZAyH*DvZFlsFM}Py`09Gz|=vn2Bb~Dy?x5Lv1xTn7$~7`ZC1IEzME2Y zKwwqonfM9-5XQ7Tr{dR6Ky$l@#oNnObx~tJ({wW0SB@^-3trLl(Da3UDJ$0~KjTuzDqWdU*ysWN^A@E+EsU$5s1_IGW5&rice6iI?G5x4+a#!z=3-=-?&EnGgUKZw-eARI#ZDG!f2acVRX?V4?;DNQxb3aB$-L$b9Vx2(oOeR4|N!< zETTNSlL*V+)8 z=aUFSm@orGHyHn_5`Vf{k`y66n54!n<;+RkSZ2K?$%=uoni_iuR4Gbdzwz1JC%Tz? z_@!+GK9AVtSBg9t%)P5k_aH!Vk`jQheQ5&LeO9$tW#$iW@9LH5ls)R=Q6)1kiDs9a z)D^Q#9h2(Cl|)-Si6K(vwKs{{Gmk>9fk{cXKfstB*T2;ndeGi^^q4dwH6Cc|z2#Oz zUez$+8BXYwD+#<+N(5712UypkEcL-n_-{10upJQ9hNYhR?E+L0=rLX32`qI7v$_Ys zZ((Fc=K(wG7?pXTW0+2ivS6L8p6E+%mFe-`EM zUivSMt+WPT81LxZy($<{a{481f@c`Y&FO%w4S^`n(M%)?N?JC)wW#T-G=DFZZYAo9 zUA&{!F85|zap$zncr%LiX>MzWIZ7h`Es+jP3_s8q28C5qw{^zt+Mapdlyw@-MU6#R zRc&DB$&1*;uxUcj0d>yUS=A#U2KU?|EV((9Jg*e34TzNmT&|f#tE6Z{>8QThV5YvN}I-1gH^}rMU``D zls=58tDnB59FVQ=f)jE1aK!s%Ik4f#a~0EbQ}=as9Ll`mT3&Ob=bpN%%e;Sh1_AC5 z#X73nX18R!Ws0=xgPrnRN_y4)V;hRlJwvD((TqqySgCDlbT=UD7;s*Ib@|axYkH5+ zY(mTa^-c>r)yK0Vh*$URE~`bK0{up0U1Zm+V6KgEBKK<_eO$}vd`tT2gp8e_ zj4&kkCtz&ojyVh|?|I~*(3tBMCU)c!wWV@dG}dx9G7Y3Wg;Zih-1LnE;dxvF-JiB*Z}F;<#u9PnrrXn7)eN#2xpyf|aE}I4(#}Jt*b&bk=xSP)R^dU- z7s&u$LEWj;g#cmL@A4pXdf%6R*Yu?H>WWsPnhDmS;-fgK_ryAbG;RbN*Q_H^uiAZWveOx1;Cyt6Z^<+oXvW43D7%z8rRpo*PBW*fN#Njtum0Z zu>n)`9r6i^XEjZRjfGsgqpI7opHKg0clG6TToo0Ss?* zA+T-JZ%umhBK2*Gy?8x13aWNXk=g%$Xb3`qLJkK?Jc6m6SqA!{-DpIxH*DeJQf)^5 zTi9oC8c_XX;qZfGi)K^DYOX@jku`_3VW#L=2Qarm`+WM6YcYHi4C*nIh>|bvc0XLX z`^SV7;9&=tif|A${kzZUuZQfA1>ixZP<Qsc|jgAItz31v?^f_&ZCVcIt z0v0nlCUfc@vzCuiVz(PM_K@8m$pP@r1OaC$QYZC6t*3!t?!!BVJui*zkX1qIEuuU1 zfjU&}2DZ;f77S#s@7`jSXm;?^(Rq5D+1sP^?~!B{T1cn!@5x-dBiwofWVI$HZ2^ty zOcVU28`%0>Rb-b4m3P0!1XlgNE7iX?5-7Yz8|6!0gqg0(J2T7hQb5W&)Q>XsqtOw? zy8&%L8Ug4s=?5_4K$(vfMvnj@i35o&vw#5G$EhVmXUWA?r?HLZ+Uu-r+^=p_*pTJf zL)xTlZsa5(y=$!XTP$QZ?@DTcaTUfkcMs%_Z}JqdbQxVVmRG)jJsZ1%d9MO{roK15 z7!=r)G|(T_Awc1P|M|V+=Q#+yFexycDnE;ktA+wU#a^@naUXFT>Cp9|Hc{12ZXz9P z=Vq3ldo0}CR+dKJn`YE3I{i^G>!DZl*!&S69XAECW0FD`n0&o-C1RQ0(=#JwZkgnv zN7|BMtqYbPfw02-0eM#<{*ax&k<1E-3ta|@7m`t1g7LrEQ#N&$a>FtIJ8)G(m{Mi| zV>4*x&Zc2LFt~C084QIqcXW{ge3-Itl}?cVgKvT5r?pZtpQnYdI{V!Ax2eRpx9Gky zc*PQ6x3-Mui~jTgtm@*V@dvtRt9wgVElQks{u!r>dKgD!vKnuUPIkWAusT!bh*p@C zgi?YKuCEDSp%C)PhS*3X4)9Z3Ht$i$H5j615b6A?^#U1t@PvfYNyAqD?CD$yxC7f2 zeHyd*!n=MI1|G^ytEqMH-k0Sr_oIryJIUoz#xkwgYrJ@`J@c5`A2ZdCI1-6>{B<~A zJF*S2R!qYoTp7_0{7-sud+3UD zRnqnGr|BSDRh&cUeV)aP;Hf2lnT$aS+rfuUgCXK4v#ZV%e47v5 zi!6BE=oJ!>5)&arqMp<=Hj97>uX1FZAo>Csff7QqD>t4wm(#c`(-#w~VhNu_anM~v z&;ncr-lp=VXdz*kKb~$iDBmNW7Y}NI0Lc3NYrtuDh>AdYx!mOp^cSPRRyCl)$xhoF z+i0_VElT3jX(#U1wP%9m15`ta*Diu zudk}YRanl9d(Y5_++QLxKtWi18q^Y?#-%dgN~Ba_H9d>u`|jSVzNWDcT!sopV;VgsFnqeFmUB#=0Y%K`{1EV_UI z;Veb;DvQofdHzws#(}%c%0u_CElx>LB@d))q)oSeU!|Az-e+K**pa7o)3Q2?A2&3$ zL%9S^n}C1%%5dOLvZE<#)JukAWI(^9p1e-i%5ITRUOKEyCDy;=Alxim0)b7#cpk!#CgBJfHwz~heR!1^?0VP4GXl8e0V^mm=&V{aAX z{m`5s$3WvHpj`wN=ePewLB(3mu^V)(qFgjMw*VABg!+uOXAi3A|NaGvwc}IKXQo!J z`6VjG{zx(*emHfA$k$}mLAhBr;<#v+N76z%-YJR+uu}Tw1tw;X>D6V~e*=5GA#szc z$y*odub9dGQh_gt&jvp$m{6tFF}132zD*F~kU4fL^?O34S-ZJM#EOxDMi;iuN^6DP zz-lwWE7yBa$iRJ--0xe#o@}z%Ep=z0cKoCRMajnF!V-5koOkt{N&3zkx);O=v?aoJ zCY7lg*+4d*djV({qjmhd1=N*aeSWmH;WuA}z>KGl1RfB=3T0F7onmFQeA_)o1#RGq zMikNmw~A8hEws>Z$f;B*TMM^iCMI%S0iC?p!`{7fE!fQtfKv<7p4wu5-+xtI zFv$HmG5E>dAXyjuBqG^QY9~vj^U?9d4wXj=N_&{GWMPHl*x>+X@4C=r0 zgBk7!cdJf}#yEUhZt!(e{>`1HRgj}f_Q+ZkWXET-t(R))Lc4AhT>V&Ih)EqD zmYX{b#e3v^l;P6zotSeMpT?TGGowgkGr%xbRn-8gK3{fyq@wU`#GVmZ#85yRlgVyU zAy08R`(T4c=7`8FiNe|CC#8yzJ&W&g;dMqR_70L_By>B$(lfS_oA(7IyBz#Bgdj}2 zWQ!#SEjB-)lq|q7C2A;_T=p8!cg}}sA9y>>t8Wm#_S`J}qlIx8d2jPu{_{ZamUzGM8xZ?aG*ggTqRY+Lro#EavmO) z$B~TuRvG5*)2e~!CP8tErF5I#e;mB;0;u8iLzhoK_K%7ww;>dLyMh7-_KaGGFAfO| zAtxg_?`8Z`zDye)6k&$k^a;n*y3dw^Y5Riw9oF4^=})!?iXaM(T{97_{Bxr3ZB$KB zbL$Y##0a83URh4#ro#P%*v;O5_X*M`Gi!u~o3Z03tq626f;WsalL0_=+$5COIqq?t zVEr98b8FZ9l{#HabNP49WRg-$0Ml!L@3h+*m=x-WT*O_*vA0mZ9&#l91jbA<)Of8Y zMd@2B)MmH5*j~Tsw581$sMhRu%5%iZws^ut-xbZCk&$Wkk(i#9u_8L9Q5!)2x1?2G z-zeyodD>_W%2M+`JWQ-SXwvM_fbv-PhO;SZlHQ^5YxdmOE++Spl@~BFZoJ+pMD5K` zC%GR%zd)LGkFPA$aH9lZ?6uFBg#d{EL_DwTs4KvW>;t zu3rRRcz_YUCW$o8`(Dvq#3}FY0;L!hTLS4Uzb$Aeh%8p94h1&TZuzk|ZP{0Q9awo~ z1lHfJ%vmvbh%q??Ixlk~$=^*smmmGw+gL@*l!Nv~&v^F@r)xFP^-TvspXWzYS$LKy zGRQ}T2su5F<=j)>M_E^kG&O8F)+I$D17BWSBHbTgYRw2LP_{Jjs_O zUvg;0UZhBy3Id!O zS{F+sk1ThW_&+`>6?PUsgdwoU8XzS;l}pYZW#9~HMPT~psk$KWf}p;2mpMnKQCZ(3 zrWKDawtXuDazYc%=!?z_oCZbA$q1l{sjKFxNoDsC89G-Y`zTVld6!${%%(|E$PBbq zrvFd{BIkj!UrW<6TH1}OoCMq`+u&1tlR*CBN_@6+K-KF1}IE#HsPOfj9KU+bpYX4p;nYK zgT14F7TKAX*i(8+@|}Q5T%walMPTf%Gfy<^z*PrMzjnMlguk)M%Z&i31#eaa5$f9T$-YD#FU8RGMg zwMdtLjook`rJw%d3vith0y*RQTJlRI?CB07X#@Gk;&nuzo5>va}mFkDQv`A$zj?{lW$-r-107+;n6PfWT9>lG>VynaCK`6{!P zQ&+~StEXfB&u%fFR+Fu{-I;=Hyo``RYN9uPw=>V0I%qTn|AO}2MSKjC+_~-jJ2U%_on0ADbN&F>#2Fu%Fde0SJ zLw2M19j<$=e?;9|R}yygbO2Z{x-nuSB=g^Lgs@`(-n?1+JAd0J*!D^Ac9LLQo#jNB zz4_h54aJ#r@kYUs963D(PKdlQMR)8wtrX!n1*bLAqhRNxSFt^xm{zdufU(Us!u8=;h=&Yyn$k{7{ z$_tb)i!@<=rX)2phxDUFi}jRwrwlP?1%Y9l!QQrb>ywvftS_I6GWtWs6Qg|xZz2pr zgZ)WQaPuqpVEMPXvbR0W(m8@-y`U?EOMjk=i$ryWdmjZ8YC>-0DEF;y#U%PwwF1Vj z_jm?)yQv(@YrJM~(|Lj2?=HtIF&%Ul^uJD*t_Ii!6^PSOu8=M?Q6UM}$3ukNxmrcx z*XJBSiIvFruDJDl-B=$b2hyWi3y$A>kZr^Y8z-D)XX3A*qAN~S0uX@DEt@i_0ww$y z7}D46FeEdDdpMTn-<6>8+POYJ4a0h2*|xW(pHA~Qju;4|e;YmI*==Gr6jsZlf$+5& z8fC9IZ0LCqnpprjQc#kCar|Z`{GuDZeQUrzD?m6j12Rknbnk`=A9Cr#k90!VnK<=E zPXbm_?GF$@_DU;z1frtU_&QRZDkJ0!rB6Z@z(BO)h~Q9gEymotPHVcB8LNV`>yiU2 zeeqIJy>@TeYm`AHQg3dtMmKS*)BD9@WhyNk)35ffHK@3;?huz}7BO~ndtrLJ5bK+Q z^U67}|B@=+L}R30#i)*~P0)&`R| zef+kgUwKLL?BhS5U-2%080Tg3C*j9!0W>1B zHTW4{ImjWjG`oVH9tB=nFdP7<6Fd8=BoWavSm-E0pq4UmOi}qNl*CwOw&s*=I70yE zu?>;%WH|<_FUbJjb@^}rYz+MYklDLn!)={@hR+Ew0k5AA4o`yi%DuC@?M z<%YOcm1z0W2~E*}a#B(`NMpLFe=byMV3RV*@l%s=jFIY5AWnES!IcGlmRV9qmO+@X zkQrI>w$dSlx#DZc+LjEUK7Jcer3+!l;#DMDMWb&AJi0T6PJsG#BD_?9@d8a5&^YG> z|9lGF4mo86h`u>eEIIYj)%QSo2p5%scsM|`8Q|bZ z%#s#qAh6dDVc+K;vy!Eu{xp!=9!@LzBoV;}+QpfW_>QKQa^^;d9L0-B6K`=XtBbC;16GW@am%-?;gi7JfFn~Sttdz9AVgn2- z6C@K{n~)QqlqlbkBkEpUNbCu?*%z*iv*hZK8VN&&I#`;j(Ed&<-<-9IGhP5O`|+a5 z#+rCP#{`EAVY51VK^|3D9{I#|j>@veUKG)Ln1P&{ls5ju<=zUfHZggYOQvIeZqQ&~ z+U)ywd@c@m44f`{IG!~{{h9#YsFvi~kpRq)(0_sC2N`ET=BbQNRO=Q%rO_!f4NK~t z?BxKJ376{i4kPwTt_&!hD{`QP5zO*g6Pfpe0oN&rA`OxYME>#oDgu+!Z2YXstohuI zo7syY<5hb`II=aEJeAj@1a5=ZkCHVt&u6N|ClSVo{L4vuqh-U8hS;0s`PA!4Ndox& z4rGR}=8W$Qw)ucV63U0%QvJrEkyn<#f)YUjm?a@niFJ6Y-Fc$Y zhl=9JPs-2MccO$dYfG%jlf@$!s0^T7PG)@K5^QbsJXkU}S$nA?_Ea1=vUdGG@YCW0 zxqlvy{bf$m^Zq_jJ{3VteZZXY;{ZMs{5|ea1Qhvt;Jp<_j!o<1YMewaz?+ZVQa!ep zRahFjB3COcZHH^E`=C&}{I4^f4LEr9`&$RFL2mAyzIT`t`=wbt7S#EhD{>vUNh%K3rTuW1WyJRuU76;{HGTD3gYPzQQ} zk7ye}H&oHM?)yiiBLPES(_i>xLDT4l$5OrTgvUlji42P3OzeQd$GkE|JRB1ruZ7wS zNPC-GlwEyTgU%XC<#9vTK^87IO@l_k_s9PPEYnfS1HG`wMBGy z{hrPG1pU^~yWz3{TohY7M#Rs}xy7|?Gq8&zVq(A>!CTWr7g%6YV?-5fGznW+Enm8h zaD8nU*5LAHO4x5-z^kc#Dn``br=co(0PQSgHip@M#Zu1XhiW%J1r*wR-!EzbEs2`V zf8cc2H8&dZHOM;ImlWo%9e!%DA>J3^pqEsGw!Nr_b}lP!2_L5H(#-KIj2d1abvM4O zy0$uj71OJse$qscPj`keq6xCGAi$y)=F44MA-=bEf|5OxNl^94&LBnmC3xp=Jb}r~ z9gi6P(1<4HB?h4K-}~7QR{NH<3;y=dl60zS#V)+0& zK*Ya3H~6Wp@z!YSc`Gn0<5lht@YI)4@R$pmvUQ(2;npW^J~mM%a!g)yE|giXqFQQg zuP_f$RLCAaC^ca%AKoI1WN+C51hK8o4Mb1Bhbz1X>?JXPqXt%ar8O)6=4=H6? zO`CX_u*ntSi5DuObHuqw!~b0{f{5~BQhL4lLZkj1la?@6A-0mb0Apg9S$qbVSBUSSqvn8PO z_-&ADE%BJOmP*s{_+sxac5{P3nLQFL93@4~!E6@)+O}`ijrh{1vP}i72GFg8f<%_m z&wXA~B`zVpnzVxyqaHv@+a|8g)ZLKmZ%Za#R89^37d5L{xS>Z8C^?;HS zgK!qfJ-z;qukXsXtDh*UByYF|DKq%89sCf0UNGXK-=`YW)SI4hO#h^*=b;mGx)V)O zK;j>`p^aHcc$7AlN_3Fod9 z7SjKWPVbk~VP4>PWUDVf=h^!H!DQR?XvU&rwsZp*5KqF9?Cl^`M7SZtl7|ZvS75Tr zjISR>GWIH;FcY<`%S_8sq+zz@+kY>mxNE_K^{uGnfO|h8^b(Y@$zU2 znr<#@pc%mmbFn>@d?EfNwIa&8@yyL|E;zT%l>Nf{!VB(~HB8`wu;47DH=*h(g8KV; z;0geB?YPh1%WUdX%$sVOkjzX51>h(s$By3<*R)9k=O~Z@csh1M^_<&@>|T~WfhXU} zJ<()HlsB(~MO-taNMMF#&cZb3wVT_T*Mqp08Gx1TpZ;&ejaz@wG%EITFVDTgcP-M8 zL%?6ydF{AuH|2-`b}57&D6NzM^+QAF^}2aX=NSjTbXU$;^<7ERv>e`P>D7mE3+?9} z83_x9Zvf+}ARKV51u39sfAy`0dp3On zE+xS2ElAKXshKg zl%R`$NTXkQ1PdW(up?raXQfgkZUI!$dYf5OlvtDHV7?&rqwV1ubiD4;k6Ss_O-5ug zZ=r|UFd{7pQ`@V_awY4XHG0;&JuX%qY#Be}!3^KSf_AM_jBS(`T+n(Ik-ToFDxCM2 zokd{VV4}T&eOj=X2U^CliJjbZB(Jh~pN*=47&CdVy3%?JTPTtrhuVHRUB(0#hfcW) zv}ip%H)+(9krL9@fW4V-K5$&Nb`^Y=R!Kk?GlqPC87UG2gGD~BkKhLPM;`mKz{1u$ zGIWojYj_7di_5A-bo`auhfZg_`{9l866x2nLCM1--6nl^!$;bO}=tMr z7a*a_zVQ+hd5f~8b+t*jQ?_8$`7cX*eiEi@U2xH{b;sYJDgrzbyJ?>C2o(oKtO9oV zs%2@v)YwURp6N4@2J2t z=wgMopJ|F%w=K{f>?%pdlEMIg3x`J7TYXMeQ1Tf+ltt-9b_Z>+DJr}qg>fyf@GFu% zS1AU?#rp;-AI-C`p8bumX&^n?3;FmjHgb|gFl;cHdfIp%+DBS4pD?S`*It43p-o#i zeQz{-Hsbz-$&nZTg|-(A0x1||yRaDM>9WBOZ36Ow+e4SCF6E8$&gAz2nw?~R@KFcr z+3bK+*4l}AJ6Tcf$Y+Nm*pJFU16Dvb(3(Yng7~+77V-;3FK|NRo~I#)l|ibD2(tk8 zT4TVK&{$<^l<6%G@NliD9~iICy#&i5Z8GdwS^>E>A&9N*3Nudt4T}oo9|@Wbb?Li^ z4E#FxeUzw>In{u)=Uj&aNe^ll=f)^ut6_)|dAs(7(!(H7>CI($Nh~6Q7Io3?^ z$gUAkU{$bKea=>?RI|aq1nk{;{g-Vc)|*e9jsCMa{YwwPFSa3ZKm<1s0^uYA;~3X3 zY@J59u$6KEBG-Qiz8-D&Ok}Jqaq3|!NG&FKRIr+F2N(@gT~X>&W6j{4$+J3uFiy)( z&1AJNH)slF?{c%@Wm}j-o8#!QHL7JgAUqGb@S4TY&pzmEKhChtZ$4-TjH7poZo$;N;S zPT`!0x%%nC8_W_mz8Kaf%~=p_UcY6OfpB;26^HRM{mld2kDvvga%=uxS|1>^+Kd}i z%mvEA1hO3k(_3>BJIqwJBRPD-GzG7CoY^yUwC;b96U-2C-#g`c$QxSt|55j#|gLXi@BQLRvjLek*uzk`nK`&{Bai1wv{gH!(YD)XdGW z8Kx5}V+u}lMTq+<{9n;ca{9`LtYc=qm14z}#vk9&gP4sB>HR5HmSdiHAyN|tjgf>_ z!t7;(lN@VD4}?)0e!JzTrMt8aCgIXDCh6#OPtpwG(RN64RnQ=)(vPBAtN_=tcPW?= z@9RPbOF`BXty;}5VNfRx@ATo0;A^>Pg!)ckFVL6+Ced+&&lPjUXO8V?nH+P@Dx7=k zp7&29TeBi{8xbM?N8+Coz_)A}Ky9!dbN9hEOb-~fYk(KZaD=eJj~`aJq+h9Xas-V` zV{J-!>w~3MPWUG4a1uzdOcx+&r9(y2T#^y@Q=6{E&DPiXSjryx1bsge(aP3vD*@^c=IP$Q)`)eP z#X>lg;RI4Y7L(RalR)o96AN>V?EX;duq~kM>jOZAfu?vHi^y8hJ`FDKssjL3Nqf%> z0|&7LEW7I5AX2r0Kr9Z9G>>Wm?Lwp%m@nUyEZEaLR`rAkknILJtVE6w<-oI;d8+T! zC07>Kh)C2x;z`$Rzb&sX%e|xB1wAQ_kQL2yOQ%qkPgN|=Jt>Yv6w_Y<{GDpHdKk z-p0r;QeRF$Lh}B1$-bgJhG&N?x?V0&!?!EqSWRBPV7t~6M^L^$YdcPE8J2RBX#m?W zVy3vEO{glg2ha3s8{)1)q)PZQI})6V4YHsWWTg8p7jAjErH}Ij`|vOn_Aa7cMDtYm zRcVE2z?|7n@$pBPszQ_{;mTXe(qLi!b7h}Aqm2@txScKJe#V(hUYewMBK@wWm=a5> zi|7N#M$p-a8<12-JO}9IqxAb*GZUq#Ccna~Ww?zH^gH`zt+fk9N4=tmj9|{kRH23k zi1p(^b1L`zx)V64yFm6(nnzv>Jw_XOM+|t4kJ5|@#GwCgzA3}&gZ}V4Idr^gBHpT4{d5h@&#+7 zp4y&md{ruzi&nHvmAnQcThWYK;)exW8D6=P{XOz88rFrHTgW>V$WJ@GYzRYJk11Jv zh!un}rDVG5sNTTfOk*!A2@Efr<=iT~11_P5N(vLUjE41Pk9dH|qEX*k=!3INl&gw& zs{g`hpwGxv4_rBJ55rMK@D85Oj@_Vd8M_1I55z6XHi)T zeLr1tGQKMar;H&#$=#U5WLA+%WS0G> zBD`)Xa8IKqcQVT}5L+4*^YQ-PqtUQ+K9!(44z+1FR(o8$bwQ$!#DH;^iMqVu8thFa zF$0>{gbcBmjq$Oi2qEj>8e5T?k@j8;FC4eRS*Q9-4)5RWgs2 zAinP#M~1|8Hxk(N9h%dUG>rqB_hoGc`>hx_wCPz*(&toXwm_ zdr&FOe1MArE=8=xO=p$OY={okJ-PfNCW*|+6GGc?OVzLf&=+fyU37jyB=bCm8%$oe z6-P3rKTWMu-kE{46tu_(l*Mh+q*ozt@mePe!${5I=&fBU0S4qkzO-G za24!rhz>{|aa%dalYLreI9V(wMl}|d0V6LC@f0H;Wmvu#dv_gb z<-fZj2fSsH1n6(}BKgJ@xhIgn6(qBdBhR|L!t}F^Y8BdYyBFveF|w=7!|xm)5$guK z$!3LF&}v#ga#f^?ly~|w6W>-=%6oa~W8;q5k&IWpb3C*>3Z1J0+npB>4^(pimL6&R zA28IvTUm6yptWfKS;uD?V5eTQug}a8$^w%*C$21S4GS_}r+cc5cSL@#l^BVeYc$d= z$?N=pi`KJdX~f++lIFl&LAxUD*Q`HFrFsoN9qba>;5o&GDa}vmE3*i|0b{I zohA}sadY9dbXA;(#CxjGPZHtBrR0A(#h5o;V39#P-P5I{&?O~9qM7rv9dq)g@h`7~ zjR{fW`*t#Bd*en^X58lJc!0BRg)_!a@*XRxWKq5wcZD4#gpAag4WSP({%Od_HEL&K z4ab?Lq8AApMu~!*0B8MOPO;;|J2W>@_AcDe(+s~y9m?@q(csC&lM7K&1SqP=d~myN(%2`YHM?aOR90&J!-(H)<36Q6kAP-D9H%M-;#20@ zJZkmfGo0*PW$2A6`R;uYf_#9eHM#*PU)k{})kT?qv>2PqZs+0%2$kZD;RMFE37Kx9 z<*(gm$keYB8(Vp8+aw~}vZ%p_zOcmSg#kfiDD*Gu0%+F_84Hspf^FJeMa3_s3g3wu z%cMIT)x(;Vnc@7_KlzDWq#Jb%aB!M|drh5o^;JAExm_qsT$R1jz$*`T;!z*-2JwGz zh$0-2p*QW0Q?W&$;Ec8!=oVDTpdG9S>HI%0R7#c|u+9Artf_+wx>M&t2t!J&vCcTX z#n80}&UbFxOF*4GN&qqr{6-pcD^Igf>$(9a-AwGD4D{Q}$vJdz+CwPicmOPQl3n!u zayLsw|A?F#N}nv8d$06}e8YleGI}7jnnV#70qKP2h4JiW+nLj9FFNIC+X*_e>yeqG z*Ycb86P^jtCs&@wy{CO(x9PacxMD!=qF+9*zm-PfB+IrImE*`9**{4I@X>(xH4r*i7hm4{wGvzs3+5SeCgCKv|IlKNUT}Q9 zT&<#fSj|eQYQMH14Xmn#Nw^bn233lbEoPRteXunzs>tc7+Pz6XQ#B{IpL^jg|1ACXQB2bx*D|7b-*3Yd`2An$&S0+ z{>GR8Nxf`qVVH)MS`gVlTV}T)AdF@QQ_0&w&y#=O# z%YDr}xpkX#z2ZH_G+}(2yX|00xt&oPy9(XwWnC|FExs!Iu!C?Y#!apYr3Q0KqTJMuaWK&F^p`f#>%O287>FTW_GiBDW&;TnA0phvN<|fh zj=0rfOXaQt`_4L>hM~7iG~uZocW{A@a90^Q7m|kGu$v!7+~o`Fdm}_lHhY#1@V()F zB1VA@aYR>1()tPsjlIsg7{QAP5;0!qW~tRt8JLa9KD7$`#Je@miORU!-u#|uonXm0 zMEWThm?tn55F4Q>5Q%(RU5Lz(TFWpZl^q?=8q6sC;C|bU!@!8>$hn#ur*Db#WFIhG zuZoMc(v*kyZT>%UT`0c4#ohKAbf3%wGE0!!VafumQh9dg03tg5Db_6hWBDeM4Jr>m zEIg1{F9_AkT3Q;_E-l>BxD;p}udd}RSRZwn!`E3rvu%Y3M$ zdLG62Nv!nn)P04)(n4TDR&xK4BNFjgPoXV-DgyVAv(Osjc^#30^W;iN_(%i{skLz& zHfMYjOcA0Tq8&6lTzZ{b-eynU;C+|6POIr^lo{)*NbhUY5h0S#A%lY>m>EQE5=3Gr zQ+tCM9zD>7qp}p0K4vL{v>jo+z#2WTvs$0VS4QXKRPH-g#I(H}Q*y98T!QI=|4WEm z2v`Kpo8b=X8{h;Xfk#+bF8`Pg{{m2#-}U+c7WYQ3)FHM$KGGs+$|zAVpC5rPA}E_x z=JF&f9a778%wWFFg%e`R2eUFm34leLlhaH z{SJa+Wv_}z2-Im}XcH*(d|>q3RsvhjBWzpNSTg0YPr+;Ae-+HQV1=JuRdE&P8uDKN z(1;d^^@7dJ!90F+T^2;1wM4Z65=5X-+lQo9-W+aAr-hd;E_>eH*A$KgcqwFEa{84% zGADruk@87zT(WRUf0?=TPLYI-=Td&G(81w3?iG4Z%V3jcV#IRmP3E|T_~aMlU0dO% zf|L7U_#a^6w6Df!c>YXw;T*q{5fLn^VmYI{jKOO1RMYxigTMLpJi<$A?oq%FgX$0e zN~>jvl&q?c3o%A>jq3vh$5c)~3+K(|7_329(SJMPMUxlyYZc9WWqI(`q+k3F9Z|@f zl4?LCuC*b|OLYe0@^lR?Ownoj_Xr*TPjH8V(_x7=>2vBOd5#()|4hz$2F?;CoFB7e zg)?Eu7JpFb#YbzywHD=8xC?t9&o4ah^QwXlKFtAk9l%{J=U=|8U^K}zr{fffrK z$|DaYTWg{p{~dM^d69YEItzc7c8$uOZXNNP&TAz&yFUE~uH;lV>w_8u#u$;kbp>{} z3lgJYe&)FU)~)Rt9_X6Ioz%7Sj&_V`RIa;+T%HK}C|z`&9+}z(*>oRxm@5bCN&#e~ zxm62`d`QCHSRZ3kL_-)#TqT?;qIoWB;W~2Ey-aA#W{rw92YcG^{ILPtdx9$m>M8Z; zhFi8{ai0igLRHtNUF|KnI3ebk8}dt??Yz6#6Q<&CSEWIvqsU2&hNkT=*qIu{>_=(bTP083(Ji(jO>%ox! zDJVL-s+O5l3>ih|Fr2FQ-i*({Ln$KXKYf%3P157$b4(AqY+WP^eAh@D?Zty4TUuob zn-Vr?$(IA`@K>49)yd*3{LCdgY(#B0S*$lC>P9w^f>_Fsd7U`FLN+RypU1c~wwE<0 zZ0pF~{VEG&t|&o9cmbb(`Z2Tmxng*=jO-moEltNCn=mCs)8F2J1=)-99QHaK21O0G zvU@kD?=2JDj7ADx4n#w^cL^VGG;bubo|ZVXx6y8Y-&W~0-TB-*+Uq*;LH+QkVM zwo6$cWnv3yZTbA;NU!xy()ifx=alc%L+oZX1%jHG+Cv_6Bc8G`8CaBQmBu>4&)aR~ zpSQgI1fAT2Ad%G(vay^;GQE=SmV4;OKD5X|>s&hDHER{yj~DWVED zhzF{ul~Nj!_8wVRrE?(h;Se7KV2(8R+XnqHC9srUd*=QRS2!Xe*=@lozha*VU>raD z*I3G==`I~jbZcAN)vhhMFV0;AY;nrJ`w?zHgr+A2_opu(24osl>KQFojR-C7w8HIE zPwj($dJXs~?Gm8+D*L|pBe)fKw3f(h!029a;%3FjWEi1feM)`xW(Ju%JbE?8YKDtq zi<7>-vLd^2kYfr$JOn&{AOye zA#g?C|N1UP4WRpT(Hc;BeIU)MX(G`eZQK4Prs(+=fT5;UK5}0H2s5ak_EPdf4Y}krkoCarMc_*KHpIXxRf6d#$7j`Jsm-B)pS)CY#tTLWV+fo7` zZ}_0aGucq6oe_q^4-#0+D4_p67NJ>GyZkfWfK+ zHH9(MtIo8b@;_9?n5KXs@c+Fd0h0*1$9#|wwVpwvtAswqVy;!#_6X)6CI_G4haiE2IngV2A$7fi{FYbY?V+V(y6h99h^76+R6M1ufh zyG_QrliQ2VTt;_}tB=rW?qw&yq@%Gsvkrz5NGO_b{tW~w6W^8F#k<5e9ccpmCUNq^ z@qC{z!q%w0Dflp8Kh#MhL-_baBq)~avE|>*Ek^wXo92K8jX{g~jS~B3Xd0j;9vQ*d zOh|Jx0^-@18w~-Ir@KYLtebGNd4jpNF^vXqwpXg^1MXm_Auf@T_do}udXcmJ>oCW# zd*bCKB7szYuA1E+Dwy?W`k^zo^RW_@afgNRGXW^u0^?v{T`Px)7TN8BY3701HY=|U zxsiezA19N?kWmH()56}_nT`w6po`IeUUepjS+#)Fn}me+#1=8tsm34R983|q}+bxd`aV6mLC^q?xj+6&kK|XmCi%KlA!qv7=Yxa#DLGLVL;rY z%0`L^CsZ=wf~F9Yn7CiP*tQD&f*>qQ+Hs!~`cYMN)@-4e*G;yoEDu`Wva_y!J5XcBdMk=a2%(p3u11u}jTv+fq_q5Rl%mp2~{YAIm3W7ZB6Fzf=$MB zqKjy3E6`&9)@nN?^h>Tzb$`D{Xn$~0N*cIw8KzeITlz=fFt^bpgjE}sA6PC7|1WM* znnB6a6eXPHtm4nOl>=(?kBLX+_-A2fS~Y_K*1?biO`D}ad`8t=J`S}==CgE+z9N70 zB*hlwCXXeXtHw+U6kC_Tl@_PEY!)m880JvhA7J=B$CG@Hb;m?9BdNagY>!){Q}z}G zZI4QV`_&;{_n|l01LrjCP;;_{njR#<>SEQcth?UHEAxdNVFZ|{HJD-~`YEY6_sP0N z^6>Z4l0S5zn_bKedGA15r4L+bAMb1-aME z8^oG{zvIa=Gyp36vO{j#b*=!-Yh#^*#d|nAp;kRu+5d%u!OWxA4k#;G@f5J(oV2LD z2*dHB$dvhn700$m=}6+H{~DGH$z8mehCNheO!^l={sJ8l9R$H9GlBy+Ch8ZD*h|R) zt}aV`zmZOagPh)*Bp5W8tX6o#lA0tfhIRXXaf;7g3(9h`+#u!;Apf~T&R*pLX9j1s+m8)gQo&Rv9CH4#7Y<_{ejTMy22Gl00h`bjg zChz}X*>35_sY0KQJT;!M>3lJgW_sxofgJTfW5NT)&fo=4j~c=2()%3)p}M?*EkNj> ztmQmW;fW}Qi+81|1YtyrSnw?-i2=yFoyDsxhKdxSw#b;ep-5*(*_K!H?Mm~!kW(Jr z-8?Xles%F)qYcB6VLCqljU+C%YpxwmnWlYbDR+C!h#X?40l$U^L(XPl|5a6fu=myA ze5}0S32Vc)x;(DbHj9<;WeJjn>(v%j zb&q1c=!6x#Vt3Z#ujU-P6Q+jmK`nH)ra<>G-_kBBUXvhNCoF3EjKD|;qsEVPG-=l; zA^qQW)U%xc!3IVU{O4l!HJ;`N`k}Ix0(7{|CFW(sFx(*7h^l@-?d+G0Ml^ESYa$|| zw4pT+#*lK~c^{yhEpdux`?UT}ajr8O=ET=SjCyt@4C8DlibL`n_vmCun^jzxB8&n& z8%>mNJg|z(>W6hjHXS!V9^7@@TUPXHnWXAa47^v&vCF%F94tT(({3^FN#3#BH~d}1 z8MwS{1_?*BIoC>Zvw!cD%n$9a^hgY=GH}uat)#2llyS=1DlHJ?u^VWfYdP-~qH6>6 zfAoU&5=l4Gw!SBK-*c*vt@5ySsdYen3ym|FkQ92`W!1s=F}~>XlSUVds&}wnd1Ii8 zX;RPoI4~cq<_cA2@v_ZUviBxP8akg6a{;Tt=}?0;BMG꧎zx!5?RWXB8;0ENf*gqzuOFt3?y|LN!|^OFapKOnNmeSw(b9BRtK!Uh#+&X24Fpl-}gl$H|r0H6D1!eqn=*rW8^ zSTR~r(1s^tiA-c%?9`Sd$i={F!z;luH(6Fi!jfUN({9RU+7%|QxZskSmvWyB@n&<|lg87sErV^bQ22$>kz5ysyC zwvqcfLisz^$FGnUymJJOaUXvLFaZMX-1eomxKYB{zIdiG1RFUX8zVG_c^GUGj8tMh z-|yp|Wd?+)M*sy7TT+838?Ri2aPZCplHD5k356am-;dh{E1ip;g=1e5`y{?h@_ss@ ze%TSR@kxKoh*mC`@l!kmh)QqI44$0AV0jH}9=zTr@`a73yj;8aq8E6xDvyt4NU0tD z8=UEV52V+Bgc=bFYiQuU;3*|7*yH1Wc|ZLfFc&|{#Ib~IffpNg-`)Zjo4@zOV|m>a zKF)MH?Ey!hpB%A6EQDx58ebRTAmVC9QKBbJk64ChNoeeISAfgl-1Qv4hva$zYZcMM zgO0Sc^37r}8c-`g89*e}VIHYRgF4}pX(^MZ+b94O4BYv+^8&*!Kk4Jhpw5gF=A7$9 zd#yt_kXJW&M!{lRbn%H#%G9Sb;3z2It^((X9|rak;Dg3nlSw1h$3G`Q9Q734N*NgG zx2bRt)Na4mV5Ce!Js(;FnQgJmdpdQ@@G;RjW;Ms?xXqv@2#Le$zcUo<6BJGvlgd3J z>eEf2w4kXkL$^-NoT^`tK;$~NI~bgu@%r0IXc`cg)51l6(>SvWYyhS}*Br$1nfT3w z62I$w@2%Q+ww6Bc123R+R*e9A7A%b4(ys19nDVe=;otHS+1uuYUNP#RO1~_^=;q6cKpE zqX}P&WWKectLg;))5xP4w|EjUBJe^IZ`W7b@d1)mcaaV_6g;ZBb7K0w;n&(?wjg`Y zawFxCd+d0}3DE&2Cohz|@^hFSMPdzif(%Gkd1_1huOc4Z)PEhLOH!X*X}9)R?`c<- zyDnS!G&)WZzWqv;C*&o>W#>(iMdg6xN=psKUn4pG(h-1&;6+3M>pW&bB1jkg)$1dg zl9a_dbsp`;3_STUr7uTDMm-2=Arl)qh%+4*Wg9p<~T9RNA6S)ysV9jMzC zwexY1cN>;o_BQ}>)=G}(Y?F}p1qAnR{WmT7Eie_5~Sut0iu!eOf~pl@&i_>06j3=4`gqFw3}QlI))<6J@*N zzt=yK*xY6>i}*I>mc@~QpCP%NGYYAjV}M}B@uI@bg@JdgAGw{buMucY_qpm z(8fN&jMR!4m~9l&9cej6dd83t*{?aYrIi?(veiF{{wsl0OBN<(rHq%wd}EIw@5IcffFlgGHO7sy;FOZRN#n@|!8u5^D&>Q}mEtTLd- zZ?+Qx&V)*p;}O>)(g8WI!*zZ&kd~JB?}QE^^2_DyoeE2Gepk#Jh2=^+$gY|XZjA9+ z3XUMp)}~sRI$HU6PVR( zYP%o-Q+mNH6ZIZ7+r4MUMNY=ZpdL&f?;)LxAFyJt#glHZ(6b}voWDGjzY0IyC~D=h z^&qvzuw9uGgKHIz5@mVUmTU!^O(_8(sM28|dF~k9s2gDq1|Rk2o|=<$PeO02zn;i_ zy0V7X2Yf-wPxfVRb$@d#bWqlvkCT!2vfP$UAC}N$qz4pHJIihFx*arJLHFl_(_J^C z#h$Fwj}QS~P{X%t>^kB$%lE&-WXR3L-Lyy9Y9x^yLY<&CVl-^_%_e4)fvY&(s88O+ z&5cbFzNf~%$g8^;8l{H zJ-U!dR6=|B8FcqcX(x|W8aRhg*!+D>Fk%g5buD`{l8;AxA;hkiTm|QN49U*{IHZr5 zvA675Y{rNb9#9B0X2~jloJwNJ!e`G&>mGhdTSCycc#|Cw$34Z{tpEy!v5RA=REUd8 zoO_4}1}y7Szkvjv9R8%G#eZbYtJ}T4ax-`@D)6O{;VCuJG1Rzvo)G}t)997(l3ul} z2dOvi>jt&qy0P^r!()||05!@=_Nq1w=J((cZ|{Th4fKYP%^bv}9y8a=Ta5@9PoPxp zFha-=y7Ckn_mc@QxpLw27T`1`B|_Y!u~tK_KGA0O%tPuwLGPQV*zK-^(blX@op;xn zctv~Aaduk@`Q5a0O;=OJoVa7Lqph9(_5=r#OEd8H*e=#K8}gmei2c~h_c%6LbnK*) zAmncLienrO$fpS(7nd4ZJ_C^wg zz&ux}{U@@K$-4)}y~T!t;1sBn#jSe#ClZl+eD0J%y7=w(QcD^-TjmZSy*cV}Xmltw z+Pps&O4DN2+ej-Wi*}%G<>4Y!L9ds{uLoigTUo)ncrK9@H-upi^A51Xp`OkS0CLvS zX*!%ECaI{joUTJ*lU#3bFLM|N<0U%I$2^38&37q`+}k?h`=yZueP2KkXS*==U9@$$ z&=jkXDJ-##Lxo~8&jx3Nf_wk$01+NUg-5rY#n$>D=gSvn!@O!&w|cmE31nSMJuAq= zjbJSd<&J4)f3tCjK3yw1$A;)3>=j_GT>!nF7Um}dvw!K-2_W2-`cC5XyRfX&55sDi zzIWTDhnONR6b^zajVfYDTItF8`CQtK^g0c$i&+lfhx7RTDbqqxJ_^8b*%eMi{3e6$ zq>b2>B*5FwUylb%m~WV&m~pO}b`k6%1c-)|SnM61f8|%fEwJnK?1*pr@-3fSEkNdW zDuvoafqPAM>xtG+*nfUQZxwmrZL?_Q1f`74>G4WZwxSM`*_XKyW2?Zm?5QKg&;9uzxNBzX5_>bgdHSdSO{&9@Q5`oU#teEuMx zUd)@l6QEuc1HAhI6BSy1X`5|tE7}(-SQsonXQ^KMq+&T*76Nsc&B z+2t$!7fOuV!TpWb8QV8r6G7|@_dVs%M*jPwT@IQee*Q#nFm|;#)9XGaTu^pcM`sW# z#`MuLc#j_mq%F}cZg##5{|)?_mmBRv_BsL!<4w}u)?CYq1K%(@QbsRTh4>}(d)nk7 z6M0)+NLN9;{eL6I8G2CH-stPrv0P;0V>D}-lZok_WwniQ79sODN-ik$y5mc$hBoct zN6m{E&wYB6XXdq5tgW;dJ$O&r3#v_mUdC|uN`2VTA0scu>a(^>v;?A}-awk!zH%v| zveH`{b?zg)Q5XE(&&%VyOUr0;v$J%(k;Fg-D{UQh9p6?P>k_*;bfyP3L#8{u>```x zu0m6v$G0Q0Zml3>EbKr18EciRl`xzMtgv*!^6p&po)D7^rGgv-PNHOuD751z)!H2D}A zeCI(f18{RW#&9$;QeO62j^(4VUEbZ8o_Z^MIMx9cyi8?XFB{>^%B2hbXzpySfyDCZ zO{u&WCI;97nrJVyB*HfNh&}5Q)z83+Nvt2-Jd69dP}X|N{bLZs31(QSO)-T{g#ym- z{;0d_XXaZ7I?2LPg_@CRi9GvYv~K5<^NA8qV?rhzF@OhS{yTajrxjdx9I53 z)Y(S07;Cd)Vi@un8nNqT${PRHtx>>Ep{?G`KPL450M8>^C02-L4$XxC7RJCM`E$am z4cwIWoXSLXiMX?HPIUkg#v`@fr3D@oLkeKbS|)%jg=2*g69&wS=l1jv*gL}v(WC$E zM{DpV-jgZHZ2U${*+CU6>nqsHKvfjb7Dh@e$_YY5TRqLzY)JPCcRpKIx3e|fQNar^ z7{CVN)_iNl^3Jpi@s>e?(uQFrJPF-bZ9GTmo*1g*JUO3?dP81mHSIXMpGI&aU?-o@ zA7<3kurNePz8=7Oo;F~)xhiL>f#nMFWX?@na-lZwr&4J=2mqxu@^*J6GOg{cK^=z_ zYO9=^#=@Tn`rwEZg#ju3s2`Lhms>-f0`VwoE@M7TiqarlBZ-U;JPUkvZz@x&LpaeG zSb%s_Q)-jy6O&n>v2iR(v8!kc=^4^RfF022BP}H1ESa_af5(GX-6j;QOqXAJZ#SgQ zibjsZ)bFK73ryF3*H?f`%vm?v)IHH%O(4A;)7_GkTrQihL5+6KM)bFz8dQKI-&%A% zq>XA0E}lbwOBGJPKfH-a5?)@HT$8&~7a4P~L;I@jfZ<&d<@ekmGSFzW^-TF4sBVQnMp9SL3YW&` zf9h%u6<8iFbK+MjF4pw&MsQN3(o$gk2tmkCxwxo{HCN^b=EV*kBpOR^5kddZS<)7p zZj&K9HIZxb+KA)WMcgC1k-SCewG*+5yF4X(&2-E98H$p)?8!m&00SGB3GYkBtPJ`*>7uzX;AF zZ_n&k#q`lYy6tK%uz@wZke@EH4Wl0L^sr24OFcuv->Ls(&h`|J0zY8Su-ii{04RvY zYEybk2B5mdeNvr@_}=$k3^0_bhL!qjH9k798N^Q=m zLi@`Z%c{*f<{v{fv;g=w5h{LBSI4#<=H`%!Y%ZHz1^=GrIDfZ2ibhY1C-hb4EA717U+WaxGV zZ-6GQ%Uaq~S@<*L<5WK|5|rt52;2IXDC+SzB#KN0AJ4ue4BX4g3|$7e{;VWGY;MF( zYo60lr}{zo{RUDzHA?H-sIE`tDI{o0=6t8ISaT2o8|@^t&w^7xH@NlQ>7oX2MZ!DM z39KSC>7F5pxn}nm&w|=&m=9beP2*~Kn#e84R@RIKOjVm?zIoF;IP*vrs_sQH+=wNU z*cYM3k4$%qvx5BJk;*<>!@vu>80nu(QLH?45(vfnS8mKhU#HUSBf5brdU~VH*DFf~ zONMQVdG)Yk!j)M2onZca5@!%bLZYsoqUup^IvMm}$#LuD?z{!F1E|KUk05L-4pAw) zUoh?Eo25buJLLJZzGqk0|xEzsJ(HtQWV_3478J_?k8QG9&<=GtSZQ01D9D;Qf&6_^%xgxCv zmOQv)39FljVx#o+n{_!RmUK+u z;adZilOmkq&GC~jlwg=!#d$DPl>1cy*ZisMoE)}U1lNa1(25hTcWP6GJLo=YvwSAUlpM_dlOmK+Sq=5+$Vv1hukng z+F(k?;EyN(fmbbLJF^ik8Q4kotCs&AGxW^B10s-5RUWGMTv@ZT8mT{|X!Ir$?tLCm zqR-T`BiXsD|M&>e)v1+A_9)%`^0#HqVaVa1yNj6vkHL3xD(XRW6`s$+J$rq4fjm8f z&rPNZw&PMPh$rp+R%_wd=+sdrkSPF?lv6l}B`sT837a&_&%wxL{&lD4w$SkU3?UZy zUil|27`rP8RG|lbWExw&Q{%Mle~j_oq0iWC1nIy@>icG)fI(&oB<3hLTkA$GPNo%R z7H_z69BE-xAEt3|U{#DGtV7%ov)&&w;9n1{LD!Z)g(3k7Xi~E=bYAa2<)D>76M}9V zz=CESp++xIV#p1TSooYFgjw2T_fcwGAXj0U1(`*Zz)n!(`A#dr?aEN8m3?s^obJnA*A0Mt~PmosK*< z`9EAaEb8%8+;lJ+@gAt{I_A~SrB<|x{i2OQLCL{g_VqfWI(s%XS`MhR^Mj|5u7KUN%papG0r>wVi z>7qOoDeNaLx^dYaeUAdkRETC|Bbgev$;1N_4>lSoaRVMvb_}DcAdEjQ5plnSn2N&- zhC)}H(~0z(-SDA#%o|VcmeBOto)ch=Zr01LZxL2o!OF2uNLO5FVnmA#( zF-mzop=npD$b@(T|f`QWyCS(3f6VmTh_ddI9X0PIL9anTF zR^`IGUAkdG2hmen>74Y3ZW*H8j;RIkDcLc}#S`U?OD~Cl3qbl!YzaLV1y6Uha9?vA zkj*0V0ej;Y7AyLB{33I>5W3T`~K)=7zT;Yz-lq3$@o%P{G=kn<1cWz(do-x3}6QYy5p^cm) z*PZ<*MbD!gVbisRScF`V_a->eEbE@b$r-Is5oL_v##4qq1wnG8;jY+M@Ua&Hl*6(S z;L%13nBFUamqu$irpL;#-n%lGODj5FaI6s_py#eilX?gWb>c|FDNxEZPFo`7#d;=+ z+#YSAnZfc6*e@e0eHiM?WXd+!kwRAOVe9VU86hf8$Mk@q2n;TRchR2vJD`DN=$i~lI9H4&;{Db7r{EwHglsWG1KreR1$H!I$rcK<-o7%R>OVD;RQe;oP!RhP05U~F3T{)_lVx<) zwVJ1teC7&5WjOjX|?8?3nmFhnY)u@-_mG75h)g9$Lnt9u9qvJ z5sYbrK0sIsbZ|zI8$~f}n;@h#VReDojP)v&SKf_(FTZJD`|0~nTiq_mql7IKIf^$@ zE7l!A7}kO&W=~74)&9txgZ1pb-o(C#J2WnM!YF8t%fy`Q@G$(`^V4qUN6sko4kl+0 z+#l{VigJjV%0m^66WJOlB-6H(!?My7d@H0&{gcS+!n4K{t;)535MfgvHf?)=flAEw~27 zwUGvq*@mV0IbyeO!W%k)T4s;}CBi=opB5Wt@c+lg2d!gFoNN^=x4skVd)}b;$=ytC zv>=tS=N*<&MKQQSZ|}eXr1L;Rqv;dco^muOv~p;TN^e4ZFoW_~lP}OoRqobuz1C)m zITLUnUj?CWZ_V85d(Zp`YK(bq}LdOtenZz({=Yceu?sz|)Z!LBgSCLo)1? zd&F$&xC3Mpq4R0~9Y262pkWt%kkH2=csMgjdCKrkT(mS>iSj+i zZB8B3Si>LcGBD>J%Hhtx9YvG!B<2T8pCz&N6#J4SQLxag8!BNnQ3!jJ%xeZ<4(Fy; zXt{rkMW}^oa0)3$Ft;(9AO%B*qg|Zw)3?OW&L)6vra_zU-ORc9A@pXlfaHSjdi1e{ zy4mzv*meTu>7}Nua2M196Be@cYwaNM4)Za4(j7TdKn`itLubxD^I|RtShmk=2a6|I zIgKvkDY?E+aN;7o@Neo5d9bGWQWxnGrudWIwrfd4R|W%O0Q=<`V-<}}ts`gyT>_kA zKR9evP3**JkywT=es{cGbSlP*ktLaRs&Ii0Jkes9kR~VtIga5~_mnVB=wQJ+UIW&Xfxf10SY*k6ShDi-@0zqQ z5`|8f)b%!pP8fotYe-cp>Qixxv##3hG#&&`SF&UpFe(Uz>)C)H-77&%sKNTZov$geY zDd_7IspGsrc#TjUzY0w=wm@)S(CXIhf`i?+ww4%TL-|m1OtjR`5;F2zC;}jZo#G=z zkYOZ#dJtgm zFq6vR=L|NAQwDJSeXkEyr$X{-p367aNe$hY(R9LVZmF8D7K~wFGSVq^t~`+RDBxLc z89;}n=dH6mVMqBnv?GBO2vInffLWlYYpfP)*vM~@Dp5j_{+XM@lku2jwg(-dJ#Sk< zmTz~oSXR1@@QYFMqpiF>;RSbIk06%n4Lm+Y@SWDBEJp#;60epeW3|r3x@zM7d;ge4 z46;5;bPSkTSuU(KOkuQhjj;6g4zof?;-K*OG>kQ#_3#156Kx2LqK2AKg@3U}K_Q2% z{OGiGSETuKz}*RN9-Ab&9qVCBZtIV30hwoqXcm=t6#r%oT<~o#$*23E;v?wx|AmY1 zpOG5Nuy%>pwM!zMZ!rR700AWJsM8QkhaIy6N>pf>j(dj#?lggB5)f88Rp@L=tO zv9UKzb4mH;w;+SJ{BSf`>1bNc5y{<17iQBSO!Vm-+&#j{IQECBah3!Z)8_GdsypYJ z4*cS}Wk>OWlVwOn^r}|uZH&ZZ4Es<&09oFtr^MEmgLss~0Bb^y3pzg8sd}t(Y98c< zsSj+_a=D17ZF|rf?zYu6a#JI&z2OVLzIF$M(XHk~gsvaErOYW z4Tw`Te$>d#9k7Ohc$ZKaw+0}GveSiK;%@i6;|KzOW^0~+6ba;K7XwY`xBtNs=LHk4 zx0_NS>STCv+_ac2Xf|}+0OaH1^OL;E*7nHI@DTSvdA4bCi8O_N zXUl+r2!qC-FbX3v5p?G5SD6J8GSXC;i#n!Ak}gc-wd4~|4?gOgt~)zmk`qst%$HJZ zQf}r~G(wjL0j#cHJLNIZ?Y+&g_*ShVGriKtT2MJke{AT*zKxmrljB>d?&#rnFlv_y zi~$huPXwcKy8ZmZ)?}Ywuq)BAQb?QPVYfJ}MVirk$CavmSq{Is_HqNg2z26?hUdp6 z`@dzK`aF6D2s`%hwlREsncJ#^<4{TTy@P(Af!*98HSa>C)13wjViNwTCX5nuwzhEj z2&-$-;vX?S^k_|M6gF1TcEy=|1D?UtHRyM4g7ImX+&!e2zn-Dz7VgOVZt-O|F;QsO zI)EOemhx`p91ZF`L!95&vrA@T;I!uhe7K#dCe;kWtQPlSL^ z%CI!{`p$2}_Ux+tGP4Cy>zRRQ(npXH&Ky1jdXJHiO^i}j9a-!4+o zHjI`I8k7zK5hFfE5__k2jgGH9&v1M~%SxVWQ|Nn?v#e1RD!|+E38V<~#i5?ZB0uKG zz39?XQ?!q48O=oE9a=`P&_f3+8nRAltdAqQ9y!iT>oH#F=3N+6gT1;=IskAaW zl-PwSl`7t^WfNRAO|?!c%n*L$kmr(U?n~6=x!=ASx%T{X3vAB;`R|#08Gp9uE0H)IwsB6JE){Rkl(D1=E#3jjN1^TTjP&~n z{+;YJoAy<39dh=*6rzy&GkUA}n-zK(?36`cqY^7ew4wb7*p`>6#VAs?+;Nys)W~&s zc=C8}Xtep43obj(rRmE{uS=auC5meQg{Ka%i%(BJp_KgnP1hMHi56v_qQ$z^#&P}F*9D$qZLR%?c|H5!XCKjI_ZcoMBuv<{ zl_&KhI5A->YA+v8T*6+7J{m;cYo=^=e$cWRyrM3NCPv=Ci@cLLf&G{XD5E;Q{!lk) z16aRbCzZF+62pHsX!v&uchK^yRBR%XZ@{y5y;H06=$epiHU<_a1zR!7|DzkLbF_cL zOgdCaU(+NklXZ#N#73;pz_Vin`bb+6qastdJQ*t)IoaD&LVw=%ld#*t!fBiu5yc3q zHoJpEePzzrYjPOaZ8#S@PgnVO=o8m^yd<)nO)3sjJ z!-1anK~52Qn1t*6G^Y`A3UH9{1@=~ep$Kitp7?8l_lmQ#^Nd(!9G6Tu`Q1kXNk*Ch z{wx#Mi~%RsJ@vplN{{A8G@uJyMgPWD5)BZkhrZrJyp(Hb8MgOVIzeT?Pob>OKvc?_ zlDn!{=-JXM7b!|Yq&yWHypdJRB6X{V8ZaGfP3O>)JyhREZJsa%pC@`TlM zb&F2lu+jjlp<1&FJq+<;SUVl-L)NUWB3UeJ2|SG5wQHLPx{OrZE~mmTns8+}GDGXM zPVxJy-x6Suv+ES6mQZLnVHd^C1$#<^jY>y;E%iV!H#Z(RGG?JAxcI5Jgz(k>nn0OD z4?G$t9M!k1IA>a!!X^{!1ePA545Obvq}zcd6b10>g`5<|8&Zk_z}HSw+`Q?zjXdzJ zi>sN`*Efh5fN5RZ)`Qu(+s7GuvVfW{cxr>Loi3TgckTuQ({jNXV}S=|en4m^Uh07S zD4b>#=4P46iYb&NHj(WArfF{u&0MjJT&}KY7^~_G{Mu+k%k~cQJRAoXAZ=cRmDK2= za_-#MsuO}wGv9fKREmBn?c3fn%EzwKykF``-s{zrs)rB`E;4pIQh;+JP9V!w-1>R| zB!I|kpoG-LW+(r<00^5I({WD!CEoh?I%SWYT0u==gw~0up?ivc(eGU{p6n*y(hya8 zk0kzYXq-EKSpcs}W{+@Ys&454sw!Jbeikhn#GFQl#{#krH!B-<0Jn^wXr=F}Z_9#b zmBPC1yrOWgARA{vtnCx#BrX;ZaLb!R01Zib1YLj5?o-p!xTcJDlLJtU%jfcg{swP( zui+(1=zcjqLl=~Ab?C;)#rOjke<=6pD}#g?$b@ZcU_9#$RyHHta+j?dO=8BnYIC47 za&E!=Z7W~5Ma{Nx69m+~Qgae@(31-iv%?AxoE2T=Cg_t8we@Se?#zlr-y7)WpSn5f zsJ?1wHeOBhg7J?fURbz=whNoUazD`=zTqVXAG}rvdI4antmPp+!d67*uM|=7_hq=6 z_Mwi>vdsb7Va8aX703NKlBd@+x3U{hc+&3Ubh5}OQA^#prmCdLV!k%sa>0t#^p&vjBp zY%?*NV-0R~-XAN|Si#UY-s^@B$U00pd8v>zML(uTj1IM*WJq3wzpW_EPC6Zf^alho z1IS5n*uQ@I?6PabD~LO4efLeoe>KLX_&OB8gz|vcMt*~Q@aJk!?CH3Taq77WprH%& z7+u`#duNw^B~`oui2B(yd!+R8TvGc-&cYdm(U2)fJPP^jz_knQcXj*pIkQIczh(VP zo-4PGrX~{2ITK_V!z*5DLY0D2I-MeSb<}&A)a~N97P1%`8t|pq-xZ-E2B<4;)RGNOv>Md%t z`_f7si@kgV3A{yy(g*j#%QOa3$)-xvlZn@boPZ?h@V{5&Lo)UdN8Myzj_Gtpv&C?G zNA&)y!DZ^-o7benpTWB));<3VvHg1-qMvLgm)z>$L*2xwH$Qq_NH@f)E%;FH)%3Or z+3vKVx(GoZOH#Y`veHynKA&&lKZ9^|dsr3)yQ~DN){tV*aA?uNK4)y+YPX(AeH}d? zQKf9GALoeQEl;xI61{H|lJn#o0)H3gvszOB*JQ zu#&KfVxz~u{v>xp>KbJTk_`@@+0%5MozW$ueXrJr{8FiZ8)Ze_^ zbH~A=vh`UE+n_!;6YJ%bmKx}FgG@YpI4ZrygK+gyhNgW*>hDnE_^V|!iU_KcNB@pf zU|t6SjC*{eyg+lzHQ=ACIQFT>yo|-h7gkDioq-^<34lOTF7^dA3=~G6$wdjSAR+SZ zDe1^Na{Lj-dKT<%=D@W-bJKwvVi!e=#?Pu6-iIy`L}@NAcsfEm?w1l^X(bCu+C@5D zQvS$^?2_+3k8r`NADFkC@F)ic>Z{FD_6sjbt#=yz8B8a)C_RWCdcbxPW(_xEguzK* z=v%*;MfDqr8V9At*t3PL`dWpB+mXrGUySv%G4$Cr=?PdDFG^86WRTX2Rvf){m*V1I zu7d(_cqj^75QKHL8?fphMNZ(%P)lB7Ocd+@x8$}?4p#7znx#)ow}p}@^o87&*s~|I zkJHH1qAr0cjd?MX5n>0`-BCi((<->*t4n+)crJQfb8j>t{m)=yr(5(m+I!Lh31i|? zo{Wh-0k*2fkcBpGctlt>aZ!Y?7n9;Iy$@OmEc8{~5)2Mx+{dwHkmO{EK6IcYu%9JDjmX zzb6S`ybX?H|77VFMgc25#v7i5WVdkS!Ux#^QNe2Fc3=G=UqQ9iA@|{!icMCgvb?%+ zI$SXGpDT`zi*9}~)`E;>HqDtSBjttBf%Q~R*K+73AXQmetqjh9{gW|?+NrRqlF-<3 z-tB;xE+4K~1ND(%-UMH{FYh#BGM2z9)u|0{cb-I3lQRaNZ^l!iva0lE9Z@3nrkhOkZb7r{|*SRSx}^nPx&-DZ?aH(CGchyDH2%MwaQPz z@YjrusJUqy;qCE$er5Ff<>q6P2c=4t&Yp;Pfw>yy))y$sjrR4E4nW=`EfZ!%SR(0V z0tGk{l`Oz54;MeR4nWrkn5^{C$CVJJ=NaOs{^lSfZ3_MNr`5#lIWxu zjIs+f6V`e>Dzv(&N9!7@L3mnjQ-RfB`ens9&sAC`tWDJ&tmwJk2D2sDh^q^DMVdW^ z<+}&+r?Z-uaQVc04_%!<`tB~2MZ_8>oJk!Xyw))QBZnzPA%MoHHx zs_KDYsg1=NmdAu)HDZbe)-38DL`DpYM*Pu^qoJPV4}#K8xu7}i2wL8-a0SPDMZA$Xd#sBuoE$2%4cs;v(0-?iV_EVVIlui;WzL_ah2F5 z(UD(RARMCl2sqrQLECw#JzP->eAm-@Q=prlQ# zx7g&`1occX=#%;#2;;$Ii8=Y7-=yf`labwcmP;5j5L2H&vNfr=qm_D&mxTUGM~m*D z@!BS8@@$V~JQBg-2lMB89(>bDX%)=p2j%=QAf0rozn<{r^y1#g&Y2HH6OTL5&H;3C z%1W@oM#(_CFUzPNSx)X(0Y(L%WP?$!ZW0jo`r+=3*v@hkEpi7GfCIRSP=m>=9B%%w z_m1Jbm(uJhoULNB6<5`z>>iELl|fH242==UrJOvVtowe!6mHXB<##JFWONiMe_2~U zgZ5#<5hZ9Tl9GxurOKWziy9ow5=D-|{y;LzYVQ+ejggjzB?15^6|=#QMwvqc*l3Ql z$sM%vGJ!Buq&1cw0&8q=g#PJtezB1(zoc+zmoUdC1&eJgr1NN&Nam*uRI3C4$O<#_ z_m&{tw&Z^P?8n-qPxXQg_%1;v!u)*hng+V>d|=(^bInH>O4_~YB%3p~H07F%+7v0k z$o?gkfjaH%FV<`>KwxKo8-nC7Fyd<@4goj1%Pi5UaigddjK4z}& zMoivQa;b1bKrrp^*)fW*n2MO_Y$k;U@oKsol>G=RWqiKM6w; z#k+bwr>AK>C7k(5qT-OurNh@d4q!4P(9>%KUvD{nT|_FzjE4;{*(LrmFTgb{q%t%b zOV1`Knad@xQp^vBW-*XxG8)yEe_w{5xns?=!#+L=E1d3RC`9JSu0LDFIXrl&LD*-{ z>!|*izU~lZ5Y>;kXQ>z8@d2{`yz33_O<)oc58dqNi=BYqse&xua{<_Sw*-v<(ni}r znLaJ|J-Ov}?0zW%MhK3;{W;yG0r8$*Tb2m|M#*VC#!NyTkRn(LWN&uy$N;X*u`_O; zcV=+G;{I9A>{|ha@rb{~eQgwC4xxA9Hxv1T)A@ zkS#k?0s@WiaW%dy0uViPsBpl%gtfX-IL4tsM|FimeA)M1KeIU^n!&scbAB5x$FVbb z^s4PVDmhz^GaUiCDtIRbyRKVC|5{u?3(*gz2N>4flT!#`BVD?PFm|m;96BuVZTCG= zU?9RB^|VG~HjihiHOY<{Jx3G`(c)_Iq$Itq-vP*wX`Qb%zSKBv|0A2fc>D>6(ZSJv zTEiIYwg}RDLdtrKp!-T}h5OTz%}PzTq1|5pq(>;OXdQn5*TS%Ixb6~{bHnxb3-=+g zRK6U_%jjea$BOmh4;1q{|1A5OXjy;p;I>PCdD=kAqBd+-aietri4T5U75c0Io8&)} z`q4V~GWcIx?`)62Lj5zF%>4t)rb2YymWxoXK+nbw(`#w>pya|u7EgxjH9 z>aJvVzcw-lbj3!qO?Ri#RYx?bd~$2?u8%eJqp{Klz8TRL74O7=LNmOwBS*n3kNJ(S zqs|agCw=Rn$`-1e%)Ttln!Ml7dd}CEo5NQZJV^NY0PN)&Lf(zcFR_l9-!$|bDb|Xl zrIvt@1d2V0{X*psfo>8!cxd!Ypwi`drR$SXcxhaq+}oD>VQ1A6QVSpTjoP~CwyxtY z=C-fui7lD8T9JIdamK)B(Sh^H#LT0%xjCEQAT1W<`|$qk(hB6=tySn5c_gv~48|?l zYIZ*@GkCl}$^WPN(v&`dwGfpFk=H#D`t{Y6H);8%-~Mj9(8?xk)?PI`A!vk?3rkwh zFFd5=2vVcIVgBc05-)h1^QIX5h85x*QMN@8`+Xk0_WcpSK@?|1bo%L-zn~g+BHP~0Is`Dx!ht;SNcE)gfx=M{bo?otL8dZERcl$ zyILe9cJ)iXJn$L;VMcATcnmP47$E^^f5l{)v^T&MPeMKMb0nkbXUF>cj(z}}|Fj&; z#!++pR^Y?1q~t}(B_;_iBTP1I%$Y;l{60qfr37p^oJcZ}WgS3!O%DwU_Eo+4K_+D{ zJyJ^-YV`{jjFW5;q;@~T^t=~Hl<9s85T2Io z7Rraa1npi|8-;l=Nh$j-hxO`PrTj6k2ZK3Ef$P!JK?_7WGx@Bw+R&+UUEycP5YtTU z`1$Ji9z-UgM>e+M&(qb9h%ro}A~(q)!^F%^6RahaiGlJ-U%LGR>eJYWigXz-a(9%+ z#fv&rtsPt@XhjlJG3o`c0%%P3mx zn>&%b9s5Kmz$oHrTtQ&smc5qH#R<6LJ{P#;xPa$-y zgX_Yr^}Yt<*E#T7?vFph<& zh8l%x3Pmd2xjSL5I6m|S&5LuP?XW9Mt%K{%6{O0&#R>j?7+V#3)Z$#F#MsJJtI{RU7dQV?+sI&QTsH^On8HkzqC@5}!AP z8QRqqc}l4&WU1O~9$gL$Pk6o<2CP|Y{&(My5snq)59QGFGUlzy8 zj`mg`n0ei#Boa8jV?*~N2BNOD-{N^Fml|@fmwV=n$CWX;V`@wuvOIc9bnQ%NmGUj@7 zb|WLPS{?DXXcdf3xYMTnYHAkK!rtVY)z_wLTCF7DY{C(~Uj~-p0gIbHFLB-E`&VjyCr9BZX?YLz*CsmZC-{ zsx^6~KM-6`D&(P{4Y8TN8vNLjgU7Lc>vN&{1ok-K#D^&4CqCE zu8~sS+&*?xCEEa~ERPP7Yb@>a3%_)D@rQfWG9LDF_r+GqJot~wC0)6b!;1k>!Io*u z*dm^V-;Ib-&+rQ};nUyxJ$VBKOve@JyR0oD7rb9CmrxoIg80#mDl+G>Q9ee9lW27G zOYu|iD~x}Row{nDAs81bRe%Dk%RYGTx5xeJ%G{TS@G>Aq4+Qyt9BY8}ftWKFBMHa@ z{2{y2gy(yo9;eX07VduX=np;sZDF>YB9jo~Bu|Gn?c{J~*1+0##3JQ2pR!X7F^`t?OVARpCV@N=C(5aB0 z^@pSI6)^LWJ>Mp8=>MEZrxF$t-o`3yE(C=>533TEw5?7jm!wGpTNClMYUrieuV_29 zJvW0^oUr&DPxP|sEyP}pqR5fYa3)Qb>kzS_R#sJf19SHF_Gn;_@r?F`T6j64lt`wZ zyEFU{zD8o2!vgmrX|PA#@3^SGQY7sLMJrdX0aj0Z<~kp~hrd58v6E(qOS_|NW%;ok z1x|Pq6>if^t_uL{MAnhsr@-_BYpjjJSn|~p+4FjZbd4RAIVTg96k$%{M^OG(is}nh zUpfFW!C7jyN`qV2S-j{fr7YiTVT=QU&P}mcUvcPGl0DU&KM@RZm?kYCEpkH2NnLZC zLTBBBG~qV{`ct!oPL~G&**bKPGwg&|$0Oy3?_YQIRP9@)?-BJCrqLMh?<lN4~LrLY{+za z6~&nQ-4aAtR!Mq$K${ZB1G89sdr59yfsh>9!o+q_Y7f}~j$4M$<{>S~hV7r4%ci`H7+LZ5m9YBO*T zA(wEqYu08Ee55=$B$bo)rk~OQ_rAqh`Efe2Y)*dfW2Vd=j9n^()3tY7`{daiGLaCF0_=CaUq zr)WNzGXZg!cQ&5DQeb!Gd$iD|$;OlWE$FKlSug|MPOsE-;mbd`iSzZ^vM1%vqEm?xU=Z%NXqNwXCf2w##uF8a zFihETJ+m!KaK+JbmPd90%AVR|>4k8&zJc!lG!ECf9}^7a9U1Sq`QJpt8-M&neU)TF zD@^%K_6dR5s(bhB`bg9M?|nTOpX}Zg_jVf2?|d8?*N_f>8fe}5C;zuwd))7A^pC3T zr=NEvd9i;liK{C?i6B%t3~S~RgeEB7t{VVQ+9<0u1mNa1L*%w@3-d8T>H5XGqx&Y; z+4vQ0T61;xAuu4}j?#Jd8993DE6=wO<)1xPq4ku2k6#J#Pf!nZ4XW3mSN1Q zYr9&9VkAi@Omv>$4l z=d~9*-KqZY_;cC5E($$E{sW-+s*t;h$=@?t7&r&S-A46jD3cpbSvJ)&li!JWq8bim zo@USW#g6vK>%R=kG;XC2ktj&J6eX#}1FV-u@KDWS+UXK}Zk4g93;9aT8M?bJn*4G4 zv`E5*)7e1u$2It$mb>>9*Wee^!1MD%sgYK@ZZ|V$!rq`6DCU?p#{zdmxfhU$M?o@v zv(N47ezZkQ4-zAmGn(QlCP-sXB3p zh)U}C4+KvrhM(zgh zfu#595R|I72e%~{i>vc>Z39Y3(#&%}6ie|-NOlGvd?=8epE1`=zfE=1GYw7}Tq#Z~ zP9y;BJAliMEk=hc`Pjm7b)^oU$@1AioX6fIaG3@QJC(KEKIAH*lbaPNYlE-)lEjItS!^$6L)P1I#0eQ2I23nAt_oUu0775e2zia1c71d+ zMn6LRPtT6Ef(=`c$yykjoUBYseuraKU=kN7bB~WK+^W)i|26RgW8a0=U$_4|iQeZM zy=09C1f>&l)mYUOx$lJGWA;1@X*Glx>JdGjk3+k&DF}PAfR7D8{s`#n%$9B(<>;L4C9O1CTUyTn38|7(Ipg6+ZmFkmqY7lK$JF7xxP2sHL(1$N1BX{Bu*HK+y z@0MY`3w?uvIH8z2&ugq6{s`iAQw%fui#m%una8_N!?cR``8)v8qd%mIzo%jaWCuYk zgj_4*Wp$~##B0(27^TZf3a5T%`W_ceVJ(Xu6iEPb$QuAn;wpCv|Gj0_E){0LK=wmS zGTifKQ#wWwSUbMP#MpywZXthCx$b6`WpGphhpKv_$>hCoRkwu?jV?nehb!><-=qgd1=|pj;5mQmOyQZ{k)(0FlrTIfixGp%<#-g5E)`d2iB%=<2=gh$i9d>Mn zzBr_E1tV%4z@nb(hOufFv6sf9Y%8H|2Q)z>QF?e(1T_FlD2pAC2f*H@n?9WRT35Y> z)4^!otRK&ni6fMgWc*0ZNWFggSIV0;Zv@O@$uMc=75ec zFwK!Wv*QGO zhe}IPzNnxk-b#RX3)}`)i_cI7-k!n`Ho04unUE}(+nHY#rD4Pv1&JG@bx!tcQ{mxlo4Xw<@5Q=0^V6=jOXuzT%AgZ8s1x)olNJnXG;b{@ zKcxoQZd)3BKWa5FDwM3#_Y9N$VCyw%`kDd~h*Kf$&n3^xqLWO_ zz+87hP^7-*?sjpL*5tds!uHLm?vV7Z-FD5|N$XLj@9PXw=eTjeNexU8;_dys^bSaH zQ5;=dtCFRf(g0S0Hmr=b+F?@ybuD#*j_@f?qlHlOqIrx_jM8}DWgna9s{;wRB*4nxLu+w)=aPM}N~+<`;5Je{5Ptb< z>so$G{fEF<0cc?sXrpNe+>i8eRMt^U9w`~?C5<}f!!(%s6XrPJiZPURY3V&3$ynGw z$nP8M<(1r0)jxuT4=C;VTY!p!!#6>IhuzH*3R{fh_8niGRM)~A{>yh5LX)c`r7^0!BJ?`3i*`y?@&aX?}(5JP?Mo8k9xbrE`ILcU(o zrzwr_k7QR%EsDXkg;8`M9|37G7z=1?#KQJnSw_bH5JEV93%kJ{P4mNqK-ob(t-R44 z(8V7B6yXf3$xKAI+PIUy#dmoBkvI8^t#oU&0=gti)~I-m;V*r| zlfW|4Jt%sBBb{C&Mi5~nn0S`U0t>(Z$<02@Bh%?RYI{sz3q8cW1mx^9`VW1qLUZZ$ zg!J90T#`Z!2ALLxfdsI|p5(&uCFedD%$-F?;U{9ZEntKME? zhX7RenoFm2l$%p6@!Vzo*EHxR+m%V>4dZp<1&Vsqn*#xca<%Eg?_C%^jZ-KA6+(o2 zH_~Cp=w|?z`PmL%%US#9^GC(L=%573(SOq7gqsKtcRu^Oe?{?(`C*Y~XzMlwU8l=A z5szd}mk_Vt%9Fx6N-vL7_k$8rJ`DN1i|Z6rX`Bi#<0mceUqN(-k%~*)))(UVCbS^3 zPGyRL2pHzyBHOL865Ax#%0GR}XGz)zMht3=yRC;&bb$%uj)VF+kK~ltsV1B=v~ESK ziR=jg=FckQRct9XdC!}lKs3VmLBb?J)~NcE{=?gXK^&6(*a<+Pn+dx=fsuqmu82Z9 zQ~^WCFk(RfE!!+Qvw-;M-=vCmZUf)pdca+AHf&SB3HJMJ0@Q1{IbDeI;~c=rJHLOa zzufxF-7Ejd_|D_1wt>P0#!I6%0NZWr1%!8!l_y`4fh-YfM84ov;vN?ST@etBv^Q7v z06!MM$q2Z_a^r0Iy@j;l@!5s+1V$f&pJ?=6voc%HM=iyDpC&e!05cCul!Of#2FM=L zsog?uEE-|)tXo-g(lsvgbXa0SVH%lUF3NfxfAmD`+*-k0-X&PNlD`bl>9JXkQU-dq}r`JKXy0b<3MMdee z|B-u-jVw>Hwe-Ar|C2F<8>&n*?)yAR0@`biZtdI7q_SXgNN=8l?*#n` zbt5AX*T%GCv20TR2hu$6Q@+l?kNVFxF34H$`=@vhIcnqRL25OS(%GtY+rY?#3mwY^ zef?6kA~qav;IPQcw^{q!qf49R#5l(O)?;7MGg*31bJP5bF)DY5oS>U42WWP|wr^dA zH!mjTBs`_%-ICkBQHts`RPIi0E`)jFeI8y1#S?KQ@9O0~UW@tGzd4`4&;a-h*Is4VvW{4Usp4;=4ef z9_o({&(N&?uJ*+MqT2ySc<)s~f&q`OHhqm~x&LvW;BawE(z!P2-c@{Y9pH=WU5Kl* z9i9#7bz!=*4p8M~)8`V#SA`;>mAC~^o)6nFad#<2A4Dze+ZTCZYC zJ=WQT@qog?YOZ)USwB%8x(g$WQi~>R**zF^-}$ zd;B6Ae1O`RSMi)Or8O+PtDe_mvz z-v~Dg#CF$oOt}4Jt0P5cyB&Qxw2G=MdM&+m0oxbS!%I^wSYj|sHs%**fEPwg(bYhB zk;u%caM?lEPK{uO`XL#@Ki)hpfPZYX8#VFBRtkxmH#qgcIm3B-K+lliiPMHC-Z@M- z3}-+E>|>~Jj-7vRPdTie7yP!v$2#$VBIe>R!Xrk>EkIR8=DX%b@!mZ;~FNVosh$zs{}`x^DqCrCI7F2EEk1!S{*VRQF(94f`YOAVan zGj_&8CIhL!QKFb3$!#IX8|x&d*m4Rf$)(e84%%xFp_p(kuhWRm;+g|CSva;uJboGO z!6=~I$M+kG|E$=B(UixLw(A9OHnE_dGtQL&h43=h{Exh-Q41c&#)nfcJUr^oP|@3D zzg4FMtsq-n)cCLYd$Znh8d{$#C9AKA1V_9dU*W3YL*tZo%=7~X;_AF$NNsZr)?6k`pG6} zL%~iRA29_^;K?&ckmM5SllzA0tsrE1d0bF0Oc2lB*FMw*nuG*eR8e<)4|bkzdNFie z8VeKP!1n{TIWhR7d|S%RF2WX*OK-RGDEN)Q{K9m_|990GgF=a-f${W?uPaucWu4wa zjcx|S5l~ekYPhSmjI=%DwLcebtBccqn+hQoq?*(Al^B+#Q6m}W8Q4n%9K?a#nAAs% zihkPVZx)*Hm2B^5S0| z9fkb@5z`!b#Yh(MkHWJl^{JMX2}FK?&49@o{hKlztRTzUZ#1fMs}D_eMUU4$?l^HI z?8Dg(+NfBMFZ$efAU8bx_T||Cc#j3#Z8^w=n2V#>8Ece})tB87n6k*P&_IWa!wxT| z+;=KAe1{c%%4ge|c5TM&CtQK8kns{(t*7T%lJ~_1`b5x3e{$cSnv#fjXVo=NX&>z` zA1gvFGSY`a2l~urCV9PHabYP2*xO!hl^)wINv=zb*N7QzFko;}n0=nz^EAV( z)&@Xq`rkpb1d&x%Pt^wVq4#-EUw8YY{7U0UcazMjYziw5{h%p`;$xUrs%;&^NLun` zx9}MCp5_e;p6N(hVx?Q6ivVylxphT`98-f!4R@$Qd<#IAH|S39^1V<>BEs$4di~ zNB{w&RW=7iqacVM>}Gq&V>JT6;7ROG;i&p#3Xu^D=(w{g1)?x*E1%OQX79mGJ|dbv zyLO2m4HTk6smXq9Dwi=T?3`4y(!NtG8!GIz%I0^q<#$bu>{RotHI{eG%Z<4? z*Fo}JG0}whz0MPHSU*;Ej+z485 zQNd;g9+zh1InTY4gqKpgu6n1pyN^_AX`M=1!nuIqw8sV1Pg)j#KW!KUd>BB29XVnF!Q|g6*h>=L^cR?D&z%g$(5a^v&Hx6Q@VlCQhQ&r*05&Qut z;exVznE_tnEtPbBWc7!z2T+cs#pQ_+GC?u0(`k_~>f!y>ix)!MDu&Pb8+^0U@_y4Czb*T#kyS2 zv6@QYJ0azSi+Fs(?m??1MCYP!Fk~H=aC5Pfv3~0AyIMe6?Kz3rC$YGu@lYsx*rP5= z5hIP-$CQBns`2K8z@FtO(r~VGyxEP__gpdPMBF<(@asLj`vS$_8?0k1kK=b+l?+lw zYOjegxBIc?CA&eAphic&2OLm+%jP_dqML@d0*0g0%K>50AAk4Qa zX<}F+89)m_%`e12c-rPrf7_N8ASMRHz6^~+OO8SV=2@U-5{Mmwm)ALfyNvDRYNWHQ z$)F?_Rwr&(S$x-|AZi%x%y8zp@z;AyXs_{)yX&Bao`@+ zrx7z|GqnNmmyq8o6~MTd^fUH<3+ANA8A9(+3KtRDI~`d7FuM902m`y-HK1p8d`D~VDF@cN0|>QzErp05@!(*B4%>v<11POKrl=PAm|v90JW%JMxll(MSn4UhX&MybcdPnv{ItP^nbJ>O zb26_2Lc-IoVY$yh_>uc@V!!($qGsS^v8)ZUmeH=pe;RM^K{55STy{P$YGw6sVEAbhHsF3H1AUSIt{5$tl9+49zoEWVDmx49wY6TF7ta8G&r=AYp z!tKl>!93^;QY12~r#ZtvXppx&{3QHHuOd0>&%k6;_~8Yf-xb_TiWF0@#b?Y5^zrLRNs=vg%)&HPGsQlC{z{ zZYlf(-JQ4%(Pk<}(*{ld{uS1gzXDZQ*jrvkd)xriK*!zGab2KC*=GrQ_tVN|1VzOt zZz97c8c{9wLL|#a(dI_t?i5HHLkBWq=Jo#Qi4N^bsn#8^N5h>C(+8f*e#A@+&b+o7 zCF3?)e(MjicY+UL^vzzCyHZZNYye`b>aLC4oit$OpEz`V4%5iwn znu}&L*(;3nYM;C_OHU|Ka|E$^kn@e61)~Hp3gCB@FnpHc`sAnfqA&w1L17OyLHtNlzB~X75*e|87ro6o z9Kl=B>}z!y)(ML+kBcuq|Bp0yZ|4ChnBCl}8i2fY;*XIf2i36zH_V+yBxf9U&P9&25I?~-y^s?GT zji0}%dStJ6vw`2(|Iss92Z`|;dJMrR6|#BiZ=LBrhqA&gu@L9?B1%%Oq9uf>rp0Iy*wwg`mql?dB9=1Pa z2M~I%E}JTkR(3DUg01T!ewX`d<`^+0`MBLq1j5-+$@R4L+65-Hv8CuGzk|QDto&I4 zLFv&}LcOzi>KmExcv}K%BbA%1ioihOet=zE9P(IwVXudK@0WXEEcPBGcJx+eu!j0j zFTT#FygtB&BqR$%#Zyv zH1dF$kW{*HFG0krv2krxi@nk%bZKp|B+MiS>#L~DvZxCebuA9n|4`x*i3_)e6wNREJxY9v^6W^$9Z!5{@Vk`-@e@rP4Q7U zDl>vTP$KaGm~c*0z{rZi<_!b$RyBH&uw^f^lG>gF?-p4 zSw@7bOV9?fVaJ>!7sMs`Cjo|bpS>?XHKLAkNj-WBEL--l4ZE-5)+wwYV3Z)%;fI45`Hb;bnf(Vi9AdzI{Tm_YQc~f%1QG5@oA>S z6~V6?uWJEw>gsAd*b{dg53^NEf!aY*bB8vip>5y-*4a@uz)AGb)rO4;466vc7#4=*3&(BD&6k9 z#(a-ZR1Zb2u~11U5M>XZCL76f5fwMlZ>2XmlUBBkiqiEss^{-hZRfSTI`|dxo;zCA zF|JN#l5nN@&LppkVmj@DI(OSHYQd`>mKzIILQ z)rQ%9!5imOs^&eXv|IP&31|~3*#7#$@v#iwGNTQXc-Fk0FtK8o(*l9I8oxF$UV6`j zyLh@NYh8U#o%1G`V0cl$M(!B&w_xnYe+2c~c~SA^bVrqLgIlb!4>5(cW%it1lp5IF&mEzLL7|Rm&fAB6%N{oslhTujd2LP3T2=lyhbrC5IjFiZ44m_lM~jI zmjx-K$Sfw0TKD`$T$|f6#xn%~-L=VR%|v2kU ze@S=gNOH|8JIIpXkmwf!()g5n)gyZf1f zK5a^I6P3>!q4DF5PbMr3dhi+IYJg~G@$kovZ@)BG9j1_6(n1pDaT#X1U2>&F^AF|0 z<~GgYrMDjYQh6Ts1#l>U&LXbH?)a+G7XLY(I;+yuy`26jX z*<1jtov^b6WJbG{l1tmjQnmukBwkOl4n+MuE=$~U=TXcFFWM;`H~rJRyjrTAC-+*0 z&=p1#RQCuOIZATAP>XwQs18HDthkQ^>ba`VYK8Gq)?}t;E&>s}tM{Op2>%1;n4>IP zsV5SdZFP7eSj8S9@zYb5DXh*=O%8io1J3IFL>WAk5XX)bi~%tiInYKxV~XpCr}=D2 zudsYBlZYQ%l%pa`-w1&qamG{fV~(SuA%t5pL4DteXqo8+sI8?63Q_hi3J1W)9GTbk zY$xLaE7};n+4sBXcw!9#TXlLDT0dYmEG3iJWp^B5^IB|tX`at?<8lfqjzgzh4Rf;* zMOU1`oQWE)TfNSYutmTc^l^#8a^I4b(zzGV~JhVu80M~u&OKs|zc@^&B! z<~&z(AL(pS?Rfs1ONQpcYKWm14ZPUOG?oaUPAQf;R-JdV)h8As3KE)EouYWuL8Hy|}iO9K0%WP3~B z!f+}eW>tDC$?ez=@xb6SvHIkz;VUnWS1dT-p4-`?#W1y#6@UXl4@m|R!n|pOzGN%+ zAtWW!?20Dm?FDJ$T0hS#CQ!$*;N1=Qj`}=r>;Y(V$*jI8VdDe zg+$xpzy*ubjSPB)8WD&GPOtEbzIbm{k76~}PZ95vu%pzctcZW8on)-6l!rBT1I#Sx zrv;4qW|P2&IGI~dXHD~y&8b`^s`Q_&xDhN%{d_k^vP5zl4Mxt_%NcTgyZUBu%N7>5 zj|xnBUCx&G36Fl`j2jt+HGO~niH7@ASyjzrn4sB*nVTV~)%*pThKq|?G>d`Oxc z33T5b^y;OkaL*)v{BofYX9y(hQ#wWwiY})~kTqou!_%oY(Gfy}1!}v$t?fKAVM@UN zRkEh^AjF``Pp9kkjx)eg(@x-$SP1;{%3j_{aqd)Ju0u$vBK8t7odUpGZ;9`|U5IlP zXWBPyedURAe&gG}QWX_^oRAK7>>0@@cYBE5nHg@N#jCa3^hs~WM|b>U)1$xlKMIc_ z%x3!VAHsskGsdT31!&@vTB3q1bA#_J(=Y5^l&H=Gr`QL}=)$>lRR zc2+L2xGESx%FV4ekGS-xzGjAEK$j`N7`Wg$OeKq#)(qnf3T!)r@=d~i>TY-J*2KX` zt7oSLpsa;)7GDpOceFfzyuuWH0+Yq$OO3mQ`ifVAI~-B)(bmtL1&P{qNF~u`S=Xoz zW9n|4rK=TYqBwPoCC|H_uD7Fow*oSB#ti(8lbr#b`t0v>G{M-l4T7H4k6ei>6A!l| z^GyL=dB=>QwTl7|EGi#6G!q9bd0^Zu5*icxapD^B}JPu`px=dyG75ad0n}h2><^^vs6~u0gFsKYC*s3sT zSb(mZhTlw+E*z6i{CQAJ#2>$IWf)E~_?bwr9`4Jp;lN)_Ab2k%*o3C5B%p3|^+cIm zi?*{N^G5|W%of5#{n?J?gq#gTE55wf{y3ec+uSC`GnBIl$Q?EV#c@*>p~H~1w?>7H z#lW8h{53?Sek&p1OTUtFjrjaEG$MU)ic@xQ-Hxz(s&_%s(>9~*vz0sIi*obW2Vd6r zK81==H{=N~fzaM}*&j&3rI>orv~00mD0Z+f8_XWn$oQcU1>IgoulX91RR1rM@JUkawF1x?WN>#qXGX9?L4!g_X8e=#KW92x9$(AZ)06=PS{u@b zqVsW#-T~tiIPrk;JB+_G-MP%D^wE=0G?ky*fp;=j_KiTm=2iE6cPFOc%d1s;0K~;@ zIukZstU6cWfBpt&^g+n+Qnx17!_3IE0-W2%GtMCTjFDfjw%9K|$uQWbmf}*uVV>`j zciWfmwM6Hl4XHXP@T7F}jTA&n1Ig47tH+Ni5!%QV-f@`Vh%vjoI(6}DVgwDq(=$XGccW4kecobORav4dXhOL_V>F6jI*3 zN}!Le1?lj?BRUpALxQ<&zy)??pxk*>t~oEkx$?6^OQHKQ-j1N7AFt*oge>F=S~lSo zqC&(plA9B`^3@ZN}?P z0^IF#ylPQe7B1jS7wfW9>jZ#t#oLZTNhS&QU4DrqjVJ4<+BPoCj__Rve1nyCDDOU| zQ0`~38GE_649d^%{k3vpD&_ge%Al^FMVKQ~F9QtH#Qvd6S5KFg9Z~2ftCi4{b(u1! z53#DOixC9L-*e`%9`29i(q)Z}MNf=c-Ff&?_oRH=+)LbW_bcd9mwSeLAYl;YTA4>? zel&;Y$xugawe;d8?V>j!7d+s^|0m+=e}xt1^6UzEYi2uVp^2lCXe77WWBKsH7sXT# z5ru=!Eir&lz)Gm@2xXMioQhuDJi7s*5@c#c82yf zwnX6OjA(Ormv2ViF9oX}MyHsDtH(q(EYUk;CPlaS!|%jRa5#V{VjCe2zO;#rb@GGW zT*L}8bMX5e7*6J3$+P$x2B)J9W$BahcBf4jiHH_RXtt!jw3K!RmWAChi9b0yR5kdt z{mIj$)dONFcd}h>WMf`A=(nfAyy4oFCHbMcq-HqLhMl#a%OA{D%VOsq4Or#X<0&6t^wy!JEZ88OS1y$Jli1|LlN}bTAU}+sfiK#j^m3$C z56b6B+lM)KO@b!nzk}9pp;=mu;Ri46YqBmW z-jLr_g=fHSZg2`#=k`|p#^!YsiisOZRvb#IRltbHo9Se5S)DLyem|HMA}^~+Tr{~GCltJApqRR zGi0{Kbr_FX4?ha2!rjeY4IacPzSedvxxN}j=}KfgMFpEyag1fjewd}<#>m+l&K`;EW+Q6o>tC)R6&T0*`1HqXa%kyLtrk zi~Nwt{FGcYsxBOk)R5zd2pCvk{BvUzmPz2%q8`j?9w`t6qJpd!%sb-vPj6E{USoRX z5?cP<-ZLU-&6q!qu(544MM-%%uYrvp-G)%Yv(n7x=N``0uHG?$wf>#`z7Gy*e_iEU z;+2mQRc5k7khds-Zu@Hw@nVE>`l`W1L4u~?fzuGAYX91JMhnr##N`JhOn;qDc06ks z2Lh@4ka<`CA}+{R8ah|t!vG7LNNfSdu-E=>;+An$S8aF7-l%yES)uL#$X5zrV*Pa3 ziSK+U!?Il}ny`HI+4~bcBmn18Z=db-Hz1?haK8yD(gltfS5^?OzhFbKW%RVD>Na+w zXT^_z+o>OXUUs0XAZa_G(hYfXH}hOxaLtDX<&K~Ko{GXD4p(a` zx5Iv7xLH(R9O-%3jPgzG82(H4B|uN?Vvm9?%}AM8)^BNOsUwTJ8}~37aACfLjSzWg zI_Z8x8RdO4u^Iq4U?&r0ghtl}2cJbtHt7%oa%~QqwP3o!?~udPd`5duJ~Y5q`H@a& z(U!r~3g?-{^b$DngIOkjwR1mE6X6)Kr_|$Wo03cWi&R#vx=O$5;)DvMn~aP(P74fV zGPOP$3Ywj=QdZPE3`0{9R3(`GsRx3Yqj0;MRs|#HiAtS0kvT zf4>wq`&>cT}$9Z zX6|bj8XahZ+27MX1?L_3N&qTeoKXsC=+}xP!{7umTnrH+5l-SAfE8f4peV4Quwhr( z`VIbs>=E!$^yVY69#i5GPnbvfwN;m)`XwJJZcglVg%TW!OY&$sd}V;bhi>xmOdF}V zMA=_|fE|fJkr2+&f=EHGa(r`@={0~+DFl$t&CXUVwXd%(;a@#Gf*SunPpj%m3 zo@yxVG;;;t|LMPKa0-!X&Y5-7_dOF8BX@(L8`K~es0s^z!lF8b7 zMDFBY6O5{by{q}m8i|#ddR=s`VhjRf@s7z5eXiy^sEAU1uY~5!Dp6(vwvC%cQ!8C{ zU&hNYFry8ZDK5aQ6ZivkYcri?RH1Wr^=hA7GD+zrDVm_gwVix~Bs1VSLdeb0QJ!}t zPeo4;FYqZ+(rU;7zZO5DrHY&c5^iN)W^8oOac1hZJRsE>h!P2szyUuHqIgxY+ zdht9SDo{EaDuU;xQ9=U8Q9ULijw0GcU}tNpT4SnVZU;7NInCoVt`g2e>*5@gr#8TX z_o$G_0;CbWbE_yIgX;{6sD5JVFm2sLNe_q0u%p9muSXqDhl-Xc?$w;uewlO4`a0fTJ8ujH$FQRzF|) z1aPpzl{!!3WAQpw#b(&1cDWAt$9!+;D*|aNLrRuekHWUzbf#H?a9UXQSyQ5}zIxaZ zR z)mCwD@200F{$IR#sZi$h%*8;X?9X8%tB+v)4(imkqU;ty8fpPg9QUpPu|E8K^^BzX z%+tERMGqJ?V*Fq-L#r^j&%-@Mhk{40Z`i68C2Zf_iI|=b->t()Pqv(f<*uf?{d6yeoTYU@oG6L`fd3j-Z@K6PrQZ=;Nq0E70#LzbqU*;NBU4K8K!x^Ch6a*|!|VC%Z;OIzpn#C8ts^=n*rB&Jyvd`WW&L(>d-yKL@3P(LSC+OSq zrr|^14YH0Ds16gEV^9^74tu2+T99}-!K|ZGCJJcv^>C=k6G1T2v0~BL#3Jm>Nrz<> z2KwpXes{IjZW9?Si8Zb{-uaQ0X9*Qjkt@KxKL0zlYB{0%wIhs~p@@3>{j~^3*I%Za z-*g{BM=%YAK$PdS!#R-9013bO;l$0rXIuFoo(*HpgC~q&&%cG$K||xf9L{aYY|+RM z%xZvS@`-~`59(Fb)G-h0XOSDSmWQ|m0{{MPSFfcJ`J^uUXOKR(y)fUMAC`wm&)q9TPyZMaDlZ1UasSHGl1VP`;P#GB1bWT`!Y*nKg5yceKld)(}+ z|0a9{iVDO~9_vZAR@dU~GP=->U#8076?)2WYQ*@iL<)^s<$aVcR4_nFP>5;d+N6u} z<4T@{@A>dyFB-@-gOOk95V|8Ve&Bzpc1Pa3;JnsBfDnkk&6q zT-u*iO}*Ofa3Xz~qYYTFPs_h}T4!T>s5#5n>Od-Xe5dCRZx6?xG0U{(C5uPSZ@D^|`Z5;Dhi*&lTUixIzZba_tDu2^AY0EL~t`|Dm=%!tm zl2Wt8rKj8FF^y^~U0*Q42PNzxusB9Ai)_m{# zG#~CDRX*k&HkQkWiJaW0x-y^pUCrf!E!QE9%ls;9;Yxy`8EC8ZB2JC1GnMOCAEkV* zE6{X4S&>2wQbUX)gKH2v0T@I5AGF35kWaj&aPIHdlX4kE-2PcXMIjCQY7Cm!!HNB%0L3ZXGweV-%U~2%wv-az3{>)^p5zDl`RPY>K zPIWY&P`O3qcY+glTpZmR#k}}UEH8+4RNtHZK~~~XM~--E%@)N7V`dGWK~xnB&os}W z36RQ_2y6D1n=r=X++8Og@U61w*9r!UbMNxGd+*(Zh%UGwZMo6cRB89IAGYdc3g?gs zaGBZ_asIn?KU$Na*bM^py>b+yqrNeYSRmLe;d#7<<)2EzP6%>31Q+%)QW_rrB1FB3 zmV#Hs9r}nYmse@45tgEC4-e2KKUOo9TXtnSi$5CX z38^UNMT@xa0#{Q|(v+yGVC<=+47lkE`5gQdZAQ4*RYt?s)7u@{1k;dK8t78ziA;R$ zmkNs%C8+&9dK{-o&=G0~9MTg`yfZ0?xk#QyyNMi=0!~y2fA@A7rx4mU9w)$nU4V9u zt^dj5;dzo^>0l*+>J+;&_a!F-r@I)%`;_n|S**aDG@iLbAbY2QZ6X`?%yFDq@R%-q zHE;wjT7{K_ZII()htH#kz+%ra&Phs$G6sG&j!yNg1mj~8W4E)Fo~7MK_?n#Pevnf< zQ79vr{YQ0k@xf2j0+ZET(VCsR7mdYjt&ncqSHjNF4}jusU6wtz#@{3{(FhHT&6kgm z@F>}lDk#?~1{r!40M-n0MPTbB7$xo$Cb2=JE2X3!uhP6Qmc7cMfR^mv;su!?S$!vC zXM)T?6V~k7%9m~CUZxxN-a)|ylO3P@*K!XYCHYs@G?sCSJ0Ig#P97vOd2XGvTaWVe zyL#k{6@*do)YQacf>mZzX;Kc5QP};(_QZ6Vnkph2 za6m@0c9URvk9v|RwX?ZgxgiZ`u0WZr!eurTxHT2uTZxqOu0juC`VX$2`qjKDkU5Kl z=FOhTtsmRXaWX8Tfi&t9!y9Y8=vO>JPRhM7wa16l5I67b?AhJ8@C_I-;3z`pGAyow z!h&+o^HH(5m<)gep4wZNe(Wj1B>cPzblpk+4Uuf~Z+l$;_YrCj zv1pG{EbF8eFx+jeS_yX9-etWhU>%dWb-=~C3@0|>Hh$G*%(j*va=T=RVX5rvsvfa@ z%K&WFZv+41DYy8x;G{ICCB$&}VmxBK-CwVLeeiKr+qh~9IxXo9m5@+VZoA&DRptXX zTU&CdU1T!c4k&bp1HED+M}QGT!Nj&$1)c!;f#P|%fWxuEY+-f-J@C$yR%pVp+(@Ks zHrrQ~^M%Y@AwhE{CWgX%5%n6)yRm#WwB*2X?6)p+DrErx5fY1 zG%v0HCZXcSb2wTrP&N>TnL?l#P-;|XQ&_w`9g3n@Q61w) zHDGkcoE7Upy51!Sbgp|i%`wVjE^!W=8-L5lzQOu^{{#0j5h|*)3O9XZ>Q@iI+LPN$ zkU*e$rr1=Fu4kQ%3QLV5C`rBA$>jX|M-928knz3TH>ya4_bJTiU@zV{gKXi_`^oOA zFLwXg=7s;*soUNCYN}gY4os5DxDNbw66WB?BgP`>-fk|o*#VpqGBAs>Q;&n;)e!^*s5(qDnZLgaLWXHBV+!{C!+HaGme+2>ibx=TCD) zM9f7vO=NWl=Fcfp1E3xoDT_$@Dqm4!*yNBOB*_WXZI2lEU?d~`)D9{lXMnJm9@;UM z!B<3B7tC~eh#uD<2E`R|Y|$uVx%_)4I8f*%^gx0pa(ZrW>&{BNp;}aAn4$SRl$t_r zbQLvPveL0P>jGv9!iUJJ%w7d!*-VKMRP>?u9vkQ8VpuqP3RHNvYoHUV0OM^7R5!w!iR$}=F08={XHmwsS?2*38 ztn4@3(la(pWteMX<1@mtDs$*Ak16A6S2VS~0NuoGGMBzk z8bTEd7UHHgySN7Bvfa8)B8G7AON$j7I}`8n?riyFrGeI6&|UUwf6xdtsa;_y=kWFc zsUR|*W;B3tu%lJvFy^LoVP)5(Bfeo)P8)@;$RTCKY~oj8q;AwX->oOq7EvLs7mM&B z=Q@S5)?tMk9FQc6y~0<9gEuul;n(3eez&De5JBfPa=_bMA=kv3Hj z>)zH(`0>^Y;mXPIt5hgJ|0fL>#=Q~2&EMF^8tB1A)1Z}la!SBOa*eiqjonY-_Wch7 z(|nL$j}#IBykn1kyera(1?X3&pA&v6_7td3I5=lJrFyPh7B3vt*hS{F6qk>yg5?8N zY$8H{5!I*1h3=ZMeliiEFz;mr_AJ+t*#flR>O1(zq~XwnP$NFTh0_$bdf+16r`g^Ys%J6O(4UxUi^qA#0&~ zYW7IZ+LC!dIBiZ@x~Dnnr*I)h_}5CJod@f*KW6^qmAv7-3iZr6$`)A;hXG8Q{Nqf1 zGMu?Mm9S;5wKh|A+_r{XQcGl^vO3TUTJI%L9pgnST=sM`byshdF$JPVz`#U=h_b@y zatY8WlFZal1waZWf1sn8m0QTY024Q(z;M(`?0Sjdo=;T+3Q;t^uIP`BEh$#%s@7v^TX*}na$Xp=od=E|XK914XD+WplG4%8>VL0Wj}4Vb{vKdzsx3*`|emk+CwF)XIrCqcAtfB0S4>ov|H_Pa-^h@uV0fAd(K(h+%%+`!Kpx_HP3b z$lk>hyM?+qx&&~Y{=}El@*8xLiF^F#$izF)Br*MuR{0W@5Y5Q??6Kn%LI ztkIp4eU*P;VV=86fY^=51f#rIJt1W>BTx31UC63f#UB$I&VuZXGG28PABw&Ejj*>x}^Xh zh71C~o5KS1ccCUnjw)`}7XJ&Od&}FDVEeCYwr7CyCwy>L5zmDKy_wFU=oKb+jg+EC zcqw23q~o^3Tk`i$;@t$wZ{25n@hHQV2-GXA!~&WLP|s-7JVejOWqYFttUn45{y>}j4)nexTI1t4+OdqT*MXI0cXiLI!=6;#ft54)FtianL<-nvDiE|a`VvYRqyNIf zN=|8cD>;#%2~n;cV?eT9e;jUdF2jq<#$y1Ys-DLmZHr}Ip!czz29K%3RU&X#A;99e zhfyF@vh1h6?iHk+h@!Gk%W#T>Y|@f7nrvWyO%ULOqhCN80q8P-1u*JBo0`N%j{p)$ z0f{WLfB^l@2#g+hfe=Jbw{uEXiSbIL4Kxb|QYJcx17dpD@IN$4)cI7rs25TU{c$Wn z0o{}=&Iln%rJfKkP~Y%ZAkZM;+e)&2)kHYikEqw7p2m{O+7LQpTF$y|SY#H9$*l@i zP!t-{avxEjZy`S)2I58;Q72X+a^b+B5KEo*%p(1Za4S5O=Ze~K%gCs>l0e!Oy#Bb37EBf}s&DbKqjtm!nNW?Ui#&x#>d zGr3}Ss51M8n>XGsWrb(CSma!*<9P(W6utLwC+?&{Mh#o(4qUqGI$I*HAu5B|($FjT zcF7XSClW68gS*Pd#ahpP9-TFv>>aYHI_+QLx~7Bf9v_W~$l(!@7uh;YVGISdUAOb) zN+92zYM6pUa&?TFS$7cEhiFV-FZKV~H-5*L|w)zcVoYvr93LqqVR!1I@NJZ>d@~8`4<8L3i6?`zx}yR1YPf(2x-`45+jU)VDB58CGWVEb2w_-XPSPEwc&gV$ZZ~h zI4*Wh1{wjlGT#s=cu)p)9wSG95kx@5wpb8e0LS8QQ}Xz+1$$i-1J&SZfnnmiJI@=U zVy=h~BR<)HB}S84PIF)-XtlzikWhW)82s=cLK3<1q%_-#-&3#?Tyuv$%U?&05Kw*6 zz$=kLYoNupL4BY17skdKG#gLzRAXaz%f#Y>a>G1W?q7+L??NGk z_W`yt1r_07P(oxe2mB}0-GZVMn6&;*JJjK>2`OtW2WIy%s0}QA z*88+7q+9mo)}W{N{0(dqsZSYL5I4Lzeh1i4p9XUYDzxc#yf+QtS#Dif? zDtXi^u;+@g0g~6HD6;|4QrmVgqmN3Rzm2k6_Y8feEQp0zEmjAJK>aZuHq`2S`wHy?ScZ*1v#&zByiN^K~bCIp1eRRCiY1>^pZ^^#0V| zvxcb3|12pDW6agMQ(DF|6;Re6T8favT$iYLWv~!_A}2W7V6RxSUcj78q}#J|(fBJS z0(7Q-U>G%G%6DW0wVn5J6W!_jz=GLun7x7Oiku{lhQBxw5hEXiUu8*}30>T8vJJsd zmlfb)TW2Td#bM#Y^@0dDLlROwnuG!&j*d`%@b8%qVh=!e8e{}#Kyr!r_J?y6qbz^% zd8QED#=kx||BJ`a1h@=$OP_yV>9RgWmBRIpWf}1RMBj)PV?tc|iSb%C+B4PTf`-+3 z#Vo>HrZ*}jvsNrQL%Cb4DYRcJiT7Zz{=dO$Bdv$-R@hS9=tc9+o^~a^Bx?64I{p|= ztRql#(LscGtuEDap$qNsF4}HoC0d!o+Lvr8p-8gw_7$(H*~)fN zGL(;X7vIGzb|@KxWAnqLVLc$p$2j%l=L~)h>*mH|0#Z%UYl_?j%e# z%$yKZE)>{^6xz`?B0Ym=%{JzoSxM%bSxVf9kWJpIs9jiz^=o!FJ>Y8>84X^vXMq}f z69iFv$OE3jT=^~{VaU5-b9#OVUySJJd4U{nM3BYM2=~g3SJSSKsTpN?F{A)6ME-Ez zRL}Tr=Sps8JlaeIm{8#7hon&4MNo3Fu@G$(h`_Mw3@5rtu_x7U9#~N>b4KEV2z+Yy^RPwp+jFo*&cL&g9cf=T9DkMp@ z!th>D{!SK0vUz#r&hku6XuI{`ZLOcjP;ZvelivX)$~^w?|0Kmj(;9re9ZZ{nT1s>1 z9YmcSd0uaHLK%(&i_p_O=QB^==|+mk8=TkdW|+lVY`Wrcx{G4ua;ArCQvj_id^Xq+XaR!p@k52ApdBF^Y&0)Wd5J1qUOZ^F z@&0Q9M&>60HB+kOKA1birxyWqA{LW8;6e9}4&RFyZUa~>*+Vz1nMM(!$kB#r6VmSS zmAtYgy7)~Zsvin~6J=|6@m{Ic?>4Om6_GJWxNaa#`nUGtQkqW@pFzJ3`Q+lYMROlCjl3$BoVCb*P0O z>$aG;ra|XDoV1gYJRK@^DsWyS^OgTA%1S+|uZrZgY4vLnS;Qkue6I0v|4577-mpFj z8FsFmc?}0M!LBnhfj(rSL9Rp_xQ}5db&KA0$Z#fc8DjSCbKFDc_Q^DXoQXjYSFhxyZPV#K-NGmBN4!@V19D-(Q@d=`}fCT-ZL1gu!49S zye*f19Whbm0b9;~he4SF(Rn_-I4((AX*eL5(0Vjbq!Zs=lu?~0Pd`$vIW!Z-JMS{K zQAyLlBxgO%F_pevE&`ZJMWJ|i@5cG`WICHG1%6jKP}UGDEZ+J_Qq#*DBKFh@DkPfJ z&#h#_Qh-;_ZJ$gmeT}MjeNE76;P%9p4~c@WGF6GRZjtqbsp{Ldz*)Npx7wrRW{574 zXt0~cQb$5@xWFheWP-NvJB({3cWFW9eCnrB{z|1#vN)L%dYj>3x8+|ItPx1yKB?~) z!?eHqeJXeDuu|x9}&KZvB_1VYVx$kB^`^mwLlo$6t$ehuVx=0b30 zMbk}y9}b)!4Q?T-2!u?D#oy{F>s`x8ZOKaJx;{FA#5EPWO1zSK2?~~L`ARk{{in2A z6Hsvdlll$BOiDlwtWU-^!FIilG~8vjSKP#r{zz?_W-(=f)C`Pkedtso1_^^^* z|1T!xGSzaKvMLfLlmlz3v&*v5mA0GoQOC-LhwADM4GWLdl;GmztT5wK(hlYk3H$S>g_h>IchwwN(*7kVzIA&rsqBw^!+d)`p>H>~-U^ zdG7aUawh+Cw_-&9+rUE@_nsXu zKt7vWfx*&HA?(H3de>SAE|N$$l31!WN<6ueU+ppRPR-IQ zu}uO>p9|+7e zRK~kw4W1BjR-`T~N>=aT01=x68`8SJ3?=s|)vj=A!8h+N{0L1&LO$jdRXeN+Xe>lsHH>{3)D@*1nA?!I{tXo#dQyiOx78tIbygw zJp3b=gL&7Ov8zHJb{jlL2fdWxX=0Q&wfaPE^1@bH5R^3eM1L%5lgH-A%kNL`=K?Yq z546s|=E>XI7xZ{~`jdcZhr3(1S!(8_rosTpEE)m$GUf#^@<5+D z3`UOt5=eoGEDL}DPA*Hs1QCeLzQ({G~G3Sz9zFQac?Lm8}fAFh7aBwd!6JpW{}ZQM{5^4j0C_Tk*k7f z2#g@zXkJ!(d>vRdK=C&7vsk$ND{y`_>?FGJ;D;!61IDIni9oY7KwY z83zu%g1gep!sc7C5{ak?)_C(g&Gdlie3ZDiGjrc04V7Bj6d8_{ybidn0{EkE<5&bm ztC2X@a5Q1Fimv-v0}2+^Se1~kW}lXY!mEC0S+Kn&^W9ps^-5vekFy9wd-u^((wkQ& zeabBmGk6nVp2kro5_;nvOBwaqvkyo0_317k@XOZ^ zJkrcwnA8K_wh_NghzfVWT@!#DA5evyW`Kc+Hd001B_?_K<}Y5`+YwKSK<*$vjfEBd zx74`@pgnOjrffD@&7H)<4;-v8!}&2FG{J_&LsGe{s356gO0;N)$c=@@p}7@8a$@e7 zyPZwG7=QH&6jpn@aUyAgq=cX7k=s*F>V>eXExKy5YKe6PxzAcmsIrI7uQLk7;ts`v z8)SX|5tZw!>KXxvGMOJJen9|zS|dhKVkn3>mdgSQzyQgK_87RS%;_Uv^EY1sCp}4u zms{_rmo_jU20yp43E@FZR0+WHCmy#7j64X)@p1>S5H#k`Tbs#>$%Q zIkwz%crI0@AwUn|Y}^)KW;H%Mlnv67Q5%^W;=jRV3C?2CQ;$D^jaBzDgGKeN&kft6 zn}XB}Exv)cx93lJ=Sf?O1w>3^-;HdpIx+9Zh$$s2f~_5uMz*%T?NLKmS#5TjUfVFi z@Kwy)0s2SOWCVa;(bLf0z}lQaz-dr@+TvEdUVdus`a?8QZg+Ue7RsrX$vFPujhpHv zlG&i9?pd||RJSZt;?t@ohWwZH*cImo@GGuy4@UXanV!&L#z~D7f{?QQ&KCk@3z_3a zo?0Vm;K>?O2xI;O3Sl-Q>o(L^$>Aie(AW>Ct>=11QqxWW;twzn->D7Rio>1DJKtl; zDB38$c(5hVcDb}`K4q}D2TqWmk=$cIzdBUinemHzgp02I?we_OsRCy=R}~vs)hG1N zULYrfz|9rYHWjw3?PzRtbytb+5iqb=2!2+*XIP=373s@`k(?{~!c96>9z@O#y=+KZ zlH&`YTq}zawlkMjsbgjNhlUiTe^>s5sj2_=E4O=>gy^v1FRPCVa)x_1z;+Al&eJ*t zQUE`uGr|%pO@k+gc`cnWGV-&CUHDQzf#z~J@yCztl2G@5tI9`87>&#Wd%c!e+82bn-2oP`_7v z7|Y5lb)Dtej%NPGL`fe@6Of^)r;c%gP^=<_N7KdvtA5!=wKpO6gi!?-kl%r+FQfu~ z-h}4~7)Ojdt4z1ZHG=uzF92*xQqe@_#a43KS5)SCVa|0Kjj}+@yE8%tVn;_;uj!`N z|N4?J%-PXf`rFpY1l3t6c=K-{zIAe6Wq;ln*9ux&Wb?n{`Gto+@@gi36Q|wWVh^oSiRThNuAMWGZ9C zcIlPNIo}fZs!ao_pxBsDyH5U8mMtV338L`H!m1svPFCAo_~=sJFHz8p zWMxRQk{yo^rR{vJZdr^a8IwjY5f;jnYl#2AjYq#r1bt=dmTg1*2M(h&2M(RX6+)ha zr%!F6>q~8s>^uLVkMrf$H!%AQT4o|c_xCl}dcdWeG!n`L2Iof8W0vxMHIvq6+ z-k6>vkFF7hYHu&qz@j?~sSQ)bV#w@v1@tT0fCs*#H#3-K@W zDnq{RGgD10S5F+6_2&prD63o>*zo8THB4hFOxdY9xuV-3%5V4MA!;tv-HbHx^s@BJ zu%9xi_9D@!E4=D5f>^gP{y@lbc8IbBJQw%>F)O9C&jYmABh^-uQo_h^6R8v5{ooqYRRW2q`vE)k??X5S&1N z%*{q< z%Hvc8Ar7(fUt?mfGlfEVT1ki*sw2-cHZGRQ05|zTXF$P6rS8Q#30r+)m3=MnhivYu-o8Ypq z5;YR@#VshyOtH~03p^b&`up~HJ7BZIQ6bV;ZIlZ`1hczzgZB5F9^UKzw_dq1 z0>;jhQA$%y0G6wdX8LB~+pR*Qmu#FxGkZNA)rnC%?T;UUqRs4Q??t=7FJQWq-y!+} zol?f5aJ>An;w%XY($EQ&ke95K?H!{)&*N7JM%Gp~7u0BpqR&9F+<|;cYm#nOj^Snm zykkxI)_FWv(8=rXUDlTT@|`Jrv(?|N?G<5Hx-Yi8#aOIbhUmb3B+@3IAR#AzY(Dv^ z@U+3NXBnUn0G?Zw;xR^C>SdbnyC1M7ov#tK9~2`{_W$}xvGZf>Qe%wiyj#jfJs1o( zC!piV;K;beP5o>5AcA0ZL`rNZW`JitW1y?+e^?@XA|xHD&N3xgU!q+SP35-5>I!W} zYsdYrA@?BkL=~5H2fjx3HMSpRjgF!2JS<9Cqooz24BVs%PDj*9 z|N8cPxS35>hqg{}w}}2VLIaAs=H*ez+t(@ivnrUJwi>}r{?4qGB6^An)le30ck6q| zXYX{gr-jby^NLA+MV;8JJoqH@A){YFYWL@P&5qqYj*uHc*Le4?MPqq$#OY&tju%(} zOcoSdmiz;V`*@4kv#<3d$$>#4401_yvSM<5my{BD1I+Gb;H53R(`EhY=3GWsx$g zP#G*Rj$eOUr)HmT!5aM0umhe1^ea7trZ*&y(VuA`qcik7XSgjX#9#iOE&JO}=e9R_ zh?6gGxiC&gc4lE<)=%AJwVNw@wX9JS>-eV0ZQH@6Aqtt4jU&|^NRyk;3*?j84PBMS zaDEQ=9&f~t1!D)_VML2b{IE1he;*RERZc@swTeYj@?xdz0_nxkjn9$Dni|bsHmfgE)-4wKXcWZ(Z3+0^ z4mZ5!Vc^!SmCqYIgzBG@EbCxs-lAt_KT4iLw!}Aow@%lw@{dbdJNOQy^|Hme?vs0t z3&)OGpX@X!EsUH{vQbT7geORhj6RA*(g1NAjs}S5SGCGw{>MT4v=6`qf@;+6u3eV3 z_oopFjUt~iX~1H{HfC22j$X&aMP9j8ChO2l@47=?_A31~yX_)*#5fgms$m0#;%vxw zH*CD@?xs(lS|uu0_e~1NOJ5>AsomIWMZ|UY$$tA&?Sg2Ll0!!^Xnv*E)C~q z_z3FTX=UvyH_@^(ITPK++U1-Z2~O=0hJw?BXgw>U(aKCgCHbJRm$^YQy8njvSviMz zZVrO7c;|ovA|Or$`qWx&qj$wHF9oEKV52J94bOi4__YpIji-zUHJB}0Jt<*ZqDMt# zzCq6|1cWiV6+5fGx#tNceVp z=H@KQ0a?^{8JW28zLvvGgF0RDWno|B5oD)@7CR3oJz1u8v!y%KL3A*|%+F=?yGDt4 zEOL@TAu3ZUKxkl1g-gAU+pCrr@c8qKZip>|>|sB|-J>D*f5m*)T`E=CSMC_4h`6O;htB8s@rkE}3Y55DeOWgG!xwNz zL9e_5kfsi-mA>VT)`Y;f-Zm*;-vli{Y6>yM`H+@0tJYFgb{xGy#kgOJyJW&PoMGI9OGa;jzfU; zhvQKZ3*Fk1gtPOm7Lj-d(YMsKpQkli?#I5VLVVTyj^rsPOTb4Ux%&M5xF&Of1XgbjH7fRW-T#~au<~LmGw4(ew zYRGE<+t^iSCGSN-bAdTJZ-|+yj!7y9w0YB--9a8~-`V9fI|zGRT2qwGsN>2q2D1+h z9POON+5k>2=ixdyU!wdhkqb{z|C2RItD;Ds0+zXd$&t$ zZQ=bkB{yd_=nnZSswSH%bG7T4bR3B()n)qtAhFT|gi(JvN5++`e9qj$yl<}_4vC!~ zpRy?~Z?G4pHNuu4|Cc7B+G#@0hCe;|nWf**0E zRTb4~2v|(GAdztPWutVwAnV)C?4Uw0xqJ8RyrOh&tUcAtbQ?g37=)VQ(Sk05M117r z1uWWN+*@(ttC^wX`}+Zr=>)=RS0~<3Sz_B=;pF2;LYrrfl-@{_b~Ss$!i5#t3`YWn z5%mXbsWw7;nKrm;axv6^l`TFK4->Px7oI<2z&5Z3+}i?k%LCQxynm+jN*uf$;{?LK31ES} ztgUGz@SFwm{Js-{H;^ABMwrk685>o_HsA2qZXwn$jQpr0o< znH2bDAjqQtb6Kvc|LQT9Ivg06JQ-K?;P0&?n)uV`h`So(%T+6PV`EIIGTRNyJ6@-| zBrwii*77a~#}XiEBKL4U;c)U{MV7<<%_*-E0XONi^B1TL;}h#mNEO+6Tljo?^+xhl z=+4z2TXK$AVltVt(aT@;)aKT7Kew}D%vL_lzt$+45DR>z9rrTJQZljbKjnIw8`N~@ z8|KEx)Wf}6&CD}HG^L`}-vhkh)UVl+$WD;~ z65&7hnV7r&)tOkndAMy*^zt%zU_{#)LH>>r4CUGn4Qa2P&n-)j>>+?Q2baX4b=1*3 zYF;#|IAStWitj`4satsr5THMoD>3^(C=e1^M9nTgNa$Yj^itZUM0g!#I$2QTxr7?D zkl)ZsA|#37Vh?0sC~4|dKiu0FhD~+1gtp*zlE)Xj2GSFQvni3mo)?BNddA@@Ai%lP z)usr9_UU2Vg`#56u-7dgmv@8Go#w?thytXiD zOE9f6;SpFEH10+~_#K7<#(LrpkYy=O=mY(kH!@ctX>F00O1+lAGPndV_Fw?}oJNlT z5=jAxEVFx@UTxBUA(GN-oTRxB>Ev$e?SHu56CH$`VoT%&@HO=T!?QdEw+ zu3b-%ZW}+@aD|}%AWD|?VUvZAOild`O-BGd*}&S)_oKR)lQ=EajX7HquPr0NvOBEG`) zHtA0ex1r8O22+a#8_~aAdIIYQ34lx50P}X}g3&1NDOamM72B z?ZubnYQo(Y0Fr~WbV>4)tSa4TmuW{C`TuE!Ye#pJ4Y!(kJu**~b$ikIl1HK5kJ5CG z47PaoY|=J*oQSz0c1HJWk5u}gPl9b}8XE`g_b+z%P{+@G(cruq0k|@f3n&0UCUssT zM}P?=K*W|=-~j6QzB@EK$__HrI9N~k9mO8%&B(&WvjoRglZ_h-=JFu=grrwGfT&9*b+7j6>H&a|R^ zFveqv0v(eT?I!kF$HoscdD!cliB*x${5=_?&D*U`-rYVKW5q3S!uyC-$KKM;T?d#V zN`9jex_>~0A?Dd9!SJ8TfqnDCI7Dz&LPBP!;RMw3^Bko$%v%Z{hx;s)2aT%x@ zFG{u=LLvilu&j5dI16r9-myEQEIyCbA8PMgTM!Jn)G>!riS3ya06_?j8NLSMnaSKc zpf0N?@VT>JiX_~Wwnx9B*wXLJla|4wnhLs-R^NqQYt9s>(5s{ z%N&R5ClyU;pD)+(Toq)>mtFavA=nxknLOP_q93D-8ms>O?dNz`gkS47{WpaK@*)mo954M z26hycPwOLiD1EyvP-GeHczbDO4dpP_-ANU*w{X?-iceqgDhK+g>iGFY*%B9PWO1( z0N*lc^+CHbtnd_%H7yKwQ&`4Z4vEWsznJFkaS9FzLVn2$GCNOwP%AhLy!^SRI5rV( zAN)##s}jdH!z+Wt+cTM?pYdM7mC+w#ZdJLbXac$=$;m!>#I47y{uIotpZ zL?r!`7o{Fv&zDjSSVOPl6)hn*^+bhU9(xDi7VXyo@?L^Ebz-3k`X*aoaWN+0<5zNk zj9Wyf3&9Fb9fq5fOyxbsWAaR2?try-z!mpgJ73=DI-51C z#}OPx`{)M6N%(?wwbI?a>0N>{%%>-Uw0A#es|?Y@@DXnada=>yQYLMF{THXDw@SlK z0CLbq^KfzU0nn*WfK0DOH$_2UdWT1ed_vLUnAUemDW)o2#%#J4%2t1XM6%I~orvg# zXGiwvj3ueZNTMDR%e4WN4dR|@p8}kV?W+va|(G{aEVw3I8yNR`$t$La$ z1EznQbBz$mTl-RSydanr=#ymet3|q`COX$lBubG5tP1wgEvLfS&>&USeN|b^>^Ece zWpd@y_IFP4S1sRfXn}$WA3{(c}j(Z`mb*>I7u6TnYExu^O%wrFt2Bs zOa3E2Izw=)m&Nbz4*h5{p#(7gU<~RMMvnjzNQH?kvw#5WGn1S}5DtD1(9LjwV5r3{FE;3=o$EN`)zxYK|ljMB~1wM0A`Tcst%tMdr?X|(&goG!xh`R zIT3+@a~+i*7&vf1*Q`)~r7mbyjff*;JSvxRYJMK;F!-;?!^u%vww#{JHmK}x`8lUt zb6zvLfB6_KXiI8xEs)8B zT45O^U;ngS#zT#?UILJ(n>M6X6vfIL70&(ycAXGL^tfsv{TJ+QB84qmAv}nO%3DeK zCiar#S&|e$?%kL~&Iflr9+KQdq8NOTi?-(&sCAAw3k zBCSYJf7tkt~c5O)Q$?A5NU)Y0VHX8>w-H_qwb5Ih}#OJ z9^`&tnQDGK{|Wje{xvnKjRD?p2!45yB54fINGCpX;6y_Cl4jnMahOghy_?(=j2xae z70c$92R7J8U(-YwHwYx)_b!qC*@WIIM96rzpj`Q8j6B7d-5K zwP7c2JJLJ@&V;=lEpBc^r2fsUTA%L@4Jmq8KB<5(lHXt9OS}EtNfo_F@7cma|Ab-> z#2TV58~OoUxxg0a>?rSOk4RbDvysP&I+SkxF#YKKYFMTT%OYE^Y_m@L+ zEwW5!^pBQ(V><&y&fSsP;IsCmbG85dJ3}QR4vpTNuPsXNb4daA0tBafn;wc_z#0Ll zGL$VS3P44L?juFeVI-J1n#%wHAg;5(0N_)}nD(c5*O+V2LZo1qhAGfIfI_H@`yZRq z$SBrjQ{(XZ80z*fRZ~)Ppyck|PGgHXy88DcV*qTnfm1YqclUJzBoo_2!j@E`Wqz0;Du~@;w!%ETqSOjy5#_E| zCPZ4v?*zV?x6amy-lM9(>Wa*)#4AG#_&H{+QE+~{XbaHkF!PHjigyZy0ksJq-ys!N zb?eM4#q$4w0yc+Hnn~g7EuxMiUS7#3mHXe!!(HjX<{C#2{yvzEWz1WCpJ1`sSWJ`L z6v5&N%MNeN))1Mtl1TM|-99|vI)Z#kkHk@5rXTrf8P~Q8pH^l70^p#4j`p|GHAOnk z5L1g;hK-)AroIa}WRX|s5-TFF_m|H|75v|EpmT8hI^p{BN}Twf;NbjJcB%NpDhe71>U%Bu?<5Seon6%?itJdA-uf z=RxD!=xd9ec7EuTrY;BM_}sUDj<9?3C7m05s&$IWe8bfvFexME)Ou!53?O7>S5fD$ zcL7~&&^71HfKbvVa@T^1pkOKIE2}epbh+u#i9tNt>DL?emFY|RSPeg`HP&l<^#&Km{Xzr+{*CRR5yon0f|?m@i?M=v98Xy!TWE*tF#0EsJ!XIf z&J3A#8DtD_mIb=q~0=xMLk7Q zA;|YscXh?RR4?Z=eN|4qH$r-YXBX?=emUsl|I=d@6FQ1`{|p)of$qrB%_%X#wTqCh z`VRyQDSf8?O*adtuf$gcLC7P1@de2Zf+#AIl_YX~t;O;bR>=C`(G?Z>*QZkavk?R6 z9KQJXmkBv8wgY`6!quzB@+!B-(P8(16V&1v{SmCnOT-A&zh}xPEQbpecv7wcqcDJT zyd(8@1G9MUV@ddBtX2cx!vnF!De<-xE|WjFxWT?lL)i8AjxUXS#~!i~3M6 z{BNqqz6?lqikqn@t5ihsnHP_^IT!!-qjge&4}UybG<_}l=Rt7NeSK>t0mmgQ>02TCfL{mPoDV>7zqN^0p2r3-e}%=ZU{>EsoR<*MwV2#Qr)xUohL(Rw6%b1b#eF~4jHvA z3tqAONONhK)vfz#2M_jLk93a6KWNQbx#SrR@KZ&&tSz{a!f_<6wQ$V^!Tb7ce6JR2 zHD<3BB*L9NVkYo^qfrf))x-czw={X8PsVoP9v<;JCd+#NV%5?~ES0teIcWafRm1=e zTblf+mc{&B3Q|Ai!j}|m+%q3B5QTnqF$n@A#%TnDaFo`OJw|n`ZbcNFxqz+Wr=i6U zyKyT)$xDcbJbUK$ZV@bl@XNo6|Bf=^D3MO{j*5T^nyQj2c@_dhQUZ$MaRZ7kXrFhS z9_fmt>#Sa?*|MaSJPHgf?g47<0A*`_KmdrTbpIC)BulKOB{Nb$<1sFhe^9sOOGnzV z+I$3*krbtzKUY9u@iu8{3XCi%r=@!!Vx;otf-!9>u;d-K>xb3M=OuDk1;Z&5o8&F6 z7v<&f-vXMkS$4j0F`BDEQeV)GF3>p5ejLH8{RxCcV4m7p?&)lSG|gDH1A{5{5)I@- z8MPtbH3Mnz`Q+G>k{VvGzh1ne9WLgcR008a)p>e<9p6aZf1Qeg%3o~N$DGn5pa);& zk+jDmDXD+&BCb3cLCJ6ord!;BD6uFUnU~_{Vq{#sSbv!}TMc%}*p#M0f&5K16G#dC zLj$XMdlh2q7Hx7W?VN5WiaQYr^6v-9LfvK(s%vuU!yE3!Q9t%;-|kuBv6qd>nFZD&jdcF1A)Q1l7$VPbQKo|b3+VwFCK4eO3t4~Jg40Xf$q;!r^aX%BbF5C^Jo##MnspJ2k;ToycjbGZ{9T?f^>OXG=v)# ztPcsB=#=;>!hgjBHVj6s#@e-NL0D}jd)S~a^g9XW=**P^B}SDV{m_tx0vB2h4nZ4k zuy!aXEGg;N^0#ErDnkCFiB0|o9k@BHW9ENavYg*u%Cd2}kQBU!k_nf%Sm!U1LZduf zK-{H)3moS8zl0HBx$XAAvw~O48{wgfijP@l!`ErHQG7lorb}|wMB8oi^O)Q+$VDpI z0%`(R(tbs{cB*RCg(Apv1;Is`OWb6pxnQr~wOCZVmWvLI**;}bY`_rhy059ihdS|C zIK)OnM>DsDw_t9ZWU^}Z2xLoChlWo41iFcx-3%s_%I)J?Y~Qaew>xCg4|lStr*q>% zfMG(?2ck%Y9%K3zKpZ$>lx04{@}k!PvB>H&A-G4;F$PT_=8`S+9vCYGp0j~_+#Ghf zd$4I9K^5YOXw%MBIYg(LCIFcs7DnHVhqQk=)c=Ac(g2n=6y4SW!X*sW@0LqMO6l`d zOy7=AE!Me842e20l6KhrP?(0Z^i!YaLYW=HtJc{-dL}r}I*^Rd{1hPO>&Dp6MCq=o zbEkjc8jgd6@IuI0!(W+q^`5PCWV?pArCqYNB!>kcCG^Z4o1;a*)EJ%yDl;_5Z>knf z{jyweY>>`T&nyrbguu)#u)uQ|coEzB9|a+j=A)2#A4&9@R_o_O*jifH$>^6g+j++6 z?55^dA=r6yb9>*iLWH^=3zHoV-rvH~L!7V7NW23Z4FJl~ov(L2Cy)|gpDFy8#p|SJQ&sAf&N`k84z6`~6*2$~sOp`VJtl6lIg$<9 zcQ=U2R6qkCRe+vrR=)2Y zOukx4!e{}XW4xEmsFyg+H#NR4x`H)apX2L}N~GWT+d#a-{2zTsM7~m~^L*rI0C87$ z#Ae(JnR$=*aK}gQek=zXueD6H>**R;{8aT$a?|5^N>LQbUI=QPa;Ta1J=%3~aSIfvvysleNqn}AEi(OKm%7h6r_;f;~) zbTf9o61@R2)OFT9myH@0EqqWv8W%3k{HispO0!QAXxg;cZpR6^j>wiie3KrC0YbEM zqoX3#eXn24+Z7<4bqj{Sb zCl)lm+{~iW6)#si|4w#^xTWXpOg#@$36b{lDsvs}V$0?jH0e}_zmWCFvo;_ur?)Pu zH2_s#X%z{WS8Hb=!Dr4n?v4Gwl$Sj1YI|tW%|`)}xp)E#3wGlgw(CX`u_kqAtoV0X z{An2j%SaT26Y1V2DV9>qqO&yxTYKkf{to{y?*-)3QA}?ISey2)Rly!#2lYQda}U=& zE4$lHv^h*}1@}CM2dF!Ff!cKA3lJvc2@sM$V5Rc`kEAImSqtz0E!-E7u}bX46d{GLy`_S0^)Vz=k6ho z>Eu8uLhL@tzP)c4X3VcS6k%Qr^?g1u&N7n_{=;N+Ns`uEs$BX4&hUbdVzdXCVl zDm8h(m{AtX2Jh$~79R+RGLW{L%8CH)?(-5kz=Y*87S>OZ5b40eH2=8m{^6u0Pk$$# zKW@F{%bqDGX)t`2!Jn^e#OzTH!zHvfy4$gNCjmGTRI*mQN^%ExWPA**!&gNv3T^ zWgQJ#jwbUUbQGb6DDWsj6l8mw3w2y`5Z80NR|J?(n$^@j>gX;fNV(}!&tkPuyB@3K zRy-LJlLpl*6rFsTqQ_uo*4SGJXH(Q|=wJeRr{?uOE(tE(pqQ(U;&13mQ^hgKErJYEqR}1th;M)2dPKf($utV>cU&6Y9fHIAH2bxbh7suubnjm5+*U zFo2mS6}ecnQpCt`Lag=5i&TrC7EP_q6nxjSsUGdOW-1xp{f^}#)tJkBUk+cCFwsaB z1?93~up_Stdd@vP%BBbA11h6jYSP9~EKqy+Y`UMxLZ6RQ*x@P6qzL(P9q!wMu5%+e zd>&?j(vM>FEP@^Tb0N*2T>Rkd?~5>0T1J6z7?85AsC*v6faO|!@7tK&nXm?Z_)(BLa%Cs}gm>PXooB|>3RkHL=%GZ5mPBTkWo^@e z`1=l|C&OiUrXll642*c%@NH%1BWL;`AcvP+G`uPv7-vmZqnq1;ts3}uJ8r=fi{WFc zRqsJ`hk1Usm$pF&upp+LzLHikqNm+_bKZh~ywW+*Fqaj=N~E7BjoJbJ<3Wv>MlSAe z)T$6u=uL5im5BvZ04N*yHS+%^%aS866vwY6)@x{$Xf1eN*eK<%OcxS5!3x3Z-`%#( ztL&mcG|G`WxQI=I0ru;u8WA7f6jpFAJMl~l8VXb{t;ax-YIf{__smCooN2S`Ci9}+A6Y$b_+tAluXmp^n-{O1#HHr5GIy8*>6HdkQ&&xrvR z0f}1pW-q%&#Cox$F})u>GbPv$sZ`1)NTp4}#kQC~mh01_njdJa&i+w@Y36Qxo}3Np z9VH4I)?@T0H1b8jEdq?NJ`63Ux|ZP-BaPwM zq>xvQo-yA9TLBC!d6bC#0^Fb=Y{0WFXRunnGv;KhFKj7Xk*YBXM9(a2(i10gz8NXU z-8ibl3Tz;5##Yt9a!rMJE+KG*YeF%Tluow@x9!M@R|W;b4Mt@_e}qX65xC68psaU; z92;JP6`{>SVxq>N^<8vndEi8TQ)4U(P%eUkwy=YH>GnS!NjPNJN#_W1J-GsF3$THh zdi((n1pJk2@Pif4Vzi2`j@26}hD82C#LuAiYYjr9R)A%eVJu3#VUqgpj1z5@aXJIutbPNcxuMAOta$J+0lh z+83AcMIm`)(>ImTqgkGPwL3IVhY&7%dGPaN8k)3PaT3W(&Ei|Z$%mUyv(ORna1ZTX zy=y-G*ws2$o(bkm5Sc^x4T~gsNq4*va=fcgEy>@6kcVC7Zs!?_65-4xr>oa(#1Et z-p3R3(gLF@H5ulJ{74tD;k4q~U@MRecpEuNla$+b&pXx)NVSL31o_dO{{PFI@JcgQ ze_7?K6KEZZV;@66M^9U^4`oLgTL&2bD~dd5cg=>MVHC;6^KKheke0|CB}(N>1ZvwD znxyF8#D!NU*z#fDA4wRTxaDatw&qVzTXAuE9SPy~wu%xjByL~%|Hsk(b4m*IWYI($ zk4clzWP@!8b&Q9J*nV;NqB-7q&)~#c#1Jg=&{Ri(s;rFUS!efoM{+h9+& zmUie!6caVRS>*fr>qAy=Yk@$z^LZ7f!MeSRuD(=KBFt38kep(dNRvPJ7uXX7>xuSp z$PzMCU2()Yqdb5ztQaT;fdz#=BSv6hB$ybM%K!tw0NS88=K^{wl>ic;7W?E)Hpl5(&xP(&Eui(j^jQYW@*T>6`pd4?Z-ZZ107w6K8+_|Sv)V)CcCT|i+egCr8UAD&xv!*NX+9~dI|uhCUmN?JfH8R;Pvt;rKU;= zAN~A|`gkop+NqAQ-jW~X8Ge)hnZ9Nu9J!4r_TwHJdHbiFru9f}L}b~HLmE419G)F7?jYrhqj_tb%swZ3)6!mop=cvnXpzm3X%AH(zrZ;TL)b5g` zZSJI=qrSqzn`~+yi$@8O+oX6}PKeg~KXY)!BUJ0-4dXW(AxS08vB(qs#Noq$ZOqAo z&oY!OcFUCA1t<+Ht#fjr#IgrOSu^rA^4DV;wi(s$$G%J0&>Men^C z`e^E3Or2AkV9T~`)3!6K(zb2ewr$(CZQHhO+qSJ+YoC|EsSnCI8i)C|zo+x7;j2en`scpJ9fRw*H7 z?WP}ooN%S`UaM77oCtcTJ`TV5s!W&ctGzsD!Whl6)*1ODt`JNC)bp&a|(zF_XO64y&D}Fg&1(ct5yv zf(Nm#V9;eZyy&iG-r+$KjBSv|sVfU!uZW-GVO+!KbzW({N=0 z>pFqZ(nh-S*I;M&%9zE%X;V6ZT;nJLeNg&!?${SAr zhK_I|F*+8`ap^Xe*|NYj7&Qo>2EPwV8y@IKM%{2CJA|wNIX!VK@f|`Wq6; z_D{p8sRbKieXy|5Kws3Zp{v(yllsr;I@Sk}OR52tf9_*&W!u{18Gl<_X4 zZ+`NUV(!o}d>Cjm^tZR}ri{=`cax>+Pj(!Pqu}BZw7mePC}o5TJcJmn5Z{v12W>j_ z4Sye9Wk1XY%|yaQhyzJF!BvEE%TO(Q$+yKYR(&`M%Us6o-gBcX2*&zE%O1?GEVBSe z>qJnuOuCF+Uz9Op5|&(ulrx6?fSgi1hkSPS575um?V2RWDPQ4Vm$)tA&t>;Mz-Pnv z$fR#q43Fu*x%_i?)e7?MivtMNUG@>FeECzz`j|>B zqCTCz#Etul3RICBt_k!={=tRSGmqS7R6havaY01U8+m``h=d^#$sV+q=+CNe7fl63 z-WtK+S3r+?G+V`%e#I&{?x=IOC`YcBlWb0iZ;B-lIaS53*P{oaZKF6iGx~Gca1Ng~ z#Y8Y>KD@!hsJelL>&@RSA<(l$@go_Y@^c)$`pY2z{PFn{N#SMeJ@fL}T$7OTa3JVs zOV;Q`0eY})^|=AG#S*k-r6$KPjxT!~YXpwfjG}Sl5)3yhkuSNhl=+)Jh!=m`^KtWf zqVh*4>Mr(>$4S0vnT~9}9MtLJR-&0X>UDhdeaUm%cy@**5NU%^NF#u*M`nKA`mamQ z-h&Uwrk36G&z`-Sm$aDpSE7kWy>ZtP=Z_57-_;(D@scm7SCvu{-(9MwOyE`+>g4Qh znv|OZgjNJtyMk=wu!h|x)q@RT*1%^Mj`aCvY0#7qU2}$wE*B;F2DomH-~R1xeVKxv zvp%b)y)HmWe!=lf1q6Y+<&5IDO<-%(qgFsZ+FsVEwCs?V7awHf2Qo16HN{fpqkS24 zab#@rlY)>FQ^PSw;Tx#rHI*w;k(`kQZa)b^Im36!1_)hZiOI22sMBee(P@OaRQ+PFJNh^stlqOCMOFu zDn3$yk8zNmoCa5j8ZrHTH1^?>@O;@qNcJ#Wi(PpMx1IhWz7d>hqtYHt1oa#$j5Ja6 z9rLvyRWeGl&K4B?5=z~U+=Yl2eO$R6Tx`B%K`FJ?HPexobu(jzALATIn5Xxzo7SiM7sM2U0&MxXtM9*XbC1U<&soBh`pUXsGID7VYUahoE_#S z=gTY{mVZNycB~QjtLEPxFNzYb`zp>_>W$1du|2bVF^L(Z!6j^ox0lt?n@HU~k%z zeI*KcpWd|w*%Rmx4Zb_{G$H~iD8CRlx)K1rYzMfamP(S*ZuRGb$WNPx_&QJXS?XmH z%+?2LHR0+_7-eae*B>MAC<&fM0c5d({WvMB=phuA=^V#EHE^e!z=ziqVeQX;jc22y z815m+K;l4yeg5TDkHQ z?{tApAhEC?AZQH7EVQu=(D1Sg*D;h9iYUO$6x6$sVl4YD=L&O{7M;`Bu5n{^%DP&M zbFoZyxuzf1aEorK0q6(wy(umfv9H6Gv5iq<+adbds9})Czi))U6G2iSohT?W@KW62 zXw2e=iqeHK*uJGnHbXx&v5rykw)BE*@{BLWq8B+nkvGfgI}gr)ro-CgDm`EFInStR zI6W&C$*;UcN~^!=ezu#3D&5Lch;4t&&jqJ>I4%c(4iu>cSdpR0uQHAz)sXdI8Nw{` zqHm#SeP#Xu{1cfpbgTh-*l{atif(zXebIP?gy1)2Y5s3q7w9E;tz7Za94CM|wn)*S zw4Pk1Xn&^KXCniqmA2&kJ&s$1!Gi)4s;36>z9<&zbrD)B`t5eFd|du@G&j?N2_R- zH|qmBdp8`!aQ$}d%&BmF<}-1mrlLbb*y3qAl&|`lVtsEz0wOk4+vgr9;Oj(=CFE{D z)IgjfQ(^@K#sCK>uA|VNGmioDSLof7#sd-HT_%%BZ^s|FZ_mXx2c}grK;r zcxMk=4Lu)La=4-0uTcBz7njl(`T_ zH%zLRsBVGexQ0RJnDrfUjv=P*P<37kV&5Q6o~|U$_{(na+@SnMT7(cy%E1Z{#svr; z(6qLgK)%7|dAY|(A@-S6K!HMSO{&=>Mpwd@?)cna1>1YCVkx?IvuY>Ur14-txnz~~ zrK~4^Y!Qqw3Mu3Q$TGvS&_aZ`*L0Hpov^L7`(5Hn8-TPs|K%B){-;Bt2ARdxOb3B` zfza!?88^y_y%pvezN}Moty+JF(P9%h>)lWuTQ@d_a%?r{X-_s8Y#Q~oz=T^gUYxMj zQomt`uD#d9D8y@k1b1PjqZZZ$rt(A%)weD3%@C58ZoxZ$ct7q_7mrx3DO2(fbMAc2 zj=qW8(L;;_B?4PTrWgaJn%00fz=T3(xhbEqRNkw=O1r z?U$VqxH8ij*}7)^&@ZQKpQQEy_sT?k?+Amv@wGQ)!mmbofdLI^CwI%38##t*U^*~9 zN$9QrlFvFOdHtI!O=*oPCCb06Xg&qxr~!u9cP1_ut87(Vjdsj{~S$_)g} zE|nPu0Zm7UP;k?a zYNik!z^{dnHO9+N)IIcoPYx^kaec{c>H`pF14I@sFDV_D0Hx4#uSFp89a6Ol0~ew< zFktT2_6l9@a;2|!r#J&bcRB&C^)`|2B)=v9wAT?HKGE*OvjG%$-h8QPyJ4&3mU~Yg zTa4L!bVLmwrWmGMX`JRTc6}p&d?JeTU7nqU=+HvKuKwHell5r-QBo`}g3V+fgU|?Y zVny2A-v5H5jpKIxQWBJFN1bP4>l~K4t{GyUzHSqy1ua3tBA;xi=kI1uMRA%~cSkXf zEP@iq!+7!^h;9^!4odDX#nb;6eZ50~5A4YUWiq}V%-2D8JXO&JsrcXu3mW9A6bUrY z_hJybiv_eA@YbNmFa0QmtQo_*sbHIrn}cE3Me|Hl40oy;jt1-h?dE-IeStEh8!4?v z4~b+lZwtJV^$4QE^YfB{+)zSwTVl88bGs*z2%E>WWU&3sPVDHG{r=fa0abQ~`4x}W zqJQ3|K`bzk)m>LTy!$tt!T2+uQ_`SB&Yb>t;BE$T1T?OrO8hd$^siEp#lgqzobZlw z*IdDfxkE(RSwywgK2O`kX#;G^LDpsA@n4&|T}^)A;L(>Zr4ix0Bl@@F4?FZB&wsQ_ z>_MmcEY9~GTHdhjO)XpSdg;H2^(@~tOr&uGqybpXJE-8AhEH`65e!vo(diZ4E{Nqb z+$h{i82~+)rYiOQZ+RafM#cwz|8;<-pvWNjLc0n6zYfsG5lKpv3L~HbR8W=*{0~V!>J5<#lk(!?!t)$=p%EMyUW1ye|~VR~F-(P;{a8k9V;K%+O=8KT@? z$@|*^i%YfJP3jk;9GWbi*Sny^bK^#=#wbIBJc4fUpfMPBEV`2Fqz!_bO28^xIY3}Y>K)SLL&u;FkW6G}sw z!#(+Hg@;H0wsE4Oz3W{Q>w9h9+m0MXYhF!e{xE_j5h)n$J@=_Z23qXqR_xztQQ*|9 zh1zs;f75}xgrnY7ZGN<=uEb=JmE%>nI>Kv)cyKP;!CTRFHpORy2)Lb49j1*d%JzY-cu=UP~M zI&7OP5KsEnL+&YLn(FSmAbuy~0vo*(E$XBnHFn0Y&e?gVSlWXU-v-F_{W;`fv3rLV zDK$~Vrzk**N`}D#w%O-YszYX}MuHBZSMhtUGW%!^vTyN`V8zSme*X@ww8VHoK2A(r zX7BS!_m@s}6tIjzj%gXCmg{5t280Act6n$TlBQCr5(@uES__@w_-)d?nPKjTYl11p zJ0oMA3S>GW)*UcDi&mES-P)ENL+ECSuLTiuyexO|eG$FAOTJvh9hPZpU zOm^IclM6MT_pdAQe*!1A%5T_o=_=nB6Y>%Rt}TfS>kz8%y2*%oju(s|fS>@mB!Y}L z)bD;)`&*uZT+eW1a#k;jdX)p~eR<3A`~%E}$%=G8nqjY`@z=>fv5oZ$_Hxu424V~) zJhZ8u8G&x4<>-c|2_;7LKoD-~{=6_MTCMYff+2|5t`j6*zTN8~3ynQ)WZEwpWgdJb zY{Bc?N>8M0v`?83TAw0jw#AJ+PgQb8k{z~0|DyQn&+ZSkZhCYgSlE^to<0~Tj zUS}b(?7c7QWi2Jm8hK>jb4HNv++#VOjHOx%ju4Mu#dXoxgT2rsaHpVpCv2{$gcO2{Et6IE3lPF)z=FKjBrXBta* z1IL;$^dYRUSu4Etv#*x?0fV4tJI(|w@FeUF{iPVrNUNm$P7}c=utq79ZqL}g6vmO0 zb6o!(Y9aA0cxS}PC~xF)fj{UlxgMcDW_rGLAD%1%dIBwLi#%wR4sIGC&AC@n0Iw&w z-tMXeC|}dIM#iGf#`jAt-U8-e=y_`&k(l($iUwH2NcuAW0t%q&fk?eG@zwjb8#qT? zlR0GOg9qXM?7_WH0@KToh*zNJOev+$^kz+}oEz}bHEFDc3QTRN(;=`%(}s-3W4Ucg zJSNtN(Ji<;&^_pJw93aVY?OucL%_zLG#w|hLX6N#Es5h!)>{(X3+v2o$kOt+fqX%} zaWHfLZwgy6+2*;Vc>H@|)mP-86QP+dBokV1#1{1LyhIDQ<$3MW$cL1GK zhlW}c-kk;a=zPcK0Cf~m4yw!P=P@_)D49{PkDlD21(8Gos=Z_7@!I#1)|ttyl9aP1 zm%Iin{_8N_SFZ+T+5|M;$pEKu8BBpDsK)z>FG|pR5zmbrm8iRvHzCM$Ru6##0vJ9G z{1C39TGTZv%6SR`j2I}*qwf3L9H+zZ@ZmjL^`#53rpr4ef&9%LR#JmkC-3R1JY3}0 zE*+gzd|IZG%3!j9OQTbi3iQxkM6HVAE%=$PHE*efq;kljW50Kg3k`nIf?tkFtX_AeO)nk;ya6~r z%PBwtq#9}X7oiBYU? zkC8+Eao$TsNfCrJxu?%1=Lh7R0Y(ox&Or>D!VT$O#91Uw5FillVd})}TLz*+&e-=P z4KYk|DwM*X+5cLb&YpQ3;d7q`)NtG|QOt9g-ddhtvOeeLW+~|24GmVsJzV_zfDb&ytkx zh9zqFTS`+#5y^A0!I=}SuAmb&7_gg?1AOEnh?ad`3mRGQ`;Ya0uqy1MZRDiC|GsSQ<8r+C(D)8nB^5$sgX3X(g0K46LS7su^9S1c%PRwsST8*}gyj?{a7{`%88I*+&oV!H;v)P}5U$#R+_#;)BdVZHz}iXC~UtkLD5^tIpRsKEL}GuVTs z(1y?a!`WjVfK?I<CJmIYU~1ApLK3%^W+B zl@kvZU>d^(8X24z@G=~sNS*;%oZA0wSdVmju?0!e$)piQt&R_t2FO~&*lRMnX>Rcc zWXdhf&Q@NL9t+%R=##IdZGI-!nwRx8>_kkWYK9dzDodg3QSF6?EcSc_uac558hgq( zj<}vJACOL?l_mec^}Ky&ec#eHeHkSQkVJ7d9;rg+b%!!#O$ph)s->?l3yQ)tK3 zP7Ci^BE9zv0Dh8P0B6|A<7=K>6b<=a5++P~^6T#59)!vz#G|GnpISxy+l%nvwVC=R*fZ_I`Vgqo=7Ync4jC4^mGr(Ki zLhR@N>om$q5~+eT8y3SxCAe4G1hp7G-}Yp^iLuyvNJhB+&va=~Okd~6O>8p;TPX_m zvQS!R8w;hEWOzJoB7N9utowA~!o^)x$=pX@n7ByJG?i>soQ{r|hK-~<M%9WMXln8Pj&Xk-s3uGS!^O5mZZc_e| zH15kul!M0X-BSwh2z|1Vu?n?_P#i=EM^F;oi;NP5jM92jj)bkZEJgS5gw#}>V*dip#_G=4Aj0;RTmX`snmZyneG%K-7Rj@c?$sS3Q+3?R zoHZ{;N;v=x+k*agvw$@6Y0k~StTUeOAcA}_5e0+GX;{Xy6O}%YztvDl8kSnyw3783 zGEI81NRF{N8TIh<(UEjtIb0T&JtkcgFK9VB3vYBGFU0~$& zH6YRmw>*qC>yB3XoyUHNXmdi^6o%LDZzbBe(QyS4Ut zpF%(9ahm+fp;*=!VFy$-F!{epJ9CFB?A9vBqzSUsLw|;{r3uT=sGis(u zoP$sDz}4S2%G4k zUYH@&Y(WL-9L36n@y?|^whYoOuayssYma!r-iOD3*Gn+9O6 z7$;lJ?(|r;EPHwJae%S{S->v~a+vaR**5|#h_Prx0Z^~FP|6R+85fqNz}(G${dqb} z|40$-8~TmXei-)`!$WGVS=Fk{clz)Z@d>UI0n4AVR=XM*ko+uRb|as*woLZl?<8-vy?n3=y6=h(K;*15oF;lG z*PdVYtlJLKZRM!+>D8Y23J2ROSr%$NBB<;Vbf&PAkuCzmwXnj zl_F-<m&D5-~t!mh@SV@F48Y%|KzdDt}=p&=6xWge_w2g2oqD$co^ z1Us}?`ED|=*!KA#>{7&-=CfcHIS9pqEGz>~g(zA5cFqFX0>Hp#gXIle&XL?%)Ruuy zPjJ@P@~2MVR^S6hn8TO36sJAq#=E zr~`%F*Ahgl9h74YY#EJ&a`Yl@o%)|J^o0B!0kVF1s5+nm!WDdoNz?|P4!5M{j&lP> zqz90B;r!UHFccrRczyR0-(mJt*x}M-U4(8zH9n(RQ#sE9t5t_>uIu7C6!xe8qa@CH zbUd~sq3=E$2fAD&_p=qFY1qqkq z7B;YUI?yKI5)(gV#?)E~%i=PajKTe`*V6X9=Yolw(4Vqztto zgR3&p*8kr)upuG8U8uJ8YZEnQ8YD0(n#*%xz2>8xvW_@5`QY^Eo#|rffu<4iqb*O{ z1RP5I^mh(A>?Y~_AI~ zSVs*(xr-%x%!3@uc5raL!`J}_EGqm!M#jqnt-uH8!(}yZ7e@7m6sr$T|1V&Rs3@Ca zSttWasmb+CJ&j2brBhSjk@NYX2w)y<#owy3+$$3v!Xb(foXNtjoz`iI}iA zC()w(hu8s%{vB~-R=V5Ct?hM^%PVz;LHG56&!p`II>Ddcov##Gje})(71y)1--1%c zX%=iyB#YsrU`~p7+>~cDyBEa6P5YUX%wrNl-aq%ez>|bh3(wn$>NdsRwIJnhPPTzUDkjw&^U|Z7 ztzT2xKSz0!HO%XV-L@`CJHR4e8~C=+atQ)9Os0yzIEM4tU84D3gvGOAXR3n_h0cIq zm3OE)blF|=?=wq30*rVC!6?=Ist$(7hp~(G(Opi1DQ_m)@98<-1+o#KZTJ6taliyJ z>1yE?g>bw^#~+O2L`rkAR+}!A%|>nE*6LeA{`%$vFPh=<;-7eWZgcM8xmE6MS zR6VhF8A+OP{PD9}WR2pALBIjMAHDU>MDMMv5qP><+M4^hVcwQBNiy-WUGBb?8VHz#)@c+!NpV zns~qGs(D*Me2T*8d>VabCE31fuf$rkb`y0~xnmvdtTl`EX$~Zb8iP9bK41+GHuAi8~9}pg# zhwgWTC(uV{?IbjGzn~H<4u<L{nkZcm`{mP_32^(bo|#Y3i$0!S0H4 zc6J)&m&_0-aPNqFE4wO$1Wkb&w%FMZK2TShZCcF#Yz7)-)x547(E=Vq?;MCG7h!yc zp*VFxeWz6MSDN9vanT5;qK+}FK#;-s11}H!Z&u3xpHLA8;B1-{{s$6!DY{xyv+5rJ zHlb-F85ubE!+p-2%Xp2DurqOx2+&Q>6%xkM?93XHvwI^Sv@n==0>qP|s)@6pEK?EE zsJTo0_HLF;F6hYP2-MN#nmSj2WNAK;)P&M$OJLlmRm94YSUoKVMp_3#WGVd4MnaP8 z_h-i^-3U@Njjnx?T^3mSwIfTovWg-NQaVLfO4o%N+mZ7hRZgu!cg6KP<35f12kMK0 ze3)Z6#8N{_k1n@dewydpZtM)G+APjE!O_&H`}Aj4PL&#M1g)&b2B#F%V0KX$*p6@c zL$Td{_%XW|B!|&0Og!By#5>vU&h|xb)uzqa1$P+t1(~E3uqJms6Yh8=|M#sD= z3Nc?ZC3({Dra@)4UN&ikGA!}&_I?Gl{NU+lKZYi=WW(af;nJGsK;{YvD3&E`y7 zP9j;0SNt~YWC>ZhY<-s@C1Y%L4x7ZdiHR$V zLak4@q+JNJqM>7S!vBWRqH`Q%eaGrE2pXiF(^B?(!aaT?7)hOC3MCQR3pSgo za|lBurrdBF&*^RDit~E{6ZnXqN_+FQ&Tg<8A8vW zzBm{+JLQ13{Pn~=eKE&s(E;S(5BL)$ijIEh#5V=dpAtTgs(hrS5V{7R(}dZ=$hzgp zuZlKrbrJ~NrOx0~z_Gala7s_34Q26Vd9pfHUjM_4KzL0dDkkU{%#ueY^}qoR4b_9| zIY10DppfSqzAr0ID~YA>Zukz*HTO+4pXQU|umLak1*-?bMAWX0a!N+L0B-moC8^T?17;$_>2SM}0(2j_FZJ*eaZ1?)w#?0)|3hIy64fivQba{ofLD zcXh)xu2X`p-Eo%B(MkL3m=;8W*OYy;*L_*p$O4M+U^EaVbsp+Qq- z-(x1Yz7N}wa=j0LgBb0o2gTNQUbnr1mE+VWR)?RnpnEZB7{%JaPHh`HV&-5K`5#jaepOe^k=-mcRizh zRLWpXX=xod2_K={%V&3%!1aJ8pMsspn|Uko6Ip-;xUmblbXl>Duu~i-h3rN-FM^ot z{Hok4zNz&n@F>5?*g`P*-B9bx?47%K8htXgY1?UR<3`ORMGxk58-o%CQZdjM*Vgu9 zBKBC4AdlHc!oI*xAV5GX{qQiLH7PwY$>f3jMsfAW^5_I*yfOzhjnY=W5@r|cMiY&- zIO;YaGrZRJaRuBHYA&*%^DMB2U(s&e5wd0{L!CL(Bjlv2-VQ}RPnv8dA!F;`0!%It zaVZ`ntXbq2e3r`$7Mhf1xoiKDS{>+t^9-j~j;P;0fX}KAdb?*xLlNeSrQAQRaC4{c zKb@qe2+u>$&;1W8`2$6UAP18TMks2BDDa;oR{wHoD&hh&X3{atYq=uj*Kj5vYu?Dvl1bahraae+w&r# zt)+s2oXv{udAdZwKTC2$-E-n5?|sDnAv?QnCVV(02`NvJ+cBApRZxNK?wKRHYa9{| z7a%l+wCm>R?IHmg&wp=@wMT)F=(pI_Lu&)HuZ6`23Ibz?7v~;(0xEnS*m1pQuDUtM z_>gkl@t{Y)%=~8uNMsa=ceD4Fx3J+s?@Q2kH&%Ty@7-cYRT7^8Wq~42*UQxqWsnlL zx;GvTnAF{a6KYY@9yhjlm^Q~A4)_p6tQ%Vm)lnpR$zPL0*yRkxZYE)F6NVn6eAI35J5HRvJ_hGyC`I>zgWmvA9>uVS1cavq)ewrJJhvD;QT0)DuLo%V_$I!)a{4Ima$KgE8T}H({{##=&*r3EnZA;mQi}QNzNT?JdZ_DyHGz&sLfWFrM zG{1K~T8>e)_O!yTy(DV)a-f$1gXa!?X^E@gmSC3Zv+B48JhdjrP&O+D4n`bxUhJ_5 z5Ar}=gjFB3Z;R

Jh7}M=;mAgp<-v-v*rgP&g6}g)Yu)?lXQ3UY_B0PQqC(C`(<; z(_kN+qfQpeBj*Byb`R|(d7Ny}IOkqVRdUtL(Hyq*KBoCDo%8|u6kz#ETS5RyEtr`r zxsb29=mir$xuQw3VOvH{z$pEJeeI`aUvNz2*Vx3roaN%%+bl9Oh>@;Xfh+h7((8*I z>Pk4${`A_?3@;(PqTz>UM|t$Cy8Fbkfv{jEKL`i;ggZ+Gr8u4-(tf?wd!C!Sm)`X~ z=zPlGoiGbeen^rrIcZ%M+7}l8$Iwk01s!E)*Om7|zSTVly3plqvz}2 zRR=ZPJ)28ks$R8_(O(myT+8J!l}fWpd9YYuJjfZcl}riqoHwwKrGqSr@|T3ioWGmD zCJh7v<`Ru=hm}1Lqz-DY#fg_}Q=k$>IBh@izZ8Vr&=F7DCIr)qPiQXiAdHXMKq(Pb zYq>O}breAWpjQ|Us2Euxx8GX&wf?)0-H0p{ zLu^e(V=eacOvnX&<5Jy^5Hb@Y@Fy;P#REZIy;I;_Mvg>}3U&jhKUuHFC0%u|VRg$L z5|28Lc4_%AuFhTo*@$g)2`-*%5u#HsHREKao0e@d8<6X&Y5)0Ko5|rPh`w%FQ20bD zAD>550_o3$Vt`Y-dNZ>7p{X5?2$af#K7`B;mDeYWobk&Bo3;6H-)0+tqc}we3sCzZbFlpZ7Wt)B z&fmI+hY=VmR=fyeDKqcRJ!*Z zk*MCYaX`i4j%j}Q{G~y(O+nc$k;Z92T*7PdX-F%D>?(}9S6qi$}P{YXq&_JLXkf->RvJW zj*@_~h_K{^wYz(L&g2#Jm&geO)sEF8^1ia@0yq1$R8TgZH^JZwnVIEEV%Q3eJYTC1 zH303l!7>~B(-^b$q8~z<5gK4-gEG-r#9&3(M|m8)uCpFb_o3`nGQ`D|V_xHN`{0ta zjl3|+Z7y~SajrI3GumC8vxmo#Wd}Sj^tMv5p73TWar-#aVL642#fB~pMC7}AWWE!o zQ#Q2fSePfI=*&J4QG}-9Ase9oIcV*pV;fVgUJ#I)o(UF1jSbhO4GG47naqK~Wox~l z7v|Z6pVO0_sHnEf5%5dd8QuM&yG%D~&rV=U&NThi=7z&6Lc%Rm+TRDM&NO^FHt;bU z-XWjat(rKN@E+H_6S20WBYEy_ZXd>_&I5(`Jdiql{t*2g6?DpiF!HdCo-`^*)gZNw zh4vDEAoN<{s{*_jQ*4bKKp{uww9?s&9P1rL~YWn zn=-CMo;wm!54&I}P3YN1b%w<0b5*vYUdf$d4v}Evt1*G$iO^MF9`;{O>llQNfKTh< zfAH}CgrWcN&rb^?y~)t4rV2%k`~sKU1)P2*uKX_);VGf~A9x^x!H>cMBUhoDDlWAc z(hASvD#_gwD;T#c3m<#o5$Zovt2G{YiOkk5{b9B+eGTAZ$Mbi#vZXd3{bcyErU+mX zZr`PlP#iv`HoE6aeIdqS%g43W5T=h>QYdq!{DyF%PG>fg*Uo1aQSiOB73uH_ip{N% zaZPjqP45!MW%BIm7&IRa`RCHX1)t;CCg}QV_FQ}Da<$_lUkONOt(&dc=n=ENu3-1S zH491pt!eT{_!fbNmxDfu?WX4{O_+g$#~IB0XREwi9;m(qB8i?VRLaWmWl4U!l{+Vf zag?scTXFF?S8~i$&wb(ker@(&-?XB_OmL-+)_p09hAJqLz1sH(=_4J&ukhT@adh*7 z7+K9d{KIlO+LojqEoUE)7&}(Xu>P#RcP1|IFUZ+_L2w2ShAvApleXd~IVnMb*8Md5 zNK2Vgl?-J=70WGW_V6>D@T~2mqmvbKe%o`^XsdUYmBPOClXu(u$z``THQN^{Df&?E zyyT=)!ayeMzan-GW1fqxf1#W0s&K&q-~38W^*#rWpXXyxJ*dmEK$OPahM^mMVQXlZ ze8mG>iHD6bzD=sWGr?@7taM$E5+OvGD)}Kd-?%NZ;Mj0?>MxjhJgID#poxlfN=6&{ zZym4v&qiEWeYN95=$HnERP_Pu(VIh`)?n4AgG3ytwB~DYCgvGX5+u@4fXr! zK5+MrM8$8hOkuV$Pk?rW>B3&lv*jJ&J@ZgE(|nZi?yvyI7(PVsNb~`x+6jdb^h@IY z@ehF@00J8P0NRBWMUv~`C8KLb{x*`PHBM~X+`Cw|Jh$34=>jU82_+k_ewapId~cSc zLUgBvNkxyX$Jwp#?0`F&D+8z_(Cs@fKb^*`&R|mml1aofn6BuZBML(wR^6wRD-tga z;%n4+th+sIM&{>c28s-wr*#)qWS7H?C6K*Mk;pG(hMx|Ey@W@F!EWc^RShn&K zcZLLUAh9(C$X+wGa6EYivsnYUrMqwy`GBl1%};heW&55;P)4~r!N>5NWwulV=D2>l zo%_d7B!(fdpNM^Pu5u@qqUiE289@fep%*s8ua(_;0cEu8b=q38*f$@J<9D0odhO}} zcMgKv>`6_24fiTrZV>37;+s54ToFv`rsdCre_ntkc1|*wL)BtFV%JsSTAACB39U!c zWL@(zwNQQ*df2duhYD%%OmyYsTQiPqxv8W2FC_G3FH-|L*>O38oFSfWZO)erUbLkP zp>U8j?2_csfG}xA#$@!ol_T>zm-0vqbw53QZA|g1fpGes_-`=lAvZ17OlJ-96p8|! zzjEVC9v>pGovD4fAVudHWp_V)Wbka(a|3gb^e0*k_-gzm*Qb!n!3XqNw~v~77rD!! z^@Yp3BVG^(PIV4>1zJYLM5kK3o2WcLcgnE-+BC5JNEP1#Z?N}{M244-3|QaRPC$sK zRz4hxnLtJ?8dG*BDNOdY)$9z08FHz)lRhjWg1U^NN%bUx15uLdHRT2MOz8&No30^L zhjlB0YV+nbUD%DNC=+xW5RnyK*7v?tXk~-N2jz- z7)f7*m9IHxnv-zZ>fbPpvqyt;k=#V6&;KK@8cR`)h`x*x)J6@GmqdLcj?pbpuDI*> zaYo)OQsH&V6$7wE@GLNPwD!C0*m zJLK0(T|71A;y9rz3Qdg}_hS}5lq)$js)G9M1L0ILHyX~Pe$DwR3ecTXfd!r$o*e%e zi6dR~RdAmhXdy)@COg>Ooujz*563ofu zg*J9k9_GXGl3~myClYULBM_@-gx@THr~(KzR6+gT^dpK*120pIW;m$yP4@R0$Xt|= z38Oevyvtk{vqu-gWWqy1<;MT@s;*C&P3qrI0oc8bl6Q@nEER%LWW0#j4VeVi!}%}- zXU1ry8UL+wq{SmXYsFN?ss!B*D0z@c5vSPz$7i3m^QSfz&OA7mfUEm2`S6f3adP&? zjO2aC)6coy6_&DdDlBNulI0?3gEHGUV>+Qr<=3adreB4lGiT8}y_@ozUc{M^o0J0~ zgxtfuPNX*IUMpl_gW+AqSnmp^6jDKMv_{#-zz!Ubw-2z7ZL=Qb{l&Kt@}=la&)-Ra3$S$tBa|O`BjI_Q@BXEVC@9?Iy_M+UN&ME>IB8&nhh#AD$+fi zv{_xc5T|}V{~+M9>8@kgBGxA#DQCY3XP7wlv@fnN$Bstr>tqI}wBIrUNz|gvUt$U< zb#u;%dGF$WX5;s}r-7%bro3JTH;V-Z#`0m$D>c=_Fw@u#36c>Jq&_~Yj%f4~`kR1! zr-uYc_bczZR`uuam|u=RF$r1~H~BSsE!-rV#v6##!q~~N^t80ua6x0!l_%QRs?&8! zGYj!C?7-$Byj+Eg_`>X(mO#lr)po>pfJ%Lpo?!4iDWb}e-T7lgYx{yKd$OpFIPe7a zOQ{WMwqIC`wq?N*qSeETn5DJSyTsC=hYAw^J9o-OArXMv#UF#(L5P1e@rQzxdz}oOp5;zm`Z;Tw zKYr*XXk>vxc#d38B@u)gWHe4m1Q_soDc;3m^Kg|4s;~Q(uDZ(O?fa!i=#vy$HYlIt z{l5!%oYr z%b|y$6v=t00`x#Fe0nGm;q1qx7aO}CQ+|DbgD|bkP7cSr6MSqDx;zoSnlTb2?7dLc zP{L%t$L*h}g#dhkaWOMDq}rB-&c{J;n(L^^w1AOFkyC}hp~gQQN_9K6H@|2=Jn4b* zYlUV^L%h@o6x!TNE0jOoCC9DNKh31oiQvM)Hbp~FZq6a8_MU>&amE#RokzO`o4Ei> z-t%0Tq{+fU6RN}S{x~F)W^D4vN;Us*`;bTFG3Xu-HLX*U#G%BCx<;ma)Y=3}09im$ zsf|vzTim&apmlILTY?P2pFcFK=5?U zzFWe}r-zuf=am5k01v}(&IEN19dB8%)$ts35R_$>PQWFWPLQg@DNjaL;}e3A(efQt z1yNd2pbwPg4ttqZT5=lwoawdZwoCsxV?hwSb%2c-yIFY{2EyOh)v_GX)&1t zGOhyYXDL{u7TyCH7r?>kU|^CGxZbaf^S*&v1akkBymOGR71j?Nfl>H}#vpm_umV2X zaQgj$50mFTK~fr;IRQBoWSu0OAA5I~<~0QYpKwk`oBi?Evzo<#6IRqJe4guvv9`xEcF-M=zsHzD45|Rl2r^9hJ4lF&)g{+(TWL<|xC_-y8%-p&Hh<6*fTQe*v{-6#`jwf$5)A$zOy%Y?meg+05 zDqq7~JZmG+2_)VvcDE_7X1gWTuGTKu%7UGurdx$d4T$W?8hgw2Mmj|_Ko@x&#`WtS zW4?zkIo%?&b-_>B#Bo}Rl4!^kVD8G4U%1`8%>V(FfUv{+Z}b7NHI{vZ^&}~2vGO{? zFtuTr(Z)>j=tyhaB{=P7w*$!buSLU7gh`Q3nD^nYVjQ6x0kr2ouLGwzt5)!`EBJjM z{T9V+XX}r7(5&QWPCmaGORW=6Y7WHc68IU;%3MIqXq@56BoMN;KOS-plK;+0GFxg* zr!;v1qFm?rLS5OF9Iu2z+XvfIESqs#LN05Fd2r}DXD62n4vR~+xR5SbW&GF3ANgHy z>Z1K(IhKa2TyDYq7wlji&j2-@nY?{AK_yqh#q!jYGM7Ok=OP$L9uQoXTtSC@!@g(Ag20YT1A~8mv;@EKW;-i|seNLS;Ir_0hRzI!<7(i#u6$`6b2;DS`TCh`hR63t!>MOEb3?&nY4XijMJ04*Cz$vZSOtDAaF}IRClRp10kU=pNyE8% zC+|zMK)k~R*3Im70<#u)Tpg@+y_x4r(0^SuEi=<%l%sMpSdQfxVDG^<9o(kJ>jeZevY0o-t>{WjxV?-=LGPf5uwzkL6 z2kh{0FLe%gHI<5NyxsY(D?1}LiM@#o?rK_fh9aI!z+b_lP^AN|LeSc}{kA=Y;^c`* zU@W+=GET; zDS-hbT5{O4)d5{yAC^~N-)FAIoUQs|YBp&eyX2t&dO$TXSkop_H6Jr->fR^1I;pv&KtAWi{{WaBC_;HG6z;#rrEkMBEd1BkHe?H# zY^)F*L9ObElN2<5V%qeFv0gdNQ$o>VoJ?v{4^;XXYfXUkH4cc+iMLEW?%XcTY^n66 zt^M$g&a+SW6%Qrmo*%PK$)pE^ zqKmv-@BR>%z+J|$px>9B53tnRu`Fmqkxr)0xuCBMp*}rdnUAFkzTk=7N(E2?_kb3P$7kpeLmF6yLa^+Ci8g?#h*hQ_ZFyibbt$6q8)U z=$rB7=RmF(Vj7$O3rmH%v}ozoUB*JUA-kjKi1R$bK&${^E$ylMDn2g(DpMtZSCnIz zUj&F*2M!YDh8YV4f%9lFFC@Cde|~K^EQ(6?Ive>-?o&85{0P%8b{dHh84}m{8+Y$; zE^c2#zk*EdkhBfjuldSSP|wj+8=R%UQ0MrN$-sAE2LhY6!EBxg1{>)LsrJj~WDFU? z9$7Ohvbbxv+^zw7?pP9F zR*Ai6uln;;O&-%i5=gCR1XxNKMfRD|M{wtF$VFlp(udDKmPc5FvC zb|C(ZIm|evF|#r>N;*c)Gl<2F10jI}W<2W#<8w zQf3%J#{{_-Rg#_3*jwA!fcF5P1F2n>hByD6W3ScL{-$dVtH^5RT7ZyOI12v%U^yt`+y^*q}D2?(nN(bo+~6J9SdY?YGOhMXZyq#z0j7H%@b|Hv46C{vuT5^?rRG7D5t}Y`PVPbnQXE z;_`x!KO^%75;=3D-RJz3C87Zi>r>%96Eet30tphr2cGW>4S~nS?&AeKrTYxnlzdgv8JFBnO3L z&BA0uzK(Ne7H)T5kTZL_cgf$^k^SfU5y7vn3am-cm|MyEaWJ5VCA&PBe-0qnu*H#m z61_~Go?O(LQVm$Ex`C-c6i^3iP-f-tRf41WtFo^bJkbkqXD->Ep} z(=G0|26GI|^djhp7TAl-{kGANLwOhc=4PG_Z`*(OP z*Mi!t$NcKRB?xvsjx6h-b8l{f9_!q6K|r)}Fa5mZR>;Obf#s>CZ;aM(Vvf6Bq&O5S zt&+PUF43aP5dOBwQp+2uZ#j>31JbfjIT6l{iFCuME{R=Bite*v7yJmE^;Pa|B>>O+ zsgEbw(wr~mLBgDW$gk!?rew}pRjj&Nr)@^BUc9S{$5$TK!TAN}Ss)_Pua1v5ZWRyD zH!a@$0VKBA2?b#EYQnZ19gI$E07W%1I@$r|bteC{hLSu$ll9XoVQG?DErKrpa!Ka| zyP%aTSl`%h^o+7~Xrt_QMb&ty3Ww-7k}T(rZ)dI3#Rbh`Mxr_Pk|5FVCFtUPf94vC z7%{?(twn3{Qtk<|K}E7S(_bp1H2;j4LvXY^G&B@)Muvew=bB>{~@yR^}rRK z*3P6J>UzAJV&Ur%?+ot_Ilg+y5!eC{JSyxac;}P(LNu;XS0X>F@VMLcf}DIMx}5?Y zT4{?|Z6JC3;ITMEkj45Yro`TSrSrH<9s)lw%Oi+jce*F!e~h7w7r}1DFQmMYyapu5 z)7}EFO)L}t`lv_!-&lMQyi34&D(apPdht_Pxm&Ju1#Ydj!y&mg;i(5{W8fSVCAFfN z0$Q|H?FS_7L8EaA#|y3 zuI(F=!pjOHSG{^42uj#4_DweJmLha_A`a`&b7OYbmbHOLE0vPS{kf8}k$VNg7$SC* zO45mjBL^=DeGcI(*9!-3JPY@KSZ;Y6^P)PKs#P{eB(oLq#onQ0WnN-}^O}s#;4N;u zu3}*08{o+3O&3f0fN$Fr33~6cd+pyl~!${nBDV+NVoBhamP-Ho1c9KEw0*e zk4%g+jhC1IV_S>5eloz_2VH^YNijwslcciw^iqN{7;DSc4XN z3iLa|WSd~rr#JHO6MqX1#|9RjyR(UR1cGRQW=-I--gHBoxpAQ z?wm!kk-N-U#u2!;{&~A47-M)k;GWp^2GAtd?&`~^7J+X@2YM;f5l5>LbHOn*3Pl3e8_q=Au+Nk}+rxMq!i4Q8Bvs*Asx7Hg0b0?q z8p>9K*t`jvH>dVZ;Mn5qnvmaAT-OqxE}36LvOa8;^wK6aal5{Hynamf(9fszf|cZR z5Z2T}f(pqknmNo)&E~oI%P?RawFrye6P?Rw;puYV4k-PFy_i?>iAX);zTXXm+%2cv zmdbO^Gru-h`C*yt)0ZPJ6(wfl_=J`y*A7k8rE>g(&{E|z(Qgo$!UkVjpxH`crJGWXZf5II~`y`|_mmEnd>GaZu$ zUV7NS?_4T?!rc9u;Dgb=p#3SAzN`@Yh*{c;ek7T@0DQ)vZdH%&Jsd8Z&b#uTsqJ0t zkQW3*7a*}jCtAS8gr0rf)XwoCmO?ZRGKEuxpkT@TEEs?) zcVCb;fSnR6pEF2=FG?9!><{E~03QO2?t?c%zyy{}ht1DsF^Nf6r8C40{kp>h&i z6NkKaW>0xas)>si|8@Fu@c?it{NOn^YnbNs_s-sDtTntX!>}DcPEKDJ-*Us_ z(TF3W1b?rZh68MI^i~gNI0Je^yaUY+?qhxN1VZx&O(LX1DAzWaE|d82YWh?qu^@X& zjwl3h*ZrZ5xU?XC2pXT2$HaACLE&!L$9)4JscmZ0!*>PdU;?iOj&WVcCTR-^Cel=1 zoU_92ww$kB^ud+jI&zFkZ{?Eqlg5Wrc9Kj;3Nb4PtN{#I1vJZ+YMjy`dYC+2q(b0) zSVsv*NxP=&uGxu^0i1|OyOk+Wji8{^H%uN{6EK9y&Vyv#g1=yN^<4_*_A5aWUUAF! zAE^>+79Vq7H$Z<4Sg=CJ9ImvbI2eHgU!sX5CKuIkvB!VFbAb}k-Sk|sOQWw&ZG~uJ zZV_#jgqjT{FEE&WAWD+)Vqy1S4+}IAG_^rGi5B4}2{pq^UT{Gf_UR<|joDRL8S-kHjdxnOL@zmp1j>oTrrAJcrYst0{g~SUNP=FnHJgl^ z4mU5!W?Wp@5u7Ab<{WwToP?zWGQt4IYSUDGGG;bZST>96vbEC-LWSy@TwR?pb$$Ar zk0~^3Zt5W+g0%O}6?!zH$%D}A-9tl<2pMjnQN5X#O!fWjcE$a`BAw(&!`;r9eTcvU zOhUco5f`KQ6P+ErV+At4Yt#mAF0GyCB;NTxjtGy!@y|sBMeOl*(s{8z@`z5oZ_rVb z#9>qlp^zS)6&hF7U7wh3y^H}uxsoe>$m?n5F8~oQ{&V1|9aeM$OndJK4P!aj#Kzag zB)*-Ttgay74oGQTQN-#2!8qS+M=h28s=!qc#&@8l`@PNZjUnWQJ@gmdx*fNw3beg? z%g(Q5xeM^be7801y!eQf=Y>xWo+$5VmJx##?E`opR42Fa!6n@_@4aJ&^zZD{}JBbFe3|ozvv|Xx@ zmbNmt9L>sdub2l#WEmw;M-}Xr3aB0{^->VItcx}a9(&g=j*ju9cr}44N+H_sJar46 z&w|YP&fayT-Pg+sxD{t=c~${tvFpXZJzI3^1$h>Z%#$7L05c5NiP9oNo-*6>uaS=w zZW*7ReEu=k?Tr#2p=Iv?%ew#Sm77D?O!|>@dm|yplv}b}ln3+?waR_J0BZdxn%(Uo z?)lD>!W@Kx62Jol@_>phce}JK@PjpBfZ*(0xR&)B2M&9*6F6M<_rJA&OcTfYtnUse z3;6Cn_JwmivP`2uVAydEd8^khYM)+^zp&?mz`{|pRLUC=RA7C2oa*2hh8YHCoy3YS zksB!-+0zzyq|1v`IDDOrs>orjwHM-QrS?^!qK+49fk)WvqqMQ;<5MupN)k%*C>;hx zoVr+Hs#&3$`%1Zx;%dn3rFx}5(Za`gIkV(`rca;FhgWzy>HNPS6>T^vgKNK*ze`{V zp!cDYapt4TY^a$c5Zmg%$h-AOb28Kd0%mDLh~o01NOnVooeBp@ zrxMON%xTf&7wF|YDc7ebUu9;_1L>(ZArwvC)25>k zPo}ALhFXQ~Lyo(_=q4X2M$s^r<^*6*YJi_G3|{&I86)mYT6BbhUCQWkof4#uEHi>; zo?kjW&f-l@vox6~5T9OYZr3aO18qDr?`_HaX_@VQ>fRHB{-#$(M|M9mZ;QIfv4Jbv z3i}-8qA5@U8OOII+N@P@&y$phik*Lb&1=lWwvRri?_17=&)yp57*R%e(Uai}ZpTvi z7L@628p$lA^E>0pnB$>Bl2M(0;cr7%F5jmNFbP#T%MsafcR(G8mYQFpT!Vk)ue_g2OB_T^@k&o zg)_lXg&re~qX23HX&tv7n0nP==l?}`K+fy>j53BZC>$Vx%(>{VwV`#b@ zG4~slzW1g{isRTk=p8mo()@2;?R9Sjaa1>kI4z2sk~H>`gy~WrImD)XTjcV<%vFoJ zSEA6nnV=3@rwW?V+Dwyr7eUTTV9JJmoznj^7*>oF6Ma?s*7r6}|0bOv+*{xL`#@J- z(MBCEs_ZkjO+91o8l$x7E+UwOmb4Fx%smME>Cu2+iSBv9V!Kf5^gRIzTrq3Y4f3=; zwNHX!!CyaP-Doi<&vr+n+>#PG>*|eNsJ4|~QenB2P4Rh#Ya@9=kL;;9+1d*L!u+&R zm`(>%5bx(3Zm-47qj+lBT>K=kWRtF8XyHr{pF5nGoxDI?-qt~C)I75a{Wfe=N!~Rw zp{KM1MTV&7VE2lDYq7_MNKN6R@%wt~J7n~&gxcXG=hj`PO5{a*sv=EX8BRe_QVtG( zuZ}}CJZKH!fB^qU4W+SDsFRD$9o5%m%Q1oI6c52FL*tj%fjmoGP?Sp(!dB&vSE;=> zc=oKe^xrz8!%;yTy=~-7f19l3jS4o=z@}vV+``1~dZQA64_?V+{}-ySL-E*v??qN=RFZjaEQ3BBXzDtF=*H9 z$)61G$3T1OBVhHji~R~qo!KR>!y)0EoHzb)ux;zKBn&yF+680Y6+}>lv?_tRNm#2? z6p#9BEXF4^fhrut8iOGM;H^S&642;tA<4usnF8g$u^Z5(GtE5~?cT%bLkZez^b z+NDY)yr0ap$_k%8=Bt1E4-OjXWuvm=#7nLSCh9V=*8T|(mgjJ{CN)~$zN(*SM5+=P z?)BEIZsuk^+qorm(JQ5#X`7Z2T(=}1KGQuOrP&VVs7ne_h!WiJ#hnrTYF{+XT}QTS zp(2c3*OUKej>Elg(aqZ9lu+Lt-de;yU|fUjS+VVutUjn9Wy*Yd?wd6p?&b{hzLFP} zkI@C=`8{~0TtRi}JT_D6jP4pKxGF(HjTZ+}bI;C>cxVy19GC4t`_|9w53UiRt`%f7j0HS%@> zd~0BIzm~MU<=<&wG!HE)KQdGyQ{oHo_K~o2PO&>u*&$cqp#(xT?r@|K?Mp$7-)npF zQSe8|YHEFgh9WS!M5=1T<<(m*aJS_C2wa+soxl-O%$Uzgw7_ZdL&(E`M4|9SQ@`ko z27*4Gc2Vf%v&ERp3ny@HLX(tIxJh#Gc9D(@8L~6qz4vBPf25w7WUiMe0f+=~2t$Y8 zqT(p{hpsIzvmEO4pay4ld8kCa5|d`<5enV07wVZA)`Jmy=ZCQtun6hp(%iRQTP;W8 z0eEH=r}`mWx3Yxk$!478A!E5h0oqQ9mTcAC-xEojk=gyB$gZu&T>PsC-Qe7iT(Z0E z*p}_QE?~R17jL)m!{Mo^QupYA=@}CHPqPuhg$l_fhT}pG>McBA#I^j|DHvDT1-J-m zDAKjz*7Atb1BIZo@fwsf=3(>cO50&hy%wASqCUNRdfX43;y?9dXZUcuwMH`f*blC^ z7%DFqJ<)rU7OW>4vs^>64V-4y?lzz9c6YvY{E=DB^Ma?9nd#mX&58g&4YG}GLKqCE z|1oVp+Tol@ibuTggrzhO+(TO`VcEe&pFB*OUN+({M7o=I9p7Yhskb6Q9E>llB4kqK zp+2I45j?GRdHiGuT;l<3JyZkydS*hFj3mY4HtLzS=%jgS@`nd0nPz4GEpgB zHr};Tb5hx`X0l7_n}DZJ=jz+x)*E7C%BLA5HuBqZ^lVG${URTc^C zI_&RRqH{h{r}q={V~hG~Ppi6-3Bkl2|Z}*1GMNC-pHMoYG>}6HsDI z$i!4PbDHJ%scS&kh{-#~?=SMswaPd*wLJaxTB<}scdU<5ozae+s#tGV+v)HqxtDPC zy^GWu;ta98OIagGNqMz5fExWX2K>12*jTFh2TBUT=NaoEvfWT>YaUQcQs5LLnIMgx z4CR?8-9=U+;wXt@_1EFQyhtBC-gdv46;lxpDTWht5&dOS8lP%Pkt${n*U1Atz+Z%I z5>aT>D*Z8odZL7+xMhU|TW29_ukOzp(HU~0^AwYCilW(s3#HZ92c_9h!Hf$N`JjQPGTHTrsohl>owuiRRJ~9VR z4#W!;D#1;96^f7yC;6Kxv%!0Ml+!|aOuh)`5FiKw|6@n17+?+ue)J{+n7dH&ARP|g z6Oy8iRA4jfq2R=0tnS!=G!|q=0>o+-foVX>6Q`29C{q`yaTQk}!Ga&l>=`o(?~T_l zRAcLjW`M)f82$!Eb4Ut2YeER3O2y#NNdt0+ zkbGw3)zJJ2)@CG@;2bw?$0(Oz%Ew+MCto1H?P*n_4sw;Sg;;ZJ?EmFRF*kPs1#*L$CijqH+A-e&6eydcG#4 z@Y2|Ht9c6h!rVpm^p^cE5P1TtCife`^MSJP!MimUN5p|Li?WCUk(m^RRl@XQ9Q8fl z)Dx&yK4W_PB@eWuzR79D!P`T3T2J>~3P)&t&qoxzb40x6HUSA!BGqY2#i+y>fOk<< z@lp=%TZt<8_YbHtWt{arFNAoiIZSsOvL&n&15|&`xjOk9;g)C;K)-W-`_oA%=DXFv zgzfY)dzQXXtpPWgTKWxbKW8)dWO?V_OZGJ{&L4=~B=*Axl$LEDDj$5ZZuc_7j|!@~ z$#C>{I;``Z-eGvF)}!IZ#FA4VP)|4l>p|D8JN^pS#%nL+e(~&KCTyTQyFsqCth8G% zJw%A!qFHWl5MY`a)THDq0$SIO z5*6*7qw_JbE%~J2?Ne%pDxk0;=Hw)9GjNnGYh=e$`4ZS)&73;e)GZ?Z<#hJw0%IZ! z&7}rvf;$%cDZ+p99xkY;={}Le2Xk5Dngtte*FT2X#C@e7l^#7r;O(lSRce+SAsZ ztP`j~P4&YGkwo>7`&abn(Nqt(FIlxh+^UoX$7pcq^#oRY1ju^v~{zz~KQ<%TzCG zu(Y@Ja9GiJG*EH=a+oB|yGtC@N|QBlRQ)qV{|wR&XudiEgb-yw#fZ8QLq+p}h96~{ z>!ah4875i(HIjmRyz$#WI?>tOhJPj15@#@t5F_C=6;C3k85E1Xxum_BX^+cCxx&np zxH4I+?6HrHKcLzdijj#TZY0!)z!p+6DgJwzYNWjHb${IEC|6@y8izTre1{9x>6A$f z{gKUYK+WQoCsN@j6&t`*gS#W|oqn^d{mw%f@Mc{Eno$_*7MN+r@clIBkjOl?|K3yL zia=<&Z#LshTR8sxn!twFW?|vwGdB{LE<`7uI#M_T-wFUPT$U_qB~wz0 zd{k9ELW;y*-^zSKh?ew&Q797~%X~=LNjQc%90teZi_oM9*_$uOkCp! zoOe|En-qGC>hOLcNa+Eda&d*J1;9nIoI>hDbniX{&FJz3FC}_E&;1M#I-f@JenEIb zAxl;&^5O^jjS;kQLC|&Pc!4I8)-&QipO?hl|GE)Z={W^5OU`e!FvvmkXvVUVPBP_$ zSFgZd;_(4V_;og2W~1CE744*L zHkHD)fn6*g_{1!1?k{oQ@(URjGeJEI`r1XlZ_cgvS%mcz0_I3xUm{x==1KU}^UP+F#Q@CdUhl+%A875-n@QBeq zJ8g_R&>G3v?rs7U!f6cJi%(KR@MH=DbLFK)qAhLb;2hPcMSlDvg&=-1PhEmSvNo26 zs6y+JSom(vz${6Pfa2<6Jiw+uT(NZeq|!|`_rHe|B__tptw|ualx#rZFwbGopF3Xzf37nIC^=d;wI36G zf72WkUNX*yvJO~R67uI^ffeTM3%0q-kOh!K4k)-&2Jg*l-klNKDe8x@F<$8#*wG6V zRi%vg^xWu)1|3m5JS^kFHX>NBTW9)oVu)S~`JhTVwdp#~y4l_B-KyX{w%|0mFwz}W z?a(rSja13H$%pUmIOmTfT&g=@tp9Ue)xx6s>18uG3$)21T1~4mg*}Nlz{Z_PHkyq{ zwfHa|f(t8p$a@Bn!KYUkFVi_MQrmOC-w2PIG8Q&p;gRAXpLFrwNVh=mhWYxGR~Dc( z4rLP=*~o#(uM9TSpG9I^IP^>w?CQ8UycYg(vwJpkBp<>o@{Rg1LggNYqu2wJ!L6hUgHznN zh>3*1m1Ym@vW0$v2RGmWb75DT{fw;=?U*%K#!YzZ7PtLPH^PY$pJVw2$3!|XsBLFY zq*I-U+axq%<^49~b>lvzyTdubq7r?u8c}3kp87N;mI{<&ydnJ0tQ3MAE`cTR|HX0% zSU-svLTSGq`A%6nY2YZlJjS(lrU-MyExa@ygYj)y6*H?M^|IM;9fAv+w7h(k@oSYJ z%?J1yflw7K_$f@5={Gi|d=+$>H_|qMbZu8duOxv;tM+{sT~CK=8oU1#;j=irG`88b z%3%?Z=HMEp>#_rjK*VH%VX_;cQ9zFD#+AS7`Qyu`DqfZZBY z3~!@igT^Zqjjeh%i^P$v(A5w0vTg6^G%F&R1f0j>_Fc{pV^h2|IE2p zNx?uGM_-k7$I!M_*2hq8%Jl5M>wohuLEuuFlpGGt1~;K@OQp*`)v zXVWVBn}3n0TM0+fRif=|&}632%tQPkG}4~7uSp%?eJ~9k`4{7*{|12`hXCHI%VmGI zNtSyqIPi!ZpV&TD(o&>LTMLn!&O8I*(v0T{MV?v?tm^r96WFaZ&3jvsXeS*=!Ad5e z5_wg(FlLZ##gIQ3Tu*EuH>cVvTegcatGn55w&d?=6F4Zolpt;;%Pk|TWv;DSWQ9E} zo6thTaP)~ZJSC_y4`HkdMk})9Ekq$-m}t2S!$tm|gxL`tUEJ2W7>oQs>jcL{w1L;> zF6c&h@tyK>E|z8=tE8shP-ta1Tp=p7vClzj6b&MC)XoJ9q3@zqyGB&=-^uP9#c><0 z?*6JP;otu+&skx9wDMUjtdGX-$*%q*pA-a=YE%%N<}Z$@LsKgiv%sCPO5QD#jz{+ui|sD&-O_As*^L`IVNMw zaZ>7t*uouqLU)-yo@UCII)WZN#fm2VVu5JWr9;hHUj5^DBer;lduE0$6y&ql zO(1R%4ouEFtt0~Y!aolTwUiPG(}-)_o34G;RZz~YPwK*A^v$`%5F_CPDV$JB{d+4T zNqI!kV&134>N7_Z;C4r9jWI8;F1s-Exb;AKq$e8rSX`AA1oB;0N~rOVA?e|MNF-tb z$jQaoyI8K-NEDw!I@SodaV?jiil3B^adbeiywGGB>31nDLKBM^jZ3?R3uG$R=Pt@= z$Z!z)mX9;Ljsv{Pnh8n3dObfrhRhsusqR;FU)xqePMy8q3w3gx$Y>|ypuKV4h@}m- z2dpnEaXaU^vo9Koq@-b!7lmUZ#^!Oqm5ms*UIS)Av#*~;wze5wa~MT>mj6eBj!ZE4 zT*dmCyW5(l2w9BWY$GP9QNld5mt2z9~YC)&pQqAS!-p z%n9&5oAEb&hM}3#c|ktaUz2YNWjUVxL&(wX!MU&IdTDZ*2Ph7lf)u!eRU8~SI?u=N zK?vt2Q<1W08}+M1jXCmkwsC_ku`}Y$C`uB`T(b#D5itS~FfcC^l~Z5X&R)cb?> zS~2m`rAZOaHi+jFlKMyZ}C&R{-*pqFw^;}7$X!{CzWj|>ZV7g8Y0 z@KO5&!%4yL#FUwJNOo8R7#P~^U`Tep-uztIyrzR`Ogk{yD0_4eC*Hg3#|2L?E9`0!_anACh^}r=*tdX>g%hFA? zvP$#7TE+S7#;{tE;tz|eaEkEM2uqqwY6(DTt%M^bWK`u=p@sedhSp4>!oLDvS+ ziBBpEt1B|r!q-HjXYh|m{=nhAZBw~T?uq%qTl9DC-h>W4xT zlCvg_zVLD9RFIEj`TXHGTQI?4j-ZYNwG|dikpKoy_6t7QXBg@4I@mF@CK(j7PZZ3m zm$gAiO=1gt?H)4lnt)dF^L{U2(GiXo5kSXz7daSI0PK8V_?fcS-6dO|>{FmDFjYLS zSJs6C^3I=REj$sS+#(Zs`D9GvX)C)xi>T_Q{xYnz`KIG*cgiWG}xuXHr-i`*}Y(6v12Q(MHCMm*cv z4D|T6?%qsB$5{phrpyFs=@NgEk;e6gAz}~A7mK_|yBt7*TjJYC2p07!xp6-v z&A+`(zS0n4S~K?a@8C&=dE+6-yRf%m0FXmfjZtuN+x!V2APiI&7kR!ky8r`35>- zmZRGA>5OCK8T#JRWWG91mm$dEAA5R5oyoz?H4b5VAuLuc$nL&Rj&z|>=Q;4=SwCM~ zqK-2;8+?NDk`MlU;AgpB32Ar1y72q8$=K7KZYdG9JEfmP<97}$A}G~Ga)|W+gM*q% zh_-m+*TID*o+XWr<{nf!;UyFG%c_o-aPJA@U%A7d6tDIQsiZYRFFhJ|dnJ{C?qMY40t8(wJ#C{`}VwgIuZ|a zc$#YTK4R-hL&S_6%a*>VY)g(uGNP`XkxN>Z1aARcN(G;YE@)cHno*^uo>O8%NJn%rwQhzQ21xwouQ_Qlz^D zM4|)nV~sbw@Pa86IBRv58y4_S-29;6d;zv_Nu*{mgoggv4JKgXE6~5Y^kIJb-j|$G z=WJFkK z&_>f<$r(F;D11QjK=e>@e2Kp>)NcPC=tqr=8`?vOslpA{M27hhsswqx;xOa`5@kKa zdvGoakMZ4$)ElUD5j6t=Hh@nx+*)sdvt2qc;kvOAEPiF(l0;O5O8huNxS-pl5Vu%0HMO6v0BD4WABCpa}NX;40vR z&vOo!#4Q{H*qDE^>>;x$L4w(F&H&(Dw{lOQX&x|>CORl8F9f~|JCq+o#k^26Fi4Tm zh}HwnsPx(Pipy{RS3SwKDVU!BKugVzB@mrpx($R8#5qoUg2BO&d`c+{0RaGtRNG!(9!~8tBj&L?Ws4LSlrn(Zd`;R3QAZU<)FZ%=^QR%PY!d@2t3*Q zJC!HM(tS=#!n*g(JxS?HVF^|#s?*;YOMj7Q-joT+3VS4Z1Xp;A(9bgn%17a#`RJgd zr0&a`p31vz4+Rezm%l=i)`rwWH9CCJZ@lr&>RN$;Hy<;izJ0-@1)2<4`y0?~U}s zNRvPKsuY~?Fz?BRbd^md8Ier5#@_51jQ3+C*9suOm2uJ}CD>j0YcHd#clk{o;2`H% z-Bb2N#6cB*^!u2N2R%k}pXNRYcqSiDEBCGhK_|U`ecx&zmX{w~8eMz2kS(}KZ@nD} zdj>TqW5X|XV?T^>#$mm_@Y`ExvX}Ekli!S@KRRtfbj1 zt@T{(FE|HCWT!d$0JuO$zlq6yn7J)GCyCRaNE@2Cf0U7k&6Yyz0JgsM9AWNS4>5BD z@6SJ|!WJs{?vj8Fm(@C2%DWXq{(Mo3qr{P7KTua1=y{oE&AbRwDQiw1BTB~xRYRM~ z&@6PkYUxy%;*K}<$S=HX%HOGR!M)LJq@jJ!7zTX^Op|)Ie{{m9mBhvfZbL^jY!2#k z%EyaK)NsT)rB*TguMj`nEi~jtFg<0?u1tcm?j_hT4!Jhx@a5p#dif;4NGd{JW zSOIFnrboY9nvU7RfVCQ?w1<6!>^|lJPT(|yKpk5HP>Ew6`{6MjR6b;`b;A~$0F<-` zp*Mp*Ye9S4Oxi?lH;YHMkTUEFC=`G}%+;eufC(f(#I^(g9su8M7#sP`QO?w86NjeJPzeEIu z;5{FPP-$QBAWSsf$>clBs3DhPwMUgn%PPCpr%xCw9!(SLIMN93OZH8nd&zZui45Jd zB^Gb4`Wj%rPXP+Xgego$3Rb+EK%Wt6>*M@6DUuKFnd*}uy5O23>-0gO5oJFQ}_qQU<~&7r3r=rN22EdGEy-DO1KU4ZuWj>3`e2dkdScgP|Ds^8z` z-KrD{6w4{f-$4yja%bE9q1yeT7u?g?+pBj4To2bz=5yOaxWhf_FmnaY5jU&990xkL z_aqF+Y7)yC_hQ@fdJCJ1&#h*4T27x#f@?x%AE&^?(j4a)_&<3HIEl^pdQ#U@qJA_{ z7+PS1_F{OaLrpi0nXf9gd27+=uKu8CZ%9;#o0dxGs#*-G?qS>Ugn^9T^(mA6HP8@f zcXB-b$I;}7xvwfKxK;awK<0){_hQ=*z5TNoanb0B?z%ATZ3DG+O>D$6aDqk{tX0sU zNxg|!j8y&CPddo5Ws8Zs#OZI^dO~{ab}SW}3#X`-`!tPw8sQkK43S24%cIO($ypxO zpffOQVDx$WWI-&-#q8jo?e!kOY*Ho}3Tf>~WWEn5(6-}>(4*bKi|==*O|+|RU}BFM zO&S_>i68J>kvCGP9n?4dMxM~N>sC2^jBO&i>d-CG`*j!miUq2Lij?T9eYxVQoM;N7 zmGQN{Sjlt-GFlg*kox%P2*2x17XRAkUWWkoFKXSM`^p!pwH*%*&9sHh(%8*- zx<%4}Ps&PoOW#vsOKI3;(1Ylm7`gLoPp)FTfglg(iRoXAGS7?=FfQU?AhaFd-ovGr zW<&p~4V7g#h~Yz%VuAsnpVRh!C2YM7Q5(%;5nIS+g}M=9*2rE|ct;TIoD=X)y+kA> ze9h1kyO+@cMm*TJ&)-1H-jxSpY$s8rlS!76&I=Oc7oIH@us{h+9|ng-d{00wS?Y@W z)IL{VoT&ccTpRF^3P{Nw;U!NZEu)isHFbT>-&-~2Bg5(mH^)UxkbU>0ine}1txzW0 zQq0*p+qW)BV6I&XDtJhKoFs+&gjD}KcXxK_L6LS?*n8)K$1I-&P2yFaz?t&!xAmZ& zs*~#83|B7zdQ`WF--|TkB_6uJyf6ZCedi4+*a4U41Nc%*(PGsme&$kC0#Ckm_NwEf z@%s0|xwt`?BEQb$_d)ShkzXZGFmK+RmIRg)iB(%50}l`@_QbNxPaL zo0T47#Q@fY$k_rJt(>D*g-=_gb7wOiZxz;l%kWHELNN!H>%61?s1As6OOg{5$SUN>}QbEMe6A_8lh0DmszKU zbxs})M|Wn0p|+th)854492mK!zFyU|rv1F>cJRX`I=LT*tx_v>&~Il>X__`&X+N*y zXOuKk$Lt_Z!UgA&B_6`z3FU$CM(R%GL7W751ABDIHa?^75A_w0(u9r91Hwp2%%2G% z0!x}x{upb_o($7_4XWHGlY8STr>+V(7L-|7Gh%%Pvv$Yh`AwNGJcZKNK4|K+UW0|Y zV}M^6GLZx@CLjaYb4HH<5=eoGEDL}D?YoMzH}a7(+s+rnvJBYIJk3r9dXO7jm63_g zJxPCUcXZ^ixR{M{B{6JQN3_=JphD@`Pd8lVqq(7meqs(ACf1|JSjBDwTvET`rt~I(CP}^wJ(NYw+T6rj8>H*I! zPQK?QYEql6F_|g;r?{NPQS_A68350k7J*Hd5;$zcUc6d%NO1Q90HlthtfjBnqkGI6SQkv3?D zCrN}CiLDgk65;+(`0dYTQeWM2!^|h(O8B#ArrK0I)goLwVNnnf~=d_H@hd zpOk<;g6gJKh~3Q>gFiAI{&JCO5WWM5CTUJt$k0uaaRzZ9v@qc~mpL(a>}5(K*tbCW~D z(QxyB=bKE-OA+*TuN{nrA4H39J4EeLyKJw>(Hkfzyo=kGXAk2%_=fWVG3oDUqkTFo zLyHfwB9Nn;i%|$~qOejR8UdIx=>sq-Ku3wIMvnjzNP~$i3xELb4!s(KnS8q@%&&bJ zPKm)HC&BWZ$7ukyKUGc*L!?F@CG1vpli+sVfqYzMTOjzeVuB*g9+Z{2@8>>hPX-+p z>9VZQ6wdvcr)KrNS#=n*QD&uG+Yl#T0W@5>47^o!@v!k6&&xN1z8H;kJ}4lg>A7Dh zq<3_Wzg6Y)z2;s~V!G@W3ZiE-+2%__qdrBcKFkflqR99h?3;3rVFCEB{Ib+{vqyy& zXY}7BNng*O6G|;|dO6T9ORaVv!}~~Z>B)L8%NjPk(Ezigfwha!T#)(G)5hUv_?}16 zf<^=zYER2Mf}$NIxk+j!sFx5jeI%9%fsVldjG|RP4zq>%6@{XWJ}ZZPncl&=eU$hL zko(JFmPuNENBoLSQ>FA=Ga2Ce|Mg_ZG^$!G}(Oz#e5Rca3&W7mx;U_;`{M_}j|} zucrBG5HXYWJ2l)2UqAoGJ8<8qn&uEezK1+U&rdcq@`HpvThIwH`K>U|;2^7+G^?sq zzsIPHcJ?nS;sKCx<}S-@qo zmsu!EOMzHz*1X!?S`?~R*=07~^KcI=I;rc#HJkw*)?)_k!~d|>mWedkwn?sm`@RC|LE?7XK|5F9OcCrYvh_=D-aeneHK=!CVT=BkiKx^dZfhyH zVflL(dRv$Gl0-8L82Y3nFb1EmlLxRqln$aOw2~WrQw27pUYNiK6R3|;f_lRm!ih6j zN`mv@8A;3V^zwUwz?=a$fp%5IJK02X|B|8>w6*DiVXizLE`Oe}krG<(ronfP2fSa@ z=p&S>2;m%A_co_{o>NFmA~K3shy=;osDZ>}2Q_pzvdJv1grhXz*xQ+bM+X&$bqk^< z&qlnzBCD+=58UY@h66WgRdZD-q_t~!YPGbhHCowt!>R7S&3TFS8Ug4slMN^t;7!C; zqep-ZG=RjG1>gXj>4ClVU-~HN9R>E>)9UgT3BzS{7s@@Rdpzy2&I5)N&iLA z3-CpZ8H#i6CQG*e$%!IHNQBf4L1ruX>1IXw6Z(l=+II z&dZ6DtVaHp!8!~3$lmN3RAeGhOhg9M?d4A?ho^DyJUePi6otZe@GFH?_$<{+9V8W> zWa&-<#z(J|=GM}$K358^2ChUrPZ17-u)fR#x#j>QCBr!ea(=C@3oG3=ILg~PtvG$W->KxyrD`%h=wUen zm+aW;BG9PM2#jFhYRus=>f)6c&IdLAHe?wdSi_gO*uO-#!#DOjrGo}=hz~q0o9KXk zxLH-Gx~MG~I;ieq3=*yaTR`upHpSt%P5+|D*sdxC!nX)z%PRH-8E`HK^+R5?sy1B%CjkBNxsiNr5zTWWoZl7Uo0&~MYWv0?+3W9~DdcmtG*jd=W! zZb#3C*p$2LO6Ggv$@Z!wv~M#lXp!~C>1e{#J?#@*|588}92}ZCmfkdFZQFkWv$x0m-rv-m?qAle+})CD^SE`b2@>~Qux8}Q7S zIT&zT(BtLc^DT@-Fz} zkZC3ZdX@)O-V37zYHk&@s>bGajfeK6KQZqmsSPrvVJ`Zz3|`-b>3N5tm3devZS%fS z{zA^lAl&^A8#uQ>G}P{OD_vacvYZsvtQ&v`6vS_4fp zhcly6n5!?tqq%_#L>&?Y;kCHUv|CERmbQI`Vsq+XG980a)1ase(aK=RT*u!rLoTg{ zPBYXGRu$ZJR{o~+(=qCn-eJ3TUXN(+xf#pT@~blL$L;|Ar*P}W1f&vBDalco>X*dh zyCTufmeP5{e#qiFBnlcC>=2_a1R}3HO;Dmit2CjPa2!RsLg4FN=IUj&OgN-H30`hEll1yV)@vmj z9ZCFcPpXTZb}Y+<;1GhYNG9ZUT7?M6n(FxbPr8mlop0hC@rV7U>oU0Ios%#iN|PKK z(RL3!aq4m-us5w+vDqX3b&>&@BvMHw{u-M0 z5d+?aRbCM$$df6H$#SYt<=8U+&Wu|ndDBjgl@R>1=J((xGeorCec`>PkqY2Y!U!>@ zlRgmu)ZZ`-Wb|*(1#SGsxS)eAv@C{{ZT)y*dBQ}1<0JcMZwh&F7 zVd+npZh|QQC@E$^q!czD*;mePl?ca-|9O1JYL7goq#J_bt~Wz85^)b|qBqRwY<;>* zKL07eZ+&0Q4eec{X#JNW{Wpw&b9=N_3i@J{TOHzmxGug&PoC@P*1)|9-3DIpt%Z%? zvS3Fxx$z83ur*cE7envO9-{=N8zy8V=ytyhAZx5 z_G_Cx5|C`xgcA%z3M?h_5z_6kxxLOGr70`dAk3_P8Xfl>+>3>NOnV(z4~C+Q`)6w` zxrh(fs%#Yn-+T47Kr)&IFfc${h-*fV01`-niEOYSH~{UFNg5WRyI9?9GDh2U0B+d0 zECA@1GwUUk{!S@0{IiQ!c_3p8&Ao{Tk5T+3FaCAWbOi=uB*Zr#JH&8pRTyZ{Gx>P9 zfTy?c27rgjphc?lEb0L4TeDv03Mmv@4ZBmXP-223|7 zXT+SbTO~Zmfaw``b5Qc!-{`#*m z(J{4{t-4twr+~V~GCJ)xKP`Sm*>k2R-s+Uj{H^fnj<=Q45jK8Sxa3BY^+?m2tp&Z0 z?{_ftgroPW1u!@d>_0yC*F~SO$Bd$=&q~*W-tCfPa+sSx4qEtYdU1*(gUwF+%W~xA zF;&p8*vGWkS$P@RJ@?iK_p@dHqhy$nhhP6Vc2IxVwucqiZ6+GBn)fID`l2wH%qbFQ zIErNemVnoyzo6UwFUKUZb~?qK{Y03)@ouFu7jc1^##h7Up-`EOZl-`psP7i&BZ>pB zybYP(T*)~nxnN9si>ha07#T-4#-7q}E!RhTIu#zw!4f_kfS2W;Y>}xw(w|PhWEZ0` z80SYLe!7lRCBh#s(w_!g$}`@Wo{?$E?mrzk_b=1*vv88kDlm@rwUWRwDiY-Nzez=n zh~XOB5aJOIuL89X*Zq3~3KC)@f+gKga0%Qw&=V(Q!0E^gq$=RciJ1xjeb&I$MV??3 z6Eo<)8VDKzm@=dWFf?EUXsbq#01`-niEOYSH~{Y`mE%srDwrJn>^rRn=RVb>;H?eh z@bg`e=r+CQb|6Y?f-;v(GJJ|4YLC#kQ!GmW4HURYVk!yNenKT?E}-V`S@17NL>I4>Q=-q}P1(Mr2^9XxA9r@J$T+pG1g}z; zx}d6}2SOoj6W$l|cSZV7ww^rOtf}%rwhP*GQt*fI{d5^yMo`yM%QfH4M%U0`q@2)T zj5PBSl-()6)!`A5|3jyNbWmQcyYj(ouqO$&Om*3qpET3TXafgUW#`>gf%tPpc{LPZ-Llz3&5{&cEmu+1}*}})af1H1|t6V4XYO0 zM*)Hc5WDb?{2D-RY^FkNo1g zNer*Aubx@tOJD=hdEU*(L3QH4rKnpGxyC#CZ+eP*ra}qEdXf`>BmDBV>1_i1_oPcF z4y=B$StoOM7LtDi8io}6_G~XE)O)`>jq4prKO0&%GG|B=tdzi(&4Y14oMp=yHy$6Z z{5F)AF)5vJsE&-CbiK-yj(vpX-|mdk2CmIqTYJ81(CRG4 z(C9d}?YUS-4?}f!hS(gMcFFhu-MgoJu-UT6Ea*^L{|5btGS&kyHV_<$t45Ci5=e!K zY_Kah0NvD#i5Z&>l;r4`o7DoNs`oI>L3P85*Or1%Gu99LU#%tGrNI7(hbje_nFj%n zOlpAEQQ)|NhP&IFvT;hm$ImP`df4rTFMnyN>IeVqiE6n&n0*yHN>!7~T5(AX#CX00Q!f zKC3zO83hi~Zuw1zqvV>nP}OiEmHzO>d#ZY+4({aanHN54Z?svDT1?dQYz`9y-v}>v zI9TqfB=$n%l$S#YuNjXU(@!O@I%@oW8)tDPHQL zko+td!RR=(ku{jH-50a`7s8wma+`)f)B$X+A^XAU4od#C@Gg4n03? z`~=RT)=o~|X5e5E%`k2Pf&k8v2$R7`dzE4clnGC?WK6HWa;LiM0Nl~xUk1i<%&|}` zt8mql8c4edMh^aS>)%?|emYbTAX->D8Owkk!r&wEM}N+$jz!w zKUHbbl<7gZCk6=tcZ@`d0hpo_>=j}T!Y*oo38vu5 zz&>fgcydL@h7~CAmGd14yLwUkhdtu_$_N`bqi?ZYmN1OGJIsBP;J}4&#jFuV;uEo; zNze#+R@oYZeqL)l4z(B*0jI0ZCCPy7bna=O=O?s2z3HcM%2`&OYzp&FkEp|Z@tQ@g zxEh0cL%H5dn&lVgPe2hs#J)0Y&YOWWz6i&mR%W$#e=Gqb_Bw|-rTqn67@lIp`vqsl z1{8b%JYXgOh;0qKcQTs&n{*n}g(Zw%Q?z!BU|@QZ7gyHbC`|H;lU08X)3+n5PdF+< z^+5i+TwKEdlLNiKy@Ke>(NP`mo7K2qqs@} z_*4fvXW)9Ob>G;t(kdjs%p3_@Pq{^Sp(XcQ8t27geadK2Rcz0F7CZCfs099lx(O!9|UF{Prof3Knf;R>SE#jd4l$-wex^~Z^W!E#K}g}&f5vNv&S5~zq$;$)^- zFqZ9<<_di_ec&YJq|~amzAa))+0b5>myd895cKZF;Ed$hx6&lXwZbD|n1JqYBWADE zjXK;?Voi-4O(foe@X4d73%op}&rR&KA~-4elcRo|Z&%)oNS03jE}aJIKm*bX0AYu_ z+QFoH)Auo_8Nt^rxxu{Gj}{kIV!;(Q6MsYvBH%~l=O!#LS}6eeGI3F941u)??+(r&caobSu-juex7@$bcaUKvM<=zdS`#6q?Mne`m)l%AF@Z6UHQ0U}L3<`~TOv$_c+ zC@7JiikKtsg@rKO!SqjQi4OIveLZ`Xhn{UEvf+}Jh!5OOZk5LFdc|OhDOJ` z4&$?H(|f%(Ea`N_nEJcj0poQz+<8uNs0&?L@V7DCxdZi}qEt=4|+ z;p15Itj{|@z~oG-#{9xo#f;PoM3J@t!BS$pT~Z<_ySlZ%*F+9F#mYI*M9=pN&RoJwbx zu1_fl7%0+t4$7Ul_N)6O_To(`Tq&ix!Al3Ahd(;H2wEFHkJ-r|R>x`*69UT=+M zc`?kq_sz?|#^on;5qR&PqHB!8Abwr%hv30qxHZd@V;0|jV_hEkCuLc%O{SD_ututc z%o>l{sb1$QTiYN@1`h!HG3REOuxpoU`0`G(`w6jBZUV_6mpp*~g2q2lCRo_3S26|_ zJF;&BXZtPdNTX0&CP-gXtogmstr_h^N7pG#lc$8*@P&?J^B5MHX5nRy>Q{51Uv|z2 z&hiqSRlBF9bQQHy4Mv45W0+aoKP@4ec2ogXqhn*%KD1K4%C*8)L8s|hAy<`#UVdbK zRXG9l-?KUm%4Aa$GHsuR$h+AeWKT}op2`v}er&aDVw|Pi4WIMe06(qA5Quz^f_g5`hA48>cTT9wof;^Wm7uWX4EMXXnMf&t zxpV0cVODtAF<8-tytMM&r`+AmK;)F!I^{*VeAtC}nFd$6Jd`0o)poKlv{Yd3Urj@v zz>dk&`G~_44dvO*e8-YbW9o9ZmE^&eyJNl3NC)*5pa#n0nwN<>~m1Uca?<`B*Qt&GO=pyCj zMUTy;2C%UGj8j-WyUkB_cS1W9z~3S~iy!K`?m%}SCag|m<65l~curydGrZz(krM6x zgG3kW>1yZtW6P57!)#>IpR>TE!L(|NE%G=Jr3I%}dK}QDO7ZGT#J~eWkoN9ky1&8Q z!B`EtG<#4=%3j+jy=nAUSK|smzqmQil{gX#3HknP-GHF3e*(RLh15_t$`}#MHc<8l zOSsa3ewxa@Gw)@nJCHp}b6iz&I6<>s;fH&4A>)HK_RL~_!N!!NmDOru-&6FBo)nxe zF7SifOS=D!2Q{exDJ`k39>?3AF<=@BYJ{z~(t+(JlAP=|>g`&VzfH`i`?6=^3f+2) zv!lqwe>!ARm<<-n7hzPP_q}Q^>Q8vknqwnMl>I|79fsKtao%zrXjan1S3pi=XT$gb z#KEzCO*bKUY`O>r^F-gQn3OnF!$)f|Tfh@YP}{o@V>~wE2jz2hEhyxo*uA z`b)`*d=6*mDoVnLxeLh(da9)O_*?IT6R$^7d=N9tRZNHg`$6Xvr_8#H||w?$|2luxWk_&HK}rRGhMhca>v55{@Dw@a-oClDjYAE_bF4J~>T(Nrt{7d~#f$$s znt8#It8YplT8C4RE7D2*#d$EIh++n)#ISG9o=7 zIe6=!qdq|oeua251T7+pG3BhK*EJTr$6n%QmjTg^z&@EQLSvum$DRTEu;`LTskUjS zP4m>?3o0ZAtqz#@@6T&mV$wr03i7h{BDm!z+RnJ5unA%`mv~D1m~9J zJ-RJZsw{wK4F`zs&;avRMIiAG)^c=O$k50B+O=v9Y7c;*ez zUdU3dR6Q+-+&WWKAcw@gh4Zo_v%^E^Bj_U|V!gGNnZ#hMZ}sWM843yf2Vz6aA-?FH zNr^>Sdmn-c;x9GPAWE|fZq8UR1pp;Fz?E`wO~4G3=%96;XfBp^jVIl~2S4mlCy3%~ zuaPm-p(_`49$#8RXF+RuUjX{qJz^>@p8%dMX7tS;vQK!p8y255%`i7ytBtU0ueNO} z*;-!saDiz$!mb9gg`TND^t>Rwa-(UQRu(EURsjgS*&}cj2iZ{f5$O^h>xr{Cq~Ub@ zAWyGJ^oL>_g|9?-Q~ANIcw1DkuJALvhsey$2sK&O?EIyI(8t3sJfW=MfO<*^zV5y` zMe(#MJ|DQ3kpj_qWlR_u`tvi0;Gks81x&M2x@J{;ojzVe4MF^ zyjP@iNcpj-anbz&5@omXuQ{+UAxQ9qz>uy67&5gR7TTJRsBxz}8CpUl=~5YiIC6|q1ZM%c9g-!mc6;oJ-SpYav1zLezXW>gPaK#Nl((No%!Vg zY{N=s>WNs;saMgzn^A`ndY{Sp@6@&QtOiYcjGca7)yTaYV!X`-16XsmWC70%EsKMQ zL5-Z*#9wMJR}9MTr-*F$CrCe<;*-z{2+HsET#@!ANAl;O7riqJ*Wa%r3)&NqTRpV? zMEuxq2c2+#+M?(3OytxMCEC+W0~t|f%p1|dN5*h;nf+5?`8ZjgCyqz~wf@V6s2$7c z*wL_0vd%x9N^B@vyCPP&qk5D@oK~-BaEJ{lua~7Y#>YDXW$;ScesE(N1B=vz0TfyR zZ9Cdgd(va4ra`pi)9^}!noWK*^33%X+_XP(l-u*^J%;TG2VLb91^asZxAN^f8 z^NjbT@aaAyjxj}ILy0>kqF4e}YfCt3*eg~!zpAKN{&Hj>9`($*_}6+JWi?H9*PKmR z@U0Fxa{kQVWhjN67dWUos`igl|C6JFreFpmkn_6zu$5mV>>AiHMO#;fUy9d%=!gh) zJDysfXJz=dbICS`-~U&o?DH4ri1%QG;bya2@>6PBqe1Ogh)UcW>{Hq<)^N1l9Fgjoh zcxy(F01`-piEIcfH~{O{w0#;?(=zK-yd^#_n3(h{wm4%! zxsgw!sG|S!_m4eG`m=$XY&f zGZ6d`l^^U3@h1-Zf4$5H-#%}*%0_41x%&6{9MgDAKy4LV_fNNHHmr=NnCk!I7KEoG z$fR-9op>_$!c-}GdwXSPKO>h6EeazSz$4~}0V2z!Zl>D)ypBR{b5U3LzeBzA&|O*u z-%6~0`L7F@WumavN92^{0IM@Z!M*z9*#EUg^`~Y!@q{#}OsJ>L7m223q1R=-C@ZeB zfhwig8^xNY5umSrLlQnuxEpawBPel;skF_9$7md}P}eBuzGX#Y--YOyvI&&g$Gx8? z<$ZO5JJn*Qzm$9Rp()emCzu_5rB?io!hBImi^WR7wiBQl0hlt<1Ta2O2dFDXj{qWw zfr)Mc%Qyh*JZC>pWNLqJM?z5q84Pg5rqIKgV~YR4!z$}RAzt^;wodcC3r%L_ZV876aT1&!^$ z)jHrU2tb`3&_u6gf>h*(fOA|`^a0Jf%sIW+*@pHpGmNK@P3)>|kH>q<8JO$B*wN6~ zfewOF+ITs>E_5i~>a^H^aT%#1c_-w^5khU!jpI^$d_tgxx~EK^2$0!F?G9 zI_^*Z;cOu=_e1U~q1^xo`5`U4F&R2`4V7%3-_8aHOP?*!DyU?8hEamjsl2cVx5uJj zRP*TmBnE#`ChMUq1Y=leSbOuQ=3^S#s*#Q&Bf3+zg7$GR`x#A}bjN0kt zvGx^#p(I>SV9p^QBvOrLo$_&L6T}Y(1|r&?c#Jol`qAghgP>?0l>e=)>*5@Sap(;h_mv6SJkoXMR*cF@X>oL8&Z@<~jDIJ7K_9mPBtB#c66|{8NF6QNY2%eZ^ zdL*d97|-`i-;D4jO&i63_`w?hQHiXJkKgzl|E5Bo00sNM`A(0B`1cimP-{w;CzvJe?7-gh4@by9i3 z7b)@x8@wO!Fjs&w>;^DE0T}ozMvnj@h=GYLvw#5HGb~4V5`!MtyJ34}c_B;6m3Hnr zbDJ+AImhHlk`EfV^HIrwJ6qwE$8FKiPw1{F8#Z68)Yauzva>=r23QJHg??_}@`MA=xp(-x1@kjZ ztdBl2e~@mdJAJ#&etzG>i}YIBxO%cu+58hI_!keV7Uo3P^MJnnq~{beY1#dn?Pq?bxfT zEC!Lna%o$9n0RUz-!8UllpMJsJ?#m2;y@DsPXTE1`f4Nw0{A-BBWORC5th%fr`5DH zSFwBjSn4ffxL+Hb+iFaLEVHN~Ap6Fy%Uu`TG9b6N2}A!#_M-~R=}W|5pWYh=y#>Z@ zFJ#+-Z^7iv%f3jMYQp0Wk@*q#$W5dsCOm;T@THiF)yX)L#395{XItBMc+o7;U&Gfk zMOr(<9!dp;mv>7e)NwlX6^73*inK+tnOj~$HkG*2+YpWLWt4!;_3lv}6j3VNi?M!O z*A#R@5#1=+sAnEC{5BFlPG?LXN#SAS=4OA3=|vG?{A^C?w@Zg%*CIp@h8?XS^?>yS!J1s(~OUN6{)Tzaz{&M?`Jct z5lSrUg4ltuF?1`+&h&}Is&ns1azl71EiA;;f}~^~MbxP2Y$JmkYUP4q)Ic|DODE@T zj=FnrobX)HA(+&QEeJ#MV@vljJ=b;UaSl^k z4@42Q=l9z@_!8}i1+Zlb?IsnxQLz{}hG2k;2i5NA)fHE}8@dS$iSaE@%HJuxKUhu| ztW$Ec1Wz4=uUQXyLE(zgkB{1M?gq5-IxdN`;I1B)5wi3Nq{tK?^CpD1EcsML%-v8{52$;Ko7Hcy;I8rd|$4zl%Rt14HFuT{NECP*L{)#y-bk}p*< zdlQdxQt+#qs{P_$epZ z+tx!*;NMRK5@}cHEZ+VJFqZVkEuERU@^X# zwp;em&UL~`Wx%kGIE-ESV!Xh@f}-4Iz8?q6KU8b+&;Kj+oMxe&m?Q;jf$VbNk#^!y z_g$9KT-_Oemu_}kp_E>g<>X~UwbNoGniDfN9RfDyC*1*$N4M=B9%d*tFjMr7MaANw z-^yEPY#3n;+rllhY`%$#5k(sw<~9uD(I#6SDd;fX2F;933yWB1*4e$I7T?! zUjyX0+#~fyhK=O4R+UStJ+3Ddt(;!8fZ|6}AQ)jmWquEU70cC& zX1;66B}!-%DVw!A0B?L6sij$hm%*7B38+h3U{QH`e$YxuN`h-bu`#6Y#eako=2_Ix zGhbp(OduY#^%*8u9wNwmwDtveZFE6cb$31K@d20{f`Mlv+94cts4>RPD& z5<*m{Lzw3Q95CaPX^bs+A?mo_&jqqS$LarxUjv>Y&B+V9+5%8|B>Vzl?#i=dkrJ<8 zlrvE$8IU}?<$knG`=4Dfh*UumZ!*n|SD<@bWN2@r{_qlDET77fD@w0P&?GtksgqRD zp+t;jrQdhn5v*idjv9ybvY`EF^RxzyLKMm>dpfg@szRTPAfczjf^BD}Ia`+g@ez{^G%RM~u|NrXXGa~{!0w`#}ge@;z?%Vsk9c>3O3#yQ9yyB`xU7oAy% zI+s@oagyFiOrwg;l12UlSJ_EKFvlZH*;x!5YI>k;27^Lc^$ zs(PxDeAlgFfAibpZDO_-LcJysV6k8Vcsr1&AEwL+E_c#BcuZmBah(%NO8(pMhLl{b ziBXM9=79G7NFLL#o6ku&U`ij;$?Q5GI?By53NgMmRp6SJ2;qVp=m2BHSWa-Lh=ArM zOcmlxi5bCiAG}yi;b6daUYzH$;Uyq`!md$$Xg}i^w4@5QnK5c(e!MA&@$0pNm_-vO z`J{+VmU$qK;kK+b?b${r_MGzpOmq4iI6K2x02Vs>GAr+Xpfo~F5p0nO%@zO?&8z`d zOooDB6BZR%OP;24Jh%p}FtTCam@Id|%LBHMO!a-?NnJyu;L+>*okn-TgX~0Q!T*rB zqwIt>vTwAfNiv*T1m-ngb{CSv4+sC=G1A}ZFpABb6@{ZrVGh+0=4 z|A~fKIfL3_LHb;z4eTW&Sl?<8;z0D?R%$Yf-vi&@xeq=+Am%Q`LgF2ppK^FQs^4PF z^!BiX1=1ag)Pc&s-Ag@8o3?h9TzxsbLnD%y%2X%bcE|C8RvllwO|SSKc-w@vsyMM} zbHjC%s%S4Cpk1oM8Ugq+kOweGP#lR|6PQD?Ks3LwWY)s*)=)0!4yihG)x_S@F@G(xVa0zP=ZF_Q~JE7rv{CqP6h5 zOo%dpTfYxMLyCg*b&ktMMCH-Jrxk;mw=i&uK<;nwPBT4RWjs}=?}*Q}hjpfG$(?Z7 z*p_aaB{@A9#Ga|of@M@h-C#n!1hOXK>1yfV% zg4slq)vk;*t9mrh6)fwMMe!Z>Pts*#_>6H*QrMGAfNYzz(iSsg8CB(42O$DF9GVK* zvb#es>)5;Ca{GI437(OU48@nYzYRQ0NFh$OBPZx}Dass(|YTAZX z+8ZT)dHKXaqt)Wac9KySvMzu$Ah$QD5^s;hHM}HD02F@N$K1XtuP8J12D_(3(|C*v z(Q`lG32~wpFhT0{k-_8)n+5Ac7PTa*%g*oa;_FE^nB=>-nQ;RQ&h4F#2~RbSHx`G2 zgf5oh*d8?A*9ix4gWPQcW$__99D=x|;xzo{COP^C077$D%ZC~nDRTL00%+b_^@{IGp4~t84PJim zp_Y{J*OcOGW=iJs!Y5F?)2=lQG;z7nWG9J4Ve6*-yGokCzz+Xz%uJ<({h>_NFac=? zmiRy#sFxHDQh-irPAA5wjABV9m8jJqXj+kXH)E0zL3y^aY|M4YX%Z@y(6`Qcu=lkx z{RA*d0UMb^MvnjzNP~%Nuqzk<)7BTvLch3OrQ%JLtb@hj+4b^44<^yQB?@s@5jaZN(*f5tqMCbQ%5hB1ajkP=+%r$|05|snK8DU{N zJ-d$53*n}mkM#M=utJ`%IRqrtaAEI>9P>qI7)V3N``s2~S{Q*n;x$`wh(vlI9}&M(NLc&oV?K&6@GwCd8sL9MI)js=yU&S0@WA9q`IR%LtqIUQ_`e__ zPZDx**-U|h@z9Y&G+)$;FQ^{jh_@p3M^;U^fz7^c~MNRuH^$mpPhRJ@BQVY)Z zBL5jUPG8n|J_|p6G*qv%#|Y z?WqAJah{RiiC}E->m+I@gX`DBgfGYYaYRp-#S?SUX&kb%QeC?-jEW;^o7`A9KCH0a zRq%^puUBX{Hfx(xsRL3Y`rht$3*KwG`Mm z4S<_?B}1IAMj@PFAiBBla+=p{tQRl_1KxYm|K}PD6SX=8%zfUO;|~qDLg-P|xDjNT zs>$i%O+vJCvX!=wSqwE&`3Gq^hj-^o4(ponQ<`+OC)mNiGBaJTOhlNH%E5>N=l#zV zRyPFR%5jGT&HB`l$LtD!6)0!~cVTuvNEm^!ZVwe^!(7()_P(|6nosA|4gZ@EKdwO| zt7UH>7R!hRPKdYQpkVWW8&nK+u*Ai$EOTr6QJ4Cfbukrc9{)Fgw?BH`=1^*JU{Y(2 z7bsD`Td9nrH1?Ay0sc2a3nwJ%n6F@c4aB;cBrYNJAnN`XRI51guI;=AvCw#!Whi@G zl$!T|-7%Y9R6ngjYtTH?;KtxIQwRUyg#8uX1wA8BT_v@zbmm$th;RclJFwEY?}a=qN$} zKpvu}h|GBu2B=o~9dJoREd#;IPC0+4zn+on=|2sF!n;+`D3cRS`tc6UR_Ko`@{L+f z*>h(&2DU7H^U8Zdv;jBeKF}=T#$*`}$x?(7AjniVAV#eY_THc;R}lyXO=zKPQJ&x5 zym0Y!)irIXXyllgIh5X29g<-tX|{+}qB(GJox7fC|xikY|0hdcZ6 zDc=_e6J{-Cmx^S;A6ZvX(twfS2D)nTps$|}JSl&k%7Q0(KQVnrNWEPF+X{}e5r{*8L4j~H zBaKR!+y_Svsq?12@7>L!9Bhzxv9M;$|FF~=LQR2081FSfV=+c`2vW>w4wa<)GnK{2 z>u)xw27y^DdMEFeMDct7DX?)N{wZXPPF35Vr3v#q$K3|2i){N}Q`|7M@T_|Ri`>*# eq6P?B5H-PV#RZlOIdFL=podTiripwq*F^Wdbv^O` literal 0 HcmV?d00001 diff --git a/docs/source/models/index.rst b/docs/source/models/index.rst index 08bd72901..5fdca56a0 100644 --- a/docs/source/models/index.rst +++ b/docs/source/models/index.rst @@ -145,6 +145,10 @@ uses, and the settings you can tune. :link: /models/sana_wm_streaming :link-type: doc + .. image:: /_static/model_clips/sana_wm/sana-wm-streaming.avif + :alt: SANA-WM streaming FlashDreams sample clip. + :class: fd-card-video + Chunk-causal camera-controlled world model with streaming Stage-1, refiner, and VAE paths. @@ -187,6 +191,10 @@ uses, and the settings you can tune. :link: /models/sana_wm_bidirectional :link-type: doc + .. image:: /_static/model_clips/sana_wm/sana-wm-bidirectional.avif + :alt: SANA-WM bidirectional FlashDreams sample clip. + :class: fd-card-video + Bidirectional camera-controlled world model (Stage-1 DiT + LTX-2 refiner, 2.6B). diff --git a/docs/source/models/sana_wm_bidirectional.rst b/docs/source/models/sana_wm_bidirectional.rst index c4bcd033a..51e398429 100644 --- a/docs/source/models/sana_wm_bidirectional.rst +++ b/docs/source/models/sana_wm_bidirectional.rst @@ -47,6 +47,12 @@ a single bidirectional pass. FlashDreams runs it through the The sibling streaming release has a separate model card: :doc:`sana_wm_streaming`. +.. container:: model-video-card model-hero-media zoomable + + .. image:: /_static/model_clips/sana_wm/sana-wm-bidirectional.avif + :alt: SANA-WM bidirectional FlashDreams sample clip. + :class: model-video-player + Requirements ------------ diff --git a/docs/source/models/sana_wm_streaming.rst b/docs/source/models/sana_wm_streaming.rst index a9fff764f..c12e4b123 100644 --- a/docs/source/models/sana_wm_streaming.rst +++ b/docs/source/models/sana_wm_streaming.rst @@ -47,6 +47,12 @@ FlashDreams runs it through the ``sana-wm-streaming`` runner. The sibling full-sequence release has a separate model card: :doc:`sana_wm_bidirectional`. +.. container:: model-video-card model-hero-media zoomable + + .. image:: /_static/model_clips/sana_wm/sana-wm-streaming.avif + :alt: SANA-WM streaming FlashDreams sample clip. + :class: model-video-player + Requirements ------------ From 14850615fe7c8ea37582f281a27d73d32d695d45 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 15:32:22 -0700 Subject: [PATCH 61/64] Address inconsistent HF download method --- integrations/sana/sana_wm/_tools.py | 59 +++---- integrations/sana/sana_wm/constants.py | 24 +-- integrations/sana/sana_wm/refiner.py | 50 +++--- integrations/sana/sana_wm/runner.py | 17 +- integrations/sana/sana_wm/transformer.py | 93 ++++++----- integrations/sana/tests/test_smoke.py | 199 +++++++++++++++-------- 6 files changed, 264 insertions(+), 178 deletions(-) diff --git a/integrations/sana/sana_wm/_tools.py b/integrations/sana/sana_wm/_tools.py index 7562bfcba..a401278dd 100644 --- a/integrations/sana/sana_wm/_tools.py +++ b/integrations/sana/sana_wm/_tools.py @@ -17,18 +17,32 @@ from __future__ import annotations -import json import os from pathlib import Path -from typing import Any -import torch +from flashdreams.core.io.hf import maybe_download_hf_repo_on_rank0 HF_URI_SCHEME = "hf://" def resolve_hf_path(path: str | Path) -> str: - """Resolve a local path or ``hf://owner/repo/subpath`` URI to a local path.""" + """Resolve a local path or ``hf://owner/repo/subpath`` URI to a local path. + + Remote repos are preloaded through + :func:`~flashdreams.core.io.hf.maybe_download_hf_repo_on_rank0` so that + multi-rank jobs do not race on the shared cache and so that the free-disk + preflight applies, then resolved from that cache. Local paths and + offline/local-only modes stay download-free. + + Args: + path: Local path or ``hf:///[/]`` URI. + + Returns: + Local filesystem path to the artefact. + + Raises: + ValueError: Malformed ``hf://`` URI. + """ path_str = str(path) if not path_str or Path(path_str).exists(): return path_str @@ -47,40 +61,15 @@ def resolve_hf_path(path: str | Path) -> str: allow_patterns = None if subpath: allow_patterns = [subpath, f"{subpath}/*", f"{subpath}/**"] - local_root = snapshot_download(repo_id=repo_id, allow_patterns=allow_patterns) + maybe_download_hf_repo_on_rank0(repo_id, allow_patterns=allow_patterns) + local_root = snapshot_download( + repo_id=repo_id, + allow_patterns=allow_patterns, + local_files_only=True, + ) return os.path.join(local_root, subpath) if subpath else local_root -def find_model(model_name: str) -> dict[str, Any]: - """Load a SANA checkpoint from a local path or resolved HF artefact.""" - resolved = resolve_hf_path(model_name) - if not os.path.isfile(resolved): - raise FileNotFoundError(f"Could not find SANA checkpoint at {resolved}") - - if resolved.endswith(".safetensors"): - import safetensors.torch - - return {"state_dict": safetensors.torch.load_file(resolved, device="cpu")} - if resolved.endswith(".safetensors.index.json"): - import safetensors.torch - - with open(resolved, encoding="utf-8") as handle: - index = json.load(handle)["weight_map"] - state_dict = {} - for shard in sorted(set(index.values())): - shard_path = os.path.join(os.path.dirname(resolved), shard) - state_dict.update(safetensors.torch.load_file(shard_path, device="cpu")) - return {"state_dict": state_dict} - return torch.load(resolved, map_location=lambda storage, _loc: storage) - - -def hf_download_or_fpath(path: str | Path) -> str: - """Compatibility alias retained for local helper call sites.""" - return resolve_hf_path(path) - - __all__ = [ - "find_model", - "hf_download_or_fpath", "resolve_hf_path", ] diff --git a/integrations/sana/sana_wm/constants.py b/integrations/sana/sana_wm/constants.py index 9192f8654..fa7cfdf98 100644 --- a/integrations/sana/sana_wm/constants.py +++ b/integrations/sana/sana_wm/constants.py @@ -21,10 +21,18 @@ SANA_WM_STREAMING_HF_REPO = "Efficient-Large-Model/SANA-WM_streaming" """Hugging Face repository containing SANA-WM streaming artefacts.""" -SANA_WM_MODEL_PATH = f"hf://{SANA_WM_HF_REPO}/dit/sana_wm_1600m_720p.safetensors" +_SANA_WM_HF_FILE_BASE = f"https://huggingface.co/{SANA_WM_HF_REPO}/resolve/main" +"""Hugging Face file-URL base consumed by ``load_checkpoint``.""" + +_SANA_WM_STREAMING_HF_FILE_BASE = ( + f"https://huggingface.co/{SANA_WM_STREAMING_HF_REPO}/resolve/main" +) +"""Streaming Hugging Face file-URL base consumed by ``load_checkpoint``.""" + +SANA_WM_MODEL_PATH = f"{_SANA_WM_HF_FILE_BASE}/dit/sana_wm_1600m_720p.safetensors" """Default Stage-1 SANA-WM DiT checkpoint.""" -SANA_WM_STREAMING_MODEL_PATH = f"hf://{SANA_WM_STREAMING_HF_REPO}/sana_dit/model.pt" +SANA_WM_STREAMING_MODEL_PATH = f"{_SANA_WM_STREAMING_HF_FILE_BASE}/sana_dit/model.pt" """Default streaming Stage-1 SANA-WM DiT checkpoint.""" SANA_WM_CONFIG_PATH = f"hf://{SANA_WM_HF_REPO}/config.yaml" @@ -39,19 +47,13 @@ SANA_WM_REFINER_GEMMA_ROOT = f"hf://{SANA_WM_HF_REPO}/refiner/text_encoder" """Default Gemma text-encoder root used by the refiner.""" -SANA_WM_STREAMING_CAUSAL_VAE_ROOT = ( - f"hf://{SANA_WM_STREAMING_HF_REPO}/ltx2_causal_vae" -) +SANA_WM_STREAMING_CAUSAL_VAE_ROOT = f"hf://{SANA_WM_STREAMING_HF_REPO}/ltx2_causal_vae" """Default causal LTX-2 VAE root used by streaming SANA-WM.""" -SANA_WM_STREAMING_REFINER_ROOT = ( - f"hf://{SANA_WM_STREAMING_HF_REPO}/refiner_diffusers" -) +SANA_WM_STREAMING_REFINER_ROOT = f"hf://{SANA_WM_STREAMING_HF_REPO}/refiner_diffusers" """Default chunk-causal LTX-2 refiner root used by streaming SANA-WM.""" -SANA_WM_STREAMING_REFINER_GEMMA_ROOT = ( - f"hf://{SANA_WM_STREAMING_HF_REPO}/gemma3_12b" -) +SANA_WM_STREAMING_REFINER_GEMMA_ROOT = f"hf://{SANA_WM_STREAMING_HF_REPO}/gemma3_12b" """Default Gemma-3 text-encoder root used by the streaming refiner.""" DEFAULT_VIDEO_HEIGHT = 704 diff --git a/integrations/sana/sana_wm/refiner.py b/integrations/sana/sana_wm/refiner.py index 3bda7bcc1..391046dfb 100644 --- a/integrations/sana/sana_wm/refiner.py +++ b/integrations/sana/sana_wm/refiner.py @@ -72,7 +72,11 @@ class _RefinerContextCache: def _move_tensor_attr(module: nn.Module, name: str, device: torch.device | str) -> None: tensor = getattr(module, name, None) if isinstance(tensor, nn.Parameter): - setattr(module, name, nn.Parameter(tensor.to(device), requires_grad=tensor.requires_grad)) + setattr( + module, + name, + nn.Parameter(tensor.to(device), requires_grad=tensor.requires_grad), + ) elif isinstance(tensor, Tensor): setattr(module, name, tensor.to(device)) @@ -120,7 +124,14 @@ def _move_ltx2_video_modules_to( transformer: nn.Module, device: torch.device | str, ) -> None: - for name in ("proj_in", "caption_projection", "time_embed", "rope", "norm_out", "proj_out"): + for name in ( + "proj_in", + "caption_projection", + "time_embed", + "rope", + "norm_out", + "proj_out", + ): child = getattr(transformer, name, None) if isinstance(child, nn.Module): child.to(device) @@ -437,11 +448,13 @@ def _load_diffusers_components(self) -> tuple[nn.Module, nn.Module]: self.refiner_root, subfolder="transformer", torch_dtype=self.dtype, + local_files_only=True, ).eval() connectors = LTX2TextConnectors.from_pretrained( self.refiner_root, subfolder="connectors", torch_dtype=self.dtype, + local_files_only=True, ).eval() return transformer, connectors @@ -494,7 +507,9 @@ def _ensure_text_encoder(self) -> None: from transformers import AutoTokenizer, Gemma3ForConditionalGeneration t0 = time.perf_counter() - tokenizer = AutoTokenizer.from_pretrained(self.gemma_root) + tokenizer = AutoTokenizer.from_pretrained( + self.gemma_root, local_files_only=True + ) tokenizer.padding_side = "left" if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token @@ -504,6 +519,7 @@ def _ensure_text_encoder(self) -> None: self.gemma_root, dtype=self.dtype, low_cpu_mem_usage=True, + local_files_only=True, ).eval() self._text_encoder_built = True logger.info( @@ -599,9 +615,8 @@ def _predict_current_x0( dtype=torch.float32, device=self.device, ) - model_timestep = ( - raw_timestep.squeeze(-1) - * float(self.transformer.config.timestep_scale_multiplier) + model_timestep = raw_timestep.squeeze(-1) * float( + self.transformer.config.timestep_scale_multiplier ) velocity = self._forward_video_current_only( hidden_states=current_tokens, @@ -631,9 +646,8 @@ def _predict_current_x0( device=self.device, ) raw_timestep[:, n_context_tokens:, 0] = sigma.float() - model_timestep = ( - raw_timestep.squeeze(-1) - * float(self.transformer.config.timestep_scale_multiplier) + model_timestep = raw_timestep.squeeze(-1) * float( + self.transformer.config.timestep_scale_multiplier ) velocity = self._forward_video_only( @@ -711,8 +725,7 @@ def _forward_video_only( ) scale_shift_values = ( - transformer.scale_shift_table[None, None] - + embedded_timestep[:, :, None] + transformer.scale_shift_table[None, None] + embedded_timestep[:, :, None] ) shift, scale = scale_shift_values[:, :, 0], scale_shift_values[:, :, 1] hidden_states = transformer.norm_out(hidden_states) @@ -772,8 +785,7 @@ def _forward_video_current_only( ) scale_shift_values = ( - transformer.scale_shift_table[None, None] - + embedded_timestep[:, :, None] + transformer.scale_shift_table[None, None] + embedded_timestep[:, :, None] ) shift, scale = scale_shift_values[:, :, 0], scale_shift_values[:, :, 1] hidden_states = transformer.norm_out(hidden_states) @@ -800,8 +812,8 @@ def _forward_video_block( num_ada_params, -1, ) - shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( - ada_values.unbind(dim=2) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ada_values.unbind( + dim=2 ) norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa @@ -982,9 +994,7 @@ def _refiner_self_attention_qkv( ) gate_logits = ( - attn.to_gate_logits(hidden_states) - if attn.to_gate_logits is not None - else None + attn.to_gate_logits(hidden_states) if attn.to_gate_logits is not None else None ) query = attn.to_q(hidden_states) key = attn.to_k(hidden_states) @@ -1091,7 +1101,9 @@ def _pack_text_embeds( return normalized.to(dtype=original_dtype) -def _pack_latents(latents: Tensor, patch_size: int = 1, patch_size_t: int = 1) -> Tensor: +def _pack_latents( + latents: Tensor, patch_size: int = 1, patch_size_t: int = 1 +) -> Tensor: batch_size, _, num_frames, height, width = latents.shape post_patch_num_frames = num_frames // patch_size_t post_patch_height = height // patch_size diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index 63af8236b..f8bbd4d94 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -156,7 +156,7 @@ class SanaWMRunnerConfig(RunnerConfig): """SANA-WM inference YAML path or ``hf://`` URI.""" model_path: str = SANA_WM_MODEL_PATH - """Stage-1 checkpoint path or ``hf://`` URI.""" + """Stage-1 checkpoint path, ``s3://`` URI, or Hugging Face file URL.""" stage1_precision: Precision = "bf16" """Stage-1 DiT compute precision. ``"bf16"`` is the default; ``"fp8"`` @@ -226,7 +226,7 @@ class SanaWMStreamingRunnerConfig(SanaWMRunnerConfig): """SANA-WM streaming config path or built-in identifier.""" model_path: str = SANA_WM_STREAMING_MODEL_PATH - """Streaming Stage-1 checkpoint path or ``hf://`` URI.""" + """Streaming Stage-1 checkpoint path, ``s3://`` URI, or Hugging Face file URL.""" causal_vae_path: str = SANA_WM_STREAMING_CAUSAL_VAE_ROOT """Streaming causal LTX-2 VAE root path or ``hf://`` URI.""" @@ -399,7 +399,9 @@ def run(self) -> None: ) if decoded.stage1_video_hwc is not None: _write_video( - runner_artifact_path(cfg.output_dir, f"{cfg.runner_name}_stage1", "mp4"), + runner_artifact_path( + cfg.output_dir, f"{cfg.runner_name}_stage1", "mp4" + ), decoded.stage1_video_hwc, cfg.fps, ) @@ -551,7 +553,10 @@ def run(self) -> None: ) if decoded.video_hwc.size: decoded_chunks.append(decoded.video_hwc) - if decoded.stage1_video_hwc is not None and decoded.stage1_video_hwc.size: + if ( + decoded.stage1_video_hwc is not None + and decoded.stage1_video_hwc.size + ): stage1_chunks.append(decoded.stage1_video_hwc) if not self.is_rank_zero: @@ -567,7 +572,9 @@ def run(self) -> None: ) if stage1_chunks: _write_video( - runner_artifact_path(cfg.output_dir, f"{cfg.runner_name}_stage1", "mp4"), + runner_artifact_path( + cfg.output_dir, f"{cfg.runner_name}_stage1", "mp4" + ), np.concatenate(stage1_chunks, axis=0), cfg.fps, ) diff --git a/integrations/sana/sana_wm/transformer.py b/integrations/sana/sana_wm/transformer.py index 633c34d3f..2129cb42a 100644 --- a/integrations/sana/sana_wm/transformer.py +++ b/integrations/sana/sana_wm/transformer.py @@ -29,12 +29,14 @@ from loguru import logger from torch import Tensor +from flashdreams.core.checkpoint.load import load_checkpoint +from flashdreams.core.io.hf import maybe_download_hf_repo_on_rank0 from flashdreams.infra.diffusion.transformer import ( Transformer, TransformerAutoregressiveCache, TransformerConfig, ) -from sana_wm._tools import find_model, resolve_hf_path +from sana_wm._tools import resolve_hf_path from sana_wm.constants import ( DEFAULT_VIDEO_HEIGHT, DEFAULT_VIDEO_WIDTH, @@ -80,6 +82,29 @@ ) +def _stage1_state_dict(state: Any) -> dict[str, Tensor]: + """Normalize a Stage-1 checkpoint payload into a plain state dict. + + The bidirectional release ships flat safetensors weights, while the + streaming release ships a ``.pt`` that nests the tensors under + ``generator`` and/or ``state_dict`` and prefixes keys with ``model.``. + + Args: + state: Payload returned by ``load_checkpoint``. + + Returns: + Tensor state dict accepted by ``load_state_dict``. + """ + if "generator" in state: + state = state["generator"] + if "state_dict" in state: + return cast("dict[str, Tensor]", state["state_dict"]) + return { + (key[len("model.") :] if key.startswith("model.") else key): value + for key, value in state.items() + } + + @dataclass(kw_only=True) class SanaWMStage1Conditioning: """Per-rollout inputs needed by the SANA-WM Stage-1 sampler.""" @@ -147,7 +172,7 @@ class SanaWMTransformerConfig(TransformerConfig): """SANA-WM inference YAML path or ``hf://`` URI.""" checkpoint_path: str = SANA_WM_MODEL_PATH - """SANA-WM Stage-1 checkpoint path or ``hf://`` URI.""" + """SANA-WM Stage-1 checkpoint path, ``s3://`` URI, or Hugging Face file URL.""" stage1_precision: Precision = "bf16" """Stage-1 precision requested by the runner.""" @@ -179,7 +204,7 @@ class SanaWMStreamingTransformerConfig(SanaWMTransformerConfig): """SANA-WM streaming inference config path or built-in identifier.""" checkpoint_path: str = SANA_WM_STREAMING_MODEL_PATH - """SANA-WM streaming Stage-1 checkpoint path or ``hf://`` URI.""" + """Streaming Stage-1 checkpoint path, ``s3://`` URI, or Hugging Face file URL.""" num_frame_per_block: int = SANA_WM_STREAMING_LATENT_CHUNK_SIZE """Latent frames generated per steady-state AR block.""" @@ -487,18 +512,9 @@ def _ensure_model(self) -> None: cfg.model.model, sum(p.numel() for p in model.parameters()), ) - self._model_path = resolve_hf_path(self.config.checkpoint_path) - state = find_model(self._model_path) - if "generator" in state: - state = state["generator"] - if "state_dict" not in state: - state = { - "state_dict": { - (k[len("model.") :] if k.startswith("model.") else k): v - for k, v in state.items() - } - } - missing, unexpected = model.load_state_dict(state["state_dict"], strict=True) + self._model_path = self.config.checkpoint_path + state_dict = _stage1_state_dict(load_checkpoint(self._model_path)) + missing, unexpected = model.load_state_dict(state_dict, strict=True) if missing: logger.warning("[Sana] Missing keys: {}", missing) if unexpected: @@ -701,18 +717,9 @@ def _ensure_model(self) -> None: cfg.model.model, sum(p.numel() for p in model.parameters()), ) - self._model_path = resolve_hf_path(self.config.checkpoint_path) - state = find_model(self._model_path) - if "generator" in state: - state = state["generator"] - if "state_dict" not in state: - state = { - "state_dict": { - (k[len("model.") :] if k.startswith("model.") else k): v - for k, v in state.items() - } - } - missing, unexpected = model.load_state_dict(state["state_dict"], strict=True) + self._model_path = self.config.checkpoint_path + state_dict = _stage1_state_dict(load_checkpoint(self._model_path)) + missing, unexpected = model.load_state_dict(state_dict, strict=True) if missing: logger.warning("[Sana] Missing keys: {}", missing) if unexpected: @@ -831,8 +838,7 @@ def _require_streaming_cache( ) -> SanaWMStreamingTransformerCache: if not isinstance(cache, SanaWMStreamingTransformerCache): raise TypeError( - "SANA-WM streaming transformer requires " - "SanaWMStreamingTransformerCache." + "SANA-WM streaming transformer requires SanaWMStreamingTransformerCache." ) return cache @@ -1109,9 +1115,7 @@ def _streaming_condition_frame_info( """Return conditioned-frame metadata from a streaming conditioning payload.""" data_info = conditioning.model_kwargs.get("data_info", {}) condition_frame_info = ( - data_info.get("condition_frame_info", {}) - if isinstance(data_info, dict) - else {} + data_info.get("condition_frame_info", {}) if isinstance(data_info, dict) else {} ) return condition_frame_info if isinstance(condition_frame_info, dict) else {} @@ -1184,9 +1188,7 @@ def _condition_frame_mask( ) -> Tensor: data_info = conditioning.model_kwargs.get("data_info", {}) condition_frame_info = ( - data_info.get("condition_frame_info", {}) - if isinstance(data_info, dict) - else {} + data_info.get("condition_frame_info", {}) if isinstance(data_info, dict) else {} ) mask = torch.zeros(batch, 1, frames, dtype=torch.float32, device=device) if not isinstance(condition_frame_info, dict): @@ -1291,16 +1293,20 @@ def _get_vae(*args: Any, **kwargs: Any) -> nn.Module: raise ValueError(f"Unsupported SANA-WM VAE type: {name!r}") from diffusers import AutoencoderKLLTX2Video + maybe_download_hf_repo_on_rank0(str(model_path)) + try: vae = AutoencoderKLLTX2Video.from_pretrained( model_path, subfolder="vae", torch_dtype=dtype, + local_files_only=True, ) except OSError: vae = AutoencoderKLLTX2Video.from_pretrained( model_path, torch_dtype=dtype, + local_files_only=True, ) return vae.to(device).eval() @@ -1318,18 +1324,25 @@ def _get_tokenizer_and_text_encoder(*args: Any, **kwargs: Any) -> tuple[Any, nn. T5Tokenizer, ) + maybe_download_hf_repo_on_rank0(model_id) + if "T5" in str(name): - tokenizer = T5Tokenizer.from_pretrained(model_id) + tokenizer = T5Tokenizer.from_pretrained(model_id, local_files_only=True) text_encoder = T5EncoderModel.from_pretrained( model_id, torch_dtype=torch.float16, + local_files_only=True, ).to(device) return tokenizer, text_encoder.eval() - tokenizer = AutoTokenizer.from_pretrained(model_id) + tokenizer = AutoTokenizer.from_pretrained(model_id, local_files_only=True) tokenizer.padding_side = "right" text_encoder = ( - AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16) + AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.bfloat16, + local_files_only=True, + ) .get_decoder() .to(device) .eval() @@ -1371,7 +1384,9 @@ def get(self, name: str, default: Any = None) -> Any: } -def _vae_encode_ltx2(name: str, vae: nn.Module, images: Tensor, *, device: torch.device) -> Tensor: +def _vae_encode_ltx2( + name: str, vae: nn.Module, images: Tensor, *, device: torch.device +) -> Tensor: if "LTX2VAE_diffusers" not in name: raise ValueError(f"Unsupported SANA-WM VAE encode type: {name!r}") dtype = images.dtype diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index acb93f6e4..2489a41fa 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -22,17 +22,28 @@ import numpy as np import pytest -import torch - +import sana_wm._tools as tools_module import sana_wm.conditioning as conditioning_module import sana_wm.decoder as decoder_module import sana_wm.refiner as refiner_module +import torch try: import tomllib except ModuleNotFoundError: # pragma: no cover - Python < 3.11 fallback import tomli as tomllib +from sana_wm.conditioning import ( + SanaWMCameraConditioningEncoderConfig, + SanaWMCameraRequest, + SanaWMConditioningEncoderConfig, + SanaWMFirstFrameEncoderConfig, + SanaWMStreamingConditioningEncoderConfig, + SanaWMStreamingI2VConditioningRequest, + SanaWMTextPromptEncoderConfig, + SanaWMTextPromptRequest, + streaming_chunk_boundaries, +) from sana_wm.config import ( PIPELINE_SANA_WM_BIDIRECTIONAL, PIPELINE_SANA_WM_STREAMING, @@ -53,6 +64,26 @@ SANA_WM_STREAMING_REFINER_KV_MAX_FRAMES, SANA_WM_STREAMING_REFINER_ROOT, ) +from sana_wm.decoder import ( + SanaWMDecodedVideo, + SanaWMLTX2LatentRefinerConfig, + SanaWMLTX2VAEDecoderConfig, + SanaWMStreamingLTX2LatentRefinerConfig, + SanaWMStreamingLTX2VAEDecoderConfig, + SanaWMStreamingVideoDecoder, + SanaWMStreamingVideoDecoderConfig, + SanaWMVideoDecoderConfig, +) +from sana_wm.diffusion import SanaWMDiffusionModelConfig +from sana_wm.quant import ( + TorchScaledMMFP4Linear, + TorchScaledMMFP8Linear, + apply_rht16, + nvfp4_global_scale, + replace_linear_with_torch_fp4, + replace_linear_with_torch_fp8, +) +from sana_wm.refiner import SanaWMLTX2Refiner, _pack_latents, _unpack_latents from sana_wm.runner import ( SanaWMRunner, SanaWMRunnerConfig, @@ -63,61 +94,30 @@ _streaming_pipeline_config, _validate_precision_request, ) -from sana_wm.conditioning import ( - SanaWMCameraConditioningEncoderConfig, - SanaWMCameraRequest, - SanaWMConditioningEncoderConfig, - SanaWMFirstFrameEncoderConfig, - SanaWMStreamingConditioningEncoderConfig, - SanaWMStreamingI2VConditioningRequest, - SanaWMTextPromptEncoderConfig, - SanaWMTextPromptRequest, - streaming_chunk_boundaries, -) -from sana_wm.decoder import ( - SanaWMDecodedVideo, - SanaWMStreamingLTX2LatentRefinerConfig, - SanaWMStreamingLTX2VAEDecoderConfig, - SanaWMStreamingVideoDecoder, - SanaWMStreamingVideoDecoderConfig, - SanaWMLTX2LatentRefinerConfig, - SanaWMLTX2VAEDecoderConfig, - SanaWMVideoDecoderConfig, -) -from sana_wm.diffusion import SanaWMDiffusionModelConfig from sana_wm.scheduler import ( SanaWMLTXEulerScheduler, SanaWMLTXEulerSchedulerConfig, ) +from sana_wm.stage1_model import ( + SANA_WM_STAGE1_SPEC, + SANA_WM_STREAMING_STAGE1_SPEC, + GLUMBConvTemp, + SanaWMStage1Model, + SanaWMStage1Spec, + Stage1SelfAttention, + linearize_stage1_ffn_for_quant, +) from sana_wm.transformer import ( + SanaWMStage1Conditioning, SanaWMStreamingStage1Conditioning, SanaWMStreamingTransformerCache, SanaWMStreamingTransformerConfig, SanaWMTransformerCache, SanaWMTransformerConfig, - SanaWMStage1Conditioning, _avoid_degenerate_tile_tail, _load_inference_config, _stage1_quant_include_patterns, ) -from sana_wm.refiner import SanaWMLTX2Refiner, _pack_latents, _unpack_latents -from sana_wm.quant import ( - apply_rht16, - nvfp4_global_scale, - TorchScaledMMFP4Linear, - TorchScaledMMFP8Linear, - replace_linear_with_torch_fp4, - replace_linear_with_torch_fp8, -) -from sana_wm.stage1_model import ( - GLUMBConvTemp, - SANA_WM_STAGE1_SPEC, - SANA_WM_STREAMING_STAGE1_SPEC, - SanaWMStage1Model, - SanaWMStage1Spec, - Stage1SelfAttention, - linearize_stage1_ffn_for_quant, -) from flashdreams.infra.config import derive_config from flashdreams.infra.decoder import StreamingVideoDecoder @@ -432,7 +432,9 @@ def test_sana_ltx_scheduler_uses_explicit_streaming_timesteps() -> None: ) -def test_sana_transformer_keeps_conditioned_frame_fixed_with_generic_scheduler() -> None: +def test_sana_transformer_keeps_conditioned_frame_fixed_with_generic_scheduler() -> ( + None +): """Keep SANA conditioning out of the scheduler and inside the transformer.""" scheduler = SanaWMLTXEulerSchedulerConfig(num_inference_steps=1).setup() transformer = SanaWMTransformerConfig().setup() @@ -466,7 +468,9 @@ def forward( transformer.model = DummyModel() transformer._model_built = True - def predict_flow(noisy_latent: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor: + def predict_flow( + noisy_latent: torch.Tensor, timestep: torch.Tensor + ) -> torch.Tensor: return transformer.predict_flow( noisy_latent=noisy_latent, timestep=timestep, @@ -549,9 +553,7 @@ def forward( { "noisy_latent": noisy_latent.detach().clone(), "timestep": timestep.detach().clone(), - "camera_conditions": _kwargs["camera_conditions"] - .detach() - .clone(), + "camera_conditions": _kwargs["camera_conditions"].detach().clone(), "chunk_plucker": _kwargs["chunk_plucker"].detach().clone(), "data_info": dict(_kwargs["data_info"]), "chunk_index": list(_kwargs["chunk_index"]), @@ -572,10 +574,12 @@ def conditioning_for_chunk( uncondition=None, model_kwargs={ "data_info": {"condition_frame_info": {0: 0.0}}, - "camera_conditions": torch.arange(total_frames, dtype=torch.float32) - .reshape(1, total_frames, 1), - "chunk_plucker": torch.arange(total_frames, dtype=torch.float32) - .reshape(1, 1, total_frames, 1, 1), + "camera_conditions": torch.arange( + total_frames, dtype=torch.float32 + ).reshape(1, total_frames, 1), + "chunk_plucker": torch.arange( + total_frames, dtype=torch.float32 + ).reshape(1, 1, total_frames, 1, 1), "chunk_index": [0, 4, 7, 10], }, first_latent=torch.full((1, 1, 1, 1, 1), 7.0), @@ -646,9 +650,7 @@ def conditioning_for_chunk( ) torch.testing.assert_close( dummy_model.calls[3]["timestep"], - torch.tensor( - [[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 125.0, 125.0, 125.0]]] - ), + torch.tensor([[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 125.0, 125.0, 125.0]]]), ) camera3 = dummy_model.calls[3]["camera_conditions"] assert isinstance(camera3, torch.Tensor) @@ -707,12 +709,16 @@ def forward( "data_info": _kwargs["data_info"], } ) - branch_values = prompt_embeds.flatten(1).mean(dim=1).reshape( - -1, - 1, - 1, - 1, - 1, + branch_values = ( + prompt_embeds.flatten(1) + .mean(dim=1) + .reshape( + -1, + 1, + 1, + 1, + 1, + ) ) return torch.ones_like(noisy_latent) * branch_values @@ -1073,7 +1079,9 @@ def test_streaming_chunk_boundaries_use_sink_plus_fixed_blocks() -> None: streaming_chunk_boundaries(total_frames=9, chunk_size=3) -def test_video_decoder_returns_structured_video(monkeypatch: pytest.MonkeyPatch) -> None: +def test_video_decoder_returns_structured_video( + monkeypatch: pytest.MonkeyPatch, +) -> None: """Decode Stage-1 latents through the explicit decoder component.""" decoder = SanaWMVideoDecoderConfig(refiner=None).setup() monkeypatch.setattr( @@ -1234,9 +1242,7 @@ def decode_with_context(latents: torch.Tensor) -> np.ndarray: ) torch.testing.assert_close( calls[3], - torch.tensor([0.0, 1.0, 3.0, 3.0, 3.0, 5.0, 5.0, 5.0]).reshape( - 1, 1, 8, 1, 1 - ), + torch.tensor([0.0, 1.0, 3.0, 3.0, 3.0, 5.0, 5.0, 5.0]).reshape(1, 1, 8, 1, 1), ) assert cache.refiner_prompt_embeds is not None assert cache.refiner_prompt_attention_mask is not None @@ -1691,12 +1697,64 @@ def test_vae_tiling_avoids_degenerate_latent_tails() -> None: ) +def test_resolve_hf_path_preloads_on_rank0_then_reads_cache( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Route hf:// resolution through the shared rank-0 download helper.""" + preloads: list[tuple[str, object]] = [] + snapshots: list[dict[str, object]] = [] + + def _preload(repo_id: str, **kwargs: object) -> None: + preloads.append((repo_id, kwargs.get("allow_patterns"))) + + def _snapshot_download(**kwargs: object) -> str: + snapshots.append(kwargs) + return str(tmp_path / "snapshot") + + monkeypatch.setattr(tools_module, "maybe_download_hf_repo_on_rank0", _preload) + monkeypatch.setattr( + "huggingface_hub.snapshot_download", + _snapshot_download, + ) + + resolved = tools_module.resolve_hf_path(SANA_WM_STREAMING_REFINER_ROOT) + + assert resolved == str(tmp_path / "snapshot" / "refiner_diffusers") + assert preloads == [ + ( + SANA_WM_STREAMING_HF_REPO, + [ + "refiner_diffusers", + "refiner_diffusers/*", + "refiner_diffusers/**", + ], + ) + ] + # The download itself is the helper's job; resolution only reads the cache. + assert snapshots[0]["local_files_only"] is True + + +def test_resolve_hf_path_leaves_local_paths_alone( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Keep local roots download-free.""" + + def _fail_preload(*_args: object, **_kwargs: object) -> None: + raise AssertionError("local paths must not trigger a download") + + monkeypatch.setattr(tools_module, "maybe_download_hf_repo_on_rank0", _fail_preload) + + assert tools_module.resolve_hf_path(str(tmp_path)) == str(tmp_path) + + def test_hf_defaults_point_at_bidirectional_release() -> None: """Pin every default SANA-WM artefact to the bidirectional HF repo.""" assert SANA_WM_HF_REPO == "Efficient-Large-Model/SANA-WM_bidirectional" assert SANA_WM_MODEL_PATH == ( - "hf://Efficient-Large-Model/SANA-WM_bidirectional/" - "dit/sana_wm_1600m_720p.safetensors" + "https://huggingface.co/Efficient-Large-Model/SANA-WM_bidirectional/" + "resolve/main/dit/sana_wm_1600m_720p.safetensors" ) assert SANA_WM_CONFIG_PATH == ( "hf://Efficient-Large-Model/SANA-WM_bidirectional/config.yaml" @@ -1707,7 +1765,8 @@ def test_hf_defaults_point_at_streaming_release() -> None: """Pin every default streaming artefact to the streaming HF repo.""" assert SANA_WM_STREAMING_HF_REPO == "Efficient-Large-Model/SANA-WM_streaming" assert SANA_WM_STREAMING_MODEL_PATH == ( - "hf://Efficient-Large-Model/SANA-WM_streaming/sana_dit/model.pt" + "https://huggingface.co/Efficient-Large-Model/SANA-WM_streaming/" + "resolve/main/sana_dit/model.pt" ) assert SANA_WM_STREAMING_CONFIG_PATH == ( "flashdreams://sana-wm-streaming-1600m-720p" @@ -2141,7 +2200,9 @@ def test_torch_fp8_backend_rejects_fp4(monkeypatch: pytest.MonkeyPatch) -> None: ) -def test_torch_fp4_backend_validates_primitives(monkeypatch: pytest.MonkeyPatch) -> None: +def test_torch_fp4_backend_validates_primitives( + monkeypatch: pytest.MonkeyPatch, +) -> None: """Allow FP4 validation with the required PyTorch primitives.""" monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_capability", lambda _device: (12, 0)) From 83e5ec414d0e566da02703c5813dfa4bb78ca638 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 16:18:37 -0700 Subject: [PATCH 62/64] Fix lint issues, try to not run TE in CI --- integrations/sana/sana_wm/camera.py | 28 ++-- integrations/sana/sana_wm/conditioning.py | 46 +++--- integrations/sana/sana_wm/config.py | 6 +- integrations/sana/sana_wm/decoder.py | 19 ++- integrations/sana/sana_wm/quant.py | 21 ++- integrations/sana/sana_wm/stage1_model.py | 136 +++++++++--------- integrations/sana/tests/compat/mmcv/runner.py | 4 +- .../parity_check/streaming_continuity.py | 19 ++- integrations/sana/tests/performance/README.md | 30 ++-- integrations/sana/tests/performance/bench.sh | 40 +++++- .../sana/tests/performance/bench_summary.py | 79 +++++----- .../tests/performance/bench_sweep_summary.py | 12 +- integrations/sana/tests/pyproject.toml | 7 +- .../tests/run_flashdreams_bidirectional.py | 4 +- .../sana/tests/run_flashdreams_common.py | 13 +- .../sana/tests/run_flashdreams_streaming.py | 19 ++- integrations/sana/tests/test_camera.py | 19 ++- .../test_performance_benchmark_summary.py | 98 ++++++++++--- integrations/sana/tests/test_quant_cuda.py | 1 - integrations/sana/tests/test_stage1_cuda.py | 3 +- 20 files changed, 393 insertions(+), 211 deletions(-) diff --git a/integrations/sana/sana_wm/camera.py b/integrations/sana/sana_wm/camera.py index 063345910..a9c8b6ebd 100644 --- a/integrations/sana/sana_wm/camera.py +++ b/integrations/sana/sana_wm/camera.py @@ -174,12 +174,16 @@ def controls_to_target_velocity( ) -def parse_action_string(action: str) -> list[list[str]]: +def parse_action_string( + action: str, *, max_frames: int | None = None +) -> list[list[str]]: """Expand a SANA-WM action string into per-frame held keys. Args: action: Comma-separated ``-`` segments. ``none-N`` holds the pose for ``N`` frames. + max_frames: Optional maximum number of per-frame entries to + materialize. All segments are still validated. Returns: Per-frame lists of DSL keys. @@ -187,6 +191,8 @@ def parse_action_string(action: str) -> list[list[str]]: Raises: ValueError: The action string is empty or contains an invalid segment. """ + if max_frames is not None and max_frames < 0: + raise ValueError(f"max_frames must be non-negative, got {max_frames}.") cleaned = "".join(action.replace("，", ",").split()) if not cleaned: raise ValueError("action string is empty") @@ -197,7 +203,10 @@ def parse_action_string(action: str) -> list[list[str]]: f"Invalid action segment {segment!r}: expected '-'." ) keys_part, duration_part = segment.rsplit("-", 1) - if not duration_part.isdigit() or int(duration_part) <= 0: + if not duration_part.isdigit(): + raise ValueError(f"Action segment {segment!r} has a non-positive duration.") + duration = int(duration_part) + if duration <= 0: raise ValueError(f"Action segment {segment!r} has a non-positive duration.") keys_lower = keys_part.lower() if keys_lower == "none": @@ -210,7 +219,10 @@ def parse_action_string(action: str) -> list[list[str]]: f"allowed: {''.join(sorted(ALLOWED_ACTION_KEYS))}." ) keys = sorted(set(keys_lower)) - per_frame.extend([keys] * int(duration_part)) + if max_frames is None: + per_frame.extend([keys] * duration) + elif len(per_frame) < max_frames: + per_frame.extend([keys] * min(duration, max_frames - len(per_frame))) return per_frame @@ -238,11 +250,13 @@ def action_string_to_c2w( Returns: ``[F + 1, 4, 4]`` camera-to-world matrices in OpenCV convention. """ - per_frame = parse_action_string(action) - if num_frames is not None: + if num_frames is None: + per_frame = parse_action_string(action) + else: if num_frames < 1: raise ValueError(f"num_frames must be positive, got {num_frames}.") target_steps = num_frames - 1 + per_frame = parse_action_string(action, max_frames=target_steps) if target_steps == 0: per_frame = [] elif len(per_frame) < target_steps: @@ -408,9 +422,7 @@ def default_intrinsics_vec4( if not 0.0 < hfov_deg < 180.0: raise ValueError(f"hfov_deg must be in (0, 180), got {hfov_deg}.") focal = 0.5 * src_w / math.tan(math.radians(hfov_deg) / 2.0) - vector = np.array( - [focal, focal, src_w / 2.0, src_h / 2.0], dtype=np.float32 - ) + vector = np.array([focal, focal, src_w / 2.0, src_h / 2.0], dtype=np.float32) return np.broadcast_to(vector, (num_frames, 4)).copy() diff --git a/integrations/sana/sana_wm/conditioning.py b/integrations/sana/sana_wm/conditioning.py index 9f04dd9a3..66bdb9508 100644 --- a/integrations/sana/sana_wm/conditioning.py +++ b/integrations/sana/sana_wm/conditioning.py @@ -42,9 +42,9 @@ SANA_WM_STREAMING_LATENT_CHUNK_SIZE, ) from sana_wm.transformer import ( - SanaWMStreamingStage1Conditioning, QuantBackend, SanaWMStage1Conditioning, + SanaWMStreamingStage1Conditioning, _chunk_index_from_config, _get_tokenizer_and_text_encoder, _get_vae, @@ -484,13 +484,21 @@ def forward( intrinsics_vec4=input.intrinsics_vec4, ) ) - raymap = camera["raymap"].unsqueeze(0).to( - first_latent.device, - dtype=weight_dtype, + raymap = ( + camera["raymap"] + .unsqueeze(0) + .to( + first_latent.device, + dtype=weight_dtype, + ) ) - chunk_plucker = camera["chunk_plucker"].unsqueeze(0).to( - first_latent.device, - dtype=weight_dtype, + chunk_plucker = ( + camera["chunk_plucker"] + .unsqueeze(0) + .to( + first_latent.device, + dtype=weight_dtype, + ) ) model_kwargs_extra: dict[str, object] = {} @@ -636,9 +644,7 @@ def forward( return SanaWMStreamingStage1Conditioning( condition=rollout.text.condition, - uncondition=( - rollout.text.negative if rollout.cfg_scale > 1.0 else None - ), + uncondition=(rollout.text.negative if rollout.cfg_scale > 1.0 else None), model_kwargs=model_kwargs, first_latent=rollout.first_latent, latent_shape=latent_shape, @@ -672,13 +678,21 @@ def _build_rollout( intrinsics_vec4=input.intrinsics_vec4, ) ) - raymap = camera["raymap"].unsqueeze(0).to( - first_latent.device, - dtype=weight_dtype, + raymap = ( + camera["raymap"] + .unsqueeze(0) + .to( + first_latent.device, + dtype=weight_dtype, + ) ) - chunk_plucker = camera["chunk_plucker"].unsqueeze(0).to( - first_latent.device, - dtype=weight_dtype, + chunk_plucker = ( + camera["chunk_plucker"] + .unsqueeze(0) + .to( + first_latent.device, + dtype=weight_dtype, + ) ) vae_stride = cfg.vae.vae_stride diff --git a/integrations/sana/sana_wm/config.py b/integrations/sana/sana_wm/config.py index 9c2cd3951..6b810394e 100644 --- a/integrations/sana/sana_wm/config.py +++ b/integrations/sana/sana_wm/config.py @@ -28,8 +28,10 @@ from sana_wm.diffusion import SanaWMDiffusionModelConfig from sana_wm.runner import SanaWMRunnerConfig, SanaWMStreamingRunnerConfig from sana_wm.scheduler import SanaWMLTXEulerSchedulerConfig -from sana_wm.transformer import SanaWMStreamingTransformerConfig, SanaWMTransformerConfig - +from sana_wm.transformer import ( + SanaWMStreamingTransformerConfig, + SanaWMTransformerConfig, +) PIPELINE_SANA_WM_BIDIRECTIONAL = StreamInferencePipelineConfig( name="sana-wm-bidirectional", diff --git a/integrations/sana/sana_wm/decoder.py b/integrations/sana/sana_wm/decoder.py index 7c2c6458d..431727903 100644 --- a/integrations/sana/sana_wm/decoder.py +++ b/integrations/sana/sana_wm/decoder.py @@ -289,7 +289,9 @@ def _configure_vae_tiling( stride_width = int( tile_sample_stride_width or self.config.vae_tile_sample_stride_width ) - min_height = int(tile_sample_min_height or self.config.vae_tile_sample_min_height) + min_height = int( + tile_sample_min_height or self.config.vae_tile_sample_min_height + ) stride_height = int( tile_sample_stride_height or self.config.vae_tile_sample_stride_height ) @@ -307,8 +309,7 @@ def _configure_vae_tiling( compression_ratio=spatial_ratio, ) min_frames = int( - tile_sample_min_num_frames - or self.config.vae_tile_sample_min_num_frames + tile_sample_min_num_frames or self.config.vae_tile_sample_min_num_frames ) stride_frames = int( tile_sample_stride_num_frames @@ -710,7 +711,9 @@ class SanaWMStreamingVideoDecoderConfig(DecoderConfig): ) -class SanaWMStreamingVideoDecoder(StreamingVideoDecoder[SanaWMStreamingVideoDecoderCache]): +class SanaWMStreamingVideoDecoder( + StreamingVideoDecoder[SanaWMStreamingVideoDecoderCache] +): """Decode SANA-WM streaming latent chunks into newly available frames.""" config: SanaWMStreamingVideoDecoderConfig @@ -829,7 +832,9 @@ def get_output_temporal_size( ratio = self.temporal_compression_ratio if autoregressive_index == 0: if input_temporal_size <= 1: - raise ValueError("Streaming AR step 0 must include sink + active frames.") + raise ValueError( + "Streaming AR step 0 must include sink + active frames." + ) return (input_temporal_size - 1) * ratio return input_temporal_size * ratio @@ -857,7 +862,9 @@ def _split_streaming_stage1_chunk( ) -> Tensor: """Return active latent frames and initialize the sink on the first chunk.""" if autoregressive_index < 0: - raise ValueError(f"autoregressive_index must be >= 0, got {autoregressive_index}.") + raise ValueError( + f"autoregressive_index must be >= 0, got {autoregressive_index}." + ) if chunk.shape[2] <= 0: raise ValueError("SANA-WM streaming latent chunk must contain frames.") if autoregressive_index == 0: diff --git a/integrations/sana/sana_wm/quant.py b/integrations/sana/sana_wm/quant.py index d285e73f1..a7e177ecd 100644 --- a/integrations/sana/sana_wm/quant.py +++ b/integrations/sana/sana_wm/quant.py @@ -164,7 +164,10 @@ def _quantize_nvfp4_kernel( scale_m = tl.arange(0, 32)[:, None] scale_k = tl.arange(0, 16)[None, :] tl.store( - scale_ptr + (pid_m * tl.num_programs(0) + pid_k) * (32 * 16) + scale_m * 16 + scale_k, + scale_ptr + + (pid_m * tl.num_programs(0) + pid_k) * (32 * 16) + + scale_m * 16 + + scale_k, packed_scales, ) @@ -192,7 +195,9 @@ def quantize_nvfp4_swizzled( """Quantize a 2D CUDA tensor to NVFP4 qdata plus swizzled E4M3 scales.""" _require_fp4_dtype() if input.dim() != 2: - raise ValueError(f"NVFP4 quantization requires a 2D tensor, got {input.dim()}D.") + raise ValueError( + f"NVFP4 quantization requires a 2D tensor, got {input.dim()}D." + ) if not input.is_cuda: raise RuntimeError("NVFP4 quantization requires CUDA input.") rows, cols = input.shape @@ -296,7 +301,9 @@ def __init__( self.out_dtype = out_dtype self.register_buffer("weight", weight.detach().contiguous()) self.register_buffer("weight_fp8", weight_fp8.contiguous()) - self.register_buffer("weight_scale", weight_scale.to(torch.float32).contiguous()) + self.register_buffer( + "weight_scale", weight_scale.to(torch.float32).contiguous() + ) if bias is None: self.register_buffer("bias", None) else: @@ -402,7 +409,9 @@ def __init__( f"{expected_weight_shape}, got {tuple(weight.shape)}." ) if weight_qdata.dim() != 2: - raise ValueError(f"weight_qdata must be 2D, got {tuple(weight_qdata.shape)}.") + raise ValueError( + f"weight_qdata must be 2D, got {tuple(weight_qdata.shape)}." + ) if weight_qdata.shape[1] * 2 % NVFP4_BLOCK_SIZE != 0: raise ValueError( "weight_qdata must represent a K dimension divisible by " @@ -661,7 +670,9 @@ def _require_scaled_mm() -> None: def _require_fp8_dtype() -> None: if not hasattr(torch, "float8_e4m3fn"): - raise RuntimeError("torch.float8_e4m3fn is required for the SANA-WM FP8 backend.") + raise RuntimeError( + "torch.float8_e4m3fn is required for the SANA-WM FP8 backend." + ) def _require_fp4_dtype() -> None: diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index b729c7172..aeb3bc9a0 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -21,10 +21,10 @@ from __future__ import annotations -from collections.abc import Callable -from dataclasses import dataclass import math import os +from collections.abc import Callable +from dataclasses import dataclass import torch import torch.nn as nn @@ -167,7 +167,9 @@ class TextEmbedder(nn.Module): def __init__(self, spec: SanaWMStage1Spec) -> None: super().__init__() - self.y_embedding = nn.Parameter(torch.empty(spec.max_text_length, spec.text_dim)) + self.y_embedding = nn.Parameter( + torch.empty(spec.max_text_length, spec.text_dim) + ) self.y_proj = TextProjection(spec) def forward(self, y: Tensor | None) -> Tensor: @@ -293,10 +295,16 @@ def __init__(self, conv_layer: nn.Module) -> None: conv = getattr(conv_layer, "conv", None) if not isinstance(conv, nn.Conv2d): raise ValueError("expected Conv2dContainer.conv to be nn.Conv2d.") - if conv.kernel_size != (1, 1) or conv.stride != (1, 1) or conv.padding != (0, 0): + if ( + conv.kernel_size != (1, 1) + or conv.stride != (1, 1) + or conv.padding != (0, 0) + ): raise ValueError("only exact 1x1 pointwise Conv2d can be linearized.") if conv.dilation != (1, 1) or conv.groups != 1: - raise ValueError("grouped or dilated pointwise Conv2d cannot be linearized.") + raise ValueError( + "grouped or dilated pointwise Conv2d cannot be linearized." + ) self.linear = nn.Linear( conv.in_channels, conv.out_channels, @@ -433,7 +441,9 @@ def forward( camera_conditions=camera_conditions, camera_cache=camera_cache, chunk_size=kwargs.get("chunk_size"), - chunk_split_strategy=str(kwargs.get("chunk_split_strategy", "uniform")), + chunk_split_strategy=str( + kwargs.get("chunk_split_strategy", "uniform") + ), chunk_index=kwargs.get("chunk_index"), ) cam_contrib = self.out_proj_cam(cam_raw) @@ -608,11 +618,15 @@ def _forward_softmax_camera( chunk_split_strategy=chunk_split_strategy, chunk_index=chunk_index, ) - return output_projector(out.transpose(1, 2)).reshape( - x.shape[0], - x.shape[1], - -1, - ).to(dtype=x.dtype) + return ( + output_projector(out.transpose(1, 2)) + .reshape( + x.shape[0], + x.shape[1], + -1, + ) + .to(dtype=x.dtype) + ) def _compute_frame_gates( self, @@ -737,7 +751,9 @@ def forward( ).chunk(6, dim=-2) x_norm = self.norm1(x).reshape(batch, frames, -1, channels) - attn_in = _modulate(x_norm, shift_msa, scale_msa).reshape(batch, tokens, channels) + attn_in = _modulate(x_norm, shift_msa, scale_msa).reshape( + batch, tokens, channels + ) attn_out = self.attn(attn_in, **kwargs).reshape(batch, frames, -1, channels) x = x + (gate_msa * attn_out).reshape(batch, tokens, channels) @@ -747,7 +763,9 @@ def forward( x = x + self.cross_attn(x, y, mask=mask) x_norm = self.norm2(x).reshape(batch, frames, -1, channels) - mlp_in = _modulate(x_norm, shift_mlp, scale_mlp).reshape(batch, tokens, channels) + mlp_in = _modulate(x_norm, shift_mlp, scale_mlp).reshape( + batch, tokens, channels + ) mlp_out = self.mlp(mlp_in, frames=frames, height=height, width=width) mlp_out = mlp_out.reshape(batch, frames, -1, channels) return x + (gate_mlp * mlp_out).reshape(batch, tokens, channels) @@ -761,7 +779,9 @@ def __init__(self, spec: SanaWMStage1Spec = SANA_WM_STAGE1_SPEC) -> None: self.spec = spec self.x_embedder = Conv3dProjector(spec.latent_channels, spec.hidden_size) self.t_embedder = TimestepEmbedder(spec) - self.t_block = nn.Sequential(nn.SiLU(), nn.Linear(spec.hidden_size, 6 * spec.hidden_size)) + self.t_block = nn.Sequential( + nn.SiLU(), nn.Linear(spec.hidden_size, 6 * spec.hidden_size) + ) self.y_embedder = TextEmbedder(spec) self.attention_y_norm = RMSNorm(spec.hidden_size) self.raymap_embedder = Conv3dProjector(spec.raymap_channels, spec.hidden_size) @@ -1203,11 +1223,14 @@ def _bidirectional_temporal_conv_kernel( & channel_mask[None, None, :] ) - token_offsets = frame_offsets[:, None, None] * spatial + spatial_offsets[ - None, - :, - None, - ] + token_offsets = ( + frame_offsets[:, None, None] * spatial + + spatial_offsets[ + None, + :, + None, + ] + ) x_offsets = ( batch_idx * x_stride_b + token_offsets * x_stride_t @@ -1226,9 +1249,7 @@ def _bidirectional_temporal_conv_kernel( mask=channel_mask, other=0.0, ).to(tl.float32) - value = tl.load(x_ptr + x_offsets, mask=element_mask, other=0.0).to( - tl.float32 - ) + value = tl.load(x_ptr + x_offsets, mask=element_mask, other=0.0).to(tl.float32) acc = value * center[None, None, :] for offset in range(1, kernel_size): @@ -1241,8 +1262,9 @@ def _bidirectional_temporal_conv_kernel( ).to(tl.float32) prev_frame_offsets = frame_offsets - offset - prev_token_offsets = prev_frame_offsets[:, None, None] * spatial + ( - spatial_offsets[None, :, None] + prev_token_offsets = ( + prev_frame_offsets[:, None, None] * spatial + + (spatial_offsets[None, :, None]) ) prev_mask = element_mask & (frame_offsets[:, None, None] >= offset) prev_offsets = ( @@ -1255,8 +1277,9 @@ def _bidirectional_temporal_conv_kernel( ) next_frame_offsets = frame_offsets + offset - next_token_offsets = next_frame_offsets[:, None, None] * spatial + ( - spatial_offsets[None, :, None] + next_token_offsets = ( + next_frame_offsets[:, None, None] * spatial + + (spatial_offsets[None, :, None]) ) next_mask = element_mask & (next_frame_offsets[:, None, None] < frames) next_offsets = ( @@ -1308,38 +1331,28 @@ def _ucpe_first_half_kernel( col3 = col0 + 3 x_base = ( - batch_idx * x_stride_b - + token_offsets * x_stride_n - + head_idx * x_stride_h + batch_idx * x_stride_b + token_offsets * x_stride_n + head_idx * x_stride_h ) x0 = tl.load( x_ptr + x_base + col0 * x_stride_d, mask=token_mask, other=0.0, - ).to( - tl.float32 - ) + ).to(tl.float32) x1 = tl.load( x_ptr + x_base + col1 * x_stride_d, mask=token_mask, other=0.0, - ).to( - tl.float32 - ) + ).to(tl.float32) x2 = tl.load( x_ptr + x_base + col2 * x_stride_d, mask=token_mask, other=0.0, - ).to( - tl.float32 - ) + ).to(tl.float32) x3 = tl.load( x_ptr + x_base + col3 * x_stride_d, mask=token_mask, other=0.0, - ).to( - tl.float32 - ) + ).to(tl.float32) matrix_base = ( matrix_batch_idx * matrix_stride_b + token_offsets * matrix_stride_n @@ -1411,24 +1424,18 @@ def _ucpe_rope_second_half_kernel( real_d = half + pair_idx * 2 imag_d = real_d + 1 x_base = ( - batch_idx * x_stride_b - + token_offsets * x_stride_n - + head_idx * x_stride_h + batch_idx * x_stride_b + token_offsets * x_stride_n + head_idx * x_stride_h ) real = tl.load( x_ptr + x_base + real_d * x_stride_d, mask=token_mask, other=0.0, - ).to( - tl.float32 - ) + ).to(tl.float32) imag = tl.load( x_ptr + x_base + imag_d * x_stride_d, mask=token_mask, other=0.0, - ).to( - tl.float32 - ) + ).to(tl.float32) rope_base = token_offsets * rope_stride_n + pair_idx * rope_stride_p rope_real = tl.load( rope_ptr + rope_base + 0 * rope_stride_ri, @@ -1504,9 +1511,7 @@ def _ucpe_norms_kernel( + dim_offsets[None, :] * out_stride_d ) x_values = tl.load(x_ptr + x_offsets, mask=mask, other=0.0).to(tl.float32) - out_values = tl.load(out_ptr + out_offsets, mask=mask, other=0.0).to( - tl.float32 - ) + out_values = tl.load(out_ptr + out_offsets, mask=mask, other=0.0).to(tl.float32) pre = tl.sum(x_values * x_values, axis=1) post = tl.sum(out_values * out_values, axis=1) token_mask = token_offsets < tokens @@ -1794,10 +1799,7 @@ def _slice_rope_for_camera(rotary_emb: Tensor | None, head_dim: int) -> Tensor | h_part = rotary_emb[..., orig_t_size : orig_t_size + new_h_size] w_part = rotary_emb[ ..., - orig_t_size - + orig_h_size : orig_t_size - + orig_h_size - + new_w_size, + orig_t_size + orig_h_size : orig_t_size + orig_h_size + new_w_size, ] return torch.cat([t_part, h_part, w_part], dim=-1) @@ -1881,9 +1883,7 @@ def _gdn_histories( spatial = height * width k = k.reshape(batch, frames, spatial, heads, dim).permute(0, 3, 1, 2, 4).float() k_rot = ( - k_rot.reshape(batch, frames, spatial, heads, dim) - .permute(0, 3, 1, 2, 4) - .float() + k_rot.reshape(batch, frames, spatial, heads, dim).permute(0, 3, 1, 2, 4).float() ) v = v.reshape(batch, frames, spatial, heads, dim).permute(0, 3, 1, 2, 4).float() eye = torch.eye(dim, dtype=torch.float32, device=k.device).view(1, 1, 1, dim, dim) @@ -1955,7 +1955,9 @@ def _camera_qkv( dim=-1, ) if hasattr(module, "conv_k_cam"): - k_raw_flat = _apply_bidirectional_temporal_conv(k_raw_flat, module.conv_k_cam, HW) + k_raw_flat = _apply_bidirectional_temporal_conv( + k_raw_flat, module.conv_k_cam, HW + ) q_raw = q_raw_flat.reshape(batch, tokens, module.heads, module.dim) k_raw = k_raw_flat.reshape(batch, tokens, module.heads, module.dim) v_raw = v_raw_flat.reshape(batch, tokens, module.heads, module.dim) @@ -2042,12 +2044,12 @@ def _prepare_ucpe_qkv_softmax( k_inv = _inv_rms(k_raw, norm_eps) q_weight = q_norm_weight.float().view(heads, dim) k_weight = k_norm_weight.float().view(heads, dim) - q_norm = ( - q_raw.float() * q_inv[:, :, None, None] * q_weight[None, None] - ).to(dtype=q_raw.dtype) - k_norm = ( - k_raw.float() * k_inv[:, :, None, None] * k_weight[None, None] - ).to(dtype=k_raw.dtype) + q_norm = (q_raw.float() * q_inv[:, :, None, None] * q_weight[None, None]).to( + dtype=q_raw.dtype + ) + k_norm = (k_raw.float() * k_inv[:, :, None, None] * k_weight[None, None]).to( + dtype=k_raw.dtype + ) if camera_cache is None: camera_cache = _prepare_camera_projection_cache( camera_conditions, diff --git a/integrations/sana/tests/compat/mmcv/runner.py b/integrations/sana/tests/compat/mmcv/runner.py index 3e82685a2..81261a3f6 100644 --- a/integrations/sana/tests/compat/mmcv/runner.py +++ b/integrations/sana/tests/compat/mmcv/runner.py @@ -23,7 +23,9 @@ class DefaultOptimizerConstructor: def build_optimizer(*_args, **_kwargs): - raise NotImplementedError("Optimizer construction is not available in the parity harness.") + raise NotImplementedError( + "Optimizer construction is not available in the parity harness." + ) __all__ = [ diff --git a/integrations/sana/tests/parity_check/streaming_continuity.py b/integrations/sana/tests/parity_check/streaming_continuity.py index cf0dc9fc6..907324c2c 100644 --- a/integrations/sana/tests/parity_check/streaming_continuity.py +++ b/integrations/sana/tests/parity_check/streaming_continuity.py @@ -23,7 +23,6 @@ from typing import Any import numpy as np - from diff_parity import _load_frames @@ -42,12 +41,20 @@ def summarize_frames(frames: np.ndarray, *, chunk_size: int) -> dict[str, Any]: [value for idx, value in enumerate(per_transition) if idx not in boundary_set], dtype=np.float64, ) - boundary = np.asarray([per_transition[idx] for idx in boundary_indices], dtype=np.float64) + boundary = np.asarray( + [per_transition[idx] for idx in boundary_indices], dtype=np.float64 + ) - nonboundary_p95 = float(np.percentile(nonboundary, 95)) if nonboundary.size else None + nonboundary_p95 = ( + float(np.percentile(nonboundary, 95)) if nonboundary.size else None + ) boundary_mean = float(boundary.mean()) if boundary.size else None ratio = None - if nonboundary_p95 is not None and nonboundary_p95 > 0.0 and boundary_mean is not None: + if ( + nonboundary_p95 is not None + and nonboundary_p95 > 0.0 + and boundary_mean is not None + ): ratio = boundary_mean / nonboundary_p95 return { @@ -55,7 +62,9 @@ def summarize_frames(frames: np.ndarray, *, chunk_size: int) -> dict[str, Any]: "chunk_size": int(chunk_size), "mean_abs_delta": float(per_transition.mean()), "p95_abs_delta": float(np.percentile(per_transition, 95)), - "nonboundary_mean_abs_delta": float(nonboundary.mean()) if nonboundary.size else None, + "nonboundary_mean_abs_delta": float(nonboundary.mean()) + if nonboundary.size + else None, "nonboundary_p95_abs_delta": nonboundary_p95, "boundary_mean_abs_delta": boundary_mean, "boundary_max_abs_delta": float(boundary.max()) if boundary.size else None, diff --git a/integrations/sana/tests/performance/README.md b/integrations/sana/tests/performance/README.md index 03811023c..e2e3c18d2 100644 --- a/integrations/sana/tests/performance/README.md +++ b/integrations/sana/tests/performance/README.md @@ -26,11 +26,13 @@ upstream checkout at `../Sana`, pins it to `6298508`, and applies patches idempotently. The patches add instrumentation and do not change generation algorithms. -The BF16 path uses the base isolated environment. FP8 and FP4 rows opt into the -`quant` extra, which builds TransformerEngine from the pinned git source -against the local CUDA/PyTorch stack. A new checkout needs network access for -`uv sync`, GitHub access for the pinned upstream checkout and TransformerEngine -source, and a CUDA build toolchain for low-precision runs. +The base isolated environment covers BF16 and FlashDreams FP8/FP4 rows. +Only upstream `SANA-WM_streaming` FP8/FP4 comparison rows opt into the `quant` +extra, which builds TransformerEngine from the pinned git source against the +local CUDA/PyTorch stack. A new checkout needs network access for `uv sync`, +GitHub access for the pinned upstream checkout, and a CUDA build toolchain plus +TransformerEngine source access only when running those upstream streaming +low-precision comparisons. ## Run Benchmarks @@ -151,14 +153,16 @@ benchmark setting. FP8 and FP4 are passed through the precision flags supported by each measured side. For upstream this means streaming only; for FlashDreams this also includes -bidirectional diagnostic rows. Those precision runs sync the isolated venv with -the `quant` extra before launch. `bench.sh` lets TransformerEngine use its -upstream CUDA-version-aware default architecture set unless `NVTE_CUDA_ARCHS` is -set explicitly. The script records that setting in the isolated venv and -rebuilds TransformerEngine once when it changes, so stale extension wheels from -an earlier architecture setting are not reused. If TransformerEngine, hardware, -or upstream code rejects a precision, earlier precision outputs remain in place -and the failing command is recorded in the corresponding `command.txt`. +bidirectional diagnostic rows. FlashDreams low-precision rows use the base +isolated environment; upstream streaming low-precision rows sync the isolated +venv with the `quant` extra before launch. `bench.sh` lets TransformerEngine +use its upstream CUDA-version-aware default architecture set unless +`NVTE_CUDA_ARCHS` is set explicitly. The script records that setting in the +isolated venv and rebuilds TransformerEngine once when it changes, so stale +extension wheels from an earlier architecture setting are not reused. If +TransformerEngine, hardware, or upstream code rejects a precision, earlier +precision outputs remain in place and the failing command is recorded in the +corresponding `command.txt`. The scripts do not set allocator overrides or GPU wait loops. If GPU contention matters in your environment, handle it outside the harness. diff --git a/integrations/sana/tests/performance/bench.sh b/integrations/sana/tests/performance/bench.sh index ca1ec292f..690f0adea 100644 --- a/integrations/sana/tests/performance/bench.sh +++ b/integrations/sana/tests/performance/bench.sh @@ -124,6 +124,25 @@ _reject_bidirectional_low_precision() { exit 1 } +_validate_precision() { + local name="$1" + local value="$2" + case "${value}" in + bf16|fp8|fp4) ;; + *) + echo "[bench] ERROR: ${name} must be bf16, fp8, or fp4; got ${value}" >&2 + exit 1 + ;; + esac +} + +_is_quant_precision() { + case "$1" in + fp8|fp4) return 0 ;; + *) return 1 ;; + esac +} + if [[ "${SANA_WM_VARIANT}" == "streaming" ]]; then DEFAULT_OUTPUT_DIR="${SCRIPT_DIR}/outputs/bench/streaming/${STAGE1_PRECISION}" DEFAULT_SWEEP_OUTPUT_DIR="${SCRIPT_DIR}/outputs/bench/streaming" @@ -171,6 +190,9 @@ STREAMING_CRF="${STREAMING_CRF:-18}" STREAMING_PRESET="${STREAMING_PRESET:-medium}" STREAMING_ENCODER="${STREAMING_ENCODER:-${SANA_WM_STREAMING_MP4_ENCODER:-libx264}}" +_validate_precision "STAGE1_PRECISION" "${STAGE1_PRECISION}" +_validate_precision "REFINER_PRECISION" "${REFINER_PRECISION}" + if [[ "${SANA_WM_VARIANT}" == "streaming" ]] && _is_true "${NO_REFINER}"; then echo "[bench] ERROR: SANA_WM_VARIANT=streaming benchmarks the full upstream streaming stack; NO_REFINER=1 is not supported." >&2 exit 1 @@ -201,12 +223,26 @@ if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then fi fi +LOW_PRECISION=0 +if _is_quant_precision "${STAGE1_PRECISION}" || _is_quant_precision "${REFINER_PRECISION}"; then + LOW_PRECISION=1 +fi +UPSTREAM_QUANT=0 +if [[ "${LOW_PRECISION}" == "1" && "${SANA_WM_VARIANT}" == "streaming" && "${BENCH_SIDE}" != "flashdreams" ]]; then + UPSTREAM_QUANT=1 +fi + if _is_true "${BENCH_DRY_RUN}"; then echo "[bench] dry run" echo " variant: ${SANA_WM_VARIANT}" echo " side: ${BENCH_SIDE}" echo " output: ${OUTPUT_DIR}" echo " precision: stage1=${STAGE1_PRECISION} refiner=${REFINER_PRECISION}" + if [[ "${UPSTREAM_QUANT}" == "1" ]]; then + echo " upstream quant extra: yes" + else + echo " upstream quant extra: no" + fi if [[ "${SANA_WM_VARIANT}" == "bidirectional" ]]; then echo " bidirectional generations: warmup=${BIDIRECTIONAL_WARMUP_GENERATIONS} measured=${BIDIRECTIONAL_MEASURED_GENERATIONS}" if [[ -n "${BENCH_PRECISIONS}" ]]; then @@ -293,7 +329,7 @@ UV_SYNC_ARGS=(uv sync) UV_SYNC_ENV=() NVTE_ARCH_MARKER="${SANA_TEST_DIR}/.venv/.flashdreams_nvte_cuda_archs" NVTE_MARK_ARCH=0 -if [[ "${STAGE1_PRECISION}" != "bf16" || "${REFINER_PRECISION}" != "bf16" ]]; then +if [[ "${UPSTREAM_QUANT}" == "1" ]]; then UV_SYNC_ARGS+=(--extra quant) export NVTE_FRAMEWORK="${NVTE_FRAMEWORK:-pytorch}" export NVTE_WITH_NCCL_EP="${NVTE_WITH_NCCL_EP:-0}" @@ -320,6 +356,8 @@ PY ( cd "${SANA_TEST_DIR}" && \ uv venv --allow-existing --python "3.12" .venv >/dev/null && \ uv pip install --python .venv/bin/python setuptools wheel pybind11 ) +elif [[ "${LOW_PRECISION}" == "1" ]]; then + echo "[setup] skipping TransformerEngine quant extra; low-precision rows are FlashDreams-only." fi echo "[setup] ensuring Python deps via ${UV_SYNC_ARGS[*]} (isolated venv)" ( cd "${SANA_TEST_DIR}" && "${UV_SYNC_ENV[@]}" "${UV_SYNC_ARGS[@]}" ) diff --git a/integrations/sana/tests/performance/bench_summary.py b/integrations/sana/tests/performance/bench_summary.py index c5d8f847d..944349ecc 100644 --- a/integrations/sana/tests/performance/bench_summary.py +++ b/integrations/sana/tests/performance/bench_summary.py @@ -112,9 +112,7 @@ def _expand_generation_records( for fallback_index, generation in enumerate(generation_records): if not isinstance(generation, dict): continue - generation_index = int( - generation.get("generation_index", fallback_index) - ) + generation_index = int(generation.get("generation_index", fallback_index)) merged = process_metadata | generation parent_path = item.get("_path") if isinstance(parent_path, str): @@ -164,18 +162,12 @@ def _collect_bidirectional( ] rows["wall_median_s"] = _median(wall) rows["wall_p90_s"] = _p90(wall) - memory = [ - value - for item in kept - if (value := mem_reader(item)) is not None - ] + memory = [value for item in kept if (value := mem_reader(item)) is not None] rows["mem_peak_median_gib"] = _median(memory) rows["mem_peak_p90_gib"] = _p90(memory) for label, key in stages.items(): values = [ - value - for item in kept - if (value := stage_reader(item, key)) is not None + value for item in kept if (value := stage_reader(item, key)) is not None ] rows[f"{label}_median_ms"] = _median(values) rows[f"{label}_p90_ms"] = _p90(values) @@ -226,7 +218,11 @@ def _streaming_steady_state_chunk_ms(item: dict[str, Any]) -> float | None: # `steady_state_seconds` starts after the first decoded chunk, so exclude # that chunk from the denominator to avoid mixing cache-fill/compile latency # into the steady-state headline. - measured_chunks = chunks_value - 1 if item.get("first_chunk_seconds") is not None else chunks_value + measured_chunks = ( + chunks_value - 1 + if item.get("first_chunk_seconds") is not None + else chunks_value + ) if measured_chunks <= 0: return None return steady_s * 1000.0 / measured_chunks @@ -250,7 +246,12 @@ def _collect_streaming( wall = [ value for item in kept - if (value := _first_numeric(item, "stream_wall_seconds", "wall_seconds", "wall_s")) is not None + if ( + value := _first_numeric( + item, "stream_wall_seconds", "wall_seconds", "wall_s" + ) + ) + is not None ] rows["wall_median_s"] = _median(wall) rows["wall_p90_s"] = _p90(wall) @@ -289,11 +290,7 @@ def _collect_streaming( ] rows["steady_state_realtime_median"] = _median(realtime) rows["steady_state_realtime_p90"] = _p90(realtime) - memory = [ - value - for item in kept - if (value := _upstream_mem_gib(item)) is not None - ] + memory = [value for item in kept if (value := _upstream_mem_gib(item)) is not None] rows["mem_peak_median_gib"] = _median(memory) rows["mem_peak_p90_gib"] = _p90(memory) for label, key in { @@ -314,9 +311,7 @@ def _collect_streaming( "n_refiner_blocks": "n_refiner_blocks", }.items(): values = [ - value - for item in kept - if (value := _first_numeric(item, key)) is not None + value for item in kept if (value := _first_numeric(item, key)) is not None ] rows[f"{label}_median"] = _median(values) rows[f"{label}_p90"] = _p90(values) @@ -632,13 +627,19 @@ def _render_streaming_markdown(summary: dict[str, Any]) -> str: def _render_markdown(summary: dict[str, Any]) -> str: - if summary.get("variant") == "streaming" or summary["inputs"].get("variant") == "streaming": + if ( + summary.get("variant") == "streaming" + or summary["inputs"].get("variant") == "streaming" + ): return _render_streaming_markdown(summary) return _render_bidirectional_markdown(summary) def _render_chart_markdown(summary: dict[str, Any], device_label: str) -> str: - streaming = summary.get("variant") == "streaming" or summary["inputs"].get("variant") == "streaming" + streaming = ( + summary.get("variant") == "streaming" + or summary["inputs"].get("variant") == "streaming" + ) official = ( _streaming_generation_ms_per_chunk(summary, "upstream") if streaming @@ -692,7 +693,9 @@ def _render_chart_markdown(summary: dict[str, Any], device_label: str) -> str: def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--variant", choices=["bidirectional", "streaming"], default="bidirectional") + parser.add_argument( + "--variant", choices=["bidirectional", "streaming"], default="bidirectional" + ) parser.add_argument( "--bench-side", choices=["upstream", "flashdreams", "both"], @@ -704,14 +707,20 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--image-path", type=Path, required=True) parser.add_argument("--prompt-path", type=Path, required=True) parser.add_argument("--camera-path", type=Path, required=True) - parser.add_argument("--camera-source", choices=["camera", "action"], default="camera") + parser.add_argument( + "--camera-source", choices=["camera", "action"], default="camera" + ) parser.add_argument("--action", default=None) parser.add_argument("--intrinsics-path", type=Path, required=True) parser.add_argument("--num-frames", type=int, required=True) parser.add_argument("--seed", type=int, required=True) parser.add_argument("--no-refiner", action="store_true") - parser.add_argument("--stage1-precision", choices=["bf16", "fp8", "fp4"], default="bf16") - parser.add_argument("--refiner-precision", choices=["bf16", "fp8", "fp4"], default="bf16") + parser.add_argument( + "--stage1-precision", choices=["bf16", "fp8", "fp4"], default="bf16" + ) + parser.add_argument( + "--refiner-precision", choices=["bf16", "fp8", "fp4"], default="bf16" + ) parser.add_argument( "--quant-backend", choices=["auto", "torch", "torch-fp8", "torch-fp4"], @@ -737,9 +746,11 @@ def main(argv: list[str] | None = None) -> None: parser.error("--upstream-dir is required unless --bench-side=flashdreams") if args.bench_side != "upstream" and args.flashdreams_dir is None: parser.error("--flashdreams-dir is required unless --bench-side=upstream") - if args.variant == "bidirectional" and ( - args.stage1_precision != "bf16" or args.refiner_precision != "bf16" - ) and args.bench_side != "flashdreams": + if ( + args.variant == "bidirectional" + and (args.stage1_precision != "bf16" or args.refiner_precision != "bf16") + and args.bench_side != "flashdreams" + ): parser.error( "upstream SANA-WM_bidirectional benchmarks are BF16-only; " "FP8 and FP4 precision flags are only supported by upstream " @@ -806,7 +817,9 @@ def main(argv: list[str] | None = None) -> None: if args.variant == "streaming": summary["upstream"] = _collect_streaming(upstream_items, args.warmup_runs) if args.bench_side == "both": - summary["flashdreams"] = _collect_streaming(flashdreams_items, args.warmup_runs) + summary["flashdreams"] = _collect_streaming( + flashdreams_items, args.warmup_runs + ) summary["benchmark"] = { "metric": "steady_state_generation_ms_per_chunk", "unit": "ms", @@ -817,7 +830,9 @@ def main(argv: list[str] | None = None) -> None: "device_label": args.device_label, "chart_label": args.chart_label or args.device_label, "official": _streaming_generation_ms_per_chunk(summary, "upstream"), - "official_p90": _streaming_generation_ms_per_chunk(summary, "upstream", percentile="p90"), + "official_p90": _streaming_generation_ms_per_chunk( + summary, "upstream", percentile="p90" + ), "flashdreams": _streaming_generation_ms_per_chunk(summary, "flashdreams") if args.bench_side == "both" else None, diff --git a/integrations/sana/tests/performance/bench_sweep_summary.py b/integrations/sana/tests/performance/bench_sweep_summary.py index 635a03f6d..b52d220c9 100644 --- a/integrations/sana/tests/performance/bench_sweep_summary.py +++ b/integrations/sana/tests/performance/bench_sweep_summary.py @@ -102,15 +102,11 @@ def _render_chart(rows: list[dict[str, Any]]) -> str: values = [row["label"]] if has_official: values.append( - f"{row['official']:.2f}" - if row["official"] is not None - else "n/a" + f"{row['official']:.2f}" if row["official"] is not None else "n/a" ) if has_flashdreams: values.append( - f"{row['flashdreams']:.2f}" - if row["flashdreams"] is not None - else "n/a" + f"{row['flashdreams']:.2f}" if row["flashdreams"] is not None else "n/a" ) lines.append("| " + " | ".join(values) + " |") lines.append("") @@ -150,9 +146,7 @@ def _render_report(rows: list[dict[str, Any]]) -> str: values = [row["label"]] if has_official: values.append( - f"{row['official']:.2f} ms" - if row["official"] is not None - else "n/a" + f"{row['official']:.2f} ms" if row["official"] is not None else "n/a" ) if has_flashdreams: values.append( diff --git a/integrations/sana/tests/pyproject.toml b/integrations/sana/tests/pyproject.toml index 7873bd8a0..d86fb7de9 100644 --- a/integrations/sana/tests/pyproject.toml +++ b/integrations/sana/tests/pyproject.toml @@ -35,10 +35,9 @@ dependencies = [ ] [project.optional-dependencies] -# Transformer Engine is only needed for the FP8/FP4 quant backends. It is gated -# behind an opt-in extra so the default BF16 checks do not try to build/import -# it. The git source avoids split-wheel ABI mismatches on bleeding-edge -# torch/CUDA stacks while keeping BF16 setup lightweight. +# Transformer Engine is only needed for upstream SANA-WM_streaming FP8/FP4 +# comparisons. FlashDreams low-precision rows use Torch/Triton replacements, +# and upstream SANA-WM_bidirectional is BF16-only. quant = [ "ml-dtypes>=0.5", "nvdlfw-inspect>=0.2", diff --git a/integrations/sana/tests/run_flashdreams_bidirectional.py b/integrations/sana/tests/run_flashdreams_bidirectional.py index bbc627b0a..3f38ca745 100644 --- a/integrations/sana/tests/run_flashdreams_bidirectional.py +++ b/integrations/sana/tests/run_flashdreams_bidirectional.py @@ -23,8 +23,6 @@ import numpy as np import torch - -from flashdreams.infra.config import derive_config from run_flashdreams_common import ( apply_backend_defaults, install_stage1_compile_hook, @@ -44,6 +42,8 @@ _validate_precision_request, ) +from flashdreams.infra.config import derive_config + def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(description=__doc__) diff --git a/integrations/sana/tests/run_flashdreams_common.py b/integrations/sana/tests/run_flashdreams_common.py index 9fd43c10c..e11ccc20d 100644 --- a/integrations/sana/tests/run_flashdreams_common.py +++ b/integrations/sana/tests/run_flashdreams_common.py @@ -26,7 +26,6 @@ import numpy as np import torch from PIL import Image - from sana_wm.camera import ( action_string_to_c2w, default_intrinsics_vec4, @@ -203,10 +202,14 @@ def compile_streaming_refiner(pipeline: Any) -> None: "SANA_WM_TORCH_COMPILE_MODE", "max-autotune-no-cudagraphs", ).strip() - compile_dynamic_raw = os.environ.get( - "SANA_WM_TORCH_COMPILE_DYNAMIC", - "1", - ).strip().lower() + compile_dynamic_raw = ( + os.environ.get( + "SANA_WM_TORCH_COMPILE_DYNAMIC", + "1", + ) + .strip() + .lower() + ) compile_dynamic = compile_dynamic_raw not in {"0", "false", "no", "off"} refiner.refiner.transformer = torch.compile( refiner.refiner.transformer, diff --git a/integrations/sana/tests/run_flashdreams_streaming.py b/integrations/sana/tests/run_flashdreams_streaming.py index 828949628..20c8e8820 100644 --- a/integrations/sana/tests/run_flashdreams_streaming.py +++ b/integrations/sana/tests/run_flashdreams_streaming.py @@ -23,8 +23,6 @@ import numpy as np import torch - -from flashdreams.infra.config import derive_config from run_flashdreams_common import ( apply_backend_defaults, compile_streaming_refiner, @@ -54,6 +52,8 @@ _validate_precision_request, ) +from flashdreams.infra.config import derive_config + def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(description=__doc__) @@ -258,13 +258,14 @@ def main(argv: list[str] | None = None) -> None: per_chunk_stats.append(stats) if not isinstance(decoded, SanaWMDecodedVideo): raise TypeError( - "expected SanaWMDecodedVideo, " - f"got {type(decoded).__name__}" + f"expected SanaWMDecodedVideo, got {type(decoded).__name__}" ) if decoded.video_hwc.size: decoded_chunks.append(np.asarray(decoded.video_hwc, dtype=np.uint8)) if decoded.stage1_video_hwc is not None and decoded.stage1_video_hwc.size: - stage1_chunks.append(np.asarray(decoded.stage1_video_hwc, dtype=np.uint8)) + stage1_chunks.append( + np.asarray(decoded.stage1_video_hwc, dtype=np.uint8) + ) if torch.cuda.is_available(): torch.cuda.synchronize(device) stream_wall_seconds = time.perf_counter() - generation_start @@ -359,15 +360,11 @@ def main(argv: list[str] | None = None) -> None: "n_refiner_blocks": len(chunk_boundaries) - 1, "n_pixel_frames": n_pixel_frames, "stage1_cuda_seconds": ( - stage1_cuda_ms / 1000.0 - if stage1_cuda_ms is not None - else None + stage1_cuda_ms / 1000.0 if stage1_cuda_ms is not None else None ), "refiner_cuda_seconds": None, "decode_cuda_seconds": ( - decode_cuda_ms / 1000.0 - if decode_cuda_ms is not None - else None + decode_cuda_ms / 1000.0 if decode_cuda_ms is not None else None ), "per_chunk_wall_seconds": per_chunk_wall_s, "per_chunk_stats_ms": per_chunk_stats, diff --git a/integrations/sana/tests/test_camera.py b/integrations/sana/tests/test_camera.py index e54d14eb9..291f7d7b7 100644 --- a/integrations/sana/tests/test_camera.py +++ b/integrations/sana/tests/test_camera.py @@ -22,7 +22,6 @@ import numpy as np import pytest import torch - from sana_wm.camera import ( action_string_to_c2w, default_intrinsics_vec4, @@ -80,6 +79,24 @@ def test_action_string_repeats_to_requested_frame_count() -> None: ) +def test_action_string_bounds_oversized_duration_to_requested_frames() -> None: + """Avoid materializing action frames beyond the requested output length.""" + c2w = action_string_to_c2w("w-500000000", smooth=False, num_frames=5) + + assert c2w.shape == (5, 4, 4) + np.testing.assert_allclose( + c2w[:, 2, 3], + [0.0, 0.025, 0.05, 0.075, 0.1], + atol=1e-6, + ) + + +def test_action_string_validates_segments_after_requested_frame_limit() -> None: + """Do not skip validation for segments beyond ``num_frames`` truncation.""" + with pytest.raises(ValueError, match="unknown keys"): + action_string_to_c2w("w-500000000,q-1", smooth=False, num_frames=5) + + def test_action_string_rejects_unknown_keys() -> None: """Reject invalid action DSL tokens.""" with pytest.raises(ValueError, match="unknown keys"): diff --git a/integrations/sana/tests/test_performance_benchmark_summary.py b/integrations/sana/tests/test_performance_benchmark_summary.py index cf07e0f06..371406dec 100644 --- a/integrations/sana/tests/test_performance_benchmark_summary.py +++ b/integrations/sana/tests/test_performance_benchmark_summary.py @@ -49,7 +49,9 @@ def _load_bench_sweep_summary() -> ModuleType: return module -def test_bidirectional_summary_uses_steady_state_generation_ms_per_clip_for_chart() -> None: +def test_bidirectional_summary_uses_steady_state_generation_ms_per_clip_for_chart() -> ( + None +): module = _load_bench_summary() summary = { "inputs": { @@ -493,25 +495,73 @@ def test_bidirectional_bench_script_rejects_low_precision_dry_run() -> None: assert "upstream SANA-WM_bidirectional benchmarks are BF16-only" in result.stderr +@pytest.mark.parametrize( + ("variant", "side", "precision", "expected_quant"), + [ + ("bidirectional", "flashdreams", "fp4", "no"), + ("streaming", "flashdreams", "fp8", "no"), + ("streaming", "both", "bf16", "no"), + ("streaming", "upstream", "fp4", "yes"), + ("streaming", "both", "fp8", "yes"), + ], +) +def test_bench_script_quant_extra_is_upstream_streaming_low_precision_only( + variant: str, + side: str, + precision: str, + expected_quant: str, +) -> None: + env = os.environ.copy() + env.update( + { + "BENCH_DRY_RUN": "1", + "SANA_WM_VARIANT": variant, + "BENCH_SIDE": side, + "STAGE1_PRECISION": precision, + "REFINER_PRECISION": precision, + "BENCH_PRECISIONS": "", + } + ) + result = subprocess.run( + ["bash", "integrations/sana/tests/performance/bench.sh"], + check=False, + cwd=Path.cwd(), + env=env, + text=True, + capture_output=True, + ) + + assert result.returncode == 0 + assert f"upstream quant extra: {expected_quant}" in result.stdout + + def test_benchmark_summary_keeps_frame_normalized_diagnostics() -> None: module = _load_bench_summary() - assert module._generation_ms_per_frame( - { - "inputs": {"num_frames": 40}, - "upstream": {"wall_median_s": 4.0}, - }, - "upstream", - ) == 100.0 - assert module._generation_ms_per_clip( - { - "inputs": {"num_frames": 40}, - "upstream": {"wall_median_s": 4.0}, - }, - "upstream", - ) == 4000.0 + assert ( + module._generation_ms_per_frame( + { + "inputs": {"num_frames": 40}, + "upstream": {"wall_median_s": 4.0}, + }, + "upstream", + ) + == 100.0 + ) + assert ( + module._generation_ms_per_clip( + { + "inputs": {"num_frames": 40}, + "upstream": {"wall_median_s": 4.0}, + }, + "upstream", + ) + == 4000.0 + ) -def test_streaming_upstream_summary_uses_steady_state_ms_per_chunk(tmp_path: Path) -> None: +def test_streaming_upstream_summary_uses_steady_state_ms_per_chunk( + tmp_path: Path, +) -> None: module = _load_bench_summary() warmup = tmp_path / "upstream" / "run_0" measured = tmp_path / "upstream" / "run_1" @@ -555,7 +605,9 @@ def test_streaming_upstream_summary_uses_steady_state_ms_per_chunk(tmp_path: Pat ), encoding="utf-8", ) - (measured / "command.txt").write_text("uv run python upstream.py\n", encoding="utf-8") + (measured / "command.txt").write_text( + "uv run python upstream.py\n", encoding="utf-8" + ) out_json = tmp_path / "bench.json" out_md = tmp_path / "bench.md" @@ -673,7 +725,9 @@ def test_streaming_comparison_summary_writes_chart_data(tmp_path: Path) -> None: ), encoding="utf-8", ) - (upstream / "command.txt").write_text("uv run python upstream.py\n", encoding="utf-8") + (upstream / "command.txt").write_text( + "uv run python upstream.py\n", encoding="utf-8" + ) (flashdreams / "command.txt").write_text( "uv run python run_flashdreams_streaming.py\n", encoding="utf-8", @@ -755,7 +809,9 @@ def test_streaming_comparison_summary_writes_chart_data(tmp_path: Path) -> None: assert "steady-state generation median / chunk | 2000.00 ms | 1000.00 ms" in report -def test_benchmark_sweep_summary_accepts_bidirectional_bf16_chart_data(tmp_path: Path) -> None: +def test_benchmark_sweep_summary_accepts_bidirectional_bf16_chart_data( + tmp_path: Path, +) -> None: module = _load_bench_sweep_summary() bf16_json = tmp_path / "bf16.json" bf16_json.write_text( @@ -956,7 +1012,9 @@ def test_benchmark_sweep_summary_rejects_bidirectional_low_precision( ) -def test_benchmark_sweep_summary_accepts_streaming_upstream_only(tmp_path: Path) -> None: +def test_benchmark_sweep_summary_accepts_streaming_upstream_only( + tmp_path: Path, +) -> None: module = _load_bench_sweep_summary() bf16_json = tmp_path / "bf16.json" fp8_json = tmp_path / "fp8.json" diff --git a/integrations/sana/tests/test_quant_cuda.py b/integrations/sana/tests/test_quant_cuda.py index ff6265760..bbdbcea69 100644 --- a/integrations/sana/tests/test_quant_cuda.py +++ b/integrations/sana/tests/test_quant_cuda.py @@ -19,7 +19,6 @@ import pytest import torch - from sana_wm.quant import TorchScaledMMFP4Linear, TorchScaledMMFP8Linear pytestmark = pytest.mark.ci_gpu diff --git a/integrations/sana/tests/test_stage1_cuda.py b/integrations/sana/tests/test_stage1_cuda.py index d0f60ada6..647fbcf4b 100644 --- a/integrations/sana/tests/test_stage1_cuda.py +++ b/integrations/sana/tests/test_stage1_cuda.py @@ -18,9 +18,8 @@ from __future__ import annotations import pytest -import torch - import sana_wm.stage1_model as stage1 +import torch pytestmark = pytest.mark.ci_gpu From 7d486ca94e86a805a912bc0e18af25593ca958a2 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 16:38:18 -0700 Subject: [PATCH 63/64] Fix linter issues --- integrations/sana/sana_wm/decoder.py | 28 +++-- integrations/sana/sana_wm/quant.py | 12 +++ integrations/sana/sana_wm/refiner.py | 22 ++-- integrations/sana/sana_wm/runner.py | 6 +- integrations/sana/sana_wm/stage1_model.py | 36 +++++-- integrations/sana/sana_wm/transformer.py | 71 ++++++++++--- .../sana/tests/performance/bench_summary.py | 9 +- .../sana/tests/run_flashdreams_common.py | 2 +- integrations/sana/tests/test_smoke.py | 100 ++++++++++-------- 9 files changed, 194 insertions(+), 92 deletions(-) diff --git a/integrations/sana/sana_wm/decoder.py b/integrations/sana/sana_wm/decoder.py index 431727903..513572c2f 100644 --- a/integrations/sana/sana_wm/decoder.py +++ b/integrations/sana/sana_wm/decoder.py @@ -20,7 +20,7 @@ import gc import time from dataclasses import dataclass, field -from typing import Any +from typing import Any, cast import numpy as np import torch @@ -118,6 +118,8 @@ class SanaWMLTX2VAEDecoder(nn.Module): """Decode SANA-WM LTX-2 VAE latents into HWC uint8 video.""" config: SanaWMLTX2VAEDecoderConfig + vae: Any + vae_dtype: torch.dtype def __init__(self, config: SanaWMLTX2VAEDecoderConfig) -> None: super().__init__() @@ -284,7 +286,7 @@ def _configure_vae_tiling( tile_sample_min_num_frames: int | None = None, tile_sample_stride_num_frames: int | None = None, ) -> None: - vae = self.vae + vae: Any = self.vae min_width = int(tile_sample_min_width or self.config.vae_tile_sample_min_width) stride_width = int( tile_sample_stride_width or self.config.vae_tile_sample_stride_width @@ -358,7 +360,7 @@ def _vae_decode(self, latents: Tensor) -> Tensor: ) -def _prepare_ltx2_vae_decode_samples(vae: nn.Module, latents: Tensor) -> Tensor: +def _prepare_ltx2_vae_decode_samples(vae: Any, latents: Tensor) -> Tensor: """Convert normalized SANA-WM latents to the VAE decoder latent domain.""" latents_mean = vae.latents_mean.view(1, -1, 1, 1, 1).to( latents.device, @@ -372,7 +374,7 @@ def _prepare_ltx2_vae_decode_samples(vae: nn.Module, latents: Tensor) -> Tensor: return latents * latents_std / scale + latents_mean -def _supports_causal_vae_chunk_decode(vae: nn.Module) -> bool: +def _supports_causal_vae_chunk_decode(vae: Any) -> bool: """Return whether ``vae`` exposes the causal chunk decode contract.""" decoder = getattr(vae, "decoder", None) if ( @@ -388,23 +390,27 @@ def _supports_causal_vae_chunk_decode(vae: nn.Module) -> bool: def _decoded_video_to_hwc_uint8(decoded: Tensor | list[Tensor]) -> np.ndarray: """Convert VAE decode output to a single ``uint8`` HWC video array.""" + decoded_tensor: Tensor if isinstance(decoded, list): if not decoded: raise ValueError("VAE decode returned no frames.") - first = decoded[0] + frames = cast(list[Tensor], decoded) + first = frames[0] if first.ndim == 5: - decoded = torch.cat(decoded, dim=2) + decoded_tensor = torch.cat(frames, dim=2) elif first.ndim == 4: - decoded = torch.stack(decoded, dim=2) + decoded_tensor = torch.stack(frames, dim=2) else: - decoded = torch.stack(decoded, dim=0) - if decoded.ndim != 5: + decoded_tensor = torch.stack(frames, dim=0) + else: + decoded_tensor = decoded + if decoded_tensor.ndim != 5: raise ValueError( "SANA-WM VAE decode output must have shape [B, C, T, H, W]; " - f"got {tuple(decoded.shape)}." + f"got {tuple(decoded_tensor.shape)}." ) return ( - torch.clamp(127.5 * decoded + 127.5, 0, 255) + torch.clamp(127.5 * decoded_tensor + 127.5, 0, 255) .to(torch.uint8) .permute(0, 2, 3, 4, 1) .contiguous() diff --git a/integrations/sana/sana_wm/quant.py b/integrations/sana/sana_wm/quant.py index a7e177ecd..87ddbd1df 100644 --- a/integrations/sana/sana_wm/quant.py +++ b/integrations/sana/sana_wm/quant.py @@ -269,6 +269,11 @@ class TorchScaledMMFP8Linear(nn.Module): in_features: int out_features: int + out_dtype: torch.dtype + weight: torch.Tensor + weight_fp8: torch.Tensor + weight_scale: torch.Tensor + bias: torch.Tensor | None def __init__( self, @@ -389,6 +394,13 @@ class TorchScaledMMFP4Linear(nn.Module): in_features: int out_features: int + out_dtype: torch.dtype + use_rht: bool + weight: torch.Tensor + weight_qdata: torch.Tensor + weight_scale: torch.Tensor + weight_global_scale: torch.Tensor | None + bias: torch.Tensor | None def __init__( self, diff --git a/integrations/sana/sana_wm/refiner.py b/integrations/sana/sana_wm/refiner.py index 391046dfb..65c472d9c 100644 --- a/integrations/sana/sana_wm/refiner.py +++ b/integrations/sana/sana_wm/refiner.py @@ -21,7 +21,7 @@ import time from dataclasses import dataclass from pathlib import Path -from typing import Literal +from typing import Any, Literal, cast import torch import torch.nn as nn @@ -147,6 +147,11 @@ def _move_ltx2_video_modules_to( class SanaWMLTX2Refiner(nn.Module): """Run the SANA-WM LTX-2 latent refiner without importing a Sana checkout.""" + transformer: Any + connectors: Any + tokenizer: Any + text_encoder: Any + def __init__( self, *, @@ -510,6 +515,7 @@ def _ensure_text_encoder(self) -> None: tokenizer = AutoTokenizer.from_pretrained( self.gemma_root, local_files_only=True ) + tokenizer = cast(Any, tokenizer) tokenizer.padding_side = "left" if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token @@ -795,7 +801,7 @@ def _forward_video_current_only( def _forward_video_block( *, - block: nn.Module, + block: Any, hidden_states: Tensor, encoder_hidden_states: Tensor, temb: Tensor, @@ -840,7 +846,7 @@ def _forward_video_block( def _forward_refiner_context_block( *, - block: nn.Module, + block: Any, hidden_states: Tensor, encoder_hidden_states: Tensor, temb: Tensor, @@ -884,7 +890,7 @@ def _forward_refiner_context_block( def _forward_refiner_current_block( *, - block: nn.Module, + block: Any, hidden_states: Tensor, encoder_hidden_states: Tensor, temb: Tensor, @@ -929,7 +935,7 @@ def _forward_refiner_current_block( def _refiner_block_ada_values( - block: nn.Module, + block: Any, temb: Tensor, batch_size: int, ) -> tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: @@ -945,7 +951,7 @@ def _refiner_block_ada_values( def _streaming_self_attention( *, - attn: nn.Module, + attn: Any, hidden_states: Tensor, query_rotary_emb: tuple[Tensor, Tensor], n_context_tokens: int, @@ -984,7 +990,7 @@ def _streaming_self_attention( def _refiner_self_attention_qkv( *, - attn: nn.Module, + attn: Any, hidden_states: Tensor, query_rotary_emb: tuple[Tensor, Tensor], ) -> tuple[Tensor, Tensor, Tensor, Tensor | None]: @@ -1019,7 +1025,7 @@ def _refiner_self_attention_qkv( def _finish_refiner_self_attention( *, - attn: nn.Module, + attn: Any, hidden_states: Tensor, gate_logits: Tensor | None, dtype: torch.dtype, diff --git a/integrations/sana/sana_wm/runner.py b/integrations/sana/sana_wm/runner.py index f8bbd4d94..d4b5c81a5 100644 --- a/integrations/sana/sana_wm/runner.py +++ b/integrations/sana/sana_wm/runner.py @@ -20,7 +20,7 @@ import os from dataclasses import dataclass, field from pathlib import Path -from typing import Literal +from typing import Any, Literal import numpy as np import torch @@ -256,7 +256,7 @@ class SanaWMStreamingRunnerConfig(SanaWMRunnerConfig): """Refiner sliding-window size in latent frames.""" -class SanaWMRunner(Runner[SanaWMRunnerConfig, object]): +class SanaWMRunner(Runner[SanaWMRunnerConfig, Any]): """CLI driver for SANA-WM configs.""" config: SanaWMRunnerConfig @@ -433,7 +433,7 @@ def _prepare_inputs(self) -> tuple[object, np.ndarray, np.ndarray, int]: target_h=DEFAULT_VIDEO_HEIGHT, target_w=DEFAULT_VIDEO_WIDTH, ) - resized = image.resize(resized_size, Image.LANCZOS) + resized = image.resize(resized_size, Image.Resampling.LANCZOS) left, top = crop_offset cropped = resized.crop( ( diff --git a/integrations/sana/sana_wm/stage1_model.py b/integrations/sana/sana_wm/stage1_model.py index aeb3bc9a0..f5eb4ac58 100644 --- a/integrations/sana/sana_wm/stage1_model.py +++ b/integrations/sana/sana_wm/stage1_model.py @@ -25,18 +25,23 @@ import os from collections.abc import Callable from dataclasses import dataclass +from typing import TYPE_CHECKING, cast import torch import torch.nn as nn import torch.nn.functional as F from torch import Tensor -try: +if TYPE_CHECKING: import triton import triton.language as tl -except ImportError: # pragma: no cover - exercised in minimal CPU environments. - triton = None - tl = None +else: + try: + import triton + import triton.language as tl + except ImportError: # pragma: no cover - exercised in minimal CPU environments. + triton = None + tl = None @dataclass(frozen=True) @@ -136,6 +141,7 @@ class TimestepEmbedder(nn.Module): def __init__(self, spec: SanaWMStage1Spec) -> None: super().__init__() + self.timestep_dim = int(spec.timestep_dim) self.mlp = nn.Sequential( nn.Linear(spec.timestep_dim, spec.hidden_size), nn.SiLU(), @@ -144,8 +150,13 @@ def __init__(self, spec: SanaWMStage1Spec) -> None: def forward(self, t: Tensor) -> Tensor: """Project timestep features into the DiT hidden width.""" - t_freq = _timestep_embedding(t.flatten(), self.mlp[0].in_features) - return self.mlp(t_freq.to(device=t.device, dtype=self.mlp[0].weight.dtype)) + first_layer = self.mlp[0] + if not isinstance(first_layer, nn.Linear): + raise TypeError( + "TimestepEmbedder expected the first MLP layer to be Linear." + ) + t_freq = _timestep_embedding(t.flatten(), self.timestep_dim) + return self.mlp(t_freq.to(device=t.device, dtype=first_layer.weight.dtype)) class TextProjection(nn.Module): @@ -434,17 +445,26 @@ def forward( precomputed_gates=precomputed_gates, ) else: + raw_chunk_size = kwargs.get("chunk_size") + chunk_size = raw_chunk_size if isinstance(raw_chunk_size, int) else None + raw_chunk_index = kwargs.get("chunk_index") + chunk_index = ( + cast(list[int], raw_chunk_index) + if isinstance(raw_chunk_index, list) + and all(isinstance(index, int) for index in raw_chunk_index) + else None + ) cam_raw = self._forward_softmax_camera( x, HW=HW, rotary_emb=rotary_emb, camera_conditions=camera_conditions, camera_cache=camera_cache, - chunk_size=kwargs.get("chunk_size"), + chunk_size=chunk_size, chunk_split_strategy=str( kwargs.get("chunk_split_strategy", "uniform") ), - chunk_index=kwargs.get("chunk_index"), + chunk_index=chunk_index, ) cam_contrib = self.out_proj_cam(cam_raw) diff --git a/integrations/sana/sana_wm/transformer.py b/integrations/sana/sana_wm/transformer.py index 2129cb42a..520c57889 100644 --- a/integrations/sana/sana_wm/transformer.py +++ b/integrations/sana/sana_wm/transformer.py @@ -946,7 +946,8 @@ def _streaming_window_model_kwargs( if key in {"camera_cache", "rotary_emb", "chunk_plucker_emb"}: continue if key == "data_info" and isinstance(value, dict): - result[key] = _streaming_window_data_info(value, frame_indices) + data_info = cast(dict[object, object], value) + result[key] = _streaming_window_data_info(data_info, frame_indices) elif key == "chunk_index": result[key] = _chunk_index_from_chunk_size( len(frame_indices), @@ -960,7 +961,7 @@ def _streaming_window_model_kwargs( total_frames=total_frames, ) elif isinstance(value, dict): - result[key] = dict(value) + result[key] = dict(cast(dict[object, object], value)) else: result[key] = value return result @@ -986,19 +987,36 @@ def _streaming_window_data_info( frame_indices: tuple[int, ...], ) -> dict[object, object]: """Remap absolute conditioned-frame metadata into the local frame window.""" - result = dict(data_info) + result: dict[object, object] = dict(data_info) frame_to_local = {frame: local for local, frame in enumerate(frame_indices)} condition_frame_info = data_info.get("condition_frame_info") if isinstance(condition_frame_info, dict): - remapped = {} + remapped: dict[object, object] = {} for frame, value in condition_frame_info.items(): - local = frame_to_local.get(int(frame)) + frame_index = _condition_frame_index(frame) + if frame_index is None: + continue + local = frame_to_local.get(frame_index) if local is not None: remapped[local] = value result["condition_frame_info"] = remapped return result +def _condition_frame_index(value: object) -> int | None: + """Convert conditioned-frame metadata keys to integer frame indices.""" + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) if value.is_integer() else None + if isinstance(value, str): + try: + return int(value) + except ValueError: + return None + return None + + def _streaming_prefix_timestep( *, noisy_latent: Tensor, @@ -1064,7 +1082,10 @@ def _streaming_window_timestep( condition_frame_info = _streaming_condition_frame_info(conditioning) frame_to_local = {frame: local for local, frame in enumerate(frame_indices)} for frame_idx in condition_frame_info: - local = frame_to_local.get(int(frame_idx)) + frame_index = _condition_frame_index(frame_idx) + if frame_index is None: + continue + local = frame_to_local.get(frame_index) if local is not None: window[:, :, local] = 0 return window @@ -1103,7 +1124,9 @@ def _streaming_current_timestep( current = current.to(device=noisy_latent.device, dtype=torch.float32).clone() condition_frame_info = _streaming_condition_frame_info(conditioning) for frame_idx in condition_frame_info: - index = int(frame_idx) + index = _condition_frame_index(frame_idx) + if index is None: + continue if conditioning.start_frame <= index < conditioning.end_frame: current[:, :, index - conditioning.start_frame] = 0 return current @@ -1114,10 +1137,17 @@ def _streaming_condition_frame_info( ) -> dict[object, object]: """Return conditioned-frame metadata from a streaming conditioning payload.""" data_info = conditioning.model_kwargs.get("data_info", {}) - condition_frame_info = ( - data_info.get("condition_frame_info", {}) if isinstance(data_info, dict) else {} + if not isinstance(data_info, dict): + return {} + condition_frame_info = cast(dict[object, object], data_info).get( + "condition_frame_info", + {}, + ) + return ( + cast(dict[object, object], condition_frame_info) + if isinstance(condition_frame_info, dict) + else {} ) - return condition_frame_info if isinstance(condition_frame_info, dict) else {} def _cfg_guidance(noise_pred: Tensor, cfg_scale: float) -> Tensor: @@ -1187,14 +1217,19 @@ def _condition_frame_mask( device: torch.device, ) -> Tensor: data_info = conditioning.model_kwargs.get("data_info", {}) - condition_frame_info = ( - data_info.get("condition_frame_info", {}) if isinstance(data_info, dict) else {} - ) mask = torch.zeros(batch, 1, frames, dtype=torch.float32, device=device) + if not isinstance(data_info, dict): + return mask + condition_frame_info = cast(dict[object, object], data_info).get( + "condition_frame_info", + {}, + ) if not isinstance(condition_frame_info, dict): return mask for frame_idx in condition_frame_info: - index = int(frame_idx) + index = _condition_frame_index(frame_idx) + if index is None: + continue if 0 <= index < frames: mask[:, :, index] = 1.0 return mask @@ -1335,7 +1370,9 @@ def _get_tokenizer_and_text_encoder(*args: Any, **kwargs: Any) -> tuple[Any, nn. ).to(device) return tokenizer, text_encoder.eval() - tokenizer = AutoTokenizer.from_pretrained(model_id, local_files_only=True) + tokenizer = cast( + Any, AutoTokenizer.from_pretrained(model_id, local_files_only=True) + ) tokenizer.padding_side = "right" text_encoder = ( AutoModelForCausalLM.from_pretrained( @@ -1385,7 +1422,7 @@ def get(self, name: str, default: Any = None) -> Any: def _vae_encode_ltx2( - name: str, vae: nn.Module, images: Tensor, *, device: torch.device + name: str, vae: Any, images: Tensor, *, device: torch.device ) -> Tensor: if "LTX2VAE_diffusers" not in name: raise ValueError(f"Unsupported SANA-WM VAE encode type: {name!r}") @@ -1400,7 +1437,7 @@ def _vae_encode_ltx2( return z.to(device=device, dtype=dtype) -def _vae_decode_ltx2(name: str, vae: nn.Module, latents: Tensor) -> Tensor: +def _vae_decode_ltx2(name: str, vae: Any, latents: Tensor) -> Tensor: if "LTX2VAE_diffusers" not in name: raise ValueError(f"Unsupported SANA-WM VAE decode type: {name!r}") vae_device = next(vae.parameters()).device diff --git a/integrations/sana/tests/performance/bench_summary.py b/integrations/sana/tests/performance/bench_summary.py index 944349ecc..195b181c9 100644 --- a/integrations/sana/tests/performance/bench_summary.py +++ b/integrations/sana/tests/performance/bench_summary.py @@ -112,7 +112,12 @@ def _expand_generation_records( for fallback_index, generation in enumerate(generation_records): if not isinstance(generation, dict): continue - generation_index = int(generation.get("generation_index", fallback_index)) + raw_generation_index = generation.get("generation_index", fallback_index) + generation_index = ( + int(raw_generation_index) + if isinstance(raw_generation_index, int | float | str) + else fallback_index + ) merged = process_metadata | generation parent_path = item.get("_path") if isinstance(parent_path, str): @@ -389,7 +394,7 @@ def _generation_ms_per_frame( def _sum_optional(*values: float | None) -> float | None: if any(value is None for value in values): return None - return sum(float(value) for value in values) + return sum(float(value) for value in values if value is not None) def _render_bidirectional_markdown(summary: dict[str, Any]) -> str: diff --git a/integrations/sana/tests/run_flashdreams_common.py b/integrations/sana/tests/run_flashdreams_common.py index e11ccc20d..be1933275 100644 --- a/integrations/sana/tests/run_flashdreams_common.py +++ b/integrations/sana/tests/run_flashdreams_common.py @@ -251,7 +251,7 @@ def _prepare_image_and_intrinsics( target_h=DEFAULT_VIDEO_HEIGHT, target_w=DEFAULT_VIDEO_WIDTH, ) - resized = image.resize(resized_size, Image.LANCZOS) + resized = image.resize(resized_size, Image.Resampling.LANCZOS) left, top = crop_offset cropped = resized.crop( (left, top, left + DEFAULT_VIDEO_WIDTH, top + DEFAULT_VIDEO_HEIGHT) diff --git a/integrations/sana/tests/test_smoke.py b/integrations/sana/tests/test_smoke.py index 2489a41fa..c57358ad7 100644 --- a/integrations/sana/tests/test_smoke.py +++ b/integrations/sana/tests/test_smoke.py @@ -17,8 +17,10 @@ from __future__ import annotations +import sys from pathlib import Path from types import SimpleNamespace +from typing import Any, cast import numpy as np import pytest @@ -28,9 +30,9 @@ import sana_wm.refiner as refiner_module import torch -try: +if sys.version_info >= (3, 11): import tomllib -except ModuleNotFoundError: # pragma: no cover - Python < 3.11 fallback +else: # pragma: no cover - Python < 3.11 fallback import tomli as tomllib from sana_wm.conditioning import ( @@ -250,6 +252,12 @@ def test_runner_pipeline_config_routes_runtime_fields_to_components() -> None: ) pipeline = _pipeline_config(cfg, quant_backend="torch-fp8") + diffusion_model = pipeline.diffusion_model + assert isinstance(diffusion_model, SanaWMDiffusionModelConfig) + transformer = diffusion_model.transformer + scheduler = diffusion_model.scheduler + assert isinstance(transformer, SanaWMTransformerConfig) + assert isinstance(scheduler, SanaWMLTXEulerSchedulerConfig) assert isinstance(pipeline.encoder, SanaWMConditioningEncoderConfig) assert isinstance(pipeline.decoder, SanaWMVideoDecoderConfig) @@ -262,12 +270,10 @@ def test_runner_pipeline_config_routes_runtime_fields_to_components() -> None: assert pipeline.decoder.vae_decoder.config_path == "local_config.yaml" assert pipeline.decoder.vae_decoder.offload_vae is True assert pipeline.decoder.refiner is None - assert pipeline.diffusion_model.transformer.checkpoint_path == ( - "local_model.safetensors" - ) - assert pipeline.diffusion_model.transformer.offload_stage1 is True - assert pipeline.diffusion_model.scheduler.num_inference_steps == 7 - assert pipeline.diffusion_model.scheduler.shift == 6.5 + assert transformer.checkpoint_path == "local_model.safetensors" + assert transformer.offload_stage1 is True + assert scheduler.num_inference_steps == 7 + assert scheduler.shift == 6.5 def test_streaming_runner_pipeline_config_routes_runtime_fields_to_components() -> None: @@ -294,6 +300,12 @@ def test_streaming_runner_pipeline_config_routes_runtime_fields_to_components() ) pipeline = _streaming_pipeline_config(cfg, quant_backend="torch-fp8") + diffusion_model = pipeline.diffusion_model + assert isinstance(diffusion_model, SanaWMDiffusionModelConfig) + transformer = diffusion_model.transformer + scheduler = diffusion_model.scheduler + assert isinstance(transformer, SanaWMStreamingTransformerConfig) + assert isinstance(scheduler, SanaWMLTXEulerSchedulerConfig) assert isinstance(pipeline.encoder, SanaWMStreamingConditioningEncoderConfig) assert isinstance(pipeline.decoder, SanaWMStreamingVideoDecoderConfig) @@ -311,21 +323,14 @@ def test_streaming_runner_pipeline_config_routes_runtime_fields_to_components() assert pipeline.decoder.refiner.refiner_gemma_root == "local_gemma" assert pipeline.decoder.refiner.kv_max_frames == 17 assert pipeline.decoder.refiner.block_size == 5 - assert pipeline.diffusion_model.transformer.checkpoint_path == ( - "stream_model.safetensors" - ) - assert pipeline.diffusion_model.transformer.offload_stage1 is True - assert pipeline.diffusion_model.transformer.num_frame_per_block == 5 - assert pipeline.diffusion_model.transformer.num_cached_blocks == 3 - assert pipeline.diffusion_model.transformer.sink_token is False - assert pipeline.diffusion_model.scheduler.num_inference_steps == 4 - assert pipeline.diffusion_model.scheduler.shift == 8.0 - assert pipeline.diffusion_model.scheduler.denoising_step_list == ( - 1000, - 500, - 100, - 0, - ) + assert transformer.checkpoint_path == "stream_model.safetensors" + assert transformer.offload_stage1 is True + assert transformer.num_frame_per_block == 5 + assert transformer.num_cached_blocks == 3 + assert transformer.sink_token is False + assert scheduler.num_inference_steps == 4 + assert scheduler.shift == 8.0 + assert scheduler.denoising_step_list == (1000, 500, 100, 0) def test_sana_ltx_scheduler_step_pins_zero_timestep_tokens() -> None: @@ -367,7 +372,7 @@ def test_sana_ltx_scheduler_matches_diffusers_per_token_step() -> None: shift=5.0, device=torch.device("cpu"), ) - upstream = FlowMatchEulerDiscreteScheduler(shift=5.0) + upstream = cast(Any, FlowMatchEulerDiscreteScheduler(shift=5.0)) upstream.set_timesteps(4, device=torch.device("cpu")) torch.testing.assert_close(schedule_timesteps[:-1], upstream.timesteps) torch.testing.assert_close(schedule_timesteps / 1000.0, upstream.sigmas) @@ -438,14 +443,15 @@ def test_sana_transformer_keeps_conditioned_frame_fixed_with_generic_scheduler() """Keep SANA conditioning out of the scheduler and inside the transformer.""" scheduler = SanaWMLTXEulerSchedulerConfig(num_inference_steps=1).setup() transformer = SanaWMTransformerConfig().setup() - initial_noise = torch.zeros((1, 1, 2, 1, 1), dtype=torch.float32) + latent_shape = (1, 1, 2, 1, 1) + initial_noise = torch.zeros(latent_shape, dtype=torch.float32) initial_noise[:, :, 0] = 5.0 conditioning = SanaWMStage1Conditioning( condition=torch.ones((1, 1, 1, 1)), uncondition=None, model_kwargs={"data_info": {"condition_frame_info": {0: 0.0}}}, first_latent=torch.empty((1, 1, 1, 1, 1)), - latent_shape=tuple(initial_noise.shape), + latent_shape=latent_shape, cfg_scale=1.0, flow_shift=5.0, steps=1, @@ -539,7 +545,7 @@ def test_streaming_transformer_commits_chunks_into_prefix_cache() -> None: class DummyModel(torch.nn.Module): def __init__(self) -> None: super().__init__() - self.calls: list[dict[str, object]] = [] + self.calls: list[dict[str, Any]] = [] def forward( self, @@ -549,14 +555,18 @@ def forward( **_kwargs: object, ) -> tuple[torch.Tensor]: del prompt_embeds + camera_conditions = cast(torch.Tensor, _kwargs["camera_conditions"]) + chunk_plucker = cast(torch.Tensor, _kwargs["chunk_plucker"]) + data_info = cast(dict[object, object], _kwargs["data_info"]) + chunk_index = cast(list[int], _kwargs["chunk_index"]) self.calls.append( { "noisy_latent": noisy_latent.detach().clone(), "timestep": timestep.detach().clone(), - "camera_conditions": _kwargs["camera_conditions"].detach().clone(), - "chunk_plucker": _kwargs["chunk_plucker"].detach().clone(), - "data_info": dict(_kwargs["data_info"]), - "chunk_index": list(_kwargs["chunk_index"]), + "camera_conditions": camera_conditions.detach().clone(), + "chunk_plucker": chunk_plucker.detach().clone(), + "data_info": dict(data_info), + "chunk_index": list(chunk_index), } ) return (torch.ones_like(noisy_latent),) @@ -684,7 +694,7 @@ def test_transformer_predict_flow_applies_cfg_from_conditioning_input() -> None: class DummyModel(torch.nn.Module): def __init__(self) -> None: super().__init__() - self.calls: list[dict[str, torch.Tensor]] = [] + self.calls: list[dict[str, Any]] = [] def forward( self, @@ -1487,8 +1497,8 @@ def fake_gdn_main( def fail_softmax_main(*_args: object, **_kwargs: object) -> torch.Tensor: raise AssertionError("main attention should not use softmax") - attn._forward_gdn_main = fake_gdn_main # type: ignore[method-assign] - attn._forward_softmax_main = fail_softmax_main # type: ignore[method-assign] + setattr(attn, "_forward_gdn_main", fake_gdn_main) + setattr(attn, "_forward_softmax_main", fail_softmax_main) out = attn(hidden, HW=(1, 2, 2), apply_output_gate=False) @@ -2273,13 +2283,16 @@ def test_torch_fp8_linear_replaces_matching_modules() -> None: include_patterns=("^0$", "^1\\.0$", "^2$"), ) + first = module[0] + nested = cast(torch.nn.Sequential, module[1]) + third = module[2] assert converted == 2 assert skipped == 1 - assert isinstance(module[0], TorchScaledMMFP8Linear) - assert isinstance(module[1][0], TorchScaledMMFP8Linear) - assert isinstance(module[2], torch.nn.Linear) - assert module[0].weight.shape == (32, 16) - assert module[0].bias is not None + assert isinstance(first, TorchScaledMMFP8Linear) + assert isinstance(nested[0], TorchScaledMMFP8Linear) + assert isinstance(third, torch.nn.Linear) + assert first.weight.shape == (32, 16) + assert first.bias is not None def test_torch_fp4_linear_replaces_matching_modules( @@ -2319,11 +2332,14 @@ def from_linear( include_patterns=("^0$", "^1\\.0$", "^2$"), ) + first = module[0] + nested = cast(torch.nn.Sequential, module[1]) + third = module[2] assert converted == 2 assert skipped == 1 - assert isinstance(module[0], TorchScaledMMFP4Linear) - assert isinstance(module[1][0], TorchScaledMMFP4Linear) - assert isinstance(module[2], torch.nn.Linear) + assert isinstance(first, TorchScaledMMFP4Linear) + assert isinstance(nested[0], TorchScaledMMFP4Linear) + assert isinstance(third, torch.nn.Linear) def test_pyproject_entry_point_matches_runner_literal() -> None: From bc58ce39b111d18a2ce0246d54a9b1d0f196cd5d Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Fri, 31 Jul 2026 16:51:31 -0700 Subject: [PATCH 64/64] Minor model card update --- docs/source/models/sana_wm_bidirectional.rst | 6 +++++- docs/source/models/sana_wm_streaming.rst | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/source/models/sana_wm_bidirectional.rst b/docs/source/models/sana_wm_bidirectional.rst index 51e398429..dbf8f9d18 100644 --- a/docs/source/models/sana_wm_bidirectional.rst +++ b/docs/source/models/sana_wm_bidirectional.rst @@ -106,6 +106,10 @@ Optional inputs and knobs --num-frames 161 \ --output-dir outputs/mine + Action trajectories are fitted to the requested frame count: shorter action + strings repeat, and longer action strings are truncated without materializing + frames beyond the requested output length. + - ``--no-refiner True`` runs Stage-1 only, for diagnostics. To inspect all supported CLI arguments and their default values, run: @@ -133,7 +137,7 @@ GB300 GPU. FlashDreams measured 34,182.39 ms per clip versus 56,932.83 ms for the official implementation. In this chart, ``Official Impl`` means the pinned NVlabs/Sana upstream -implementation measured by the FlashDreams parity harness under matched +implementation measured by the FlashDreams benchmark harness under matched settings. It is not the SANA-WM 80-scene benchmark result published by the model authors. diff --git a/docs/source/models/sana_wm_streaming.rst b/docs/source/models/sana_wm_streaming.rst index c12e4b123..a0e0acb2d 100644 --- a/docs/source/models/sana_wm_streaming.rst +++ b/docs/source/models/sana_wm_streaming.rst @@ -109,6 +109,10 @@ Optional inputs and knobs --num-frames 241 \ --output-dir outputs/mine_streaming + Action trajectories are fitted to the snapped frame count: shorter action + strings repeat, and longer action strings are truncated without materializing + frames beyond the requested output length. + - ``--stage1-precision`` and ``--refiner-precision`` accept ``bf16``, ``fp8``, or ``fp4`` when the selected hardware supports the requested precision. @@ -137,7 +141,7 @@ are excluded from the headline metric. These GB300 latency runs show the official implementation faster than FlashDreams for BF16, FP8, and FP4. In these charts, ``Official Impl`` means the pinned NVlabs/Sana upstream -implementation measured by the FlashDreams parity harness under matched +implementation measured by the FlashDreams benchmark harness under matched settings. It is not the SANA-WM 80-scene benchmark result published by the model authors.

CMQaN=QW=e2qM{S(T;rcIV z$ojcN3LqB*mBpM$^)!(%4^BwMx$Fxsu{F}(1?hdu#;*Phi^U8Ju?ZruzFc|WnK>_` zHRs7$?P3_r4%*{ZtnKg~9|46f{XU36Zf!LDRzpx0>ydopoDuyqkw^UPj+F6F_88{B zzjCUyvt1BTeqm#y{@602w!_1(hvC4v2fkpmO1c6{XqIO~!X^aa5ie6IuSbN43^i~6 ze(xG|y*?Dpl08sSOAbPC>jV&MLV_lXOp7E=U3Gw(R%N|!HObyl^rS(1 zOSu+uSgkIPeX|P%JX%6K3WbnPyCX@WCUG-$T7h$YhvJ5NEv$KVYPZub3iMb7(-BfW z3Qz9(Rpix@&E~M~aT^&NRuwpoXV&HW>714vpUC%*Y=q;4%5LKBJtn zjK#~FVn$fVpU1egZSA9p6NJWY*J13Q$^=ZMBzaI9qyc=t)D?4ce1lNc*3im5GSlXwK%Rn+0U z8>!Mh80r!&Q4Vr@UeZUCOH2@(;=4ZAnvL#}inpZWA_K2l9OfN2@6FgdCNes;kY0R) zBsIN1ETS?Jwd8!DLh=OtE4vkI04))y?-JBW9O-ytg`OF1OR{IM*PL1Pq-Mn5nvfvT zR9wNak^BbmEcw)?b)ZW0Wx4M6sJX}0r}EM5=Y);4y#f__hfY*9Wr{Y;n!7wqY4(GyJX1kA>TB;iZD2_>!74; ztb0yU?+i!YF_`-ptD|opa47oR#3aI!Z(}#Uru90Z#R*J0RM7fBS!NJ(^QZ+}O%+Ae zp}R$wZxXxnD!;a=juqtrr%4f#WWwHI=w*AEGzrN(wFlng6S$gUckvJRdIBqo7S5D| ziq3PbvDcq-&{|0AlkoU#XpM55B1B>TJ z)9<=;o!~97y~O!t)gS8!aXK-Avi$xhh@qFTvu%=tK?f138m;O!y{JgOd$PtOz6s0G z1wU&sp8|aGxD}QqoOzndz7_*BS#OkYpN~^B^GBSC>FBtfAFT_UU;vs$^iFT=Y-h@w zH2;Fjkyc|0cwUn(WQp=m%a)Q3vEomGZmYY=u~1rCwn|I@lt{~C8<;RI|D6PNHQbq9 zF*w%C3bD8)#AG=la~bfNtggAO*B_Y3tNejzJnRxOP`K#Dj9CsWJi?3VZFDj)Flhv$ za#>x)xx1LQzQX z*ws`A!txeXQ@i0D4KhHSl+y*CpC${7!}L_64W;}9fm`QDU4=cwH>J*z(9sH?c9WmC zRQx#k-s~KVp(J`9^Y1s+zC>QG5|@dv$K%{c6O>ZadFjV~ql#hkbA2^x3T$2sxz43q+i-6@ zR22}rRZO>ubMa8M^omltd9Js@D1X*Xirns|RA<^e$Y${;p!?FxM8F;YZ7jSj%Hp3f z_C1N=fHYA#P=ZhmUu~5vE7_tuYJrPIS-G$!A?kamr+(UXPkyV!0?G!NB#I&nBwYd; zcb~lKilo@3hALG^#praen1Ik|Ft5Ae1eK^Eo^S^va7v%YN;Uq(mUmq2o!(YG>vWV5pSnxS$ye1eA0E?J-PX7waYR&}h_BOQ#+|UFlC3S<7eRf)GK<&4ZSWchIlW*q_y0D#|@PQ{f)Ke1NHnDwieuEZI)Wy?W<_mdlq zR)9bjYW6*-VfM*8{Sl)EAqDa$rOZ+iXbkUxjVzWB*1w-;K3EYnM?fj(XJg}n7ol^_ z+I4+faHH8ZoSd8RNc(e$&l_?bt*9K8Y->q5ScW@i7E0SOw*HsaL{T0;DFHyb(M_2J z8Hi-NI#JQ_9{C5pRh4Xwda=3Z?%5TlGDg*jPaBRw3*2qQlKWxA#p#ddBX_8vL57+9y@H{jvTp>AzfA(qp5j;K-cXQL60ph}K zyBB*V;O+Ey48A=Nbw;K_gONi_LB8tNGC9cSF}p6|N=gJq7G?*18&6^P7lmw_&=FtZ z{Kn;90$sOZ8QXRGeFXj)pL`BDdU~5_=bNbTq<2zEXT7BbH=yHx9-n5mv!}+KPW^q_GD6O6e*O+tux-u$4rXphvrk0U@$7OFKM$#vT>jwSedYKo9#f!38zDx z5*{r>w7@TOkP9-__>$KgqJ>Xb!Q#BMu9sXGWnZZ>E00^w2f*J}mzoNzOALNKXQA+uy028)oa zod7bnS%R9_fJ?&R?(M3&cb>H6!|*6C3R%C)dBR#3iBg8iE%VOnuP!Wd2?W=VjPd7m zs*3@?fu~Rpv~K`_69+gI>7#3MYb$Zb8Q~R~q0qk}5P??5dVK`XuEJT~&g-mWiy=y< zT&~W?aB!g3T8VG2uQlbVT+veHWc_npX%mV^>A$w{?`al$%w)?In6l0n(?A?OTdZvR z=+rnubM)gRYuA;(b{`Ya%Y_mF^-kZScqUReU9N*X&fDK$Z3DiC5{B_%LsrZh-Mk|> z++`9};Ls~jsU>lZ2^2DFw`M8yhvp?gI-Z+7p85^pUKytWDYh-s*i`XUy~1cj5l$tP$(A<#z1 zN~>n0i^GPiInIhnWgS6Ze6ExP^bYFxDGuKn zol2PJf^G%KV&UM+AZY)EpBF`r1qur-VmsmqEm9^ z=aS>Qqz1ntk#u)A)8W~3PvP50aeZ=Cqd_(QA6Wlk%t=q!Tn`IXuIR&b`av0MGoON- z^Kd;6p7N6X!EBI2aYABo4{$gCS+fK^SO74nYrdrW$Jrb;jgOMWx0>|Pq-K4SSaiK zsNLOLxokgJ-aRToMe`(ljo@Sg#h#)-wT!7TLNs?1B4&8=qmH$I1gA7H?$svzG45swDY*+H;TdR!PX7PWo+fo zVfGJF*I3y0m*&B<8sn0da~wGF1}a%Zq0{{OiF{`^IQvaJ{0o@1lyQI$xP&t#=U?%H zKAx{VPOI~@L2E}m%9NQoTvEcqFQ&OE?DqId4XQTAbrmek0Sy&=CLp3471&e)-eM-c zO4pa8AHteV5I0H!5umTj(f7b0v!@ZK;SVZKLkb%en+?B;W79{XGyTvS1?7#YImoTkHpMB^JVHw8opHKgZV{(Ymi zOE4@+eev}za!#a`dW2w=je}R9oXM&yk*Jo&)l0e2^yTLO%=HF0%xDQR+{@|ZU=d)& zMe??=&rn0zN39c%pVftG63p;gni@bmo)34rhT0EKB?5K`${GKMI`JnJO3^t5Jvi5I znzjqV;#L|aG8)eik6Q^$aOgE1=FNv7|H-S|&K?d8s4?p&G~{2@hGlX*NyuZhu}P5UBoLsOyRb*3CBDc*G4 zlHN63~*|os9SiRlZyXm1c%|*|K68kbGU5yq`@PP5#Z>TwtywQgv1U-PUA=9UI z!B}H4DI;HtwXE~Nm#^a3PyCy+Y&BOXh7c>wA7s)W`ljYn6D?7daqD3+t7Q@VlE|T} zrn>QEO2;J#?Fd~le}kghCn(ddcoU8CyJ$!ikaJ$*L)K)I`<3a8G{yb<&z)BA4;W0g z*$prt@a@RUP< zKXZRVmgAj(er3?-JXkE*J?FxvSU6?5$5&k3#of5d`^$^M$Z@(gV2w=no8 zum}|sTUb^BchxRreHBdT|3Tq0y15WXlaCfppnaD@q`HtxdnA@ds&tK{o-rfg4A5?I z5!m1>@Fw#AB1jxAc2@q+TXKZg!1qANVcw{i1(GR(FH$8sV&f1ABW~Y(c%;SKMsv_& z!vjWCZV_DQYF6{~gwN1qH|+GQ!YfZVWXayYh0?Y`F=Yc{__=68u3ux(YW-p0wi?7?;_B#w5eSf?miOd?8L0v7qCwt2SE$Q8jAF~dU{ z5Ww&tCWRx2gL@oJ z`MxL}bv?g?H5jXagStju1i`35hFbB%^C^R2ZTjrZ*~2*1FLC%*$r&bBD034{stJA2 z&`~}9o~}@sWgRjzIkWRhuN6cHJ78XH%+5bau8Dh-XAY0nwwyQPVNO_zDxkb%_G#DS?HSFx z+Uq!U();6Cbs9jmYV9Lvl5)>SU-(CKJ}O+P7vB6Yy?m- z==3mQt0FewjaCaL1|$vQcI6LsyOe5gLorUaVK_iCzk*EYA0o`3wODXfAt{9E0~34J z7nM8ql4dgsN&k`&EBSu~jMNQK;|6N?>+TQnjyI#QM-;@m{b*~g8g60gNiw!SZ;uwfvk0_WVZ=%>hcK~kE{cck(*}RG z@)?NNUn^cQ-s_j712{l#@nNMrhMP5-TJ#{E=WB$@;%Ju>-GGHiD?xL#Z1>ae`_iZ}NmT=~|3w+G!~o?-?=WSn z#r`FaU zm5Qmr%&s8u7M%vp$nd<32>4_-xRv;Gsq7@oM|D>_%PI8}c#AjwLAUNGQH>DkI}YTG zM)E3iMV;G$fi;^9mxLNPxgWmI>mI4%!lTXdQ|qld?9nhDw;%~vSC*BbJZXBJimWu> z%;V$8Lr$=nk$zzIviMIFq*Ic*4fnB7K6kY)9kX3#U>50(z@<(Zn5~h=0hez89byG` zxF1-cV`i>;#8|GM3`=+ovnOd=f;~GXv0kJK7OsQRdQgkpZWF=l7ZSPt(u$*j zYT2kWD z(Gx=aR%Dd5$!ZDcH#fWVuSCt-XT?=t~MoTzpIPNN! zO3>mCl)*DVkB3=kh8C}ccPb!NBJn|5`zK^ulR`j;Z22gmaV+P5KvflFo3BK)F;ktm z=tByVZ5LQ;S|*cioRM>ACV`0VQg>)%6Z73^(>z@B{15hv8W1++DY2G_U2h>)t5MJ3Eq!B}lOsA;8dWoC4fJYpbiwCof156TVrGey z>wZ}1|5PLx!u&y&U$Qa#@Sk77ALt1=vyB-8bRW~G4>w2M+RxY`g{__~J5lW5q^tViK78@FAl9OaDU5>cAw6FQNw+b)yHWqAd0tKhH#`Ru zBKI*Db3?i>B;en=WeRYEFE}u7YR*yXh78k!&5c$7WSIv`m53ER3JOgtDn}@-j zgz?%kzx*xKrg-d656Ni>Advkj5(JB6k>5JeOu<5&uxC&{DWeKN+FYXR?uW6+HWglG z#OalA=Dl~}IU}z;>fo>dtr8x*^FaR3N-^e|1xCGV%P`%Kf6Kz~sRkty^E>TsX_H~r z5z5TNBFYM}6HN4k%O~_WUf$t0%3O^@?Lk|6nEdjlz#nM-^`PMuXJ!3Vs1j$?zlVaa?$M3y2wjq>|LPQ-Me0DI$C(BZkQ zp!ZX((QzOReu!vvk~Mof-VJ!R-UzD3Dd<`17fZfGEtjE0m%z z2J5TM1m+Y8EPDO|N&^zQxnFdLp=rQ~FJfZ&CI(wFQ@03|ko0&&^`nS8S*|$IP92II zw=S7zX6g>zhv>&pf}4&4uxuM-teqBPggvCW!0bJj23)Bj)}nfu8eJ3Um23&=I2&FxZh99Mp{ebh${seAGF1+75r29Ye^yxQ z`a*U7^u!=G5vsTfD?N*54GsV}rl(|_;-g9oLmBE#L$kbSQ&?++NF?FF)`;A~GW8n;cPw#o*?{)X5)6I>HZu-z3J59RFPSXx%c{i)b zwC{3eJRxGkWr-jM|0ug8$^npHT>^flybIc$W&V3aybQYF=E3X0Jt-)0<~#Do7WIa6 z9p0lq0-TNvk#-#6n6mUs5W)XYc9lH%9w+SJTy zS`MZHxRoJiU<}7OPPW3}V;_r%K!`B0{L`|;F!isBp(Ed7cIl!HsQlChdDtn1kvnH@ za!zZ*u9>9*r4tar2hcZV1U$If%w8On=&8Nh;ALb;TTLQXqxoQhdljGuy>0X_q(aOr zfWzxxap|G0)hJRC=fw0NvXjds#1I13DvFr2nEabI&#R4L%_*5Y!Yzt(0DE)b6&q`e zq#Gc}@t~UAND%|!ex+(JlAIT316&|_`6R ze$8n#2@=(l7qxF$0=4+_(~CzVN^ezUCq||vh>FOVIf=yb2yX{Vdto}^wg^fkG`dts zBe};(1qDfw8|vJgA!pVj%q|UWEd^=Q->v$CrQd^_|K6VQU**x+&*tder|4PYwKlB~ zpRLM>6#yDq>b3l1eKCuI1y52I%`v^EkTw5hh|xR4in=LT1ovh zo}dU%RN$Mt@y8xpesvf9SQzAYOcBP}lI)n|IjAuav%z5(;MFaJmCq#C!H@pg=ezkV zh&14i8cURbcWV8ai`g{Bsw!H@*t{32pS2*de1$lnVb|uuWDbs8w5g`in$0zK59|OL zk(Wwb!0E!(|Jer6Ve?BB*B$#FrRhn-iH>fek9u8&3f@V18w> zyqzdfR7Yrl%rHmWKWWL<=`A5ehgwhX2r8kS*1xU4r7kYAl^`ocGx{gGMYqC=%h%HL zo#x+U4MYhHjOpmed#t#wG}uoGCI z?FX2Du)=NnBtZRz!XQKo$UPJv3WmmG!}@1jXPeo~Mu3n-Ym-4ZK_ z;4PnN91b)ETW3(s=Ya{TGZ zYpOr1+kMfc@pg*YL&p>PySO#95Wr}uT=8!1&9KR7W=1-j+gm9mel@l1iyWdty$(b# zewv5%-{t1(t0uq|Ml)EC8pt1pcPS|$^oB&~Ro~oY3O~reC#|M)B1|f3zuY3uVkkFi zHf+NpK+evNgGVo3EEa@KggKIwkB|*idO!Lfi*^8y1-xjZTm1Le3a<=m|3b=0(scjo zir6Ih9X#_r@>S0X2LDHLx*b1e&bPKn28P#^i=d_!$nU z7$uoHd)FLZD$^@Nj{(hOpczrNEUhrmy$2##JlP0SuU9rp;WQSp}FhC2Q+pX1em( z;ivMv|M5TEi!%p+j0tB{@tqZKqOSDnhZtrTx#Q-BDkZ@$&hp&cUj9qb=MMPtfK5sJ zGgef0_bQYS9cd5*8EHPM$$Nq&48h(*EE&&o4g~`M5Cs4D@$q($^_NjvpK*~i6mBXn z;f4s`oDWfun z4JS!t%Mn2f7x1Q1f)w`2W3PQO`mr|e$#BJ^m#rUZ=@7&9zpf(ZiwFnJ_;G%(MKLll zfa_3E^dmpt@oY4we>rP~8B@j1xA+;h!?TOr-`{afD8g|=CC}!wy_!5y+gUF#z^?Ra z5oQQq>d+JpzBsH5sw}800;xOw`JM9UtrmgngnZwM%DEYu?y=B^y?j^-hL*o*7 z@4M~GPW}3+&`CE*lBcibY5rgte05K5yEK#Z z@nkU;_Wz^JelWkXGB`akZvo$j}x9Pb8-kFbSF6S*RUy z7U-96H{%|vN(FL#SsD^Dmy7H&>r_tdLysNAAPktD5?=j)T5|JDvJ z?bS?*gB7d5ccCrF_6KP291Qr($eDGg_(Rlmm#8?A;OH`LWq(N1D`AY4*SM>ot6&0p zyMk~I2_RxznxQ$EBAY<59WH@Bs&w#*!XcQ(%xC@$lRgSehsyp%m8%_kM6l@(7)Zyf zns5bH=&4}f)6#dOaDrJCE_{A157J8~nW!~a#;V-E!?J`d1lsX?`C77@wRlB2bWWzd zcGIvv5ZfZiIK2wQpz`h|_kEbyjjQ*SAO;)^^dEkjXCXv5Enu+kGOt!%`eH>bP)!#1 zF~Rvr>9wQf#xoCGZ|4$#+>A(TS`K!DRIPXFc%90T^K+4wW=hBu1@hQPGjxZg{N`5@ zzE?$BCt>GX!y9vMFf5IPNz+#=n(v;9)5SZPb91I^;ZCyRh;5PcT4_uhNp$++S7B=~ zMU=NISP)PvxcolRSmb4nUQdkgBW`?jq_9Ymqq}s&WftAN-goGxu5uM}&twoAeh-vX zkEa(oW!DAaF>6`+`vSmf^$VKV-NA0QWG zTFI-smig}~sC~d+*41?`8ski}1($pqY4qRZIXj=0jX4MZ#(J}oV=@6%mP=`0mKY|@ zGaE%LCnFkQPVIm@Th8G9=J9P1surO9x^@m_X+)_7NoHhDQf{1Jh^NCvW>l4*Su2r} zV5hA^XeEs)czFzzPf?oFKQ-Kzq3Wr7w|-Hem3Z@BeUwCiSbV27L&O2c8n|UG_{gp%c6-|Q zA#*LdVC1ea$lNxQDiVQ}v97$5v2c%|f)${f#125CXN3s?;$ZNGC56SBDnNo$FWP{B z2?S?B(Mqivr+p+give8Pkpg;=FdAk<0TTArsPKL>Uly*@`ln^yoo4 zlJ#@V<$lM-_}fEKS3sH)((A}Sl+?$I){kQLAJ?0YGUNRtc+0+NbZ%Y&Y+F5>tLZ&k zk7cNew?`}lqZ>V`d@}--$UAf-*EIn71*V@PAl>SyIR2Cq{XSX`6qudD*s^73t@6jr zdXcO%rkqYf4YC0X zk;%Y!SGy~z=L4$vU1g;#>zqr+uk&^99WlF6z~q;pdo7=gl*Z_u zH#dAX6AW39LzNvK%&D6HOElck-t|V-@ZEm>KgNC~_$}q0PdggEub{IH#A5lLb%;vL zWE+b+Cus!!{vjDYvz_=fZ$NP;?=BcpBnAKZm;#Dkh!3Yq;5!CTfKIXw87pghBl`QT z(=6a)G+quRj3oQJA!ar15O)aV1_vc-(CulMNhy2+bu!~C&-A-yI-?h#_39qCY8RZ| z3;dEH$?fA#HTZVZ$jI zZ}l!bcXnkAd{=%*DJOLF>jl77QSG>M3@|#?jQ$6Eu11~{V%xMPMkbkpq_~BOR9Ev> zqawq-nWmVG$SPsCs5<#_;K+n2ql_+EMI_aLJB36#KD82eZ7U~!N+JmeHkt}|&_Pwp zxAF~Lx0-Aq>|a~!de%U1en~qrc-l8j75i1>ZkMkN2k)A4_Z~d!&HKvrSlp;c1PZ@Y zSs|QRrDySOVNrZPzakG9GawJqIKhU}5elpP0jvIT>Gor^;747o?+)Fm9Y&)GTtgJD zn;BG_8w!nMzVViQWHdjcIPYn@+N}-hUb!X=&g`j92a7mjvf(=RQ5(Hg`21X2#p1yh zWXg!_9|j9UM%xTau)v4SEQC&K`f+D2<;kH|60ZBoWS|4uYCO^f4Hx0T12@ssMo+)z*>>bRMmEyWSm-n za8p+w!e)F=PxNT=aVaJgDpB%hKd$;ZQl-$KuIFAM#@L+@&TB=`(GlV`g^q2QKWs!v z9)S>vx@lgoV$;*NOpm+2a~~@I%VfH;qbvBeGmQeQuC`UnCeUBJ$_lt5(=5z!MW6dV z-ord8;a_EUAK?4<#-~>GAt==_Nk#rc|L;Wa#&^DcT)^Vn!9!A*OZ<360}ijZWs|?q z2W66%0j%IECEsP&*Y|uZ9lb_}&t`-sdFmEU7+?&~g`V@YnSwb8T8Yu9I^F4I7fXsJEJgOhPxed`7b@@LLdyF`OS zN5OX;PBJ9yDRAbedj&TdGh%Ao(~cQgJ|Ox46Z1XfigqT<^oJwP(kY_ZCTU&-^VXY_ z1CLng#q)#S!1uiTbd)=)IP6MdqsTS!2&>rK@T`I(NRTk&9b!iS7D3cj&%x5%!d{lK^QsCxmuklxZ|tr<~>g0qtQl5=TtmpGW93oZ<~BRh(&4&v1TGewD3 zOCD!GScN^7rHCk0z8UAOZ!!PIlZT^e(a@^*ub=Izm8neF9htB!uaD#fxhEB85elx; zc9h#JimgA#na1`X;52>UB?zN6_O?NKJ-;~hta^;C%KR`9vb+jwjW6el`@7N*eF#=y z9n9;#1|>(+)8A^nKZMfUGNela=O>dtTIeR?J(Z2=P>Y^}b!I8w9Xj7TUE?b@z=b&#=g?f0)?>Y&Rh&#pcn*H1HMWTwRx=>En8 zj4vfU;p9HGGRW6<8^ac3Q%G`gDgFGp6cV1e^?}h<`QU?BU2f}a$gkGsQ&bX&=w9W* z-un+wC?FCEw#iP|YVit1$`f5%N%24L`OF$Aj=uq?3s~@1f+x!q%w6Z`r?xqTYj)Hm zaKJ0iF_bqwtwNr&v+A;BA?$89Y7#}$Dex@4?$>hd;QyIHa0FyTR|!Y1_<*kBiWW(U zcF9i{Nc5E)MSgMl>In2|HwvzgznnVzwK$%+CD)pHHuEnfF1^4yw*w8&f>uc}yl}Xxv*)b5U>f?`vs}O1jj8;Y0%At$uOYjp4J8XPbKPeN$6ucDE-=G>9 zuQCFy0RJ_YqYoRLkc&27>Y%B;oOq-HVa~vDT%{@rK`v_(&Q&YwG(h|JrNvBbw?M9i zo={mAqO9-nMw-g7bDiM7LW049fC&E}Tf575C!ibNevLLiL77`#p-9cUAuy)D63x^u5Ei|>B(+nZME4_ds`>W%cYc}(}b5{Utgjb)*DDJo{kKi%&|on{}b zr3~2$RuEb%Papl8lJZM?!w&y80G(JuOAIeS1&?@eA*tg=Fb8;f zmI;n_DyhHWoN$Bgvrl#z{TSHen^?`hRQ$&y6~~a`P%n!n;X@r+>99q!N|a7j^U;!@ zM#18|cwEV5m$;pOsM_&e&rM>T;gz45mo(&g0BbIkgS;@oBDHBOX(k8`I}zdL%c zH%pr{G4emUGZDI3g|sOtBw&+kxzjK60qWCs);s3ELc#DT?tk@S5lRec2J2q^W7Kga zjbN*?alDn-80|(!(NqeGoJt1Js{Ci{HWY&o0%yGx`kv?EC^LhybKA>YI|b&lW_-b1 zsv(_b#74?801Y}41QY8%G|g$C+qyW}(pX)El{a10Q|CQ4Pz|nUw4BwPD$!q6G)Ogu zI-Ad0qTOJKTCqbFuU*Ly^5&-_t*8N&V7K?S!Z3VJsZ6?P;uvN#RGU7pkFn6}Dvf01 zkR^YBxgSK11y+C4J5_|VEL+m_L{N=oS9?r=4qAHTrVRk=pCXIPn!=S{lB9(KG_J1c zf;Ox;ebId50nWIfpyNsu;CX{h#K)Up=lG6XX6h=_&(#+AZOZibmY9nr89?8XdJLuk z-hH6ySv=9`$5x{gheMvsxgUqz<;c&^naP=TwsZ>CkSz2Uf`*k9WXG1R`P}7{qF39Z z{R>Rs0zQ>vB^wgeb9%-Oyj;e>5e+`0JymNTP0fV`X#{E%H?88gfpi6K6v1&g-!Q40 z`sZS$C2>sP6gwUfApO^jyPZd^B|Y{KHa*&x^~5G93tTuU5_4-@$<0%4o5X)PM+wBX zpI4ZmO9Cgs*M9O>+0r{2qvjHhcms}3)?R}#=>M_y)@go{c6NvZ6fXWttOjNYR7}?u z3`r=M+3x3c;tczAmQ=Q9Qv5~k@)-shyC*Tf?9w6$D1@`zwu48|-X|K64Su5BJWDfM zss}q<)aE*$oPN@#-v}Gg>C=t&!f_pV|5gnUF~<6%wRDm;p2HS35UTpL-vX99l(`kQ)_O9xLX*fQK+CDu3?$f{(JDRGc9PmfFhaU|MiU zSQ`Cfi3xqUuKb&l1=9<(JS(52RPS466LPsvQ`Idd)_Ll&5Q?Vdkti3nDm&C`qAVoQ zLJp-JLj3>y(ht&#gi6e}{k(Il7j2iWMJebT9<}#)=}@xF5wQ19D9b9%QJm?+SWlK+@?-I4ezUl&4%1^ct9vnKdK#kmkLi@!ZMTAB zdxfkE>he4~#|85cDMsFas$n+hJyY9^a7Z5Nr4S=~%iF`-RrW)()O%}<2g(2h2ujt8 zPA+YP9V^=FxNwT^=gDDit~BD=P;#?_f*z@kluvscU&BnOHXe+_C)IG^!^HY49uev; zUFHBB58DMc5XvOI++P#QWs^AKeller<&?(>59i=50Ul^gmn*b#^ZZ5RM|Z-pLUdHQ zfexiPijr+lFj5i91WTql!OiASZE}^1pcS7f49Z?}78=oBO(8!2{|5+H1g|9IqI~oI zc^JzE{pOZw-_w;B1MQM@Y|2Fst7{N{?(xLU645GqW%R0T*{MVj0*{RZ=AxjxxW%E=AW$d@rocV&)C zjP`yJ>iZyGjS*9NtqJjWWyY*Mw7l+&qazFXqLVGpaw$6TSET)Jwe3oc!ZSUIlN~KH zzot=UfxMjmS-^~il^l=(CQgmBSh6SkS0ZzCgLs3;vl~s2K}HCJIr_kY@y=}4W1YyB zmbw`)A{)Z*8C`J<>6=qx6(^S}QilDiu!%o{oAJA`>=diF?JOJ4k-5#_^bPLIthWTP zCUG+B*6;=>$f*L@Q`v6KYmyXOb&Yl^Nv?pUBPY$=GwVGr?w8TFGpK4 zJwaUdv@oJ)r9#xvMQ_Ww!komvcg8aoqoX~*?_%(x1ak~*0 zPu}fVe{&HR&L!?UMV71X&w^wrpF}xWxUf+X<;Hx!^g|~bc?00)e{^mZjx0eB2T${>n=ilnOL`C8rDxaE-ybcIBu0sfrf7NT({_yr6s=!Hq3`Lp}<55m> zgWa`(e*KPKHRa|*IK%hS4KSp>j!O((v>Tl(benMZj-?eQIU?3`>8E8s6$%qhVAP`# zZd%zx}lZO{76f`yE_8 zE4EN{Q||zTx7T}T_A80E-Pg0qFIl+9_!3a zV^HL3C9P* zm$J@oKaCD3&K+QYzwllSpVJZ?zrb7FwssbPDGgN7)=hH~F#%E~7V+vsiNyj9Gs z-2XG9n_kf-S9~hxF=k;c9w^v!>eCVjtjr-ShYbDJq41pLE_O8e3u)?;&eQqOy-C~H zr%~o9JFRz>4la))qo1Facf`*JJ!Qe!&;S9{0eqL<} zKNp3`eb!x@CGY!X*+ieoOVd_(K1pb#?0U`isBWW3qCO03YqxuU>+gryx1l>O!p2M-aKOtXDNL@N1e@2(cG-PeA+GSD(PlC1THn)j6uXLw6_)Q;`nJI%U)Kd;=y0$ zczRbgsTtZP#9&@3hs4s`-!uVzkh#EtGGf`25F>ST31)PRMhd6e` z%V01ux5Y|==whM9Q!cODjbLu@O54e!BGd$T8 zFej6R@bb_)sbxRc4R(s?IKS~Ut9BvVoTqxR=vK0wlG zGEht?4?&(hbJw6cd?w_6*cOQK=#>U!$HC8ALD@Lv7KUK$`tHrL z8aUc*;yk`F7kD;lb9-pVXoED z7lsSE*ktm)Zw=Ma#c3n>Et&Cif(P`oqYyY-tZ}4b&am5rfVy1o#j79S2`F5Rf+fim zi2hrx{r}2)L{SjDF{j;;0Q8`%MjLHWWRIuS`3MSuAVsw*T=cWN$?L`A-@xqN98)nD z2?QpL-~yKrRrN(OSmZIt#J*nc_=68N#A76WCQod{>Rhr=`9|ni>doT73sv<9K!`++Hiw zFk!SMBm{=9K7&6KunW&@izKk)K?2A&MdIN8#A=QR-X`B6`-^rmlT2qNjiV15o%`z6 z4GN{gagB>|dT1le7k^38zJ29odqkPFUs0&ES2gO;9Zk$pL=@#27e}N&7LKcq7()E5mj~gv2|9J@}MnI4=(M`Zm%*52cmwVInX2|{3 z=|Bfl%7pp+kU?`PL9X(ff;L&-#k0|OILRre<~))f*=uFtNMdv#>u>76XbBo~9D zFM=PvC~kaq7b)~Q*8J2@c`wXALtE)+)L=!eNLMwaxR@m>gOgQJ(ofCM!QVYmcCz)$ z8XdANwIKkd|IQ&`53mX#>CnU7X)?j9#wt7}(V8IP> z)-Da49Wh*nxjGeC-ed6r3vnda`9S#FCZZP`whV8haQZulS!;B+f;8s?Td6@%)G=zt zITw-j>A@#I5oYPBGFLmrN@C+Q@#P|$atsJr`$AP)!QJ2ikU$GST&xi$m zu5)|jF)gNYVAvKj=rK-E>1XC^p8qGvL;Y!H^Jf?fpdsTBL!^s=m4p)0G~DaAJ9w=5 zArsb_Guli#i{r1=W@2?~q=!T%KI$TK7(rrct%WFj1!}zSjQZsQG`mo3On?tqpf$?0 zKSLL6NCqL8Hn{q->N_LTz6As%QOBC8lFz*vN)9h@xR?l)3`9i--n@^#Z?#|WcbekK zmXa|r*`Ck&UBQdkaA{W5lPkFX*>RF&#JYfivj|$4`bu(NwbW$tW6CgegR4qf*~K@MtPWe+e8wXnJ^o`32ij38iInWU)Ud2;yL778+e z-civppjxyN+ieFs>!9j#xH}lFFoiyx`l_>WX;(DoBR2f#!Bh-5IbLN^~V6g ziaSku8%dkMh(dmGtlR&c@B!OEsBifex}d`j7{{F#DM(n z_wlP>nhg{J5Yh{?I1Ax4w(O*%h8-iSIvQ58*JJqP1DN z9OR7IEN;4RCf4kUvLCi}xE+FQETmkbq;x+zXPBqjBG$6q^^ukiZzjZUJGRz0@F_?( zw$?6g$CY2AtdTjwX9ouQFWr^tCPy5Td?y92V>_MgdmFW`B1hb=oqH(x*^GY;LAT@Z zUuCVD;2X1=Eq}ouz0oz%+b`;ms8ggE2{UAv+;)~3e~5+<(3Ujz|B9FvL_gSHIM9uh zP_H_ulg0Licu(I-H0NupgngsbCW*ey?n6#<)2qH@(`|G5_UN1GIUI`r!=|Z^*(M$E z?*fW5#bto@d}LXsHB^cOIG~gi8fk5x>P^x~C-_>`Us!1@$jRW*tmffn-E#W2?ct!N zqnSRMDSnVTK4p~*Qlqip;ut-~4^^cnS9i+oIPVBlvV1~D0L#IT5} zWG`cUj6ct{fP@sl+?e86~mI#68w)e!b7xi9XlU4gFN?|cviMAtty0k#2hk0Nyf ze#=4pEH|wkz*K+^hNLMT$txPkw1=%eiHGZPp}tML9zm9PmgN$7p)mSZi|Qr5Fu{ik zgXROJb7I_Akh1E9`>RUoy|=*Kx`7vNb%A!n9JFB28ggLlBPC44nRkSGS7%*`o+w_| z(L_knw3wvtwambNbZeC2NWqq%NGif1oD2WFw92H!_~`97N#xh;3s|M4tmq|QMzosX zk+lH{+}rcyzPFe#oGeL>zu;0T3E?OXJC}4;K;wNNCw1FBp?^j>RMRw0wu?<=|oekvRaDiA=4OETb($8FJdO1WVb#V|WS^y{(ed5pb(nN^F}scT01O~L(&wl#dg z6^@QOvSo*V%_?p^jJ)Qp85a5q-q;B*C6?8Rekb0QL>e7cp%e?SQ4C~p?zVV0k^C?m zY1D2uvwGK=Y!YW``Norr^JZI?++m%1wzJ8e7VuC#_CNUkT1oC>|9c;$Ec-ZZ?+#Ow zEI(w9k|)z$#M9*2C!8OO7)pvkUoRrIS#6>6dzjRaQXSubuwVLVQj6r@)i?)eUH6yKK*E@e!}t5o8OQz$7A|v!=7Sxp#%5 z7`e^3)lIkVxGW#1On-qd`G_wT;wZRxImx%Rh1xm61?oZcV$(O$+I$o1MS< zylFkkWh{bEIj7(%8&oBaOPbr_a)Gd58!w6|zwgMoyjB390H;iApSq$fn=Z*$yf(~< zU2}lIGEi|@0cDWT5W5w2$Wo2;#k43yx0zxPIHZS&r8&nO$JcF{|=9< zoLD6Vx9BIpY@u3%!9 z%O$om1-;k%>RJ5ePW@v8?LD-u^&yZ@F?0k01hfy(^IXXGcTw6LrY2Mt)85#z3D2tW2RW)kH!wB=9IUts}D`a7If(0Vdy@Zeb^WI}sH8(VWOL3IvtBF@9 ztdPB`B`Bd%RhMtWrK7D>uMc)d{Q274+*%4& z-z)aPOK#qTmyCV%_8fbnIGU>P=8xs?xPJcyumsIinETn`9tsmK);9+cZn4;|%$om^ z$%>GC+;wPl;r2oJc{Asb!xyuQP}W4)aH;D_v(0JJ|4eY11zStFhc`oM3U`tghG(Dv|(L)=mO3y zH4Tp;Y2vOI9#waZP92bRKyNu-Y1`3S^%(@Yb+`D~Z8}c`2Hw3y+Ic%HJ&N z1yLEINhcxnn)Tp8o>5M-eGX=GO=-`^H8c^~P23iIJ^>zWtY{a1PNZKpg!GZPGB2Dq%`9BIfgioZK*kMEH`dE$SaJb?KQCruq(u*2aD~3w z^OyqHq0uEP&!$Vc*TP$l4x++;ewF0Wn-%R1Dg{4BoY24IrU#)Cg?ivU41+X0v%quf z$gH2WJ?>q%`$#3e&O(S?R?)!LTebENL7UvQhtxr@3x_Jwb3fVg*hVjWvY-DGE~U@` z6byP~a7N6Lj|`(?K}#b|{O%Wd7nKANU2Q<8*zC5zvqN4 z!dFaek;6b2n`%~m@>bcJ%FcQ@PYiHQs5{4yEWh@3Lzgk4j9Gvv9Pr;0Pm)zCx6grK zprz3>S2!nNK+()PKub%Sv3{y?EfiHPOGR=THQVxhY@pP&v}#*l)3Rg~9x`kU(h|uc z1g5wXMAE!6t5l)BB1G5GsNW0+USFQhup7%&<~X}-w{3MSd$OG5KnpwS;+ZW&PxhYF0BFOw=Xpr zoA8Faob|_`Mb0D<#3i=+Acz%@eQgBLI0Ee{$eHSMCNQMa)p5_b#Gj4QfH-$6S0Dq{ zfvx6jg0;aWaMs*{kJ#mD&+}XW*>C(V8eb}vgKm7mPZVf2xVYoDM!PlQ2;rfcrlH{} zHR}{&S@v^MNhu!TVs6v{IvO%dJ$f42fK(WQAqw7ENkKdYKfz^b_6b28o-PW##L!Oz zvhGjVCRV}&i06)j!l^@8*r8PmS{X{X+T++rF^>#uH z{Y&5&Yt3lQT5jNsMS#F09n=L)9|jm`8cLh^mOVrd0zofHNWijZ>?Vf?F&I4gmBZ!?X*j<2;oF7<5R;725{Dx0ZSjs0=U#cs~M_3UvPdp)+G_7K`3j z)#jyl6~oI~ELUO#mDuol9B|s8 zn6OkNf0j%Bt~89V_d+WF=|KKVy!5C6Zm{6?`?3bZ7HD4^4w=d+U%Q+hua#JCCfwHw z)^xU2Gg~E)p1_%$j&lf}M_Ma~RGsHllCX7#pj^fBjZ@62CtdvuM<*%o>Pr1Ug-03X z@+oGW)&bPIh5PRRW32OQ%TGsju+RGM;i#|gzb~P_7p|%U8cC=+D=FEd2N(R(_O4$C z-|&0mOG9?LyOq?1bg085fe8F^H#7$XOGa{mNf{4r#9)!aYXH} zt#I0KZmMZ@)gEbky!8QW64F`+Lf^XywvG=q1*i`yE_81jV(!8th}8aTy2|w%Tap5sMNPWHM*2vPp}AM;zQ>?U z|HO!cG(|u;zEqGNkZzIo-^9iEk?{`u;AH`4w|kwNmnTZU+sqP^yt|6FzF;t3f4Kr* z9B)7Ev%t$8DaLTN(4u?0b7cqQpbR7Ar$Qp?dMATW^1h6bhGc8 z!VAwBCjo^{Dwt*SWksa9|J})wQ{gS>3e<`uH)a7X&fAS{ zv)=w2uEtWznopCzEbD*`1Lm34g8iz@*3N*PYrgAxi64u=rs0Ec~L&Npr zgh@L?WebYk(t42@sqWvLf5T8+w?Qmp@W^z?ygbq-RWPSM>n;)=)6czgi|f5aHrF|{ z;?LP85io%d1<_kxtX$5nN-q9^~P6q~vLz;D`+uAf@vu^PwE+T&oBFpv_X zJc^|*FfwG)slfDTe}pd-ZBSCpDwyl3cyL`b$MNxBC9JP03wM=w8&;(sTN@#;@amRM z6O&nOxty`p>Gs35^Ae;8p)3A`vN^5;BRn;~F9wiYh@qbTC?7o;eufJpnQEPB+Fo!o zO?*(ruwQ|W74k!MJHjVcF%GXgR;ck*%IrsivIUVivYVEkN#^TXla$kokhX|$8cP_JC1Unr(`#Q+ zP(Ke^-{yXFT5!i-QgZh`Hx$jQs&HzVFTm6T;=+=|#XPd((%mhl-%q|eT6g}hfo=^v zIw?)&x}YSKzN_c@l2;~g7Uqa!?Mm1l)k2M-t{g}>F{-{gB9Un>;Rwn7bS6u6357o* zCq5S@Ox-QK9;i3#2bM~X%BCb{8C&j7oljO@E#9Ei-ra|#5v_S-r)b&wD#Yv-GmA_)X^x;!CoTie^Zw)-rA|P~+5X739^PgqqU*L;U z3Mwt)z^+j*`{r;hXx3v5PxJgzukN7KZ3K;MkQ;E|K=!g%2QBR*a`%AQ ze~%46`*1a(pD(>Y*UYM@N|7>hH)7Jj&Jd30{Px-Y&YrU-RcSpa zpO`PTZ0atNR!p>S;etv`ohoeC`2cC6-W@lj8CYRj{PBE4RcBMM0-b z?^Q#BpIMS5Yyw-Ysf$uf^!OG4Hi!_vaaK2LyJFr;@b82)i;AoX&h`=RIk_53(HHU8 zWz(MG1F&H>JXlgr5gTau^0A(EPraP*}6LHGb)YRqh@?h496W!ug!KQg_!%S1HuHs#bfw!m`ZncHkvj1?N-RGXJD;ATU zk?$6pM@MoZG*oFHBK}Xh>5QndG;lZzxQpUvSB&B1D*;Q7{$pq-We2$SZHyZY%t3zK zq*F~olriL3TO0ZID+39mEt?sP>JbB7M4)|;GQbQnc>pXsDkDdL z6HIXgp#^1}0M%=J=bS(KTl?IcV=$L0WX8SEcmiU{YmLwWc<%={Zp30*;BiT-Uu`3x zXk9*a?yyBVm2YcWoA&}GhK;PyzOBrxdD`KHH92nGAbYF$#7=s68{fLBYdhE99654k z5!n3znLuX0@GxptL6g?-sM!*UMi^^onki6`;$aRgkV(t=>2EVOc?W<+Xj4v;JVgJY zPk6kY-VCKt=!yKKLq|r4r(k&Br4Go=6BMf%GDQ%297}LIS(OZzWb1repT-+Xh&vm5 z6`luVbQjVYJ2^8-OSlU=QK7HVA}j(i2MEV=KS}3-QH#XhD!sq9hH8`K)&RO0-mcR# z)3Pyek+Zl>MKwwD46@Mpg;kWzviM2uQf1yBg8V7jcMFXnY99p52b{{`A?&(7q;c%M z56P)xT@r^uYZU5wEp+LQu(bq(ZCg3I1n6puEe?JXwWc*Odf?a`%_u-c$SYkp9p>uT z;(vE0^PIvUT)nYs0PtG&wjdiu$h)wAxZ2tv!`{QY=;T$}=MMz28wqDsMf+3c6gwPw zzMTb7BTP2~YmS-1{U0t_TGbd7Bs*tN+zGh{~VDeXTW(I9!Bh- zxsJH{YV7vXz4o`L_mm|??dmSB<_#!vhWr<${oc9`=W?p+6qPOZw45IxtysmwhR>)~ z3$-NYln+RvpXr(XeYA5>Ui8>IdDC)RqM}7 z*DNk`ELN)GB8jOlnP_=dV@lC2k<8S4$8)#g;A7sX2xRygtMUL3rTCCCHY)J>cO*ONs$?+73O{!@As}FHwwOg z1-R}>ft5tlkAh#+--Vx86@BJctno?}M_cK-&dGKc+UJ6P@`thS$fFJCf?}KDK;lg^ zKQiIIpDROe2nU*UH6#?hPi(Sj#Cf4Qd3mr%xu3L)-$GJzi~dNi8}>ZsSWPp*Bi)nm z!unlvN|Tlf^VIZLWaii4ad}RV9tD=RCAFKz*s*|?| zp`8apR85YUStn6t?8huOw_oehU;eS^Q>(gd$mI;iMHC1wH!nFILmxO`UYEKoaD2 z)DUQx0UQh!#i7WQ;Z88I-<50aXE6Ul->Ch3dJ=mi+sI%e$fS(5&w6c-+-&#LkG0#j zd?Edf6FEjajOea`N4(Gx)r!)oyV!Z<3~%KR)ilfu-z33VY-UqnaBfMkkrYS4mT5HbNHcil&<906TJ>fAdeyDP)RTA*1EhbqZzdxwucE;eq2pBIz|V+dCzwC~)?%izG71Wl z;q)^|e;vFx`u-RH7uGYxy-8lK{WPr~kW$~S(`X}p978d^#BK{Z7h=P{rh%>p8wJxO zr+8W}TAhP-YPa?*Yht08)#$}g7Ec?mVDmbnO>=qWZ`FqIGUy62dI$l#DkDdL3oJ1N zmRZ06`r+?;CZ_XK29rSxJy$1x2@_ID1SP#Gk^6EQOUmZ`BPCbty^XkWqB8VZWLjab z=+^OaG>`cNWzc!~-Ph?o>jlGr9J)mtN=jFGU_nWE#Azqb=GDYLk(sjUSa0Z(aNtL@ zF)($!L<*=B$KzoQD3|U)b(C42n8&%@p-?$4%;qpwh$xkjsk z>LHfGfCh8X#2V?N_m<;=21VEI>6c#RTBhJFK)V${Gp{0?74cBdz$qSK3uSTjfP&^H z+WuL^b2s_o0;0plDvCXrP#b*2ABDMi77dnw!k-vsMTd{oM?Kcl6S-YJRbx1<7!OI< zdea8l>o=dE!YgJQogxSQ&LMVsmXQ@X--6-?P(+ChwMw{baNsy(&$!lp#rC|4ae>U9 z|D+WTiX3%x#MXynwBj6&40Ya=f1*pCUEk^l5qH&7Vus@rn_2eV}je|GJ z!q(xTJYBEyC@^5KkVt-@8(r~C#$SndO!wN+JX*A%-IZhhY<fzFegC?SDYw zh_wG3%d-QaxeR?r8y-J;lBIGGMs$l2vifjzUF-J{V#O;fzkj`tuS&`0fq8Ma?}nMt zk_`b>2_#@vHcA6>)-u*uC%st6kn~`-%Oqr?<7$1ElFZdTw$207lPPY?;H(Iu%Ea`s z_hh)4zlIIbw(uP@Kl!)cH%k3og0ZLaTw%@Cv9-73*7z-@#F}E!w3+eOj7QPmQ)8g z0>P4vfFE!Fa*9mMnT!;i0GaL>5w-3V_Fv1gYPXIUe4W#?0&v*`6#N@Lh4qAk?4iks zNRF3`Pc?`l1ku>vr`3?Fv5-GCCOz`&L59dL< z-gU>OB~f9`XHieM_RySXfXge=@C7`ptqiza3$B$Q3BT2RLltk*-rz{?5*BwN_|ez* zgyBJ`=K+p}Tj{*VkxdsBTB8BYd_Y#_lJvo4mYDq-h1nuRWfsgYv5~8&RQ}?nPVRO$ zDy|FxfnS)U&zu5VAz(Q{^;f>NRZWZ(Vaid+B!^kT(vk4VvePDRHNvf2hWoHmKd`Ke z&!3F!{$CqIVw^iZtIdvYbJV*yQ?jMlho(d|ZvkR9;OHb8IxQ(b_4T>lEIM{=Z30vE z`?R8=&orF!k~09<~$;uh{pZrGjyF`H+w{G!RP$uo5HeBh=^yE#AnmkE`6MQTP?Kb_EG?{K3VEk_XYPS zCs%S=EPvS(1EKtDz${BdErAaY0|Ej-WF@`d%zuvH?Le4Ys_4l`zjrq7Y5TTpI9z)2BGJ{Fyu`8eCZi2u1k{FqUgPQ(c~^l_0`)OdO4YFCwR-U+{W{c($5% zk**gcnO65TGgFsSuoIG#F;w6h0hltD4KjQnBZFQeM}P|~aRinHzyL(}DW<$M1`Fzu z!C;s0NiSqZo{$^c{UUR(?%sPti?W=je+vAjobh72xiUDO`}K`*E2 zAI{A;7AW{XkV3CN6X%m)C4)|t*LzGz=&H9f2wW0Qy(L>4qjk3%8$iI}Ufj!&t=eT8cDFHCTk zo^FVC(iJ}sZJ}nzSR0Ko%aUx*T#*u^G*!<^qwv5#`l71ojB#&)*-e7ptMF1Asftk9 z^U+tx7)CQ4KD!#7Fp}*m6+qY`?lf22cLc1vkFvu%6eg?pgYBDqA6@WxpF z0L-ZPF#T!K5iR7l`Wz;WKYp_;B_%)zvby}+&bFXC`YvQ#B%$_B_KvhDG?y}e8Ga_Z zUiVbTXZ?$8S_NLU%9mvEdEP6u$=HIhBQm&=kCGAjprdLbMl=ZlryF@#9H?-0lAAh~VAX)juYvUJsN%=7#DyKFo`i@+R*o$V z?7;I#q~9xMcZ&f{Kx6tk{(MCo2Mx}tcp_msqj?K3oFF!EJOVI8E{6GB9A5VDVou8N zV!Y{?7>7VHI?>AZ>y;qd>m~Yrkw3f?7+wut<&0o>@F|uSc9;Eg$)zg!QpGKZk;&Bz2-NO8(|B89Y(bzt6TRLS;A)0$%g%v1 z$2pSnG?=7?3=3}vF8j0j*OcU@0>(q3Lld>gx725X1t z=jI|JTkiXcRn$AmEtpkRYp(Ln-njV_eniC#@!kf^L^^R!kzJ_);LujJUC$KzroV#u zx0O)^U(h)-M|&=4yDxmVdB{K$V1S!u^y(~}u|BX1znuR`<4SUJ5yJYRCfsLq@YM1htMv^|Y1OltuLyYkb^oo#CN&q(O z3q7QU9B4iFMQJ~$S56;|&9jrHqo<1M1aI1(0)(o&sr3#g=&*Gy9EgGymF?iNc$j8ImwX2DpOz=qDccipr zJh|@luR}s;x{^Fj_(UkvalrDqm7whx1vO5XCrOQKF4;;QSahO6mwk+sNhsix$mETX z0T7P-!5pEFqGa1p)f3weFO#&1#$35i0-8)zKXKB7sy*Fsvyl%t=f<>~p;ngQzA6{6 zrxqiY7eX|Bt-;e8UTv7zBR6WlKzS5ZQYE` z6|Q-6iZohES9c*9g{j!*oaIJ6;_!nUcF9H#t%l^&w$Bp)$%or?0XMLMnRCkTq~jpa zg4K`r^P-H`cEi`%;uHp-a2zb2B1Y@jvnSR->6L*%bV$E5BA)$(-r1a5>^C-$M?)_7 z9(cDPzS#%}nZ$CEBnh~LET;%m9x7{STAt`l*qviox?rjaBfOe(jagEz!j3Lp4#FSo z@upJ;uA3GL&Xz{Tu~%jBoVNpp@{zW{i*kWG5Z$npCv-gP{^kX#;>9s-a^(g$uAXW~ zB({VR`Elnz2Uc_YU1p{@0%#HS>WirJSNxs8-g9FBJjhU8hhvk&2s$Sf#7!N}WGGD} z@9KM)U}z9bHC7*r!ivDLVtvRLCZp}9A{b=6h4e!AY)?ia1OV1{C-TAT9$R*cngx#u znR^GOC~Lxue~eM*MVhD64Z1RL>$jZ6oTEGo^Am|kEMKrvUmfHk!Kz<|D|@9|s~>_x zsPgaLBYMn-b|0m?Apb3C$o8l&2u{by9tLjMEAR06jCpRw;$Or}J7rgC=|M;wx?9PbN#Vxo=Xx!F7kc_k43@JT=|1o>(g`ws z2mo_lBS(M>EO7*uS-=3`dVnobpaPw|y-y#<7J^acay|FYwp4%A=K@UmvAv;pSTZm-+yv3ZAybqOX(94hqF(0p?ge2zOC z0c0m}9Zd=PZI6+LDNxn{#U#Xm&Dp@I(4{t*!eMV=GlqpR7Gnt&hi(_G&QwBKdIefH zB^+?l3WyHc@6BzJZ$c#M4*ezD;Q_p zSW8D@mf{4?HU;Rz9@2>)`B3q%CU8By-u#G`+8S} z^u^G9NZy^+ntJ75oqL%*F-WXn?uJO3hE) zb%gNRZL%?U&*u6UTfYw{3q?68{aT!STc9Jv5jqe8(HFB;gx2rN<-cg|%F1%^O4ggh zfqt~~2fG`UaslY*O*GbUle<#*fkW_r@<_~5j`x79H`x1#ZQV~gMdqmWQvlb};@FKzA5u=bX zG}9nT5Efa$0MNvR!V&F`>AAciutqaO@|xvkd%5V-I6To=pkDgnvbo7YULfA%#M(=OeFXbFXM;xTu5Q<+h3&r}V!Kb%+upp7 zR$KsuwUxt%c#uzhzlO3H`ZVp*L-m<4VGY3K2i^f-i;rBluV8)~$c}r!3Vfx{vP1mK zbas`yXYl(DbXqdlL>q@b*}OOl?rFJ$Yf}NEK09&Md`m`Uhq%4R44n2)D;vXPp2;ok z6$jofvl&^qPcZ{}!adS2W|MlJ$t0o0eIMIRZ6sElS4wG_9MI5aStLL5MZ?AY>QLpX3RPbp zhXHHU1!MVN#|Zk}9TBV26{9eN4TD2#lhl9Mtfl=NLYt9+kG3lq*t)P>{<)pGDXxsw zuFXJML-a_-Z~OuYM*bI2G~vhl{uW1hg^|H`vvOfNNW$$wt;yherh@Mre2CUW-7@2j3Jpp^_vKoo<|mder~s z{WEn%;-chWEYXY6Ro6lOb8iSKd%85TbX35V18%X3B-YlvB(r`e(f~`DH}avjC;#rVM=z`dH=1zXM|is_}OScLo3?a?Z0E79tGfT>Wx$BUka5*nY5NhjHD4f#aD82GoSX5ljFau|PD)0&mb z3$)*7;$4HC`KEnmHm$Lgi<^PUC8hyT3=Yrmu{{z$mGWQfJv9pw>np3eF!^N$+VD^7 z;c#%U?fhrF-CEO=xpEcC=xoAhDY|SJ77*J41L*L*rSBU@xYi_o5KWdp%*4-80Hx@UTYcN zHs5SabHkBCO1nF74A})7xJy=B4$gu|zZbuaoi$Hu-Qu2Xfjk0cl72`GTtMyW_#) z`?Agp*t-@yF$*{3a`;+}8UlE42k!R&4AiBXW>;50`aBX@TKtLL(A1Dzg?rD(nk6nN z{DAp0QCPmN+~he32eVny!Hb%{+n;nr>)aBQ`W_}PG1OyAvyUlFaB@`M5H3)^(LL@9 z(+rSl3W#o!H6eg9h(X{FozL_rYl|eDZN9rJPic_ zBXI57O<7p) znrPghQ13RpzQ7B9R0E3>g?~!gB+`RY4(}(lWtwGp%|I0t4Z)J$;V`R8z9*{1B2~tN zH724MA-^{(sH67KDgXys5geEtvD*^Cg*;NyVA}H3T96&X1jXT9g7Of$p+MTWRN%DC z!8!2!tO(4wlT*X8*T55D?N(B+)H69=k&5S)qC=|O5Mp*)T9wH~7Lcyntdcjiy5!p~ ziv}A*oZN@b#2gYNjY(C14BtS@e`K|`@$XlQdwx3UNg%L8j|Gu=-aTz)L}2ZC0k4j- z_QwB74YbYv@9_t4fzxaeDdEoyfKTc16M0F=O+m}pK^0HNloSQ>NSa@JvtCPMhQiCl zSgkdmqQ+$WlkwP07Wh|;hlyCLsyq&wC9K(oi)TNlHC%BO=tF7*y6@|Tv)#zX%&q#7 zJnO;Vlv&vOH-*fa7aUr|c82`)*;oCu&75BK0$g;}hVs;{3-hlv?k(kN9$7(Ty<5as4N~qPl>-a$ZY1W zhqG@wjRm7(7x&&rd@*_2ZaT*Ds;z_287-SJv z*X_k6O0GCQaiL3LUlc`TU+#nkP(7##5uRM_FZzvVP#mkp(eT1hHP8>7EEQp3~ z^`2}D3sN#gc&Blbzcun1z)mdFaT0G|ZGJyCJ6uE2>Ki-+wr>g?zGlgc>X4l~nIE*U z89sV&&>IrV_xk)Ar|VHSh&t%gRO0#M3R63D&3BK`d67Q_@&GENi;je&W@YMh@Lc5O z=4C*cr~rP&>__2}ly}aTvUSql|BDh%9C}oukWnU>EiKsyIo~=-W&@>(PBJXDJ{1*k zeUWStDbOuA@$d9U?wO}|zQ`zO?;Dmbdq2pBfn$CYkqXIHj8!{kVZ~Z>9WupB(@6N! z`lSGw7e!(i;m6LtJSH5XgrvfPQlo%Regk6`UZ;_5mIVeLvyd`1Gx|yC;PnYc0??5qj28yo&Luq<<=R)WYfgWU`UG@ zYwz9tuY{nee)J3IjkLrgfSXRHbWnZN$;%5W{9|2nA@S6VTh!CbkIQ{jT%z?j5WWK$ z+`rh7Xt6pX5&xQO?{NZ0wgW1y1Ezi=7&|`(mGJN=bet=6F@Q~2@LbtzWs+UY(ybGLGk$~$ml7|nW+Z=#AN$$gN zaGy(WxBqO&ip!Oih=&mJ9>t}dz3N>?TGXK8nE4l_j#e3fOhh0W z3%^ot?J?s0neO}Jqnrza=UWm!*X3usmXI8z3$2WNn^a!>gUB=1>Gl~|uNXlK^_=si z`chhWXpShg&0D)YZKAq6SbFJ4WE%{TlV1N5;Hoeio$kIcQIYw19{&>c0;#8sKpC=& zkeFP3(~0`%3kGf0%o7+aV0=bx*$CA2M$|#FpkN0tn=EehBEf@q_SVca?KYU=*BtcL z@<(xGQBkPbV`AR|se#U9`o_>BNz+&EW`%@HADR;$ltcdBmUDL5bruTgOD7kErvVjV zJGeqUD-zfOHwr+5*9NhQo#?H0O=s>!m%7FM-F*ZTlOIl>Y#VUs1|@TUxZ``R|_ zIwuxvsuJzJlsbAPuy2o>Tatt-lpeT=rr{ob%Ymbpn^*{PCnBTpjoog|5oKKJvZBxv zZFdH-2iMUIJ9y?688IRo{4rc+*D%J6`WFY8Cae_=xu^TusZlf5>TKoLAWzctMQzfS z)|E$#zqHkHLfcWnUf|kt+Q=(YYdfF$Pnr~t<-=r#7P)HQ^VDf?D`9luKV_5Q5JGzN z{M-is14D%S(N(2KegbDj&;FZ33JJGdTyX)V?)ARh8YOL`e6At{tA_~S+Bs$SYaNEC zPiF$0%NnZUYc?%m@OE$LYukp+dD9aRhr24UCziog^xDvDPQcQ>;~zWe{(l4qUzyrO(zjcD9OT|t6-k)za2DRrshaavI>*40ZaW!Yvj?MpKe$WA<$^oAv zGi9Ny$9be0P1^>doxS>S^v#c4ey6T_%1m6$(xsMe)zNxx!Qtc3HWQCp%ZWhx+{poq zRrsyB=jm_{GZX-9{>$?sPH>mmjol14Tq8lE!8bc7#Vf*3SDZZ|gDzOO%ZZ6p%FkO* z@a+j4cE!MLenoThtGcRgegxlSVZ+C*M&p5&B4ZK)(ZXNBbs z6V~-DSAm+sZPWKl@ahhgPRWiY9jSPnMF8<2o^cW?Ffw^&9#^2ECXAg56)?3y$2Jr ziixZP%3&XYMNV63aO>FBBhDL{({$)q2I3jWimu)P`g#=!Nmg9K_CC z?qy(h?XLj-9mSfV8-|T`AboUHtWdO9)0SX=E>c~O4P z;l}ZME2|qKh*3Z4`d@X44h6d*O3T&hfLYCy8r<@+44;75xqiv|; z`63ax;KX0%@Gb2gc*mmGRH{gA2obdgqp`1%8nzoLnj}og0mUD5Wg(^*z9X<6s=?)| zjQD==!P8w9+8Xe3nL)mxVI!-))|pbC^ez~QLpK(dX=TE967z_Vw8czABZ;`JDVd30 z2S_P3_irH-@u>2T=5`GrSZFpq892tt*bL@02AMT_nSE8guS&#RrS(9h;sm$K&lxbV zf^{Jw`cqvgBQTYMan&e=zbIBEHgDj5)Wo3!WqzF504V3A?MbQ%3$C8gh);PH@7t*< zWU<{{9A2O^*p2xp*Cw{?0tnq)l7?gLa>rOW>ex%aQ$D)>VUUccQflHVC^epa(Xp0_ z*kDqP=dOLFYJ8bhY?V^&dX|DY-Z@ojO|N-!p@vg+{Q8QezaN2XOm!;FrmG$_ZdD!Z ze!$$rNtzqZ2UU#M8us;Qn6GExEawrg%Z=6lbnPWR)ap2xy~h=^1ZE}m9dFL4PUAub zuxPEBHreQXoL#k#5Am(*&9Gm6mDl$*{#%O~H`tO%Kr^#c6TZPjJ-@AEQ@pTLge=oO zFUMrsODdb>)_cS%1+n}Ni0-f39f4)Snbn9q#8rloq-S~q!|*85V$`;OH@Z~0GI#)I zG1rv_+%V-}Y)RDvLcCpO_aY=7Mqd4?9d5k#PbyN>ybPF)e)k^}i-dwdOK|Neu~v1GA31+KfUVR)bO^VS1)j0<&>>K!@X zi%9LIO4rJvv0UN1K2I7%F-F?wu2myq=^c($ugL;2d<$+ zp1^a?1Yq>7wbdXKFL05tL5h z`2k|&GvBeqWN=B2WpqCi791s~uF%0I3(0RuSf*&RJ#4i2ZxEK2Q9+djog~8HeT0_I z@BO+ow@ct4?RQq{0;>cR;ip}jSl@i2)-w^;R0CjmD#r?I*u+4I0c@%mbjrWy|e zi@5=5gly0Qs+%VYx2A+0hltf4JiSDLBZb`Ye6b1d z3_`RAWJ6tF6W7&A;yARIbZYvjmjhWPjE54Gq1$a!KCst=`PjU2 zEBcYF&%O|JntOYDrA>&n<0e`CioIKk!hEbUFtbS@^5Gn*Tg(#Yv~mSGbzMQwRGK!| zS;w$DW!17TG@E?j~@`Z3} z=|7B+fus_A76QxwRTZ%~z-^6OXHT&{H+Huu{{yC|PW(t9_q>kWeNeivT;SyBB;e~n z$oZ{ZSee!16Fcu6FuAIWJws%yf0j(lX5YQ6Qjs+##@(=f=7qm2J`i~Y7jG=H;&Qny z1>+oDhBA454WKyuQil`)d$;ct_P-y;MG}R99VlpCwyGS)nP+kyi?4H43_^@FMm!(~ zZ5hLVp!Pr2MXzVgao>kQ-$>TNkwqLby3LqTLzDwHG^#^;O`-MgO|W+cyAdwGi!?7&O`~imb4ZBSvsu< zgIp`no|Qs-1IhuSWGiqstj1EMru%>DDpVAwuIfsYP>oxePn>-PzDw}PRn0{f9BTIw zBY7S*$K~Kfz}H?5N^52%aEV<(6}6KCUR9kXnDL)Vx`E?(Zogq3`i3!p!I`(At?}=T z#)9S!!vG68o zOX8jUarZ7HM4)hdDHh$j3jMX+rpZa1mR;*)I<1Fabp@wrlBJrUKkFobZlN#Jetvec zILD6P+|Fq@a3d-GM)y}X6RLSPt?FbxnM0&Of`ye_1cZcg4wU8RX7~htLMr*20m$58 zjnbCCP=FbC2^fO*RGo1F8>pc|FRI7`2cr9fqGn1=^F>Z&qzWKVQPj3P+D&sDz4RvK zM$H5p!4R0dd~H2iG&3i%&!2d=!y(uhCu)V4f)Xs8Dw3vbC(W-vmER@rM~HLd2=`iz z2VC;?4dD7;F-@4i+;~i_dT7}Ijdgj=CPi9oOJe{nCrYr=Y#UI=Lq0K=dd(H%ot+&v zv9J>rSy&Bf!dXh`L&VreI8c1&6uBR+`VubPT)B%BYtKc768dHc&+V(|)jVi^^pef3QN5(LO{6H>B;#KwbrA133{ zDOhN7+mRg-e<|!a&LETmRM@dr@hb4C#XS+94@KJrWBW1v?g8;m>HzJV9I-Vsn4LDs zK^YkvhSDu+XAmi-^+;`>%`puMYxTRn69MqKm-+)_Oc?U7F)2)m_|Y4i(*!96P%74virjh_;58W)0tR|;_n20oLAi=}X?0j6euVn8;QATKibWaw9dwbB?6cG`D z>CDoN2j;cBxc5)%r93GCCI#}s0fTEAjv|mpgqU2vY!y0uS5U`3Xb z_Upnw&Tgh(HU1TIJq%Nc#38+gm#|v}ldZUd_g~9JAPR3ZVsM%lJ++1=lY~gc>wot5zvC5$3lmp#S+|9&OesOloc$7YA`+=W6s; z0tnL5ED0uK#3m#iq`kw4kBgdNjs%T~iE!S-)#b0eXHo%obj5gIMNja8ijaS+o1W$X z^Splg*g|>x2Llp4jAikRtBdL%enVU>fTtBel@7Bnv4;c^>`;j$qNNO>K@S2c#ipG{4#0by2AX>N{P+30BFcsF2>%8sn(<~@`&(8&8aM?zjq8M0 zlMP=j^Ys0HDB*%^F3}Fm!u{F7BiHAbnCiz?4r=%^{RuJxAS1h0BS(M_JTU~8S-=3& z;hwif;p6rlmh+x6*Vn&yx{s7NHR`-!OJKLs=l|<3toI?OXY%@!tuiZ8iZQQx&qt7G zwb6WebiOTyzLP|wa_KSS|= zga?-eCmnVCwFWrSDu;}1W1$u;l}$SwK?TBvyv>+@Rs(yC@YkD~C4Mxlv4n@CuX0XI zt|0RPNjH3EL;R4vTuaD!IT%t0$#Y$)AoxzENNAb(p`K=t0D)yk7w){()>Rp0bwzV? zAPASMXjVTY|1QsW-x*FOgH6g19S(UknZ}1PLt8vB>3{Y%zC^5A2v;dE2ex8~v_@7h z)b66pIsGZc5daR<4Fj-eqOM<XUq+10(mj^ctWmWG9o^Jv9$u>02;;<40^mZCMkyJL2BpPoJ&8(Xd> z#2h@TbWePhMU`MEX`=({!h5P4pv=)Zk9$$$bm8z6rIer6YVsNf=7;s+WEl!K8jIX0 zhg8_5BKl+%tgm+L_#ETd$IJB&#mYEy2rs49_Fx+}36Js6iZJD`q5a_3-gk5NKu!c? zF9>wOyZB)`#noD8@4ht2mFx2;{M0Gyu9SUzWUNFIvbm~;dp|r^gL6bQQ3xr>Xt|ms zVVdOl@bMevuajUr#UD3K_$rV$zZtmc`?x=gFFOA*YMFzjF1QVwyAilA3%Tpl>Rz^9 zb4?VL$Vuv_D3&&}gSO<~M)uAaUG49YDa5Mgh=#oNB3_nAUA`I!EmBi~VhBo?3CpE9 zZQ%K-e3x<}yUfaWBBgNGB0QhrErO&sM_NE3{FZA~;(dPEXLG`sV0l&I60UG}?OCDK zecf+}Hnq#Zyhd$)NGKy58VKE`oud?Upk4J-5FSG{hKUhBM->v}b1)?a(pUa5z>8@i z{teQJjbIz3PcXrP}VThP8v?I5itlL%x+*)&pEasyc0^ zz0f)S+5F^ZyYZMl8#5#2I9U%f1QMGQ+oJH#sr_5I(A{4?Jxf{)C}~<3nQ(1Y)EO43 z@YR=Yt#FqxSl+Xp*E?s1gAlAo*|Y+LjQd{*9BoNj1`7@f9o#}4p2!`F0INfVUU0sw z;xnLVG=RVmc!C79P^)UGt*Lo>SN^H2ppXhE(#Y;f0Y?!(Tw1yrMyyOuud4c&^j6#| zjpJ9*YnymD8PC@%qL3dH-%wcXfLe%H=-Xaq?i&6G9xRB)GMMs{`Fu;cm+-f`$KZ&{ z0|qcZX~oce?{R^(5YSrs?zyV#JWR^h_Tkq7fLuqqh+#@3b&`Buo6dBK-*CD6U>PVS zjQR-kmDkD&47@)Jo#cLhz=ta^ZD?qd)|p=F+YaYxM~Kv;N?~>-on;_Ew}LeHXEAZ7 z#m{QGx+#4{@nwJ-0jM&v2{HsI1Pi_+M}P|~G6a@czyQ*dU4vFjsC1Isvyv?ijBc|8 zjc@=%I_WfODDv(&%Ob06v6m)+O-I8gb?y1a?T8yUYVHCSMOL0;8C`PWY@3v#lJs{| z>ZWM8o?zEH?x5#_#dSd|7zjB+iH`%F13)onhiiJhh4euMIsm55-Ve2T`x(XE!Q1T- zEkJn$ADk!$Rv+&#R5^&e%54+DpNwb-81qB@P(A<@<(?+<)E4i z#75Vkc00O7cf|(d`p&Ib+!=pzeu!#zRc4}Y_hRdYh%D$XUPmgu%p=l$Ael|gFj~cr z(yl(tB-XM_fpxQ8PBhmkQ9OnPXwf}Qe>BB1*Eo)bgh=7rY3(|?ibaNBbxM^@+q8QL zhe8SP-#Wo7IW>F_-sn9zKeD5onKzqds!DD7d)*~#2zynUtZ7!9SdVPyNn~(4vqv>0 z3C(KI(a{_$wkHu~LCDn4I5{bq2)qeJXfJ)1Dl;Yc=GN)b9$@g&P8Hq|Q=>1PAM(?lM%@Uwd&R?VY90qt? znqq#=&|U9wbsa88OHFoIr1w2-U3%pj9o_9~j9=oCVwP$PF}=EOBStK-L`qOxIGNVb z?u9es?6Dx||eF-4T+hV24!y;o$ z%gAd_wmg0`0j!WAIk;?kQg%+;9bB2iJ)wkkfA#G#>1Sa~oETq_1;zlsO(>rmsF->S z5(@eP76ha;ZcGMv588`IfjUU)crFLFCTq#|eq{*qv+r0mt#?%7ejd?CfA_*Ic0|m^ z@Qeupa9QgbJ@LF4p26G{I5QJuPbp+_LO+?2%&CODOV|AS1jWRe$$3(1q$^HX`+de; z^c*K+6&vU83!6U3m!Z=R6zI+pfGctU_00oXFZA}Ix+CCTv-qi8WS%OFc-fd#+-z@?-W zNi8>b?w$F~qnR2EOebrm6r9y#f!Zvrwa*-l*$0`0W_Y57K>zx{V-vCGIIwO-IrQlV z(KKJ~n*c-G+I62)qlc|I11#td)W9(YHDV*R0goxNse&-wX;u5@&tA0vYiZd#t2t8e2(-kT4S9>CNw%DZr*?@r!$p65T6%QA__3 zKX>S7yMJ~(578ImX7>T=wuN5%lzpi{pMp7Avy8jPaKK!zof1d5vofb!PA3u-2@;4j zknqa(=Fw#M%DEqzRdnXW?$*edwQ>qLrmBx$PmTWnOxs!PjY7QIQA5U$RFQ+)Q{E8< z1+x`e$KIg>NMv5^t{oo?VTy1a&ZV2Qfj%vtrfz;EmwfRbHpunRr_QLGA__e4vn%$E z2F^B;1+?aNVV&8}io-*aqZ!oz*($9{+u51)%X4DZCO4CK+hBjUZrvX0?%vCM#N#X_K(MXC zZ#gMVom&E^FG8(xoR z-g%fo`To!SAvsVJD9Kg-SNa&GeHe%OPL)cH#6I_rTk9%{O@(An@ zP!D>S1&5qA?Bgnj{3o*hfkY?roVi%Fv>BEFW zrOK55Hw!HM#es{SX`c0-KS@L#uV6l{BqDCueGeX+C{1B}dvr=1`q_4dgVvluqcXh3 z&N}j%9t;5svB1oWUTGX6S~s^s%&()Y6Wz5&^$y!FC+OFtt+^189!iR&2?SIQ(jS<= z8tsDYF{&2Z4QhZBW%`N%rzy68OPV?JUX-i4)zpmuHV^KJmOP5)up2|HJ)?CDJ}o;U zFKjd7Eaz<4UI2(dcfUzF(Op;pZM{KK}O-koY2Am+1y7JHbCf|4tvVRyI&S zeMY#y)&L*=qhs#)UDbllU82%8mHfnvK@}P#-NJEOx!&R-w)qFUUNGn~xHT}C*?hu=vE^LoNP5hpYnJOF@0sn8eq&+trC=9L70UTY>nqUcoIl9gN?lAxpnkISHsXf#{G!bzxJ3;+^#I z2cMKHe+f6&YQTl6FTT9dHnZaJ6Z8rpln{F(PGB#MyotF~*Qi`r#xU)@~1vKRZ`i zAypbBQh=DUsUcdhw91*ndLtJg@;YJxU29MOlkNFuL)CdJkiWH!H~1^d-Iljj8l>zE z=uH+65#9ORvYXG0KmkUo%Xm;k{NY&Qtoq-5_d9Zw3J@Ki-wv(F&qleSOjl0s7R@Ho zfVgUVwXhfsf0QtGDOm(nr#>L?)Qv@#;6P^W-G~sx9|G5AOUh+$NEHt3+pqmDQFmTC zBsWX6cqEZ&2CvAJb=}MUMkuWqyggFB7Dl)b=>sQ@h$F#q0>bB1QnUu=FdxDH5);+d z+mn706?vjmO=z(+hsT|B-J)Op0Px??>%is622STEf_1eBxT=c!u=L>4!OQwjR`~lS zrX(d^IwFjYW%@)OSzDBCK;GZZnHw|(*n?Va6v~^yEWx>Lol5hBVC4Hd$L5}XF@%E- zGt(As(*SP}C||8EQV=zh3;4i+>_xyujCY<1ZAI=?dUJCz8Y@{r~kteRJ_*?0sq8;mXsaZQR97ubr6WM5-*yCp^st0yPgJ z7^{&Ht@3>v>*0WRX`sr`n#etx1tfrr)<&5t1~hR>`4X&}RoY*8={X`$KO6rU8`s?Z z)ous@1RT{CQP4nvNT69N^g!WkHaXOMjvycR;{jJ07gOoZG+{$vY0Fx5L4Ha{#1}IY z9Vn-KBSPuHTc3GN{7?}VEe!bg!)UT`qxzG9Jov^7L$vH#S578^sV^?4IzAGUGf)}$ z`RGQ;yQ#dr=@?i<9;{gnRwf<5(j@0W#y00D2IJpM_4M^ZR8)8;5(^8(E(FTTTTQk5V%b z_SKXh&ggG3=v)i?VLeHl2S7Sbvkmb)xRXu@&Q`iJ4iQL$!RTn!()C~ot&v1 z8!39{?VrzXqGH)vSv2ZrJz2U3Z7Js;cvO+i^l;Z` zT**pWRieSmDU^=Rk}e_T=sV#HA#g$|4$f~AOxBvUbG!0UNCvQ-;K#IE%zLq1WCx?B ztp}B2MB^PJdt_DjFqNp|(94+lLqS(8<==%)?0zuKe*GM;MGP3t$En!3PCo^bv8-*_ z+NW7ub07r<8C}#!X{rkW!*VFV4RKd=3&*M;q|g;SFq4G2O21p%Tl#Oic}!O|0B)Wa zu)r+QHwt+mH9gyxg3CHe+|q_P6DFT_i6@a6?xl>(4X0QU#^ zcB?pnz#pig%$w6S`{^p!x@rKaG!{OGZ|ZT8pkni$Q{zH?hask&tan&BO)h1vrx<7) zs?{2f)@H)F4*C;hQ0>2so ze1v}q&~-Fd0|P;XqF^~Lr=w;#U%~I*J=vvNj1`qN-ZfLuziC49RBqViUzN!)WzlN8 z=&;iR>i(vD7XAAQ@7XC~OCsH0=`gKsP;d_f9K^UpKrHm{vq zO+259+1Zk6l%f*99Btv0Tvo{tx>0ZoAm)Y3zA~1W&Wj5L_P1(pGk;ZVK@4W)!wAnz zwYx&P@_WjwKxmQVePUoFOo(aO_Z@uCx1rl*zo|EPL5)1|Ss7cgvo2EOFikrtRJCd# z4GC$Pk;O$$xrBmOGkNBXJC3a>xy7Dz`N7A?=Qqc#%7=z24{wy-LNb!wGarCNDV8Pc z3ytGBIGYpLz;{lr8HB$9O#?F}-2nv#ZFrQA_9Qb+#6>cRGO48;dAW(0ZF?Iy4*^8W z+M=C+m;Bv;nJI8aFji~`IQw9IsiIn(D7)`k;#?}$dp5P^e8xvx+j@s=dZ6bh-4O7m z-*DZT=BLH*9b>2(rBC1=sq2P)DO!F;D?d$8gn;4d2QkNpUc`%v zN87m=Sx;dy-O_?yX|3OPQMiK-%3v+Q~=m_cfur_G=rS2+poiRpomJyBA#D#`h}{V(dOv- zpKzlyLN_6Iz-!JPZM$$=O;r@dPeTc6)^KIW6uDdcFmr$C{I7=Av$kv!6yiK%>^4#E zX|nl$tBciWv4+D$c&Y+at{(8-3O6_}#eYbJn`1AX@t)4z2b&JW+FO)-!-1gXPLl5y zr_&7ScFR{s^#TpB~$(DCLl*oYf-{|EKY6e&}CSHLhwJ zAFDIAsb^Y%8W3s(hy%ilp0waw&CxxBF$osuK=+gE;>OxDKZI08Bt!aubPtHDcF*2I z;S~X?Y3STw_oKx!^DstL#c>E)Q9@z{uFkGv<^YWQUf21Zy3_>p!RR9M0U2$gM<_tY^7oaZ)iAzEMm zOxa&vzqe~a;m5V^Kzvhu+UIZ*KRU7 zDx*iy1XTGNnkYHkP`N5dc;Qhp5xdR6C2#eG_DFNIkrY6eMg_dpCe_CmpI<|P4gw+h zE26jC51w)(g@Mp%teE6BCHg#$A}B*)?%HDFC0fg4v#?>8Obxcc=dd}dJK!VAul1yS z$zreMo*3Q|`hRW|gQqxFLhsnvP8EoqG}u_DUwAM#xRhae%j@G!xYj?I0$!QrI4R-1 z@c4am{v>~|0O&1 zlr2J;UtOaspm{)Ea&p8LArygM-u&UG0UmSUKOqqx zDCPrH&)zu+`yri2Zsq8z& zxuW9TFAH-*`G2ZU=tRSnwoH1fSQQ-6$3fAwlCFNJ{V`qGX0$oxa`vinHLfe1OeECl zRq8;is<18&@3P)9`tPWcxviEpO78luA)lV+6Kzb&<~A}m>>`IR&_1e4YpjN=bay8KIyNbtB5On8(M7yNs-Qs?5)ehTb|P{p}@ zfLtr1DOwI$X|d6we1A(RRcR$7%=kq!`dbTtPK5EN-|j(Sh~(F~zlQo^nOjRP^O)}#l0P|#k86`BMNje`ACDM0MwF0ss3A`Bry0u)U0+{RqK!2kl#IPh?i0R4wpW$Q_!&8 z0EX#HkWm>zzAz^Pf{N(n9X`@|!rb7*iX3WA!lvz1=U9S^Srt6r-*PqR>d6_ZBW|xG zdXYMpn%FP9MLbh1I2Z-macAm6%)W z*OtQ`ruab)o)S5L-|hu3`-umqSj%mRTBP(t%XlF=?@KfT!+s+XGBpRpEiYRQ$LWas zh8o8g`_}-lWPMj%B8!yJQyx^VnVr;XtfyeQgqwyM!;045*-thV1>=lR*BcG;{wB-k zKut@C%QT07C!<8|33x)#)#8r2-Q3|PkF(h1?R9N-rApO`2MGl`v)WX>|1<+Ciz{+e zX^rykzt61Ko^XPMS3^Uc&wr7ar_d^~9D%=!2r>0??hMy=d50QD42I=lug}Y*M*r1W{Jyu8u`qik3jE5 zu%kZbO{1E7LvHDuH+^XER}C9GDYDMb(v#HPyh#IQ^Hptkl<%E@Jc5sq=v{$A8)`Mw z;J-i}ShABF>HsOn6P`>?OoC)GlD#uGKCx*BK!Nl=YwHdygMC@8*mHq$xT^~@f_L{{1)Mz?^_7bjLf`V7Oc-oJGX@&RRsVC~z!ffqzv1Vw5 z!Xy-IO8-G0aMWJ0!@{bHZn0Y9b)9NTIk|XYIx%SGhi!%pT~wR<$^WWlMP4o${ za8-By7`f{0M(Evb`MgutI7B=o<2+TvQcveU2awG7m2ZgQkUK{lScUL$R(_Vt7MhgH zb2_80BtX5fV_Mj%VJYyez9gliX4o0i2k+2ek7ZG`I58SFzCm@$icoDEj@#gPbDY}-h*Q=YYs>^sJ-@i+w>6O9a6FqbLZWv zoB;f&xBxDfAM>tc4~cw5UAahbmr?UxO7B&LrW-5p6exiaAbD>oO=4?0kopjP3;OOj zA!?U#Sv4cE6;K8zH3eGy0cpX}Eevj|dgFg1#b5o@bwXK$%V|SDm|kX1mMzW2$T9$! zTh-VmvLl@koE|zKTK#Al><^SE%$AocW)fhN~mbMJ-NA&@)~m1S^Zo+AGGcLV zGYp9E8$gEH1~5c=6<1Le;C;;qy%xpUeB)vJm2u`E59h_`{o}Ly6d7p%RqhqlK*Il? zBNs@VnY*{nuI}aU<5>LWA;Nx;>2+hfU;U)?MRoeRyzol{qc|fDxRQWjvla+Fj0Sui zKD!R%VoaJ7A;vnZwJ}SagJqm4bt)1ziBKH+v$DFtI^Dwtv-Bd7sezvj6RGH4vL1=; zY!f{JTO=T(+KJO>V^1vVC0;1c?NKCI?ud}fdX+7;zMl30{w+lQ(|-#Srs$N3CJSHE z1>Bj4R;M|tlO8I|*hYYbuSQ-%WL2|v@*23p!XJlFFzwnLDql;936&SkDE}dG`%Igp zoqO0pb=0zNZF#_??IZ=Mt3ea?ONLN||6fWsOH# zk`pkhopMa<>yJD*!~iI#vU?nk3x4Q&UJP9<{oY3n4qJ+U2*roP^i`<*r`BeTN1AXY z+R6d1dd1M^3J{NNOk7WzIgW)~9CF~VXaCIL+~o41x}+is&LR+B&AP!`H1$ZwjHilQ zx_)pklul7PFq#3?N>g$s8m*~x+EeB4d$0hEH1(FBi4VUdRX5gRiwJOPI<7$e{cDJ(6BArje}_2y}+8D8Ca`)3~cy^WyZf@{3k6DDpq<-@po#E_B@m!^G&( zZZCtE3AgrGG4aSCJqG^h>dhReAhh)D`kwIabSkf5Gs;%{n|e zuHf+KAsrAeq&aTgD9vrH0I_RYVI@Fpgm~gpDh8GqkIcZBtRRsu?^8$7_#hExIusue z+W&c1QwkNynZX6#X4Uez|AP{>1tvXl`q3MNvOdpq8n;eI@PEK~`T;*W|5k78MH=gT zU5q`ToVIJv+xsZF1tI;zM-gjRilX^HBeFLjS)B=yUUi?wf?+NPa7#oJ@M4SxXEaxT z+ekKp5jH-{AM@@RlD5Wm4;cbHO0K95-FQAo_H0L-mu<+`YTTV13L5It6hQsAZede3 z%q+1jfeM{qTJ%nAs;)Z^2+jFs^n?1+zn^$B>Gv#dtg(TbXJ4ARid zE`S!Cf%t@_uz}=zd|q563}Z#oX=&+fYD?GG0}vIRq1%;#l3D_|FVZ8ybreIy{8r#I z$=Ty%9m=n*p&2^w373yYU|Mtm??e9b!;P4XuC|nCA>`BP(n{E0?erg)4t`01>41N) zNH{$9Cp@g0BO&h)<#x9PiUv#<@rgu<@TEtN+MJ+@Q+{<)F=2Y%F8;h=PFa`Q}k#bgANEW!U zC6R#;_wcw=XfCYow(2$dgrXA2qoI^q$;=NSpduE#jB-65u9_FO4=<5vv!E`No8%52 z668EY7jd4#eC0sk_#K8T&02ZnVg^PKLObr@79im(zzfLuo!7T_QSRHL{T>1@1O{}G zZnqydzMwKhTJs9kobjM%(>cNv^L^Rh;jA%TJY!g`>#5BiY_2pEWm%(<|IOC8h|4u@ z0vup!CY_!pbWd6&B(1S1rLuuUDLwB+J%$n7?w+iBGec8BXMQuLAdPB>MiC|utQH*$gsB5Lrlv4-C&i^Z8P|G#TX;l%c;`@b9S-d%g$X^22#U{UXAk+1c z`kmHYS>_7?KJV)f1iQv$sbZ)IXIdF~L4shzNLi+&P`nT`MZ7&$J&QObm-A(dEMzUy zXLDLB$y67HRx^Z!K(&1@dpa+Rh)3_tL(bp=T@okF=M-wXz`xVaA9keuatxrSc7Va3 zNVMeJ?k^Jyv8^|20FEAFy zSXL&GF|YKTKqE=!g2fFSIoegXBWPXW(FEQ__m}mQ)d$>)l2hR_*y+_q(>9?PVIlbZ zJcviQ3aoATzKBztMFRq}MXRh|+8XF}{4=v$$nur*$x)lG%iBDDUX$RfRi;i8 z%HsYxQMIgSlW@hF{e;RRfiCE|Ht>6;y2`SRqk28}51VBlFNAe$P5bmf`1QKt4=;(ECKL_4g}js5FX)7b76T!g};glmS&kka4c zVAA@b2p1SU;#mllch=~s=9rY_%IUYF6)KqTTm+iBLH|K2*PmhrW&V&u+8=lX`@#S@ zxoT6N6w=Erwi1c}bp;-bGefMk=UO@xR4IRKIBccWUvV$uEwa#HDHmkH*p#6YBQ1Yb zK(QIitgE@Lc+CaWw2YR46+biY0ULsUa(CUawhGalg_%Ng!cm0*GTVSZ2Rd0+5iS|3 zQAL4W&PeLGNfJT4OPO(cvXa*m%g-7x8oT5`=458OX8ArBpm5;G8-pn;xc@Fq&6g$x zbs-TsHS#BeCeq7XoKEz+CE-&YRion&5053lY%y*NZ6}QyJ5*aEP=XU)o#ex7lb@0I z@@=Ooc{bv#zo^Q#){O?me;TUaq$?mTIP4Aw5}+#6IM;EhqjkBCLjT!o6(;n*rG&MI zRo+~DwvKuw2p0LVm|8==qlfncV}K42I>Efl{M9Ft!Nk`6zS7i>DZyEfC)U8D#Qp3P zEz?$LEC=Rq!}G{IxzpsJux6kDqyLz(k0eNGMo*TNu9vwE9YfB|&Ql>04UXz+X6J#i zSbc=FHNyK-3gMJZxG=E*RfX%nRHM)A2w>#*f{Ee21>bgpl|+$Y|6h%Bv9|57VhklX zuX-|*g)FkZCvNC_2Ld_T@lOcDvy(|Aw?fJi$RN&n`DY`K9a>HR2uh{ypE(B+!gU<} zCzkcq(NW!SVlveKWxitG;JO1HHFl}IgCsn;t7x_F3P3FcTp61VFOym4Gp!zBR?MLa z?Mijs&}*SWY{A~o&Mb(KUr^*!EDTS;$6y&CW|gnlZ?mf+{-XSFky1c1#t1S82nBN< zBS(M}OtA!(1;7B>{9`Lr_zH_TETb&-s#p`|M;L=qs9>|hj=t0B72bIa zjaIWbD}NoV29xd!Zd!S3=WLQ?_8N(;CBEda7U>sDkiqj8yJq^F;^OW3V9Zo=kjcFO z<%+GfK8$0ddNm=%3xE~0LAp?mG2!!#TN%JwH-|Go?Wk%4=*%MUIdFuI38PPH{k%> zfo2ZDk5uT0g=23b`=xGTYIX#jx~oOSc=D5S-F@g%M)XrF+p;t6I|%;ijn~Lt47rao z0NA`b()LDx)r=0f{wBq}bQG1*5xP944s_V!&f_^;7QW?X=REmHo{{x>DSaYro0wz$ z$jl3`At8}@?5C#=svoP^QccVpBis#i{d9XEuz%NZiVB4u@!b)6_j4C{t=EeZ_yw6A z5|pPBjG0m%Vf0(b1N9X)h-MQ(Q0zn3@R%C&IY|BrN2wZkZwfD(0lH{vVo-XXv&$-* z*)5oqm}SKx*5++&*N7Qg3gZ#gJx9z27?%9{!vxaFJ6AJGZ{O? zu^PGG6z{Z;R4F_gfJDb(){myz@OE-`gYYvpMU7FK5K939)9k~WDs$#(ER#F*5*y-} zF4Ww)y?iC7K{u27lkME0W8vU1*7V>&F{q30UYN3>6&WW09Vo+b07XKfMFt%d1s3Bs zAOaUeqCnpfS{rP11lLH9ZI@F{|s{!`vJoz!7i(WzuU?sE&@H$-UR1?(|}GC&8XRuK%W6{ zw`Kq=;`Pui2Oo*Q$TGFAJ&_j9m76MmjEVo+$h@c0*(1O<{cQ$V;<`H!|BwH8Z+k6^ zrc(gu2{>*-iYCJ;4Y(C>8j^}`iWI6I#MND`%%+H>YC$Xo9k0EZ?k7r%YhYQg>-am! z3CEbTP3np1xw1;0jfkIGvdIKhc4Fz-zTTcbGVDT;SrD-YVuPY|pde0a<%MMCM6Y9= zI(+Fp(g8rzMR9YSRi}Izu!N%!g&C*#`;9N1x)UhW2-qQ?798=iAN!y#m)C4$8Ucth zj0`dfBmt8?BS(M}OmPI3S-=3-<;mckWidB`(R$!gU=_YAUj?urO|4J*FY1B(kx=0? z5Q!l9d8hUt6SYsHgXXiKnA}KYLBxz`tler}pOf9)X8xzL)o{n|coF-!h+mNY+3*rw zb&*e!)b1MkGzv?kDS9RJb=}-xd5D;y`&yd(`z9CJ^A+}!(^idRPTDZc^$-ht9|vkR1vCnfdU8+*$gm)$Yp~U_*2C#<8*l zd`@JB4L}8Z-2CYv63nTKSei`84z1R7_BWUYK`oqe!T($49|K9V9rb8280Ea|1n6{` zgnTFHQbTz;brqi|0}F)Jb0Ro0W{Wuwqa8@lNM+X3_sk^fQC|Z-{YsS>H<_vyZX=K^ zw@?!)b2HQZHQ9VC&(lfNLNF^K0yO1p(4{}xS~Je%71c`;=ULINY5KTnOj`m6-Z171 zaW{sdQ#z4_4`zHq2w^kVGp=hQ=_>`+Sq)6N@H;74EF!Os662(u-3Lp7>&~S9>&kf zRqhk0Hlw#79z-xBOWd?{pzw$70BDuSI z4(M>YevqZO*ayb^2~NYq;`AG)D=Ax*C!8kcc7cbaFu+t8nTV7ZCLRe5j8Vf1$7|@A zelq=D56_{-?3KV-N31Cw@Mgyz%&{bw8`#-Il@=*I*;P%B_RW<0|A(5G4Q5J``M`k{ z*AtQEr2Da%p~nTTbI{zKY%=kGT-*$?oJcqa`6i2Ptf;3Ss&rrtw>FnxZ^)QQf{uY} zR2X<-3==7Hu1v2nb^zmW$4~Y)#&;S9p*X3|BPdgNAdk9i80>?nMP*rERP<1U2y)1E zwWwID7(9~phO2@s0m|Gj6c}v18+w1q%oUW0#Hk}M)d{L2nFJrWD-H1-uV!>H6hlhK zL#19S;IT_Z_)U8DRxC{U8@EpoOKnn-U5ef}KCKcfP$!{XzsZzya4SG2ZZLiO=cymW z=O<$C2yqy>#!jeQO$otJSuiO5I|7}{_*JJ$Fv+og$2{0TO!ZL|PEd?U3PLfdWGCec zWQ7PUqfeVzCtZ>2?}*^uy*y@r(2tj%zjoYvirW1bG@YHjr#JhTVV?oUKP}4`waQ3_ zJFs9KZiOKo!^>k+g7Rvqx-HTljvTH^l&=&znur&RUW8i~aSmIH45^ObLl(zDr(Je#K0WUbVug#$Tg3R)NGG==vaJ8aT?2E*4}T z^%?IGxz$M&P#c)>>`u9URtE5x}vtxv$l>$FqfG!ztm``ReA za;2p(bGB_dZ9|Dmwm(~g>_L`dLMzLCoDUd4kiA7)Mik{}h|lC&Iv?(pJWCgwE7X)i zNI-2SW)J2%#AeP*6F}DHp9hYUUd}-XnftX(_Aq`8V%thdEy`Y0pWTK5VQd6(Lz5PR zjGVyhKzp?Y_!w(pc#66raS=5ZvKw-o{6-u@si7f9I*pto=6ox8NW5&$Z+|&-a}p!q=1IyZ@%^{e|#fxt4o3o;4-iZebV zM}QMdaRinHzyRO6X`9*dLl>$EYi|dA`Rp7Xs7^*?^3;7a%OM|u^Y!t`?p>q#E^Kz( z4dYC?l&VXur)LmFtax4^frB;WD5B$~Hw^7Uqk?F%53!u_A0-F;lZ?T#dR)=(io`Zk zeiEhI0qH9W%W?*jEHb9GzEqSoWQD#*dYhqC^rTee#v9t*B+lIdad}+yhA>LnRIVB= zO%~`TW2sZLigNcjkF2%uw{5Rz>^fk@0EqB|y!=;*KgW;yhW>;xTXYVu*mL8@YNY<~ zS>fx_^`!UVU`tWdIQ2&o>vTzw-~1@T_PRoNM%k-FFV>H~z?F-jt2OC1;@^9aJgA?% z9nA2QMDmzUv^owPFk-nDrJ$o42Hl0xDTw~N`rrhA90@z>(Fov05}jCC6^-@O`bD<4 zh94C4*?W~3rlmqLW9-|ya-?$x7Y6WE-~`XDLU{x#e&mSG%fAA3O%IeP<_A(`k~E||4B*so9m@s< z6QPSp^HMqOI|=QP%%m_G(+>18@wPHEJ{ZKoS=11HL0~KW2^nerh$Nadl13sQs{zH8 zsB+G1qpp!6&me@cHq#&Q;eV^uCg}lz6O3x!Kb?}p?hKS6!>C2bbz$sxTo9Ja{)ZN% z6S3JjI#723J{(WA9GDvK^qUBTjn0)CNfknKVFs3rrC^(v>j#K%--FF5VIJM;^(yiL z^vfVZYRMe>7Q$==k-3_QJ6?z zPI7MIp2LR2O$**fXF2VR*7R+f=^4s`f@doEHZf7T6(L7jSP^Pih2x8W_gZzSem=p* zfQ)}Vu&h{P52gsN$`Cqf2MXrUW^L0m$a$Z0Lr~X8ZTrDu7;?v_S%1Xpl6MwB?4{y9 zn)N|8jt}tol=|Jx@_)mR%8M~<6>ZBH))hxaOsDw=6K%ikPxzgvw}+!?lzrj>gr zoRqSY^heIYfQRE$o;Z=p!BrbEqjy01R09$(1(%8~#RJLWMUvNsN%0~^txDW;4Xzag z@Z{Pz5&5wyY(q%>=C^|voZnj?_^uD&QmwP{Y=z~E%tt!U7k=8(N!9YLx3_&Pt!uhH z#1S^%=xy`-BA1tyjrKqBQaAk`C7&6-6Ak+GU)&drbM5|2_b7re#)0JC~0s0HJ zny}}>3$zwpbeel}&?Bs6#x_Q8^&t~FeVb<)^We8;)}g-DYWe{7d^>@Y@-K!6SMiI~ zTU*EQH=>s$YgfJ)ZsU1Nmn@xympQQg6EHibsIG5Kp;tJ))+XAIa|>y6UcYI}l03^^ z^%I4!3$k5{h(G)2{JSZto~E@(=0-^C`z4%Nx=m}fWcWViJ_3GDBn!6z;A9PkzI54( z%RQ_ZL2WSL=CGKbv^8wNChvVI3y&v*X2;yF9^En$)k^u=eE=_pT}MHT#LAylA9xs!gtt`;tqT9QQ8MBU)u!;4332jl;XZ*+!aUuOdz~}Ej zu*d&(Gd&mpf}qp42G{ zk&)-NjU57&U2EWi#8u2m5Iv!oxC(f?7*V5ujZiUrpJx&NDfeS!W z@VF{AUIequzO2c^Tqht%jbKom#Q`}ZRZ1Cr8`7;taCpPoCZ(pIyioe+n$<}@5CZDG zYTY+Ua%t5CQQrlY6p)PMpr9=IB*a(T-hDps#wk!L&VCv>m&8`qUG6WU~$YZk1^wqboa-}f*ZfW}w zycE*0#x@G=UwAbZQ8Yy`5d0cLD}1nwe<@W3{jUg#RV9N!jv*=4Om-E~W&RL6S6aSO zUp{R6#?SjXx`#nf z^bV*84KvTDo@gJv9O6QdtuVqWrZl$>P6$WbQ}_s3B`02(TMs&xnYh4AnA1kY`lsJ4 z#$mWGVJgm}d_r$Lh}@%3wH$Ae!iLJbU4>fhAgxcCh_O#HG|8c0B@klt*hc9ez)YFo z4RE{tE>5^wC)=r{C^3siArA7ZVKsyhv zj~J1wHorcTB_BN=$cNFClzLbzEnU}SG7EG+ASWV=U8LR#Ss4 zx8_in8r;!v0v=FAhq0kQ6GcXKpv}RIErreds;|^2eBZ;1SnR=NrJ7ga?~Odn0N=BT z^Nh#GBWK_?^C-j9R9{atC06MUxKnSKa%Imw_~Q#}F4y+70Q#)$>_>%*SwhpU>x7RB zOJJDKwF%_?Qsk@|jri9Firp+YtfAM$x0UDju&=XY^k)6}ne&Ae=ud1nyDQOp`M3z> zOR=sYXP)Atu+Q=0x_B4k?n@hqyVVbEa@vgEDLukZsv^d$X&#Vv|0Cxz*Hm&tcF@mG zL=ifcKz{;AQf3K}lLAW!wr>Gd?jz0?@kuGmR69RII%xn&}wQas(L7 z;?Y9kDT}_IQz}~v7bt5xeTwHpN=;3APhxehH(W!5T%A%D=xj$-a`Eb^p25c9`9V9x zJ~3z!F1a}(>7cVb|5Cw*u@tQN^QMM^G|_ss>vChSur30=V5ZAuQg`1w3Y9&TYQaqEWUr^7AAR;dV!OqQ&J_ZaC*9$mMR(=33IDUSp2f%iZL_ zy(ZCi|Eobrxgj897HUvmSE$KW@9pJ%6DO8jzu1}R>_y$UYdNum62pZdkmi9Mluac5 zXL#}V$igK-P4N4W;KOJnhYGgNdYKM)2UldWEIZLqpJ6>~>avgs_tHiNWeL2^&Rir0 zjv_KFY64>;DB`A(MJW4?O^>Q#0>6}#z)9wgH&4ca2@rmWF*ynSd;=h85K+leDiC!D z@18hNs`-ppsoSLs^5JHlEv*RglSe4Ym*Vnobt%moz#?M`HSVIgNCvu+dD~5whFWKVYR1#=8~YO+*E}W{tSt zDBiEr2^99Nc36a9oq^}<^;wZ5yVr~>JB6vpzU3jn8 zsxvg1mxdj8Tkv_dYP__RSSh{9OAa~`84<-8;R_Z9YbFwJwjA>3JIZ(gW_}NSlA+73 zdxXJ>EZ;E$=92QsoHF^=?2KhX!Se6i!^6^feE|>EJqX4-KWb;K(o;+PSYZ1*&{Teh z1s6lEAYnXUT5QuWJDr9@63nA~Gq6#%9l4zl?d*-=By2xf?)a@2rO& z2io?lfGDp(JW$mkt{lT+A?= zM#PVxW^dz<1xhhZ1cL~mXlym>mOzFhalR48 zE1-ss3(uFdC6WKY)WFO^JcS7|!=GN99e>f0^LlJ%$Qy@9ihhg+$9s}FGfiq__mEa= zezs)V;$lR%dqlE;32s%r@}r&_d0gtt8L@gT#T1WHw=}Cc5c~x4I{8_rO`@&}u*tt0 z1E2Qqj18WPyt|^4$&y9%L0GwO6jw+Qaset&UeGs{L{k@`&Bh0Gu>3U!loQiz=V?Gz z(Tz+hb3E!3XlRWa2Jq1hOD2-iyc3|#%q&43RSrPKHAiwL;+BnU!~YW1BQfHEQ=b}Af3 z3Y|+3ylR&#R+gLt47&hKku%1l`jWZZZMWgl@@2u z*I)Pud)Vt$S(-aJQT;kl@jibyY}BX{PjTM*B~|61LU@*58=%Os95<1PF4o{1?TY^p zkqh;{JELFzTTyJlwyVZ4EXGZO!r-XBrQuOw>x~~o6_?2^BlxMhVd!blh#Z?u|3BN} z|0W#idVBK-N+L=ri1nW*mbIejlbScj_qpyCxgNTWrs`Go-52k9;D->vq z8ZP9bLNOxFZ-V?iwhXur^L^6>`+gNQ*L>R-vCW^u6?DDIo<`U5Gy+kT=Zx`bgB&m3 zGA%i)n}7Cfo)wcusQS!8GZ`B?T)p97pL@yKPRZzAzfolq_}#Oz9`NG zBnFrj5x~*bSDhIYP|MVPa!2pkM8uUSiRRS!@@Nv)NC<3y#OH&LUluVNKe#R1F51(5 zMaqXhq1#y0?dSl?N1~-VtoHhVaH->BIfKY#Kjtfj(Yo;qX*NwOj~+y!=&l=KiB7Ji zo2cH@u(~RUD1m2Io)I~JgZLRSwFkiPulQ;6lgS)yZ=F(SqxP&4y$Kz|TB~UkN?Lm4 zAa{4vh6()L+Xz{xt&$Woik?7m$Py+ZksGKlIkQVAeyCFSBTh#eYmHq-u8{Bm>2xb> zj75`AakdwY=(X~HUy4Qw-q8QR5qdIb4H>?l~e&nt)0$F;~z3|;a*bSNJ?ViuZ`DX{40r`oSJiJhhTy=~~5 zU$vBHETA2gn@fD#+pc>*g}*2LkbxGi3I{R~)aROIWHiv*`rsz}+4?usXq8Y~0L(x# z@jx=q4KfTMBCFmbM}QMdvILe{zyQ#+X?1eHT!Ue_62GE18wYfG+nVd;025YOrT>o| za-cJOy~*{qOIG8mNGwD=^iMeU`^>m&p_^6zW5qEfK5A8t?cHB-QTBqXY>&OSjLjM9 zWZi5;RY&9!Hs?#V*_0U_>GKsDNR{TPez}8%bF_>X2qG-umSZ}&f zxJ6TX!^@w+-Jte8|Cw(}{=4(=Kz{IvWRw!aeH7wEcZxF>3af^NPTb18>X=Ux)^E$w zy(Zx7k>uA=f>*xTsK!5j+gXXLU_+<^!Xma*is*8v1oyZ;>3`Vs?Hm_o`KVck)wiAn2fCDbTroknC3&VPs5f?U}m= zI&CD9=2PVgf8`RDe#|m)tJqaay1Zb{(W)&or0sxd885~iC2-A;+xD;UwCERpIN5&* ztoNe*>jsC>qu8e4OiT8eQ7|#GeI!BkDxPm&e6YNgg&52T}T>2sd33 zPf{?f<3^{y6blg2mnqF&cl#8+-^_M~9o6#im3s zjKN?iuJzdL_(s(32#-2L!Hp`#f@%`}4QPhtna7F4Suq%JquaN7-RRlwkslQ0~UXzBt5SVT9 zr-ofIGB=6`77gw;e%EdpCm_NPIU61h@r?BujJHm4s)u_Jx$a+97bkm=TCyqLqZa?i z=2r#)^FHl)H~Ga)Q-JiqWq$zV=OhZ#?tn@@)l(b=VLu79L$GEMu3LIfs*Ojf@Xkp# zSb#0qbwGn7v{b8Dyaa$f^{~~nw9e^D9`)5nZ{Bp6Dz*~}p8gYpV$J!kBd zR|4m2@)mN(dWg54JU;NOKsZid5`T=bW%E0AKsUj93WJZ5jeeESHbfy~fyHlFDiVr_;NV-gzILL4Mq%wAk`kLgeJ1z)JY- z0nSXuC)5i*@0BZKdU-3fddcHu*$6@Yl9sK8_wCDx!=F)ui1xjC6bD(+@ngAj17)3Q z=b7B0x@n`Z4HFxhL00fjO(Y3a@C}0=--SzY{m?nG1apjoIyp}=*FIY#W-?IPdc$ls zV8?>DkWs*Vuuief@dqd8mT+@!&6>ji&elR-l<9o2c`FmE2i*ny=CNG4cFa&)hA{jG z$dr+e4OUnRFE#lnJd=Y$an6T31&tb#Im*ZaRd{<3sbM{omL#?9BahLQqo0p%j&lHJ zK$*Yae8rY;-u$ysSJj`qtqviI_Ka4P4OnatU@$H|wdF zQ($3AH&Yz@I*v{GPy5s#;J>(->lSIwT(ImL>no(AImc{oMM13`PgIWyns})CRU-M$X8H3x+x2#(pHn*2!#5|0= zbvAnplj-j0=7j9e0?=T1>i@HV@yxy-6;>sxF_$-fXB;V`U;@M1bLbc5-bI$l--QU% zucF%Zs4Sp4e5ZlK`9~iI=r&-5b%%w`TN{R&H;IxJ-bt%*_cWwpHFH32e>D@2bwyG3oG5k#4Iel~(I zhTg7vZpt-crn`6CC~}JAgdz6p!%KNEu`ceHa8$o!iWk?!KnoKjz~0J*4EB)EpVt3Wf4IN`_e z@+yfFE$xMvif^Vu+1i2O*wr<}H`@=(UtR|0g5NKl!mU-V5fsxo(@cR3IZx_wxpc6@vtEKz( z=j9L~-G(7}neH@dsIq8Er?<1nSmYzn^0YDJ5IC6uk;tp?w~-j0*FTQGsP6?*H`T7? zr?8ywLe%ytTV+d?kkcf`Mx8{-KT2oBnr`$Q+w*V_sql!?$3;$K#81pi`X;J=-05Bp zHqI2O`j80W@(9g5c@1-(YgZpM2h-pSybK+t&LHIe^(o(}j#cQlW%-1J$g_)E%_I8$ zp;iA(or~SoY15Y>Ce`WuqK^)9u2V2TDIp`fi9(i|gvl@Px>e(E6uL9ssgXYRtv(BU zk%NUYXG>()x)V`mFWjH5eIex1O`8P8>Fg>LX^mNFcqEWuIo}(1@~QnbO2BVdh-qa( z?e&Dp&`NQP3&;XHLN@V#bloPJLj!6rqy`QikPc5(&r+#u4h(k5o!JTj*{TJoayrq%)|mhgZ~TEaMw}4^zVwqqJ5@ zprtz_{~;~#6eD)i8-s(Gk>x82rirC-rHO@00df6+sbMyn_IH;s$c-vyf~Y-`p4c>U zz8J%W$&bV50Si*S^oxd{Vl_1`!ed_#1qUhhUH+b@&_+ANzeimflZu>^MUJ^Zb*#>#fz6aMRG% zP*#!oyjOzHe^-~Y+dSz@ zn|In)F^7`jRYQ08W}6u-u@tTQnl<5MzoSTmg4z~-^L8|6YEfD~fLhS*X7~3aM`Joa zX&0>Ohu{`Co*D=}G{UA0_su*}8K`@39)Sm;{7%L+*lR)0R(2}}gVy&9#3R+G`~B~eV12EzcDa;k5jrrMR@yJ?^~7Qh9GWJrU`ue^C9VJ_);J z3Iw9O(siUdI#-n$allu^L#Qo4kSlM}2$C|`Jj-GM z2GC??oTkp^8e_P}0P~!^M4ZuoB}$j*F_W`P$w|gQM+CPl3tLiptAnSdOblvMV<4N` zrE-_PlG5WXlEmPM8|&*RoP0Z>1d8aHUY~YLifqiLq_u$K_^ zUmC|&@nusQm`wW-`}oL0A>TyxSkaZtma3U))o*D)=D`yEQ3J7cfd$^^{<!3kpZrj)4gN7 z1wc{6qS&|dakKeIl3vOYupkx0ev0FzmpL64|IJEDIDrO)UYotxFodp!yIGbz*e@)- zSb{X!o`{R?edrgC)sD-O@GW~5mNGlUpe!3Nm%Zv|*8dIDyweVPYV1QVfZPwMq!VuO znpl}dF9kZB-IEUOL%w$n_5TS}0M6DD!?OpjAesEJvLY&k`mzojnr5UP)_1ru%NIvf z@GRrfDw`<62G+lQlH7LN*((|3r@RA0hvcw9?#_V#MMI)KtA_C`vA5{^=5K|$e;Vf3 zzen;(ch+k_pa-!0^xH{nfWGutKMn-%ag!7o8H_Xbgh4ZN4V2swq2L$|bU-pZXh;tN zR$dCU@7^)6hQ2<|Pb5-jj0^h0C`FTYKFuG|H`H5A9&!Er1?w#qRL(UvdIKyb%%^*Uy?d&@SIQ7uUMywB9ufKiLifR7hQZUW|(yCsK>$ zLm^`zOlcNedb8M%nG`^Vp%ddj#nJH0z1W(xh}@}pA)~3F5Cl^h0>8a{M$0rJ%N1{i z{aFh|QjGt3?BHNA1aMQl!#<@2EGbg}%ULiVN5R%3Uxm4%@mn7>h3GByXa(kXuqD4} z53dOojfmcebKGfp#Zm#B{dDkRev~py)O#-|ff0y8V3bNb)(2j9x9tarDo*wzpn?Jk z3ffuO<7CAQ^EviE3szZ!NH0l8;GjZofGgP|C}n7>Bl9jHv*$U$e;BeI*TOelJe1D3 z)83Ng9=_?qqdPhicrgrMaw2SGaNrhz!4K>7^#4D+$5Mj53xII4Ntvb!71sejUwE%#7#PW)q3Jc>2T8GY)+|HU3}LVpg&5g2V0g+`TCu2V zk}3y36>0Ny!h!4=MVIp-f)c-fVqk{y6W=CKT+LO`EMk({);%}|u%*FLB7?xuA8!+H z*Lqg2d=pRv%0E!8p90jj7n!2J3^}<(DD70Jch}tLbmbC>cJFmD4>coTceye7-7obLQprDPy)%3XSKQcj_{|`S#7dy=EpL;1 z;$!F+P91;@UQ{f4PLfPn3}pRekXc1<>eY%_D=S-D>su~~hlXcnGQj0Tfu$@pk)+<1 zl{3mUD5W{{Spw%Q=%xqR^BK?L7RJpM91um?UnCc#L+S43>wvg;UNYF?6>f_KHFQ6D zc?XdQ?(;=`zxigux zlc0bbu}bR5(& z&D68c$Tg_xeW5jP>-@-HTM7VI=AEW**R=o392BrICEC0k8meI#5=p6rF0cardNoRN zjB=a$_~&+w+SM#H#@A>-Uw%kE46QARQhog?tJ1v^RK!AZ6-doxF|RWjFW zN`n3MrT@iw39E^A%a~sGtugCPoKCs37}Lu zp~gYJ$J8Fo)eYOdOu`M3q)UN_bL zgNd{ZU1%@#)w8x}E2ELDzSjWUkbFi^A(~XCOm0j%|48>zRridbE_SkUmT83eLdOJk z+eiJCpD+*;EU%~a)5z~x(_#*1zk=IjlLvwHjC1&M77MyiBDm<@UJgHgFQ3^C!uy4; zbhkqXz`1NuR6?*bbvMeoV+Tv>s7ME~K+U2X=Ch1$&kW-XXx?h$uocRFbK0U%k{u;o zYj}T$G6wEyyo~tUvRpKfnh;M&P8vdp5}y*>n>?{#X#OmU3mBt}%Ta_|b?;yTd0Q%$ zIC`Ct`$usoVymXvi_?ii4}`;bmsBAX>+{mV53Cn2ns?I`*@JjR0JoT}PkT?&iyJPI zOWk~yCT{xry<|-Hz`dnZ(tdykkO71Pwmi`LkG5u8m!Hk3k zpH^AiU>sv`{w-byFSpeWEkCg*0MF@Xz5kUi6x1y*`q$dujg#!8XIhRvA}#U`cs=tB zNH`y9L6nwQnd=tfE^f?l*tK>mF7^5#%(y>^l9VPlSmGF;`(cN%(QH#@XEO0}IF{mY zaD2!$=jKot|Itb8?I5hK{@sVS>e!beT`TBf?h2mex5IYg!C9TpyJVtJ;#gh{urA8` z0z$6`CQKk5T+hGrh1g++I&kp&l!5`>3Y1QYfLwRbu?@wtU;I8X_Z()1LoEc?^F#?C zh97e1pZlI1?fC=#SEhCK@(5D*$on1|j`r@PwSzugFDtVF*UYj^LCTjKb1L2ceoPX?GF?Vd5 zq69|KQrBRmX4{C8wH}XVAzP9Dp|TNWs@CTgDU7!0R}|SAp{Uxv1395YgUuDln2v6? ztArONO@;Vptfmh#hFqO4=aEAouRUMucBFO*eG-hBjJQby z^2clMH{@Yxt@+jceCo>2I3`lFY}=81EY@k~WyO+6$H3J$#$Iy6nJtHSg>S`#LYhv@ z3&m1vBO#2h9j@N~)g?S)_itIe(T# zwunqT-9+S`FRpBznq5Sg#At-0wnU1?%g$pMo}<_LP0mgIIBzTEa6D|madViI$8siK zO&@8Q-WDU0+|&@AiECw%0h`N%RA$s!dUx~KU^~=?W!zX(FH-4?7$qERu%I%e^}3M)uH{Jljtj}dUs%s@+GvBRNVB^QOg ze{RdN+Nntow<2y8uLhDCt*h~x@noLQ>!Eq^Wcy9N(_VL&pRK$;$JBXx$hVq)AbC4* zEZsNXwo^4cEaKa<%fWcJQ3eg2TC@x)|0@Lx*u3NKV!;7T>9PfHioDajvAHp!v!3z`}Ms51QtG7%sobM_-gfD=q{1eRIA0LdRpEBSZO{`oYH0oS+d*!7h( zvo>m9;G^sbv2L55(b34P1Gj!7gurX;#kTCa05cMzaXnyQzVDR_o4JE8l+F|K z1WwHc1;_UJROAa+kO)rY8bijQ2Uw?f_n4P6;v3B)_f4Y51hI`*i>gi)#_De6ewe}W zCg32arTzl#kHH&THEo*&2LggA{_%swu zvBv1K`rK1Qul#(Xk~N8=_5cet!vTTOE*2X*7!`Y}GDU)@whU6#<#TJ#&~!2^mCxK# z%$Dq0HZK_#tVr1Lga3noUO8-muNP~S_C+V-2OsawM39kcKo$|VN8X$eRh|tc6+D$2 z+?A5s_)q01jZ`+`nlu);xdg9S^UN??F)8wo5ztLCpIsNF-o}zfg4>6)!pT4(w6;$74VxWJyvkYCNN!snQT6u_mZyKcknUgKJ#zQO=oWfJFC=6nqYD0R~l)cd=b=fAa zLIC}upBsr47rxR2j^Tu@pHP#*j)`%{?xmXqu$=^)aVr7r`6E4g&UBAVM$o0}x_oJK zEcy%kGU8_4qmF_9m%-y5JYLrLdvUE-b z>f#F(1Pg(>FHiy}$8<(Da5BXzIx_QBc_R}tJZvPvAWSS0c3QEk)D-0CKrmtn(15h( zPH&Tgn&sYk&|j?{g$c|jWYzGegJ4O9aq1<*Ma!V|Hn|>c3a7FDibcZUte)dOM8pR# zsVO_kW{6_8-%!f;DV1SChZUcqd#=dS4l=cS?GJ0F4YvrIVT~L!@xBaK4P7<~;mIDA zcz=scW3~Fu#!-Gdd}A>eW5DE}J$3d7#wp)yTa-KGHNpyw;A?Lc@el{#pc($lue9@y2F*M6COJ#utKmhfK?iqF{`SSwY@N8IXFy$fK;d~pBb!_;2$*bxaJbWLn z)<_nYsbf%X+e3o~&cLiYZ8-Tn`?Kef@wa#B->4|TR9c1ppa+Urb4sWleznCC&+931 z)sVJ9Nne8-(m{#v)mgNUVIXHI`$HN;LD+}Yxp(~8Rj0=RH9dJf^2YJJQlofkGM%dN z5d?rivOmtGxb;UnjjTpueknQ&4ItXL;zv|UUH=mWRn*U)iwU_Yno`;UfR8ll?!U4K zqe5ha*|>pmqX=4zc}~)K43w&V6@gd}OPp5JQY1N&_%wQOQ7)?xR?I$R>oI+i$wTaL zAz7K|NV`sHQ=lUy4nx-Hyj;562jhOhfzo0|?;P;X8hV0e0jJ+8YwZhv(NHT+@<(oO zDW4LrdI2|a8@Csb8D5|~$wW*9BV7Fjqj}nM3rRL z5FJAg1dP>{jJ_JzkEQ)_(!u>=6mLrBOJf=Z$sOF3U~K0iSVkuOjU8Kvqn4FJxPn~y zreE3&p4ypMDbUQ^&mf%3w^uODjpH|`$6J1l6%bVpVn{rav4`1HV40wTg)LOQ!gKGY zMn9&>FwK2VZnZ}*&iLQ%ULIYoFTyM+U-!^}syJv{1mWRV2LUH?b55n}=xt%|#M4&f z8jco9F8uVijC8rE`+QrW-<@rS?xN#C<|LFT>{3&f&n+_o$!d0+jzmp=dqpOQOtUU( zC`}7VS!AUM*8xgBt>kOC|09-x@PSf!J|9RTJK>XZ#n6=vdvzqwDh7${KUzlq&2as& z*gU7^8ugifDkY9Td5MWEd7hY`&9@haP&CC*Oxsv-pi#GF9pXeXqxE9ZnF}uk`FDcZ zBC+j!5xeN0s)i6=T{LKsTgnJJiqS>sTDxX1@>bP@rhuoiG*R3Sl3$Xv-Kqj^xY9t8 zS8G6Ih=Eg0jC4q(KF*Bryze)Hk-WURZ-#poAT&kHIC{nOiQ(MoD0!x z)hIS?rCd2e0N%FtdLb;v%g$i*3&bli0smTP=1h|_RiwYEh?GnkIoQ6lMQ!90<~MF_ z1-slpwrSA}veHJI*CW%}!6iMz%y7bbg^x2>0MjZTd1@F&darsb=j{iF^y`v-JcFt> zqQdE0w-$LJ8I}y{y~_OZr+X|}iJ|w;?Al;>G-2s>NQV_X{->OMg*`q()FA7mwLoZV z6%Xemu@^r`LWX!~O6QsiY?@9*#_4}1%B$AZ&#Mrdnd@pfzskTDHEk%0=F|~F<0pH2 zJ&5@f66Zp}hYdT`H(mXY(R!M!vl`yI4sicZ>e3 znpsV8obC(L-d(094Ct6UGX;j4!<#zh^xZed&jD#(D9W5) zjmuDRn|K;C_@M*Xd7_V1ataC>Tc#=iu?Gof*Y0|2LM5ENTl5zbpX0;5;povbY^PE` zMHQy#J_&|@dOGaUTN^AUdKW6!$Gj(Xd%20&@+kPNu3*dgF+ba-_94EUXDJ9K_tx{j zBFA$>%KF`S1$$V5i_8r%ZCkLUKU2|qoItexK5rTee_liDb-j%(#HsF-9=H(XEyd|N z`~?9lHfuf3ftSREZ^qreOGru-06jDATAs!UoClS6aS1rhe$HwFPV(f z_Mdv=X+2i&+)sWs<6U+&mR5)6lR%Pg{a&O^X4ce#) z6mzARTY2utUIpso$eZirIYzBZEuJ-!}$SZ1;CjH&1=PaLeL}o4wn>lOZ0YiO`(YRLe-Cv2M zW@(!GPPWOL>{s0_L9sed?4o*eE>pxu5cB;L0BH|SK@u$IlkerWc`r)gM6?He@4$#W z&$Jn(w^ATbxO)B2PtE>ak$`(5pwJB_6hBwWd1_}?8pHMkhX*vYB@qz!rBY^K=`?{&TR_00@Z_R3e?AzDW)tsaPM^Ea& z>&oK)0cg>Mf_)d8#o#f|9N%4OP6ZJi+ldeqD_u7fIY=-JHuHKC_b#Y;ifdrs5Z(@0 z^ddi{XBzAY*)x!Ju^j#P5hZhKewkWFuPgzDor+Bagyd~l+`DDFkepd}*0I=$u_8*n z`CH5DBqX58)hy8hmO<|#44a-Vys_$t(IJd%(XV^YlfJqtvJ^;5B>w#V=}BYStdH?r zKy-|{ukkM!L_VG7>Lk%NFkU=v-=lI!s{yT|;BU)6raGbjZy^DM@*I2i`E<|gdWG_C z?rA4uAKab2lkKtUGhOJH!2anrNyPGV=s2ydQ{9&xM25s;Y9O0^z}kUpUG8q6tr}?a zaO5p6;`G{3GM?6CHsTsdQT-CZ%rKZImR{Zlo%V|=P=}sjd&v0V4&oLM+x$G|U@WuA z+3=jOQG67UBu;rn6xXfan({e`5t=^=eI8UiV1+|p_kM?5VosK_rM*lz5X_+YL;vw| z0Bt>e{m^tI%@^56HrY8xMZ*0hEM3S7G~{7w-D2^Bw%9(Kr~ub#`8ZNl0$|5k`u*)m zn#{NokQhpd*Am$|N(`+XX}8f#IVFf}g+!qA>16yvfbBy8tmp|X1bNn}W>zz`9?a`{ zGdvV>d((O($kC(arvDM3@Sc#@1LOF@7g+2h35$^6Z=3J55y|Fai>sEN+#`EiAum{e z#5LLKEao~2@~Avqq=P2U6%2=Sb)oHtF~cRl#WZ`Br5WNd(8wHV4vi}iCZdsKM0|y? zI(AY*vE!HzrrHI~B8l!{SN-L2^mgoj-1h_7Ifr{NNf~9;XjPy8(ejxI=~UYG*+%)T^3grhr{cwFzC_}4=E|@YzEsl0Zhx8t2sw#8O;Z>IGl^bxTlsis z)=sa;UaDuwi_l~Sj7pd6@!Pd35^~txO;6D#OpgL$?-8g`YCw)7OirY^EZvPttc;7} z*E9*FEa6-04F6^jV*Uz%(6bKmRcnsiGkMTfF=2!Xy|%Wh+H&=nVBEK&VYjBLN?^fo zuC{qsc{s`D3>v=-=!6hTWK(7>hDj~T_Jx4*Fm3D7w7o=Ud5`^U#i9=ZUvB;JJoIV; zBcLtWl#LL(DN)@XUpjT2QfMVDaK1yOPiHt7ZJZLgmbSf?=%j8BvN#JXV^=Ysg`W zUY6QmyHrWgtEWPAwd0cNcMBl%d_8L@W!ZUxh4cEaNVeCRmWzfXdhg_1U9f^<@oOjY zfVosCDnhZ`g{N=Y0k*3u+?h*o|IQ~8P29_CvQauQDu3TVs<*OtD)`(o?o)ZL$@rJkop#ehPQKv)0Z>iJ z`I+`IfaEhFtunl_F!QxH`E&SHHqBRTorxH#QN&}OM@n<{j&?sHUy@5|Fti7xA)l0x z5NoPj?cFA(;N79Jk}%Zo+q{wo@2X)A)(VlPE6$+qQLx)zP-DkJiy%(1u?;j<%+y~S zJ~{B^O?@VC6nHAr8eZw3$l*`Zu=7X2EIJ-~Y4P6^);S-^YgjjU?q?VWqml+{2~S8S zn+s0XMgI-)_EO)V5zoW4=vRtODQ)AhpHBBC$i*ZfYF(uCY+F(mfK3x_B3pN)LGQe! zOwr2j>T5ndJH6Qd^%}fw=)`e+$6i{pX3h3&8SPwwhlnVMY6;!zVYcy41gSPLw+5&4 zp6_X%HJlGM5}}u#82A3Nck>Ba%J@RAgkwimAqEDUk>5EVjzaG>J5P}10=7!T6E&FE zSJ%fy@Uc%+Rqe-bqOaaTYPA|SPSXPVCPgwGj%jEr@Q8zZH}>`~%I1R>P3DKmV<%(b z>`rCHUK7*#_>4R*s||Fh5Hp);tm6fR&&`RYV69EX8gMU2hFMUAQv!(}WIcXBa{pVV zFa@E+L6|sqn>_NEfLYZwulGEQN1S97l@;kkvlQTB$6NwbrG+o!WkLr`U!?D-!1Wie` z@8t>BS!oW`uuz`2*#0#NyMbGJm*#Q7M{ikeXsflHX~#@@JJy|CItyok)vU=L&g3|N z5B@!LsL`xnL>-T^_(K<&0<2?rR@3mOxdDbCSyHlAciIZokOu$Fz%Ip`;26qbEv5 zm5l)O7-D$I4CrqQ8FQW)6>_I5NvkSCuxHWv-}rV1%x`d5Byk0$rjPN&lbotNKO!;! z-_pGg&%PF^SJ;}!e~csQpsrHJvS=|m00|w!VjCze`n$mA1lr<9GI-wPT%cp_yZ+WM zq~@cd!(;eI=#<4pdNwwKZG-#SHcZRKy8`Z=`-0Y!N3J5U zqnZ4NS!h)Uzse;potd3SDF)QFNb@Okub%!YsOKbMX2U$BtBwYD_9tmMGuraec z{n7=*$--{lg(3OY7Ys6CqA6BZXg$wi?k>~5+LLB`m_7;e4gq*MT$H$Qg=SYE+`@H+ zOqS?mzLZxKh})!A=x%(Gl*TfETMxwsKR&EaUIrGwXe(@T#okQ-uv)knl?F%RAdi$d z)9Hy!8DB)*D1lVszx>H@3Pz>C-IN=Oa~?(#93DMWNAM~+pnoG4#Vxybsff-MT)1(g z9Zc9(X3teC!UoGfccHE3=JxPbZ$k3%N%P)NuHeHFcZt~1S1CiLVk#~p3MoGW-o;QE znX?bL3xo$D%HgvN0txoK9h^eRF3CQ~(r79xZ5|R769#@FPgX`t)ZAoaaiEP{{+e=K ziChnW=|QyV5KDugLmet+2(t2I6sTNzabIq&G$4-I&9 zj$V(ZUX=1kBtPfINy}zBjp;(o&RJ=^rZq@#N&sz73Ti5}hEr_tZ&3axTdRdc`pe^r}}{diXK44nGe|81RWb2+_w8826X~ z0@}lF7pYK$sW*~epkSh&27j3pOL&+)kzoKV^*dw@8!qi7%%)x|4~d-QslUX6#l;^t zU=kvtpMCR7BT+n!GF`|hVX_16z`6DW&gQ}EtHAWn=Zm~rVF*vFefQEe;q(1}5rle& zTRv;KcG*-C1L)-NDYyn^JQSwN`6}h_6~F-hNX8PuiUws%{4&DiIWEP|Ayp%h3KgnaI@~y0?kPWjRq!V=eY4 z2@=jD{lOCqkqyxJTR41S8@0=Lr1=OnY;hc%?b0{ z3@~63rA$K@px_F0h5cPktzA%-t_eEoLQwj&;HVLc07|q-HXs(oUg^^^!I(LUt~Z9e z!M4D&vL1h!=rr9!-Cl%cxJ&{26i_#_^5oH*je%3ofDINg0F)ReJ3Z%naoZJz@qJN4D2F#}B+>^*=uv@uDLik)<^2-MXpK^I_w31EgA-V_%mZQUsJkg(DQ%k zRS0gYZ&6SOcv0%el}1Y>wAUlB7OKWw_j1K7va82ks4dfK|IK6VB%|RsW((IsA)%uE zHIYc%jN?_72*79~)N3iR=^>(q6z=wD676L2y~_Bng73U)-Vh{lF@7T>n(XFd(CnnD-Q+bL}x*fqm zz^ErSC+BncuA#IVj9p)&O_dc7x5IUr>43$nX<7ULT6|7Bg9Z(mP!uNKF#|hr2@vyXeng&RO^tGlgh?eVwTM#6Po~kTM)NI_G z8gCySccG`kM>rw&AUp{BlyMT^JIW$V`(7}oemEEVn8_Qm&)+$`zABm72imV8L#b3Y)FpK3Z}epK9!OobhaRR7yZg_HXt5T7R1 z%A9kCm3HIby;KQc>=YX5Xs&?t8gu+!sBNt0oE50Yk&vNyhMuw|gi zc#Qkjfvt)uO*2_8*!JtNF5bnj(qWSG`7uPmDy4bC;Wu5GLNpp@CETiKiALH*LU^nl*RpGrBB(ak~3R#WJSfisj$~ts!})5 zP~=h(`=MnLL6d|GgbOk8bjFvNpaV=bU(aAFkmH5lW}z6!wcHBz@d|17rX;Dw#Do0$ zoC-zkAVNWbk-vkFAF^%e~C#MZS%d@h3)1Zr?U?PLcBOjofZ=fHyq~5gCZOSqjpUh2_o*5 z`t_A@{UEy7b!BPZp}M{7+!leYx*o{clM@@OLCCxI&7++g+`S}j&Jh+?Yf^o8<{dOW zIp?G%WFZE{AA+2oEW=067{(Q_0aBKV~RED+v|m+W+BGNAl%UTTaZ) zIsDnCky{Hk7F8Ozu~5QamdxG;Trv)7Sx#_caKF)2pGh0-GL2rsU%`H(j-!v0zKi}& z(4n%ueO1vZJN*}UUg;5WHex^t4LPvP^#DatBOPR;?dBOdv%Q7@{`nj zYd5foz0eA_*#1^j(9T%=kKExwKZ2f5B%@ zGCi^1x8_k01a2OCLv#Izh16%r}3!JC_|^m{SjZ%M@NSO97dCw?kHp3%p8h?C6p<3}Q1MUb;fQy6#$F8+urm1_DGU+-Mm^D^cri4` zAWLO|0l)zD=9)ZV#NTrOkRf>^iZAJ3?|aVv(OlGM9W2)J2j=UXip3nxxRdpyG(`j( zqJAC#YiMgjyU%a6h>a}nX6D?UOe|&Wy{8b;R2Tdct|Qi&Wx=zkKxLM?K08V+-%kQB zyi8pk3*9f)*w_Agu`8^CtXo83PV?w1Qk_;AE>McVnbp%MG+E|}D_j2M51niRN~ z>r{? zO^V$=5TiEjT&W9?sPzYh_lD%)!u{Xp{h@sCbRxiZLBCN}5X&8N<|_;#On;0gK?)vnSLBM(fPL~Q>3cg8hYdg`5v_s)H9~>mozTBTnRtn z>Fpt}3(dDyPpR`wuElq*&EAB6x2TEMBkkcqJx@Q_3^qKGpp{G%TBEs@8~^QpuV0-5 z+}5CbA3WqA2dL`V{3xdjrtvjx0k$S2-0a-G=FS!uFV;(soyd~!@dw{uQJ-o_KN}Y` zO93D}4f`upiQZeB{ioZPCSp8*N}_HLVggL>l92o`1scZQgs|QHwiC;z97WvxDBkka z;calmR0VJ#jiQ+-xo*ZrRWI;}XX!vkRsl6>jFOKuAUv>+hyJb3p6@~_@LHm3OTuf{ z7EdgB7%5d?)<&5qF(ipu2au^&hdwNOWSF(3JzUeBK&b`EAPZZp+Q9&U7t@V+GnLRR z%bc5ki?RT!@E76OagS@`-*em8lw-9|SZ977V+^l8>jsHFNqjS>**pN;X@1Le*j5a| z$kBb?+4b|!gX#~)5DOT!YdzpV3@b4qaqi;M&(a`q>#&+|X}!K^k8byu07&KD1kIfx zn{oN6IfseCYV~p=n%qp+`t!-{`Hp12FpqdmSUhIPu_xBlMfdZ?FUX{f*_u+M)wSIC z$x67DYh=kho2RKdA`?@=5jHcMg4(8@Xf_};19Ry1nd{Qogj0D@?3k)#^p|>jM9=$- zi{eif-HgX}T}a+0qD-C_Tznlm&Z-<8SSqli(YB$R1^4kJ+?8ZvdOX5Hp_gKk4{|0@ zBF1(C?`o3uA6nfkAp+lk1zvae zq|-S)E*4=SZ6JsHpw?CfKQ#jLDTYnD;w3|k_1>{==LvT5%B|*|=3Qz!ceT9{ok}dR zVI0`!4D+EGloNGhspar3C1v@7Eq?6W0sPsr}cz#+j>@h2kZPM$cCMdzU$^H@3Y1N{md<9KImMQ79rA>#SbG znB~I5+sa=mze%Wt1^j2bYKd}w^FpM=QMV|ADBItNAW=5DraN{-@B?1Alr92jGOoQ6W)a17 zsigiv;QGJ#=U9?Lc1u_0HD=Ztj$5k8TeV{q>7IJ~E4}@YdBb;EKQF*>V|vagA9F-hAow}gLgO@ z495khkaSlDOiPY)2RNenP4hDKLd`k?W$wkD&6T~4Y!UFjPa>G~YO|x-%>VqZO4;2m z3HLeCVo3*p6;}dU>+W=ki0i1U2c;q5fYcqxw#4+gmJtRVCBjC_JHDQ5m5J57w58PD z(6dus#5qMAmwr)-SyW7b{Hh@kkq=(Mo~&>Og0`R=e>l)p`KC?4Msg#B-Xo~cPNI06 z(Ue`O<-mrL{ap9AqS1E)O|$7BKv`D`FZ!GBc497ozH6+Wy*Yf< z1)`J@<^{2t81q`|jC8W)LEphdwovTS6f(O0tif6mMfwVz&JYalHtzVzA3?z4`XIxl z01KZSA#zT4RtWZGPPWp<__bbmK=-4WDOA9>=OZ$A@&_F@eGB8?m%U?7sYLp61mtZv zJfR+|ePM#)EsyOc)pth!%0^pb4fq;uf2VUCE+1 zib`YF=iNxnalcHb)}iyC35ZL`EO2n#>Bqa<|6eCZ{=>l1I9#?u0=&jZ#m#G0r6IyE zcrh6*bsSr~$VU&m<&l#LxB%~fK}!Uc5{17iP-CsaO8h8>hbl?bjMZ1g0Y+wCh}t&* z65MfA&S!Jj!rO*(m3;`FRyT|i{b!(#bpR-8*Q)P&U}W2QZ-}s(Z%9^&)wWvfXqq4q zD#W#zYJ7+^v1GG5{fqsR7~%3jcC9)fEZ!-B1OjThhvDG{DLRW!ePY~b4JT>Wa6elWurWZUA9#x_=<2c|KDB>aEYu+Isg zFfRN{mQ;)CoR?x|&ipN|6Xc{`+yp))A(D*Olk&^;8^&um?%L~>_DFN60`t|t&U}z5 zm~?B>_JbMoCwySu!Np%FJ0=GR3#%x!Mm=-X*Kog!|Zmw5-hJhlDrT zjHL{m+Z+c31@ef}dK^=`AaTlO{0wNg4i!8mv-bc|aD2l(nJ@CQej&cH(|gqmdR~-& z&!AV+gHRus+6d93WDe9&anrzMkjP(R9%{5F>FBg8P?$IVmV=pikV+p|>dOK*xL{mi zcQCe)P5vL?s^vkvv+sv5nXXF(`h}vLo64$iwS6Ojv9`=+o@W@JiQGxE`GR(jyHcCo z_LMCxf`2z|QjE{IE$u?A1m2Kb6`Y@M+MxQ(AA;(Y3kO72BLh*Kqq84~1QGc))K@Sa zirMoSBx`@v7b1T2ou3DW_Oq-| zom||T#}(@gdun>H4os>cXSl!9;YG7H|L%caB6f>PoYD+jz4$g@FpVLY2c_z9%%T*# zA5P6}2k?(@lG_ox>xUbpE9fj~1fiokHE(Ocvsm|zK-~iDt12V{e zjNwJ74?XJLf0}Y*080-ofO{J@Dmj{FMyHRUZj?*R0-rJmF<4=J{vq;k8IGZ4k7%Acj8+WKr?@S}~*WKI?hLiky} zl%(1K2s>B?$O!-E@TAM|Pl`g#fI9L1s<}`=qYP`Up+ug&p&3Q_Toz2igsnQo>nfb=v%=w7k|S_cVL$Mrb6C_PS(Wvu*e$0x_+X>^W0(bOHC%y*kJE{ zffdsH*NaX0d@EVK^&kQ)5-&(Jd?{9oDWG4BG^1jB0e9PGBKBxF$xC^psLRI@9Inrt zC>cSNI{_l18%EwSX83lTmd2u6b@%G#~tVNi59Sz4pG}sL)(&T zV;wh##S;Td;P14ryP*H58!L7XmPyH;Br4d4xr(kAa_qVr$v}~1^ePUtM&nr2g|uZe z;&%7_|5nIGF>9yc)>3X4oM0b%%aZl}eHu4FXux`Zh$_oZq+kLez;Ww-+}+t|NE;HU z-q(>s9+UF)GCpU1bJ%-LrtE@$Sw<@ynx57~C#zCf!~mUS1|OP<9Kf!;KN)zm>Aq#F zt2u#bWmf1*`+{5fk*M24z?wv)lUYxQ%A_?4#rk!_?B(HsTQgi8Q8@)J<>JwKzw7-6 zEsTq*HZQtUxVl1{F)^OBa*b>nqIzkW4k?pT+;53NXu|tx-IKf2N9!kHeej>N75u@o z$=YQ%3Cc6G%9ikPcuUuEU}is^+RBK0;;0M=!BLh}iBp(|C>}f;}jB>u#F|>6A-w@dm{wMAs zt94$_^jL!om2(OZK+Af9*2HtE=aouwB9BQVioqj>iu{oi2JS?!%khL#`vWC#YjOzg zhdLS8TiBge*MD#Woq66f=dfk3WQ_=w?(Pm_!#ssm8Wo;9HNg7xGKxlw&?}A1bx@2i zr7hrI=DZeriiDkWXP~{rvX3DHK!(AHpw5z2EDMXI3c<}xSE93G6v0Z54azyX@l$*? z7|#j^I^lshNM;{V-&Q@2mAPz0!cKzrlv6>H5NphsC2<)ew)wBM5j!F)Sow~-@NE<> zXnvv+JztJrdH+GqI!WaKrxUHtb(6^+ge=wMh3nv(H;>jZ8Jca~zMke2JLo)WzxPE> z+_MM`Ip#TW;ey#0mh$jk2$~DXuF!t5vw5c^3Vr}xFAo1Cst4R_9en(Lo4raJ=O#dn zxkU4q)N(!G?skq{;KYJ{H-7-W2%F`Vo0W(f`##`aA{)CnMvoJ*fS0F9k4E?{MMY}$ z^h?l~gz~j6DzFNhpIP7~q|Ob2+H&zBVMAYQLXUjx>S`f_t8TwcSl=F(XS<(ZKmqjT z%ZrBKL0Qj8mmjhq4~2$%E9&wwDs#Ms#-#{E*=r!FY)s--FOVrvDnKKn%Yha=u5|if4KLNY5evZ!SVQ1 zbT2wB&pR|a(u15R<)~y(Hd5N*ODza9`G`#k2wo9>l_8s=XHn;Q*lC=4jhxIay_#=63~~`_kgI1&RB{3P0-vjQoc7XM z-8>{ibKp@m)~0zA3+U$0n=U`8j9vFdW51?!tFL>|D^&=O2yI0B@8>}qo6q7%vIr65 zDVA}|+_Sq;hX_OtE2#WZ1zUD?rrQ`xu+ry8zS z$ckxLvg4#bcWK;0+DE9?pwN-)W~^?#arJ+%m&4K|RcQ+-;>I)gkc5M(Ws6fVeR3H9 z_aC{4TTUOMb=!Y=!E(2$ITmtLsQWH=9=y_L04KnKK0}|3kZ3lH4H>(EZLIy<(AD85 z@^H6uCXVv2rHB)mw3n2X;lOD-io8!&gPb?z<`Hm+?7ASkj_4M$QHBrT;bymXy15tU z%o%K@0St~4xMEK1A}%qq54qed!5+BlM$WZqt&>o-Xm@MZx)NbFjR&Odk^Dy=r$|B! z3%qb*{9lVxI z^33M)Szj(I8AD19>10p{Ifql`F*6=}hAe>D1R0(FMh^~1HSG;`c9gvOzti^5LD^VR zv{7IaO*Yngf2-iRQo(1EgF))jz7}13%_YU6B-A8hi1J>d=uwO4!cGZBD2y?jhGA&6 zV`ydAo9|VhsNoxrhbAy*jsS2bn>t*U#P{TBZYKAGT3IOIr*-vS&-XZ6dQFoV%Ns3@N9I_%VE; zR45teO{LjAnfrjp7<|&`O@+v9rh7S`k~}>BnKT=e%kZC8J{+RU1^1RNON;shk9A~ma3f#PU?yuJ<2y&i|D9r(!E8ZL`uD{w4Xzs$U{eX#xX?;ikg1V=Yya3XuvGOqAe@DJ zw#Qv!cFI+YlIApR)tNI%sQd07F640_#ZCgkxv%ARSsKc?z&>iSONZuPUr~BW!-Z4s z9F_O9s2n2hc1@=!A3syMH+Dz>Lq`ZEI?ZC}3b%(aUlarzxmjuE143^?RSNiPA55ylD{6wc3W42^;{z%-hg_!n6_n74HnVCgDCH8& zWT+60CCcg1<5`X27W1OjQ=5tAO~$*4_^va?Y}XUzjZ6@9#hUE#Dur8#%)1yAoigFP zps9SbE5UprWkC4%3CU>j|3h8M5$uJQZ9iK3ASzGlDtGl>wVKa;m;n>ktdM7-Oj5O( zEvGn+7@)}Itu6qAg*z2=`-Q);&%>P*`pOte1+FDc_zkM)zo*hw}(i!KTW0f6# z0qMIVEwY+sp&T@(Ulm~#LD($|xp;Wpz}zadT%uHBymVTO|B=x@o>;b3?EobYN zCD+lL==7ltZ*JnQNk1YQ5?vM#r=FAH68 zT$%YLXv}I5WC4hc9>X@R`#wDYSKP~!)v75WSg4^HR zMJGK!lmk0t#PPxSsjM6?|JzXr0jIzx;i}Y&t8|ByGR6FkmrtpR{&lk8>4;p~YnIbpst1mtRjaNRky!Z_)4N@ZLQx3;I+4 z*ZARCxdqMq)a(Hsb49Hn}udnq@>GWQnrh9QD6LR9e9^X%K^otD}*{=q>pV46Ks zam+fjS914oo_DK&I>%-+PSXt5!1+N>{DaZ(l5^rjoucN zJOl09pHgF@qEm`>#)>PWCXTwp({LwBTtLj_bX>DDGpD7JF5ytWviI|j!FgQ&)1vWE zyLY{FZz*^UU&Bk#O^Fbe@DRxaDV7nG(uSoAd@J$MNzpsRov40X!M@5O6PB}xxitpV z4xTqXxhoQfspWpL=4Ys~p}sWZlpx;BoMQi%e(N?6JzskQe?>skVFCOQI(ba0b@+}! z2ArtP?K;aWq+99MH~S5z9WSk+AJjV&YK+BrTnNs(07u$M#S`;WPi1;?XFW>WWa7j?2f>8UQz2O7K%sJ`5&7DS4Y}uzH#ieHcBx8w<=Nq4L>M+ z;2%ngC~(|geHsaC!A-|9q4Lba=CsE3=wfxxyu#>e$Log8_#kf0V+SQh;Cupf;`wxu z3mU7`)%PHHDnD^vF7>UOWeg$mVJh|;TG4#Z5lO45ax+JccbS7g-v6rv!lQM842oqm zw^oaIm`t4pXuWQ&c|o~=2W(*Ez}~i~NaL(BtqU>}06>h-qep-fOmqa6S-=4N!^|6} zv2HM;Hx_sZe$=+;K5SRkE!dll%`p(ws_$4&?H9O1LT;LhHW z#D&mfdEz01a0Wz?0?ktvqk>XD3+a5#QIbk?A)->Vi+EUj=)rhVzGu+7@&Uzn^Y_Nt z)rF68Vh2wb%@;OHvPSmQ{NouxK?;oJYQDg-;oHhJrfy>nL=%)xi^M*~R$)u@l{j zt_UURYl$#MQd;{AFixJ}`Lc}TEEuZvJ}qWLEBLK1UR zRCEV?HKCV~fUs7o4Yssu>ZOTAJ3=dqvHY+$(~->Rrd6%i~{a{C~@w6^B*lF zR^ugiY5^kH6)0^A{krSjy(f>wsMECZ7a~AKD?b{nASKt(mpK3bR>dCNr7=qIfaW&@VZJy@?FWUd zX>@3@jR7V|YPDNB*r8+AS&7{w-6I=sZ!kFEJ57oWoEzMV^@Dz<|M84(dsDnoa}xB` z%Hc7D_qTVPWo}r#GAf^+eHCfKt@}$`n(XbWks+SQg{b7eP+E|WK>+ucbkFq$<<6BA zXMKn@%^cjNi3?ffPyJY8H$e{#$6lU7_b@@g2cdPsVXKhZOA}Fl({?69bvv}uRGgG` z2-XGki_z5TTwJLloS2NHN|}n;YE=cdn=d|Ob{7IW0EAEhFBUCsSVzDSpfOQmPKNpMzcGvu0#^#&KO*~-}fG+LX}B9@DV z?wY7D5t5DeR`naRx0fzG3lSxXX-uQCch`2~!JPQYwX?-6eOJ00$=}xl2jwT3pbhs~ zy0F>J`sdH~3mvn`ao-*{v>48;%QJJ&sxCkG46RH;xKIx7;RLtz!y?t=%-2)km{_*3 z)qV_nuX?%*2TC_L3|n63WRq`5ham?`!|P)0`$N#;tDbp0D>IXqep-XEHea_S-=3?cfRp; zFE}6Rqe=lQ;F>9uYCUpg$+4MJ#5?NDu#|3Bnz9=Z0eS)KPx4~mOhYv1u)KYgwB&?< z@YF0yR_2#}q!?n~e>`+xH8o)I^p|82Qu$QwB3z=MOh&Ut=PlWHSzyl%vAaqGKPz11 z_)7t5MVW7h^!vCn2vzI_91(KoMC6-n3VQ&gF_c5xwv|B`X9O%8Mfu8%@|8Lg&6#e_ zQ*gH`(y;K@arjU78jADD$tw3u@4P3eyIimzwVVH?hwMugDH^H^=8lpSu1n^nWXW*& zqi?6)I-(mP zemCAU{R}mndA2cgqA8zil?q8ilqkAga3M+^h;Z)M0*b8ZtzZK6*fTC|l@C9&Y!xrDA3SihlD)`pZZH=awX!Wa zqon*(csJC4 z7!EQZ`dg)cw3A*EUky?sv%Y8H;{RWeTZ2|ond{0D%vBIBP)eH0#t0*@q10=!q-(KN z9Gb6quR|!B?{WRDifrQzk8!%Fo;h|s!<5iCkKmx-L<%sJI(n9UcEL*Q!?o*q+F6=! z^#;fULAe{90&~RQRzQmG+!BKPsN?pod`o=H_>jtUrove?S(w-4C*t^nxX{Cgal;F7 zYhr&u{;@QuD43cQ)l*r;(e}`?5g>z4AWJ`r3#BG*cw~yt*@zfN8c3)XKQmnYIW@!> zGGpx!Lkca>N{)aVry>&g~X`IVt}erPUy>j#uEg&F6{{FSKb5vXr>vabk&g;KSh!a3ow@} zN<#({U-;)uSs#d3?LJTQvll{lrE8uVG0Tmi(n^Ff4YiaFeT#c;@cd(N)dIyO+(auy zT_U8d66O-Bq{LdKjV9wXgS${BDnMa*Na3$qnD#J)Na2{}31NImyM6Z=(!PY_%&8l& z7sg&$d%9tp_7U|AaT{hc7dz>C-Vr(f#27a%TWOC{>sZ@LcKYzTe+8rp+MIS5VM66m zHe+m>p0EwgIc%AG1R;%6Tl+`VF3y;Qg7;_edd)rC(lR3T*TM~Ap$apIQTHF}o2wu- zwU2rVRvr?NvQ*`KGQo%k6DTS?1e6jO`=KG>BRh#r38uYiBWlhr(E8THgTK`)3H%=W zm|Z&23E2Uza(){a_l&`&EgyKq6wP`!J6g4-#5zZ(&Gkuoaa;4Il@CVi@48(DxW$i> z9}AG$x(BTRn}H=n^=bE@F`K8(ypyG;q=l)r7D@`&ZkNqVqT^U)L!Q9rDTDj=-^aqZ zkUnz|T<`y&8PNd_U}5BTul2FiuFs9Q%A{Q;eyld|7lB#zRr)IG=xPETCSpFNqF@`| z2@4`i$~h9ycqZF`O=OBBC3FPRzfP-ricE?Dt1XP4=sScJ?=Pvt!1={=bgUFfJdGuy z0$-y*8Ug4s;R`YrAS{fpqep-fOfdwOS-=3Uy*rFt5=ssY&7~ExMF{4*8y=`6u`X`t zEq1U2UxumjwT_FA9vz}zuT4aVKtW;xTK|`3MO|bYl_OBR4NR+ppEuIX3$ z-~%{%S38(ezn`&p*jW(u;_Xt9@ZfUvm5l`FXLEcp6^F=^zf7oOU6UMR<9j-Ia$3Ul z>l;;%X_df%_JaJo(Om5tw*b+H0P&^vdF%Vy*A<8BEVDVbcj~QV>oRw29i~1`_Ck(eKMQB@4A2h7-KT z#;?3Kv4U z=RP&}ZP&-6`-09wMXScI^I%6UF_Q<)u_LNUA1vTW{Hh@!Wk^BUd^I$DvFRY=z9GoB z933X9V_!)5^-3WE8=5nT)<;?daFe;ncrF|i+qfr-65Vfd^`^;!ih@0^=PV`CeGMyx zcvunu7&O5vr!+1o`P|;qJ?z(!xgfYg{NBZE?5skB?-#D>%*gsnMCJ}{chU;-r#B=H zO_rqPpWJ6__?Qx>Teq-6>CUN+E^X)DOl~*apM9~Ij3`UkiV*kEK8(GeE)b`Nj7L(E z-+ADGnCHh!o+H?yUq$U4-R5rY1c7B#h8{MrVXNNK z<3`jIOREqw-3P`c73vMEQI)K+>!1EZy{>rr4H)g6yUDPwJNlALOUEU6Y}*5hvL#3A zbQ7mMBB9vl7)PFhaN-|b=o(79z&3rnfJGBJPF_P@N1%44@` ztu7oUG|NgGhoYx!yxqCti70cKqEu~f;R5k3)7pDdW&G(K-f5M?Mdz^rI?01oYjr}; zSaXd=UULlHCyIMZ2#_q%Gi&i~cUxEqtx!|aw)`y8-2~coc7GxJdSkn<1-)#gQ3|!WzlQ6-^acd6?P7~^u$gW1&s?F_4)S>w3 zKWW0(Ou-xwgS7L}t10?iqkj%ss;+jOxC**z(|+;s%U0=sWObd*R+_x@DU^{F2aKf0 zuTzpNYCqVhnC+VS7}PZ>>-ZSs47|XpFOs}Fc*_upSe${Ar0?7jexoqqqLc0c+$RCx z*&3DgpH5pwmlng|eMk2rnRq^v!6(F_*TCk20IWWsJu_L-`IQ&xE{e*643Ff9Sjz|p z0II6;qh@ZmQ1)Pv7{u63p#^5t8DaMH!&0h^c$6J9HU0p9|5o{ehna|zQD-F=NW4oD zJUg)gQk?SLK%hfotVwESs*C)Y|5#DUu%2u`Jbwj@nty$tDw!3(3gV-DNfI4hVwHhE zm>7&7^AKz@&j>OX06~@E;lPeTvP_8UA3z7QU@#{4<~_$M7Y2$e&Fx$H#3rpbF$OaizRQ3Mb=amZp1 ztj)ms!bT0f>5r9Kxi|?19=5p~x=QzA#~P%$H`ojfih18v52~wZp+2=#avX1*m!8PT zfBQt@g!R(6>VG`5+DQ?8Y=~Ar|`C#LTfw2si&eW2T5QW^g>^iWLSi8PE8qnJq z7cLJ6O5;lO48A@LFT#W-G*a3fN)`@%1p(C0bQE3(DXUaRP3Gd);wHu- zn9QbL665PseL;?V0?Zp8|Mvdn4rKt>*GjYSeh|?14uGW)Jz_A5AyoOP7aM2C&9LnX z%nUKh6y_BmMg)`%$Ono@N?~=RS<#w=b4ORRg+sSY;c$h{b;UzPUn85*F%AU^I>D_m z8l0Y*Bkgj@yeyN9=OU2FR{zJ{W!2G#(6Ls<`b|CI1{+PqKCL=KE_@({_sCe-qK*$C z(_sLe_tby)oxB20TF632(i(ir0=%Da--g&`%Gw;uN1*BQf|2v3S%^DM$2`RqTQPcHBUa}l#3gG6;E^q`=j-vhKwtOE2+7&&exfr-C z$34cdh6pX?YZ1mRBnQXDq$H)^Ox~jPP8SPye?OsoZ`@me|6ePWjfExZ#z6Q&7C3uZ z<5GV$w@Uh8K-Sp9Izk1L?GTiJ+h|#M{pcb*uX1ah#kW*t&KXry(-ga3*mQ~ zNRiV?%6Xur9f8*W81c9P%$cD2$Drjk&3Hk?4}?P7Y~rF0oz%YrATpctC`)`!2!Ax@ zuXK5|+Qz%^f03_XH)h-T>}C5WFXr8&HExTx3tx;gwquDOKZ7I~ln)<@UQ2;9kbM8; zGT#T7`EJSke!RV!A0lpA= z6PPA>?EV=!UWHVHDIMYI;xA-8HyQ!hGPe^c4bVj*J<+4U1(rF2O9J2kz1U03BQdtY z%xjIy?zt{H!6Heqel=$Vc&bHR+VF`4-AOYaE#*I-+Z~=?Hkb`1b{(Q~-t5Tf7$UPB zl&Nms@O?F_;8*8RcYrR8z7#b#*RFO1?fLo$Wm-CqZONf~85JoT7oC| z#8>+(fwtI-&#ynxo|VvMC2aGFmwCF0vqki)6|_shF{GZ93v31A49aH)rFJbLp$kXj zT9InTW2EJ@09@N_r`{vc2`4Mpc>1IdQRkY%$|^|aRU^M_422op7gA}h3FH(y{uTXx-{Sc*#RymqL4-v3= zT@NdBw$9By4#1$Xf6T8Vk!yuT#lg*RI|&5$MHb2fyv9}+0tA)Y1Z|*Yck6kbHX8B^ zdrb{j9T8^yba*mnNTn@ejCW6=&ru3o91sU6jH)6kJ!qNE0&!?8WoT5u^C}f$HrAfS zxX0jsHf>gb54LreORtRTj<2jV=63JyqW(?ow}{xS-K)%RY$D$4O2&9m%eT8c^+TPs zfj9oq?$-}+nm|}4xO3<1g(w68-ab*ElgY5rrrsVaSsOo}_n^%G|9@}!96dU*nVeXu zi1!K?w*siJDdYGYG6{&>-XGx$OG4>Vc{b!z+cFrS(^e_Qp<5gh$Lklq18}le=TZ9! z!%9!Co}N@%?M0Ja?$>wXF`8i@zO!$VIht7_f_Iv^_yiWBd;RMOsgje~(jY!7Dtv$4 zr#fbf`)U@?7oJHiCvdtDUINw(ZR#@cuntt_YNl2m3XQYFh%eo_YXiKBMY`Pt^h}|! zC?hnEgazrr(MD=e<*WzIq|132JB9R7&C{_ym@^Rn7AR5xKn5rh)OHr%kP`LjQu8R$(S~JIN-eA%mGm#h z^^{^`v`x4HCG82EYo?P+llfi@bimH0u<^UvsagTRaiGmwjOR`A6oMh^~9-e;w8{cpFL*HgvufT?rBa z!;O(QWd;ObKIIMD&CRTAiI86Y0&>W8BS@^r6;)RQ&AAP{4tx;ouS2#vt_7%isQx}hcBzxLmMUG}q`+;=Ua`dyTn8#~Bl3*TCM79Iq_4k% zE<3|v!0C!aKqgAAKqNcG^&0ByspD|-mRPMZ3i(uH4z#~hj6M<8v}+Q;{w?wPK>i}$ zShhev2J8qJKmAnL=9)}+t44!Yvi<-o^N%=Q+>Kkbws2_|!aE?=S6`U+2%n91ykzI{ zSj6O-=ga-(@@xtk?~5#Tc=}GHlB!7bHK2mYhOfodWkf$gL<7Tkh!+ zbf8aZvPr%@4mE?!BHUrSECjXdlzULf6+=2F@i05#~8ag*?L z3@dd44m@6^)S*BCdKsPSP@jtTNDeI{GiE8TcB;d-(fa2M>hF(Z+qmUh8s-#)9@kLS@u++O)i8x|W)?dW!%mXX#qk zJ(4MCWD+C4Y?MWQ5SVm9GSC$|Yj~Fj{QiRUx%LmCFJk^$pbo>U_Nz^yPS98rZ{eri>n<{3Tr?n^T>~LBaxel)aSzUdrw3-jzuzbhpU23xcC;;op?g)1TUuN+ z%V<~rhZlY&ideo?K!WD5e^nx8@w?KX)3=d^N|G~lm(B&C5QL6P8X#ZWtBDKzAc#fe z^Q`y=0X|k%BuoxUQuFeYX)#kT`NH?*3o`B!>n%Rwu^E4qy?ejZAMRhoVGk;A8X(fJ4s8=)YyD{NH&cEtLGikYm&xc-(Ej(8k8)e*{=5 z0LpXTv)urS%Z!#$-P4$8aI5~2YJR6_p&D&gHQ)sx{q|yz7+5_vit2)L^FW6&D1Q(^ zUuz30rWoi%VlxRp5a@{A&qtX7`chcJVKH@U;H1J&Kmtt+gigS`mcW#BnVw{p>d2vp zjdTeDyE3J5XV&vma_F7t67Dk23o;of3Pew%M}P|~GX$0ezyPm?24U0)6cvo@TzsT_ zMrXSf$VL%EN(NF5q-GKUQvWwzy=M@n63jbi%#$5W+LdVpf9BRd62hEXm)0WCoESf?0>yJ z%??tgzuYT4ScVsHHlGA*!b#H7Jb1*ilIIj&5*;wJwC7EUP6D>XXpIPABJINr|4zk| zI*P#=6^1beZNhDj{5IU+6dR+)&`%4L6+T}5_sy^u{hi}%(s#vIfVI&SPxOgJI9|Oz z8o$I^c|}mPTL#lu*60io4i9=&Va?80ub|oM#5ClXMd?&wS#bv9Ywf z#ULQI5@EEkTGiO1?Df1_uVnM*!hja0-2&oqklw$nbAEF}6DlR_Eu|l}_2@EmgS`N;YDkQ<4~W#%Vvv^f z8H3xTIk;w<wig(obuGUDNUksqgRhzut@4vGERy#vj6Qmp!eYhmw-kBt|^*L6;_I?_M#r)BG z+gm<3OiD(vXjVp*8;hGysIvP+$@_0^{{F^`7loh~@#vhIV=-Wmpz}?Flcl#2ezc?y zJG|{*ukZw)vs-O|YSg}e-Wz0wB88;`KSKj_c&UxtO`cOc#_<1I2h|lb+8)T<;Pq~W zhu|y6vfR-VTf8Z6qf4L>I^qIysGfe>f^pCSklR!swX$_J8dJo$^^#Nr*aKo5t!gK< z31y2I3(uV30ey_J4i>6-=yrr^QOwV*z z0!lqL+%Q#h(l??w(jG+&L#!HlbkMw*&ak6PRp5E2`Pzfne2O2l@J|B?V#Un^U92>! z`>WRySd%P=^2;ArLFO}jr9s-2dSgZoRQ1m4Gf{^X?Bf4wWBrz4R&CNd1yi;8sO$x6 z-LfZWVK@tG16SJ)B;I1-j^(e}d$DCYF;qaYmwR1+{X=133kb*C&s^9{GCMFnyQ<`d zD85^t#y^hBxONXXoLH3<4QN!I0!$R!lRsoKsw@4W-l7y6d8GsDCY zt4>M2CfS6FfZ;R~O^UEi^UB3~n!mad^E+IfY8bP2X~P)#Q8z-&@}U-n1P4GpZzA*Z z38JfudOPE;!*=NePi`8#J{->1KpG?=qlq%VUTuwp@6}veF&i7!AR!x2C(>@f)taIT zp~gZ=PM(q@YhOkuX{Svj|5QsL1B#$e@k%()qvGZLtXk9!=kaYkp~$wng9=TLjKB z>k2X&04+>Uqep-XEHVU^S-=3ayF!22XeqzMXob9v?rH(G7%J^A(jgAae69U@A%7O_ zpNk~DUe{S>4+kBhos3|Xcb!Kar61Z0V33C5bghW|`&MoU-gt$|ij8>T=<#HyS^ zInaT;pPy(dx3bs^TvIIEsKI)lD5>f~@!BCw%?`@Y3Vsu*XJ1C6yAHPHHZri{rss{S zNvu?{&cl85x`4>-`Hj?lyh%1p$9>^o^JaeartJNo8uGr?5ZC>?Ca+I*IJZU709cz8 z>Kfv&>mzUELI2{>23kJq!6xWX(96M3a)5jxtNtBuO1jz(HYc)g*LS>W>!j6P13KIptp^t2j%nO1cMP)sOAAKEj~f(4o>K zRE#)U4UJ->)nnK?RvjSw>eLyX7$92pfxH_2FwHnr=)bEKSM4>l0S4D62!o>8Bt=>APWHe*$o zpj+T{{Yl5>?ISI@$O4K0TT`TiKhwZa?GC1mx&~|Xji!cU10E6HoSf`yqabu)(AjlN zB6o93EyM(t7`6)EJ^pGcCW*1WnHh=1&G>C0S_N{R`{!mA#U3f~2v3K?%(FVTM_1OD z&_H?}G+D5*Q8UKg;lMTZ4@tIXOf37GYvX$UU?>z5;8ScH-#(KnI!BhC|NmZ~4%8#J zcAm$<07loeF-~g8_MyXS{Qr%u*1w1Q{-BS~-}=iBeb2uP$GfLrq1l%!G+78l=7sHt zXmf@5BL$oo+RKwJATmuKQvd=cr+-bZFmyN{6-HcSbMI#Ew23}C*Dg_BM9!#e5=?p2y`@EPn;vC7{jm9mcC@FTxk*JZhq=4y zaz~UwA;N##mQyXLS zco!fyHkGC`1h#GBY)YpdFcwr0H-YP=b3bH#J*poq-#oqTgZA<9SK-5=H$4b=DcC{R z-itN7%;v^UUJqs^tK7ul=}^Q~zan2cy7ddz2%{imO*_qB+@Ipjn{G*&{rf$cK| z{=Nr}yxlSoM|3kAiiYr=y(vx2wj6dIA`qG+;61&mpWE?;M!D&KFLhqB=Z~ok=k5{oEibBGR6xs93VtAccVvu z3oLR3mRZ06y@uwd2MmkV5UzOo=|2N;cMC}*C6URdloBK2=CBQnG`sK9f8TJup)GUS z{ZIOpdA+13{S>gZ@HLer1uahjv}o`3@1iPP;zKedU`+99PB{{QG`2Ja)d{tgXDG(+ z)%K}?4TJ6iwaxO^NWca^a4ZE_joyePUrdI3h>`2Iq(!cpzr4ClHt2y?zz)TtJ+4mc z_Nj+Gvb^BZtcB?Hq;%Z~kwj*&7!F^`)P_tj`vw>OLC4O=g6ktP2Scht7{$f?0=ftV0$el0R9WN zh*t2(4e+2mYWW+GN6*~}UYH^;k$Q$hc_3AgVPL}H_~uX(*bJw%5=FY^e&fO%|9E$Q zDz5~MV}WD5``QsOfiN*>c|Nn+KfBd3f=e9dM{Fy8d(A(rvW4Mmg+NCqO_&aEP*14{ z<#%aGn_F-Qahk@TYH+=p%xz*u;9_SvlK~F`5i&viKjxBV@Ey;DClZ+qGKxB&i15`{ zRef3YVBGQx`0KVXzyYBMZWg1M)S(d19JTSfDP9ha=jvEi3fq=rcJY-uxLFJjS#lbH z*sLQ3g>p_*>}RmEG~uji)+Y0Rdkp1CyH2kX|Gr28{TEJ_LUFz)oqQ zZ9|gWo`f!nLDNXD+ATIiIP`t|YeiUPo@<+0=4QpWFJ*nY)JUT$`&RfzWpdr-4~Hj! zC__pPlU~x+8fqxQa%&xb#vLMLx2adh^$9CVWzec`V&Maq=O~QSt)$8AxgR*&gMuGw zm_HbKT>)H|fjr1xUl5;gQ()otXM(tA&lh#P+^vARLLq#UQDiycDvCt7&U@Pkan;nF z;kRkOMc$0N+oVWt4yDcMXJWH#okscO8CAAuxY68ia6)!EWV+$z1ma2Ul`(2h!;K!I zYEydc!RjO5*9+;RL(XS-451&0bgKbS)tyqYdSuO2|DmFVP_L#9z0l^{zGa0$d9cuf z>RtoxRjg>S>O-U4)FDV6E_&Mqg+;MB`;f0}$(~@=G#H6~1K#-&)Ol->9mH~&xt4i8 z;DS>0&CUkm_izduJ~G$d%nW6hlKucPL`FMc;GqoZ-zXSTHWzqx?@Uix1QjsSp5iFl zrCZz8L{A~)qTU=cPhPqZnJ$;`m;7)}ouO(9%^Mi0)E(@a#KD--v*mtNBi2*fG@1Po z4=>MMEH>~S_iq7mMxBhKh848EI~J^g9XuUQVO2ymDG^JkmB#vKt`8-E1`Q!7Lw)rf z58`KMVU8x#zF*$5RkIpwSL$J^M3{bTxi*CxXWPWWjl0B!m z=>M5PN~UL;2sA?59aOsEC1Luk2ThT@iCM&+s@9)!W{-NET#*r$ddB`ihSw`zj{HdC zK=j=wK&t*i%I`V8>21^&%scLDz7E#7wpr|p__2BsXASSrN7A*|jTvI$aN4z{R}X zJep;qJMqMwZ-zMbHlC3OT^s;ivo=ProJ!~Zu@`GpM&b$q_Vn+rywz*lv4s?-kX)6* zz9|ARo!x8&NVU|HdxsmQ!jen7z7h|s+z0w=*Nt+W7zY#;5;ZEA1t{^15->Y-BPRYe z#1(VJh5s-Bqqr6a!ZP3pG93g0gnOe$fD0^h1eOKB0OQtIg2lpc1MW3^gj_$+ z=&ulp7AqNL;&IQ`eblU;os(t1eXSbFZRNGT-@|8GfqJHh#Yf5I8#A&ZNfFuG$-(9j zUu+8Kg=e9JVBSZUyU8q@G4ebWsALy)6CqaUWjPT9AXT=K24T88)?y>QDK#Fm z+GHf3AyWr*uwy^EsOMvm70ju0p|(wz?e43fu_hX^c3mo-i@#p9OoE=_bs0%2 zZU>c-PJD(O2t2p5BRM`A7t7Gbe+Zldfm-^_7oTeo**bMLQNQcdCo$~9(lSrrQ@W5R zs~Tgo)y8dKIwY(geJdt*HYhs*E#23$N6-V1h`QDl1lu8ON9qT(j1;fy_`0+;@bk3B zD(VxZgvn>|05Y7ETVem@xMHgWp`Y&rH6<4XiKnWHkgG;=JtI3QH z`3e3<9O)ko?NWm#be~d&m@AaCK?k&b3JP(fSNt{2C(HY?ZTX6ET7V?w!^2 z81Rq){896;%C*@PDLobUo}t))+VKntC@LQ)yfeE?XsvWzg_$4jF%n{H z$5I$Uz(PJf3CBHWI^{clr4HhP?KP5NK@$bwxtOM)*~+l|AL{i2bBgIHd}6y@CuA9E^GqrRgJ&!;Q5*gKKe;6ZOG%K^kuM z%LSFpNFTpDQK?vZciuaGC#Sk*y@cC1bx;(e$thh-U!0o6)2vp&^!F*T7be?X@X()2 zcO>b|!$2~t)p9SFp2fXVOpOZvP(FiB8%Bgd_<~MmIC#QUF9D3c8#fbwM+*fhOwzYm z)fuW$I^Z3;B^p`+{8LDAHmMB zNlNd$K(Wclk4;n}JpYK&^AZE|%HRc18Z%5DsisGXfswH&M&Zpqi4Q4ZjaOTf)PqKC zjVtT?IPYy3C0pT{(+y>7d$UMix9>V^iE<-~2%zh#(aKrE@;rCb*KkYKMsWigiDQH3 zk1n>h_%d30a+=%_)BQriv$t!sC2nxcIW4Sp!jFXj?l2@q<8A!ij3x+!nneF7W^W)i z%PVcAR2Ei6_BAHYnY$%ZZpSJxpo`Ie3h|pO&q#=Tks{eav#?`d(#;#}!zHZ{7D=!^ zU~W13Fkj32#5GciVcn8_kcpPZ7S_nnwXVrrIf&l7dBV$JjM|j~(xfMX+a@5`Y3tH? zfX>!(P{wuhNx5dNO*0nZ_+an;mf%DD(dsiY7!}i_(cc9fI=b#SqDx<)RjU8)OVKCr8 zFDoTf(>BrTADA5o&Eysu_=byIi50BC(XGONU2Gsd)#dHWLl7V-#QH=Li=)ag1)x=F zv`Y2hm^n@eHcKoC?Aa=U-M|BoAh6()A9O(*$4oVvFtX7YD6wq#T9w2vgezbu=zuFL z-@!?%9%5O#LYI{Rlj}8Z+4*<0?Lq4ymb^JlkH9ex!NV`^?!9$S+s&}6ka{d`P~Lda za!p}ko6o>zp*j7DeW}43^^Qe<<@*VtV7E4o)&)%(&qdGB+q_y)kqGlWTs$%P6LkCj z+o=`5tjsz;m?gQkDIY)L`UZ>W1)Xkx&NAPF$?Q$)-&^_?gpae$*&Sy3xhw?g+H2Q5 z4VYbz^f+M38%6@k6QQ0`n%YEIAO&<*^+3||vb=PyO)>;46iOSmBs)2+zg%+mFBP|q zzq;Q0l_Q5A69qcu9-G~fsCG7hG+DUp?5EgrY#8}Tc=T)VoQq*`m%!un06Hh@VIdD( zt}p+zNmk@{?F`%e7FK!Ls;)N__fau4Ig3O=e2+QtsWXrcx0>h|;3Lf^==D|SPw_uW zfTULwhhgE*r(Sy|Ss=_==#k>Q5r3b0I#MQTttB|8rLxpGX^@P1k1ynwPs}$8#+*Hx zdOLl#xQe@7OeH2vzZt>%YxaBCjdpK9_kyjK#CO)4S_hS|yu+aoi_)NDQ)&WFM&Ear z2Pqdv7^j}kuf;IMkJY#R6?}j@hu32=4uEw>92Shf<75(!I;vv#6Vwb~#1aNj%>Hc- zz@mCjcb^bd7Px}kfa&c4k#xZMj97@0W!CSb_90&G!RNwDQJ>fhptm7>Y;M3LnfN3( CtB28C>Z;wLjG3&=^o*qim;^t} z_f|%ZdVl>s2A!#)rTwP}0N9!tIR20Pmnvq4MmA!4)`pfw|M|e%nmJni=K%Eaj&EjZ zWAJw-3;+Oqy#Brez%2m)b-E9k(#*`-_^9Pj&zX@)$i~vG-SA}9=V`*<~ zVCf0~{_pjn!Z5S7`k(qf5(5PS_6PllCSYY__hx}{2f4;QhLq?tc>(unCO`pnF%e;^!044 z=sw!z(+&V&zQ2En|9b;L8=Dzf{udh2z~H}41N`{(#&mG}Z!`bS{yG$WT+qKV>PL?M z-Dm&we{HxQy8mv5|4sMd|92g5-G|NnL;j!fq5pe}f{!?$YyWI6p9J}r6YSx~a<@+%jP^fU<_7}- z0P7C`|CK-D!;yXx6zY=*FCVslIlz?uvrT`Jk>cZg!Jjfr`zOJkKZ*BolY#y1sgIb5 zqn`vX`DFUXet;eS(=~m>ko%Oeya51+fKM6jZ&Dv?K_Wj%`}4!^-#&toeCYqxhy3w~ z0+#;tf#>fAgbHVhDW_DE*TV&HwB$pQQYkBap;Dd)p_0UO$Os z`ETy`&z}EDVAD^^|M_SC$Oq`_rwsMwlOP{^2ekAlql$hKSox!_zmM9Fx}h}w`Ne(I z=kp;;e$;RE!H*vQ?{WMg{_Ww9$2$0gzLlY#BlO>_>7?E($10;inbYck5CMU@T>vCd zv_t@+3ct8lU%=RbQTh>&r`YZ6)rr#g(@-*8e0J;{ zlFxMM9jSjVqhJQ+<>kQguxeu=5EM8~GdcwCZh+1Rri8W-7;i+^Du=z*z`4*uj`Nls zUwU}<$r#PZ1c1XAltNWa(%4gu$k{T!oSLu&iyk?+mj>;Eo z#hK(!##>rjSYjf;a1?+ZE)+m*W+zsb_PovIU$6s@RW~?xW#125{WC~3(YSH&2S)Lk zUF=?io8;vp=rd-mbw=C!Re(T8#nHY{*N{d!6N3g5(|aJctuZp*)!J~eYs^kFEZLq! zanH8PesG(s6m}|Hyfw5*1z(c?ptwy8U?h6CB@L?>-f5`~zk{u1O5WSahNg82*a3Qa z@0@7&=c3xLPEOJ#pt{p9!!h4Ueq%>{XEHTx$A(_~_MOms;40HLRfE?@W@1rKv7%-H zVxwOYMbJ>8pG95gLM=o`{C#Ak_=#End1 z+-Xp&ep{39cH6;HPFBi1BZdk1B?4iq+biBsmVhkAU~aTIpKdBb=dHSs$NNMon}IVB z6R)?$PwZZ%akmzO%rBHC)PUPse3L~Mr#@{ElSXP$Go+L2HU90rhPIi;hyX;W@K7|4 zQMfNnXD<#MuFuas)@$T*u}3` z&4xlAaJiEmy$?js3+Ov-Eh10MD8Yj#bv`Pd1UvcdUCt{4FQ{B=VV2Ll{`=@E`Mpt> zWp~>u@|#KEWLI>UWFi%iQ!#_WbVI_Z*U^fmon4PKIR-dH!t=Ztns%Y7m0w0b5X?w+ z)!jvk(nD9c{?Rw0-OD`UhE){4OAjMiEOBDUn5}gU290lx`^KOuy=C9a9n5b~#Z1sO z=>){Ok7jXjhU^E9;Kz7lIiA5bgH`}}AUcrPB^_g8p_>g~@R?G_Pn_RWMv6prga5{I=L&~TOVyKwRFyET2W zToQSN8Bik*ceegq;g{+9%7hP;q&q!D_8a_y=3O%gIH{9^1yL+|6W0R@H>VVC{k12p z(TXZS3IZHxh(y=8MW>O)Dyz6uie4KdO_-2T)Ej@BE%W07V(yCrze_io;7=OgJt4n} zAHdvL^*R&+juE0I77cQRE+|5WKT_(oe%{^6>bm4Y+sN~5;d1uNP9(im0wxv7iLC3y z#k!wl3nC^TOzBmQ`jsTY2_Q5Rd9i+o2u*pl&<@s{9d!3~a6ZZ9_f0h-uGz&MIQ4Q$V1>V*^H) zgHVgcouZc_Cmw6k%*(0#1tUajK?^m$AOL?BbjA~LuXU;Lwp>v!c90ISh$%({`%ROk z>jiJ3wa=8S|9RVcFnhCPOL}4q7iC7&i+*Xxzl%iB!z)CjA;wx~KikleYDiS@OMrIP zg>Lq^G)_@=tj9{52W&@ssM@cKfq>ARALNp!<7DGQ+vR7z+$swWYi+T5-}8HodzcfjxTp<1IEmmHCj26b?^*qbDm$hYhiG% z^YcA5qG#~~sD);@-(I2?9knl|rQQyw5!h~~yJ)9_p>d=a+tlAYnr~mjT&@8I@#iOu z<3s~zYHf5jF_|dxY~pi3;Dj%1!A#|RvEi|RR z&=+d`{J!H_npG}hjJuoi@X&)%&P_WcU^ARJLOZ{D@3he@S|?U{N71ESm^$9}193yZ zVQxW$Yb=>)NOrm{tfZ~iVU(JlWi;S|3rjD_l}8~|Bhz+G=b)!~#YLeOgL?1TFr>tA zb8MScBe#-x&!(F-&UWm#AgICQ;7ZqdmXwxQC)XY zW@ZJRzihB0{;}-vFQ5`r?!EvJ{Z1?rGuAql71P!ng?}P~513YWgLhF)Ec>&&JyP!` ztN~Xcglm6I)4v;=f_5XmfV}$brNzQEKp8a5wp&LVIodZ7gAE4+mpU;s>EHY$=`oV& zsfp%rpWcNJll+iTZ+x$32(?q%n?D|%#2o0%dA`pI9noP(RZ4|yg_gYlrLm~S(fW>> zn_l}HhDUhl=6Rw14`iTU>C>v3@qz$C0wS3P-Swh_YEV~|0$+Wg|4Z>rO29m2Hf zYEptlHE}GDo(FX+f;-W7rrY!Yn=q_%aDn#;rI7&E%baQqYzbo!Aa#4laJy9{TGED#doPjup zix$K~MeJ4FxBqHpyl)u~TQGMsgec%n_a4f+Mc@ zniO6BYbh5r_q86KO*o(6%+WAnPO*pJ#e||T3)8n)B`f_>OsQO}3kaq|&4Z#s|ZKm#C zT{LifAAWGQ!U~>?;FpMuX|I;#j?LXjl_zQlY0R$!M373;5w53*Od1y4X2B=PvU z4tUiJ)`{MmApQ)rX5M(q%=?Bajqmg^59y}qU+uzt?@18}o-Dvf8-5AV_W;w?4th>Ihp2(gdT_?ILdN1 zuX_4cWLJdc@YE+r6M@f<7nfX-8RmX{@+04)v~yL^aizVmS44`ux4@OohYvwG-gIBxW%v4#vUyGDAR@i>3j5MNDX@0vxo=sYM1*-t;>1CsDIIC$+ zmm*IuU>#Obu21Mmo@lh1j3D9$214H3O)csJ}6J$IB)5hy;Tmd+@Clm88*&Vi7c z*mc4@kub-5)v}|s9{Mn~BGlZy=NJMOI{~KFdrlK_>c^hh=@oyW0$th_`tf%KT*Y@R1?}&iYCz(FlR0 zq)-pu5zfq>ljbO~<@Z}aa&83d7eE4w$(>_2blPYXo<8j`7ez28jLk{K822pz)BS~u zW*Nl%CQEizznP@ly7OSahZ+R|xp`lB!IN*k(XTTN^P)d9&N;tc6KW`ie7 z=F)aEodmZG3cO7!>@(a=MPF1abL-)%rg0^$8iRvnm8=?B@iTk`ki(qzRLvDmfs7Zq zH!Y~C)k}&J*GXiA4#r7Cqe^d}cy*sSh|)ry*ZROY00-Gxn$!=FA!s!;*g|ZO86a^A zWtUSu?cf-0(<(|U(VNHBymC`&Noz$XxPRi_I}_%Gh+t6~gcMF5f?6p#mVS~ikw2ki z?NoR?D!%!Jlp?V-Wa)uTPp-bIiOGsM4&Y*whU-~Cy2?_`yW5|at(62$dq#>gP4$U} z%0*hYr~b%r9}AQe_8Nt0%b!=nGL`C?lbsb~9g8D9|K@a)k9DX<%nF03`@$E^xf8A= zcZKyQ-9!!9`VU1ZFReJdTWCfQi-dThc=qFXD=KL2Irlk|mnJC?m_ya9lJ8Fc?3Y6W zn!Y90errMPl9IYJOSL2j!laTKu#|P(dXpEarHNIH#m4X31uB^@VR^e=+&g6X2h77Q zC!WamNH$NKRHR4%6X+HKqQ-OqqJeJFTX1m-hhr72S~3-#POrsht0>(YL{wYo(&|LT zpLxpy(!?9C&FCU{WvE=~tjDYxg&76-7iBA*sLmO#igAFRbve+KcfzZxP$w*eo+kVz zCaT*BIX68~A9flr?lxkjUxn~IOg@CkR&IxVk9RoQ2Z(LZiF zuL(Ml{lPeN*^rD@OA>2vt=fqy+uMvnQ|poB>2`E)DrO>fD9N?6fkxdP2;^MF^1duK zBjH<7<#%^Sx7J!u47rm(1lhP70zqxAY_&AxcIY-?nlB!qT((f^d%hYX&!-K92x3$k ziUeOm*JNBA>V#7Uz$$A*+b)WmRZb3QWKvf;JoMiGmj~P3C7dS?aPP_XYX1k zpJF9B4E@rnmĿ()BUFRrW#P47e3CnJCDCR8#TN84Z`)#bXS^eUV49D{);wm`S; zEQoRl09WA%nGQm_Ncjy1sw!zGc|Y_D6Tlzp|Espj(mWAfYx`jGqF#8@QqMGYGx@=V zAfFYT=lys&`(~^Fwb+X`#xiE>3z%a+PARZaobzNrN%|Pbs;fw&>KWPj^X|mFTGyH% z92lyP*$AI~yG}E(6N11x(Kz~O_x@uef2ox7Gn_C3)i&33q(5m2Ua2@T zHMv+NwlnLB%N!ozk4Utii~!fk3mwK3aGM8ZJi1!dNv;c&n6TlzD47ZA`l6D))8IAX zln;p702D$(VniDE%*1(fH|LEB+yh-?v;~88SlArYANn>=aC)Kdb9jXX11!ZZdh65k z-4)U7GG1+O?H$(@L1tG!ts_*fC4ZK%S60JV5Y0_EI;e5%Y>EW2BMZWnds`_B1k0}i zL!=gmT_GMI-v#cjhQhG^SP*+t5{IS9f=9yI7&7X6wz^;%_)%^clNL*pHhd9~7}T9O zccZXrHS`8jy6=Ln$hFHLHX8KO1yGE2_9D@l4qC2D6YpCiEcR&3ISbpqKm9`$1sTcaw{4zji?RAEItqvqaWW(D(H@N zJ7u)mw(yL_L3s=kO09Aep-`G<>H6ig1Wnn*R}+22CL0t~+ma(UBTP==LMUsmAg8Gn z+MCmZq&-_l9XjVBGW?YDMGKAF0}qY!!r%;SxI4x2QjHS}UX_zV_w*|?< z)o76J`lp@dDuxZ_e9`^J`3skl?bzf&Zy*dun_vaoU2EvE&L)_^3+iEkPot^1HUnXv zBd-OL$Rh!yPnItAR-tNd!_y*nheDD22)3YkNI1hBQ``6eHY8IK>pc}%5lp3@fh0_~ zyHpUg)p1>;Yku3ate&1uT^>vs*P3%$A>^#jt^4i_D6* z|JRy3jG?o_zDwP8htMZE zn`4cZw-*BHNu$gd#XmA|E)3P*pwPKOEI1Wrh=B&P5<2D`0g)+?^tIaq!KR6!oplzL zlJTiMG;D&7$mg!JE^6;9x+51QioO5%;g=DuAd^?#Ej!+sY!zmb$CbWPya3i;F|)au z;%{#-P#`S}^{V1Gz0H%f3X~+O@7?xTV>)r&$Utv;B_cP)(&78xwDHw7;$X+1WFip? z;$Wej=7Oit`*?Ro>WLb&O28~vgILQX1u;>+Kw4)YK^ZFA*$fvlUgu7zEqa^$ z9Lj1^{Ncfjz=9%?Awun?j{;;4&Z7^?yK6u}__KUMn`=7PF$pW~RZ~8{a%v@Gpb1?~ z1-)W`9aW*nuctdYwOGI59(nZ@7NK56Hsdy`4$y%U2@;H|@JnxeGTG9m(=!YYVV7Gh zR=0|Ed?N15%+GS5Lb@+5&pDJY4xvyox{{$2eUcK6{s^U7i=HU2lscGlr+~WsppRN> z@?wcZ=blLLJIr$+*4xg-!=|-iG14lbx5sbmq;5sy3vYC{G{)Lrs4^}1bdcnVOXZKd zFT&zBQ<%uVPuAEyYkZDff`aKTg`{q-)X>o11pWlA0EmXA&=QCjdFInn-&Fm%XP^Wu zA+Ex%9DD=e-FJZ#B)($P$O~PVY~TxZ3@uQ)=x3Jm&N2`g;fNFSRpKQ$ zG7azSs*z7E=E6y9Y>r$*%x_*e+SRdBxV5unWTXW+^j*rW$M&2+aCDNo63}j(B_O@! zbZt$VI2fj({5EC7*gn*+a5`Y~i{uT)+enfiO=L2CJlf9KesAK%rq%g9Ad9etuJNXf zufX~h_u9*QOZjl~_pApZ8+{diVjua!kjMZCi&>qs_nceGc3$xF$hTU#b0Rwp4${1Y zuSE8Dq;YUjLG)ANUpt*4#wO!P+jFK%61)22rh_e1h_0!rGUQ#W#*znwVroHFu-@q# z{Yz_N-jT{0^kzNFtyci$j6wLz?K{_$lJ)E5L#&)A_~1Wa0lJ#mp3Cyth=N6tJL4wd zal2*!Ev__agWlXm3Z)<`>a)GG3aUX(przCcP)2R&osra=#`kW z@QQm?3=VV$zRxG|fv=Qi?d>#!gb=psU8lxbx_?w? z7!7%q@lk+p$XOLvKWUcN1<(FU`LU~yZE`q4rx_l6`9r8v zejJSYaYqSDpXm)$jPx7XufF-z?+qPZ@+1+kO5oOPJA5*G-t%SWBtj)X!b$Dgv=#GF zHQXq?81J=+&yGpCOX=FoBxR-1Kfi%?=AZ3vT6;H3Mv%qhI&l-oWId@ZXQkk!di*XE z6kSsuCn{`}t#w_0j3zcvgbliOD6^Ng1Y-GJ>6~NNH@sd2Bt?HrnzE{G#2zeOI)@ux z^;~4x;hn)Vdp?>X09z7rAd87EcJ>opjJr}LhYbdt$cEK!rUG84XY6a9eIB#d98tY# zCm!M98X0hbL@Jmu#6bQtPWw88Ny3&^OKml41GoW{6~6LbkL&8$95!+q)@30SYOrtB zP(3N-m+7{1A?-4%&WCuCN;pg&uAALqs|xaIYx;Vqs$l(i_tB>?lhj*Rh>_f93L?{P zWGBt!_{QMeZfBwTv_=#p)L@)2adI4lNaZhz*CW!}9&~Z(yX@!eolw__bU@Wv{6fp- zsDKFRS#Zj4ko={zCx)5N@WHN4q0hb5S??=}$kU_N%3U?ZyQNMr1|4pDoAEB@Vp={L ze5#av-v_~37NZkg;(COvw*!>Zn4iluD5S2u@qaQ10*z#ABH+y8wuj8qyfyEHd_QAFV=!th1krPd2vF2O^kzetPo#CB; za0i4N_|5&ZZsD=miDwO$%Ju43(ZgPzPIFEK!hjLnnMR~?1!`*y8Zdf2rab};8^@a7 zlq?H;p9KpsUnQdzxCrLbNPO2}#-vGgn##K};pCusR~uLoca!%>CD=OmX7E{fm(KnR zYq;bL(rrl6D+vFgy9F++F=kMCHRW8hW%)hU_r!ED4w1LZ`_MF6#!Y6PYM#I$<%5wa&p!hBt|KR7yv@hF+FOvi& zGuqIzBytG?r;28KPDYc4C`V2Z%K&*Q?!BxY{nmPT*;DN(6oel^80sPAm82TqVVej8 zvAnE+`kv-(=V1ysdh|8DP)oc7>>M&J5%1mU2Xv}=&lTMNAbAwU;;9D@r*H3rF5?WY ze>h5;gS|%bll8O&FPvYMHquehq_-_>PbV3$2%o=MM;^zAJR!}-CrV6qmSfon$;x90 zTlJzjm}{+=vBZBbGjDw>JwmuiUm~WwUl?uCGW9;;Bq&ug;tevkjBt~cDYEv@y#5j` zvlee2y5;}dvUS=jQ(#aWS|cuR#(Rq(=#sN1HQ*ff!r4)W&|QhJ$8A5FJZk{eaq%Z@ zB-ZF9=vV%;FWvWUh%6oZKB?Qe&%)ZNu$+mqR7yV24vyD^`Z<8SQ*uoa!2^zz>exO+ z5fM%B7)b#|#?*(W8lSZ1xNoX~=47RFy) z;ldc@j&Jc0--~e~oDWzVIL4owxs{&M7KV|-)UHF>nbSKrbwPX~kQV!Rl4hniuH%d2glUAtdEQ580B-uziMWcd7%SB57n5q z0Y~m~1V|!Mvm!MUQ5IJJkv;DIRy6c zJ!7#&gj+=9E}0JCf&{f}3B-U{KQ6zTkEA*&{CtT#%7$+%)yUP?6RR&|if_$4q!UR- z+MJ>z7SEsQMWnD}z!N?hmL+CSDcMq%28}*n=N5%^urMK0Is%c7t)y0i#@?2fMyTDR zEAM_HPCz(mJAMC2+GBeD3v~KKyi(WMF~Po0(TJ@DWJ#2oO-MiXnk|V=c7Xyg>RN{X zCnx(BKE2eU3^_Se-o(8?SzjMInk zILlfjygp)q#e z@86o_rOdprRLd#gD}UAHzk%d(jp{9l4oM#f2^2_zZ*6Gu^K*@(`s@^SG8uBe+i%Do z!32>YmzluY%nCHLw-&P~Fm5IQT2IB1wrQb;R!_*~S^m^$ONFPJrN_&kJRM*mu#mZF zyBj;iqrxQ)+ViOP%CwqEPD!|BrOz=AU}eUK%lY^lpTZsZ=E*u05GmFw^3y3wK+I6X z#X-0&TU#LWHTnSvMI%@rWA6tu%pTY7B|< z15Oo}P>THJ9$p~2Iu73N?`LU*EYOXQOM-xe1K+~{X6yF5q>`_zqe6Qomm-Pp2n-wujpC{F2lEa0X>>fo*Cl}XDqi7 zO?x-YF_ZX3@Ixy$xvk}1y!x8;$z8_W$uyqaW`K9$8Rf$l>FgIS`7PvluV-}YEotG= zgN;`;PUT5yi%@K-PB>p}RD|>e4x-s7sb4I$a(9)95gIU8AgWkT?8bY}r`7P>ShjbV z=o1|wfQj|Z!2sLD{Xb-v6*VJyY?rj?9M<2RwxZ!inMk`Nr(oZ#Q9!kBW}0F zh36Ypc8-bc&=JGHdrvs3a}eSSI0;Z2);$@`zZhTfrD)4ss(ROJshZs2rXiC=aQcRF zl>=o8V+WF5d`dBZ%kX_3vbiZ0H#p>J>+%PoCK0O96PPgG1N)JeB@zdQN;10!JG+4c@EZgjQ5u&Jo-Oqw*gS z`gz(h6JgPx2uYnInxG92VbE1(s3D^1-o>4(08Og?%d1x{(5WJH+>QV1xT>5a)uh)? zE{`HZgTcdqQ$OQPxVC;}dwi~TyW!QQ!|mFK#*h|B5iL>5;q2rH2KYM;kdBjdhnRZa z1q1~NL4Gxc#4}N&#I-QRwMfEBGp8l$$2+`X@cmxi2oUIwbh;x`kr94g1740cj-eJ7 zh4fT$$XMEFZ6w9J6ZhPF;kN5U;vLzPfPM`84yAFBe|+L;0OGi) znlPs(0Y`0y$8LJowhuqXbiRH~-jD^r*Q^nE0VftH$omjzv9b0F>^25lkH~U`GwSzt z(P{qq#yn!9j6+l)U?{`a|c7z9rqbtPeYJ3?6L+*G($Nh1eu7+a$DdxQ31*hIN_ZM1 z%ky)<9;u;=z7F7N8u|0LQbZ(@!mYHQ(1M=j$wANy=9Vj^R0jedfFNsvEgC2j7IK9i zUJ64o0&kO>0IK#gGh64;Lu7H!!7+nMF0*2G5h#14g4MiT$mKAZ@K>%yMSs+Dlo!bu zanCdJ)Dx2@YUWQ<(WX^wW_;yvzt4qd$yQ^ylF#tt%KP?p6N=@~p-d_O=LAC!)62C* zeTbsu_ZD>8oDdUvjfD>{4U9DlR)=vQ$@k`9vr{am`v_s*!eiSfUBi^KxmOqy#rECQx9h>~>% zX$zb7rU+Gb0L+g3`7blcg9m+9M*R8j0f_?Tt{lEVLPlIy$YgKuE;#8R>}?@`l8Rg< zHdkkrmW_SB@S8UWMLWqz3Um4XOwQ!CszOzMssWPIcRThO8Im)_)5OEf_M-2R3(^{A z0#d0&09=SSH&eWz5_e+C*eVmJNG#btA(d(Q2t!J*fN``(scxegnX;A`gx2#{3c0!D zU!0ef{84UpSDS3`qih8%`ABS82IdJ?AUr7osml`jV7>N>-XxkCH^{td34+2JCd18uYTlCJ9nRDX{f7~uNQnW_8jTLHQsK`@lE7I3_3@VVJPiNNY z<~kvgHb^Y>nlR!(ZqYTJ8O-HhAFhs+IJ6$YFW`_H+GjTtMa&_-uJ3bVG)KI}14gXT z6Mw++fR=w%Zdcy6@F6?nSD{2cIykxA6#OM->)>gV>6NFtz*nTYd9jbt6@N9F=v;gV zalm=1AbcZg`*TtXlb%5C;Y>$sndwA=NIsSfn8iA|Z`;adz1F2?om-RB7?uGGBh(aE$mE z%@nJ%-3W`jBo5vatM$bQo8%0#jD3YaaGO}m`U(qKTL5NCRC-oL6?3yLQZPPU{A)#^ zISkEGuG9CqC$eL1_)bkOn3TO7KNVB$uyMhW%$+|lLbQ709@2HPkBN%PnbMS|50jyk z`N})d%vl(#=S+m-gIrLr!Aq9XgO0m0c0S?}Tu8+^MJQD!&2I$}s?oQ}`20DD@J$ zk=%zOdzrGB<@WME&d#Z;KhJ-k*3w({^2W~!oPd({VN@t*>4rA)B|;o$>cPP{t~Omw zZ->S8OwJp@6T&1{$VY%kcwPRMpfEXa%h`;P)D@Nn0-PZ`=R9X39c(ZNzc#W`bdTKN zJ+&7YSk0Dxt@wS%0&96E7GoveX8Wa*kI!XAy4NfDmDO8G-3#==kR1i`nOSJZ2l+G% z!ok-Ku2>>!|2R8*M?TkvA!CI9Ei5d+aceE_IC#2ynG)wbwA=2c58Tu&UE3tg9`#(! zq)VSX=-c>VZBe_31w|SVZM7WRKEXhF9GIywrt@KHbU1eHTv6jM9)5L-Z?E%GZ|mtC z#o5>Xe2A+U^8`-OAuArakXJlqFK%oh_t-gJiRPVl47^%;TBjttjM!tD2>5Uzv(ZP| zy6q}LqyZ{mA;DY}aLK0Wt@!w*_8R`y`e#N{GurZ0Mj~~RXw6b?-bh}rd+xQ7@l7$3 zOHDzEo>8r#-5U>(b76?$R`%Iv?v9V*vuL`$-^+KpuOUo9#Id?S9x!ImvT#tsYbuzE zL`-Q!i@sjH)MnZpUCZGnDO>Hjii&u5cGye7oiBPW5TBGgAha&Z*zistFbl_xxM{>p z-0Po%LTfQ5c)dDbIG-2f`?h#gPLT)4%TSl{WbrkkYK9mU2P@$zyyFYKh~JXJTly*=VpB#$m%P>=~bb+#4j}tPAO0A=a__|EB;HF&_=4AMH82T7K_A|yF|ac=keYM9e+<~lz)QNB>k z$L%Y5##&{NU{ut^FL3*9EL^hK1 z#V@R6;7x&D9tSk@#jzAbRxa{*a~Oe+=-cS8S^&e_p7@irf(c)*%SkE(Tfz%hX5P-> z=B!*$YseAhO3PtGfckaO`k|XJ-!vEDq{DraNXRij&9U^(XZ+sNOYnxh<-FcIv1ZZZ z4XX8M=ybNvg@mrER;v?waXladN{uoJd;W08I$5xBJpfCB(4rf)N@HD#pDDe=V8O(C zrrTCEvd9OrSRGAUkcOIbqB=rw5*ri3^5;ZOU=ox9fQq#1WK%YLq0qIcm>o-uYJwo4!aNFD(a=G0u z+Qo4@Zc6*YBgOQr5ts#)^XT8^)CdJ>(sr9}?D6w1z~mE?f7HNX_wF2F?A#*%kY{oo z-`bQj0sSCI)3p$CKl0C)O!yJTQu^p@X4ujV$?gPUqtpWcd9bCsI~|O7bmTGFf?{u| z!!ls%_v<=P4+1@cFpN@#JX<<^=C<*pECa{5#dpM!21#aj&YXT|Dc&OkQx!BFn2&em z>*hXKtblr!5j4JF85f5Jm{tdw!O2^D2W8;j5npbgl3B3`s*rTeDuoM#))xS+b`R3V zBFX=;*M*;EFT=%!AcO^%57AB{_^Hms>;^;UxNvKB3N+D4_Gj13dB=4-pk33wT*hhl zd}0gd2YsKWBF)jjA)bu>4@;@jao?l0GnU+}kM9R;;|!Um8LfzF`M#*w;)7P!FeILT zN5t_rtzW*Zb38U6#V}EJg1&#T_ z2=(3XuxKh)yvgwjyoQoP0_o8?cmHSpAAaa zNTh28H^=*l$BogeaT>ui%0TdGqi+)=9A&X(dii&8)BXE+0KCum+3IVq%OaV54UdT( z*%JGfN*}|ZlR2H#E11}vn@xa*QOH$^klv$`yvpp5lfYW5Z)6}B{=vsrjN@0dp>fJ3 z{J7}&XlK=o-F9|Vq-(W)IMEst=oKX z6j4CH^S*8cIl}C(&B%@K9kind=c0G~1L5s~C+TozlPqGH_;=C8u*a1g0TvP8&!IdG z?m_9r5|(wWv8L1(!Bf)L5QYSb>puP>iw0OLi>GyFhFvK_#q;5K%v}y^|lU|8i3Svv>YpHPB#nzWM z45csY%dywNWv3$h9)WwEm}=G7a$XvbQQWrp6ts$8koLZC?M5w!Vt-60DsEj z3hckLhsR{yo_$TpN#7IRvQDL%A4)9AJali+y4>otH|dNp6t$02z|u6($!|)lpur8E zGn7ucGE~fTsaU%&-UV@!w#8cJ`U<4X$ijKsQry`|V}$b5#bMzpn5H&~+zo>OF*SNF zR3kI}<2uH?w!y`HkOK6Fv3bj5Cr;J7FpQfM&<%yTmokPT!*-`_nI&lxCZfhZ7A=Xf z7%hAKi(*EtiM_48g`=eyd!9zni@%2pNm-I3`5Bj}!4Hl4%y`Vcsl*^L0hw<(5FziT zBwr=pd_Nm`6Ftdpa3Hr@|C&W zXhrqTh8LP-mP+&IPlC=(VT51b5OXr@=qTc`uA&v&_q3<8l7Yadio`c`W^+Jfs4>6W zm;}}47&CMz?OCM4t8_SBD5D>g6PQd%f|__i&yt%c%t7lnuZ9Z7;lzZ!ZC{d{h&rFc zi01O1VRkKxYNX@>m3K~I_IzdI!lY1e#J8Zi@Eu4!ad6)$vyL#8}RT# z1jy;Uib)8~LB#&gF1nm?XElztu=;Y3oM9!Mg#$^^E5{TYgY@dLodKK2gD`y7idSPu z#xS>yx3nXwKIFVYX~ATM zGjwuJRBXTb1X?{$!LfZ)B!dp&odk^ah`vS*8vE#;pXYF|Tdleou`qPM=%`hOw9eR4 zIXdVV`gmmk19X)yez89_Dl~bM+$mgh-;y082%4V&Ts2Z=H)xu3NaY$gOfJd0zOnLL z1zik$FRE7Z2AhOh>EA%6G5@;1&%JaI>1F>|u1Ll?fEFydlIU?x&bM>PCm7!vmcrfJ zr+4=292b-xh)OeIl{fvGbOb_M*Db7LsRqy;`-fuQ28LBJ} zTvixwJR;l&coy!sy3iDVY+7CPQLTqJNLMGk$nL1!wb8?HS|nc}CjUwNsghvTpltdV zvCxqkKiz?PsO|%UVn*M=j|mKRkd_*|ShJ+SWqQWA*GGXL2(&6vDmVR+Yw#2~?SV+E z-S}`EllY8X>SSUOiDna!b^}b)l@?hBW%xHCwvNcZbUdSDUEDdI=ci_J$_^%@3cY$d zWPC*HmQDfO>H|Rty$bf7SU@kxFgzp4d?hia)lHSX!rF-YAOBk_jk-on*)85{QrRf# zh1}FF;^tcJ^wX4j$p!?8r4cE!HN1RQ_WT5Vh}VP20Oy|mCSfJJur`!Xy)uHlII}HO z07?H+ql0Nyaz+f3d=Zp7$9|`z;~21xn7l^twMjsAjw726l+JTjzFO{>A88wUU*2ay zyobZJi!6HOfr7;PUi!QAll|g}ENfMzs7Fyab)TbrYa^-VCjMXuvxHxRoHc7b1hl9L zJsIUi#j61~vVW<*tj@G{zrHMl3WEhycB(5-dx6Qhe@-6u=4f?&=%7n9SysFV)O$R* zL&+^@etpI6U5TocRxlR*w+f6UZ+K12_-=3Q7Sca80=D))?FkOlv;LIl(FDgm2GV_e z@@BsE8W2pej57?YR#o(>&d|#v%4PPunbj!=T(x6YyjL6{uCsQ~%js6e6&D>*Dn#B0 zrvx(R&xD*x1EtbuQeW!VdYY?x@Zf|pP*{Aqkb_#>aAh+&gw37j1ju-C#?iB?bNxR6 zFF?@0SbsLsg=Y5vZu;{i7d2<`Oo5-8{iiolWuBdZU}z59m4iV#5+0ugF=k3n(*j!Q z02QY#F#0r|b7K@vHomBhBj)YE{_d_y{3}P4olzl}%0{$JWQze^VJey{E zDcrrAZDgiTH&D~8q7R|KZ<+MYI?eMtt;}5UEyr8R167AU%y47R1%Z#J%=FCBJT1%C z@5XM1oT$%)M8MmF7)SfZv60dAGb(ym_>s2wS;jaG@DoE-vYQ-OxwXZD;+UlF`&Uwj zl4hZSMXNaa(sDb4^_1 zCotAnS{9Jk1jw|voiLxK(kG-`+3Ewc!}cwhokey)2ttX=uCuq%xb8=_b9l#}xV9(I zc{=nZ^jrsiy+g5CSQgN-d%F~cu!B-q(t38FPtH_bf#I8oBbL0oe zsg-lyT^?!WWdf8cFiy*De)Cr)DH0zO`$h078z-#tP@_AzR>A*~Yu@GJ^JK{R0X)cu zZXq=?EYZ;lVKCG&-xA24UHnk4n~OE8X;wRF5>jN@FmX^KT!=6^<|bG2h49+%f?iA@ zONiFWnn7w|k^-?fWkg%L#E4z0)A$^&s4?FXT0rObC$LllT+Mwq?2&AmIwpDx&c-ax zwiXhqoU^td)C2V5qXKxe4){W@kVM<$_~MlyVyaBy|0D*!0guV8Naz~@jWLtJI&f_BZ)+Tvh~$}F%MO5RZZb^b%=-kqNlij>y4CZIFiK(H5FZA$B)KMF$MZkkzj zf-!OWu@@an1w3HK;M(3eJ*1bA&g+ZeU}{P<+GLzw%Je}4N^}b^% zv2n|PDNkG2Se8E;HNZW`mgPA4jasA9*Pf>)_x_@KS= zw=tuj>3@GQY(5-Z_ouyz5-Jd)8{0zcs|TwTl~4nG(S(Po;v9bf@R;F!A)9UHyBNeL zzy8vDf-W{lk`c#A;!GlfvAAKA=8Ffdh()&O z8xb?eRxJ>K{3pMYF6mxU&1ADl4{snSskL(>5`D6Pf5zg9bnf2Z>%>AqDa*S)vB`f* zoS*N69b`ke`YwhQ!dXai;ZO#^JQTsi6Hm_&cQlfVpwxPuB$Bf3AC^JHNlMIqlEqGh zSl8a4 zv*+XTfeBsU&DJQ!54vs>q1fW>BvrDppM51Fx=@!Pih9f8Z=aktXj-S5>dx4{~X z{S8z)X;j(_P#LE;YEpTaq#_@xUWbC*cyDa3&@7bme~5HQ`AIyNB>447)~vmfE7@l!q_^s%TWL$iqcVU&Ror&sMF=m9mtP7k1(;YY{eH-(-PID3>f`K9HoG9iQ`7^@a(5gz_2YHr_8l3@U zG#4v^qa3_4sDCyEPz^HMYm^ft+|wviL&e@xx%UO_mNd3|OzIfXB+w>#^I5&DMM^(W z^{1eJ=}yQry7Y&g`Hp4m>96zYL@zH&4|Z@Qlt~O3_Do~})24tlD>%-~M9SIWtKC<( zvz3=iEWpPT_2A+u!j>q}H~3KCII>jOCv8bl#Z;LRIa2E_8SlI8Y@s?}+aYm3iH-@H zm6^2HCaGbApB~E;$OZDbA6TE3Yl3V6Q27V|e+7j$rYxp2!{k(F1h;1Z{&* zegl*qqhk@QVrV=IgL$y`^d7*%m3Rb?V-L@QSdF_(UTeBdwWpRG#S08GMmf-!EXm#l z$zh!Fw(ubwE)pjZdUB5G96I64D%_JC-Fe-KK5sI2{JDE#q?NTVj>}#81)%`k4E2u-kH&EK-kMo% z0A74umR991F=v&D?SMUGF1e-AZ?cGd%%0c3#kJT^t0Q?O+_R2*j>uy1@Fk|y0TmRj z1Fha87_iTk2DV<^eeot+536O0A~q4)RLv^ar{Ji#d#Lg%h-0;)Q#FGi2>_aWn&Y+f zMIH4GQ~Dz?>&+*nTCj17f>9`LlFv>^?Y?k?3i2nVC8w)FxZDjXxtwsFXGoI=_rC$B ze-dhc>;CN$FBGzCETqL6;~;{YX=g(Qkxm5jLg|C;QXJEBmS0Qy>1f|U)^&K4f7G%}1 zKm`H8s}W34m2p99TuanTHT0EXe0<`|@6^dv>QNgVrQPh)?in69;-5qF5`ZkWMeL;? z1vcwnOWmIE?yZkbLm?^|;Ag~XvKmtG$Fg>KLcqK|!d8IxolfySdJxExw({uV$kP?- z#@O`}XsFBldF)oL);O@=(VNX#4u7ZKp3g@IFo*4wJ8roWPxzeYVhd(V(UO z|4>t{7R@e*0&Xi`Dj*PpXj%v;_ATjflXD^_NrB|S=4olB?BB4$ouBgdI z-(ZT(?Gl&&lyuX$SOP&PX>a~&k-X84?YeN@ZvmjxX8V<%L1!BzL2r6~6LeYgzO0lI z+8pd5*M-ALw3B|%-TxtMpv?MkcC zs{d0Y$Eccr@ZXwvYUMzGmR0m;_2DJYExU+2F*x{+j2Xutq;NeB;M5(0zECb5T$NXb zZTuJo*NbPj=WHY5;RL*z{~RVEo^6+AkBKr~C3cv02Es*$`Yf_86GYsFHxVlWU=XL( zpST^O+^QY0f5wch@Au{N30#C@n)B4fIY>MuT$*k2k-4Lu*-peL%q3d+_O+G9o~j!MI(VJl8XS zzfaoICmk3P!s$azHr0Ea>oz`g1)RtO<)=qym~Cob8spHE3qmnh7i!L(%<`{>Ag$+J z;z6J6!s`cD=Id&EFf#(iakvH{b6&86kxpswcF!cqNQswYj#gv_X7)!SKc>@yGR~QS z3156y^#3N|yx`1#Z573`90;D~gAL-Gq$|>Fka@4zTab((l@tJ`FOTH1Ap`x}!CbGQ z11%$wZ-m*{sjOT+tKlE{p#vdEDCK~vmxj@OGa&bhdKiB(wyNw`mo=sBHV(}@Be$as zxoCfpc-S@mC&aEu3s+^>m2Fwfn@m0E%OP~Pok3NHtL&zv5d_zxveFaZ)1c<12 z%G6Sc%@D%-j}zaUynsD6tu_wJAnoo+ssE4=*8=ty-|c@chHcQan5_on03zi>m5OZu zRP;%Lvw3iUy?nikBu}6Pm7B+Bz0jpWJ-EleNUs}KjS_hCu=9s7b-3l)O18+fU1e{j z-%ih2Eo#uxy02|(3Yr0YJOpq}5m{drSgAiO5^aV4<(4{*Cg0O?}J_XeW*LeS0 z6-Ri)$ZLhfiq|*`>K{qmS);jXrG?V^PiSN9MQb=D=sG^^yUgO`?IIjuCK|zvgwLQ z@Ax?Al68qJ*7T#WCVZyJX$P8B?FOxBS=>h@1%7$Ol{Vll{z>r9U@H*y(vx&~1^mKB zi1$XsTB@ApH%2{Wh9C9wxVToK=ZeQE^tRRwVoqTcEEf^xnRfl#GNg)2&L?q3ic!2R z9y~N?;cBSX%nrX|(XOnfgMeM{M?yRqZW>cYPXqSvsIG8>;1sqt0iHlwc&xxnkH~g)ZYyL_G&HuO!pj5J7GM&|clYPa zf?8^KDTdZDwcrbxb0$Fi8SyiD;=QbU(s)6{*-LHN78nrd#2S^M?>L0N+}b@K#OM+vtNm$fJwdgR+NHG-=qk z4ji7;5yOZPRtnn@(-BQ-ew;}|R7F?*85~K}S~12`8`zuR-D*|NN_yx&-cm3>P%@vz z>Se_AfUC0&_tY|2{vne%Ntmvh>e{R!BoZxTmGNay-ARL?B+X!NRO&uj4n2C$e2)xG zsAjR^mQF`fC2jfG+E})`flTiu&SGc=|IX3y5Kjgi62Tdz_`(*#W_rXqff`+jJ+`~6 zLQ+6oL8FvOWZctP^#*@u_nFg9C_my?5#&x|D&&NJjh<&WVab1?HPEa+k4LykZ6T;a zhq++KNfL?xe2VjKrm(=#yguvXRd5q@)5TUvB*0oiUu_ix645ZGT&1(3!KWUQcc8tw;!nNot4tlw(PfFHU{DUy^`8-I+PaF5M%>=H305> zZ~YhIQS$LSHzx{)xb}=~OvbdE?!quUTq|0-o=mJ#G7Ctro3XLh+)C6vw@$XE=d`KTz9lG00U!u@+u6mI)lzJ|0fUHAoUMA zpiLBsQ`uY+Ru3%H>qaGS`|iu~W)j(-(?@c!uB0_NxhI)ZCvMcFHXU8fShl3)d#pRC#Hx}Hc zrKq@~T641kpeXwp*Ou3@TTklPShBtp?~-`(kvfe0?18fK(OD`sPk&tyVJ?V_lpUA` zlU>sPfVzAstyffqeu9*!U2MY6a@=19b{QmJ7C`wC3b)n319==LC@AMMRzQ~c zT2U_x9auXt9faCs^XU%j+`zrXMINb3G|y4~vmhGm4D3fod9y=ZQDMwK?Q>)M+CM?= z0~IFlXH%^?Xq$)}@+D#EaRXRM)74n%-AX#P#$JlQ^?(y+c{F7qJzs_W4*&{J%~ z&t^hp+dCR^4_v>~Spd$bwTjmA+bOZ3XPsMACs?Zn$qpDAmLsa*8_OqukcrKRdetxQ zsbQ99KlZm)O3c;6mB6t*Txv{rP51(WT!`8{CBWe}k-5(|3fS)aX`RAp_YY@T^uhn) z31*`pnVF|=^0Lx3u@{apaA%e*ny;T|-$;W;s3ijclOqOKHx>~Z4&*sTE6?aHR?7sS zsq{S=6epdhr^&)o*iOXAor}R=zsP3jP=it%cm?dAuf%XDzb973&mo^c#;0w`4-aV3 zi_@V2pZAaDykQ2Mpwd6gmV+Hmr`OCQse$-g0BBmr<7yr1*0fg)PMD;KaDJ8x>x9{C z^Z9tsqXS5V>kpjcmBq~ENz9j72}x+T-pDg$jEmk8LuSsdc3o8wUu9G7loEphH_+kL zfW0nyl$enqj?HarV7Qa1nO)9Y)V3>P(gQu+Y!4dfG(WKRWv=@g#`iH5OXNZLfa}aX z8up!3xN8EDU`QwGDyv%+LIm_n40N(zdYHJf2DU0UZra2P^M(ZNO@b7*Po{LFs(h{s zaKO$M3RSEQ!-C^%(i82(Kb%$WMJcJiltvUz0$vX|*WjD35jngxXO8jOYFfCyO-I(j zlZ=FOti~-eOR@Sv*ng4Bo@4T@b4zoX<+RG}(C{e_C6RkfsUV$Na;b1H#>c!Xa9{tS zoVymHT-`ZRy|1OaIC@{G>umW8{k+jYlQqbvM;;+Gm4{lu>{yQ&I)J!Aeb^L`3wD19C|; z2ALy+qPdCzVCgk?DOLQ-H$=>7L=i%REFV0x)0CDEq^)U!NHoA-g3?(uXybTWQsgviA~opY*x`?rM@0j$O3PIWt`!gkResYT zCj{f{!lyTQ?J~OcS#VK?yN-+C3lYHP59f<%9>X4~+_|YgYhdDvBMB_VJzJ|j`m;6V z2(<>^t0^!NpQLjmO%WggZv}Y?sc6JF{`~;{$uVYyBQ{*IEogzjpcS`2$a|$>0CV}f zb68r_oy2n+F)ggC?p}1NQe-PeV+eJ78e;Ru#Ii4b`h$RRISa@*jFrAj08`7AX7}nx z68^=e)c(hh7tb((Z!j0!o3!v$j%B!GLmW4HB_$_{aZwg4{^m)DLykp z4D|{8f_9t)TXsd?c>j7GdyY!PU0r|z&52M}hs)_Ha^`F$Wjq6sEZiE?pRMP8y5e;} zq91_GpK*%Cz3cksqB7=ks2b^X4BdMmcuAeNCQRb9L-Xqr38t4v5{}ZZ+@tnywAmC+ zYNNFFx@TiJN`ljejqPvy~Mt5*-iHJ?Pc@@ z_eP+YH779-#$1RiV>2!b;yrbSo#`!7)tc6g1b`IiB_J8)nS`pMQTR9cb@1J} zGK6F@pjA>JV=e-8S#Tow%4H(1*EmgtOHxnY+~Lz+2{`Ur2r=cCJa_!`F}NJJ_?tQF4E6zh!O=+(@A;}{!_??JUBwaj@#xy#)Wz-feBD~Bc0Y!V8u%pGS*fcb} zy~6=+;*TtsYG?F0^iSyAd}a&LSSZhoc1wO}U8J)kGWp}G1QlZlCQeP4fe3cU@S2|? znXUwvj|?okh;nTj%B^TFxnA^6u^OrMk+-eVvIa6+-H=Y~{MJ5nGt4?4%rpU$D%vn+PJMS$wZ$kck=G1x5L^6pe$?RE1i)0IZaZ)}EN2|?Y6Gxs zea7tX6O?jwO^`kY|MSW|dq1w390!oswY!8lXl|M-2CAT5Z>cH0}IZK*_^58PzgaIHR;6MNXBSP?D1PUmKAgr(eg0g@B!V!{i z1OCq~+=QVRd@o#FQT${p7``|t!TEW3eCooji* ztclcD6k%IfMapKr@F2x5%~Z~A3?6owL4>;DjhA7EDa?;^cX)1vy@`^BK2tSJE{iH& zM$V6fJ*?Fpqk0rQ(e?)}(d~2v{!hDXkZ#?=VDC5vJYXwID6Fq!SCOY6->v z_p-Cm?#$1|ar;lvoCeHThii8ys?tKgAU4)*2AT zj3j{Rkd}(X>`do&Y@Dlw60f;)XVw!}kD{DVL)mYBjZ*hd`x&*Ec~l5z#`K=wS<4`r z0a2V>mEEwwdSgu^VX~#i@7QykIm>rS-}o9_Yl6^%C`|wGf@_Ha)sA5gZtAt4?HP=2D*AQY;)HOc0h-yt1l<(+@Sqsr(qD>aVi+oV?fTX zsX)whaSj%buT#sJcLhXFqekp)OgAeYaaGgP*WO2^hv~&lD{HoB^ z31==um956_J`UZ?tk>&20fG|oXciKm`ef9)_Fma%$-u~(BbbB&W9vpfCz7{Ky7irl zDqlkrfsLlD6oFy;PrjX8n&6@8z%&$ejqFG4?!px$dGvijL?s9Mpxk0YMD5!LK#Tj3 zk+9z#;|D?>hyrn|8?IV4E9T{!y&u_G1QnkSwCfsUr(kPDSUh>~4a18D4zBb(IJsWAzPcBfeBNa=y47LQ-btJyo;~X*LIkX8X=7Bb$J$2K! zIJf)m?Zn?tf{KLip16hoT0T`@&<$?Y_y;h|NpKM!T5At-ELuShs9F@nXjSOF21A}` zHffxpdg?2&o#&A~B!v&rb~mLEPm6@h6w-SnjlAVU)Qe;t%*x=z?q&zAl1e4jFL3{LFg$Z{rL}jS%gUF8RiyvBxhI7rarhZ= zAj*ZJT_z&xF)=L2v2{11rJ?MXK#Py?SKL>L)1WFacP8Rw@{&bI1ypjBiAmPivZB_z zUQHpTkxI$M4r#rJd&mT6C7nV#tcD$)1}D9ZMKwK38`pT$Y?{W_|1s>&@TlX-XaI$_ z@-ilIpPRhp5UW6h*>)F1gC`b-?L}|JNz7ehg~%U!-d98o_14hzZ9({oxndqD=aosU zDHrX1A7{ks2QP3tg35{i^yjOa;Yr;_Q&VuE2jTP{pIZ)5) z6PmC}t;O|eY;>G+YVwK=91K7+_VJLTKE(|O>E&8^p#+oW7pS<^%u%grnC*>Z`eS9K z7T@X#PVM5~e20@z(B5-cNJA}=GS02fsxF!{DF)^Ug0y}s^?tWilSL;ynvwCevb)Sj z{@_Hdx(3wgj0&+hM;a?-%0AcD2td@{z04JolwSVE$xwTbxmUxiS9KG@tIa=K;yifc zqz+#IROPMq;!@)3PRggV;*=cd_4Ptkt$U*q*ZzX}w+fDcFi_>mv?(+{X!qQVzk{2+ z9nJYw3{&u|?}e3`oQ;Hvp0~D_0_{)%5qr7W6#v!J&PZpl740$^X;DMZYaA8KNSu!& zIZAfDcFF+4Jc^Wii~9RjsC4k@k(4Ct65Gn!q~zL!i6;Pwm3&r8eMFw?4AB(R|GI3< z8+ecKVo?^I2J*4PgRG|; zO|7=BJv(%(a>_t#YZ^EXKvwgG&FeSfhpRO;Rd+v4X{;5;`S;l+uO#P;$>_-G4=b<8 z{J0-8-HM}r=2KgMH3Wr&_m_FRrPJg)zhN(yl2-w3HI?Cr33>1+8!RHj5E6j!^~+$1 z@SE?~@bQY}J$H3fs5L_>16Rgat0aYTdCztc?LqdkhQiW3^3YPo0m`Rxz7}*;5sv6- z7lebQ^)=h%$Xfgdwfkfdb$N}8PCDOEWG6@GEvmHn%Erli?(u13Qq>gRpU@JcMl>=Ai+{$Y%e3IHpOi5+ z#%6tq35qz6A|tv0xghr?)sS^Nsr#{z>ClWhcy+3$c!E1+#R-|;4JRPAQ6$`p+ zo#_lzX6@A6)-+mn)jI0nPF&R@iqxCYlLe?GEX&41KOIJgcG?oG3lJP!--K2}a`;5q z!8YTXf27XbC8_<823bBwkew<*@4>V}V!^kcges2FI?^CHrpNgHMwpeY{R6j{Pi_G? z)1_guwiR3nVoLCa18|MV8lL?mdqFfP+5ZyN_4m@ju}YoB3T9NK=S}N*uQtM}gmU`;V#c znc&A-k$uAjEtcWH*&xTA0)f1y%FvK%BEShQ50mf#*dTLtS+? zu4&*Oh?4$Bzdw!)x4eWV_%l=|vF$ov|DR202!dS9$${K;5=V76D^jBiLmS6<+O0kC z{i>LA(GEp7FHB>6F6KitYx%+!jD)Fl^tjwELOtVu9C_V`Qj@mR5%Ak46T zp9L0&ovWFKi#tgF1QDkSP-D_wLnx2kN2T#c-}LcYaU$Boh@u$R);J4v)+%YAgTfV&EQQJ0nS!L zOc>zLr!-c|bvEXxH4Z>>LP9%twZDPD7=0@2*F)HQVd7(c`}{wv)cP+oSNY~?y6m7R zl$T7!Hk#Wb!YuchMx80FU#BD^ce&m1+w17pQ=U=79(J$VZd~4Xizg87`N<+wlk*tB zS9^ZtYi?=VwjQL=X6Xb$=nUixz82uf9b>0DU-{1dO~R{$e728~d4+xq9CQVx?JqWl2)bN%4^8GE|s z!#G6e2L>kE7B2$()ePLm$(GV=ixNLKCbs;w6|ITm9pPhIG(_Wpvv&r z%e06a64JQZm6|2`uPL@&H{M!1x!K zinw8~zZFD$UCCBp8Vq&LL=klQIA|_g$P>r1Tu~PT{X8` z84()#(Cj@M##!2zxN@@ZM9h&T9W@?1+$4&btZ5D?XL5f`sCH&1=`)9y2y?~x7%KS3 zsWb!K|I;qlw1U6}Om{+Ffg5vKI56P4dkXiy(KXXg^#3~VE!>`gBH^}hjb{Fb<9NKt zlz8s&3R?RG62`Glg6cB=!s^tGz&@6g5&On!Iv6{R+wnosiF8TY(KRFlec>Vf0B})bEFMnBL&A#-sn70X zOWE5V1y3T~0Zzy8w7V*qNbB1f>Pi#UGs~$A)wO8Z5shgEizJu2ksAn?M>zb+lh}^x zw)r0<$8+ka`Jv5CRo5WK4{Y;qAdTH4`pO;Xi7kyRfVKF1@PWA_u81t%XCOe36Gk(| zAbq7DGKII*4;sB&T%_n{?S<+uZIReETZ(AC5oUn}z%h-?vM3HwF89}J$sBSQ!&x4j zi~T2Vcp6zPGEy1??63%tBXJT=4i`pHN1}D6x_<_Y>q*oFGeJ(NW+y%mx{ZPtmnr22EaxV(@On!5d+}(C_f**FpZ`!tAEB&m z7ChbMyG82KM^{81%rkHH6 zsEAXS*t?qih@9J13Pf9=N95@KlR;X(BpShfSGm}hcvc$ghiWPH95^q$tlsPGe#<)e zyNX(IKWwA#IYXdI?}0{^!%k9IhML3_9--(5Yvy=XQb z2(JzZlR+E_I(1rcx||+;o)q8dD9_y7{%&%jWk`rhX)WQJNo+17MfV-ndha_6B4D0A z>!jt^B7DdtNbG#&wj5xZyB!X!CYTtInxrWea5<0 zSQ)O-z!W5Ri*HDKl1^$h z4CdWwb0lp5&fB0-3$ufkAG&=_kW?tJi{B<@SP6y_g?G=G8r0kfh0@0C9V4}FAflPo z$#SffU;o4Vu&aJVjz#Z)z{sA*hgLL~$S4)R;zL!Z)HQ_r$@ssMLU59muiH6J{8#}_ zq1}<67wQZLIpFABz;zo(uW!iEu@$J^8b+E>hlS$Ssw~QEN{P*V4M0idXxI`u*4dsucQrh=E=F0_wnws7*n<75EAvp~n@pYi3dNr%rZ{ z(1};B{ZCe_jm8t-;2+i<@W)+0Z${u1z1#6$1ny5Y)zC?FhR$#~=DYz#7&|^GinCWu znPMdgw~$oMG4ks+md(PWVu5gsIQ1>?mANa#2NcTMwvKW@WT_Q9PuNik zncQpzJS>9NM*uL((IIjJ@chQYAH$5`-NJ_GHS!y85-i5N5IgR`jiz7tCA!cHw5prY zw*Y-7aXTKv`8h*i>?~DYM3*(Z1VanTC7_Bk^%|2U=o8p(FoeYJh`YCn)}Lk(X2Mu^ z63#qjKj2b)E3!hUfX4f4FR&0P4^ro9D^A%Dbq2J$34XH670p!IMxBmbH#1>o%bY(U zYtuM0$^rfBtNbg2zGdEau@AzgtU|TYd{f6%htRXOFH^c*L;jah0@ z`(0=$Fhn7wFhrFJVMaLyh*bc#ZI3+!R7Tgz=MA2#6c4dV&SXHD^sz`yIi8q`PD2M) z*HU3a3FB;V7#Z zs?*;eT;Mbs9;x)0;$?jU44l>K_4f>w2TPfw1VC5{c{*M)vjILtl~u_F3s$7;>?@|I zAVh}0vZKDn1r3`A;rF@bxWv3t{T5DBC)$^>amWtX=6Pt7z{YZ&NWlw*mVo5o()-b8 z{8%STWhzKU31mvmDM6ZXp5M$PC-OflAgJ_-QH%scpR^~A&1E3D9&I~TaGkpD4*lG~ zppmp17*_~0LLw001ai;J1!FO-5~|=>jz|Jl_d~D{u~C9N&OBBKB20b{kqYy}XK8`e zQs;VU#sNWd=Pc+JOEa}P-nc9gWXw}5ZWfG#)uK%sBN!03p3n-OyngAH%9jNKwyXo| zU^bCW_Cwp+QUs6SX$qgRKGM{O!r(wj?f$F`KfpVktFWivIy9L5M`!8n$eBt3caX-9 z=rt~+GLhJ6A~wU$6wGEu;z%{&9!`QY3)CW5vE&6^t)DNWkSRnm(U?TPq5Ej*`$xuM zkjqW{J?ng!pHIWJBOE}pSEk^J&Z!Pqj)&LI5F^0ylz3PEnW}685cK8!JnK(OuMfwl z5r2njgl1>=_3pi8ol~~ZmO|Fm}rmvSF?>L>v4&04Ra9RN6OdB+Ffw_ zkE|&#w&Q?OFu3qP zT%sGc7r7`;yWnh)c|k@X-ELq>Eghq~u$JEAXwhT9m1J zkSV^tmC-!@FY3X&s4{{_JqP-z?V-(X#KyqkDrG>8$(m6Neltt;?K;*?rnhuxdO8@< zW18a%^@Jo|0mR_jy1}u*@?1`fHCExHKZXtoOtBHYb&w`^9?_ubNaAq-STKR*tH7A1 z7DIkoyscUiNIhQ2f_mJws!NOQfYn17s{N()|9 zZO;CtU)80;t?@!_;uKz|h?+i57VV7lNZXSmeawD?kv3xcN}2Wai_t6!Tt(y%SoT15 zVAlP>k{v0}Fi>5I6;mOa=5u(wkn$X0cgu0D`P6 zQt2vDJgr?k49psT0sfCnubew5>nmZL+5!O)twy`}OZcSe$u=}k-WYhpOlfj4g|#Wg zeNlcPO7X*VuoC+SqgZWlRJFe4Lp!!x8uaOTJ_}!aZ}i=qyYg+^ng%7G-R_$gNQpV? z3Z*l6m$7gsT2M!(<4Mbr=WYcx-3l^VeQ~x+V1oA>o$wb4i52kSP^Fjog=3uRo_q`p zN;rZfHYioXDek~U+(qno#}5p)35(o7QL8+sWt}PH0em0%-44O~N}OR?C7R`xJV5&` zQCe$Ikin?o+P!!N?c6P;U0Jxt3>Fh%0OP=7Q~PN9t0c+u@C#&;`EE^R<~TpgP1BFp zo#u|a9AK###M|&H2nE`gm}D#a{)K!CH!c`uZ9XZ&!8RoJdeoz|1@Lkyy@eN!;EO9Q zpOOo_=Xqrr%t`BB>;)a>713*CFn1JE+20q3k$y$^rP#r+hDZ}Es~&zq72p#_fr0d){o*< z)54wK8dHj6L5ppK&|8tz=Ij&OJqai3)$}ivN{Qw5DEv$0t!~LpL7u?(ikpTN<=E&~ z7MAd|08A*mSVPi#m zxy0yCA9v{oe+H`rR$>(&{Ko@YXUV^fP<0l#T=$`bf9-c`X9}u--Wv2)PL>4wDn#{n zK^~CJ9g#8Hx&yxM96=9$hDXiesgLt^8p1po8%mz^m*R@a*|={(nY{%vXHX~G8%*~J z@-Vk-VoDQy!~p_k0t-EHN-=?n~9W zwrs)y6td^m`~uH+J}e2>Xo%XWtNwc(R=XQ}a*T5^3v32WGlP|bHetF#E38FQp7am_ zZJXog{WAqtB&5dvdWEk&H)l|(OYy02yw;+eJ6Uq;E`7jQ8vf+whlI51qN*qQ0oy{0 z$8~bW69WeC(SCh|)~rHqzm=p19XV?_Cb1D*$=U1&GxbT6;k!I(T_7H7<7An>rVX&+ zUfPt!+oc)%=*)S#Sgi$2z0+#n$?h!bR^|ZMr7sVhKQUUpZe=bmmnNvw{PF^CPwf4E zOTPHJU95;`ms@e^5H)=i{lZXgZTz$zZB7t;)=mCl#PVO^?ELkBOb5l^D|&o10|3yu zc$LTRq!2A}JTkCB*`F5ZI8@7%#@SyRw_I~;aXlH-(BX+!Y4iM+R-dGv0olU~2$5w(*|w${H&6Ey## zlA0mX1A%3n0#ZA7bOy6steEc&l9IDWm;(Sa60vkB$|j9bQ6ChV8lS7&_c6vxwV^l( z040@ldu0q_KFVE??YRQxK?OrV2M7gpNYu0YrABB_!aRMelfmvUc}UxUC6d>aT+7VF+@t zxc(hK;RmGe1pKGTYl*iwV(_`k%KAc8b?4`7I+NlKmAC+Wad(bZMIS4-&Qj7|x}BF{Nfs7Y496t?E)R)vyTh(?WRGr*79zjMgX&U^RP=CssFqBXG$6#BQ|He z);iTd7kFZanJLka4m5O_EN*$o`+$BT*+5D=Q>#8Oum^!+@~i?(x@K50S$_@xyOzaM6$y8VWm3a61h;C4`edL=MJri1;eY~^h=lGF9Ce&of#S!) z{HcEK8*43xH60tjf5+`6UUlbxY&6~vpR*4?PfVt$u`7k!Buf0`Y;TDcDW`4(fy9_xMPT1^88YFUi z$Ab*d6VBU(ftCxB3;ZY^@L<~IRhKjYOnom@g~hsXls&+yS?zZ($(Zl_{>PlZj0f#S zL}!DG-rzX!2(FaOhK}Ev=PruQFRF*}7SwhQcuJ;XX&+>cRiskBSl)m@XT5lRd^g4< z1$xu3`UOC*lZ*OIiT<`op!%%AG6fWBO5}Z@u46s{7ucr*DebS-j!^uzV==9Kbac?xd-F4^SMKrfv%(kXc1Y11FNX|O)<(U7mOibw-s$^ z+^jyuyQNRtH^2n8mFW)>;tx+m)uvxB`F94Df`ih0Tz{UJXc^LZnWm;j0V63^@0_oU zb1w2@`!cr!6^WtpkU9*ZnMKWxV`E8>X;*vwS~S-pv3R-W%;g@xXA^{*WoRocu$Ts# z7h4Afknz5;CpQu1j+;PF3VNZ^os^-;Usj-{DPH4=k_xymkbBhr!Oks-rjH#}ro(;i zR%`EfLmpE~t+E6V z5i`fHf+iOxZRz+11@c8wT_6*A<|g|kC-eYWgFJM>tyQKt1(Lcw&gxCGYx{)W(VQXE zAwjHVFt}KXI=EQ752`9uOl6)N!S|vrRpTuq zd_BQ{AYMB6tqXZMzXJO=uoNqx7o>}=Rw}Q55<5~d1;)x8@Y*O-pGnE^Bt7^8lA(9E zuitq?n%g1neTfUGV>q`k%m)`8RyTjTQ{%r3VimkH_Ij+JB7^72bg5W~ic#%k>P{`E8d znjAJ9JfHIstxJHh7gu5e`u!96yD@!9Dv(t~Q;Ltfo>wmL^)r}T)1?%&msZr*FRtCN zH>j<#aYdK5Z@r?;`|FBQbM#j1OA0V=pSQl1(d6`oZ%q^`IECJBsi+>R2e=nC!QKXw zu)>_8iDLWFq+LFLL*?VXJMDFc-5F$gU742jNj8)h^*A%0lP&g9IS7lp9sra%w{_b} zqqZDOeM=HRzx?rLF<+WU0*e&AbyAftWl*_U$6#Lyo$99t+?BG@&jb&B~Z@Avw z>!ZA@&FZ@yPFM;#e_aMwg%iY#Hn%2is@6hHbE52s(`Z*fbZ;3r7x=&_a zYCWTr`35vaxy_5TR816*+CSEeH2%(TulL+rgaQUDEl8lu`-Xe{6}~phGc;F10P!U& zGp^pIc^gal;8Nm^I4a?cJ0>S_LxwYHDwrp<*>&e#vHwn&^jHY{KLE=02ezp&C85&y zPX_ooW{?f$zcYgRM48M!Mtw*%QzUuL#fBiR;DV)_x=z8 z0K0b+smyFUAJbS<2(5Zn-DFBoF?rgMa=(|4@X+4k(FjuJS0!~`E+**5TXA`DaXMvj z2trFl*a>h3n1JZz%W^<*cwZ>^9P@!%LBWoj5J2EcDYk@m%^yQH_n01&(p%0!c$I#w z3d@m(S5V=cRxaUh%MA~>5^r|%bEJ8`D^@w>VGlTlE;ce?Mjux17@2-NpKpifL1Iot zpSmu*1OdFbG2H|KAbrfj(M|M}PVl9i{V!(~4MDRF&?u+chMoC*r zhqVFwT6wP%n$iB^aka(xbrQu}u!gy%>y?X&fWH@K5$PbzVbp9G4`_w`wGDq*+X zO3+abxrL#Fm%R~)!yF5Ybw-hFGnLB`188EVIt8-W*1+lwK{d2jAO`vchCdh$S}&H$ zoaAvNkN_2l3qTta0ZxHGsT#+A>!Mdb;%EJujmVT(0psuBD)hWK?A-``RYdCkeFk(b z6no8-l4Dm}EOB+6t!8|kG?UZu7Z`5afW+{By0a?805A8M*E;S6v;~yvmkoe=yGs}=!h)QjJvB@yG8Ms&K5na#W^#Vshg$%wQ_0UZtofv`eqOW z+i!vRu3V!Lw1i-A1wxN{C-xz)s>Zp1&SbZET`1adeZ!MU=)?#6>t1I3fS3mT-C{L5 zmb0mXn6OyiG40n?SWIb9_(P=F4XEUzatH-OgyP?%Zmy8S6NyNC*4*fS|N19~PjS)A zt3$D=@8}yHGrPdb+@oYTp<&Md22{$DU6>v)PlmUI%Q7A&GJ#Z~ZaelK@Sgv}$c3d> zuO2Ll^7SBIy(Is3Kx-T9G6w|AgOcZImIv{;ZUF*DtI`n0|Z)#F3oe$el z=Uy6RWFQ^wAy}uk3T2LhmH&=pLE-}sio^4i9;P%lI8!_jYU|N4)x2=R4l0?!vKLzFw8NrbTh9!bmu@WYmi*uV?C8|XYq)g&FrugFd>479v(8dz_b#(ob@*U6F4gs| zrtSrnF}BY~7bfTtMChR>J{>IlF^VGA-cE5m7ZSDEbDL{_V0a$aG5-*f z_(AM|A!eX{%Ecjba&L_h_u%=1-PMYu50RV|C-pccU?0G&66F-fJ6T65GFO zKU0c@GTy8h=i}(0M6mx4buIesB{7=5e+c8{`8sDu$%icMQ7cOLYDP8%fp8EqGv&k6 z@kNe?yEq}NDE@v}#x_0yaG56rLNy;%*r7@Llx(uuX224+9W*W;kI9(ol2|uvngW+O zg)8c&x1>&v|F?LQGuq`fJwN#ZzTnbAQv5XZ0~GeTkvz~93ZM+<>6~-#kGd~9 z3t3y7$wR}9r=im;bFyKQK1_BFW9VN^h zqJALQEPCHMq115Ej#~_H(KAW^-cvg_#4YZJDMn7cN6d`=#7C7_fSR|k0^sAp&BV~x8<&5aF?>KFu6M2qL5Qz-8B zDEDZ(O9rz7o+suH+qO4;yQ9m_3~ExDCoXlJ_CotF9S|OPa&|bV3`zTJ@&qwKeRY(H zay=t5w(>y2QJpNBa{(rYCbnGs6&X)8yZW2!03mTw@${eBOGrk%RgvNiiXWJ??87kg z1rwFpY#*cK=Ad~ZF^*9Yks*3~DCJ`ODujwfGRaU68$EjH2pBNu4Gw3+Cs($T3AW@q z3HS&(n2Aq0&`HPZuO1?(9+osjoy$$YD8Um?aN5ZtdIKdd$Q(sO2A2K)wwtyMP`=#vzNGU7V{$(i#$=CI#O#2s-<_qRalMX**QRMmlu}wkG4OEpgm!j( zHiJM4k-NA1*E0I2k=er0nleXlWQT@v4lto|J2_yDewGM7*r3rv#zg^*Gr;~ii=>sI8c zu{L_s>{6+?sBn$OFDYSv@%K17m7Z1hxC>DM>U+m#@5EsvS00J2PB*o&H{8o1VD2q+ z+1op@O`Q}hMQgq9*9vdN%9S{1^~+$Lr--aSROoQxK`rEkmCL#5H%>YTa8bHYJGW#k zC0F5#k#)L%ey|OA>4cHO1ecn0h%4^OLm`Xf+yA|eiDNqO0_p*P;>tvN5y!<$4EY?9 zF1aR8ZoDs@r5pBdPDpjeW@!oT1Q~}4aI%2PfUhC$XdSh01=QD1-!}Nt$ZZ;nb5AjKu78U8@f}K&h}pCoXiRHK+y#e1S3;<(>OzeR8FU zx(q#B(40&WcX>j=YG8YI;P0y|kN_fc2kh$DgFw8ztjE?t)-;#BjnSw86RS?TeLn39 z=W8aRvAjSUr4wTvU)wkBS=kJxzb@flE9>vx#rM+T8dL0T4Mx~U2#%mfq=FfYZo2Y~ z$5fu@m;@8|?{20*y6Jc46~2`&A6e3BTs~93B^MHw^~l$-r%+FDxe1!(D0jf4@o`e@ zXUtyA8(1}6u#FdY>~(;oi{{|9DuCCXH0tf3d`&p0d3L`;f}kH@XIrP=Tjh8^kvdg> zmfJ|5_PpTRq(w8?x!6M{HoDo?GZrvMSP=cnV}yy}uy~)C8ACr)A;PjOAJ`=@TyYGi z6|}bM5sWX#CacO7pK_!xw>CA5r|}JL)7=e(*XT_HKVZVRKu1vd-uS(8U)Gv}?PWfQ ztW6R@{Q{n%dPZtZNN@Kh<9gkbr32Kaz_%_7KYoBEDHpR_%EH1XLyW=_pGy$5!@*o5 z#76 z=|5&1t76WEpMpZHzccWEMh`z8mX6mjm!^hjp?B!2zZYoyG!v{J%`Fo;#Fkz`wQ&`A zf33%i9*`j%nVx>BuaO_T{UbXYcb};P?{C=|!+?!_t1Zc_O_Oh#bU+?VG0RB>yq%Lt z23>=dYqJMOrJ-C=@K`kFl+iV<2tsoCM?&}G`kEig8VTTB!dvo-}Y+(BCubiqi@)zXoY`;j*RFH ztEpJyDXEdlpE?C0YJX%h2nf%yBLLrMqv)_lP<^~x@Idy@ zly`9q6|hFSoNeZd@?u2s$ae}!qAbcX3%6@q@VTthdeXj(Q@$UBG}NZ)m`gc5^Q8v|7Dt}G}b55^TPxXh)9W(j? zo|@#+piF8W`E*?1di#!)Fem2i_D=W{g`>X|1Vr#Qco5LGmH-nK|i zgdrfUA4}5{$-taNIPlpF?y1D0=xfBsjJ%TS z*<~c{AC|>vIV{gRz0rY5D2jPC8AeYLzG8k1bS4n*+h zztw}LfPR+oS;Kq z)0IFP@@%x?2nn5x&+pW9Oo3`K5wFiZuA^Q!%KJHDG~*y_$omhuSaXs7$f zi?Rn-sBH14)06m*6c4l$Cdw5u`peKaWpSO#I=NqRj?dSJhEKHuBS14{klmtUdClPM z`4^|WnvyUX_v}6L0jO8Q&F(N$eKCAgC&qK{jh8vVe~KTo_qIFYuU)!?eT~rQ(!R>@ zO|Ef`CB@ak9OHpMl(h4hAiK!p-uy~#XUIG=mee|hl$Nm1V;pSiMHs2sk)X<$o9mPv zjQhS{Mj{r+aQ$|0K7ENSXR2rob9fd*q__5U^wU%%DDBsgiH1zEmFD@`KJqo0cDEt{ z{$I0@w&M>>$tp#l3i#ThltD2*JOtS|FXokdQor%=(%#Sh7M028^7XKBp8;$twnm{L04xx`|u{%Tf2M z8sd(FR^G$|lBQN`xQU#ZtGaVJQ%L^~&PR@y*X;hCDkXyJ(bE$#$hSc{)b9#CcGp^Y zr2h&C4qqw{C+d&aYhmbZ-Htnrvt>N@8Y0=`cSfu(Hc+ydQi)Hx5itra3pnBR{Nx0R zL%1y&sfXNr+0_9HmjCPzay5Xae#qHL1j^Mcz_oHNU}+591%G~@r_xk2{cHglJ(XB8 z40Rf?7dX0S9SY7aNe}~&%h5ROKh*09IK)fCXF}~MCa^c%kRgm63f8KWdsjz-dcJ}% z@g;1IjG9?OUY{))wK44U(;-7{N|Oc#&E-U(-Iu8#2RS!)(Z4e=ttZ{ub7m7*5;~#drY5=4)fnT)*5i=Rf z8KhY5^Gd!`M&rO34%eV25BXxhTi}kO7?OhqHL%G$etOw7ornC>Kv%HCaA~-Ivww>o z<;0EzlqG5Np7&u5wb+c}B6v?PPn&2v3%IW+&?W98HPS|A*!phY(ac&r8*|D$sK8I` z&F((zv8t;XnRQGm5sd|2g0gwM&Fm-l5w0$5zsr@(v5&a_Bb zV?hg0)I(OoI{)A=I(Xs+Z#*L}xOz3Z7VPAMTqz$J(+KHA(P>R(8Js)Bw;q4HqmX74Km5+U^;mXi4UO zn`>#@r%MX`sWf4~Y{))Y@OJ#90QJwOv9xY4>Lk2j@3KYL7DuR(*=J5gCw{BY2ZGsR zd&wzec<^RscjPj!9=yxIVDZ+F?rLRE!>N0)H2(>RxdbCzsWN$1G4vE>>$pl!kR8dlSSCzH|G(DByU@-(az3qycRymd!Lb?GW>uI6yz}|9<5_xrE#f zd!m_K4#cvn?|45`Lop^QPc50Si0kTwmENF=q*TqNe!(osW8a~ zsBe&xRU{2kZL(awq$UiU_siS{f!$68+WpivNnb0k(`c!^qjWkJPw*i?>+q3Js(0y= z;N6<4!-nj?VaLOt-3j7Vx4AUM3bZXzu+!ui#LE>0d<9RTX(2F%2n$l#%xv|xyF~A z66ca47k1L2a2bJ^lEF673c8)IFlWR9Jp z$0j?VO|(WH;ub=VaUGx}9n~y2|G?Cmhq=L8tZ5$IVC<^wdGF?(|K+WP-2}0}&k6xq z&O$i)YSppv*D0$iz7&D2Q01F&fy>~24X4`cJ$v!>*C45#p~ue~%y8VCb1Q!T1>%D3 z0pmxekwrKnK|ZPQxMMgellvFgFPy$H_g+81ypW^Y#4oVE+4Lp3XG>kXHt3_hUth5! z1Xz4ZIu4};dBtViB7_G1K9MSV8DR+;Z2VS9Y=Ht+vvSwGJhOZgqlqb9=I`ri{y0!xf7j{1_%S&so>*5INXqXa1D(+_6BbX*Vxa$AhYqNYX*1wzEw5{}T(9ZLyEsb&as&{qeOMbbDvLvVTTPR>~vT&#P9 zw}{TPSxGAIYq`A>GfN=K+oLc?bZ1~)W^CO*T^zosV(>LS;(`ULxgv2(hIRIQLP2Tx zd~TJ|-=@x2TT}a>lz-APYM;qaGV5I^5a56S0HZ^IVFU^&h#;)60Imzb0Kx`0CSjFr zI34_n#y#>!@giO(IF)DEVQa`cqt*N^UR{4Gwb;q0oEfMjB7x$d%F~HAY>w1P#vNln zxFG?aQZQULiHVYE8F;)wZ$_?Bj6vRH{}Xy(91EvR1(Z)V9bf%t8KcK57e;JIb#Xca zCqmAw=NKud)e+>QX;Dc$%PV*N-ro9~krfq`xXvDMI_?Th_IDWUy4a8sveN>;VV5^S zhyyJaagHuVDC9Jm>A=qhp$Yk@LPz=lBN!#%(GCr^DjB3pmlNe}3+A@kM!-4%zGro+ zB*C<$APzUVgRj1nSO(S?_lBOL$rx%5)80{aw={*Q>9h7MZ(G1n_R2-N50TBhpBJ2; z5CF;g312-QFDacUxaom>q%@&q-7I;*6kcW~%o@p&*L^{wN6}i!OIZ3dV&`3v}DzTKI# zeUNndbmU5W=1fNS@>u{oH!$_5RsyH7ar)S#R8NjQjSGHny0#2@FlE)tM^#PwPygPN zV;MIxwXMT}4aBT=WU5KQrhOi<(J}36r>7|~Ih4trcGVOrAS4u~CCd~#e3hr$`H$Ms z2TKR0V7M?R0Kk{;z$B~{i>^Lwz&K#d^sVDz$eDPFN;0ebwC?1sujax5DQs;)|WhnT;A%ZVI_@lp9HzrXRRDj*Iw$51p$`A7OVhq1b z;wM%|s{HXfo_+kjY_Sv)Z@P4{nQQ**`8JT|q}KYl&BiFme3+$OnNQ_L7k%_83sybs6iaiFjQnoL-M zG}gouPoiHTwLG#Gj?N>|D{lul6TJqIvl9&r*;3jHhi-+y94bZ#Lxn4CqW@>zp0C$> zv`Y(Qi+ue$mBEcxd<1TbsY*N0$8jN7>ViDpZLo1r2#Imdh2~=%v()dDDa|2 z1_-2}m2u08GZ8*QO%ugNscPoam`8Y=5;{;&!J({hNp3x4QJ!nj>*;?gG_1JcS)Q4` zhtFVn#hlG#5nztjpT*ZmU+E+Ep#YzP1c7lA^N#3n>Q0L)J|7r-M|xYDiBFBEo&w)3 ztY+q|)qWZ%cD0gZ17P}+t-V<}tEF3PQg<#fC}#^XGCiY%bXhqJKhioX$YK5E%q1d! z3{-1&*bP4aKCsz^dpee;_0ypqWlC$RSD2^K%l&yS8vFhREMJQu z)>a(!m#Krl`%uroqeo5gr=y&Ukh@NDeO7|2RbSktr~o5dwb2dAuJ6%i*dD_z1dj6; z9}U-1tN|-stWqsJ>a-P={U)VsEJH|H{UW2H zT%Twhk(>y}E(mn4o=7{PlU~!<%rm$4=U5McR}Z1qI9)<;yI>ST;nTQ8_5w|?dyx8w z*Z}TsNWZa6qU<}R;x6qQ1bS!Vjuli~*)-%eyN@96PQvJW3Fr@bO%$JVV7QXTXuqvM zVCR1@oo^vtKw}M_S~$##zN}F1=40t$us|vb zEE{LEId3mX)YpgO@l3P$aKC14!LK*;q*y(ak6~S+dOF^ZG6|hOH-b2vr}uCcdx&Fn znZs;MxkZeoy#}1df+*o#49;h*MqkH3p1NiHrWp#@AJ2Pz_U^$%LUL@Wg7*clCJ_qS zshiv4LtR;Q9Yf2i{Kxl_juIWxJim*f^MeEm`E9H056##{0Ad~m8@V|!b#JGn%rVk-LB zC)nv8o`YC*lxP#he%BUo`Sm}Fl{4wOZN9cLDY7xSk%Cp6JN&{UW>O$WK_(gbL@6?g zA;`73-+U^Xii0-xY`UJYNVDk|wqZb@6$cGZkY%ud#Y-2pfU@y#0>EJMam@HeT>wIkPw1(6P#}| zcb6(1AJGwN`=HV`yWb_zH>rHpEMl&ivbNELHRTPqfQ@)s{gyXjEQZynnKG~AE)ZPF zV?}ST5QXW>X&ZWt4ymmd>>E>;@)`Rw-r^_*EU{h^%Wc>RDvVfUh62g80{9s|%7QxU zG-^9iOnwYPt5JZ(wC{U*%aUKgcEi+OYom<9`fr&g=dQ`(Cr%~6p6W~?X}ND(HTq(# z!eVk7#?OF30SVW6@hh9OR`@2zmO%!<_@jb<3X8&05YURmg@fKzMEO_m_7kR${=;C= zFXH12=GpU6XmvCaN2MO|n^c;@c63ITr>*q;6xDRLBEET!Pvc^PN!Qqtut;iOi-8G} zHzVG*k~{6_yp_Idi+`lkPuxEygGK_iHHEDBdzB0#VbT+x4mFFG6MwFdtx8mpXZ71x zVQ}W9_x$d7>b6p_>(SKY%AYid9OpBmH38sp$NxLhEuP%(Xh}o--*OfN&38sCQh7V;^tpM<GG zI2>hDv2d5}rBfeoxckYEkCc3q2H?92g>qFK5cl~va#T`p1%!Qn@E_aanuW$0nK;zU zljxT62v-MU&}%R1lyJw(?OZ$BJtvr`QWxeic>2*N>rCXA6FDgoNjku&wt=O0j2|YX zhb58e&87u)F}CZndf|2FUovZ>ST#&qCgb^h&nYci-lBCVs1IT}MQ)zy(J4>Z9kx|M zRY@F7u|OnMhd<0|gxCDn6M;tzGYA%}eg)!mhEcJR;06Jbgwn9%@1*@YE2b2Yb=D!| zIHYsFqh(Z_5if12dC8B#4W#z6?B8Ybsbgt7eEV5sFYAvG@<`8yQ87j>V>OKmqGIc#vH;GLb=oLf8}ubFblKv^yK(OC(};LXN#?&s`t zJrh+E$b_l)eiqUXkS|CrTyb&UQIwd{6Y3?g94d%WclWSqFSj%}mF${`J30N`7j>dc zblNR2K+q{JGIh8#WWH0}d{~L<+IY>%2_a?Ttc*7d4V$@2EiWV=nR=Vhk6(PHq?|Zi zamr%j#@vtHp52h zOYBjwh5KzcqGU^%2zuOgr4enDtE@@B6?Vt%5kX)?0xi^cG0oTK!Npb#r=((1O8q^#=FM7w7StzM_$!jEQ8p4C-c!k#*(K$0MY>L{Kv zLo7_1s1N+>Lc4f|U$DUE#L$KbggeRGht(@Tj(&oZNO7;+wvYxdhVbtX9KF6gGXt`4Iu&tcmm?kC(|K6y!7 zVt%{^p3sr_voRbRil}Hg1=DX=!S%yz*?*U@`0)g8xph0;g<#DS{|Y7tuD^1|-~5=% z9C|yiIMWokKz{|bz4;JA=Ro!&Q20dtK~R4ZMheo0`ni%z%3h=*E{6zTqTsk>o|C{=vJgmLQ~f6yL?t`%d^Tj+oFdCj=n6KY6wv3szj#WbQ8-t!V02Z49Pq7*zO`?` z#F5x!UQKJHbwi?6k!%Bsvsl!26J+?kPfr_XUOO2?gi*M@4FzGL!XNWx(s^2BX$nc7 z>Y3e!mrG>jVO$ZU0YR5z+8hbxOUxp5{WoE9>Nv^T096{6;fEJ=&ug%VKTm+nYqahQ z3QA}DZ1o`ST%}E{XyZd%xr$RAHtfiY$aXbVU*(hN)>64j0lf*qPU`*xMEaTBjwbCe z3AaDE?{s>G0pASKLbD7q7$`SMzkfKQGGCk7@*XJc`2hRJAh9o?wHcSxM&<3oO64q9 z7ahvZ`Qsel1^`yAbyUVov{h?YcKMh&r9*?;qnogg%(DghB!!=~J-#ZRo9butIDxkP zUW5h2l#?+%$yBk-*gfF9Hc9Y(*>haRsgVxiRxG^`bGoa0<9V%wf2{AGyTau5^aN7ti30r`?Un6x#R#r(QC+Y`gMus*O7H~EziNZgrEfit`T zn`LxZfVWA)Wv=sBiml1eJ8v`dw^DxFw4*n!cs@*NY;uX>hPSSctilcnu67DF^^ltP zU{~Q%qgBhn^OVOn4R46fH}89xTku)%k8#MW0XAd(W*A(rT#^uB+j$d~la}5?8?R(E z3VIlx$-JKf{CSIysaeP#O7uIBXBt)w{+IS)C+eLP1b_K4An&n^EiIJ`C}^eXf8g2& zHwgztB&7l2e_ZhzO0DOf-S18@W=~*`|BM+AQ8uTjXP#iH)l);6Vg~wK#1;v(CmTQm zr#yyP;57d6k8`c=qsz<$c5pw8Edl-rNwXPofU^uUuKj4P1TIM^_?io zreO|vRwHihxJ_6|By7_D*XU3e^3C6bawzjKGg*vWrMKHrS$}1Jdq*G?FeA?5G%L-@!a#O|T_sHSs!nrFjV19l_S8)dd{ptyo` z@cVwAk(gNd{bf|`e3-NCvngRq%IJ^Jd zZ_mg!ZufR#YAfU0{dut3212T8t!J~;76TfGwh6ap;p3vL+w%~uwup1MDL^H@CF*vX z%rh>6k%(7qP)-LyE%WCdGMU?yFpH5rB?6+#h13g0D%KPQ&e;^;c{+9 zg*_|n9$}MeK3KP)f8EvlyyBe2KQXH=?NB{N81lM7U>}>I%ftEPV@2%x{**W`?xO(! z%Dx5!DF$AD_i&%cbB{Ul!l~!z!jxu2N90NcFRx6xbPXo@CwG&e{)y1e7Y7o7?A}yB zgCN^z17)k!94c37_%)d&7688q5z?#T7OZ|>Z@j^Q>q4Pf82nX6pEAP8JpF^5811S^vkeyh{?(ua{ba;gYUpb za6->*X2Ax!+0qm*hWOl!Q6$I0Bki_a+AEhSew5IvrE` zO%~WXe28pr<_)7|fr0Z!?l)yb_WI+rihkxvU^iu@ka#W42Cf2LeC=Q88j29o*ny_pDWqzn1geA+|b*G*zIV*~Gton2>?G5|vmWtz7b*o13eYiX3K~vAQY(=B* z4j%@^?`y-gore^Bg_E}*(>|#35rHXApCW!NoNUQ{XMWr)O?Ds`^tu(I&HC8>|U+@?Uj`mvdQ_Lc^3B1+f z-U{*c%TyT8q>L@_A-wor{(Ff{(u*sjV{ncWq=m~49oZK)yum$eR-xESFIyRmR!O25 z0tcXsDrWxjX|ENGi@jeTn;IO}fu1*hRV%qT;jcBdgaSlh?K+|k9LaV?3>KZVESG_O z8g^1@_gM(U1Mx7pG?jt{77V0}XL~>`$%67slwiZlDp;zI+Qu1qCPmw#X{ApDkAR27 z9l!~V2p9{DG3_BOXTJ^VdCc+X-pG+}q{Rf7KY^?fQZk-d+c^;{~s=7hQ8{J&c7t`QJ1k`&X9w^EJ1Jw}Oqostu z+t9E(#O#V4M@Cn99uw!9p1xw@Mchm8^V6P}68l7(YbeHXoJULo<6Xl^&g5EPE5HK9 z`-|L39GrOR_c2%EzO272P~RyFbHzCJ&n&iMGGUPGHpIWaQ>hA&hx6G`<_4;v_eo)u zVxmPyAhDx4%WR}`LK|rOVtSG1(QHvGr~UBOvjYdAf!oPT`ZXx3f4@788DalBFW0v@ zA=doX(lwwVg>RA~co}dDRAu%H#dE(sN}+tgiNJ!{mrz1F2%&i7jc~OxbOF zExKW2*dA(UuT}&e z)p3exk%im3vnOKN9Y+@a2xtE1>`lU%T^s6HlIIPk*{a0cfH4`S!9xkFQ8bBGu-{_;QCrVGPz2k(%7`Es#+>ySePpNICJ&q^Ha0Ck5SpjPZ9J{Sc<6%?~He zouK4v)%T)f@NdX*`r1pfpYn#tP|kJDjo10&nb=@_YI2u3486&#a9hk)#qD^N+{1yx z6?SXZGdg0@;21|8lw4y%3*SVsi&DsM#Uv+r__nter5T0Mw(s;8p`R#-|Ie2BBTB1; zC+HeOuci7~e%yVkb0j!PnnysWM^_@3e354~b@U9~3xWkDD_t0iKu5bH2nEwh>Gfk; zL)`ohLJU`Cgq7`w(b7#Oo9NIk%3*-$L=Q((W(YBl65!7E>KK&AeF%FY({9(PKqeS* z%*Y+y0Axf$$7h2aaa%BPzdp)wU1bP4X{%Aqi|%{cIxbUq8%W+))TJ+4VpaJ#Eki3* zajEulK)#6U1JV?W2muopMBMTm2_g8v=0 zIvmBc<> zWk(W5#gm9)S!BS+l1bHeJWLPiHXnJfdgodx+?dNQF#I|88saoKL-E*Fk(u4UiH_#a z>sO9PVgQY`A$NZbT$=^cN{3rYMiNZsqtT7FY@>DL|H8(Bg5bS6ZA^&~1z9qcuq3+? z{&H&3#TNyi7%ARfMVrs^; ziB``e1^lwO8ms`?YVl1<6@DF=;lWCQMi0mf_a;|;JsoFAsD6JtA@9R3Wrh&yz;cX)=pBaL;z~Ek?sKci%~+M3zf$8M?vojdt&YXg ztoggE*sePX`9x8Wmq19!gO3cVo2Jr*{%9hN`)DXppS)AhT<0?!`7cTXfR8p~%uR6x z6Hri|h4vueKHky@*GxQQv{T9U{t5bwnMs|473&-V zfAO4F9sVMuw9?HW7G_2`4wGY9=P3;I}10alc}XB2BVO;VPcbf+h&X zJJr3Gt3-HgPrYDsL$r@dt_Tm=#Sd_&6-1Z}(^ZCm{3{_glA_T^EbG=swLGNTC>K6H zm{Bm|gE{6KhCzhFdM#yFB*-w*sd>YHmHy+1F-qi=qL@wiyi-Pym4C-Ds>+m#u{&6~ zQNHCvz`UX&Icd@U-iovd8HlMZ+NQ=M2r_OKx1zfSVXiZIKSc|bof~(pVLd=N{otvc z`wr8sx{)xov2&BS_Bwl3gZf3e-+!HLo~Sp^3Cpq*d8E3s*?(;SdYy%DCCjVI3}7^% ziBxY(KZ&3~5Mm6Hs~QV@LwnMeR=igwA%Wg>LNO>mNg;s_NU%Z z*?17JG8iWo)dYeNJ;f7eZ`=#}Oa-D%QR(sV*wNr=DA`~9;(2e%E;#>}mX90#5n)6) z&NuA3J>bJlfU>M0^>tG#*uQg``nZ4Ntq(F-RWymWJ;D%2X5IYkp~in zAZA1e?cFrFb#R#dIwlyXSwuz6j%;nDzXai+?wMk0%!R*?k?cn)o>&Ydm8Q@?uxME` z#Ww&VGg_x2LQ*t!@LE>30GSb=R7ClIjxcz98>2cpuPocxzI$e*_*6;UnCI%CViOdE zw}CID@O1Dcd-!w?`H^TgXD>jxk`fn9z7ChskaH9bT5 zulP-QIwxsSwdR`8_U8Q>{)!XWVD&|9YZ7#=o$OVAvh@En6Vi#Iv)Ry#mGSdJAVzs!{g=S~}hYXNQ1xwUTt#cR6(;D*}JV=pK)ED8aK()9u}+hDREFKgQ6{ ze`;JmRGlm~(S5aP{)o+>nCYY?Uy=1pC@s}Wry_S&K@DoV`}Z?}BmgcsXT++#Q`Wq6 zt;9c=JZjjznLivZvBF7?VRAi9MUlf}%_>(wx-)E#YRbPr97|W>g-esKU!s1Pf$-ge zQOYu%fdB9@$uzI&c$ZCt*}?j_IV+M>ZQX8`iKjI~+9VdW#>Q%5`#3jpNgU{PK4|9i zpF8=%ll}&Bdq-+DGsnIX(?^KB08eB?G}!*L#sl!JQNBcx(Zt)b(6;+)OKGukgHDn| zNx6p31W8RH9yVKQ7%nGKiv<0N&~i|-rb!YS5?hp+mQsZ$oAeKFL~7X%F)BwGI>2z5 z8*$f^&Df;E#6W@XJDv~i0BjWt+SM-;c7G)LxR2EO!?1GwtwMKDUH)gHIMK2E1#K2Z z>bFO=S`@wHA6}ejIcnb4JSIx1&)kL3BZBJ1VQAgZ9Jb|Z-L`faGt#ZMBEnTKo>sa? za-9}|&s(opZdcW%|7~;!4!$Y04QLke0V!E?s^bgyd{Tp7cA=Qms-34qG+A@KMU4<#dY!{;0!t77XjtUCN90}QZMmJaHM}+ePC&800lfvZpx9pNv^PL;`w$hoH@i34$S9=Z z@fcFRlpomOv^iso2%W~9ha8fi^(2TK_Nm{4{LCg3&co%otF69Ih7I9DZOuQhmXHO3 zPuS$0+|OMxhvBJUzHRs$8iLuZ4NUxvnQPeXtVh(IQJ~Zl0$NCK8_~g_S3*mGK*!q^ z3-YQ`S4DT!>x)9&KtY#eT{xX@h%cicfyr&BA{5@H9DYLx+J*;%c)k~46(5zZoviDE zzy1X}Il`LaDm%6Uo(m~O|KooD&`qE$Y z)ZX40Z--W+${>}DB!B{%1F4UlK*E%}Wd zwkwb)H~zKb_XgsMd_`QA_fVS$BxJW@#5LaxMKn# zOj-2L@4g-*-s#7jfIw1|n?3dP$x!1;k}Q7|3;}Ht2zBEV@e^e(`dOpd>7(6) z6EO+Vg`8d8^nt7NTT07l9;J1Dn9{gil@FrU$g#*L?Yo1~@67Sd@V7R9g7%wnv16cs9r$#sS)SBQm)B(09&`E87f!;nQ8qkp~Yl#>-) zLIRQQOs_YbL>>&wYgflq!d&qNMi$SoytU@iGt1tY4mgd&By;iC0!AKKY?zQol^x=h zk>(*7m#2&)k`3xHL-dmhX9WEjP*_|Ea_))}`VTpkn1>V&&^5QTQxE?!--6Ix)wCBQ zlYI{ff~d>d)Zh*McG)7l$pEPJbp?TEVo@?T@^>SXg)Sz!q1S!ts8R8ASUFa|3phy# z2-*JcLI!0SVIERsOLW{>(Xs3v>^Dqc+@s?#&^|>m@m_x&9GH3kg&D_!I9)}Yg(3QV z&$j2=Vq0eFQ*{7dH)Ufst&v=y8!~XvQ`3qM)wgi+qlZ%_K;PJ**>cP(Pv_zrsX*}5 z06Ee~;+l}=eTGo*J2c6D?tmxptZF3S8E$fwPEOw_3H$c_A>WC=kg9(H(kR-Fa^jc< zs21We)a2agVa}nFc%F&3{o|pd1pS|>k}1n9Xqzk$<9Z+u^MKx*m?Dr5<+Dt#Tb{}Z zDcs{twswD}Ab?E|u=9i)(0DUfwfGzR+=LKzCSoMo_1T zm7?n;F`&Nfw|?17m=lBpheL@v_#ngT7b0Z~Wj{Q44| z1T*r5*E_a*&;J-j`(`o^jDn{7P&d7c!ij4RyiO>VpRtM)GY~{+y0P~TobG$Au6W%z zKkl5l8h?BF$emn&ODPYsv@&xd-J^4NJwkw7evxu5sgc{o*k(}jW5JD&>ez8R7&!&< z6tA*LA;F2FbCoUNEetetUnm{+X7KVN@(cyludsS;X#8f2mSn{VA}Qp4oo3VN$ZMeL z3Wk^O*S3$MYs=3Q0mizoz~~D|V^z;Cx?g3ZM6{_q=HNC>E~Bw}(PPy|#bt_cg$-Ez zu7symy2b2rR{1%sbe8pnN*?U_NGv7#z4|gGRCSR!F9`A?y=9mefUJD;z7lZ6JTh9| zf)fv9uea*D5`vLGL7ar z@9mdg7J{C4f_3_fN;(jKr<-`6joH#Y5)vkoX2|)(F4Ak{Kpm*Bk+UGKrATcU7vdEy zLyyDAwh{&?3xN`52Y<25cJBhkG&o@c9=vDwc2X>YmVo+GxQQeuN)E)ji@$dV@&@-5 zZB*^(Y^)u)0w{zfBI?;t+NgH<^{b8$0?zoEMc?&Lh^Sw0*Mjsss^qHujEX0upD2EM zIX`nJ-@G0f~Y-aIRF#K!3e zt+mxvra<-1c(K{IFrxLBOL(;??5~!`7Jt6Vw*ltB+K?1|OeY|goOpYeetRWuVe%2M z?+2*1H8#AFQtmPW0Zt0qn+UB6rFfBrQ`-Cu*T{CStdwH-wyg zb&hSpynhgBKBC}tjTcrvOGc73Ie1iu)zjOFP0kP0ub+t=l|xI0hDGa#swnBhZzaTw zJve~Yx;I&yW(zcz28uMQ46@uuGq~*2ZX9oqfQ^lNTe-cSn^9=CciI!H8j>ldG~UGf z{Nz--r6o-LZf^5pxMqSbow( zx>=7YC?G`=GDwX6{oDnK9K97ah`hUh6N@GM)`i>XUq1C3Gis#>+;v@7iWbBo^B(!* zMVZ4xD;(o8SmJl>Jb=~)eTrrmQdtRWCRHT@?_R_8Hiq@!VyBCZuS_-}x2$N)YPNl? zgjPi0$on1d#nh?hZG}?SFD1|U*g4|iKX!&1aie_BxMoA6@~l&t{mE)9$6(ixinCcc z>DXUq#`@u0{n?0KxK4xlwGn`xlH#sozlrYO-fN|?EBa7h(-7HqKO1ny#*F4mZeY+b zD&qI2jG8HiqO&z?7W%rbw-$p`zJ?irJ?G9KQYo+@wD61bpZ?H2%L#ty8*(Lbc)nWR zgt^7x!*UWPWJBwwr;E9~JouHU@zkXG*>tf0ZyY%!o1dkdAS7GE15XYm0;k*FR zy#Rh3#f5=UKaA4i3ZVNr;>5Uy`)epm`_gFv@)G#F5keVGqCTh!4>5(pdU8d8SWxRK z?b;cs)+Dd+%1Icd;N%)qIq3u+^k>%PnXmzGt}X4MNZL$-vt>*a%(o*&h$q&mSYkcq zD(&4+T|LG5?!Cz&upv5hkF7|&EA;}_bSS#{P1-Szp( zF2@h+Vh5?OTwXd7vzl?+saGkM+_+iX91&IcU}jLab+@K{VTeJ5u@K5C`OaNp8;*Nz zt#u(94e(g!lQlJx9k|8Om7RSgsuPq_{5f#nKByc$?^Y0)9=FxX-<61kea963Yb6}; z4f6c_-Tx4k6=v-T3JAX8?7j-#%X4@gK&5gkGAibpd_A5p>D zF|!5zwKHfH57y}z_V68veM8Kwnh#jfxIaM#ov6g?PM`JSbjJ^s+x>?x{~LqBaXGF= zJ%^F7M6T`^{nIJGfk3t}i0IZHW1V6O0`UyjaX_(Gew_s{j|*8l0VJb@w5iYH*SA!S z`j9~&8IoQA>8MQcvp2e#VRKfCIn) z#AkiR3}OJHKjGEFW=W3JE&|A9fzEYMeLww&8Dj%T!vXm2<4&p;dUh%wJ8-&Hd5s9w zku__ELUbZuPg_7Imsr_yiHqY+M_QRcJ9{k*L~G(Anlkpd&ZjP})~SBWUX{;v>}6y~ zg}SCnEjOTWa5Q@FVU^i=bXp)X)zc{3NF75yLL9ULL|d0oKPUc5rYolNIf(LZDAF?C zAAV5(eeVz`DoEU04e0dGh~1;xFg%%zl`P?TLH+{j#$@UufIJ6%BWoX`%xUUrWxtIU zsoNPATi06IRWtmQOe<-_4a<_uN<&Cn+lj5}6G$YZ5Nzm-@H5RlBxqJanuZANuNH-2 z#K&o5Xe*H&O$(%HBDjLtmt@^6sdMDSRfVU7> zX|%Oa8We1PZet(@iE*VK8&>}%{WKv47rjIc{! zM7TTfq|m`ZbpnK}Z+hSahRo!G$~RwrT}P_Mn}i}v0Eaz$|E-&91D#WSd?f2cas@{^ z9!2b`fgH?u+dykBGC3OSdAQ|-yRwD3_nQ5Xfo%(c;rL#4uE_RzZ%bv=?B zR^umXdGeg;&n#2#3B&n21O0J00-q$mZemr!&>2mb1o;< zlb7&|KOqSmQ;U52nZB{g=Wl7ReV0_{FBB2q1&oatAE17u>48z~yYy7n{ z2KnC_?w}98hY*5J%4xzSHjB7@dfg@dx4SsmzB7Q`0mzKnv5lLaOy|M?*Kv>~V;Fd% z*X|>eNj=G*h%s#nPa-wpn08pw8o}-bV#wdEJ$FFP?z|)_FgP-Rt6i~deBihu_DNX? zzQi!^UII`ClJ`A_7Y!eL84x4W$mu+VzwIB9bEUFs=CRCU%KBtuP4L5}|c*b`E95+WNS#VCo3 zvw!Y@b%&EbS#}TgfuA8mVlpf9e(+={@@|~Z){QgD?f>tgJ$kyZX@+;fi`3%y_!w zx^-sc;Po>Jo}gbBo-@_8rO=W%+EH zPmbLl;cc@e7L39Z^Es6;J$he+VS?y1n32e^p&d`+#kq5?kJ8mzSC!}U+J%qR&nV%Y z#u_)57SM^CnT_RwK~WGm49Tpw6fI7(+ewHr@aiG>_jb2x(qRW79Pfa(J~z^4M9a}n z5tVS;zQTFEv3^*I#y)7kDy+Aqe=U2|*oc=(D;XUTU0`-0&K}iekKODYC~AT7V{}QQ zo?Qd~Ce?G_g8{X#X>!;kGS;>4b(ocoL)468km6l_K_qp3TR^ciu&8^s#j`R{3x9&o z$zDPFOXG^9jhjRDNALp1y!SBz=C8qGdFTCy9FA(I)VkPb<{1j>Tp_e{Q&tt&z}z`c z51(nnKMHP6k$C1F04UvZ5nK5shHN?9L-Wij9GkDO>w4fj>@lE)RRE_=;T{WCdo=BU z4#}JRtnI&#G_Di%L35~!SpD@vs%^8{sUx^-(5BTD7q_@38-DmmA7q5zZ&agE zfYEfUjJhlf&}dt475BM@MXzaz9(TPsX(|M$(evvze^(dAFfZ+u{3T2D9V>Iq&voXyG~}UL3mCfMTO+Ql1w)4VnkH7D<^Zi>={?_J87P1e)ii(0`)SbBJK4++o_wX9 zj8)-ISLCk4_`tb(-}=rlqInluoMIAD^l6RqS9T18qs-Cq1ok&(tPb*|v=oZc7>>+2 zL#HT2wi6w8@3!BGq_ol(^Ywy}vOR)|-a(@=cGXwWsw~EJjFP>`nY}b%%!L!^UjW5x zkg3QXxz-J*nb4^^wp8V{>X7&6lut-r>HO7H#;!k6T|NK>lPM}=Vd-$Cl>^+Z6J{(H zC7a9xISc78i2cIiZ;HS=&IE^kLm}ezSuDM@hQuQXyWE_J92S57KGY=JKcT$<$}2|} zVh`KWQ%0Hj`V#h9Npn>lk6ndza2~x(mZpth!$IC<;vfmNN`>@+l8Z~Be>ix6zn5Y2 z9+2p&W>GD>6x-T%B3%*)T>m{>7zrE`tOI&0Y*pGF2Q}dv+}J z&o5`q!LpHkY(y6kHPUj1$vU@yY96HIo3uS2Tkb*&j%nLXT{Yj|I%o9>6mK|HOW zjq_;5AOI%>g$XRRHwNHB-asp%@(Q9FW0v)zjrLbBzxK-m%;h2SFV@mO?FBUnr33iX z_E^jxh+O8a?Bon7-{JS?1%~K=x8F)qNQ@DwV7_NUtG<>sCxi%M-?u$*6-<2;7nYBB z#T0q{7d7H-z4JE|xy$igwgl5EJ)0$lROedz*2fNM^b=1sn1Wpiy6LSAE5mch7UQl8I_sY z^uuG-pLH+%eO`23=`kt?x#)VxS!-G z2P2J`fJ>5Nae?si_^?(qPfEkLoOMrLt(0v#fGuCv(ZLa51}XcXLw!8&4((^?uoc;v zvoYXm*i`+D*sS83ngYz2lux)vf2LFc7X+5!_Y}AlgfDiGs;um>i|nE`{Bb7SKOOZA zz6^``x^qNp{xuZvLF)wThd#SC|91&Crrr4^I6{ZOYAC0-!=e=_jdOrqWJ^KuD!9Bk zv0{^59G;Td6T852*kbCj_94hEfvs-WZko3cQk@PMxk=zWVi;0p(k$&e`-(TnX5m`O zVyP78bgTiuS`##on6h`kEGt1vgK&|m#lHn1&^cijeu?i!`M=amdzuILA7x}54SX$T z9{#i(2mbhF+QD>1JlaIv?$Q(3$`cO%GqL!9fm_#`}!)4iS7-|yj z|Ft(|m$qjqXs#N!0*9@Rn}8x|8>AF)4u5GyPu~@O*ei5J`-47ZaD6D6oT{bZu0o4M zY={_LxjsZ7JQmM^`X9Nb3fh_muUoJm!iJ&chdesY8fx!wBVJ&oUTa3fAfJAQUWwEp zLP>zLXX_2WKBcZ+sMMCl2NbJp#@7G9tpK4xzQf=`d{lf2Tr>b*BMoRkx`P%8!6I1` zUSAiV;SA;v3s8`E%b8hR1fOCBgYCYtoSA&0gWTmo-b0X$0_gj>EqkOkMG>mD5uAIf z#lfaEhqcOLrD>!lK13cfM+rJ&Bf3ogoNe1t{es4puic2IB9XVDo15T&`q(E>=y>j$ zVA3!L;|QUn-5R>!q^V}j)B!l2lqFeHGsE7>nIHin3SYBiz&c+;Vj*au^Xze{dID0u z{$@jzmR=OJ5_1J#Z<%{|fpp$(cNx4emYLX*c2T47_ShG!n)w}|jgxGB_Uf%DgPt3= zNoyF#W9ck(Z*TT!3UuL2_$IJVR$i;Xll$W7?U{=K>KG(smK@4MrkKF^8)}jdLkOJF zFtUq{>`WMJ0OfbmVM&(SFt!OPSxT1M1}+C%fOXQdSOXAH7=8n-l$RWF4f>&l*^lAF zZZ8L_b*4cqz0Yx|%xW74sqS{ZhLli>O7X}=s)KRX6*gr|D{k6UroHc4G9yGY^@SU96%|s$NgTi9WygqMQO8`fUzIYZUBpT&zp3VL+ z=4n$ukpyuSiOK?N)osd&&MVy#w>G0uDseyY87Jcar~baJ61!rpQ3-ZOHysRgA>gfP4Dy9b(lMj1{O7Dx07@j_fCn_jFVTdj0Zm3i zMQv~FyUUTZ+%`Xjzn)3n10CRj0%i57Ql9S;k%8nY zH17A4P|rCIC!H@!1vnC$Vz^JHp{4{293COWHt(z&Ak)7rd4yp$)^`Bo>)s_tjGoH1 zgd(c5VTM=;qU6&`-qm9Q&}59>0GGVlKH7hF=sY<-uWKFkAAb2NV@2l)R`l^*NxPq1 zmu`0Ljhp=wD`P7z)!fXa)zT2;kMpq*I?}Gx^#GzNb5g@aV6FbiN072pr_-{#X9J=FGu_jwu2j8_vUUlFutG&d z5T+3B4MX;SuW}+hk)N|ESC7I3dpzlZAR#aEiJI98MuLJaA&+nAMfWKysM8|un^fCC zJlo8~Q^}#ccJ${MB=2AvARP^!P1fsTJVFewaq2BS`G;NZE~_sLF&sjMU)n^Nu6A~a2VwX1~74&HARyn!W8#3ylD zUSVYdd9(9@Saj4vwW02Pp_0}oHJ#OI-n)AC`2+Iv@iM0sX(}_p?^6 z*lA;43i*tJhkO1o`3dz<4FK;x$l%j*h7LMj6%V%3)>j`);5W>8@U-m!$gSI--aFhP zsWFPnv+3}`7=(n}BtjXDt@p6_=0|_uFFzkYXA52}2DAY-6r(>VIx_C6&jI@GXm5}- zrv*|2(Ob}JPdzD4r*}qL7c~YmJrG7Z4%OM3T4R>exI;e-BW;me#rF+qn)wbLTOtX) zt<_K;i?xdQ%B=i$P-LKmef%cPSZm+_I+SpuYo@-7TJ0!rJb6D2r$MpK`7$C2ED;?P z((Bh|AOyiL18zMCJ&oxp`r?P>;cUCebO;LoiBMcg$$;qz zkD^IokhHq&2VF)O6zRSd>=Pd_IO|UzJcd3dm*FQ%C{#F+X~0d9Ww#tZKS5Rqq3(b{ zy>eR92AbkqeZJCO7t!XI#T})a8#^rz7K#$?a|YN(S5!?>(5}p4^Vw|{F=kU8-6+DG z8W{mVXqGVqGMgke32G{)mbJP~vU0?O!i8*Xmr|Y0wQZmQx2rL6+usU4bi`*Kp$Vxh z=Ja$CB|##R$`9mrLgz}jaBXs(7tEU5j(38M%ga`?@^|z$lnuZH&Qy4ksyvQACjGVA z-PJ`x;F9*6h(@f z8p5pgCB^I6Yyv-?@Ddc_aAzUDTRcxL2H%K^+!e-F5u7$Hj4bl{M=g^ai~CguIo&m0E-g+9ZJB`EKHP zxv?+LYX>EeDG2psOw+JIFBvp^75#bQs%WE>BUdwJU%O;vg>aeZLf^y*drP3uX3RXd zLJDmaA zTQhKW!He)h9)mw2?kNdWWG)dZi@y;bAubC$9#H;Vm{VV-?U?nxPtrJqdP0fNoS1PNqDpqX5L9|lcXe%*;2CWE(3hm~%I?X73M$2c9Z z1Z8hj{F98UG-WQ$Ev+EBlrr=sjK}y(R7!??>!Q)m!iN=jTq2B9_mYMcPjq$f+nGUs z=3Q&|)zt=u2r59o^djMqG|5T}k+gUqlec&!R$+c1ju49r9(9M<7{{IWpTBI*7`LVB zFeAS*5w3W0p#_RB1qDwhHYHklU?+G$ZlxxVgv%si|~~ zY%i@f8L^xQ)V`-nH99CfNo=Czm0X*@B4*PR^h5r0N{eHj-6bCR zvU$o2O=oT@3RY?q&sdbUJLG0tlAC zfCIn)JgrP#i?W@gZul39abidV*?mzlE6wdo+5<}SGFk^rq8>9_pQU4kZzkuex~{As zvdYefX!JA#Jd*2pNVdFOnCmgTCQ7uCYY z*(kZmES#4uWqvOrJj3Igwj0IK>Wq5p0&xb`z-o-1zh^+sX|q!nBwLPns1=OYEe8+N*-%?(mBl*b@ES|L(Gv>g)tJu4;N zYq5VsLJFvXl5b7BnHFUo`T#L~NtQ2=XSG|Xhna!Y-C8#ou>Yht?O2KdN>wf6OyKNv zF-wYKqxfD@Ut8VT-Pn^N2mu%Jh@6Uj5XuiQz3EVm~JJQrYFF_u-Q*ZQg~MolvExpY&$ zPE6Tx@4ub|ZegW51PZ)$yoSKaei~@whHlI0 zIgA`?R*Hqrp?XTNwkcG7sFKdPi5EDm=W07f!=XwbOY(kM!;EgEcDB8l-^Z(W68LMCwfAd=VYVF5*!Y z0mz7|?x*W*fmd%O+ULIF6dc*LHPi1Pb4*nL01X=n#mgS?=7Pm)$v< z{HlW&ntcI4o$!_}Hh#&cQt#%lxnyXyN@-nKmamgCT>R}BtxW820Db6}@z&q?7rSn+ zbF3%^j$Imsy1(9TTd=G?Yr(Hqnl(&Z0W+bl7}|z(*5qHDETFOY)&yl~{cJ}?0Mh+k zj?yYJ&Fl02GzxWblvPOl)Z${Cru4A1twAmKqFcPH0{mTsOAW`AZ3|Y$hhKW9a(eT6~ zVWH#`J>g#)j6q0YMhPERZ7V)l6KfC(EybD^NIIE${PC z+)@aR7>4>asaM$_RZcf$WB@rt#r>oik)?Nb{(`(;VAZ*q8`pWZ{$;H~a<*o?4NT=p zD9EO^&w0bplWtwWQ~To5a4III=!1*`<{A}g9-89FNT_h(mRTvhg%h%xl0{kwFQQ*x zhJSe9{fLvc*3~Z7d8&pJZRFxk1M3+b7>D4?CtEB(&P?&Xh`X>DQM8S>j);}3u=aM% zn4jOv8~vy1_4gN!k7FHEJd_^Y$5SUKYN~v$`k(24Z7YuuSK|stdwVRic&MG4zclE$ zAsLOm&hs%31giBI^`hX}x|7T2LH+*lCR-9!G{ZxL6Wr);TkGA|+^Rg*a;BLHq%-Dj zi$A=K6FR?f@^GLk%wPl4`F3MPj?utEcM^P!V^}D337dXP^P{}kZYQX7KVkc{m+oZ^ zA}WQgK#F*6&w!It5}GtxV&^S@f@`NY^bJy#iqg zi8_jEAIMh(&2keN!HlXxVPfU|rB!YaRudKI)@Y~AaJ(xu!)Erpc zOyw+9@Ot^M`O+lXNmoG`rtcBlG_)vTTVjh;LZW9+|N9GxXhmD!c+aJg(~~mCY9Qt| zEt^2-mF#zkd0?8IM&`~f#@i$XmIa|X)8f@H^&*&9u%g} zD7DV}b2^sJTWqZ3JJC>XKP58GdD=%0SPO@eFh6qdBPKg9 zszn~1UP{na^w9pmaQ`1{P&2 zFiG-*?1RCPJFlAelh7pyv#9gf;;{3QOOEFU~fyDe={?Sg0I-x}PjZG;2nBwur|c!Fsd1#o?YLUk=21hv$J(GP&=W? zO~*ZzzzJZ<2(RFf%gZTW+=<(s^c1FS>)dI0A`IQISQ`FX!B50Qapei7+yq{WS6~ha zTWim4W*uieU6Z%X>r}PyVsd>fX#;SaLxfBPrcWg>~%)PCv+8g9Pc{@JXnbEv)qkL zKPZ4$1h2Lg*iwn0uFbyQwiP*%D-X!kBHkson|7O z27ULr=2$Aw7I|eq{kq9n2uB?>GkoTWT9c+uOF{F|r<4>dGP9te2>659P$fKiASCt7 zf8Mf%dqjHndv?2dm14#gdlFw%Utq)~!bYE6>S9Vp99+_OEf`Fy?=HmGR(U$TLJdM&zzGz04=qHl5_ZG8TcxMduMQ&me z1YrGdMDrM8Ky;t6FKv=xu7&YqOWj^5Lidi8EkXcEuEnF$uhH10w&U0 zwr|PoMjNa}Bml<^X-tzV6KQ=z(?gH(+z+Iw$0pdEr76vf0-g~pJwO^3kVQ%i<(IDr zex#wL*C38u#BL|uTx@T#0D&%bs%Y2J$E1yasCS~k6_$e1P|PG?%`SVwJIjyoJw7+< z>=5kNN}G;tJSn9vxxn6Ci{8$DSQ!`sza<%`-QwXPj!hJfh}E$$BA#zD!LC2f9=&WEp2=r@PRq1jZ|?h)1NP zhqz2y${>+))fT+L(`A%Apb}ulaTzSuFod6c<#EMH5%ytDqaNy!pkThQLXb+w~yzdE7KNzkJ#VM;hcQ84h$a_&?;V*#5pRNXfpnA|XI< zGt7(rVHnYeIl|UrYaY=GXLO6I_K&c!ypA~%Ix;eQ{oFp|eLiLROCc_*toJn8ZVLy` zQ2jgDQ%Lx47A)#P(cFs=dO6R@q)^Un1OmNrOF8y%sLIGcBoTMuoRh>dde=K;5SH?6 z5wl(_gn-=+Vpr%%NfE58F)*OC+vI`jpd^jHBm0_dbSL_W=iP|1+vD@2hrjC!;!=J-FrsvPTTPPH;#GfF#Yuijl^IqBDc9*uQc$*TD zu^Tw2*Jtk{Z+4`!OSS{a&G|#8k4oQ<(r?Dq+n|@Uh%)#NC<2545R9WnKw$(5NS4b0 z3%~#qq~4}f3&cpky~~_t7M(HNpbPyXmE_-)5ceT&mYEPMI zrmV*x%g2{!>!ZVZJ(;w%1=A!|;bR++2bVxk1L16a+R(R_*m^WvB&?jUhsRfon6ogo zB|3vVY-tjOegyOdWpTCu0Wtb>_(5I)tHccK1hA*^^C9A6!+ca4UI@#k_=d<~ikjY` zRuxHEZRvpE^zdXju}7YFZ|2&UaK zXPt5O6@wp=IKJ*vSVY}na{s0^Blm)SmX0{D0K9m=coubEmlO!~c)|R=UE1M{hX5de zs-x4Gm~VMBGM7kT(j4wSvAO8GKI%Pq03}2&9)vUr<~fWSv8y4wHy;^Q`UCWL<{kat z;5Tl%oVS#^#Cf+jAtZ^gCo9RM2QXiCci7UxTjR1nl9n(J7XD;wicrzPQKNl;!duR^ zJx+l9xJ~}K4EQ?wRsiEr&mZ&vjbw&PbkAHI67$P?89m`=)Tbf zyv*v$hBWan(?AM>Rsa< z-$XLHqmJ0-#9u{Os75NENcHiObr2d1CgBkU&9wu9Du&s<9bmr40t+kt^h31-(E$q< zcLuT-p7R_34zo2I?Ro5}4-L+R&t`qd z=@=y1YBrSRXn!bk$-|K?wzuE74ilf>;^eY9^T$APHlUDK^Yn#g@G85d;@Hu7vG%P70o#NKdmt6#u}iQ?qo? z8m=?JW6vmEYB=E1TmWPzlhkQ5daBZB@5RFq5SDDXTKb9kNs^=Wq#3 z0C*7IyXx>LlK3)fY|K}Tx={n!0}D~*ggjaOMVa!{f($t;T>t~?3*sy+i_QRne#*h= z)_A$v`4n5Z&$@*wP*y+iV0R=SFnu4cenDfHqva2N&OTl0d zD*f1o%qRem1r4lM$W==pO;e+C;sjHp*o+~KOe@&iY*W?m3I_%rf+(!LF)@Bx`k))7 zZ!Cl6kH*Ng)Ly9Qui$e6bw&I6T#>@gHa<8Y2wGiDxxlD8&3O@lkVXAUbrB@QKi z?A1Nna+Zq7hH{S=BK=#rPZs&e{xqPi4 zBMPg=6INnkQnKWZ099%u6(trWEqx(iGJ^v!1JD7XVMdPt5I`V_0t3JRo2cPJ6J#s3 zHoRKV*Ff6w@WNCp#qN2t(Ss{a{b9t-{ajde(^8JtP7@&aWJ~r3y5<={G-}*y(5SIN z#NklEJd2fWW8Xa*%iT)xwl8o=wi$jmkhWbR^eQ;SyyOx|yFBi&r>J zsG)mrb#^9E&Q8Njyd1{ntyVLS^9J7ytC)F$=*x#VO2_Z?4JLTg&=6^2qyU-1d;@|q zLX>E2ycQ%mD5Qn{(0!k%wdj&72Fk6WRG&Sliv$lD1$cHjQsEKcV0It*vIe|-S6(MM zr2KZxALQTG+Sk#$Z0fhN#Aq%W_wK{ufu_HP-ZrhUWxR+L;O(uY9?4X&Rv?V}swnRd&!jy(cS+xxazA{H9(`!|pS0tLI5&Hz{J9aL2g^hfUgX z)%C2w0VEZMfc-M71E@8un-pcrg4=eM#!#Q&>X}b_h|0ER7XF<}0@uM~g=pxbWC=@I zuj&C8EkrLflmvx9!jVC#7giUWlwKErE_=Tc+p~(3BRzOU$Z!&jqXmqm`zdjwqWJ2Z zknU7>(I}i$Ba(+eIxyPC1pU#EwUO0M3tcPyQY4~~!UQ`Bh?w6n7LMAi@#4flBZUbo zBo7#D&7r!Jj9QN;;=lR!S0*<)ai$&`0mw4a4=4sm3FvsEMi5~H3MiJ#0t3JRU@2Ep zH;t2&E{-%8>{{v*ZRJFEIE6jwQ^LF;%u>lYYn-dd-A{dcDh`R{dLE$(l7RjlV~V&x zD(IrPF`@vCKytr2IY>gu#{cv(b%B+zk8#8?X^7XdKq3{jvbD2jkNmh&O)gVPxNl+D zeNi`lgm>}KsI1Hlg=@G%P)j^RWS*6~yjdzE6;p;uGlxX;4BGYqjO`Tl_~TA9+M6J7 z%>q*uFqSMQoRdYSX(f~DtjZe9cb2v4{+K9xala{t5FRCzrSVN60^@*a_dU6cuafKb z&!XI-ESM%d>*5b}?lSD2JO>!O+p}5zATLb63b;n}f$-Qb@1X$`8NoE>%E#&Lr-ecjEWO4^hE*VZ1L*)3J!kM=@T!2> zzM0K3Tg)fbKRK4DUxq@SjrUQC0wM8c00@4YV3er8lGIWu48ASP8O6GBQ9s?H%O7P? zj}^U@cK9&34FgW#;}f$g#;V>GPC^Y%YJRhj#v!Uut>^D-9F%z>^#~jDx~PKwwJJgC z`8Nn*_BfFm8Welp$wsW9Z=4{nB@4A3gpBFv#vi#$wZXmdBCPo9K4+O>B_^h)ZX!)( zrcjyedXAph`@I?c|y=wB$dSPKE@YI9nUHEx@r15W1HA`mI^5~!j@CvlF za+6z41@uuWYdrt^Ue8XsqQPIM%5|J(=%%{|(R;4vX{cj{BD&0cRSU(W`tW)u>DMJC>9as`h9Qgx9@H3fT2`c&XnS9 z4Nn)cB!(=ZE+ai0!c?fmnP@)(HoUCcO7kj28iUE7;ZD28S<#j_|4%-9VG*hTn)1Ft zy5+^K)8lSyG=P6{buMLi=BbVFC-~@+d|*rgJx!rC)=VwNp8ij;s8Ca)4H6}e1GX+# zA(E8m!3;8ty~a5w*8}6xd|D^YM3J7>K-V!X)?SBFjiI_>y0$QdY%cnzoKJVIQmn`$ z??E6xgUEHZiS8@=`ViShTw7uhxEH94y_LKPAITB8QE$fbgM)XL%rn(>Hylo3rMtU` zQ)i}rn}CDc_44cI3^zm2n?AUI1xf#L)|8?|eLyX~EN9&l;8L4MS0<-WjxM94-iyb8 zk4V#DP83K#(DAtUe_7L~T1Hjs zZ&8dsy%=sh6|;k4%Q-M&sYljCrQr=d@ztQB;=@YX`KVp-`gr)!qS z$f__cbANb-`wvgnr-W*fqAwJ%G&)}Cb#WsXoNe+~=X}N$Y<+a&hL=K(6MHRO?qBvf zX=*bu9UxBFIw<5|a)CEZN0jzag&tp5%%x}l?OiAo#U4Lt1XzYi^#wGnaXgm|2+B0kVC+?g5Qef@;t;tvg_(^glw4|q0B;YCtEy7OGqw{_Z&Lz`b(J1cxmi%xenkM^MkScXV1nt0>fGhXqc zZlT2xpUFX*WDkS%1CqrO|9Mlh1U0Ja6cLH;@Mv^XrEBBDfYcN8)6&l`1sJgcFq8fG z$eRN^LW_el&dlfm;fa3lryBu_SLS*#Zjx?>k zeRTf2Fx+l*byg`9iq_pURJLn;%bhe|>?P4uA#Y1aAW;m@J=U4j)1u{vl|FMVU+Y&% zLPlRXafR1z)g{-rbZ?fA9(=AZ-6uj+p)R`tT{uap3B-1HHef~rPyy3}uLFDLj&!$D zy_HD?6hMFctsP7^@toms+$nlp# zLa;AIvq$Hj`fwF73LRiNCsu6x0?{ZxPw(l&-Ye3S#&5JjPw|RY9QLLu!dlZL;WZ zUDvuw#LN$gi8~Wxd*Iv+2gA0&90o%B$)3_9b@AY6@PwcZx zUzqMAF|4xCfl8=(O*gR_0=^aN>@NBAB1_~V+6}-0i&xUo)3Vp5BE-HEC>{$yRnsE>?-?ht-!`^Bfcvx} zoVh#HA(X>l8Ug4sn*=ZpU>d;rMvnjxK%$8(v%mm`$9gbejO}ZIazwG+Dz4WRgSPdC z+4aq^TQXn+eG`vT*8^?tE0Z~0gpF2zX=oV|cjT7$n)pFq#H3{!=iF>K%#>?exl$F7 z%p+>cGEz1~;2E8djn)vI@Gb@Gxs?$DZ}`fEod~GUXg73_=I+RHH>eVFU^& zmdgMGzyRz0BGm+P;__U2DpAu_3jbPPWLH##2`zSePN2Hhi6=@X@zf?sOebaLh}43& zKoM_PTiw-ytzyR%!Fd${cUMbD)Rt5#BVB4Rub1SyJaF8~Mn|>sDNc9P)6x*aOv~8w zuXe)U!KnIufC)!G%%ukhD?;A99z%6olTFCF%z`IB8o$pk*@)E;03oA>HNh6$DKXkT zA=ao5%4TrhZc&LZqgEQrtW>t;Y#>v>BNA&(@~Pni*$U@3Ch_x03B<|an-SEP%2fy7 z9{sItCSMJLo-N@pUZLi~dRjFvM`$^LO!cOf5wj_it|B*$>t83itkSsVq;|EzblS>M zsTQ1pCmFe#zDJ(H0(>^R>O{k^f*ZCgkp0o5NuAY+D!ULfNn}&)`lJrH!8S+NA{l*% zq*gtihE7*dCIAnxG*dN*`bFY&_DUF9N6$Tr4IZeb+%a(KN6V*~+O;3W_^WEXUZ< z2gyXAL)Xi$_RQM-Wq#;6xe9MzW@tQpUcj{TGe*E}Bx5s0@OcnNj*wyx{cV&N*eUbl zpV&iHRKUabbeBbSKV3MER1B60BZ5g``TW9eE%~pPgxA{m9BZ~phPzN)`9KJ8_sf#h zI#AxasfQO7RpCP8)5%5r_|+{`(A^YL8Y1WE-6YXq;j}?LHYX`pp+{Q?zQ0XbI~&Ju z1S=vs47D02 z`@y?@X64PHw~{Mc{Yd`j%uIk28WF?gbZcEsU0M<;I_)7U&3SYe;=mkZ7_u{g6?Df@zqB(y$FHmJ%A9wj>kXSnK z6Cpn_!*!{TE&YwnmIA6^)3~J_$woW5!Nn}cRNK~|O{m)YD;+SrmEjAdnu+|az`mPM zz&J5<0sJ#PEP+9Lyg+tH*cypppk>cO2M%r{#(3^`U>fNJzP1z!V^ZD03HPVklgVTG z7UDg~HU$0s$PG-uL1*Bg-eJVd=Vn+cIdq)CIQPo!u>{F2A&yr57eQzB2fvjMw?m_a zk~G#t-WwFL8__MZY@8eBcLn|wiu;Y2O#s0rdEVZWBDp|4_G2lwneq1hBU6Qu8X$qV7M2G+7kja&80XcA@3|IS)y7B#7t{ zARqt1C1_#R=u6_h|VZ17yO6biN5Ek{*&-I#v(b z*E&i2U9yFK80%bDhPc8MT%#Jl9juY~n;af0;C_^DBS|zN%L&$1*w7L)#(qL}Q@-z} z&b{0NlXE>7?_rm1gLAxCFHtoX54>vZJqLooHG&_JKt&<5k8OrUC{eUw)iH>Cq1T(a zyPzVGXiOI*z@{$s;q1eM{L7JCzw)MmLGzkhh>9_2l*G1iG0;=oD&Mo%}W{qZrotMFgREyb2Bd?H0vX3h2Gl+rn~qy zD{7zo`GoTsT_MZ!>OcQ#o-*;#K5Tt}wbpIW=P^Xp6_U-tMNc)>tO!($v*xJ&1aps_ zH57dMZzHHs!7v1*jmtzA0p6WlL5gbIg-S!-$5=Nz?^Cd2)xK161}Sqegs5R)=iMeT zk3^K5<7+XxiU zcJ4%NjdSgAmt5c1Ijh(QYhL0$fwTc1GHvT=Pl3L=1eTA5`!*xe-xStO3je`4!7@z) z4bJ8flBNJR)6J&dl^08t(Bl;imnfZqa{j432;X0V-aL%QjgMyIH024cK7-Yy&Y+It z4{1f}7oTIxhBhz1G}lVjuhVf>UVF@iAsvKVJ;l~;0#Z9|LKaEbOPed8GISXKVjG010+9A#Ib$xQhEz8qk4(h2pRFCUBVPsA}xe zRDRO3AO5(njfmQ3xD8}AyPCf!8jsJV@LviQ1GoAcFpwgR`s1$is~xmR0w(aC2@d5q?imR@fX_}@}kG<{zke-sT`9f(h9Y{{!8Me zy@taSC}5nJV7U{x`nTQ;5X>H-E+hU|&g7_2!n&F2$tcL;Xat;cpmi-Zm(T;y)&t&T zTiP7iidiKd2>`*^u3cyen>T`bK!xzsEc}!~DS}_x%I<_MnRfiAk+MmwuC;Fnaz;Gw z5sf-Uk_g;9Om2}Ywv8RE^*C=0nqX_aRQvJUD0o#(AX2p)0x(@ z)Lm7YV3$dI6}Sv^A+6XyVM%Ch^W4_5Xvm;gQb{5N(Cj$ieH|L~UCP}#HHJdgH9Jtx{NbWGrQFj0f z@JeR6rmNs|q1Z)@#4mw~A?I;a-HHEbvqR+0xv-x`{@C?hzVI;N=x=sYV;DiE zPI@bGuXyYLDOOZi3smXmM|BMcApr&dBATOW(VIhfL-EC9N!Oyh2o>~ z-oy%ZFv93V*=>TfiMSyAqNIHMv>FcD5W&+ zeRj=sVD)?t6kP+9tO)-Ku)9{GD;l&sM7x97-sf8k+Uyv?)eS^cXSClnU>E(bk3Wnm zKRlQV2^)s1+FD50S7Wq;PMqQO;xZHE==Cr7QKj1W^~GB78uf`!1%ac_(GVX3tqv>z$*hER96ApFPL`= zR{T!~1XOLMUhN!OpdiOs{w*~ZrT|(+=g;(xI7a5-N^4ZcV#hDGfd7}!87MX6wPLSa zJ@36jm?fc>+}=P%aM8j-Gz;wXhyXESt$sm>vk%rUT6eW&gv2TMmNcR3F2F&XH^++} z%2Gvg`50Sd;WLxaQK{cIm+eO@sdIkuN}VXj*-|g2@kfgLtU}k!_|}LntzmAj!r!xL zO}PABTi8RIq6AHIU$Jw^+k94@aA6W{Q>|RZPnQ2bH^8DAn>Nowja!xP7~kugH2>b| z(!ynj$l-`|ZJ^&vKp8cMZa@?I6W+szz+pWm{c*3T)U zuT96iS2!3DPB{k6z41z(a%!0 zI<@c7=F?(*&~~u0Nw}oK0vw2#L_VP7b&Z6wmMlh`AJhrIj$o%?%qy&e>$*Xc^|fZe z>(SBhYto{BhG(&Iw;C<0B%&e2J#CPH)&opPVf?)ZE;*wE*jVvj%d^p_Fod!gq?Q7n zHGzo_M4_-Yli-o|U_76OXkDx;FP~e`%he<9@Pa1S@TPA&i=KktBv(ElTTD|{1DX;l ztb)3yo^KG1E7_ij(&8NhM7p5wS8miY4@M&nAnRewJON=xAC2){lzB@ER{zON&lOz< zb7(J&IJo5|vaZh7@iqaDcM-3{FDP9;diaJheDQd#dO{4M4Ckc?RU6%9a*&tQNCOL8 ziNA^X3M-fk&cI8p^w@q-AGwB8U%VphppC$6TwVIWGTjU)3WNvnl%qyaVFU^&mdgSQ zzyR}@JM>IyMO5=~a|fu0fbx2}T2i_0DtrXhQ|a1WzH%|I%{i#@{)S2N4$wM$ z@W!&HY`GrWDOLfqt#6{LSnb7HxH5j&r@lm3+d>%n2~dMkH`xgl?Tp{Ff5z#(P87-| zZwo#Cp!*JMP?vR*KD5F92l``J#}f3*F*G!(Cv7Vajmf%(Fqsh_qh5^dq6{9Ti7-kR zT(K&7zAwbbIz*KLcpl$T!O!vp$)NSB>XUQR3A;%F?Ee5&M%;`UBDCk7Ad!QDv0x&4 zuw*M|X>J1M(OyNLk(9DCKS;o-hHIWaI2$*tyi@;F+wR4%rf(KnIRmm zfR?NDVo&X^L%pi4LpB<%%RO7 z-0MV`l5kI=EEj@$>N<;Z+%SSu>RUR?PrkV+u^j?nOTp3w_cT?Wd@&$MTL0Lq=;p=- zmajy9>uW&tfMo}bL-ZVZ)_U>B1|&o*dy!#4>8;Pf4Emw59^{&qZ_?rn49+!&O}SiH z+A`BQ#w5CF#Xl&nrw>kN1)nYM8_285T+wn*;vRKwQPb3<->vT8jhlbv*-pHDzFiWxKHdbQ)7M{LHPvvb?8D zI{87X8MvV+X1A=)0qs7vY`)0Sv%D49GHUS&k|!VUK(e`Fbwtx$R{Hjk9?Oe;^{A0` zl0N4;ox_`HshyzSj5$kpKrPSpVWw$KBT(d|uZaa;IRLe?ZIvicUa{elE+U2|NGeE6 z95A(tkoI6-8B>Wy$7Rog&a$DltkJ91559NhO=+FxhRxcIo>FX`l%FxpCHu)Lj2GH; zK4h$cgG?t%=q#BCtT}2gjtA`?L8`T7FHmAZTGbd$)d>A$GnN0TQCYT*KC!DwchVhpMtj&GFzTEr zr*2hr{(CU=4*&}fndq$;4lQUCu=XA|@dcAVkbg%NPgkC&|JsAZu>vF!z*fq`-dcO) zgXIwcqFRD3R~@hY21Am2H}jjGC?%pKy zNe*-i9oSJ<9k%=)A*8giOa?Wenrt;m>8|mWaGVby7#M}v&6Ocp57ZRC#bX$%y=sU# z*n0pS1`Tqub(Ar?)1e{!#$^CL;p4t--Lc6P`z2r}iRP~z7@e$I*;r|oWjNWs9| z(LLsTV{VX?1q>*-VG=#2)J0^fvAt%dF(XQ3=^Bv6eLk-25R9%$O-}Nec00AfU1MP- z4+I}FCY8^GNC}7VH_}v;5P++mZk=6^Zoa~R$-!Zy0;Kf6qD0xV>hEm$% z8l78EA-{ z>8StF1@PNCxv^K;*QBEsiOK~M$!Kp$3yAZ-{CLTLGm;JD(gan4)bB&&8Z{f;Ujo3` zGWG>95>OA}VMdPt5J04fEVIA>%i`vu)m^$p=|x~$CT90iWP$^{pltxfJho%<>YEaY z+UC>mQWcN?hMfgGRw-%Bjbgi=?b&~iz ztvVRzkpxS2jNI{lY$0eaLT-bgdfLcG%kYnMpjF9J0G$rMHG4M+Y)R|eK}eZJR-pTa z&bktjGc@|mZJl4Ajg`0jK`DBdK)0*94qs}G&Wk-iQ|nM~Z#4`!He~=ntuRna)A!fu zFrAxmh2lb}oazR!%ZBkoHk85J%*Rocd=(2U9;A(Y1WOY0*Z(4bi^}E-LQU??*f*Hk zz4QfCbIJ*Da-=Y*uI;7Dx<-H`AF#{Q!%@LP2u2{OM3%ISD6oL1A*)Ub0O)l96fj2& zWyL7J0z(v}WE8ONF25M2-TGTm6h=jczc2I8HY+A}-ba)S%{Ao`Z{WgN{eqRjym$65 zSTXXJdF`%BCeohgVAR=U;w}nlfPdeE0ZQ8O`lsFADZi-*1I4b?rm15+9Cs3d!XL|2 z@7-dZWy7Xh9D);f>0Lj+b93g9s$5CvDYDL5c)7FXG;z++o-LO;6rzPQbTfMw9ty8U z=1JySv-HO)o7y3eLIkKOJVaN%m+_kTU6l{!3QFUks?BvezsaP?00K$#XR;h!`8xHw zrxz=wtOSiaQ9kSi4!;=6wVN-64!ahbHqX}L^EIpPG7l{j4&RRzHr=>3P{}sh%yoogH2t zF*v>N5U=nZcEBB+NWnrE#;4 zuTCrTl&pkml~QdRhd~P3F+$HZxx820e`3>(o&F6@jB9bDdvF94&q&yGycpo!KcRNw zxr_BZzDOjjSKOk6&(1Vz_VA1O03A@uS5cYzlXHEUddTRJF|GWBF}dZr2q`9B{JvS( zZQ=_oY>+k>j|=)iYS5BhS;jIEO-Xy+@~%nN7C5!2i`Wa48kBkt-Qp0C?D}DGUJ^{r zgO)-4VXFzVNi4b3c^OQeDd1m@pP$?RojvyoP3G4?SW3e* zMk#<>5mmw12=!Lax_ebuj}Bu#yzSYxldFaod)mSzLEUTSEuoExGMtSo4rZE!pXmJB za2t%K6b?{~!W56qt%RGJTg3rDjy{XzC1C87b*bAH#*QILBTy!gjbS6zjy= z814BaeM{L2HcoyZC0+*85^UzgI77fGYtd@f)<6cDXK0C3-5j)EWknD;MsFYF`b*;- z_TK{y$|;XJVQArR9u6$S-1yw^nmry4Lec+*LKx^A>5Y50e41iMNSLw{pdSXo`YR-A zO6fj7puu$hlhhPz*MZCe7?}_etXmfp2R|M+B=!Z+jO#3Ys=u#rV9+Y`+X10T7hRR; z6%m|pfkvJn+F9P=;`q(WtzYEJs{^c}`3y33DjEUEGUE*>4nYa%SffTDVFU_DlFK{* z@)9I|qFpqivQYtAvJ4(iB4-!)W##a#uj9(g`;4HAhSZwoe~eWfk9MWsDKHp78X}2R$>j)) zC_ws*vXB6z6j*wogGI?;ykM1xDl(jCoIW{mhPr=lPvlZ>XU0ts^m0u%YJd3 z+Izo#KCOK&qd+{iLFki*m&2kF`yHKBi`r_{V2|`&!BOy7u|}J8g*M`<_Y65VtWpUo z*e+lC6z)u(`VVVO=;dWXLS*6n$Qh#-q?gyO5o84SqIBOqq-YfEzUFcz`n#OESZnXm zMNx)PQ*p}Q4i#c{VVmY&x%AuDQEb~-;Wit^suy4^)(PFJpL}LPqAx0GSQiX~vo&}w z1_D+aSzSvZ*w{Ylkr)KCXPq{oDqTY6!@w0cg{J`AsL4(H-=UDFggowXo?M#E@mvg7 z58b1guzPr!__Qa-R9aHb0n6u+$h_<2J$4f-_`(!?R~K50+a|aSYZT&Ve6%D22INRa zYJC}fo*|0EUCz8}*?N3wy=i-ijua!iMpe{ibkM05#pS}^AEIcw+o4Bba#T`P4Cvho zWVWV0vWEW0<;HMEt7yj$E}!y}*!4}EaMt}Clinw=N9{Z5LK0>$2q_224e$Q9sz|f; z$xz^Gh$Sy|hXMD<2?(A0t-+8<)A0P-UE@UmdD_~CEQSS|kg=Rw>GEZ?zgn=e`Urm^ zR#&i{^D@JqYRUZoPyf_6FWQEfklQ{r$NoN-LZ)hdMd>#JA}!Sio&E2;z8eph^LA3v z{Dm*1W+GxF0C{Z#-lzccGU>BH05ho3>Xpq@%L>~Tz^!m09pYA?)W>L$-_QFjqmL+Z zjw~T4-uLFI+|)%F*nm`DXmJ3fe?RVBDQv2EUVfGt`GkLb&eY0FB89*X6!|%L! z-{+6dn@TZf1X$XMBAU;CR2_-3hAW-W)&S_id={IuZ7~EC=D(+>@}S!6Wv}9?QM@d% zAHXW>vr&tiqPL@uUn#PIUW0^42XD?R-hwG%?d7TJU2fIXY*s&}Pn1Ww?>ALfDd4Mr zu+Z8WkiI{L6o3XL{$r(4OybblE?t0Q&aWXK=WLkiclIjgk883Kn?;V*46m)WXN&2` zqxH#WtlpsQyYGnlP@hRbu@xP$Lb7@$Hh=r>i@Jx_#pl4q0@=FzP4rB}+GQ)w#dVw5 zAE4owFK=v;Q|2NdGJ$_)vP=csXjSfKzEC8oqsSR!(XT zWP5sP7D`mInspHTqT=$jyWj#Y5Nb#^4eo}nMzzLPYK!ub2_6ge&K-vCvv%A1^tmVV zfXF8;68ucQjW5bamcIRIy}w5N;)T_Zp1sIA%0rWcIS=kI!iH^*JyfpZCt=n1$45j* z?DSgn4b_z^B}L4<$?u&7#3aU%GqQCl_y1Yx$-M49%+ef;U&A@ zR{xAkOuZOkA?0Z@#EqT4}Y2 zO$Exp6Vrk5|81*Lb%l#zK1KAwij`-|{cuRXCm{4hpJqw((j>_7Hch+Dt}ow zu2vhyF0XK+-qbsABIWQ@or^wFK7+&4sC9FQ6kW|tD{(oWq4wz0R}}^)eMOq9>=of) z!7xxSHdi&36Yi-#3Q!f#`cD#}x9#IQ2w${0w>>IK?;$@1XB8h?c--YZ-Vh%^Yr=(3 zAYc>Sm7x&t`}CiODf*ODgX+!n2F3^k`(+40ha|MdrY~JnMLNX7fBy( zB~Ep#HU|CXOVR(@4Hsj; zbbTb_WW*axaXxav$e?3l8|Q^=I8uq?$mWSqiDs$Eirj5ijY!=&a}cC;&K2&jNB8N2 zzR!m!56sC8F`{c3ZU?wEvoC6CCi^nQE7Kq~C11}y$BR7D2}rRXROYu8-6Tp-AS;(F z!4Y0X_Wq}3`>y_Wqf~JN^?5OejsyG(q&5}*_rNmM1uz;=6VTB{j{p!rqKPcCfB^Ra z!QpsnO}ZeKteEZz|5U;)>C@}Qkw&vaZ=*$lpo&|yP?hdKC$mj$C+8fY7zW%1H*vbz zr*^zC|GL&=#irjV6*&kEvlh`0mI7>^!Ow14-$v3Gf-VehaQ7`$OSVo`Su{WjKAA17hskmF=AkQdgOaLp3o8p7^xpVa?8m0-5% z=A;6=tZ43bV6^Mn0!(L3NNxVAiI0qvqtflwdND1Qp*(v}(i67<3!-5VnEp#M*1AyX zC?%nY8T@rss$CBeSlrsApQzYA+YgPddT6!9TpZSP2+%#cr_|<-;-QrL^6u1%cJ}Ay zn@NyWA=hZW(8WHdJ2ZXeR$jm6+7zYN?z`sH!`yEM1ZSjJj9tca&%?%w(|BwHWKgY-u1ZDko`l*nOEjoj_$ zeZb%8apO2y0g8eQ2}laHBy?{`x5OMUNzRd*U|Kl!`?_1OLI3KLMbsUfO>sSKZ8V^h zlUDrt(;MG(#*zamof%Kn{CXUY`ehhP+BSrs@x*D4Xv?Ddng*62)l+z#MhF+P>alxukD&ka;HU0a`&j4c<}$9EY*Q-xecTkA@_Pav&nXEFh2n;m6-u{T;5AA21eX zmvrAksKOTb`z{4=HyR@Hy3Yn88>e*pv~wN1=NB1OS64bTk#JHib?_p7qRv*joe%X2 z_@2dwI;B`8nd4R}zp(*_=^A*h!*Zhqn#Z*he;yhExH9(yFdj$`K)FVb01!bUi7c^z z0QG8{{Y-R-HvTrP#U%RQFJvS9&9!sHJs#39>yfiN??LG}*cmF4)nRPNXA<4C7&^rp z;EfYoP5QkJ7VeognBvI^^6{4k__&9^oL&Nww0^A0dUKHwE+pgsM#$PIBo|sn${i6k zT-s_EDXbEZwtvebdfq`j-}U)w`$PiTgsC**cnUEq#}f1HA*U4vY@w*~{XXL>aW+1d z(D5_sFS`R`yLMpMNN@zSX|s%PW!10p?rJ!rhSOH-EUasBDO0|Zu}ytL+p1SG#T|a? zvwm~gG*N;2wwhhurG2#Tu#1r4vR-bT`QILg!e|C?TGflW%j*!qU14={kQKN7nJ#3K2hE7vXA@lV9dinT-J7xJSx|Jo#!H zgNX@K%7H|NTJ>R;f1;4)v1hVc8c*a$+;&h4pUst-WGH{7?pD|cw2f=+FW^~WkquWv zUq$pgscVMx-@#XO1vQ!#%FcWIxWfHIVH!>$qrcNhUQeX{!ia^W>_D;jO+m|xVxVBR zc6Wg&kBf#F;}GYV}PuO%MuX^9l1bpy1p zgcr85kTB4Q3Q6jteiXrv6o&0o==l6`B4&)Ph~AKpQ)%0@N&$q( zTxmFHF+B@!GfYvZ01>E6K0Er}W!YS8pg*MO{5q2HJA4hNggldr3$jQW0jM(YDJU3F z1z6CdMG#>G5-6I>00-ZWN= z&e>O$ty9=oh8YGY5NC9oi(vb6S~0JP`VKeLWE%U^-}Uk^UjRwz-pXALbON2R5(onv(eEZGCUN5 za^}D1E+FBBtkzVd&6dyasAipo$3glnT^wtYl#rs#4~M;0d1xBBrn)gzNz>+<{_u#~ zion_-%=#5X=$|RI5s%%@_)t`&4ig7Q(ksXi1#uQ&`eG~GLSp+1*WX5*C*$2<3JqJ_|M!Y^0>#FzTF`bSoZTHmY>mxQhyyDndGqL(NyDQEFUJZp-|^mzYp@sIpT zz;AvhU0llg6XJLQq!&;)*Sj;%EAN!jZGw?eVocNl`b)8NgPtS=@Y=K6LGT;AId@#} zWwg1jvu3NK+ABYDy6I}7x}1y$ zPGW!woxHM^#Zgx5gL0Y>)#e0B!_Bn`ElOF^jen-n9fvlV>CjLHip%C5+UG&i4&`kM zEQ-Nv)Nkh|g8UJ37#4Jf+lncF*+C;}yNy0q7D~eBGin8YKg5cV{~@SPYKy`FWwbB;V z1d6e1KAD8vZ@*j3bN|7y;lC6x9iM^Umce_zVzFvE>(81hmwrM=_~8=4uWr?ud$g!u zjN$!g{<_(t5ZcK!Lc5Q*)>k$99qgBA+iQP2ddiogz0wk8**NP1mI*ikaDZns^&KzBtCf~^mc#Z0aMFax zn(piHy6S%XK=VbM(g_?;V6=GGDi-{M`~P8x`!8qCaPRZy{vx#M6bVs_QqFr&(zpfE zTNv=uIv9Hp29yy=Ipd?5Oxmsmn$F_A@7YpeVC4Jh{?>fc`}PiTs?-OUuYw35o@EQx zrJm2w z4>ombO;vwB4a;9eHT{~V!-sQQ%ij*=pXYxYgidP6NwIKv7^9#^NdItZQ6%9i5F5rD zLD6nxBw5+BeQ>W0QCIe`W$%U*7~o94>d*7`Q_RhIQ024q*F6#$;k^Y6y&0FHloLG_ zP!bhTtrmF-Fuoh{Of^2x5@AA|*VLNDcvw8k4fM90T-`gNmZ+tpiG5#l=)bBDrMn9$ zh*bO|URmGsL+o%-u@_a;*&leR>_Q#F{Osy_rK1df89!0>xfvj`kJn%rR(#Lb&@&9^ zldkEZrpmmh()U1bl#H7FA5LHv@XRxwM5RJLS;>_e=Y6|x8s5qx7IC1gc}|1!ODL9N_;(7Y5!wv;8#DdmOrXy} z5qzdKwQ{ho?2h(`o&IHqyVeUyGi3upi_x1xg#ksW=lkC=?^+}>xwRuSUxq6*JdjQi z{Cd+O{_>AXHIitVgcs*RD%TD@!FnBfBuXA)?vPAYcxTra6po=dDwEk#w zcaF;PjzKy>VH1+-ABO1g+~RsYtGXd(30AOs;@}Zq4spbeo1!f*z$|Vk5 z#Dz>7y2`&pJ+0&4IP^rCRS_GtuUY?s=J)-+lSU&pyeEhTXxWhsC-eb6D51>Wf(6Uw zLjzJXF50X=4J$cVtWHeC19zji-Hp`YfP#D}v1`KrVWrsz05`~_Z3&Z?0}ot}^<`d< zl`)M99)CkeNc@6x&u5m*0zG9or$#C7A?=^5hXo%#m}b(sTe&d=yehC*b8g*`Ogw2C z8J6vf*fMy^>m%uQ%2@xJ7;(|CuC(lrhizjoG)B~~z+wP}dE>6m-YtLfw4eVxq!P~v zmBRS@H%z?!vDFFWSS=ITwsyKW8Ln&zCgtNyAM>%kY!?n&sxsi#ArXEp>q$2uV7t$` zTNXDh9X6WMb!@Zz;(TU7*?4eL{`>rG-zv zG-1M0a3d^<6;T93eg+FQ*Th8G((it|N)ZT3ICvXQ*lw7BR#W{GkBJv)P5>>XAIwVN z$x}E}WOU8nq(_SfX962CX)AW9%7p)R%cnCYE>k8spKCxzWiXM7%pos8JOF=2CSY}l34LW1H-s1*R&W)d9R(OwNbZQMHTCUJcPhYq`5)%R+Yv^j-aaZzVAnmdeQXA*w}yGg#+C-ltqRK%9vd5TD9*Jr=Q6R6{Q9}8)Jjebk_X= z-mvzRW^$I(L8B{USXO0cPGLKyw;u~HBkGBF9T&qhAJp{Pke+!+Y_ zCxU_eI@f8yCr1{&92sVfT-IqFq?5Ci9i~T?V%w_1@|#SnkrZbUD4mhyu5tR(@WUPm zQTA3|mr5FsK?yP_jnO27N(1vwHSxmpn;jf?VA-FVTW56qtXU!n@@v+rVeB`bsVSnC z3XfpD7$XVSjt!+RFx=Iv7D*p_J-j8(R)QT#(N{Uc&y}+%6sc&Cjoxep#n}8;VNj-- zrY}tMyC9G@*3Z!OPL=(8Mh<3HN^0_$!7byJ+IQroYAm(CI1BqRn;f>3I5*=ZJo0p9 z$XU8W4sEF$;-LD~)|QvLtm?Zs2x58{o+_hBkq$vKccXbOK(8CXe;mw!ZA&KUvL(vv z*Q1Tj{eHcE5NFHf!Z=IgflgQaM(7<;)Yav>{TA`G`4B zPnD0ROew=Ye_N%yYEzo-Dc=KpL~3JP7y`($)h2`FVieYIv)V(h5`N@{YF3cooe$_A zwXo{JflH7#84xo;HJ;1_z`M-(hrN% zPPwFf1<)wQ4hOIp6KAWGF5i+L1p22*8|z`( zOwVZ}g;y%Xc1<<1So1*!mGUuYjMhIWbs zHV3dCd9i-DF;kNK(Q^1%e_43O567u0{H?_LUXW(DDYbq741~tOS&B#4Ce_F1Bo+Cs zaBzUcK&2w~BH|bfb<;ZKL+?XGJK#Oe?_{~on!!h1RyK;x-#Jdji-Hf*o$ypou92cm zs~jifBvNAuJ=KMX^hb?BygrfWKv@{xc|<^hs5?m6L%LKbaoQi_5Vb-;V+n(7lK+Xi&CE|$X0m^@~ z(V3)?==L7IP>8H28P35+NTE3@p=P5oH*zN^Fgko7jt+j}dz0+R{21~;@m8ffXm9VW z9BL;whGpHK*Wnt+W3M=`hh_~Wrs%|J>8JX7eX17^Ik-G9E6jPI&~k|($sPKBAXA4W zk6h%iP=Oc1KaQ#!L@^2plJB7Nn`xr(lLUi56AlFhyu{zO-p8zO=!e1$)N~U@Na*zu zbWOT@|HR)s08O~n6g7gfcCVWoDK3e$@wsxO+#F8v$~SVv61&BgvBo6!O&FlNu>P2+ zqmR}i1_{VdQwam^81$zcXSQR;K-|F*RdBn0?9jxlLq)?JL)|d2)cB;olYIi~je@4k z5Tku6#|7huhTT{9PxEmA19XUomD%lMF{a>oAqOZhtjE3pkA2dbrYQN z48mQMT)cms=4c@Tq}9xafZ)U>e|_Q&7FWuiy7R>0Mv4leUUALIUKft`+*m`_t)5N) z0FOX$za9g=BcQ18jeHAa47Cy_ z#<8Dj-$7g+8$$kaBX8>ATh#M0yJ^snwH5I>Ti|wzQ*52pq4))pS~?rVB6z3HuvEUa zhCBMM?Vf!8nId^!uH2eq9q%UrRCy-4TsCe6|4O0i>cd`2B0@7z&G>52>v?=A=){K^ zN9wLpffT^v$c=3Og2BwN;Yf}3Jmz$8&qsygv)e@%i!Y*3m~YY8wLbeeS!vFNfnTuH zi9$z{rTPmu@(xJ>+wr068#+?BvsI8&Mk`GrYyjrQ|Ld#dxCQ5Gyq30Z{0Mek)zywA zB3O``apJT*`Jv8tLLiFm_03s4f|@BYvVxDftSU8_z_x@-F&q$MqUilGfd`|sBW>6s zXz0Am&n;!btyzA8ye{o(qFtUphgKqEIezN(R?8e}I~vPk`R`=D1nBieWu30}9O^WI zetQaS$k1C+wTG!_d}M077DhacDdG#z*EZa0mnm~k%Z{5iZ{4Bn?uG=(;?GBgS#i=n=X zjxBcO73y`cqk|nPQ$Kkf{8Hbtag{@;)4OGrQ)Odx%{T#_Hi3GE_L04;BHi&>^=YR!ZM#Q8@W~8O{ytVXZ*_mz0=GgmQ1wR_uB#>^NE zelKqvSW5zNO;r2Wc3t-HEzHAiiR)}f9kZMI?A_vV{StjTK4CK=vu9F$x_K8Xin|RO zcW$e(Ju?QD{OtW?Oe86nh_OK zQYo@o2g61QGIJIEg=KDD8Th7t#fCusBNSv`Gj6BGi}at{q$C#W_!33#uu-?4$iS9A zwd9~c($5{iPN@yB{sznGyWaI0N+~}yuY+`h-q64!Mgs&8pRL z&8PO~xrMmLG?HW6nM#nQe@vXSWp0K{D@zf&x&)s~o2*r>e0g zU})~qr$}!>e&#tlFI=^J}u8$--+qhKt|C0RSJ3V4{2^J+58zkw*eF>kGZX_ad9>V{Gh!oC_T{kk3HDjn6516WG z^bOV`7Hiw~jbQ%{cdcv=8*1eL) zmJSkH(F{cjyn}sz$3z&Cctmo6=7y|vaB}2l7?YSYe|ufLBYS3UGPBVe7f-jRrQYfC zFiHA;2U{@B@*H8qfJP$q?b1JM15CD z`zh&`@6bAk;Nxl0IEu zoPPZ0;l7!y_U*%hIF(cnFbMegZWGpLaKWRZ(W-+D+j7gKZ7`{IIItNxz_;Wv2#I!o z$t3dj^>Ox#dCo4lEXQV-=&Hk+vkPNtWqodNsU2;VJ}|xj?h-kjJb65R=6m;fy^<0p z<(m$lV1P1}ASe`q1z7l_MId1W3MiJ#00Y1P<+dZZ)drBE=jwAX-d@_Deqa#_g52=L zKIVT>@X%9J-qii%7&$7utOjg7jIbN9pyj=EInQt6jWM9&WP9c;p4m1p3!a!kGteIL zH&`EYPS#sns5iCpVxxD}2CEq|Z2|p#sD=~q5AKsn5oWpTzVtG%WB$frLY28uSl z(6D$so{DqR7mEwe#6<$+1c4`YiY;o4_I&mPp~H))?|bN_Bket9Ds@f#=NaeVEuMTk zy2|sv!{R??l+2|_XzqG;dKx9(04;D?8&Xxbjf2g+Qw}EiMvl9vzejQB#>3t|f%^Gq zZ*3ItxuE)BG%vT1Ymftv9N5*{eA@&fcL!==yH+X;qya$KS|WBPw|kZ4+3@;;9>JCVU2uBaXPI@>jaXDb7{;pyAsgBs;tV06pBhnn*v7KQ`5UG->V#AE@2>O?grpA7Ga|9fWr@0 zyPYK2uzL#CMx-|-FI2Rogj@?Fh*pcyK1q)_a^g~q2gv0q7DP~+sMmOzb2E=HH~=Me z8`ZdwJ+mWY-g@gSNmdXbD6wK$O8b{JU+Eyik&RogRWY-fokGpm*NQmmP9R=hXaXc~ zeAS-BPsR|rjED%#ZUXQ3TN<0~TRKJjY0-=GHQMg#38gs_1s{npf#lBhl4-9~i>TD` zLQWWdDAjV!+Tb_eyVab<|A7F~+AHDjXn;}2Kf}?Sonx78?_!2;G;3CDYbUGeb>Ll2 zZ}0vVQ9^^Xc zt6{1Hbvb0?1=}nrnE~jA@gh9rYkLXwQH|8nni<56VXuenBdB=_t`(PgKHYoTn;;z(ticPFMSnU|usZ&dcYEe- zq3m-<*<(5}vuO{gR^`yX)1|Bd(`1IFfquX+^q~4wP~S6+uJ()sPkgiL(9h?q3J-0H z_y>}UlB2{E!T(H(Q&VV}7h|D!^7Xf@#lk#71-P&H3fObWI{FU(H~fM9TXy(&LWZq@ zG@Dcq*;)h&X7B2I@uX*UCG{vuh-goe08nzg8`}=^uUWx1MF$+M=!_FO^Sp;%LZ-Jv zYC;xP5M#aJP9fN%!BA>!91-T=Lw0*`I`iHx+VY@}z9sj^im+aZRcO!>A(czMVKb!z z?w?0rqWYZQOdIT?Y{mZLxzo(poX3y-IWaEA#O*OEowoxp>o>@s8-|cMJ$3JWcahmt z(!7gb2DtvYVCG$678fqYv<{*EueD4?a+fD?_8vdCBf&g02A5(s3x}{Knz5*i5WT#irD!+ink_GbS`f2gWNN)3-U(IU!aGm4XWqFUF7H zVZSFMDnY@!g$8J~oA{uR3ryIGdqJm==JwqPrw^PINL>@bIwWOuz{EDy z*5U38BBgfSl}nVX(a7o9r5*WicE)KjcjNaTxEP>|4b5fY>GmfL9?y50`B@qrR<**% zoW5uWt}fL@`M(CAI2DR3f5LI)mR~P2u@cR6_@8>OsVz(xiynsXLf4)#*4K3QJj)qx zMr|^G-{?j+K_B}M`J8)>{}EY85Rb_Ra~F+u7H!7o7)!*KOEBTGQQ^Oi1EEs0P!Zu?l#a=w;4Vn3ia` z@MYfi(Z&r#!!HeTd}a4*|Dhf%GSE|5$Q-O^*(ptFN$fhna*ulO`|H5KzBKdhUCngP zIhTinZTt=}zZN}*g7Gn;+dN1PPYVnchK5b9N-EkEPa}-;0zlQ94ij~t24i`rAK0Lo z@_~-M6fG$`XCQJ2KBt*$#E|{qgs61*|v~ z_9IffHWU*mm{xUj9*2*eOg|2UPjbUJV*&t#)PKOwNnK zh0$C^08G@M2fcZF>80OU;>H_LHB|^7Dw>9$nn<;1v?S(cH}UP}tQVjpLN#3P;lW*6 zuB_I#3jZWgQ*0-y*;lJg z;*aI`AbivUUM;fZtHx=~A6^@+5s8H%#KMGf<;KGF_7p*Pq;^ZJm++uaCHIeYz*MZO z$4FQN0gy>z|NMOmN(TX@a*Z*8;8zm=^h<@xsXbm90123*n%!`WxHo?!YrhxB8rhGM z&qkcTe;4YWMQ;=T@IAH+&Eu$y?}`oPJVVzvXJ=JRMUU`Cqiy5((?m2PXRf#TNmFEo z;&i1_hQ?ciNdCiOPiHd>y+Po_3y*|TyYFK^9fcK*kL;9~pz<4`?DWkIxH5vryLHxJ z`Z@EZa2Ju0Xxn`cp^;z`qu3CHWSE*%;QuRBp@KJOcwn{)rCsm`yZG*?vY&cMbUDkaD6)5H;d zN%K_X`G#kBSO}1Odr$f-Lp;Whl)kuNo<`x|^M;>2x4z2ITWbjQH#5Nc5oNw3Xi8`J zM#gTE<+KS3-W~3Oi%gBnvb2uDp1w0ZugaFCp1)CNtondzc!ucqDOB_QIH3?{U4$4_0si3z=7i!B zwAkQKZA}?brF{#7{q7!`1B%oE4rHE{PkWaU4P+x)3}lvE9v~T5ym85~?R`cQqY; zb!S+iFTvx>wLNrhj5naaGW{6@8uO+KYCp839MqJK1DoWS>2h<+*Rmifbk9axF_3y_ z*tgqy@u#9eoW}~(tsavo`}=XeFIW+(Y3ScKf}FZFOxn=5O}ZwAkGoz4MHlMU(<7J3 zNW)fQ+Siy`2TmuF&!M`W@5N90*wdHjJkZ@iA=-32|Yf?5jZnm;5$D_r$q z&VGE&CJ2OD#<^gz|*-?y|KeTpqO&V=MpbBnh{?%l2BIOpb@kgMW8A2#fa9 zF0Q@h^s~nEr3pK3y#_zDf=RCuZlHPhWg#xWu#NwqOBL_yzj4S8>4~Exy3KVK@FFMA z<3XBbz6>X#g(I8zD6;$wauD!wUQ!PY(Z=--8vAXkB9x1Qa~fe|zCfgGD% z12w(Nv%Z38zT2PdeXhU1TfKmOPpWdMkXiErhlTG6!3&ZSRvn>ja^JwjJ}e6a$yLxX z;W6n&-GpEM@-qsx4g7B>^7#!95ei5C+K=6Ziv;qL8@}jN~N9qYCa!Rm9i{+Y-sG_MA!SVOfX)4QW>Ku<7%eURGJ-aVg~2-xI1m*6#GyKii) z!VqF@ElppShf^=X*0srsmO3g%-EEPsn+=>K^;&2EEoBRqN{xEL(1U0Qej3hEi~N{Of9j@k#D z0zR2J1MZSwgi^En8WU0y3P3W;3@8$a1z4D)Mu1@i3MiJ#f(yU^@V8<7FG>m#-ky&E zKASqvGQ4uEwb46-yrqAjf{VFLy`TD{yOf7^8`ygW+zZXW@$Knvx;WCAfoZ+Q^3wW? zk!VUpz@b_9EDPu0W7uUSc1M@ zG_$XW4`l-G>7QMnqB72h78s*^|6vRmN=psU#GYMyD$AUfysnG<{_Uj_DZuc$?n9|i za}g>AyRIPxlov>P5F8l)-VVA9Quzq+(b-GQcknJZ;+1a#z`q`=FW=y^fay%GEB4jf zG}?f8-sfEc!}>B5p(;ST@ESs)!5JSIw-e80V_c7ayYBv#z{KwR`v~Q%P3p9mOl{GI z@|_x&CI4E>yk2LCPECN0K3i9XvtZR~ssB=%{@>!R+R+6`-(2}Cii86EBn`qkT%X*s z;GtG%x>WZ%zwG8Ipw+6u0V;EM>>59fzfSEbp10oL|JofKS8t%*PsfRSJwgsw&02iJ zc@yfc;QFIvnTbCIz{w8~PGN9fE_gv#3t3{Q^QgZ79ce=+#7b>5)lPqVwJ=4S@&dYB z)sJy}vO5l&!Tk#kB3C+KskrUK@?5FKJRgSMm!%{of-ddxMUZ8s4km($bB$!vz&ZYc zabNwp+|p_T3tWQQGD#Z+Dm-4Vxpz+HUopi0_dfK>^SU$H^45W4Z&+A*zkX!AebCfI z&%V_w?3@Q%OI(a)ifOA_sIdW8J5UnY=g-`j!G`6XDZGND17jl;OK1ikO0etgR2~cf zQl&XG+V_>>%RLA>2D%=G88~%#Ta-Vv?aN%T(JK_Zx;v@p$vGjD)f8_e5O)O{%|bo8 zNYdxs&mgio-Ezta`vZ*PI5|$O7{aX6N1cxJ6r_EjEc_BW9nVJs@d#4OBc{L-&-0E# zyOqrM&fYvLavQ)!nO`IyKLX(sK3KEUb-j^A6|hi34pz+OqBmX>SqFu6Qk#u#@I_wC z2!%dYpam+#U-oy5@NWx55_4E+Go8NHq~$p{!{yKgF0oOJ6_Cpv)mj?2m^>7^fSd05 z1+)%)YmApq$E}{21Dp(JAlP7|BHhcTIc3Gz974@#GMq@eSBTZ^A{hn_OGP^%JRVuf zM&jE=U>zjS>KI%PoZ5mQml+Vzcd*)XffwM=j6LxX-RcP*M$R(UD_h){oro%|lI zP8_?z$UMQeQosC@Y#&oL_$qjal|?k zQR*HB0q^3M_`=Sm4m!am`<2KTYDM2vvLUR?vxl3j8J9%7v%)07kuPloAGiX^$`F(& z`@tzdpQX!sgni@5j2#y~We$@3g$%k-a)r?Sns6^( z=^s^H_4W4E#D2@$Gxog9Wgd84Vb+AQ_d?yl^AV@KwQgnnQGf9@>#VcmHjekU7vv03 zk4#j0(R+9}0iH*Q&IF(eO)V94$Sd>A1K@GEuhF5v29FMWgd%TGjjN*nw^wWzL|?pe z^AE&y@$7T%$WZYUMe@0+%yH*fH_!R$%H%kj2&8664`C+Aj2!kr@*ESU^L%KdlpTln zU^iHNT0U~=q_$73CWdla-&91fGN1-9B0>tWp+=7Y5I~}dEVFE-iCBv10@2)+ za2v%EQ1c6m!AoIeqa6{L?1&>T6YLs64~RZzX11{!e`DY6Ep;W_HDss_)$x%DzyP}F z=F!7`IP7`Q-SCFcMge_ifSjLW11gP$k`KS5Z+ozseE5wb_B_p4P9yq%7sX|(G2{}a zNLvBa_~j?K)$^Kfe3LCuNK5<$xS194vpcmoZr8_LZ+Z^Eq^(tLPpgZa9m%=~iWS}j~ zWl-(|Fux)#!*~1s%$GZ40|mWE;1$nZ)KM`2!H4fvqE&dW2Z^^OMIm8ruV`rW&qirS zO&M~7y|;+wv}P@-=oBtOrlztt3ieTnu1y%*|6$|lY)pYrr0VwL>D{5|iYJn}pHi^fSgpy8e$ifTh z8=FLr2Cb8fTzD*&#ty$Lkb~63T%`-oRb({$y1}MEull>OG_bp&`mel70ql*yI zzs)Q+BMf7IhjYmziToSRp#`XtxZ024RU7b^B#^-skecJ^bx}S>US=RK5I+;`9Iu#1 z;%8fPAuyLK1D66wkEisX`ozaP_R)!IZNLO3m=bX-=J^mdgh{wJDz4^QN!L1Sqc| z2aVm0zCFPEl-KTG0OsZzC5IHE^kXg#6>#yYR$B-c1Bh$5E0ESFC=Ju-xqVI7AJc&7 zILMY277{Y?BzxZw=Pu-DHeN(s5_MB#PyU}6QbW#RG2;&6^AgP-sq^BdCYCZRA337b z_$2&A^^_p*&XfflZPj&YtVDrIy)-+igZpP~LX>iwwlrAJE6%yzbzWT3pL5tF-=lHEzc46%dTi&ESvA$l>tc1m9Q!>+~9FB78aO|T-0AdIdD3;3t3%~&N0qbstCRi@{k-vS+36(5urDm6x!HRosIk6m}cTUT3 zJsV5fZ)$~uCUOIGFX&uv4;X?=SrruS$O_VYw%=KOJQ`!9f%g1Q3**hq`s?!N*Ug?a z6EZ-cAV@a)=96&)(5=Eg0YeXscGuF8!iI!th*wOL*^#YXxEMOlRd3Dn6Z?vhlw#V2 zTqdxEH*qO{o}{KNAyz=k`yex(G!a%&C#L-K0KOuhfV&z{RTCk$N7lfS>0Ci-U63g% zV|aM$GsJ9DCXz$wzCJLKS$R4u!$v^@mF;DP=ORW&-iA#GYVf-8e zqy6w*F(qV`+aIWkkBMgWsesMUX02%K)@z;XSh{~*8IYqj=$-3hZ2##@EPNwbbn0uP zkjhu!ark3c*b>8Nu*cj5NG>6Q=bt=BH~iM*6}2`F3+iUzq~?v(LQVo>8%GLRil;eblS%wah8V==I56YHYu|1mZgsa<9bg@EM}+I+7_moYxz5NjE=1J9e=$9Et5qA&nDeGxuPFe6Z*3fZS7m78C zq!S$xBg*1Rz3Hy3-if#<7VZ-&IxLdpE0$*jN}M>d_*=0?frO>Jr-ldig#zRu@dD%& zaxG7d4V-6wnl~dY-cfzoE3tuA_0ytp|EWwWxs3XA6`2|wa_2Dl$k*iFl}T7GJbNe9 z#nav>*Be4u?sqR&#yniDHT=M7b{}Zl9ajFDvo@VsgRHeCx5Z-mVI;f3D!$eKX2W3|l||%LUAtX(yPz_J0O$S&m}x5?px4 zm7e6?e6!;p6%K_p(N)iVfjkB{b@9Cgd!1>CAP4M~l^O7{AP=+;x?RW3q zxD|gg(Za^=^>UI*G0H}L9v%}XiisEOcb57i2O?Pa&bo1{8QR3-$Wlqd$(5^03x5}y zS{%>7JBMDfhSsA0gqX5R-T8nFE~6HQ6&UCyU3TZ>l+5yddEUa&1MQpNKO&eq>z79; zQq(dnDe2s{o^MQO|2Jf*%cI)%x~lHCZ9J`~RN!+wY`Pt(#9tZF|OQ3X{=v-{Ft+ae<+Xk%N`O5|1{I z*-}Eh&Q~9(Av@1|X0xMypho;R=kSO=(lUtz4|@tlxDb6wQXDh3LzQ-Yqu50=eBtx! z+W&VL;~QxzoRAK}S)=KC>i@9Of=bTST#ukjJplqFQq4^?28Vw7nOO*FETMD)p$$Xu zT@dKJCLM(8Wgg7ZK2^N9Hc8GT44(%ce1KvfC4s4bEqKJGD}RMNZHh(N8X#g_>%Jcj zRt=o9fNVvItB|7*_`PLY8zzCHwii#FGXW)#M$rzridI}bui=d29fwl~i{gky@5MSt z6UF1JXn?7877n+^S3S}`g_!FH?yr*G3Ce_`y|HD_e-jo55o>;*h8XS{euuhNp8nXM zzX=X*wW}}y`BU?C3Und~!Qe@(JevGAlKCQwq%P-bVYg>PYX8g~yLXh%6B~xK88vEP$N31s zmUor`;F^j+6gXK4`inwSugC_ma`Yh8Wf%Jj;Yp-yl2#CdhyQ)?xp;C|F3T$y2l0ph z{1emu->;?GsWO^vj+2SynUNT)v61QoTxJF+*Aav3FklNSaTzV2@E@@!gWS9wu+BfvljfOqWsH$j*%qB!p#-U)+ z{Ikz91%wPW9t_n1Se6|{I^A>`um+N((H`bEzra+`eX!T)sv8u^`T1( zYL67CGG}ukw7Htowx#$k*5AaRa>P(5%GxysIr(6(`fX$yXWo+k{=DfJB;k`+KHUc= zJb=fOTAS`-o*y|Kbp7thH!|G>Fe*Y4;JHSR03d*(i7c~#0P#9upwMgBn+0q32MFir)jnxejTS3XDuRX-+WNjIfuP^mwv+o39k@D9A3a9!e- zFYcXb(tPN{g^A;8!&sW$eD$WdFR%qkWgmZ<@U2RQGEbJ}f$+6Pws`9@g`W|e80)5K z0Vr@>N>|4oIjcTjP4o2fZk2h7tk93O_N;}wss$ZUrqFoePW#K4lR zW?8w1!Nl>W->$NMV;f2w{5+a^ljm05cHb;X$Az1b>^1$$)(zk)oAp}ExhjBCS9WCf zhv3pVD^3Au8wCGCS7gB~eik!k8E=$!VZ!36?7o<@gy3P}84~QXBm6{}yt3&j;47%9 z$_zz$#l$GJ)g-XlC1`A(G!5VUOx=p+=(*F2bc+lyZy53+n=z$A zE3WA-K^EL5;C%tR#FH=<=|+zLAb_HYEDL}D_bwJ9qCAwxb1dOPf|aH_eLjVwOsnE{rjDF- zTdtplPaPl0p;zm0x|{Z$2ifyw-oBp{(}RU~?b37`6TyCzY3}X*O z+;iId@GO`2E(tq;^q7hTF!F!8Z> zxoxL5h$QjrJWH5eCwG{E1M1;ymy-Ces+tQ5{ai?t9Ud~+#^mlWPFa4yA-o#bbl^jV ziL3?S*b>>a?6Hs_X3@fVYHK&ce1|VDd!!uXsN_Hz0r)cUASfI}50G%9MX+H65@?po z0t3JR=pT7B(J=@DNlH;iqGgbwQO)}gafXa45tXNDjVC3#G|QK0@SNyUFcyt|bsApF zvmEi1>#4s4sHc47otjZ`+s1=1fqQrz3~KLqYo<7*DPID0^SvLT@;@@@Lq-c_{r!}? zv1MAEd{?ZtGhd}_6no6?{{v^6X-&#C*-+TIiq?OT|IqdW?NNNm30CwhbLp6BvCpb< zleSfnY|->4JOPWKSLIe^uJ940xA3GHy|Lz?=gE0S+S~A3su4aeFQK9*mIGDv_hrk< z$7TaVj&a4h8xC1P#lE~RNex$3H4G00T)Pxxb2f;a9ruO{1-REYX_FG%S*(}Ck*SwX zJxJwrYvk0pIfgje;8*DN60I^mrX^R2;f!hyN(yfvOCX&lIGswB(qV$8 zr&`qOI55M2{N>CEJxHiE#{=v`>lam8xM2#&0QpUNGP_#xmWOx8G{#-oe!SvSqZh9O zx}uIKSv2Z=ocFHa(>g#HP=kP=%>W@o-L^Qdsw4{b*>4k~vW4jT=aaV1{JkD8!Db|{ zHKFZgqbrOW#6d|Xvy~%*o5)3lp;&3{k0XQsaCjfO7H`%drd8F=Z0qz;us@9dV6#;# zs#LItKmms~o$c!NM?UvnYlKo<+P=PjO$T$=|LtQ<0VEP{r3&GznG=jVcW3c{n}g9RrnLa9->d4GgflC9TVZ^X)tHQ zf%y=fNH+bkAa%H%IVf08QZiYRJsZ7EKW3DCabRbBZ8Q8Ju`3J`9@{E5jR>e1Ckkxt z1S6~C+<9v74+dev&HrWZ5t#YY2vWw_-I_?hWl79(BqvjyiO@rH( zz;Es<@iO|oAq$&%4!^5&W1g>Xkatc{&B!BXE6@!?>%)Km?_57~S#&k3p1$mR2NyBN zpuhL0tWrhK05fn^VHGo{LlA-1$8=ciFL{)3s_4NX8f6-r`-n{wJek$n`C8F;#Jq?Fa-0kvt5I;c6r)W4WqDdJ;W#20?l|frXke)9=jo+ili7%LTHAG%Ad6 zep(_J?TAH4)j~25i#>ZOi1kDIQ{++i;pKe-i?xWZGuI;0zoF`{SMQ{bEb}5VV9$>Q zV&J3CnM{g#nDj+t+B57h;ajh8YBZc(V-36d4YP;x22CZVyRfv{1%8-=5twzR&4Q1~sV3_uI`a z*&8`IOAQ07BQ#0935+u1$M?61;xLF7m{lC4Hp0h6)#zRRFG2j+wv}`i_q4r*(>{k0 ziFlS89mUF(+9)OvJ6%U(5j~4T#W#|Exw?^Ld7{nW@Z7J~BEty`ZuEmW1 zEkw41>Av77KzS7N6>;2sn2>lkV&zlFaS-CDdmnA~cR^MSB)wXuv1*thl{9ywwVtEM zXv$J#Sy8W0I$Knd4?%tBtJLVWOe5G= zB&&YzEsH3L1|O1DMwh0YbjjcMRFu2O(;QFX6L@k*BN?f!;8a7p?6u*rci}zYamJrq zRf<2Scu7Us-TxXOI+u*hwEQ0Ya$$T$n~>X$wX$Z>`?ox31{44*v>&P`7V&gO3LEBH z%eO;vTy+n-k(a1Q?3y+8EH`7MW~rJUT{ArEsu6ewMJtE?9xXO@{7Thd&*k@2KB9`i z0^~bE^?w6osvwaWM8tbpJ6c8#4FW1|@d8WJVK~NpWMCH+Ho*70A7*F>EgppIq_n`5 zKh{3S7i4{qHD)iTzDoQzWizd-J8c->6nD)EqH2AcyGIV4FMYkdZS8J zaXR9+-mtM@?9Zkv^gUx;e@@S{y!#y(3cU`Y%WLP1Guc)xV``s{Bq8~zmE4vDkZ(CRLIR3kIiGq06q!*z8(42}Zu|0lCm_?$o~FBwuZ_n$CER3MO# zwd34(GrOd!l_EhF{%MhCtZKnHzPSN}Fx{fMZ!E%vv88n%|=O6Ph%RkgWz?zY+CaATWGkczUE)3#xmhL*zXrd25R2TvzJ`x>LPW^=&7armGNR2~fsAYDhL;;VHs%>ZN1Q31Xsq^3|A=z*jSlN7T z!?{aiMA25wSL83+Vyr}E)?21K-P7(Wq5O8kwHcAAJ z-?1@=<~D4G(oesTmJvhn94XlJ=y1BV2Rw5O?QmdhUO>oJq6kO&j_sc3753YH_~afN zsn6>(`uC`ygb6r_?JTFrLeS&*okw1Eofom?crIx?)w z4nY1eG5dyNey?LX*qFxns%80{o6zE?^XGtj);~6ZMCa7OFjZRL`o`R*nMmsjAtIM= zKZq$iQxiD1{&$mWG+n@Ux1t&{ms-$hfme4!lSSNB6)vZE%=gk%GL+cfni}Y(L-pB- zAf7CVEn4L*ZbJK%Z1`h@7gj@YBi(`q*jVT9lvmV-WXWz&eEEL&okir726qoOjmJNM9W(R*_pNR!{qyc}&@lblUr|A)M2oqNA(4M6w~U{B3NVVfLJ-Jp{f!|Y z8fY11Pga~xranByEF-WytGAvcmho1yn?AZ+ltZYc=jHHin8$jpVN{A+srK5QKqxEN?^NRAX>@ndrp&QYcG z?Zwj z8=&B0o6q$|DQLCe7R$fg1YdOlQljWR{qxtzVR{J!#s&@an9v zv?yVngUSkyFR6%$qKTRK=A2JRPBp99qR(8_3L4V%JCo>5UiPx4k>6y(MPmxA)i4EM zh54Oy`@vXY6xOjeXd52@&wp~i>IT4m_|v&L_DBuHHl(}~2#m9*SJ$|LTLUo%%I+&! zyl!Fwsh;05(l9QJ9PybCV09 z$?NmplFgdeI~c)6q7zg0Yaj<>*h!?vOF@N>X75Y7SSo08l%syjtaf#)?neSK$}a0F z8DcI7n?RIT)G~)2iDusB?z!I1H+w>LPP#j%Vikw;1(il42p285e?cslPW+b=u5j3& z8}90jX#XCmZ7;@kj3OT&P=^tM`KtV-tCENu#>oC^=l4Jf=1xqTB4@LAmZLx#1%X;N zg#R0Ni~QK;GC_5@6W>;JFi$Yiz%UF)oY(z*Mu1`n3MiJ# z01LnX@(s?Luw2D!3Cx_eMb&>r8NpIySF9E0Vh&jhysZ#PEF1E-@6>+q^1gvi;vSJu zkfi{+TkAw)?Em4J3TLM9bY90-ui93pn7iA||81H${V0+`KQydKMkASf(Ar)XO=ifK zzJX<$OIB?Y)ypm@DDU_Rk|+0~&l#HiK9vHmHT~<8#&3#th04a7V`sLh_hB@^)q&&6 z=IPgnC#_k7`FCRJq9Ky`WimY^Qn}s9KSfZas?N`QR9;Fo&NB9lgm^%(1I`DWCC%>= zNq96{KjgcW-h7Q!F>U6z@DoYf58j9rE8jgzjybK7q`;(uCa@!svbIo2agH{F09rt$ zzh(qs%*^no(hn)fWfOQ}4JU4CqIY#`m7$A@;#0c&Vw!>p`lsUE&O) zZ33lapQ)h0#@GI|2zk1qucq=hl-F-?C}85+o3^JMTeT&Nx9OT{KH4NcWxT`&I{kkS z!o$iZb7s2c*!$_OZtlJf3RoZ}6Fl~)3AVfd|)6f zdXZp{s=C!{^60Kr?E@NOMJ&rPzsv1uavPcIMl<*M6=`BD=CiXwuOu(FyOgcn`tu;o z9THv@t(ll4Y;$D7498;N1kWePf8Hn`3mC=}toSfV9^EiIRB_kP32C@suD!Z6ze>53 zcE>FWIevQ&GWVd|O`r^KpKMDvf{@Z9b9cAUt4d*FT13_7tqapZ-^)AsurGYzX)tV> zdc$WIr0qP&dpQWWQ4l(7eZO0v?%8HOU1OlW!1!fBjG)X^^u~&S0A$mY<(uVVNc&a* z9@WCe%z9C&;^8G>DHZXeuPU`1&mW}%J3j3kb4rOYHwqzUmYSBLP5Qe?(`Vrg;t4Yp zB#7=)1Sr-)I$3S(NI;3$gjlyr1Mm9qi5>ErguN_W2oGIc=(D55Uog6S-1r%m6gDEA z4%yEI%?JG21ggL3lixj72|^gqri<7fW6)xMO>V(a$M`o%MWZ_7kWk>?nbv~mdnYRn ztGgV~I=NZECrufZzkz)Ka)Okv#b@xq-e;oEvdAPm1zsDu68d+-=T|VesCjT9`8FB3 zj`UZI60}8_;|yg?vs1-qMuJvq9hl%z(cRV(q1GakCMQ~LWg^6$5AOP~jx<`N2Wd;O zr%J2$&0}25?i>CY4$HMG$5mBlo#f4S?YY0~V0mDE_&>;N4SOA5_d z%9B-{5C8CqYN(oH0U>A#N8<$R9k8n|9*)AE<`H%Rz&km?8v42m#jOvRQ`dldoE> z$xrfj_oj)0eRp@@QMrZqtunqQSZ6@wC_bsjqN%FJ7XZn}gKD?!Z)+}nqVjNWY4c5a zfO@PGj!h8Cn$>mztyopT1wDcaE*OYkPvjGz$S+7~AD_&9p#^s3pXu^#Zx9(^3Q!Q6B!#ZH{dvD+sM0OcS49E|BF4UB<65XQNISSr!+gD=sZC-ZM z^%B*oYj{fhcSpVhp^i&If&DZS8LW4w(`ZPc&iUFe9tr%Db$YsUSmyXzAk$Z>;CoSX ze4>28l89f$V-AkfSL%87StNv5$r(%qw|?~w0-~&sN@zZ~!=+)vo_~lLH~X=y^tLC+ zSrl1>*NrlFq{#ceq+*IeOIfNhxB=IhrN(HmMQ)&QkN?!>)_pl_QHrXO*t0N7;V^HU#gY4O-Y-YQWbVkG9 z2UmN{yp*VTtv8`^d?O9p3?dyG%m>IaRqy!%(&hp~;(^!gs%w)Z?~kIoS6~gM$4W1K$KtU{ z!<0cW#cMuM|JWVy6YNSdcZtU}wU!>LMbPNq>D43F+JI&F1Y&oM z4;UpoQi{J<2(>iVEeupOdNx=qlhRsU&7h?!$vu(Q8lBt(q$1fSVe+42R| zVERBB0mw3s2QW545%FvL5?F=^n31Gq$Q#%Xu29- zLT{W4s4C}mBNr{H`CicQHUlzkuel3k!eYv;iQ@oV=+o^{+_UeWB8AWT0*;sF1R^WT zx-00>O7XPsUkm`2Pn;WNlS=TURG*%=LU03cnsoYk8% zZ})B@mDrAt%234L>ZBe0(IwAC_zNe^^oW6n7>q0P$&Wn(PN!{alPr~>e^Nzp<>@6X zw6I5u4V~R>duNFCuCFJH21DVE9J&R_%R@}=7QIx{_*(3>!EEhMJS+}kr_c6HjuLvl z4olA}MTWt0;z=E)K6u!-6y)+65yub2SMT34qN^XkyLHMKw`U_KZ;n7|*)p{EV5xSF zzSys8>o@LLCIN@r0+nrn)P3mFn9Yc)JxIaoKtr#CPcxIC)2Ao=&Ak>JV3XX8!83%n zuIOt}WjJ5cIZ^aZyC%GW8kpVL6QQR2Z&NA6ZOgjS$kX(CA3Nc=zkGA*6@^dfh!%U2 zAKt{-B>E(gJ`%Bz> z*c*k7DlcY5|Ci3TD`66yZ{2JOR_I`A#{*8I^UtUH_m+UliOMNkq(WZ zf96FnWe1Y7uflpIb=^Up`+Fx9f#8vDx$NoJOx+Gzwpw2=Bz=5qJKRoNGVY3t&`#b` z`q zh9XFs%K``huDieh=)qpUupioNn`bs-8lU_%NOWU1pKhSng3Ju5c|Cn9eaj+HXkH@0 zgSU*n%%|?D%22vI{GuyQ6VBroIFBA;=~WNY)C$3~Y+PR?Xf127`~2Mr&6Y?}lR`p} z&k#)_T6*T|x3<}p>7imktnv4)E+K#eHe#yQ^k>8(3hl7$X}kL(d6xL`V%{75)gRcDe2IZYaZ?phRwbtxTmC?8ZATCe6-;jx7d@`eUG5 zaJ&Cf?wcU5`)U(p)TysgRl4XL*ugkDp)_rzBt-n}^bMO7>XB*aZ-7}bH zUn0yL_K4IY$8H#$f#FULoezC6gV*I#>Cqfn zB5CAN3^NakesdA{f>&KCBiRY2?LW)7kS`y;(8ZH&*kZ~Rpgypu{JsKf-!f2BeJGKx zC(VPp*k~mD5ZG5akXz?Fvb#g$uhsh(O9kp4=pIjAMZyLHt8JkyYV|=vA>b0%M;16R z1ilJ?=d8+7Z!<~*{*A9KpF}jST5=6i5*6W&qrxD~bp1O@eC5Eh-Y(DO5jN8uDwb3j zC&t3sdKHn86#uZs1_OxW0a_VM8~tu0%@%30Wg{C~E#Qjs)RlrP_I;~dL!wJDR}8$O zttM7w48DT)Usq`gc7&$qi%REC6qca)(LmUlTe;Yq8*{8i>q}rm3tIApQbmcVKz~~} zNx^buRIZ*tVSeq^Sw;FddcKTWmJ#V-#>U77aG}jKe~Ax zchvXKH9Uez4zx@*2Caf?eKD|MoT*2)uE?)#V_8dsS^uBisl6V66z}IKE$13^!_5=H zd*=lnAUMtV3t!6Vx|=Ddm)MUB&3;En$Bodw zxbj9;)Wq@#n@vtMHn?mnUj|*ac>eUvVa%{Wn#F6s&7&1lBq`&e8puxiItFoPC@mm~ zB6S&q&-ve|Ei!p1wu~*}UPM*6xhH0c3EA(dxULTIUQ$m8g>C`okX>sxj59oi9>VlV zsP@Dz2E&$EdLD#*wu6Tyo6n00ykWV^Y$IDa^>^QJgCfJS)wuL>wfcIQM6+@Aen#5d zy0>6}f@e_i2R0syvet_b`?&Hg!d3Bp>f$(sh?aM0U`v6sc9pEl)*yz@3H<5@59;dp zfum<&mL+T>Av-_Unzuzcvph!8!rK(7~FVhvBNBA|T76{fEOu64^;Vj7pPe zghzu=js%BiVs1#^8S!acJ0i=+T+7(5>P1Zpn7S0k`hj{ayTSZ{?gJMk9L?(PM~9u4 zrVuE?0!ZpeH_owX!GgWm+=~GP@bSOU7S^)){6Q2Hm8kc-ZD?b+M$P`7Q4*K%Z|up} zwr-w{JL7EbCFVjpO;P@*0<~fDuniM)A5(}7zpMux6rdg|+AfCed!T%5mWFuE&xVYo zINj)o$W`)TJ^_ZOxTvd!1adfM!}ND6lF~!9C3%;_7N`5Kd0$8hJwoTHU`Lb@b$75& zOe=E?Ck@4gh_;W%Rz zgKyl-^XKznY_Q5JhxU%dAg)tJEal``_oHmFejVKGD3LyvoS<77xM`{^@VE)upJ6!l zA|w=SZg;`M8WB`fH~^4Q`bji;HGULpFnrxX<$Z+SK>FvzOgCO79tSQITGG9oGr^Vi_4#P;xEM5QwDBXn(pWf)VkVDYAK(wyt{}kF${)Dk(YneMjq~}WYh5#Znwtu> z5ana%gRS;tx(s~>gu@%#B>CU}n%tYyzj1b(>vyw$79Yq?4Xe`6^XcKTF=CrgE6Rrv z#bIp1v3HSnK$WPs&*d|{lVPbFv*L5t&Ylnx!+_(Q|mD4Jq{NL~;I!ui}20=yYC zpk?wVsMEui|KPv&_;7^f%YKf3lr~`m)aJ_q$?Qugsv{MOIy zzKONAboRXupu;yE3^LD5jr~jAY`e%18rfaI;`3&32ur{{XQ09~^w*VqMNmOPy$qBwEsab**iRLQ~zrWmM%};5SO1Z)yLqIXE`i>QsrY9|Lh_(eva4IK8QNB zlZ=ID{J+B-+1S;o+H5)zm>(%EgW<=ap=%ue5!4y;Et#h|RwuyW95QK0tt&%R!j z2)g>m^U*8zFNnRKBR-_+Ca(Srk|kW8W}#y#mm8arxRX4|dC~}w ze*ADAP+nj|4dFyKeV-3s)P`e5TR(ZQ?Hk5Xr#f_($o#0+64jGo|S{tkT4i@%2QU3PBOFBN71x>3~GeHNZ zn1Vno)&bnXagsG`(k3qWiAt+>1ya=#fUr{72p*tH$@5o;WR{4&Uhcu@d@6yI^G(yh zNdR(i872IhQwWE_$`EnMbul-{doU|Dzg}CtYt~vf8L;o-1Mp$@_DCj>JvVxTsdPG5 zgovJ&5S1QJaRv9o{Bb&ye(Bz1T7rr0R{E-ClV8h|t!rI`%icc6tEzZ7*UI3>b0(vc zKB%R-eplND_9}Gk@tL~nQEAGwizF?a)BW(biZjD8BIXAWdCA!$bf7-p|&h*ye*e?gUZ_f1L=1wFJ z!d6n1;EKm4=&RCbKA9e6`2&QbZLj7altaKwM6|`&gpz}tLPQ(>*26zSv;NH#g9;yy z9u)~oNm9#AJvH38;+1VZr1nP7YSXUb>oa8ou)t~J(8R86K<}`HuC)VQWSb33=5c~w zCy3-1&qjEkpU~!_w1VbTSg1PfLXx4O3AuLs3QElN{{?!U^BN%USv?7K%#@vdc^KQQ z=2ppkPEuT$&&`PMFy=pJ@2|Isp7@jl8QcJdiu`Q<#OgcvgZizdBn&U5^)`HWOOM}o zY(0T_eC{nnAabc^lvZ$iKA<)V)Se@N-9vtE7@Yjh;7`vtRM&Tb%P%@3I@aR-gV29B z_&OyuUWC5CjJjTIRL75swJTxv!tr9vs$k|L2*uq zlvik}PVvq0C}n{!vz;Li5GY|WH6*W#%B54E-U&P})ajog9ckYR3UAS0$xZ~@`oO2F z)V`Sj3$kN`KG&FoTuh4c)R4@zX_;NLpRe2;)XVL?UMz#J=eaNtthxM8%bEcqk3VQ= z-4UuW4^4Z|nqznyYmN#MOf$6|1NEX40q`uQRn6qfhsi6?~j;)IX1BLL1Pb zMvkB;p(crJ0_!*c^U#q(qBBzZuEGv;UVM`0MOO!+`rJBKbMuG^DnfZE=$JTo%}`tc zgOk8;=7gPZiWjKLbGt0zL%(ijmRP+;aCjt1aqsvrYtDQTB1MBc z7~Nf2vu@h-)2T~cwxlCqTvY~EqwPM*b&Bf)-Muz0_7E`f^2o_dnkQ25{?$?6#pz7{GPuQ;}Sq3zkSeSTGa2 zjeH6V3ez!bbN{Yv9~zyLxWElS+=Cco{HX3@v5)Sz3LZ&>Fpxs0`2%D1;%7Xa z0d>mO_5P@X7Bk2NQ;Ze?Hr7EFGa%;lM?cX4{+Mvx*mm;A6qrH&M#mK>44ijCnW9OqH_PDGv$TyDRSoYdmB}1&dF$je zgOCm8f6xdZQJ|P)yS-i{Ozifp*^h4Rvcg;S$b;KZ_iJ zqX*Y^V(`tTBWEHS1&3aDB2PwbMfa|IkiMV|9>32ILnhopL>;0s3=5bjwq9Z3P^5y|s z&{UP`^L3g6$==ylLMeyto26`qECa$>mqOT=Db!Y*+1j}Bifxy5)XqY-z5;5Yl(=D; z2`>;v{WS6si+sWZe<_wV_B*A?Gt|}+(d*^w1TR6wlEJWw{Hx5H1`QI1As`%<_ZA&& z{PZ*gLDBzvLij}?^n@$Ez{=F`zTD;QD2cgJ(5nEMOwJI?Y$^4{j3=lKpkCjnA9ByZ zoIE(&-t);;@L!A*5_EA6wkoR@C#^`ZX%Z{47(#|d7Eb6M);vg@ugG91kvp)Ng z!a&I02=|B>Ad(9Qy>+K{5IC(WaS}WRj`W~x&Q56V*O<~V`}sWUdg;FmFbYJ)BTOFR zWKi7d2Nq#NB_KWbZF(a@E`jtt*LVY6`zOMQ(KW0bS7vO6pyZO;-xqxPVCta0d^_)k zeGAD2i0S5Sd?*%<5PEP%&SaBS!25W#GHd>ZL)n#Y33!hLYX&EU6 zJDpEznGt(q+5~b5ejNn?OzuV9yZhc_X4PRXoR&1%tLNah9FQfo3CVA{NOtkTKLTI> zn1IZ$;y_~j9PL-@ef%*X^$C1DAL71xo()?n09HL_e<8Xd6=65V!$5f{#rs@l12^3C zfPW2$*}_W|_qp)8QQYpYvt1We>`=8O!NNtB1Zko*_pKYsFL7*_5eM|%GPGSFFCW6M zSfUIH+HtFkohP%8wC^DUxjoq-a;{vT%s#IBp=k(W{zaeF(Tht)G07)y=D@!{s7tkv?_{L=~gs$F6dSG;300tJnmWa{gnh1e( z5G=3invAR}mCEjhvX!TSC7QTmY?)iGE1VU3wdqg{V%A zmr~PY#E)Se0HOt)hDiz(>8N}c9wM_YfJ%Yw8tF1hYZK)^r}KI{G;2PI4Dk4)qs(~E zA(sq4Rf0GwaL`U$W?>BH*XynmY|j_CRuH>Eo@;`vLaS9xoDf!r5?C27Z!vX@2hE)L z{^)-buORs%;-|pP*X!D%&@qP3FeC6)=|fC1U=7Q0q;9*T+p8)v;;|r`gEj$4CPLCb zd*w<7-TlUzdfot7^dSYj6?^vkJ{g%t-#taJTqz`Xpu zDBzM8Ejup*xT@^+_!;WsfmxP6xg=yY^(@`v<(Ix_R%Ci+{@^rz4AVj;a)DfyCVBEu z>gS@t4LB{`ZOQjdrAr1Mmkq&UGl@}j4@bVySf;uP)V~!C5-mvoXBp(0_8rc%thPk< zZT@OMrFH~o=QrPDWRDF=F_-p^fDykCF$+O=vVq^c@;BUyKYzbv=Vf69Ko2p|j497I z%Yx69mT7{*UXg2B9cTcabNB6N2;h6j`Ur7|EX*_v^f7%e4YI~1mLH1k$!8LTBwpQ} zkWmMD<$`l2m}@_y50>?S3T04cv?$O8pC34c6vAH5aS-+N#32xdHv@41$uav&ph73C zq9ZYa6-KpSXlIRQQ)=Ii03fr9<9iYHalM#QGKl#dVco9)`%WJM2iHDQqlL&AI{@7c zu>ss^+SX|r+s7?jBk$hqN$>_iFOz1 z;Z~Ap9B}!%YPR_6cw+J7sm|H8EV8lzVZX~}-vBYM>)ZtA*h`MAX{}4H;xbJ_(>FON z(!l$L8(egKeeVq(-bfyx<6yQl_J}84>B_gE#4&q$CT;B|l>ldndFo&v zc(C7LoV>ZV+;l;fb(1@+Y;>!CSz_G)T@oyM!@56+F;6m;tUrYzrp=7ZyV(M#BIQN; z&7Y3keXZUC6FmsIz5q85V^`HXXs@ZzarYzlRaN!ghP`0s#Z5Y03AbV5IZ8l^5){+T z7AV-)vGOwEmPWWRsK9HSMs;oI%R@VhF4QlQ#bcpt{k4yyiqtVCF zTKHsinJ({b!hA$Ua<>#bM+jI4Z>ZCJpY(eaLyTcb|KvEe2_O>AFtYM83c52*(;S5S zN>VXOTpo(LZ(epbZQe6bs~9HIl&5IASGF7II^||0<~M}*${worR261z;JpC9_8v*Y z-Ri3`>(0bquP%LKUB?RrviE`8?6fpJf!3eHFuYu&5S-0>RFOr7AvP%5=b|TR%6)G4 zF4MDgq6ErQn$@TC(~SVbj`sA2YqI_R7z3KUt1c~i#$Enp5f^k!pEbt;`ZlZ%QyG#7 zY500KjtoyDCG%0%jFn_GB*-Wek_Ue(U&gvD^9%R&kXj={mVQ(UYjFu4PBj|K(}S0J z8i~7NuT5&tF1iB=4oslHx`@oGgbDqgJFC_cF-BCmqjXZ^8P_mFXn_1O(o!a{n#iV> z`pR#_PnvQw26I!c4@P)Uvb1hb$R$+Hz>uVU2@mM>ynyN4;|=B*N>^@aH`llYkAq4t zc_$%S*}J`ET6jGDx@Y*aC+;o%^Cyhb!J4>ixX1~^KyJR=z`+ED9bAEdrP_r z_YFV&lI+iyB&HSgZpJ<)l)OWOV?@PbaXv477-y0Q*5Mo-{7^jj$r%sAe4F9F>79DQ zdYiHeD$IC=-Zwrgqlh;46DkeZ>rG}2+}%xG;xD}4n6b7R;2Ks<(+FCF<>U4eKd=HN~-Mj zdn7nFnfj7^&wgt{j{2FElTyD!-(Wsqly6L{K0i$Yd0y#JXbNV$mY~c;Wye^4uwR71 zX-sKDCIgT9{9x$RF%!k5ondr9P$w65mygo}XXhYAKHOzTgs0pM1BH(X_szl>1WY5> z2DHk9(qN=>xA6O)5i{D%bs^oB8dWAgMS()=8?DsqwD$E2i)P^ub0~jzTvKyM(lN$t zR#25i+lSVCq@dn0!fG>I-MRM>*MGt*`c2Y9g7C^)6wvT%{a-asy46=p4T3&4>$UP_ z-+Mz{P54te+DFO;>LG}_UqL?;bY!)dyV1kkmIanwq$QK8g1-UOa$~r3zJ(%za4_t$ z14f=JD=Z&~8O-^=A7K{J`NooQgyI}f@t;ktt>=Ch!-Lm>d%rS?%Mpn*nWgDwgc&SliP^&KTTZe;HwG}>HmDfiz zv>`g-aEWe(wETCq5P=xY^LV2Hd`+^Td-!LZ!KAtekg)ptZ$Y)UKSbGMI7!=Bh&;VdNY*AICZs|B z&yl-GlVy11o$kb=>DUG1?v# ztS?b$nToy>vUgvSB-{Hb3w#HC@K5I?lAa{$a>nkSf95|7iUL5sY^z!)^qYIoGyzYH zg<61%plLfT2fO&UYcS?)K85~C!Ze(J==u91@%hIvHrG1PgSM2HX$N=7xTo@v7N~!- zw!tM~2={MOtU{*VfQq8G;9?b&U5CCBb^3yGw__0c43lgF*kihl&u}l5^kM|p?;wZjuVj=7~RGGDh+FW zMC{BS#bnFCF`a9y4%P2{x?J9Y2SiG(2Zxm+=_nT3iCqd$u~(^K6G3-UZP36-hCpp4 zDEL?oq9q((leR7{h7uniNU4>I+hP}*vx{9iX_p`X_eQYTt*xlu3g)!iTHl^8BLl8E zRJaI-{B1mGMWx+UGtPfEH`@t{ni2>zsbyN@GV8F~lZ+#B*x7U-&ImTF&{Mex2j$-r zy{e_6`2DNB4* zh``Yq;kTMgR-LD?4WE)y<3lg|z@_`Rz?wNuMwTfaSCW7fG+?FAt;`Z1nuO7$L)A** zSe3y5X3GT|7=a6c!K}A7v=OGG?pXH!S&aQbJacCV)vV%y>=Rn<0zuH_Z8SIy@!oQ= zfI)?;*IEF^PY6Y)Ey0z~4!oq+>u<6Hy3`Pd;Y#L)?(*ISUaSqT7E1e`3VWP@pfUFy zO4j}^M___?oo);22o_1$P`;+;5lyYg>6OSy`A3znXb-*9VM|Tfb5opTtF=& zz|NPza4C3#MAMOnztfSgw?*`6VM5eRH-rdpUKLy41JbInPj2>-=4z~meNqj0Rx(MZ2Z z7fEOov9H*Ly;x2PpBxf$YdZKC$?}r+kyq!+K@9Y^ow4~K5EX_Dm}xxl?1*z^w79eNZom!Fbdre8+}8 zsqR(8*TM0M9br!KBDypwqFypECDXN*2sUbIJcBdkKVv~A;8H$ZPi;(DYTlO*+QAS> z<2cuD(C_0#rAfC2x1_czk)ZdJkfI*<_Vw2pmFDv8AE1lql#XxDJ@E|+-sFFHiuWNi zoDOAB+8YfkgD`}SNY#YeoB2_R8c!wmD6`H$b|dfMPh9l1J$bv;L_TEmaB0V>n*A8_XbW}J(QUo`*1`1I3f2gUjn@ zG$*oOU;$Nna$FB(D+hX)oyrxPB_AHMW2#yv)f&3eHpDJy5`@4t@mj8_0aPueJ8_PfXut)Z62{s_ zu-fuk ztN{|aecr{PZ(FBO9^7r@BgQ49I$S8m(|xc8WAc(7a$~LV6BX8F?>DR;QyDt7eM_M4 zY?Y+HH2ZgRfqgE}u*DuSq!^yZ{rQepL;Jbc>AzT4T-T_bWLQYScYs4`w83%S%p$@q zD%zThA=mN=EVo>(5f`c5N=%SKBWe{yFRZD=V+N6w%6hRvR2T!CS#F(uP*~#JZ5)3V zNi+MQ9V*c?U3(n()s*0IK4gP+NuGaaO)Xiz?asU$zKiZ<>yj*N%#2F__A&s8(JyWs ztZ*5l4{$y>kqMwm`vAs zo=V8UJnm=0_>BJ7ziWdpgxS|4m$>YX@?i|<>(k7~xNTl7CXt_`DS*K@NWq+@sINDR z-;wUG(f!SX{lUJa@>WULPzlL=T2mI=Zs~j``9zO<9kaK@!%}l z--Mrzd`V#4Vrf^fny=v-Ig!G?bIo<$XldH6D9tH= zL)~m^DO#Zl#t|**RD>rTaI6*mxBthO*Atyq0aNHuHaMd~9HHks@!|bEva2_klFPf< z^Jf8I`sTp{ypG13skK+wi)$h!Uwbb@I(|-DK4XJ>s&Q!rjCM$f|KB+Uvk1!c-Dn>i z1g8C70#15n^9VQ9zCgZFGo$E;Lk}Yb!!?pG)0)}HwJ$691S`KpEXLgDEMk9sib||) zZ|se-b(atX0I_2;8IF}|a;K41Hkc7w(_&&vZ>kCW&oG1vbLYa%=VCNdSYhhm;l7dQ zX~lN+*x+LD8VIajiXPw5Z*xLV1X^VD)Sw9Rd@mUU{sy@mdD0YVeui9dGyaE_MHd~g z8|D68NFpgXDm)(SjvgB>phDTT?T#&cJ^2!CJ6J{}9AtZ0g6Wp7s-|g7Pe_#=*^|#r zlJ;F(-ab&bM_#S!!AK5k^Piq@=w^RvUlU*QO;EgzGOd*fp`M9ddN(RQiKWzi++VJg zJOb@QQ5k;PLFGHmwbIT2rhiDiw6)^3GI5NVmp=b44AC<3eQz#78~r8)hI?wMV%^OYF`9k~1Er-=h-$Drkbh5mkb~A*iqxEwPmqd6*hGNzoklR~LM9*N? z_meV`?%u)`!)vVhe`|f+G4ohGS$|GvJ9r6)ROcP&)nL$;AhW_w6%vCitRDyLu2nb_ zL(vBu!MVN;yg?igLkyx0R3I8O10VtKn*i->T%ml75K{#VND>VlJkkVqvq;aDVZ$0P zBCcvYJz}O)0RmWtHu>2>w1ZH>o~agVjUqkvR&&!%LfEP({|om$V)gZz7rb=f@Ufqg z@M6P1Fmb%o%k>g2BvIhAeFjyQpUGge7X6JSsp#b4F1cWs$9TC&(cyN8PId+K7pq$> zv0Pe7gt$2Y_N#Vb;EnAw{(O>*V#ax(1t>pDGL(zYmgyNG5O$JP@X5d4o#+B?1YPx`c(>`sphp%c@tT4lE9t9or#{b zdEP4_$6+Y6@my|E*XS!MFUhA1!2NOnFoz$#9p%fT;=ZsP47BAc4nawctXjgP5hR}27{Bor;YD9EQwH`IYGj~E07mD7oR{NB$rFIrpr|- zruqsUoaTXx6V|1>MgHYcmN5ylk{UQ8v^kpjW7KT=zqFkgf}*u%NmG|TQykKHO3?5w##v6g`QtDhNWpfDs;Es#E@ekUv>LnoWapKZuq;Yr=;&#m zJ>8y4IKN?WWb0RFY=*Caq4UffnLq}kUHkoh#MYksXBf4~|7_{uu!!!#E4KItCRfj{ z@k(27xs~RbV-W{=f$v~2*MW8baC3+rk@7Y(v5a@Rm&b$1WMD>d6B;|g*EuSj*nVFy znhjCeMg>yC^oHumAmzQc(d-8!RR%WF(+6y3qwa0oT4d@;8)ItnU1=IHXj)?5CqY$H$&Nqq^<+AL zanH0lq2)ZL7D#pTqw#HtkOgU}eqGgtCJT}SnPUK@cV8VB1BEG3?9J!DVu^XcP8BCz z84GdC;Re1PB@}XW9Zj5etc-duI)Y~ z5Pr-a#?9KZBP>?$C<}MYMWD=@C?_J{uvjOz6R5@Fded8TQ57r5g&??h{6j=_kt%0z z)(crIM&*eX&QdpFR7ya~IA`a6^VZ^wnCna+m};xo(v-=HXDvJ$)5zeaWAbHznEtXl zM}avOS?*+CR1!I3f3}ua?y;;oKPz<37t7cNX>pUGn{Pi3sXWJC!%XQ=JvyFSMdt&)>eaG7@$S{G^PFmWb1>R|1|c(iyy z)*jgvG9{jl1i6{i?BppCr@Im>4Z^O?@~nRmu8A1(9Mpw{Uox>}E)sx7=SHi-xF2m> zYz`K!$JEtlZ)`1&tvoiG4@WJo%ncQ!1MTJvBntQmFGB;~8}4|bj7Ote?rZcrwMkvF zs<6V(>J*uBRKH6p%wU40arGx6x(NpUGOE%BNbT4)8TxA!LzxOw45;3jOWXsC1Len_ zEp41zKfaS;OPtqjw+^xJ1j=X!ypV09il+(!1BvAp>1nU%B<;u1aa>L?=PFfg_g{F; zaSg<9I>v%cABEcoZ&AusdcCNBJ6zhCb=`PI6i*GD0U%o&FKSH>o>Kz@``*25?ZAFxZr8W<*+K zHehTomp5rrz|E!8>A&t-8gLtA9fM6F_05Aj`iEWgqQ*f0EZ?e&ApY4Yc?8PHmd;y6 zo4>#!sw9RYwt6BMc|K0@PjtgMl$71x5(Gv@X#4C^u*(G`kp5gB&-{=Li@O_;RTqG) zQwbHbPg#Zq%(cS8dYcss#}Q)40T@o>by{tv#r-S(d0keA=WISrj20{8&d<6>lmKO& zGGY{|)jML0u^vn0hFR0yc!^5EJRc_@fG4s+H=L4e*XK0%jgP>dJ~iXf!#csI^aY57 z4ZkT69;K5}r%l=Hjf=_T#G>FXiE0TTi`OP(9O_%ylP3_sWy{Z_fAVw=F#mav?fw=QC~+ji`Xt zGUSVV=91h`*hO;-1}f5HSWPX5eRIItIMvf6Hb!>4g&|*Tkce3vGx5%InKWws$w=?p zXb6IA;FHUNO{fmZc`TV$)_dZB&CyA+#mW^=!4v7EvZl!Ykad5!8;2-q>3wo$JatYq zsg{ZLm+y#!h;sGuU9<1{k25X25$(K8`9SH1`kzsh!v$O8F2w0U`>tljl4CfFo|Koag2d3kDkm7u;sX$>B!NTl^=Yp)zwYKE zjwDe;5%q8%KkY8`uY^E`6R9|Jo8`~M^X&c%-T z0jkGm3Afze8U$7mOuKJEJ>Q^jNhYTJd7tO#Pb#~7v87Nc2_xl{!BO3At0Tner{WmzW zc1#x4i5p`0JJ_nhZ*mmt{Ql{|@)Zc|sgaTUGCY}ob?yV6GNkf0F$UQHLO{L0)P4?P zTB^}cBY3TXR!Z*LvVyF%7^W7nPvoqi`l zvHse3bz({MeV!&Oulv0zvJ2D== zYvFF?sxxZm4bQk?Zro23Q7DWrZSG>Z8E*Rd5@96V9g04vVW1-p5s*Jx#`a@Cra;V(7-udtgu;6zCiqmX~X z;EAvupbc42VW1f4z;dUzSYnCJ-aLMHUrJM&!UG5N z$jle)z2p>K76@g)($n_R3TiBXK-?SPBC^?J=g>$!aM!|f7840b`kE-a%HSv_xq4s}Ui3r0JNzh$uV@Ge*+ zUz^y8)wya2L9AE>%DG}dMzdpu5H4jhH}K+JYkcPEm!{Z&J;+`v87eN8*`uGhnjE;= zj0&keb@KyxIeXN1=m~ezY#t6g#m%bX`eVmjkUo5qSb;Z(T)Kdnd%|u24gvBaRW~C) z`$76@ijnJVS@fl>XQO8J!8{yd>0E*9=&T6G_fp+Va!~ACcu9uO{gc$i5vbu$W?1 zQaf>V=HCI6QqQQ&&~3G3`q>GPHDQ=aX@9{l2v;_m6bWJImZ$MEbzY9ajWY7iKEkDOvLIwD4@O4_Nnq#iPypR5P|6Lte2Z= za`Z-aE)$q>(uetf(rUeoi2QR(PT!j^SA1{^twooH>gzXSYG>DgpeN?@d51C9h6>r% z?h7c%qys`rFppyIQV0QG72xAfJnVpj?)t*%Py)K*XS?2$sX+EEhYtF_X_Y670x|cF z>f1_R+f>kNYsVcDrcQH22x)@4z$sz|Izi&4Q*F@c%Q0B{8n43s& z+#4hE8b2K0xR8L(6Fd!C7G`5Oi)r96`{F~aR%yTv>GEjB0qBHiTa&q=o4tnxs({6} z3oLkAYs2Zah241|2t*XM^7(BZFvyWcMzvlY3txuOJ`T}=r+5Kv7RV^I(lmwOHxXbv z#kDB8BpX|#g;>+nJn@?M$I8by%!26(W54|%j_g1@sg7=FbKA(UJ;lt*>ls`sSJ@EW zE~jy@9Z1yiu>J#^xXmE3p*y<6Is>pjNnprLe$_r?K&1uja%@oZ%Vi@6dT5|}an|UW z6FkNNE5|SC$wLVbOh0E0l{I;yC&(hotUB|trxTaxdKMj{&C$dGY>S(+6N9ldOlzBm z;GMpFP-S1N96&ELiD#}a8p*CIK$-+ae2(K{ZlgBx7w3)Qs;F*19Y?r)%5BwG02iZK zZvN?(@}Y3wgHduI`a3FTniKVqHFn($nX!uAg&RVkWdoX2FWY$gSP(2Q{lyelvkQ?D z(@2}ZY3hPU6F$g-{o|r{zM5z(RVpj5i0yVIxKqKRdkLirn_yS}26**TVUnwq6zeBU zQ0Vw$o06#Z$4xj#_AF|_z`Gs1NO76~=>BcMip6z>UlAtdu{BcG4|tiy(Nk53s?AIF zO*pey^j-9PR{iF~kOw>}-^#TaE69g?l*?8uKhnX_*=LSv5#S6Vykj!A{*th!+GHoD z65@ii&oi6q_J=+dGZh6-r_4i|4kd4njje@fKhjhj_E|~=n!~HRxv97zce&AG1VrX7 zWe0sb|x)n{jr^N3TAGcKyB9U614H5;c>#Kp!B+=ER>u_OLw zbD+0B_p=M_q!-%-g17eAwelKK!PMZ3<@(8;&OG3H?3unQssmv^c=C`xkIl#+&Bc&H zNXBw?>&-RLbx_GAAUrBTm;Qz{6E75|w(2aK>Rh63FbUu;lH zQ-Yz=LEwL7?dt|{J87rUf*Qwz|0TqKhKa8sFRPseca4@4ToYA=S;QOsY>bh<8ak&v z_hsfm2#9h(Ts=ZIsBNcq8t>{7#e`Kg*R>SE4W}7~mAV}{X2)H_%jUu(5lqkQL}$uj zTV#%3mV;cVrk33SAJOhSD25u(KW`!;krJRIHF%(I%R&v@mm%WRQ^mKbD2Q_&lhCBv zCY{Ev*@U*sji(dzJBXnhiQg`Cw#!J>MY-8?0dq$#4ovo&KghqueVdBYpY~Mw3R@q&3B*kr(K*j-=GBq~peP87JH-f$~Q>lrv|-TI|9I zGBx@(QMpTVc)CWdc)sjb{T*Dr7;TI^g4yR_XY3KVHj{=f?#bO<1m7^dXo0B+=W&cA zThZYdR<~`M(hImHU2$i1%ZS&mBWi@3&TBIGdeadCw{W|7jMi69Vb-vp>tK792dpmJ zSG-78-$@Z2&$eFFiO{A(10qjg^@I&);J4ffo4}0`TNi1xz}2|sH%9z+WKTXTHK~Uwy4mS-+4l-qEn8rmw-V@?f7vn#A+e2xVFM zVL&C&xO@fI{h&bh^&N-Cx$3JmWA_uHlT>bznYCjC{#gunehvg}{q>8`ifuDzj&rs7 zenJ?p^L$-i^RJ66sjnOx6%uy>TpuHZaxfTRq=j!JA>>4>r^i?aNA5kd&JvO|3c4>q zU;W=KQ~-o;R7$9l-n;ZtEy#=Bp2?{{CXqd-D);iaU{%;vB=)_q4nU$8lL>TgpeyLd zv+Oh%Ao{(?fKi|KK26Tsr60jxkH>ya+~obYe5}ePrx?VOzpaV~#2K}HJirJ`-9U)6 zs}g0$USE&vzsau>h_flF-_96im&D&V!=l$Q>VH%wAk&dyh`p!LX78l?$Djnl1z(Di zPZOr*iYCS%*0j2jkSiK>kU*W*kUKwGV)V~Y5lKKJ%tIL%i>vRkycodvxO zt|vt_uvaRxk6qZj^<~vxE?bq2MaTM~QIKZ9rj4o$zcl;iDs8G`%7?G!!E&lmFk8_K+#98>@f)dVypnoP5Ey(wa+qXbXjS(d2gP^ zn}&&5%+)p?nF*idxZpp+-;&rWelm19j{EcIMtqkiDS|6YAq=^CF|vN3qgqm36x}c| zc5})id*pl{fqtmmWtDm}Ok#0W_3~^B=Bb`VK6xfx8xKU6#*im)P94mJS9|3QPk~o{ zOV=}GSTTulZX=Hy^a>tMQQwIoIz$63L!M)f6tI@p6XyBq+6-OD4Dsy}#ei5rdqeje zGbLuC>*WJv*SbOEL0THsl?{xa-G^&$PCgu=Yuzlw#`zrBenzh_GW)NQ)vThAdOV&V zmliOPUG_|85z8d)S)lx)r^5_hHnWBpe+SMxA9peohyRlJXwLIIo=}g@Dajue$|o;V zPTTej#7<-Lz}a+3h_O0?_KP$(Ry^pO2yZc6hczdnbG(C#pu5<;;q3N5#br)D2BZ^z z9f={g=SUQdkx;=yW){ z2)02*=gcioF(d+3^X=PA4yoxQ;M zm-wRX@P~KfZ+V%WhiA%v$(s()M|b;cFPhj|N%2*3{f+jGXm>xSzVWDks3G<_WV9l+ zVZh_yE`T!JD=01?hq)FbMX+I}l4zR1f(QUEyTAbNv;S`3g(xwu$+r)E2EY&(^R>eA zZtd+!`=Vg$!PmskKJ<;@jgzWEuhIDtAffUHhJvMWgnQ1@nqr z0Hj7sI4@6K(RU|3uWNeK@xMd{ew)|P`RS5VV}sQpI{URS6>oNj@6yR4rGMRyZaE%5 z#rGW5sm%r`bz$dXPcSIt5H$kF^+tK(#nR)y7Vp7&xOnM%hCulkzzyJKu>!y47`2z` z=~NZGMSKLv@`rajR^P)+jQt%V=}rG!LXVorizl%k6(zOcnNF|TI?^TGw%}jRdE^LJ z1>caf$>|+LR3LvgT1+t8#lOz^z-`a}d+s47vVmcA$_N#WQ~&R4Vl zwiJp4u{Ni0!D)!w|1Oka>~HDU7`VC`h3Hp5BXJ=s^_=hqq)&RBRL`e-6rtBa&9$S^ zo5$!^b;<0z3@ou9h}Mz;&(-~n7e;tF02a4{FLy2SM;8ru^iQ5dC|m(Z_Hv1L4XOrv17`^O z0W$7+8prnnLa4=#rB=Ujfwj2oxLb}mz&LNcu+pNxjywU4U;}o;$2{r=t2r7aCJR4z zohaHM$=e8jEH_`)qjogsk5^WA(El)$dvli0gw1{GQe)z$*02oYOwG;k!bsSD-l&j8 zze1p;&Tld=6aRJ7u;prRo9}Hn0xx5On!-lc$epYNqeB>fc)eJX^x?C9_97}j0{9UR zq7`fLjzbMdXjfYhS7LH8UF*?QQJ9ixKy&2Yoh#u=8^uPW+MGTGpIS@V38EPW%Ig5Bo|bEDRY zbcxpwj{eTELwP7h-Xh}o&3I|ch^^Dft$aEmHd}MFlhMCdoVYV7S3l7qTWB8WV$1w_ z1Zo*5k<@Db2(17AX;*54bUDKvJ^50&7OOff@i_Byajpe(8O#@;1{Xs;&158u!y3cpMECdQ^*m9OLfK7Mt_ji&;bBS?b`~lO)q3EM+@Opl#-Ko;yG2tyLqy;gDRniaixEfD3m!%# zWfyjmNC;>xS7;Zq5iEz~gc^IL?T_Qvp&z8*%t1@l%M#fu+qelF@17ZD%3>Z)t^k(P z6)l;-K3R&e0iE7GniS8QNiYJ{V{ZKh$dN_fH@S>R!Y40`<;P=I|o9ZWz?;Zx+gC4x$lIJ%@#`T~OK)G+u+%NyJmc2Y{ z7=$8C7~Pqhfn&daRx!wsQa#v#KHn^Txt}uglkC2%ZQN8*XvPk)c6@Gy$b|U1Is*Bz zgtW0|#{cT(SyctL8iA338v=mT6c`Bqvjxy&OUChl+yfrq6BslS@bJNJk2V$dd&i}3 zarZelj9Y(_{p&IfewxPr`+r8Jgjecc`2C(^;)T2LgEEjOHQ5dp-nZd(2IU|yFg0eO z44Iv~HI~0Csu>YR)=1;8^Po8y-7p0W{hh_~16q zT7QOf+;lBkaZn!-oAZ*3Dww`qki)K#IEB!=vIa%NH>d964jY=$o%*`25j-XGy`e^0S#BPHdS{q3} zALVzL{0I++8q8tjnL%j@I-}gc2bd<2$|fY*X=WCW8JDgH^Kawpjjmn+LwGeC7|s(f zwZvee$xvvQruEhlo!H)0v;+?m<`imbxmT3hWYII;LLb5KBk(1oGlpUiZ1ew%DdZXd z84+&plMXbVxrHeV!poh}o#nv2Sa|{HLshM=9y6nrB)H-)CE9tR@Sfd+P+h2qu`)*q zPP)`7*LR1=9>Je2c#dLM8fI4@^U9}c3zAX1h$_4FNaGN%TR8I&>%4*HV0HiN0J?4b z7PYm`t~_Di`wpZGPuca;;!F&7P3JmQnPv0Oqe8BJMw+k~I=*Z=HqcEm7 z#zFt7YU?Q_t6#Z`{$CFD?KNCu^4u0Wj}6EixxFWBih`r&mJo!(tSLS~^EPwN!H(Vg z?UIk`UNUUw&ec4w+#XO4)ykGCIFFTT3AM9P>2uhbxJo@sRH`QRtzrr@f?S2KO}YX0 zGnuTO6KKNI*#nRsBxM(6-I;S{JZ~5Vw1d-^#T6`Qw*)1hPJJ8k_zlsaEjl}0MVZ%K z>}<-{BQi3j`ke>S0KsT<#y*=w0)Zm-j3a4z)2BzfPo@xo~6w zs(1rx=MuBCetl(1*t#P{MQ}`8FNR`}d5fXlqOz&`>VwzpiG_|dC5vOhmq-m6=y3c? zoSe7bWLX@|232MHZ49IbVW4o@b-#u?>kbvt$w)OGRSXtoWT}KTc{9PSo@3npYr{ix z#;Z!OB`|;sDa_U6PG$wcLlyE;S-eu)l`8u|WgVE`{4?(GE^kJcCu2y0XbQN@!!WnT>UMv)Y zPXY33b!1k`7O`&My0UAS9utcR3gX{cZ`^sc=Iv*ziR_9HQH+TT^|x(g9oiKohW@=M zD&*ud;L|#QJb$W=%*z>r{d$CL{mb23 zpG9FiGDcx;0}eT`_cRBw!=02UWSNiH_mq9lm^6)!M4BXZ;ntcrdNa#wQ&LcgN=XA>|#k^Gs_X*6E9>3|wdWfI!GPnub`J7SG5cw>(D4u^JqY^MaKa z@;18=j@i7R(2#KrZRv^G_BNSAm9b9^9>Lh5i zR_2pptvgtQz|t-%vM<~;aBe6aWE%lZT2e@-q~G-`(9p{AJeWoajIRwXqvD^Fd0uS) zP(>e-?RfGQ!H_VpLDnp#K`%iQAnRr0+^a;de%Qozm4RcG1Vwv#4tyP0xCP71O9Yr0 zs+yraT6t^KL%hR!C}V*=&XCP&Lzd#q*E-`0rV8(RADvnTpZ*wi#Y56Hw@_q0y+sjJ z-;U>--y2~tjw;xU4sOy>X86`3kIj#7I8TVoaJJs_tqY1e?MWb$#sepIT*Erf;Z4eh zvN6!N>{24?2tcOie_*3!osnk`eP;DKmd+4(7Xy`?*L}JNanvng{i{M(Bq->I7j*ga znxkx*|8RpzO=PnhVhikG(!k2I&{|AB1i#=aodFVmQUj_$J=(HuXG8^hO7Cp(v=^2b z6A%tQoozplWAAm|>Xp$^ug7k$One&Uv{k5jo6ew$d*ID>97J4f${BEC&EShGQ#c(6 z9%sIN7BL73k1~<*7sf|P4m%BmHmDnaDKf>8ev*7XX&?$HZ(`A++JjAAdft^{4!K2` z7?0Hlltg;%esnw9F8lP@UCH$mNw{2mvEoXT<6+0W)kK6n*=W%^jF-`a6T{^Ob~3DQ zc47sujyA1WLZXWZ+~L`(`_8jPJ6?n#%H&Y4i;_YL_|_|H6r{(auM`c`3nYEi#^5v2 zwIb_8Io^zy%gkse5Q#RtZ6pYFYxfWahxA+$R=%=J4t;4rRPo9DJ%v(KEdNHn;$uo`qEDgIeLqdH-0hL5sQ@Z-0ewFw$G}$zgWq z=(DCcg@O{3gl7Rh66vN-&hN?pt8$7#!?UNlBqs$@ilU)Ssx?3-={B-&S?pA6!Xhk?W(yYZ*ld!&=4ONE;>_e?HM^qeBrp)VR^N2-PAYO z1#YGw_92md>3Vz*rGbDKX%8xCT$@Nx?E4N;o7j%_v8WVN$eUWu@5(rY#KJnLhr7s! z?F;VudcBGO6uy3eiu9uC4DL(i3V)jtf@#BG?)tqzp$y4C>k|M#MR1<(I}&5|{->gL zY}#E*fjwF!|LWl4jVJRf*(V_6;-G>$Cd?nrtOM{^C{`dE*@-@}*)bwx;-9@jhJq@U zT)#L%7xhRFNWIt!Oh=QD(l#FgQNUxKA}Q>u=KH!wV5ASvp8XD6hBnm!2p)@zmxWul zQ6fe9U9YX{O%9{l3)A0{BsTTI{OJlN6GGd{b-tjFwWteD4@e8vEF~(LwFw%o{-o^C z;8&=(?>$e;Owg4teS+NV4XWxcDbZOaEh;Xyr&jK6(tDQS@-(?F+q#KR8t=7bGbS<_ zkO3=B1Zv$s{+z?6nFwXWdcxvYt%@U7))4TDI&g0Dm9$!q}8>X^^=TwJc{Ue`k!$j z3auI@@jWb)q4elMOMUT1z3#=lB*Pu} zwhb4N4Wof+tKMpa#&wbQ5^Ew1Z9sOzS`mY+vx-o8&HdTCtP}P-M1q^xPtzU4`h4+@-yNrY>R~s2y!N&qLyp9ES8kqTqL4ulhUb zp>tVjWMhhI8~Qd)g54y;glENa;;{vhm_;mp!W+09R7yElpsX(x^tT#|qoeSOa=jNK z{3T`iHV1S3m?NqjYQ}#npHGR409u>$^J>MsdE!Uy?8DGSDr|kxss`CYatU`W!w@OT z*#=ykhldP0yneu9li)Eb7d~!BG#(~7`G-<-)L3G>!&IPbi3Of$dfA!TCpxmzLVN3> zZ>(4RG8ME#YBJ!Dc=;43+d>D)dnR=w>D901sP^cVfUfldP_h@h4W9SS4uU_b{yeGR(_t1wm=_xIAUJ+A`}<<^j^YBT$bq zJzl#vt3m0vZTZMZp?eUP$R+rfb6xzu@o?HC-#2L)k>^yu~*>#;-9#THvstn={R$dOS`N2+5zy?XXtMMoD9E}2Tw0DrEOE|LiPo;On#he>@ShmCz^IF{ zaUu$M*#)W+6H?97osHs*tCS-X0LIvo!-e$XT9+C~QWm@bWn$1!DZ>sV9EtqVY65u$ zrg)+&ZL^cPq~iV-vDz6-%AA9p_Cuh zKp8J(Mc%4<;SHRS2{1ggEnJz-FP>48i5oK%b1z&bG#l?8s~7G}2M0cCCLV)HNoc1+ zDwazD!qJi&kW%@L%?|F$(ETm*nF#iRe*yZmooWKmqAVmmPv3vT1m})zf+(2eFsM#G zJjewin434W?Pf%kY$OwECk-%P~C5xRahCM|g|R zocifx^eIJMd|KbQVhjp@O&~ijJn&C6DpXyx5VFcK9>Q_?u+FWFf*^zFw&IU=VR^%( zLwdiqYO3bpoqVHkNV6q=C zi1=8R?h?;&h9En{H@6IW#TEx3U?`_??-7{K*kWJMS;DdzSu=Jpp9rFGB*6i+N8_(} zm4#@v$z1B@ZqVIxGCI}Dw&Nid?Gwad@#^Rc2r#hV_3eX05a(N16$_jwZ)Dk)xaBlvb0r?m{;x3@AgJJ$MOIyDuYaC@82KkXqart$mKFiO0 zvOUAY43Ri62iU%ky}$820Pjg<$Ya`6BL)~7+UzE`UR91kivc#Fq#~?tnmW$Gr~k;D zOwqC}Ud4^AwCJ0{Yu8b$XX8fkXFI-5gHPnavdK~(8a#zPiRYG41e(PvR;Uy0)!e)l zSHlu`^T&YNX53lz`ez~z_g%86Q`dwbo>gxg6jwjwRTJNnHNAD`2#${&tnj_gi^osH zY=I)3!Y-GJfk{d;Jt=aU?Vitq6e{#gM)rjdF2Ql-ndlzk^aW?|wf+5c z7Ma2J{fE{Ml>r4_O_ZdfyEuCbssdXMk;VLUm-FCXJcN1awm*xIH5CwW2aqqNrp+ypgC=1?Uc0P%m@}b8LP|iOKrC8_yC$F z`l&PNTLi7^4-yaM&CeG78MgjKz&9uweDM?t70f+74pw6}I!&5;D#}v4@cb9@vM*x* zCj&LhdIkwS2vA{BMf2p4OjU|n5Wr|~EY$sc25v~Fh}vAEj-oPSFP9Yy8C$Bglbg1G z>wWUjxhcMyuvjxRVsT-j)@pU3wb7`Dc0kbh*Y&fGiDhb%(_gugn1F37(x|p@eS9`* z1wRT7N+g2}3&f?;XFOQs!2hy02RmMEr9I)pCz4ZOLs*OstsDot)2zZkpEYE2_1M1p zD141on4*x3Nl#wc;RbID0=@4-pCxRpOg5g9iMIkc-;sB#^|eE_;p_2-hJJaLk0dG@ z(ay1>pz_*qha#XT)wN-i_INdzA(X6CpP9o)W*faj$ zl(qv3qe44>0)p@m1AWL<&Kgac&dhLdEhP~oycQ9QHGK0%n?OVSF27w>iR`Z+$AE$Z z{2DxPs+BEc%AI(?Ly2sYWY4?yjaXTCbc%Zxjbsm29Z|*lwgmRv_+pe+JSnd1WweIc zkem!NcEkHP0kY;C3#RI9m*qvl5uxVbV$MF&3*1L-YaMR67?UG4*f8@`!J zCgyIkT7r;6BXnJAXiuKb9*L*2e@z$b(b7r7P%H&#ol%R{{Xc2RZ9kvgOFTg2}wrdq^7!&F-CkWSSeIH+VX7(fQ!QC+2J z1B>I!9&@*n9zAf8wZJ1@E!wDr(=Ps)^>%U@f+PmNu{`o}UWpjAASzE4*)!QIaXeoE zo`jHRoXjUGb!mY_;XjVZGeTWl*2jkZl8~dulNY*Uw(>?@UWo>_X}U~c+2iJe{HLDg zuLyB%pIX@PUx_ns$WWvFe}Loq_%bd#7B*mVBvEXq@$*O%RCzpP-pa=IKoSp70IN zFod3>sg4%c)MG7WlzoUOSy;@}IDs;P>>N~%)vAEnQ^46*ym+|eIEWQD^+K>-avIS6 zmByqE&M*hs%}UV>WoS;273lxrGm4)Q-$5ctsD^hJoOGu&V z(VED}vu)V4gB3_?qqF^TSer+?4FTaOF1i~0>$#`1*lGWdV1!01-!`MtgC|(4?3>Db zcLvE-D$myrUcI<%FzONJhSf-!aX>*rGm`_9%uo?|j6X*=n2J72-&MSV+Q7J_&=U;M ztbcsoozF98LOm#JojFD0yo?=Dlv;20gDOPmJ3ALRJ#Gss9Y&QmSg^OJV>$5ZA8f`9 z4_)e>Hj~tlfL`6p1EQ&yPw+Y|bVzVk1@_iOl$lB=40frBpqV@=4qmF;it@sTiU+*7%UMVwprwHYUjErP zROmU_UjcF&gA*=mX)!r6^fGjyI=Y{54vYCpI$BDdbxAZdA!ZsH`UOJd@Z!HZ?=#V! z22WDrgq-fz{XJvZZ~d{P0HI}nW1`7I@?K#9EFtR?4xXk>8yHzd9Jp=6lJUIiKZygi zmM5*0Iwi<*_eD;Am+Jqe7?Fx~CW>|&<YBRZGlUcupSBBcC-$57#c1(tv`i3#2hh+|<4|4t2w)u8+ayg5ehB4|JB~K@W;J zIx7|bg$s<0>?kroF<=jBS#eg18QC*C472ngXBbe^!tPA&*}%X*67&WsoXXvP{b;lG z{IW5sP|n5KSTeCpUZ19YjBcS^YJ-L$0`2DQfsyav0rE~(0W4Ro$bYHA#k36DdzX)@ z3QsdF($&@~U}X0}k##3joMsBU7vWr!>dNmkrckKAoPEa#fF#DrnMxOu0cQWXg9s8e zo9N)TM#9CV7MV4EjEZntZi^5=-;UXnHOGS5gwmfQBuY0mR6`E|Gf%Wp`&e%f^8mY1_uzrycg!|io{(%n3)-+t* z68m%5l}xZ<_U@EUx-+zqfFE-gsM&Em!Iv^i!_CLBa>7Kg^-uexS_m19qY_VklQ>30 zeOGx;n?tN%29$0dU*(AB(Yc{;m_^>F;_@RB5hn8|@D(dfq?M!-6Q8>C)|wf3D3Tq( zx&HA#{`l8}>nYrFl0SS3QW|4hK7vtg>Mz8s=YLMa47Yx&h$p&>BB+RrKzaQz)jgEa zsGTBAK9R4ChmMK~Bw`?CG#xlm z8ppJTkO=qlgjk+ZCmE&{y(br18Wvow&@8MWAc_c+KvsH@MnH0w=&CIz*o&}jwT|wn zwrn{6;TRK!q4H4T3M#m?rwG=J2P{&TAjAqq`?3X2$(-p?g|K|<7`>ZZf_q-R2G$1e z2c{b)pGuCnGZ_NIDA^jcJ6kL@A-pz*K{1zUDu2Qz`NYM1)rJ@l z3hJhvIYc{Z`$=dE8m#h-g^|)Tb{bZx?x`Z8G!W#v_=}8 z;35`e8?>6pc`T9vfRN)cCN2{ev&Q8w_t2gU^z@xb3n7)ju(v6~4nDE7Xr(J+zQ!UcMO%)Y)wRkug3ZWs`lfk+ zdDX5>W6Ro{q*-KBJ1$#AG6yiENZeWGk$C5jqj1w?(Z1W~R!U1BVZqT=>-XlI$Kq*;Dz1w|HguR4{sL_0h}SGJ_}XQMusmF@eO4jnZ9qEujC9-{wfVb<=Pap zpgBLK_muC2`ft_RGaY9^iHj~!$lHP_lwR=x@}3Q|yLNX`OOxra)5SeI`4J*_vhoqf zzE_JBG~^O%<7Y@2& zSJpJSHC@J(1wj+GXzqjx-^$M2ihqa4GlVI_)_(3?WmQS5bCgo4>7Hmy(BjIJ%qC`< zYVQwoc~`(qL{Ex_(y5`CESRi~t}AOn4pvyWp2+pWk|63%z}x*7DS5$lk7~3=394us zu5A&j#_~M4;l%*{^MY6C)Gm_RwpN1;+y$8Xr4KTq5@bj#Ksx;7(6uM7xJdB~USAp} z=nKy5UiMd?el{RWuomg0+}F)>*UQ2)^2J@{erOPD3&suFaYo3JoSm*o1$FA8mscfJxlEHoxqR?d(-rK9)|ln`%Q3*1I{+(3a!zoJY=e&nEa`oDOLN^c(#37iDwnj z%pqx*S0P^6g4H=2kIGC@%2L=N+3#^2>FqHUK1G~6;7=_jUDO_*f{x;1=R0Hd_m@69 z{NVPH(5~n8C6^GSF}nDLFQ-4Xu%obBZ>I`|8EA^xM+s#FL&`~W0&7Hn+um%5bAp)< z$aySa@^BR4KTm;Gz2551?Z^@PvzzO?VdRsr)6)(L1&JOdrpHRUk+8I&xR=8gyciq78o(4hI z<}laMm-#X58s@J{?g=c-`6dXEAcV&EEPXaIe05IMrq>1)VtakSjQDuzjfZx_7tAX=_sa zf?-Ad)y4q?bQl+l%tM{byk!HIfu&<=>`BA~k9P5uOcSXlqukqeq(ZG7ZHSPE6jN-e z890S++U%X%e@{)(S8&pW(Y33zrpJ&1&xmDm$PGkTjq>OYsgxV&tcKccf8rPh5{*js zg2viZmGwSCPl9t+uGkA{|41}G`=@Gdy{N%`o}$Dj!IxGD;5cOdhE{ws_phdQXtLWh z4Pp>p<^v~leo;OF*Ko=g#=QNi8S{&gg713{op8Gq6%@L`GRxA+a23nBSHJ~9xtsM7 z<9geg`@W;^wvFbl=tz&zPtU>V;qX37d6=0f5V%+eYrn1hbyD$ANvlQjSE5hF*sv?; z3SZQRy0#4H&>_-xrT83v0l4_%VY!Sn(wAi2jMGk#V>&s_)M>Jp#oE|B7+f^%l%cmD zHU@&z>s8Gz5ML?^q1**0qmAb4_;JDI0SG-9~fh&KW8URVUUigXaLIr*G86qUAbv zP$6KNgjXA}A?VA@n+q;!51K5u94%Ll25k}{sshBIXX2EsfE+$7>py`Sg=UmzuHEunlAy#kv@xV9{aQ{PSM3{hNP9HJ*Jv4Ngo`jV0-;OcvO z95`{c8<{5IrZ%>5UoORJ#r#Y3*u2)1u+QIfkkef2=!`$#C;pN&l!pe?eauuePn@oE zXSm+8@za_sJk28d*ne4u`{0`2(A`NgMAn_Q79BJ;hZHGN0RH{lD$fXwZAb4X0~t+vmaxr>!qu#FB89I(qD+ zTw>a7x$7n1IzN9Gj-NiE6TP?~b7pQpWwN<12pnFURojPu@lrJB!^bK!2_(PAqJLi3r%?GiJTQ%nY0G4Y6Hp=RvA>{5NA1!1({D!ZK zGnKs4dvC-0*}ejC&5_@5wGpSDcm%4Bl^KbHyAT^yLyeU)kR2@xNS-E9P7uVQjSb#r z)@L_;$4<{)b|vzC(Pp#M>lO9_s>quJY;3^=e(@dx$K487Q6uyE1pLsQF+Zz=E5ncn1T81kP;Ahm--ju&tl>U{ITzL4Zs%slYV;$~WS3 z7zW}pAZ6VG8bk8_gFH1);FFr`_q@QFDH@2b>+KvQch#pe-EK)ECV~#Pd{fok^OPPY z{hW?&Nivw~bGDMTabb7CGLyv38#4w6 zh{NnCOJ|va(iq2Q3xF|5{Q(IE`yBUGjSxC%-Ft-_cm~nENeb=rI>#J6``uB>i{Du_%lMH$U{n!dvSrtoguM7JOw!& z&CkzO^iJ^#C8DUO@81<(h73&avo?B_1Q8{cngc!@=q-ts4&}X`VLRUjBp7i0H?s~L@9l}l4sF~# zs3MC$Mgc&8EQO2VUCNnDYsWK~W8M%Hu0B86vo1Bs#wY-t()a=W@i~5aD4LUpyM6g% zk>Is8g1Jp~j#v_(3OB8a4kEVFtD7Z%0O2E0OD#V+|S4>kCU>k#%-sondrM}-MRmTjIYNi4ft{G8DgmLrN7Av z4i80)7v;MDVB%(5VIuOtnwpOYO&KUMT392S?ZdQiJjI2svXF&dmjC##*N$iu7R2ykURE_xMRWgY>oT786^mbHUS2-<2?AL{M_Y z;GY>E>j<{9VyaF9v_s_RJ2dv_DQS}c8kKn+^G<(}3_f*LG>F+UKG+N%0LCQ>wt^(J zO6@rRA1vlzt8+ddA&3WgA^pB#A#LLiu%-9v7X#l+Mg0{pvRNbrcv2J!a~gAJ85N!V z_yAC$=&(ZmD9+hpnSr*4Hab$HkiKHLaOx#MB+KvJaD7p@TXVA^%!O5WtuLtJ8iIEm3 zoBU}UyC9pW$mxc=lgNyD?+%<8bw8j%4Tq14FQbDt{#J|Q+<4*vPP_Q^6@WC8U^|ig z8u^(Wbxqxtlnvc?u%~#o#VTjt;u`%|1_jC_vr=&j(IuD6ElYt;2$E`5se@Iok->Eo zaWiN%sqj{_h_y*(-5x25eOb)lJg*o<4Ybi)6_kfFoyvv&t_-;aV3wA4&;gEwA(^IxwLkAyAw#z%raQf`t^(^+pYA)xd^C zX71L>fN}%ET&{VKE6gF1JA7uCek&x1l-aO-Hhu+2e9W$t;NR=6ouAK(9^ooz8zPEL2 zjhUA;c_GKghfW=tD`sEY(d&m;XIO>ZUcdf10Yk>>il1l}amC!fmSpN-SfvFAcsIbA zj7i)M{PfhS2s(%^Yi`&jEIc6?%nNkVI*_(3O zB8aAmY_Kah0PtBF8(xX%OBtdX==MfS$WKd_{w=A-wC6+@!wmF1L=1soPDm`Mb5Ie0 z%LvD5n(D{HI&2}?;5xKkgzHyax?z;IAMQ3N})z zP3+FH>k#rAX+g%%a_}<;2JVV)|+6>J$qWpNcWT zi&^uw*$0W+UX*tI{@zJPHU*9YL7b>!X=_E6D8gzfsC<7h7AP)wr zMiy^aG0j;a*<${f9#vK1+u^-}^ct4gNNgzCJZ3H(=au zgJ|AZ%JEBhp;xnBYxC?3p^oC@B9$hDm9K|rbKL-&@*(Ms++@3%gT3Ednet;|)*YBf zW0KB(Cv0g{NatVlaPtR+ui_i+{;IeE!aOyNhSu*7#D>gCMlE)LhKC|&?9e~)muffJ zqa#u8=5i?IcU);<47$m6G$G@vEUm2S!GDRQ^aoWb4iwLhfFNzu($~d8aQa1E&s79? z<>&0;jFaef@Q?7vkSVi9X%`7h({*brl%1wC?T)~BWxKJOtO2Xf+DaQT(1sIKE?KsMN*|6tb~@H7?3xdz*>{eGO4`O9%2} zS72<$$X6bwd8PJ(CIpBd)}oA~OZkmkmKuiZJH|N*R=;3UyWg|dLS)=Nv=qnWVqmM$ zM#(evLvQ6;p_bTVcrT^3v5Z-{FQmG2Y@@{BZA6tLTt6FsUxi^q8^|}_E^|tJ=T-=9 zrx$KV--7aA?L}8t*kK6Nz1tu27RpFwONdef309+{(yZmgp#I5G#^K29f~m|)plo|L zrbtKRCrGRS!1iZD;>JU}t0Sys{bMGB?EOGxP=W`W zk^Khct6eNwQRQz8;AUQr{nU@sOn|9@el5#MmTeU3Z0_L3g74|4Y~xYFKFggDsmn#c z3tk=$N5a3OQjgGaG|(@zC4 z_7++06=2r2O7L1D$yzHqG_`0?$FKJ?ikaPha#Y+^5 z@;mp_T~~pKklRYUbJ%!6aO$y3E<~&sPen=p=A6_-7AMCE1u%h3bp* zm#GxpD`0SSF2}zs0hWgF(tB!uWPHJDaHH93v`JChqjW}M_zFx$6hk{+H?qP85ZbJo zBX>%xj;{3{Q-KOh(04y!7JWavAZ(N=ez90T=7>g*9lz^lp-*XXeZ#0pTy9qov>h@D zyN@w7vd$cA8Lcs8GVmTrzqh)wD8pv~{Xy{(qt!a3>)U}>8fUIuaQ4VLFmZG# z@!Kel9UL*@oUV|*lyCJ^lOARU=|4;+lwo>JwQeO@|A1)2;uT@a*=QZ@l&E@rtMJi{ zkuXsR&0+^DQA%qzYZqV2l7)6*$FLy8xXQmz0j}pVq(hkc9nT~-dc5sd*jGK|t{$l( z>2v3p`KrM91takYAV|SbjnLZD_A}LNnCE%V$7K^R7 zry5~*^cr&~jJ5yEvO0?IL*rMO^@QQ{fk+!3tzow90^|PlLAzJQ8Wsxin~yx*?y zjqys93}k_8jpW0?6OKc7E?(hJPwnf8S%ot_j{)?{)Sm>qcyx;PV17AKO`kPevE#hvdOyhLH7aL)} z{A;<~@~?AZFGG7HiqlJY@S4H!QFGxe2r@jo)ZY$O;rZ?Kgr45r?zG{AD-qrpHk~wB z=q}5WSTac+fF{ecz1yciCG&rG#Ft?Qk~3Lhb=n%@J(vXX1kKV5gr}sOewW>nJ~VKs zmr@#q@=T4ThE5y=^EdQXAr{iQ#O`Y}ntNoa(DIxkH87BV4S`Sn+6}s&K1VbEI%8-Q z(4g(pVZGB%nSV-TBV%m~cS-f<14pqoWFnRdyl--hv~pCv;D-VCZfEzpImrzybwdBZo}c zO)N5h`ak33>dE*s=Hf5p2+OzGSx0J_jWvvHMXikh*YeI@h*9QF>WX!ZV81GJCcu5c z>J5mo0q3}mQy1T4Hk8M%RKD*UO3AF^>C~Xo-CDkVcL%oK2I-`zskp4ZT?9k>HPA8@ zqQLU~4fX+ThNQ&HJU3yQMv1R3D@qgyJ@DwZQG=`jh5`fS&1uq>=#4>>g zFi1dMgRDl603wMdi7c~#0Om~hfPx52wPoXXeKS$hp6o(o043mO$9C4d?Cxrbe!us^&`#KJWA-7!tY||*ze~vix+oHv-Z?(;A>x}2`5x^ zpXL}2_8$UiA@#$5-;D(-Vq0z{?7<>FwsMXpqnEux00y^^f}GmDq@Yffk{Xmpw%m(| zId$)q*ej9Yk3F;rnBq>0(qXil2i}A{Yo{0Eskbk*LqJJRIFwtshgB^LvCyZaCg|S* zNxt1R1*56`To)sC_dC`-bXR~V0)&hYZy-m9QWAo?bZ;ijkHE$666Y~i<{zrW>6Z(H zG1qY*k772Mt60IUM?_3J!l`jMz5!5e)RY3=1)QKjl2?GzMuy!*{f34d4vOL)IaME* z+(dKO{F(vEgb_)2<;*tvlxeV$l@)Vi*|a)HrC1&o2Ub63d)*ODf#1^BGxH*(}p5bD2h94W$zJTN( zqo8kU*p-{_Z}A;hx!7`T5+#Jrjw(+a^KROGun`GvrZ8`Rt+ZE&qyhkPQBEZ+aFJdu z!zW?bmi@BBi2YkRBv77LHrF|ca#}FGiOM);RZi>Nv_=`jvoD}f;k12$#rbU%Hk8*y zowbPu1;`##L8o@6H#KguF0ZGnkpB9uKC7b5K7RY9aS8|AoJ--q)dhPJLLm)97l+^7 z8VI|ji+O=|aFEO2wu4h85DZBu#QE;wvBCW}%fpgpw3VmTo4Yk5>o9`yc$iQz_GQaT zP>RWRceo*DX(-~u?)j_Yc>=QZTYFLIt<|}6gKH3Z8{Z^?$=o*X=I@VkJuOx5mEW?4 z!;2-pM?YR%0Y?6>fEoelGVla2OhOoQTt<%oB8eo4EVBRr=H8SagE80^Tl+RqPdrPH$Ql*wZX<|66ZjX;Ar{GI|zrS;P6${E&Y`hZ7uie_0zl^Fs% zR)ws!!{Gi78vPx>;E355aLu*idZQxZ%Iubc;Qwa&bN1gE-ST#>hoTz%P&g-Gee@@nCC@ILuQg)Gl-HN|c^L+4D<|*}6ZtTtaZD6Qdf^ z&+O4`{a$lE+YX%?_KeZ_9G`?Q?R1l0CQY20GEsC;j8sJ6X^3u6L)n@!La-HeIZqo9 znLrlPmY&BuHhOqcx>3dc0IGy}P0BMV!%Bex^`;mh#?3v?&_;Ybv)$6lL$lDb>^aay zdA}ZnX`-Fi>@rg_39`D5z6<~bTEe>9*uug*cjBe~I>3Zg*y2I$^-??SQi(RRWqKnKMVPn~87X4DBRnvxd$i~o=(MPZ%SC^(W zJ9Sb%mR{Mioto7;B^C!@CC*+N0oXFN6(}lD3h_cCM*w0dk|>hPJOJ_fNovn-onQI? z;u11?-nlETX>7X6vQ>5Vb-0_#*h&rNI9hc_ECP;haP#?_Y*kTAq$3;Cq`{#(UPKjQ zCV}q^r(SE2sjYGy^lC@=-uKB)oncUij|}zfoUS#Mx+$|4CJq-9G$}BnblVl(uK%i? zNR+kS-1jl&`%Yp61u%u_Mmm7Vh@OUOuJTx{J*pHo7Mv&>P-Q)X&>$lE9V(;}vUHAn z*yS+1vW9(pur)Cv*`fos^kaaG>^`yv5ekfnhj@Xr^>F*js&%bQ>d{CZLjStpJQGG4 z{3?UmEa5CtN7#HmgC6ZDSg%&8b=HWbp0|b~a4PMV?VTN?c*`swYF}dHL%Bw3!yTO` z^ra+gf7I$CE)g9X;@{+VNBdhGx<5%-n^azX*LPxyh_R`Cm9ASU&Ea58d-`SY=+SA2 z?pup`Ai*QGf`rcdYwV>=Ht@mrZzQ?smF8CdaqhQ<7BY2iS%==P8^lstTezjP10g56GtDW~~;V?u3 zZbSBA(j7?wO`mY$6;{JeC&i)OX1LWi zq2!1*hvy2QF@R~7j++el#)#waI-VTuseryF%M)DngZkT_SIp-Ie^j?7n_4m=11^+c z#tSL%&%KHgaJCo=^LyeI`s>~8%n2g~ksgSEZZ4M`STJ3#O6V2W>*-gEt1q3Srb`lY zdUfOyTJrZ+$jH3~KfjWwT z%)qx}39P#lMX{?$*FOSY&`b^gfR^N5{w`~qSLh_M;C;%3aPRDUsL7cCNd~hMaF#j^ z^?<_L0q!;&T7oi=^4RHo*{)3@F&7^kwHB9HE0e7%zhL|K;MQ*x5IoN!Yvae2rf#?8 zd!Du4A<~`ERAv{zY-aui7q17K<*{tn^J8gtuJU9=TbE8%NL)BGEu5-aFX-v@O0C#p z5>tN1{C#|%Wt()4B{6J|>Pg-iEpo*;`UepK%<9*})>CLZdK*e-3khau#}q>AehQG6 z?<`i0wg5j8j)&d_)2VH-?)l(~P!Uwk+gR}lFr$Ah&XBG)snk=f+qS0c+&fzp0C=}r zO*~cTIM(P_blI58m8OpdO&Ma{?ZjAcFVRO=XPOjBEbEU>o7%vw33q3|(wHw-*|ugB z9@73CAR%OcTm<9^y!!gI1YavMGKl)wswUE$4ZfnGkfbclFF~WpP`qzduSlkN6w~YLQOgb2!)k2Xb>hsamfzm_Px?^=Lj= zR=sAE!>|UDJO&}B?a}hlpTMcJ#ZoqK=XH`f*QXnbP4azDdQQ#Uz?SX*!u*J?8w67; z)a``P0pUk*p$j*uAJkTfQ*yvHV$Vrpooq4>z!-gk#p`$1oaa8-D_h=Ngbjt{MVsfT zA$=dun=0JTAEV$E9G7_2tAZ=(!#44d5BywT{#Oh^rfN;eZ1!~8pO55LzDLV-%D#mY ztH{W|9@n!(l+4V`#jdRG2Rs+8YO(pR_)Xd?70(~2tU+1*YA;gzY3QqZCNjP_yu^rr z8kPmuQ+_V&J!X6ov$?7eF`!nDQr+Q1L8%Yl$h&Z#-=<=^X`OEG`@ws9t7-V<=J)x0 zSKiMEUE1DqRL6o1tI;nl{U5 zRqQE1pb3WX;od*bBC#M42^c9iQx;rH9XD-gG`KolL_M9R(wAPw*62lV?;4Q*2Ul3T zG0?!ss>^)9Yg?ES&cLMoW_x<`1h3Z~GlFC?eB{tXk7sSCHa+wioi}f!R+Y#TTjE~Q zEfiJk<_&Q5Hh%$qS}vjwy_xy>nnVjZBsbq)KDa2hbclvr#T)Zu)dQ0iu(iro*^cqb zJ_efdiOgfqC{mpfNgCVWm>&%oE$N&+TSv91o1>}etH7l<8*_vRjODLeSNFa-`e%4E z)Af?0;5zV0g`n|pOxrBA6B0s`zL3oNTc8#&Zt2K=tafKriOY9 z{R|(GwSLErM;xW&zA$8K;paeh@;;8L*rxjw<6l{G0Nc;?>TsEEd;)(oAfy6W5+S1H z5a|?eyNb9VEq~O3tS{C%*kL^|F?iOGe>@K`w+621W{bT=_VdwVBvrGXYaEIAt*7Uz zTOPq}GQTS+`>NWZ=4b3z4?1^MND&!;LrPIKBOL<{r<#OAK5mG-5cIYjayBEts2`k@ zN=wbzjbd$*9_|Rx8qM@V*?6V>! zKXqT^PaP_4U0J-zYa@uAGJl|)XtKk3Y2Q3)t4h214DD#iqH(SBd5I4sG(#?!6L!U~ zj9z6jNmmH)u2etrn~Z+!^ry0a32<&VRF##n&`3Bataxg7{a|&&W2|BHzh(=0NE$@F zov0PM-(xyxb#LL(!>wX(!Ihd^fsM*JZcf>C(JX>gZO>eiZEriH%=-ZR0pISZEZ-QG zl{n2r>H&p$k)1qJKU3cFwkbY^POuu8G;HJ#(P^jyh^Z-(xe(uLb7-q%d#-yn;pZgQWj~|yoxY$cpB^u}?$}OLn;0`(eb47zi0L>$96pFY@$Di1*%*xm zOY^ZUF8ZjM8zqLfgT`r<-8Gu>@2ZE5(LmIn_q=~PGXZ?xbuTogY%7e8wfOvF3jTs) z5^7d~(Wdz8j-!D-Y)$i7V>x<)H70uOpck6OD+y5Mk#dX`2En}hH((pr?rH$BhaJM# z6Sm~Lu>HA4@{3K=j=#LJMyL&S7S~j6URhZUf>WQ}>2a2{qH24igH#$_3x1a?sV42J z?iKlh$uDqNq^yJ4%KJJjiYZC#h{W|iE^@#epV9su09eJ<8ImCzq79}$CRUhqD5l zEdZzD>83e-Tctr6%XAJqOHUsHvk9SD43(-c`~`V(DC&gG!kYMg)2N3%gOX$Z>x4^i`$(|rl++$VG`GuIBHPRZ%Me-RUu11$1`|REVDi%e(2C|k%z4MmtkX*|Ln=N0 zs4?JsQT7RX9}yb8&fV_Ma!3=DLz(kV)3~aPdOjKl+V(uca3fxWtlL@d$;2B?MwNoz zYjgTW%OQ*7YMNw>ob&Y6-@iHZP~5$pN9K9a6I&H=|*n7(A^=<4LA^a-$|Tqbwt zyMp^1%dhVd8mbpZK;ng|b8|2La1K2sHlx{SDV<7m>^}*4at=b$<_B~zH*Ni^`2>IU zDo~X$97YPer;o1i(CV9igXSR-!7d)KLU64i4*vZ0moCTyEZ0X;42cOVV+gIf!IR&` z*XlZLZH=D!|2g%Hwixwy9Gd6zeZ15U-1%WEkYZIVBgi3aMaU=5LI=~4VPhaGx2X7c z2?b>eB5B*bkG+D3Tt{C|m@Q0Yxd)ZutTlS^QgY33aACe}Pa(MMUq@=N728@q8B5AR zqRJ3y0gAW^EMNagIOl|SM+F7bMBl&wi!%^zndE7J919>`wYrR&^Pp0qKY!V2s>&~o z2xceRip-j?if7Dz?Dll3+S7G_SEEAObKDjRSZ z3(;|XQMe<_8z;%MeN3n4Z-wv^%hA!LVohDtTB0YRA1zq(yyL3@o)+U}R}|DeLS^u9 zSPoX>y-1`Z;4Gv1;iym#HR5T-VEi#;3Ip&*(&U#|q$Tyjl>qI(d)CupKjrS>Uw1{c zmmrWpl>Ae%0$c_{fpE%s)ny+2M3-90&=&n@EkaWiZ}NQbi|&c}dv7EH&s=prS!l8l z&A8aIgR20)GHDpe>vX2l_U;cMNe|84J$<_1f2A=3J~+L#scCg6^UMc>_Kc$s)K4;P zhllA|JlI`M$h#`rt>2(rNkK|76>vSdWC|~MDLw~Y=Tv-zTAB7VK8G5HmOM13ZgJme zXv6c%A^8)qQWH&^ZmW7r$488mD%B#Eq(s*GXSp96J85j?$4)smxiEs(*~)L+t8V{{ z_KN;^+iYOaOYI(Wyba7IYpkthf+=8I@W@`ST8v8TF+AJuN%D99$Edkl--l=6ah7VS z|I^@ujb&mI5bpIDU^R~F14D91Mefp2W+gs>mf*`?{@S=4jMxT@ZiW~&=M)!R$Z3y1 zutKBdS-o;9OFaWqXCqsh*mod^5qIl`AQ3E#E+_6LewcVf#CKM(TY1-J!SL9zinr^x z^F=D;mx){Iac}?O4u5>auS)p+=$MSA1soXe&H4k@!4sh*Q>HyX4~Gzn@@JQvhHoMw zA^%89e2kPD-l+|WDGk$rnIihxv}g$D2{T>=Jze^FPW=L(!?v-?#^$%P-i!&KRhtEx zlQQ|2U`zsu$g}i8=%Uds?VLwx&;XCEEd?31k<|#zB4fS{N7pamnkOj+xzvLY6b`P@ zF@=?2%A6; z*01nAp*k)pIc)HiNb-@2eCZiXc`&)4JL0Sj90miavw6do02QGdjkrUQY=!Jg=LQQn zJlX`~IG3o_+^mAy-HJxFpPp-rrqRL>AvKG(HAoru@GXl9`jrW>gEwfjK5$p+o<dQ%&#gLZVa^Wyb znuBAgNdMl|#KW!Y?$d8+zD~z-iEVG*;5Pm=V+Or0e3(E>u#yO{Gq z4znEPV=#Y!wguAO;|ixWD;C;?B{+>cAiE)hPXK;ob6$npf*s@grHe5+-PjDJAxh_5jI z!HXFnQ?B@)C@*o(r(61;%cE_VV264UWY?+l85I8$Fh~qFb(x;U#qvI1f+ds$L9) z7R}C?xr7YieqqPg((WNC#P8VC5+*Ww$XMwBofbk^tuvXtn=j*p?4zsaDzpoJrIurp z3^tC|-XVc5JJJ_Z#PakgZJs?g|I`aA;2#%>KygQ<=Y|&b|=gc9FFOxmgP?H`4f{&fqG)i;hf3rXt@Z zU^JcUDA0{T+;uW_ccIaQexkK^k5i8lAlrv!MoU3X^}%{`HRXSNT9wxVc6R|VjWXk6 zUO%Os(C__E*hS%UXLR~EqkzZ;bJHRhwz8H&9YzQviTcN6Sg!$;YPjw_fIFGqk@UT$ zjMaC%Cuft`NXL*ez7i*v%G|5_o0W!>hsPAA!YT9fA}4_evc{OB3R&H_$K6ozCj!by zZ{gv)Ae%Hmhb=F$=atDNdRTN~@~LvpMZ~NbUE{M;CXvD+P|s0^iI^pD-*Fe7fTGmJ=VU@kaCe% zi17rk(H)aCykbWjoymI6YBPFDqgSMe|7;@{vBPKX(%>#*diOYG)SUXSv=oz@Nz?o% z9U;X>E&*x}mB&lvBwykzopjhIMdh3!+qfG-CyWj-)U;fr|C42Szdr0>r+P09BrU7q z*^Wck0{gQ3Ph`eIqP)P&s{hKSmP@#au^C?%@I-%QS?iqM+-k?CgM6`#u_SW(vf(** z=y;r`;nL)Jc?Ak;>BYk~jA8%#)RMBbXd2?OGb$0kRDU5N_>+?L4>}(r>paqG$ZIi( zBKjsl`&hqnEvJXJ3bh5vQm*7t@&~S4kce|T^~OGvzp&BuK9;aSg#)=8Tc23FY1vc| zzJujGIe8d#`RZ?6OuM7D#?iopNAa^sR~=RXjS1lZ7(gdkyGw8uuJL6@qcRY%_saAJ z452-llNgzK^?_oiIxFRvb!0v^{pwhlGb+?~#s=o@LWwsYgV(A`dj)e8Qk)!geO~Nz zTuG_~&FuB27+`|wn<&?xqxV??u9h12Hh>xd_%hQBC@cXFfmS0&fDk}YM3!0L0Q4%6 zWU4%Gy})SS1^;yV|2i5K-z$3sddS98(enux!yb|h3?2fx;R!1xNc)<+)=jSS<2sGVrIQvYx~CJT9Y3#4Wx_`nPw@Q0X~q<7pVu0L%3` z@)5kqa08JmdR)>ij^_iml3X*4P9@&0N{m#{uqHIG_C>R%`rwoi-{3H7nf#N5&@wRE z$0~!-UO&)vfhXrCL;~2oL1OZO#`$Lb>+uqJ!3KC~Gk`CbFKZc`S{|2w!y5W57P}Xj z9<}?@<-{(E!+$?Zp$(DTK8h8=m7E_mIW}?yUB%nHv_IQ8B-;%5Ee>$E9qe?QX@n?&FEEUUw`lBg~iaD&>qC zZaUGJ`Qd!>ET>$s%cv;)7aE7E4saxYf-5t8fajw8@yVU9LJx(jHtA{KDjsWeSden> zS?$f%gC;Ca0m}?YvA%H3`^HJd*HP)Ux%rqtONN+SvkV_ZiWpiaaH{ERH@=Hqk7OQe z3+sp4&V}%+2F3fcD)w7jszf#hVVFgVg-1%gN|_wj|B$#LgO+-xoQmx2pZnGE2m4vu zaa)M0CM0R8_*M`h=)EPit-S;)IFYD5ZB!yZqRe`;p|IeQXcoJx!DOzHS`VbNZ$VVW zH~d0RKI>t3H|HawBbVI!nHz9O57ib-idqL^PcW25yvrJHxBVKBy%X-s74}pnCVJ`f z6Ev2EWQH8_JQNDj=Y}xv^JB}<110Iq<6e0`?5ekeJ5%l8fm->`h_0iuHVbAn?-@{9 zrVP67!2I5%H33d~`yisC7uqD;`Yfa2s-9W%_Om76)SIHB7yDdQI2yz2glofu>mZGN{Uj>ZEC0Ykd`(N{E25<-ZO(!;CZQX#OM5q<#nE+Vn*Q3Q{xv)3 za>pyNVk(lWKSA9aE{lQP%}g)C`C@-rFH8rb(<734!yep7=3YNv?b3&h9)>FhOp?=A zsFI)e(Oq&*SqQf`@?=&WSn_FKtnI_ys|ypp`nwx^)lHj* zP2UUv7dhf0uEme%hE+y|(9tilzr`W^cBmd57k9`e*eJ54CngnA4<410ZkmKe@H=DP3i5T{stSsyJjGI@S3SyC+hsw zo52T$19~)iS%L{kLH7A(fh%Lj14QCPVu(2H6f>ett?=DI>~NyV5YX;aW(KUqEQ>LCSA(tmU_A*ZaJnVG)wh&(_nno42OF`S%Cc}e zKGZNf7h~lTFidNT$X(-E^{=n;dbvZ^)|97XvACpZBY)la*O~>v^l*l$yV_kY=D7`4WksWuk9q&mtDeQZ)@`-P<`M4K|5iTr^|< zEgo9yU_nMI!#fuYtF1VLj$pL>*&&z$2;B1G{~mi4F}W|(NM?3r{`K0uxj*ps4gb+o zzF4qxl|0aV^%GuN;ceNZ?7?NV*{1YN-1{C5s^#$1AOWGJkB%B zGlP{c)VyoU<&&aHznZw_^q_ZfiYPNNCrJ(V{O|N_(LQe*x4VbvRGzzFKDu(h%LvHQ z_U^^dfYnX5P`i&%7 zfTD>)vdcIC;npmJT5qOELkE-E122;Ut4--elUuzGElEHIA?v4eVT9sQo`mX%3_&#L zx!ON>Y{1$$Ek(RG*Yvonz5@Y%p%EB-30#`B`7N*$cliyS=e9C<88*v~Jh1tU0JgJv zr%FJk!7pj#9N3ome+G#5%~42KFL;El1P_sOaH4)9pssX1BZMqGVku<+b##~?s%}~i z{)@76plt7gX_U?YLa{QfGtIslJFr8ndFjA{NxKhnC$Yf|c}AYQ;=ZW%h7TdsWJsBn z&tz&l73Uv=&F+opsPHie+zhpxIGPt?7uAKlP>u){o>epb)rA1(>C^q6=Q${kJcm}s z2Q!w7;uIlw`~o3f*Y$>&vlMipv`*gaDimp)5DfJU3sn~X2~L%^J+dwRz}o>l!my>C z8+4o0mms_)Vhk9JLxd(gf%(BA_AD-{W~)8a`Sr56mg{**^kM)B49%bK2>jG|HwnhM zL{(iag*;#{Z{&~}ge)bn1?EI&T;eb`w=c8LL#oahEX|U9rdxc}0|Zv+fm`d~=W_K+ z`ULkw&r^5NGIxXW3CMnqUl>7OqR`RJ zO?CxGZ;JwoHj#uV+s@0C2F$)!gMnlUeQd@24=CAg@tiU2XR~~7!er1#@Ric&QJC+~ zU_KcY{aC0xqYFX>n~dGFo`pFUc76{q!1Yre1H5?eW&QaEk$xbZufczCvBzxI8`0`j z!$n@oGExPNk&&|cq=w869n%L_w%N8`QO&cad7`m^;>{;?zDoCajC606NhMstY6a6_ z4SYIfg?w?Ij-0``E-|wG%QZDGX{cHyJDf^8iKm&}G>`~dE%fE@c^e*ZCIB9HgAw{l zFoU-s_NfafQ)`8*w6KRmjuYBAtoN*Bp)qgV=^Lss?URz382FH`ib`|Or;gKe<>8 zFj_!P^0Y>e03d*(i7c~#0Pg1Mi_$2wd^FHx(oO^B(4@{JzaGVYJ#BXbQQx-}d|8hI zz-C0+JpLlHqv2qaY|Zu8i?j!3zGS_by%g2CR;~~}>dJITH?Qj142i3P;DtSiDq@k-`o=Ir&%ez09Sog zHFZ^;$M5%3e~D?LQqiVlubZvFE`Eh)2#h$UBts@cd%irZr8A})ScXKJNW$34+-lTe zKrq1<7tWnPI;9(OBDgpT!}a4VOvLl!TK|te(9sUxGSxpS2H08qQB{Go` zwsVKjoPHUxdQ5RR*)(J}kUlkvNzaUgsLDG&q%usg+(a5)7cNFLLWnHQwqch3oSVJv znSnJ#nCZUMdes#Dl<_NBpmvFj2Hg!R#cIBv(@IH4vG3Z|f%cF@r(K?y6(U;CsdW+! zcF)OjO`FIvH9+I>#k!KI9F$^mvf^=F*!R_jh15ll6>W(?+as;gg6gV?I>D+-g)I&g zKl2Al`JT8C;K1yXQNJl-G6yAq3*0ok;BW@%*kxPsBmkB74hJN~0CJe9Q$CNQO+f$f zJ(LyN&9HSSWsHeVyc9t_TP2!nN_BX=B2o?;TjVoPZcq@X8cY@GFU6H&WN7?5M_qag ztac#yRuQnNdO86TE?Ei8I`j~_H$?LzHLEiU$hbw;U&03ulsEid7(w*IhLy_=j0jT~ zcz&fXrf%5z4pbmQviR!-wq2|qGk2Q7EeyjIL4lSI_%ncL>ea^pC|u?W*(l{9r^_&r zQ4KcRsMaf8{G^Y!CCxus$h2w_OYWrxJHhe1w-ie+yI2L`S^pdZ7xe&t2w@$GG7 zg^QV$F_o?maH_!B3pBJ~rd!cSi_D`9Q_j8Riz;ZB18;ATPWd?A)`SAj*g+#z0u zd8aVm+)G`gxol7K9aus^+rItwFV?nxomv0AE#XcwAd4>MRlF;yuo%6>qNTOf#|x32ZWZq&t(UXF zVs7iz|CKr6qeB6e7H9VGx&##HLb?Z*&Gm`duxT!@qDs{;;;vDZ?U~#$Y7E2>LK>cs z`hX{i0j!RV)7eTMHm-2r^URw${BoqO4Cd1GLKG(?NJjswnxZ3=nfwfuG2MuO7@WdI zJQLD^WW;~aVEBvBiMD+ZUPd?#JiMB+C6jjxwz2ijEtJ^DEGiM zm?p<&TsZYYq1&eLf7%-(>4=alVuPz#hVu>hoAoA*$G#y?j}0vhV$-R859P*F`oRfp z2W6#Ta+Zj4pP=8U!ylk4VcT9MZa~yAFOA%`tzjovYFOp0DN+p~{o+0~AgM=2<6CAi zxs7CzT0UNfMbcw(ea*Z5=8m*WbSt8f*IR6=gKt{j|2<&2z6sXL zR8F|+wmWuiI;vhUbn-3qT>hM0>KTPV z4B3iTvS)WL4KDvXzEv@Fw)7*tclZo#NT;<^-=)_#t}AJyB#Ct)y*0UFCeV7??Y0un zE|w0&qAB&XQ3EJDObiA>&f}Ql8k(jEUoj(3vDRhH5bDCbtML$LW-hN#cg1+lTa41) z`wjcPTD!Xcw9Hm=;7>c;P!j=fFr0^DeVR-w-OeuHtq!A?dt3(0`DQjw##y{R95>=oDe@*izkM|b&m%m6q&B*_<_i&&G zcwhQKXp$b_=`h!O>ek7R5zcdA8>-5xmHwF(n4R9Da3HRRL7r9s%rM6;zE$UiAMkpt zhUuvh5WY;|b7%aCZcTm-2Q^^K42~KNiynEm9;NKKQ2UPB`mvV7C{9=t2y%b9KE1I8 zzfH8NAy1d-FrzpXm9-7plXcu@O!g?SJs>(B~J*~76IUz4qjc6*NsHZ&u7ogPZi90m9Zm?#YnB2C8XSs z^4uBPld0-Jz+VJmGegdDuqE?=vMOL^v|{Q~Z39NslmZ|YL+X3mawY?L~xo3QMpY)l|?#S$*;O~i>? zoR2SXi^qs%%87rd$*S=-_h-rXD$*fk>y6@>9tHc+ouow9t08Z2hI=~d2J8XP+Fbd! z?XI5X3agHPtSxIt^Yw5Iy$}y-J%QYx1e~rx+@U2{;C3!^xGWq{81aMpbm58Djg>Os z61@xc<)_V{uZ~Boq2xjh#O^tI>}k$1Z*sLggXWFHb9~Ff82Oj%vWXBuEsfRae=}}I zqdW2=ZNFm0-G?BTM6igLfTOQIUaB#?VMj(4G<(DnGM{P@o}%`avcc1%0bzX<(i=u) z1+Z$&U}G%lKt%0Z6+gP`rP!#RbWh-Va^0gYEHO8R5&a8#3&&V8wodJs_5>VeRyc+( zInMY5s0$MK#00VUMov{6>K8GCpKoXr$1%V$%HosD24+u~8wITUz44Y?cprH0Z9t^As6~hLF+tH0X(8HcQ0}Y6c@vZUg*VN?S@tr`7#sKR<5;+7!g)9J#Y=1*>54}Ius;}JWfd98?Uob~Z zWLk=Gx$*a#wTfT(d9ZQZpZo9)VTLOxCWwA?9E0%{_eDFpaIB7F&}wfTi&7&ODHT2mb(3AY}Hto z{DFJ{14|SCY-CO)yZRwsxJ46(ATm2j28iSZG%VMfl_`oMyX_H{?_ILXt&^uK7)7x}- z@(cRoI85Z4->h|yJBn=tm1|+NE+O*&6v{-hD)prpF%A*vxJer3p3H!;+JY5arLmx_JcMtstP$2^ai-9IY~)O?@Rlq4NWT#L{UV*!cgkoUrAYV;SXcl1WUDIr#xnrx|E*QEYHaA| zBe7h0%ufnqMU+s;rf*?g!~;a@Aeb85)Pu3|a@x`|Z8oMOYa@VwtNM@AJ_3Z|39QL= zZvN{0eJe&y+YW~a9zh*m<72I7KRScW4;YRNR0UILR{FK&HIR%;tX;x#+q5eOIR53x zmLU&E1&sxSDcEaCISe0(@Ps>$xAD?aVkmNbims-&l*vDo@eX2j*vWsGkjNG{c1hcS zUQ9bSLB?=UYPnMra??KtUYnF)QIhxXlh>k#^JkqmIqE~~zVW(8`FslR;X-r5E4 z$k1ZH)>gq`{xR9_=(J2Zz$v+rNr$h0S@9qkEX~WB!tI)x@=v4;{{^8a$2kGPZ({p$;RUdE#j}+tOj25)hVuKUo5IWN9&_JeTFdz0Nwl zq*WaozTZ)1J=;7SXO(y18yXGkZ(P)F{8h#lFZ;dN;EhF$)u~=sBv+cPC6JH#L|YS_ zRa+_Y=IfLHAu>8d5SZ~EL$=+gIt=`YUC{g>TTCG!T_qERpQcZ>Wq$y}u~ekDE-HD9YOM5&~vuxZ#+ zWvn)owx_po@0~&C`oc;n2h7+qS4qCTsbos&rFc}=_Ca}NYP6U?laqTU`X5u<)Z5|z ziI*KNg8yw#S#7oYW!Z$pv#RO(bh?#Pt;X!0-i?V3K@hpu-9C4oiDVU6@&zecQ1q4` z4a2o!k-CxTc55?#bA~hNGP|%lK5oA;hUfq zz(lWHP08rI|5Ozg@zP!R4^{eEbZFpR?g!y0M}X-~~nS(Sbsx9Fz)p6C-IkV)82 z@j}Z#J3Zn&38O(@O+N^OoUeImLjA2O0+o{nzL-H!qgkacz_%-J5XG+;P$CLkqn7$g zu|-#L1c!wHnnn7Ss`d#&aQ$H0YDTduk*kAyX*(36@>SUg(ouDuHbYmZ{F=80P-neL z?7-)+(YW;jRZX$-kt1apf3@qST*tn+;VfS~Q8Afv$3z@gr9AI7hkI=7#x7q zCO+2=w4Ve<4DWPjA>KELjnHl{>U}N)aiw#DzcJ_Ny2T0e*li9Uvg_6v;`hd`Ndskf zm*hbX#9?>eTPz3rZIU81wJE(NPC=Mv@@|kOVcCSkYnqkOsJE9TVQI^H_m8 zYWZaXg&C?T|F%9uKk+P54p}S^>n=TsLS%!m=SN4CR#HwkIAXnL>NE!$7|{O}E`;bKNR9vHcfHhe@&LQ;>Fhdi1#JrdM zS3!a>q_Z^oGmKyXJa7Uy61wDdE*Bj`QtpSxq4`36I)a`Ga+kom4TPAWC6=D@Y|M#B zGfp7q^;4T(OEI3vFB5nw1r7v+x(!Y1?6HfY0bqXI4O-els`warG{K~t>AaMT^g|wA zJ?Xcyfk3h{Y}6bhr&-wrVV?Sl0EE8~gGc-H12cxS9c zt?nawU2y0FN#Sf2s42N2CAiO0Ev7<{4gBNQuH8Ag^#!G^cL5_y!A>n;!(0WrazN-2 zD7)XjE)qmS@x;VDY+lR89U!gUaIdis73v010#JKU-qDA9F$*=!=m1W4IdzG3h9)rE zUZPXlD})JTAAca-IPoM#+LrDEs{D;W{@e5G`2MirVY@ZR@7$_X7P}_9 z2R|H*--B|+d=<4ZVPh^9BW;wj8D$+u+Ec`A2nzVyzxO2+P(pT46 zPW&3q38-JYCHWG?_$hgpPg3bKT;a5K8mzI$}_V+@xjCS;)pI`h=FX~3hs^N+y zCbX!n$ROYPH)^&$xnb5uHuc2v=aEh8Vk9>nus;q_Sj8_LVb4CEkx?mlIEhBX+E~CO zVXBoUlnj-rm~&3{g)6iE=14HEb_pD2Fsm`9*6&C{$(tBuWYO1MJ1B4H`-8ILv_E)6o}J^G9>5o(M5pL zj-bS%SA@XS1hG^Ia4NxWINo^^|1R%4EO8#;?V03B>p@)-`H2$f zh~7Yt{dx9RLl<_e0?@Gfh%Nn)I0}x>!wk0I6Cm z)lw&(mpOD50VxdP%7wB4%7!6bsJ)u%J1?)Q7m=LK233|dSBU9Idmq<0WsQe^)FZlJ z!Y#^52mma?>4L=U=~I>V*4>vK&c_-cZwnN#B0zOp_?AD%_2kF!$ST71lu6nkMAvtQ zt2a1`#RQv@7%%tRHLQ{43=N9*3~r8=KZzull$qc-S$~<#Ar+VsVH#-DmBk5If(#lm zY@;f5-(&_`c-_Wt4gwG0%)EH_7V^k;q_lXGcL_qrQK52rN;&B*nQQkO2O@}8Bs9F> z9VTF|R<~meLRC4%Q_4H#8!Wjj>Y#{O&YWMA9uUjv#DQ-B#R9y7_)p506ex@@l}kY3 z{ShpxKeALj_nvR#BZVp3&G*FT`^vxuFPH%g%SPz^dUJD@ru|l#_^~mk~Q8RFuze5lBs>Hd5AI^LvfIVNI zbvTdFwB=iJO89q|MkfYTuagYav3s+Vz#Lpz))|m$zTKB?M|rp)ljrLr{KQ>s9Tya- zg?E^p{i#*OvrH|wd)qz17oQvY`}D{IVjmHXl~y*|&fZ_>g_1u`ZJomR!j#~Ce?ft-fHsVC`o z%em}AAMDI6|8@zPf(($jQ#d7Q+mMmdIt+G&HGv81`Sx>I7~&}x8L+blNnp&LEJR1u z@sa(3{0pXo4K2^<&uR-4C0it8w|)4YsV`7kL>}P@3*I0FMM5dIe+GeTeldj8w9>Zk zzMw_8Zb)h-_EM3NBo3V^--YrRftWg;C$3=HZ!AbArT=emyv=D|CU2VMyQGbaA#~}W zOT?OoO4)_7Hws^qE)z(D+hW9gHdbx3KmLG|%W|eVMbHYIlCtj|&P~r=Ab?|^JYm1GeP~L;W&!Xkmi#MFQdf%g?!#Ivf3{c+nSyT5SP@SZtFr{jBanW@`ME=6 z9(sOA=Ho*ptwWi{zv^_q1&4%kiFXilrj;@4s%4|5En^` z10M0VigWG?Z%hOlzt$<-ltkVIsh)d5CAvc0Emxuw(fZHx7yLJa26v(kJnz_AMKZCmPP2;hc6a%A)iai{&~0zc01=F^b-!F)Y=qQUFQ% zhLR)NwmI8Jrc=aA=XUC$d+vW+%Fu*LH~l^fiK)BnjlWOOZ)*SvzKa9Gy16v>eTols z2H4Uu^l~dWmQorWZ86;0!b;rpppW~=7VaA;AAP@6=jsg5-mXiVV+7>gTVGO{7M3l@ zF>cwM`(u|<;RfdYyaP=DVxonrv^_jY!XVzfL09=3l1Q;}rK(K43(vhhe6{o)d^H=@ zY}nKe#Iv;}rUzrI-V&vp#(c>Z;f}A*u^zg>hxh;(fFM~x7kLE=X>2#-%~rIa^E$ z8#~bu53+2(f8RLAex_BkkI@2F)3ma3M3-*#y{-&okd1}uaX5IS-uZJ~j@|K%jr9Zi zs-7q#z&zi7YR*<%g`sg#({D3)>j+L#GKU8+UPuTt>_(3OAb}!@Yzpf*0M;jUH~4EF zfEdr~sh^86y(x; zuAPDiROL$LND`!t$ud8&S2vYBH`q&xdiR8ss0`#BZvKrUoE0!Wld6a-e30zf|9Ge} zC~_S{)^%co%YPw4hbr3?TU;eTci1UlzdI9(I?ByWIZpo??UuPpZEMTOy%s5!37_KI zt2%lqZhwM*azH7grB#|nay@u&!Y=xLbK)|yvFJ{&_!!VV189Kg)Y}|7e8=Gpb`pQf zQ^2l7oc2CF;cb-c_#d{xDKpW<3D1IN=t~Wa8JR{nB?-K0%o=EqR=>UO{v7Fj8Zx$c zftg}uJ4Ts{r3SE~lJ4y;@>u1suKQ83DR)(nhGv00pkULHL+tTfXq$g|z(-7vD-eAqLQF@_W{%hN`U8OEj78m{sDXp+Z%}hR}(My<_lqy(KREiW+jWun+3V)@# z!sUJmy7T=4W`Ii+SIRt=62k&U{P>z)n}es4@$+UKXrM6)a-Z45jt~YWy{@>I1Pt2i2Q?Nt+b@p*s5vU|sd@6?hSKgZ}i_3xvC z{rzw3_#k^rz7ksFN3UwqtAE&PfbALAN(uXi?X3kg4#(~YLlDy3CL9J{xT4DyGs8AI z>4I>i2C4y|nJC7ocekZZ2TO^GKkyqjhMaK;erf>O%63pS{2Y|dH=Lyf*O*c`5?*0F zA%}F@YtpZ{OT@>FuQejjJ42^qXKq49Z7xE{g&Y9kPitbEq5hd;k8%89AVHAK9Z-EN zZJ?IHl6|A=Ovm7Khv4B*mY!rH>XZ-0_IH+!ce}4gSOB<#ClQM_t5e0&t&u;hP&|)J zuty_A_#zGsxPk_FduR`dsU4Sl*53HE+BO7f=n94OQHDQf+XRoE%PgftO918E4ik+V zn>{zqJo4{fhC(pdq)@tOCloy@KLClaU{r5zrQudCQ&NzCf{{9$)4d#B0!=VVyr_98LRVbHZp47vx4gBr(8_N?^<#JohW{} zr+4dk@ObGEc{9#yI;KT0?damFJt-W^-ZpS{KsddI?Lr_g3cSa=y`c#99lVF-9e>)? zU>Gpf?Iw=gk6aXfU4o9sI#-S<%et39)j$M7_G~z#CA?rxP(IXw@Azbd?isXYvrEch zFPq5LQ<`|-q5VwOG&aE&NArcBKx}|lm$2lC6a)LC*tWN$5lEVS zj3&O>Zht2^Jn~wmH{^IHFDJ#bOgPM+W{;kzBMmL!cvAHQktiON#f`d>z1O#mS^d@cOe3jpZSoKmr0ls8sS1) zkLfLrBpX_jy}r^lrT6M;r#RXvpv0(9#4sgxLI#vB#UGq&qKSBCFXX}AVfh@w-DNv{r!BzA z8W2^x?Clb~dM!R8a=hF<`I@ys)KiCCD}&n1fPgxrJ9NSvt@$Ww9%t3(k;F!+=TF1`yR^!eaTF&_G<_crmKBr+FTYhFF_@t0=tjx$op7 z06V$U_28&Y3V;oh3b@dk6E_`4-tGCfK;ihaEvC#B3mECTUc;MH@C=E(`V*=XdSyBE^<$(2Lwv7G2YQ49S%rf!FKqKpgg7nf@29M+c3E`R5l< z*$PGrCQ%<7XzPiJ5oNH9bC=Zmkq|c42YkC~R}UCir}3R7!ue7CsN%E1RkCTiF}x^W z=+3X40lEIVs3D5z%=eNASd;e0^@KoH%L8x?6T>Sy8E(J?5Qlv=uSjC@FdXyh6fIfb zUt}5iZAukmcU2VDZ((c?UxQAGndF!atDP19>-g-$F+fzH5d zmMhU)&Rzfiix$JP(R43MEI|wIffxR!)8^Xv!vwY)XaQ$afyW(Q!741nyiFTqNdTR& z5(%0hog-0TD~Oo!`!FEZOQ zPuxKp*-(pAhXqTxIR33}{pS!DV^g(x^E|04@#pjvMuAkA>h0?<3FQ+yc>)ituOov?& zo8*fMWSDG+vDocHS|7Pmf*Jy^7HNKXQ?L*7bA$KOHRL{KzmQ$>*T1O1r7D4Y@3%$^ zo=b>Gz<{@jT`Ak=iz=YPNzKMtIa%uA`GO72`VFo^A~uj>o4lOjci(5abuqM(yzaW| z8@7RYK1|cOsdz|4ROv?#$Qs7^)0w zX=Fy73b|2)eVe6i0j1NKj*^X5fk!7iims`8YVguoE|!B&`;Q(}5Sy|KoPSAL$WLnE zaPixgHlY#gw!V*By4e7*%4#B!rr=~K6?IIbNi4A@N2qn^+vQyteRLw#gX9&7UV?h= z)WESI8bjh86P0i)%DEP9%fuN@(y$(JkQ(ZDB>cV18JB3G8`J*lYYY-k-q-TD%2VVv zOGF%iW}p2%>DMfCHE^W?IA1H`7D=JhOzPqDY(YWF)R}_aEmvdO_glQ6R`d44- zL+=T$_{6z8uDKPWn^-hcUumQITjnlA{mTEs0&&~fPeCR=*+wdmX8}FF`twj>JYtXl+bkK8!1|EqQ4%f zjS#4jfVTF8i-L3CfH&C+$?n!Xe*+21%Wwp8;V(glXXTK)DU?_03sK`cxICHo+ zg>`3dxDq1clR(^~9h-ZHT|$KLZQ{ej;UTVSx6=xq$0uVmP_G0_nB2dFjm5*7HBH`9 z3okZ$>CwwYafVgA7(yfY-hE2EZQ)UsHX8GYu)<@p`31!9ZcHbaB;miLNkY02ot_;uA%at4N`<-EVpj7IoT37)rSGeb{=>)k>y*`lta_@S@S=6(` zcH4}AT4+5_wSKQa*35XBxEfL}7mP(4KA%8zS$;_X$()xntr;X!!xrqw;@9dD+;~c^ znd_1Fn&3b(vj;F_K^@XGMvnj>fTW2mvw#5agAt#14++zyMW*$uWJPaWp(&GFIzMBp zo*d{NTpnCkG#r_vGklAgY^dGui_l%k<_@KKzmxvItS6E`8EMYm-rbi}B*Qv|T6xj8 z_MbrWOH4x*Nog2!V>@WBj>f$5UkNMtG1ak6x$ z&sRPjv&zj-P_?;&kRpZ`9$dff<$4D55{s7@_=E@`^P|P`OpB)JDBl`npW>1mUaqvN zZ8oTD1pcbe9xqU339sZYAW0!6vIX-|j~%ll@KzTW)WwT24%_tq^qK+pvg+iT9Q&V| zq!S2jGN3}(RFA*7cflxCJ3|sA-N6M#!g7wD(=e%5{^FUP2I)gxhfTccN*N0wyX*#S z>M5VQ_A2L_&~bRI)!aXLGuj1Cu$NU`2og?*Cy6>}#Ru3!KyW8V?|DGi(ih%cAnzs` zY`885&u-IK46fTuv5g299)WXF zdn2clp2sAPK5V+IH9m=mYZyF!QXrY0k|+MqI;$_okqdNTgV zlZL#7gT+9%#s1WH3SFgnk-BH?+1vp#WeGzopKxJA4x()$*E4mhOnXPvEV z9H(3_cB-}LW9faLUDiX^mqALiL{Fp=J*~n(I7!r&esiF(5Lvj!n%xpYC1T&zzW1mo zJaLtHTYiWxD2!Y0|3Z}WZELhRh2i_omKBYY0J2{#%lxF$SXVIjEtPsrRFd7Y-n%Kk ziw>XcpSm3(I78sm_@j@p)6|f1-9mW*#YV3Ks+A3l;gQrR7b6hYmuU~>Ry#m6e;DYR zl?kyQiAYEGQ-?=mH7WJ_Or<67ei{LYGWP~BXh;EboJNlT7+|1@EVF7t!g+wA3zLy%^zl?8rHGb3K0X8zIJ3U^EvxvU>eC+jEd{EUx0t zm|_)M{nr&EXya&=eY)B1qG5XG2QshhKHQ>8G$zbl+ur>c7JEUSh_U`u9L)k6x7tZ0 z>0x&bEO0kXa-IVId6KSr45P6TNl=qRc5{MaB}bdWf;$WtOWB#ofiGK~khz}JRgsfY z3A7v~i=HK|5kqF5ruuhFY>$bA)6T(jrk6HB@k`e!6^GqwoE4_rUP4%E5UvC?0^AMKKbowRU@1U`np6EuwhmLLxdPa^!9lMP# zL{7&=b?Ep&Jq3D>F(KcNW}jpku$?m4hebx+bohmlpfEhy$qCmRA7avEuPKzsW2+Zj z6882GcoI7(J`nRSxt?TQY{ot22E0U>vB-i0zpeG~C=(M%V$dX@bhS|l@T3o@MD630 z5kygw4vLX-vAjD8=N;#w+Q0m9k*Luz-=#a_IM`@%&lj`G=+8OISiI&cONn4B0bomL zK`rHfPcPs_=Spbs>GEL!bwrC(@9i-?qP_Z6jjp{hk}U$bwF{S<>ZqV_oI#H5WENI( z1KNpCcs3o)tSZ@I8YF>o&h2+QSB30#$W%B84dGr+)@#;EdoRy&Wq7)pPhVV~Yt7_u`AmLxY*}aS z04%spy2M56{ux%lBCxyBei@0<+K;@$*3;EZYX}+v=ra2eC^$h4L0%(AFk%P_NRrDu z0PfC@M*-gf{rCo$(NmJC1`704;g@g0FKmrkQVwB*DP$o&iW58)TwpUw#z8G@t444X zo%#^y;`ZX`@ObTX>YIV)V374gcYOD(z=%~v(IN(l=pno&;a@ua$i3i-!?p@T2IX=G z>Vf%lW3}liXBv@JNUh3uUmrI-0Op(-h>)5kPNJs>@nS#01UN;a(|0QO9$J$Mt`dFH zSkUE*==BVS+3tT`f zqw_b5dakS~jTdDpF1QiQIaAiH|InJ^Poowe75$tc)6fY*W+;Pn--2CD`f$TrVt`M5 zj@9E`}JEXMCuiv%GN-JC#M|JZy886~)~ zOQ?j)9zQeUbj@;YwNr=3vcMi+XG!5?xAoU>YZuH6OJc!#(aWAjJ|WUsY7H}rF=$cx zWlKVdRhU)s^kgVCmcb~#;v&JWJHx;hS}0Y0m`mu|#QE@A7NSGD{&GPz`dS*nM*%l_ z;bIm{Humsc32ybZ=yRp39V5>khy~y7WIs` zy-P!!@e?k>-H%|wFK~D;t^5~woR1YSC2%vFxvSU-#uodl=J;dy?nN(q<8%LxB089Z zo|l;*W4>3_rUBBu7*=!HE35Qd`!}_shKDGgtBQBg&n`CfLzE~o0487z|B^5^i?@9y z9TL$26kcSnH6cW=Sr^>MfCr$v0>#)mgSV22N(iKP)_QyS`itNAOrPB(i@kOg|E-ny zdS9A(4IqD-1z=vPp+=VOac}=R*?i7%7QOEbObmFuZ;p$HoAylaTtul2b>}oi4e{d> zgF#qvFb>GInxu)UkzAXn78+`L*>UZA0`V21-MMHgDCPZ25$!KICp-xl@X>dL>VDlS zkI7Ey7E-PM0K_zl+8RhAw0K*Pq9r}ppHm!0TP%Z}OqFd2%Fy(&YyP!r;%6|g!1Ip` zaLWiCvjZU;Jfat>BDw)(TGgDh$ufBI7>zCP9PV#GW*6rGnsiRI8#_SEjzdPoGqFm6 zsB(U9hH35J%EvLo6D$3wM{OiSUqv;A0-4NT0zZrS5NkiB;xyz&AZ?m*k&!xGXPxUy zF_7@<%LxFfY)_n1Y9l8%-HQ!9;)rrL9dZUY|8VRwBVDB z5@(RCb^j&nW24~@j=Wl6c4eHEEkR69HV9=-xWTH%-{Bgp$p>Sx+>cZ;M;VzzhPIqI zSU!FmGW`yr#Qt$_lmmrFt!T%U%T22$f3rn^PrVmy>BO>*4#rP5ttm~kT|_(hUDKfM=Cu z*QuJM_%zDeI+#y-t#XK&bZaV#5ZZK2z=_-QF*+Vvp5oACgFopCa0e}?OJpR)XH((? zg&S~+gw>37gz2BD)t{hnqB@Z=f{7tnk#v9jMl0IT>4)Jw2KL@%_s@bvRQu+m+>=1i z8S~?xX_XYv6heD3I;?HI z5BO06;zvG9+$T8?mg_;wax4I9kU;N$;;4bvT3P(_sMti2F{EgdEN{gy?Wq!ClH;X2yG~95jl$wSLX>6}x33!ZZg2s!9>YIl{#90bU?p%= z6K4(>QzN7UnrK)7T@NRHVo^_^_${8v&m5;gx0mZVPhsXiMk1sxPem@zI?zUd6%_o# zamMQB0dC=h@Pva7R32AP8YwI&kl%c8ag21?8xrD@{~ex3y=@%GyW&nQ0V2!{e^FES z?<{9Dk}q{LI%!S z9(9N+nk2A4<@ZmWRGF-K?^QfkqV$y&zz;|MEh{>gevu2?-Vj*7rMAMPvN*#qu&%u* z{R6VzJP`y8f)(`x*Pkw)zIoUJR&-y|<9ag7!IRDEIgA+pHk21$v z0Isb#E*k6joo|Dk^o8qV28NjEd+Q76L_FHUK+71$5Mp6 zWILL^<)+xF4}pK!OHmz% zr5C->b+m?Y5&C7#`_2Fwja_soXCxVnyDLgycOIMkuRUe0VGI;+fQGj-O%1j(=Laxs zU>btdMvnj>fTD>kvj70=DLNo>8i>e-)pk^^&VN3eJyKg%%szEE6x?f)4kgK?kJ)Du z8T=ev@;@hpx)b}yb)a}oo<&&!L|$DwncV0(e8%i=Fu1mp`MWGwH{*EO8J}-vvGd7q z!CWcltzlGLer2j|2nR00iS(buVW6p$OHTjV?iLt@e4l!h9}sKUdjuTC$7&8qioxkjzGm3RAC}Ev*!;i3Cu?oP` zl8}v>!mBPN&kw4WQ`yczD<1iDnnqa$gv04+_qOWa!1fc_q>16ZPDt&J7saXSo%bAv z7mJTXC0f(9+O)SBGohriS@DB%kGzhxf-M}wd%v0zoJez9;GKsM=y0skLIy(rzJaxI ztD7@ZQJBRyx3DGbHjH-FnustjHK@*tPW&kjc2}e>I|e7eb1q|M3kVSjHAxlA!gz>} z(J5-)moMYy=sU0TLBw8_f^K?f-UzNdRDAJ%v?+uHCck|?y5De~$0xy$*`s44?Vuz*SgqMjRH2oI{0VgS%= z_SPKdxzdXKu8==Ha8%l28$6;sOT-`d9aR-S0N^C$)LmUS^voI2ZFDu-(Nk{>!wq`? zi=#{|@eAXg6MfRxNRlIj;8zXTG81OI4LEE34P_mYtU zV@=WKIt7s6O3Ku+llQSbtv(u67l{2jBz(OUrgc^K|a zz~+~i@NmH6eSj6JVe^WRbADWpmo~spY8u@66ZmtIV)23~;Pg`d7*8+_(nvmmdHHMc zOJ>KgkFw`(RmI+c-Sez}c9*)0jsA7zNdFBwwU$6?fV(KG5YKsoToS{G+*OgDk|USf z=G7}uc?rnPp$EEC3GS%wkG}T%X_r#Bv-4E=c?*_)&MU)~BL->OD=GPCf(3dqx=M0c zIQ9}MCC}yi8z3bXUtl;tQwmuH$H2*~d7e;2(=kLltq&paOOQElu;1}FJ%8Y_6goE= z?HESPF7T`>v@-SvFm3@KQp84&03d)Oi7c~#0P43kjXfcxY(X*dbho8>>CZFjuPdko z$MQO1?5DWbj&w;`Y2-;D#Bic*x_ui4BvEP0hm6kJ#}VHm%8aY;(raI7d6is?dt?;@ z-JHk$b*cCZ)sfl4wJ;Y^vinc+(A9HPjAs~%DR){d{EYX%M!k?uh<`G)Q=!-Y;V9os zJWr|R9{6Ud-SNc}I={A7h#xo(#QwvSe<&X)xKQ)9K`N)`_WFvMQ@mJd30!8aZyT818GZrnO8Zh3c`!Cdnl^P&d;ds*$C5a2cOCORe2rqR`YhySUu47k(OTqDm_&wTk1CFxEs2c#|_>L2WON|(3H?NSg-3VH== zk{)eDG6f_DHI7lpVh>m#IH%p_NCD+KlFt{0ho`0uzyCQ}TE$8QW#Eiia34A-MV9=p z50xy+rRoTw#_ix_TS1alvTwz7bqdjs%IwY_OY)A7;+eOu(3_2(c9*M+cJ;2-Ll*Ap zq_CIk69>Tay^8ECA|7lZ@4|~6ep3qHjeKDNH#69rB%R5L@jzvSJ@yc~I_E-N*5&t1 zrx?X6Df3gwt^@Wp<4kB$D=wc}b&;KX&+yF?@x1pmyGv9|mnOl%g6ub%6$Y{^d^d0x zz~lBAB08Yh8&0+l9)1V$yu^A`?q?zP7a4#Sx_o-dcm9?eXM<*H5)Cm@$bRZ?$n0!O z+GzM@%vp752F@Ya*HyDurirwMk|8nKw(?=k@O5hYrb8hx>2SIGBDwS(KBQb9iKmFU zL#*orWDakCl<(M<%p?Ia!L|$?S?A4mTZs)Zpszj(9qLn;(okw%0_=4Z7a_kR$uUGm z__RFV_JF-nds*m4E3*#nfaI-j{NIG^gD>ia5}Hwzt~fzn2|0JLc`%p6XK#K~0oK@y zm{RYdc&vS=D$u=KfGQdRxH63nC^`Tpxz-~`aAJxID3Z%O0QVBCJHpuAKwcK$B3+C73V9e+@|EXot>ti-8WON9}6V)RYvx8*qPnyK@b_F?h(`{qrnWj7>u zeg?;7e%lYtb0r>rWtTZ(E!v3!^#Ww^m^NUrt-V~I-99^1^)DoG7q8-f(nQIV+rZGg zg($f{uge`7X4Q${muRGCMr+rRGK(lX4rA<9Xm;K+HP#WrC4tq|aBBfN@GnssPAc_) z=tXmZDj95Wx7Ny#0PC>8WjCVU&UND8UAU-tGsa{++?!{ zg0-R~&~Hgu4YMq?QJ=$FTuk`1FupjPmmyo3w$4Rr(dHF*bHh_w(0;QjIcESmi(&2q z1&MgLL%~gyl$aKq$y9bZVT-#s^%#{aw3T4;(2`K-O=Vl;ViEo~S>fGMNCd=r2^#Z+ zn_P#r3jeVY<*&Ob>imHImIYf1lNa{7h{HAd8GDi~Mx3_jKx0EH2P%W(lufxYgkAjdct|dMk4UsR72H#F zZtRP+@uBv~V{CY6mb7Co=i*^?pnMj~Ki+P&Oi?2L1{5vaEr=$zfE=D{I==lqmqS?Z zsIp)6!du3Dy$ev-X9TJL6zuOI?U%KT{!#ya-uC6Vhf`%{58(*e@TVAq)*eCPk&4s# zU7p2#Fn4j2nc$~q0${t$=TjDTZ)bX+zS1oW0KRHeL;1x)z@#5yfAA1M zb~XcaA#6{aecFz@eazvz%KP(N*+U9xY5E9<2~ft+jCsUk7|6!iotlP; zz@Ry-wTLj95C*IC)#N|@Qm5Ev$%kFx>~vZxa(Ox?HjYY}Su`duA=$g+K{{XvP$jTR z9^GBO<2BvbC9B0*ndVnh3>OJLaa?I_bw?OPQR8>1w}vgvH_&A^VhWW+i|xyYI?ABK za{Fl7Vh1@^q}T=976By5D-DPJ?fn|?@&5|PATupV0@fbVa9wQsi6vfWdaAzo)0S?2 z4i|-t@mJsEQs0G!tHVgT8$dj3A;ud$c=249D92|0*(6Nq8^QD>VxQqP4j$sn%ruVz z6MKZx7VwInTp$uAbSN{73pW!Ks8h!Zab*`tL;hsrgDjusY`d zrpNC9rK4B|Ln{C9KDBiX&Dbl7xUPU}8{1o2;-|Ww!|~1`QM#GNId(?Br{JCUnBuZ% zDx$bRnNT|%a2|Ggg~6iOGmrvr4$e-D?8Z)ZNL;E}CebzdqNDa2`#=Yx4fTTjk>uTn zR&g3%dVoxI$+@h-k~jZk#{F^ticWE+pZ?wnoky=1=RM=9*|F-L+b2jw@hxMhH% zp_NkiWOQhYvWs1i;}mhH{q%m0RPOTEP$COhtQy3%#!T0XC3I9hx>kza3gEC!pcR&` z|9Gf%(L^^>kUhkYGqM$2WAwSI{=zy|*glYiJJ;`gLc!(VG`M?dTm2ZchS7FwjW|D5 z?%qhpfB28Lt$^@4K|>Zt1W^An2C*Y^kEz?fq^jZc>4hVxTb6)Gycv`4D~Cbk&6dom zbFCe_zy&@XQQOY=27g6)J*fa^K$ySp?`u%6Nr<*VyO3l z|8tgLPF%=pPO&~t8>8mRcO zLps(DzFtNSFEDIK_lyX)p`4ZfitXfZA({RS%oF*P#vCBSdO#*97b&|NeVQ(`;NNtK zk_H!SJ+$^FuLD?G1f3XzqA6f9C@<_*)~4O0os1APMnoix=uAlC4E$2jGnro4uBo%H z_w!P^2teMFCRcmJ#vTey2|ka`1J90wZ|Hw??K1MV)RCAi?@G@(g)*63xV1YVfXv1qze}aFz?Fw{uY2250KCmaIiJIwcTCqK>sk049-o1Wkid9d-N9`nZhCzo7~8)G7OaD>u4*d$-Nmn z1vLj}Lfx^-yP{Z4c=NWR=tzj`4`y5@<+6We3!Ejp1ivCM$WpZ?#F%yO*1OTx94)_o zh+@CY!I(_y7dBeO9A8dRX(7zzDZ%t;eM8lA!y>;v1Z#A>-3AXZikl~GJ0X)otAce(z>Qn`_=LkhL5H`4L5fTjajnv|gUuJ4qX?b4Xo<$TWMn z(^57!iMz4AwDl>8#@!9YiuP)Va@i{7y{(H18}V*)X0rZ)#RLkTg}16z)~Iq8Hx;vH0q@Y z6cDFIyEr}C zZ3-rVlHYF1*Qb381gwKg-6o&>4==N*>2u1y)tf?cRQHvC%Zury8UcthoCh#=Py({NMvnj{n4*a+vw#5JTPe0!Kv49Z zEvH{V4_m{Gp@7L_)7Y#pL3hLt~0vOLyW2R7bnEE%qr0wnz#zmC5cA~4tt~~8J zFZ?*i7HHbEs#f;{T*0wQ6bqE_;h>pEtP-FHfxX<*h)t}7*crp*=dDg$u3T=q(D+8g zp&>a$G1yE|cY}K;wgk)i0s+jb<8k>VTP{HLd{(ED$Azd8Ikc3FM%57^qDc8nSWrd3 zOfyIiPB7wjoeDu)i#9n?pdXXeg*s}0{IK(0VUcnz+%eJc`OP-P7F>Ux0PbDYi7IHk zQTN03PUX8WpU=RcYY<~y_Ri|`>28%CBKqg>F4+$@jS|98ANa4)DKl)5h2O`J$K`M~ zRQ(>7Ptn(-x6*$W#QZc94;UrqqL_J0O;j*4C&ChlsU))%f6nySWr*zqh(ipn|8X!A zN%`J5d}Be(Sl(w!@efNZADhi(xt(O!w;cu;P-Ub}?0WuG^%nS;!`!+IzSYcGowrBG zfmHM^FaL~>DpPw-;ka|4a8}uw#4S8fwheg>SLoEI!o<0_}g|@`7&Zk(_r&{3%siU(y!_iW7A( z!4!5EQ(Rn56sec8CDxXx%Cti9n8hr0^i{Gjko|3iY2zQjJPTaac?<4rrg5%4R)BKo zW99(^NUhJ=%Esh7t(rO<_#H7rEB_ZOQ9;qw3=SnPeN!ouGs?t}2$l!gD+!69QiL#Mq78q^*%B zN*UE)SCX-Dg4TJ?EUM{rUeMs#sI%}I=rF$yZ&9P5O{#3cHM>4V zqW!G*!IP{fdHGjPaGVWHGVUB_jqk`VCSAJ-DtP4N^}@o+k6KI!vvSVxwb=80z{9So z)g~ebYjdIrZ{|;X@eK}LGJ$a11Oh zBSkP_Wssnn%K{4Eth#^z&YD{pIKA>JBq-ACJkx$r&`ROu__=rpuYn$%CJszVp)J1A zTLh}80dAmlaE6W_IhX82Jv@60ykv1v=E`4bkdXTUTNnGYJ@!*j~SG{f*Y z<%*?vH{){&WQUdW&=*Ty?Am7^#TqSQ|LqTVyBNJdLW601a}zi&x~#<^fUCghSGiQt zOXz8K-v4x8_EyjCgsjB@koaB-Vjv?}Y^nkHdc37Keo?>CpE@?$YbqXBo4i?X!}0gS zkxL+Dgb{M2l`(yYs^~+1vdQsOCj{oR#SILx=r?(hl|WSCzC30%qR8IuEKZO&6~zkS z6t{n_FcwM;YWXsR1Oy4&23CJWWVXxMIf5o~OYE8#@|AF0@wq@=8u3{J(-Me~Q8BNS zcXwW;7rHHkMTu@lnSP6ubvV4$Povp2^A#}BsLhbdA6nMWkzPoAdD0aGx~1t||D$iE zJ`JD6E9J!Qny{Rjl;@URIwQC-V>HqPMd>S%MgKR7gKX)#3pgaP?cdjg8ILxz-+=q> zC{EGxY!CBA`*^_}*`|teDnLUwNqM_Pu~8VC0Qmk#n12%d?{}Zk4EwgEJhlzbc6HgSDEw=sh}&8 zCB^Gu5cNVX^2diy{KIM-)INsu@X$r2-_0ULt9ZKcT#yU6(^qj7kKf4HUG9FeatJC) zM{yB%a8mxkhFzg*im-0bTx!ML5%sa`V5Jn|ko*=;%I8d_-++Uo^2hs|TiLQ*S!H#L zfci%6+}>zh);AC7u!T*lxy)&t2~*dVO5WuSd5!`VEX(GkgpkJ*FrRJZxCjj>?42j> zgc_i2^@Nfu{StMl zbp9nF)Nykm6}*r2$7KWil`9N&H6Y1z_feCRfC;yED=E_erlP1RS$h#5b4+!gM7%*1 zXvXpG@aW*U^0#IpB0?ZSMrKnxvJ=pg)Szx2*a2{2XgXA_Ejw3GFMz@j^lg_}-*jx* zjuHV1inO16Fh-4+BWW@`W(5r94q2t+AZyHft!vn0eUd zpb7GaHPt!FWcwxK;vqulA={Sfzh+jDmMHCh^IHi2w?A9zoZ5DAcj*iH(vL{rZ5g=Q z7~#byMm1dA&r5|j3n9QwD3~o0H&V!Z*`1Y21WoM->{8gT@;eiW>M^=JlL6^VEa2(S zbZrtu7H=~ve%myGwt+xPdEw$$quR0mR1;z?=#BXP3;cHkoHDtM9Y@pS9eUFe(tmN7 zZRc<$NNA9Ma1lggk55fy7O5c5HyP;U-nC-8;?W$)GC?slq22WJ%LIa2H-4;P9d zBgp8znqH5DKr&?r*flE$NxIYcJ+K5Ay#?nO2thd-30Iu>8A=?$ z-jPe>5=5s91GHYufumyWCHtPQAfmF~fm|$Q*3CX<+WVpQmfhMW1JO znf;Pt9=c)a4~3)=9(n}%5G8*s1BJmd?uf>`Gex6oD4EjKf~%k6I^m_uruOtJ&@-mR zf8tKA90?c4kdC5!R1mhw)o!anivF0M3J8C6Fy%`n)sEfZTdFw`gRyC(+PVZ~9Dr!+ zFEt(+&k2i6hP?Xii<*Xu4;llEv=63RKZC#FStHf&)pw9`LnMqy1*3&I3X}nCQ9A*g zP2XnJqDqx~S)MoWEe|%KL8zot%X{kl91^6U0Z{LiPt{r^pKG;VBralpr&ghN-|`Dvs48b<6g%opPkR} zO+?JcP}}AmlUCa^zDU|PN%LpOS`}48&kE6>*Yt}kEh3=RG~=V*e{M_MIP?=bqYRXf zcIe*iJh^`YBqPAV&^vvqfeP9e2vYl2SP-pHZ4S@XcuruJJzM*EaF22jxz~*{0oCs) zmwi+?>p6WQpHv4jxJh^^7y%2wS7QmsdLN>G<9)(IPh5qS5b1f#$v2QLsQM!igsAv~ z*{^LSt}AebLliUVjJ#%m7^itIS@zM`l)lkM+0XSZ#7F z=h`SMh~Ma!4g&!nH)(maWiAxpY#Os7w%CAff=lFa&wfB)xisuQZ?;bA9Z+$x8%^$s z&7u8F3kj)o!tjd{Ohq`G{fegtnK{7?+0rG8)Jt+pcHP8NYzAiMxmR^!jFmVw!ulytto~y zIvbRWOuP8QymsdL`2{F|T3BdatxzoAY~4Gp7n%@Tt`))A9Kf`Nt!kU5MG|c61C@)b zRD#yFlB@Pm3_>vznt>jbP{z`OGL)L!4R7r}spciYtTN2!+m|tNa)UqYa{1{jkjby& zb!E<)WJ^)B;ouCfpdauT1JW z*rwo2djIrx5+=gRsGf?*9FC4El^qS@QO|gRo)HFM3Wd3PI=GYiieZgEG!WC(DN0IZ z4l)$Daq}!ZoS_u}Mw&zOGe_cCaAum!uQ%`0&KD#aELg`Q=2d^vvH6GOZTQGbKGGhy z7`H>XO8?-QVztb?I8RslzI?<@qqQo}z6;MVWkzQS$4*fn*o9P`%6_uJGfAk+Zn~!{ zOnLO4cfM~Q{`VA?b_;!TNckKSM4++4YsJIGmK^c%N6~!0SO9d@n9(06(=p25=PENZ z!(fQ^zk$BeJvbUR%j?X>c9TGQWOllY?>b}KmRe|fV&(EA^DJSaehi0xd zq`e@d$W$`bf3k|;r(4)Q&|a~9czyQz`dluX_lc@~UgK-aPVOd^0}uz2V6dz3$bsdB zpfOeye50@%)qryW5jY!}AB<{^8C>6Ol>-1Qe*)@=$1wT`mi<0UG-@&x`W#zCIv4YG8!4BYxl3V35X9EQlR^zE-PAEn|`4O zzAkAtyI@h6_l9@{4bu$1>V{o|BVOa$WMZ4}O#-Di4VLitQTiNyQwocmoiv65V03%c zt6`R;-9z?_EqknkJqVzhfYsh!7Cg3vCT};+0Qk61<-4-_xs&*c8{^?}QR=Y1E$!W2 zz|k`ZK7gAA$ecSl;%)RM4KK!0ia1N~PZ9cWdC<{HViOa`&9XfQJ`A5=9j~h7D2o{% zs(536H_e2l0k2U@g@wbNosKfvT(U$Ewl0ZbBWWcd2|rXzWXoct7!Cxr zAw}L}2%MN6!cNpmHz%A!A49v6!?rQYU6vNS=W+npIyU|d<8&L~boJr3B#E|+z~H(7 zhY;7}P(r>U**=5$pbeBUuUfTOX-QnB)+g3V#0icY+ZSSMyjm&W=?|jJkHO(wD2Ai2 z9ZrSBAo`akFbO!6wKIx9>LLVkJ6OEZI2)I?>?!)whzK}fai@}Sv zCXP{{aJ+`KnY78)3YS8u9ew)fz~(+a4f_`AGngE2d7YvI%FY9lQH!VwL8uRF@-s2z zH!B2$w^arS+l5*?UFLqH~gFoz7pS!LN2wDZ)P#_&_P#R z&W$BF#LO?U6jT<}6qDU`=p@N?L5K!Q{I_{@LgK{&^DCWc1)5}_h-qIAw;iQZ!$U|s z#ifk8N0q2O-TNC;dU`YTq@KLf?M$!WmU-5~@0YW9Td8a#qU0iHwvlorM-4WpmEZZy zv=@uXYCk@7?Jf*n?qOmY6t2I=7obC1u$kxvYXzEdMtGAeq_aG)0SPl?*L|w+@BxJr z=KdE}qg7<_{H&-M7@ddQt^9j{VywWKbaZrDEC|o^`8?{UKeUJ6cMPJAg|I*t8kp&N z+xeIW-rio6ItlI^r?eKp8`8#>)FO9?wO7K+9k~-v8Ew{@%LOA~EDBBFws_(9cI&9^ z`g)qfarOJY14kDJI;m&Q3~dT7D&ZwCEDblQnyYx!2aOEegeSh>&-zicEv6ng6g0tWl7=DxS3^ZCu>S}LY)h(R@0 zuqfnMeCa2Av~@&&6m6#1D$#l*$C+wDD#-CWFZs)5*P`3DW}yFsw3 z(n~WRqaD#M``7&4~f^Yi*SK!CIB^sn-1pA zWPS~LJNl9h(U@4k{YwvS&^;+>?{Gm&97nF?d*m(mK%0uofeEhX-?<84=HJm)b(ggbCpd6m;vewmLBRGvaJ#S*oB)*u9{z4|(O8AHnsKN;XhaKgAD=k@5J< zPOH#vh_pg-^fZBjl8q?>e95)!-|Y{rE)$SGL!ZaBw6B(tUaL@UZgL5pmG@{pW68Vt zAENy}O-@GdU3^;elI=N|7q68WX`z>`&3jjz~CjI|GvTYO z1Hcv}_Q4Tx6_MCgOM0h_oFm=zS(Bm2I2a9L^KZIqEwCAUQU|L)-h|i_vw@LzQxPDx@e5dBvyjL29P!H%00(*UXi;@AjF+Y?lflU z(d!w&*zdm!$XO2iME)4@;Q1}+$l-nY*TDdLYL$nLvdEw)Im5hkW0XjSq(iAy%Q^SS+Ip7f62SYelk}#GDq_0;x-Hz9qBeaDE69JGY##s*-5lMWSG5+W_-n@#aE9 z63={i$2_$2Tw|3;jD}%4`J@Z9vTsaTp`LV7SmZXlKA+6<>fds_?2rad@M=Na;%CU! zlF9!u6?|E>6!hy$r#S>X+YQb6;=vZW@X8JieQxE-%#!AV26u%QTjI5R@m;<+Tcv$% zqDCtsw1z3Hov_)0hY)!fL>hOWKUht_5*^^R{W(Iwaq`1375jsC6Ba zuXgokEGBK5VFJZza}MCWy89f{oeJwSS%wy-bG$kH`axQYdp?o1Y%}xXDkJ!bee^G; zD#be3KC0XlG#>mHp9;x+G_2~5Uhr}90~KY49Nlk$h_`e&%mwv%6Gb%Fg!C@qfHBF>b* z4eI9A>Xl#ND6J1E@>H-;<|D(r&)e6gm;I%lEKu7Osjndni=~Uk8h&Lyf*T| z!U=ggUfAKM7nWfWT&MUc<8j5N@&PweN{Hby`HKIbK2OI%-@$3aJM0*b5_ju*pKak~ z#~5kT9cHw~2Jv~Eoa$1xS^9OQmuolPT4A5IonqktP_nHAUs&Sj!6WJ_+epe8A7}10uNb1QEfHQAXRj%`zXy-7$Zx6Bwgt$r_ zd?Ud2pB+|;)z5`VDut*j)-t}%|e~FAkpNa_5LOV1R+iXw)UfwtYLiTi-Fisik5$m&!B`z@lvh$X%9VYh@ zP&ygm--TcX+3LyU`-+qV6X|Q(`wlvd4>S9I z&^@64}Z)?Uc zGL*}M(du0`DLhCiSYM zYmF{opb0?@edQ?tw_=mGV!#}EMZcg;>@Df(DQ)&pcC=^3_v54PYTK(X4y!o|a)T1n zIPa5J&lANBmjf^zSkrgF>7hZ(ufq1MuEP?bn2UA{`evz*FXj%Vz$lWSljhpa|c*9u-^LN4DpnhE%<)dzZLf>Yv+|h zABZ@Q*s~6)#M4t@9`hc=wp2~%wV3SUcSOcy)_G6Q*mOd3UuNkEKM0huC1Af$&GoA6 z$tM2Tk*a0=5?&<+m#Snihc8-|pZOvS{l`;pFp8r`T1>#K+aO>ie!_3c;&arpp)s+h zvjvF+17o?;?u8q8P5O-SO`1R~nL?BB-4Eyas)`($C&SSwj$5b}lo@21!B}`i{OELm zr!6Z(y(E&y$Cb*D_}bycLK~xq$Z@0O8)}EUqN)s}aWEWSB=Q~`p3y?NU*&4dWBtvzgSMSI$^U$J5OGN1BOSTga6|&#COyb*%RTm7X%~S7dgn4`yUhHsI>36zLWVAre8E3e z<^5=3}Q;boNtJP0OEqm@lr#EMz$NvSm@Mg<47 z!d|2b%>zB^A<#9Ye2FWNGGd>vfqPRFxSb2>0h`q20>9I78O~9HDNsg9C{GBQRQyIA_MwJ9R6gj5%Q@S8)fY5i}UNe}>%$Hb89Bz&Zk{_Y#{84)!AK+GtW&0LjIdvQdg9 z2XeiEhWFzrCW_0Y}7ov@oGfTk25=akSm{~STw3%MkI*!R1OKwd7s~k`p#G#Hn46Dhj9v`q-IYu{|CL^^X(Ah+t>5?^X#iuFZ z#VpJ>9E2LU^l|ewgS>7B2Ug@%oNt5q0zK zFPb>2_|yjy$zA+A6pg)$=0@P5BOMNNDgM(nj*{X-lF%@*c`l-udh8rfRA+9N zHo2YU9@qMB$KD^|C&QxV$w1DMB4zRE54wx0%p0#Q%bID@zUaz&*sJ!PMNJ zkBDW@P5{Bj^9JL7R216clBH&9XH`~ZM{Z(g3+lL>s_Kaf6jsq$zi=OyJ$g^vr73Two2x?7Ld+41WS`NFLd7*=|b86LhZ52<*m4 zt|E-rY#s4-->Wj`%6iH~anDiKGE`^pMkvww|D^S)%A`yqPC5J0PAcCOB|x865|al9 znIVhMSYvoy*_9S}numASZKpG7&cVL_Duv_c-MZrU2 zyh4R3s(pvt6Wi6)jUczgcyG#d;q()KA3)$#v=Iq02X`HE+E`+GO{?&PAL|zp%9jU- z1*1SgsNY)Zz3{f&Q6oBCnDOcIdHtG8Q)S0;;@MEjr2>IP-RTZID4ynLun`KDNo^ar zz5{POe06$6))AjCqe#a%6;5yubQ~5fa}B?ho}fb}_(bL>$V7EbKFFCv7e3-dZ>`=E zeO*GkZ)yp~ob9Y2WOcNO` zvks_c1ddnA&et+#&By#)rzwk&uZ=7fgmMO zxx8mCx<_d$q*>ZSa-8C~wqO|889J~09dkF4Vu|vYPmT*{=HbE|p;h~CT$vP@l*!o+ zuvOto!bfKf5i?edw=pL%IKdSS=Wk(PO>_;C3Rr3B2dHX@xdjlKad< zibKu$`_7lyFSqMg-pYYT+o8s7zrRwQzy#{wp1jBdg|nyaydz`0MIhA)UWiA06nu#k z_-oA>%Tm*~f}RbDWOAlp08_c*0Fq(3-(eVWk#L;%%=kP2rQr@JYf+SG@}m?CONHs5!u#?HM%CwK`6qCBZKM98<hceqhg#jtp;xfILG_LAt_&u@|^2Kw$QT9%XluV4_-}PNSaE2d~bJLtxpslO=jtqiBkfXN8-1Ww2tn`L(Pr1HEE17|KUPclB+ z1DP+9w^(rovycGYnU>-W&}R3&7SILB)a*q#zXTOL{b->~U8P#j)oy27WmSr~6n5mE z^ZC@Es>?lW!7xT4*S;<|ttu0I5mRQpx`czGpzWb|RfCE})vfZ_sXNJtuVYG;y2&$j zJ_YQbM8Wsk#T;$Ecke2e$B~87yQ_YcoUENj-q+c@T zVd<)?fvre&cvLnqTaHxT`#$i|vG6=whISb(E~*q&<&41Tg_2Vr9y446P(7HPcJiT9 z70ELG3!na1M>VKjG$+B#wR892pZ(Li|e3Dj?N~m#H4l40VZ0cQP z^}+K0G*D`!Deb$Ij?`MDEWYcCjS{}V>hkEC6%G!P1h0PPtE?-CgZHaX`WM11xFr_q4bGpU?~Rukj2G5-iyazMh>)U7;rK3G zW~41QHXfB`G#0S{+(v!5f!3rQUfOfdfeH|KslB$F$X8+D$^$d9WM~HUa#ca zp~`mmXoa#MKF3Twh*hC@?mbPe;m6oO9A5f*RADd>B<@Cd>KUYXzJ(GV^$75G_7S8r z(e}A&Ek{Qu(g38j?`bWM@AssSX2auBh|!H}@Hh(GTZk2Y(2tD>;5KLh9DoR2Ls9FW z*bBE~#+B2S2+kOX`TJCNLx~k2#vjFu5SGzW#V9$Bo}L4@2EgR80Lfvd$GS)wk`iB~ zbK$FK`w)TVSfH?QUG;`Rqg66qyF)2r{8Hu7I>A+pkkJnp*LaZ*{KQh9 zTgPj1{+DoJq)zaYWc2|W`X>VwQW;MWj z;P^3c`|l-`PH`(QnU$Xk?@3{GB7c6>9#@|!;9c9RPWL$e@G6;_?Vmp$%qsSIeR}lF zVr;5DsdaRHn%Vb=fJ0e|$?iW`be9M~wuoCPLGI&C77D_^tA!#KIBJ?Jd7yB^eT8YQ z4iB{1Oe7SP>Yus_kY)q5IoB3&~S*tU+=p6&j!A6P^`QJw{hXy1Eh zGFGuj%Kf`E1mm^B4R3e03fKPN?a8mJIr(HCpEB~KI!ha}gc|{=eCauGhg6_MhrV1d z4cOU17EKp>CP(N-kT*(VBr~t%OtZhFUei_m^|5L;u82`oZXUO=U#droRpL@uvD>J_lu;1Pacnsa*oOR8i5O z*z4s$qDa64KYuKz(uYI?q*Hos)SFEX{pHp{CPqf@>)ZEclrqBk=OX9I{vGu+J#B(U z7!gmxIMrLVP3BE?DbxoL{1=#%9Vu4q47y^oa$~s?cqpNpV&YiCot{a3mSDfP+P4q! zWOu_AS{MY>dE>_U0XmPlbsd)x0&y{6A@==VyoZ#6Wgb9h`^?Z8Li-A`L8Y z94S-MzNVVF&dJR(710h0q9sb~?}Y(t>J?5^>=1_k38vc!{s`NBf{mOK0nGB7DeSl% zjbEPY1T`MpE@YgY z?P%rR3$q)UyTf;3#wj>#6l^mj>}6=NQ5pc?A4_mf zpEB)3oS>E@{po0PC>h7Z*H~3RQz6}b*}n&AYzS-fc~)jS-fe{pd6XrXeW1QyB9~Da zi{I=ZLL1#uk4O=LVUXA z7Za5ZV4G}wYMc*WBa_XJI>zp`jtc;dBPDc=%3rwL8rejNGaWY0pE z5rAB)iT!?Y0Wm1}CbtJa&7baw#kW7oK5hCBTwaStC^@pvXU;-TwF3b;fHLquC`urV zEH9%*5MpVVc$&)s2mme%zySI1diu39JyK(_4r?A^)oVY>!3`1zuq8SVKjmnN_5zrr zjMKbrzvc?42%d)E;dpS)`z+Z((?)wfKgORc7dai$`QsjgLvf?^#%3#+!?^ca zMlV1drX)ISO5OL9XCP%j4^S=^RHI4SQYK*zJ1tpO2}7_K)bu$Q(_DGXZbV1}?Rb>8 zf)g{h-(i1E%>t_0R}^!2TTCnn>tAI2u9PdveYiQfL^K6sPeiEQc`^$HE%tHd;5XCuF--|Mc(Z3d(xMW$PBN>oD|i|NogSB;!s zq4U+@+TTJ7hvc1RpLLBOkx_irDz}+LkmAQ+>ez0VxFYN263y><8x{GSBp_4KG7V$j zCZdtJFsnyvDBHC4u_#6`Wlk0hjtWo9A)CE-4$iZEv)ElA76BJ#5S&c#iF=NI9Tnm>b`jOwORGnb|AkjE=EYdb@$iXV_9e+@hN6+9 z!6zkPw~W{l2x*Dl{(lL#cLE_mcT|e)tO(D*(5SuaSpi9jA%~sJyqr(Y@pXM$eD;V< zPBo8rqRp<>suby2q^GD$zp(M71Jx&h@Mfat+QpG-*y@PCw4NDdv9ZG+v>5VFxlrti z7ziTxMyF9nb+y-8fl5P4{EU2tUVvW8(DIZWO6Z6>DJA)CS=A7fpWTcvV#_M>#l@qfEW8 z7*WT-n?!d{hVx-YD}^tb#^F9ZTV%~(sjpG=B1thiSsVk8XI)`jSG{?xQlIh(J{1`1 zBI7$0X>Sd;i+e({^>}_)#Az>9Sd|nz8_bRi4VlwMiP2Z`?U{KGUapy83U7lc`eLb& z$B)NYwbtK|X(8=aB5(N*%?}KOVN2y1!1)7sqGaPBO6s{7A}onSIN5RvH-Sg}n`_(U z#8fB-jLS|s6G4phSZ0X`<5$F-xAoc-Qnj&wPSGRHk`l8;nr$11=@5#A^6Do-hB=Thr`s_^EQvJmvn-?vu4tHnHB?w`Z7B_QqP zjyY~E3JckMBeA@RC(v4$AF3oNJvWLXwzA16jZ8!jSPS);<=JDXUAFTerm5b-+AlQt zd@~V&aFkvc+#x8a3F99D59+f9lT}!Hm^06vOc2OJ7!H z)*2Xc*&SmNvrA(0Q9D^WJzJj#TEUslgwz8pM-Ji2dzns5Dl<&a+67a`% zOBx6ca`NAP%f%MatCAPflgme(z!=+eGMcw z7eItY=wNj&+0&wyTp1QaC_vIwNlCKq#omby4u}XP;NYN?Oz*4e3uC7z!%!;Pc_?w5G!bH?UHzJkM5r|L zp^`6Q274qxnrG!uvHEbcZ1fdiscdp-cp|@QKa0nhDX1_!hJLFNt1=KqHAT(D z`%hf8oj16;!pK9|>PtpyFQ63L?75%oS-QfA`{z}AhhvDWqqkEN=i3RG*Ni|0DEA7G zQaDGXf_XZLeL}6ZOcUoHMRs|it#`cphpGwA@IEN;+}xsvX$W4ia%&y8Dwr-E$tC=z zEu*)TLUNe6DvjTKtUXn@{TYMr^>G0D?_K)j7D_AA%g}VkuK)e0V7D7KUAlEtvJncu zSlT>!Q7R4$A(~tow31pGdlc!yNT^=uR(;c4;pI}9{VF|=wAu`(3n0MEwIxRnmTthY zE>i#fMbO;Kj%~hKrI`onkB{N`jiMp4f*E{gnV>!Y_{b}Qg~uTu4YBdLOO2HYj1hD_ z{!aTr)C6EZ*Cg2FT%o6ybWR&J5XR4$YvqxLEkfMAcS4vG6biv03n!k$U4A5~XG`S5 zt*2t}3409?Yv_3~g*WgCeW8Mo`E_br#b!yU?m$J}NBs?{5jvDGP9ec57kaHgf`m-3 zUBBhia-}kUAp7V^ zT!r+0We6n?_Hr#EeQN90Ts3euzD*Qp3>2Z|0mfh$<|U5kZ6X!5zx}R4CB)#{(&8v` zR^~$7tC~?QkQoGw#s+mDU6oNSYV8yiD@A)1{yJGFEgm3hzWhK=e)RvZP$%%Z(yhsD zed_}2mPA}U5a1;q(OfX6xW%n6XL)ft$L$=dWS!d!D;HIygv>%V> zv;mG;7(Xy>w$&=E_RWi$cpgHH2oAsto2lg84;nM-4y%OX;&t$DCw$(Bj!;gNUKLJx zwYDB?%#dvn)M=1aPDNF;oj=uPfAhs=E#hNbIr&;FhKg(0*>C+z(@FKbuEvvFQK)6; zMq_>n+z5$#UEa8+mRkE%43G!<4M6|&kq`dHf_eO*pgSrfM8N#w3}wGwu+Ry??QibI zUg?MFRnZ(IUq(0#GebD1a0~JTE*m+GFyzVIQj^|5@w2{pKWIL48CKLOB-?AefggQz znJ{21Fc@i@gV)i~e)oHt)6~WzHe9Wy`Aoh{?@=rLC889DxIOB?J_il`TYPFkr#I#+ zVXp2y4RyvC(WM^->vHM(^!Zh~W^+lqKE{csSe2ADC*WR|)`;=u>{Z2fX&g0g7c&T< za)YvPpYNK;)@>P|ZzU!7PNYdc$-(>GK<@%b)v{A>V&+X;w?uXI@n1o$pG$|0O62@G zS2ylWEKG%rQ(<(rzaUfQ)yVb011`cVD+K^`%Ec2d)Rxiwppn6 z{V?OJUL1vSrL32WpU6+NwQG}Q90lKQAYVqf;+<@AO7wRyHd7mKQwUbur8{GpTUt8? zwWI%7f@%5`yHhCg2z5eyi-3{5B~t|tZOCM~{HUAd-mXPLpKpy|213b4bokerNa>Ft zU)Wsun5zc6d8KGwe)pG|`>R7AH)8as1N%7;IA3Fh26DW8`;~R!9~g&ud>a0BZ7&(f z6#Q9rx)6qm2Z;-}4~^RpPqB?{rD|y8KoBRONbGtxIo}cV_Fy6p@@qD;Y;V~N8`m1t zO1|411>Wvbg9dThNWd&f#k8jZrO$FmAwp4JXc6?VrNrfH4^cjg9=UlqqEsN?nW@z1 zQ0}*E5EfY;jL+TlBbgTFzJS{E=%>RZm)SI%7}ru{<8TFIU$ThR9kD=*G8TKpgL#K( z$T#bVQZj`XbF&_5xg2>$yP{;$R3B+E3F)>X?=tDKTasF}NsY+MNC9bLKvcFgKAiEx-Evjo{eEE73|W!UB1GfqDIVEhd{I!#2>pnH*! zP%{{8##bg8?oR@|+HI4g6jK@|X)QkGM;cuz@gL&Epl0( zqG3_fR6!)z?{XOw6|~vt;r}fx49yf;a}Ls$iMuL$!w373$W=M zhA!ad>F!E60!^|I@oMV&G|q4>Dj&ay_lAm!YUF(nAD@9wgp5-N>io*}T;RQF+;8Mo zWFcVU=bqF(YJ0sF1#sQ0n%mqk$Ap(V1^!t^Ql8FNbg=eN0VDPLBc#9Jfq5ufu;Q3MH2zvX^Y_`@w+>q9r z4==p&pux%e@_!?*6lyH)N`+rT-p_Ueq<1i?6>*{%y;_)mHVUS+FoLZ#8}lk4Ca6{- zcjT~EjFGElskV*!zAPVptC3lc`fQd)9fFTOMVbDPeu6CkBa-TFmjPT78R0}-wKYW4 z7?O^@K`NxU2vlUiWueMa|9u+A>TIAFC-l9}prO02DQy)u!(H5GVX08^m-@fF#3 zPhc-k;6N2hrzQC2kz+#Eiw72c#N~h~B)k>LD1p~)wf2E#V8>{{pWcaPSl5Hp^>!S= zjcd@Uy|O_8nc#q24WH0nBcR3KTDyxjmz~o~?c&8UE)lA8ZV98Ttnn$jn9!F~x;-c< zza(`Yw!$qUA{_Tz<(JYmlk7%wC6#(Wqp%lY2i;K5s$vWRdY(zPb)?lsIH94)-Y!b? zR^ufNiGF_W91o61g%8b+6-$O25n zKIu5G|D8fZqL~h~OGsn~w$2pcFqUiI>YNW7CzWw;v{s2nq3U>jb={VY zrczM9qK28oNAU6Q0^=%iNPqJjyK9u(NbR-?;0;1!xK=k=flc+a&ae&ul;4E$<)iIS zbmzEUF=Cc)aEkClp!H|5H6o?F_nCYJ8~%YYmnos(fVkl!+_*x>Hz01VH9B`N(}edMC~L9g;x;)_ED^5K@S!l(rbjgnlRHX_!$i(sWmE15%DT!uf2 zo*9o9^7beUOZD@g+Wr?ux+hg;uXGR|?0`xoJYDG=RL z(BA*4hrB;#GTF%%Nv^*E^uhmeOw=&2`xp7bIk=j2=_zaQ!lf?WL>^)qe#mty=mBT4 zDyE`()Cc)JMhO_{tsYR~Kx7qeLgji9wrx4mJ`twtO(^v@I}OM*vVFJr@jik2lCRk} zJg^@8YCrwez;OX`@>*R#i*Vlo=H`*Ax~=yEEv$&0II`Y&u?^6W+!w_-j92<;!_FIz zkSKBN#feNY<$5Xq(oqCCChR$0ERa_q=H<*)`3J+`eF$jt6j}PpUT>k+o}f)w`3i5$ z7d>@_)wFJI$jFK5NM4ln>0Zny4R z47;f0*3rPYYP>t%Yc?TaWzd4!0$fmu2%Q%6ylNgy&XIQKo%fEz!p}&dD^3EYU-w=Q z??CF%q`zn^p`FjUU&&*L@q#_<#NU^7l7=l0;qwnyPv)}&ka1D=XWsZkHpbtuK;kJD zwfW}odF}wN3iuLfFL#%~7W)VjC-%dFKz7b3uDhcZA%{He^bac~LgayX4hBaFcx0J| zlCI+6aUy~#<&@tLKQ-=MQTNhQ(CW1|%pnNDXqI)`tFf%Ewx%Es5Xz=Jl$tmpoNXnI{32?puNcO8#1pat1A%zUUFF8Yleo$ z6MVET zl>D=icjoVusOTdJb%SCJRLh8nyS%Lt>Nz%kcSSiXZ{m+sTV6d-t?yOXG1YKoK+hW- zN@W6p{O2*uyf{NmiCmh$&y-{4dF+p^QoOk$He0mp zjo!PF?gL8s)EY(JT@eR=B=MY#C+|kr;zn6GQI`yPMdXcA^mcFi*Ous1!%6a?5g0_X zBf>Pu&mg@plmFCx%MXzK&*2KX*@T1&T$A3r&M{JB`}^KXqm_KG0vPA30J zPXVYH){HzVV~a-ItBD_4?=Z#LtR;9ULAKk2u)Dn){rlv%0SO(3WHFp&udmFrjf35Z zZmFO!=*C#&d(7OEey$ryyF^0EL%FcF19BklQ*9_R6_~^PA7}RW`1BL+vMd! z*geKBi<4hzGf@cF+-t%#`mSgXYn04M;rApPDOQ$5jPdgrf<@<*oMjrc$2W}VJqF28 z0p#HXFpgz4I29_=EMQiX0c}6!z7W-_weKV^if}wh*ZI5;+3qqZbSvo4G;Vy|NgL=@_%5J+hGLUv+jV@AMa zvyYgbsz52aASz)lo9h)##B#)=e0DJGyhpC<#j3u{7WBd|+iXYx%UipxdPeAq!#j>r z!2qoF`S+Vdv#ET$eFk%`eF7Qr@OVd92UAm*-O^LT@{1a;VcU3lR?A5u zXDA8FIOay)($cYVKmFL-Hp3Quo$sj!+S~j@!m~Rh#s)SSTsT&`Wt+|JM~^;V#o!ke zkKKY(+yNuEYDYm63zd!uuMsC5(oD|3p61&>K~ zg6XP{rbXf2&Uk6@skEj>2&34uvqD1a%fTI6`2++!bfBm8g6(-rLW* zy@DkV7up_I98B><+h%{boNX2>Ze`mMFj&=tx=b?kvH8L)rjA~rjmqtwytU7(P&iQZ znlki>ssp{LX$}3_PN9Z>)-oo_Ws=qOf(1D^y7#RaB&?(2?qDS!OL{SsbC1TCL1+r7 zwJhX8SaE9@(cWUS!~VnmT2%xgC?5oca!}}?C5n0heU#jC!V0%xycVQ{6);ObR|j$Q zEzu|R?oDSlJA&*(+IjjDnk%vBI zX}~)xN`*EiHrd`SIEAvMEsOP zYO7pW)K7B)=PIK6WgEwwk&HJ+n3l?ufVge#(XPm(gwR`WTt-+f?jjw=I4Gn3*JV0< zvYOfC{C2a&)yCpN10Ox>6Cs^}HXW|D{yNDbi1GcX=R$W$w=4=AsBapitna1iC)vKV} zw@~X)-)Ev>y_4jwPEW(j#9NW`Km+8G*^U;cwYW-Ycz%}wg>ncW<O7^AAJrF?rG?OJM5a|B zp&8c~wzUxVOT-e4^|hnMo$)rItl;v3rv1bk%Jqv2MA@lj}7I^8Q3gIooY z1ULk2*cV&i(8PLGC2SfdNzy_|1J7HG?XyAMsbKi)ZAGG#ee1ci`TDu1n602WBe>XH zSPUxIdakR8YOy46uUvMn1+DG=(Su!!$XqzE`7Se*3^a@~?q+&9&E|M88z}E+qwFbd zGAQFUxCrTG7dOx}!1Bqq0!;#%GyCK4cA`TL(iDCQS^lukObPvTqnQa`@ znr;+?T$Sr6;2qWXKd%w&7Ms8o$6H#jcncn}Ra3C^$`nqD@xPtrnp;6D^zTDy94;hC zO>N^8_WTAwFo8Wj zz=Oy*32zWrB+7X5d8$f`}X-2s1**9IgL$!K^u_{hs)?MH?8%LD2sV>8SfH6-~0@ zj{|y}K=Sd}>co!yoxXYGxOlwN6i+36CsU({@Kk~yxa=pG(q3>Hk0EbH2ZCPZyRlNu zaO2RbJ~q68n~xQyL{n%j{r+u;gIKm#LUn*wlQB1uqs?t)bv#At#VQG#<7);Z zxnFaz0y59Q5lSr|QL}lUkq7~XfIav? zR_}NN8MW|^sq1i4NhVV_h{P~a2m^1on8;`-K^JEDEJ3JWijV5S6HvKOq6`U62-&i0 zcc?SJMd+dUY$IpzwT>019>?cGLt3znn&e@)4q%k$Y(UQrlN~m>6OBIRNm^AGcIrmo z`IBY~@ZsoBO-dskWUv77V^%k`jFrGOTgNl7AxRV1;N@y2@WFUZQEtMGd`Z2mc)3(Q z2CWq}I>t;NJFmjB!eSPS%+o^JOsa{5@DZ--G3@vEl*6UgWA<%z&BeWsYHbySia9#z z1^Zk5hc@YZn4gM{^t;x3Qrj9<9KOXv%1CSpev})OLM0hr;p!uqUQ*Ov zgp4u;*`&3=hh@5yQwd*3ux>Nw?S7|7&9Pc^gbfT2BMOy&8t-k9pY905VqsYXYj^nx zr6H6N@LX4C7oE5wDJeIN^5B@VoW-D*D$1j>j#ZF6473l2X1`?r>K(c5?K0jPC)CJ_?h%1 zE987~-f>oMOnkuB0cxxHVU@!qY!Lqzmg_})(Ae<2G|c)0$wXT%nZeNVX+`*9A|RDt zY*)LAqSOK#aW~=wd$FF_V``r3<%@j3XdBu)`|=`oe~|qg{|eg28|K+W&s;PlUmgF137nI>$5r#xiO$9&(Q_T;-N;ZWf}#T@Jd@Dp_^7 z<$0_eI(Z8m5}^Df%`}>@1GFCUt)D}m;C1MN+OrBC70|7t=$bg3tO=1@aicdpySGMvLB-{{O zasGLe+1<@dRu3a)N>UZaTJpSmVLnv)G>~Suo(22(#X5^afNNz)>@G`^A%v0J(P#=v zt|nD)7n!|{@&dT`7oT*!apRS_2!jOd^PZ|A9^^p%84-lh3*{rf!r4^s4?fCuCAI;| zJLo7Y1STjGzEB4pnk^rnm^XWp9v7cZ)r@~Xg0SyFr=w-lq?%n{sH;QXw z*Yif^_Ze|xtgzF<)p(+yYTay#+qr$&oi=cv-=;QbIZ8a9=pR@eO_!ou%7tYE5{JKf zRfY39^%gOeB8v@PiwQdG0578DM?Ix~=2$X6UYJNrb@Z$`JzGa6t4Ta4$l*`xr}#)1 z(x)Iz5;UWv7XRpLK&bJ!gibLedHf=CZ+6Y;E90212a9*aDs0$kJB{Dp)H?b%7kH0f zl@`I;M?Hj0$u*N8zaPh@fvl*tfQL;-LswHECY!dPBAeNwWK>PmMvI4Kt^_` z-e~oe;V&3JM(rrd{@L>`8p=((7He)QjyJbEhaa+{Le{;^VU{Sl8}$Cz33Rl4s_ z0r9*)#`=f|;xF{)(HL+&ThBXQjTstE?053y-(UzNOG~i`LrUN0xo;xeX67MErzh3h z82%b*ycaemx)gG|<6$|4m$)QB_x3^wnVd6uI>kJ0Jy zD;VY5d?ZVk2+ZFOLSJ9Xyf;)(?uLW~@(Lzzf$2UcmfCsY@UmHm5xE)ROH%`bB4wkX;+ z$rnlh;2O=VmrxkT#)W(h`nh5y{4;?1FBrld)g@IAEP!lTc6haXl+rrTHAf-=YcQ@> z&Z#P#C09FL;63ktund7|9$l;dCY3q%>#&Yx=*rOuu*P}~Z?uYz8DOu4B7n=#t22>{g4D38a|o>Z+4_bz7eQq4Y0!VlLO`;VjAud=DSaWRAx0G zB3|rt*OKVMPLEtY5m=f^@L7rfK$fTqLp0T^r*tbDxUt8O_YWItm9W~*%pTmME7uP) zl>qTHXZkGp8I<|Z={i5^X9%8Lw`+WWX`_ha^F09(KITin1#)sUo<%&Yx9KIecCq4B z<%)auDw+eYHjqgeq|{8UpAUOdjmzZK1_mhcRa6xv0vy#zT@pDVaw>b-t0qYFP2>55 zt2VtJ*W#d1QL8NWx#AG5fd%m-7#~jn)W*>zGddPiZr{||q2TWUp|C|BK|AF!Q$q3-Jj!7Fpx)YN6}#&&?pwz-utY1_Tz)5ePj=>76Ci<`qoSs(2;)M3_kUiN8v{x zM{Yn?9j~gyEpd}AI4tD3$o6d!9dVlQhn558s9t4~0{ff78AyCdCxJ4CQ|oJ7cNvce zTrsc1r;2QbtBDR)n$IQnVq?ybqx0&~;4554s^jSnK#JnTggC)Gzf$Z7+F+pG}W z3f)}Tdpy|7oPcrW^pa>Tuc%PrN*$K67{hp4nm;O&AzI>^NNPou5c(hICC1`@Yd1LK#>^Eav1}{ z2bbY>D^(zCasNgi8~z+Q@0*Zs#rn&EJ`z4cZ=RhOfV;DJ7~k1NAkr>cTN=<}$c&JI9hWs$br*U_^Vhc^Gyny=O>sULtM3o| z0}IJH;evdANodyZvye$8*TkY#g2J3I1)Y}WisT&@Q=nex6#a^9!vy2pFeC-;w|>9!zi%Eah|IR24I&!q}*u3l$9ZnQf`F@ zqJ6D;8-`D*v@wIkf3O0RTobD8N7EScmb(Fus7X{X_+h8Ph%%rn;k?MpHxbkLt$_B> zAISpYBxnY+Q0<^na);`vcZR<-_X8Me3X8m?ft^)n*a_W~s#XQ$r|ph#aT0LG@~AbD z{myq7M>aT~HBw+7<`gY>rMS!AI5X>Uhg?X|#fLy>cJ07ga&vD|sZn@5T^5CTjG#sJ zrBQ2RnPtxL^x=VG-*)al*Y_~n`z?@6fzQ|zk5Aq$7RSyJyMkWtcBw?MtGMREbl^}W za_wxX{S;M!FoY7l*HZgoC$uGouuwqwXx|E9^=OIND&DT zTkWh-o-#6TQWSZFC7Z>=RuiXPJ*dEB1_tGLA10y$;kLAGOCbolB(}FNu{kI{&_tQJ zi9GtJEWrX}Bq?Pci8=&Q&eI3#+qtWv1MY0epN%%!nhxrgrS&Cbr)a`-UD@=87Wwze zlI+wX=yC|d_+hUfa?S zf~q<2XRvj&<{33>L*s4A-|DD4AgYL3_B`JIe+LxVy+6b6Dp2SSwK{5!gDu>)XbqPN9D2*e;0u%Uc9OZrfh4 zEULh-J<~S0AqM)`%burE{I2dj#rbt85{W=91H$bz-cR-7VpS!lC8PVM54Uw^*2=?NC_yh+YWuX>B>#@_N_Y6#*B~HxN5Cm&1!==k9NhsRE_MuuOfL{yInbnUh_we zlWnWon35uxtRU07C*tvOK*WbKf{L?(yYZ^xe?L>ox_az97`zgX9Byc+fw%gAH1*P^ zb7yg>jE-|DKC(sHe=r^-f7<(rKE7%d>HXoDs}yC3is8#4_!;B}$7;fmgXhI4mTJ#fkCYc7cHin30= zSOBr5Xy~z0JpoxMFA^^2zCQK^9mPllY*a5|JO)UCN=3i@F3Y^5x&^2Odx z_DcfXyq}i|v-5W9JQvHwFGoLEGO!USKZFb{XQM|TVrigQlFK{*=Cr~R^1X^wxV3VX za7_;?Uj-GaQnj0KpwDdBzRAX*$>8yAlnI~Pn_*Q#>%T~1#|Ich8VS3}isnrS)Q^_OGw=d{w-dp>K8!KGq;WZi3Ltc) z8EaPHK*|w8#Y`R)XJy%Q2r9sA4wezMCu1^UfC6WeuiUDJR_z*!=Y65SHYu_oyfRBZ zfO;)C*%<0*Y>QZMk*zcyaEs{1jtfWJJaF5vNy_et@JSGvp_X6@*-TQoF$P-Ux&ML` z^o+q^Ja6_`DmM*r3bBmF9I`P;iWfPJbktkl#)CteTWfWBG#J0of^;gYIqE@sbdn5^ z679=}sN4LAKvD@xh#-bjPem5bL!W0Nmv!G>3|??`XUmkWJ(TCkS!-zsh-#*nN5Vn$ z?jWd=y-+wgCb#nLpE3(x7~fI(Y>&~-J}^J+jH^lW!ppi+%^TzPJLGI=yqjgi!eiWK z>*5*CJ64T!q^Z6c&6q~w>!fxBker7Df}wv~@xg*bWOh&yftFf84e|kv3M&CULE9SV z<$@d9m+9RK1Z}qT_dUX3A1j~>^Rhq(P!93eAnsrAUOWi(t8ozTbsR4`AzVwedMf=D zf;7FC<=HfR>!qQL%bdgl89kzPuK$9<^&%<6?nN%my=Hb?(>3A-!e?X;PGiXfu^mK<`)OxJ|Aaf zHZEoIN2Ox9zhVm%i`|eoij8W}5$8kiLK~5Q$)TUi&IumFBoEO*ZbU5)a;X_(8V226 z>3qRo#Ihea@U|m9R0f) zLF1Uc=YZ88_JGeYKSa$khz$6!~=AxR0`!TTQqMzokx3 z{P#y8w`M*%BFXjik&nW)mfOgWWm4VFCq&~(gKR(X`f1}V|ALqjj}N`@g4see;a`aKv*v}bP&WmXz$RHe2YHUgdZDX-tz2| z1*_*GxS1A}a9gn3Cn!j|D!XH`)Djd56i4afZeyLtTX{p2Fr_zcNtuW+`bPa7=4gXC z);=Eh!^(-DA@IKQd~njfu!SxMiGbYe&+}U$ zBC<@iC?KIMqS!v*^@>@pQq5xkxqXAHwhU)U*(jEiN;!Zgj(T!%^e)z`&8=mq+GMBo z)QO)j9K!vK!t>qmdD=^`u3}}|rtUIgj1HD#+r%+=8`S`+V^5(4=G~|hcu-sr#nmN6 zI~JZ<7Ps(J`@3X`zuH*R*-{dqY!e_vU7p8H%+5pTshvMAv1@xl5pcRw)}Fo(i7b)UGtDgij1yg$opW8ZEw2W>HywU5y1V2FV}TBzh?1>o-!*q_Q!}n^7kXQ#9Em~l$lUZJ-_fIG<@Mh0 z(n!?A1eDr4jFPX`=}VT}a)CXK?RO24N9-H3$$g+DRaDa6c{&rUUWj>tKcfRLh!ET8 zPYZjA7Mr1v@J1w;hQ&P}w8<+Xv**Z~k13Ys#Zc60rH|Z1Hc{0^r~GYR5A z8IbuRSa7uk=_xey$`6tbY_C}ReE^#ATBCJUvkX~E$)7}M-$!n^sFa2QoF6E1c|wNR z-$&+jF|}i~=NIsEz*;Y?!|^eV*PsG0yMYm1kY(^qgZ6lzE^xo}ArUF}wsH3W(X`Gj zjAKvk*Q^hii;)*gs;Thi5B}L1xXfs|o7?#`@AI(Ww*n1)=h_?pNO~Jb*DA8jj#)sF z!ZbW09bBWCjjfC2llpmPU153DCd@eu{g&LI$0jDg$`4DK>>p~rm+i!J>CsvE(3T9j=Nrp_>{Lv-hW;y6L2W z5@rxVQDsUHG*TH)r$O$-LWq%qzm3@cyuAtr692Ung}p@09YAcMvnj{n1zWfvw#5XP3KO`pnse5L}G%gWAeJ-IK|tN zdV{brSRs^V@NN5~9|8z4{e006qn$DLd{DqSm%5xXl`!)gF~z9I2TGcSlSA@ zIz58z;8h_B282T*!KJ2!jncB$z8sbAfh?;(7;vVmE3)D%ddOb_P*>(p<)DBdbh3ea zsM~0yvCzs1G6*PJFbh_8woM42}EY{_jz0 z43r)Grwd-VYKvXmbwPW^W+lw$YD_eK@)?-v7%l*L{ix}MHa{OOSI2UaIwoeC4T0%2 zXt*iXq5&-y*bnNfGIPXmhen1AK`|q=uXH)M(!(6}%s2;gylS&IHK9N8R#jw? zERWig=0gYucIA11UdZ4;fxzC|NqJwGSmlgo&3=5ILZeyU_shY$A?ygw`WIC+Y0s8F z^3AzD>(nWdEGLdNF(-MwtF0VTgv*d`@=$L>#sM=fTY3( z(vbs`vhdSx=ove%{(NiLa&)UR^TB92ZZ8`Gb`TS6Z9TOIMaMVp$jdWrE}Db6wce^m zq64ce?&^gSifpHBYb9o#v6zbtNKZxZGMoi4d=LW}Uq+7rCYXbXEVFBab67OL@M(@#kg9{O$-tP4ZatF?dqs37OxgE>#VzV=jx9V6VRSbAR zxo1+sCGiLbV+zdclKJYn)Ww^!{s6@P5ddAL=4Z;8LFbwc8&yy;;Hg4xhmh&YKB}=W z1J!o{);B)QtjK1> zdd_{W`S?fP#)7+&JI(`0EU5^2u&r?h0ih}CFzK{q;ur4vd;E$%J1SCdRcucMC>f#u zcRwIbbVJS%u|HC7LRw$2g_}9bR2NLGU|$C{!zKmmVwbFgUOC@`ka{ohiM#m^M1j{H zC|T!WS!Tu&kuhh7HPOCLx}2f-_cx`Adv|Js3wnK>_1E{be9nRcm}?(rF!=;d-i>kp zaa57YL1iChb_2lv>xhDI=w}Got;-PopEnu_NqQ2?36y1XUIf0V!e ztK9oIr=mctVHmniSRmtsurmdnrJ2XC17%3Lz&J1x_5(6VW^ui7KQN<(R`*HBw_jd zP*hs3mySVc5O$Zk<(*v!AGo%h?QjbtWZz^~yG>Z(CBx$E`~H1t$mac!*;~%0yT!Hm>!>UR{;iQA-hS&44P6eaKjd zR0j(#WhEe6rC7*&RdrN(NcKP|*f>%={n9TpjicSr8Y_8J*3*8`y{VSlyGUz#T+T)r z47TVj0+{EO(X*`RtdHhov2n8(U~wHcjmd)nHYajUds!utFY9+7RT5g9#2Or2i3Fcr zuTuw5>y1}}U-RVL<7xG)!ZFjJW3>%1u%#EbiPjW#xMvQ}w#jsf8i%8ZvrUXdYUef28md(%HPB^CRD_%z9@{hQ5t=t4%g zVtIog{7Pw=%6|e2WgL|5s^c4%vZ@HHf?Ndn4dskPS~^Pe*5V6|@Miojjw*>I!XR+M zh3%4FW#k1~o@Gkum7ZSP8{&HO#FpFyaXvuW!qxe;ilrMir;N^eq|g)g{uZu6o}s}# zg5Je+hy=aZCP#Jk2Uw+?}^!>iM{ELMf=Bb?+cLj+?R9qzC01Ev@a z(}^IOeKA!4EQVUsFMpg*Z`q@QWNX&p!iQ~imC`?cquJuH>bZ9eKd5pQ8rhWksZ z#>=4h*Gqurk6@4_x;8R4t7eg=D!J~@2TO4YY_FAeFduKk0F43~X-hhNg9ZWt_$w3> zgBQIxcl5*%?L5@oS+Qx3b5(T@CJiFdZpU~^$MZjKn~q(9n7kA?+4m5o6fSQlKR5;f z|JS^|y1e70n>fynTYW(CQ82e`3(z0-+`35_^A|{celF^GM9=>O3lI}}v0I&@yCRnS z!KAJ7F=2xd*gPXpWCT`CdbQ)IjE=!W=Fj&usQ1e@VHJ-PUS4S?nq&xxW3h*dpaGp{ zG00%6Eu{%}dPyjB_%lT$&s^*kZ^6#@)yzaKt>wbxmUb{te*IDKnOClC*PyXzXS}qLb)nXo^EvVg(Hyy$P8$BL`cvjgI+}7d?4M4MtbXu~8Zbo@Ey| zZwrKiK>3r4Z>79=>XyS9*ckFq*)6TdUs?fHgFNxO7vzckN;)M7Iwd1ZgNu1ry)dM~ zP1(%^8ntMz!t4qp^6aVw`dT@KQ!+y2r?xM9vsTUJ%PP*O%DK;h|E*ASiz%FjT2ilZ zp893|&P;+oi7|l->%<>^k=^&E_(&Z)UbrM7Vej~z`Qh-BfiWI~rJU;?$4exfWJR+2 z%OE&4xORPvSq65S+E{_D1mq2>U;lmbZI3<;E~xoGsugpU^wECeRo1vb8%E+6E-n8l zj|5b<3Qr@Z5 ze<;QC8y&zKBjAuJ65({T&~_wERYbh-xLD~r(QyH!@G+edHTn?MaDx@{Czb4O;uo9- z=%27y=9TyP!RqzU_wA&XaHFu?Bq^c(lM_*PCyBDx1o(rDjHAkG#my(r8rk@0j8GMo z%2BUT8n7_b%EaVxAsEV07txNmknYorDlyQ1SLsxOD>5sb{>`dvH{-C^ivS+;fl0IY zia^(CxzxrZs}vI&;fyKh)DQ0gUJjiWQQ~CsZ&6aP!RF|0+ib*5Fx@hmt5za-DK27#p0r)Qb|Sahc#{w z-_^R&B!muYKTOz8xB58G4jbcOJ1OH zr|jWXXNx<$=G?bXrkfMga2NY{;qH^v{-B7Ylc%X+A0!%5|Q8+I27u zKC%;qxS}iEQYP)JH)z8k;UG*Cviq^tGej4MSKxXOk_fK2RG{64xF2B%*xV*;$CT6Q zY2ci@TOe1@g(O4=Kku&Cj?dhZdWf<+K1kJQ1;se26aHw6S?YNS)NXQ%mAFd^hZV+sl@M` zz*XhTJ;7^qWTXnH=F0!jkj2`c*Z%<9MRp*zlL~oJ%l&A8P57+IhyZy&hQI2WHROC1 zddi&~|I}SPAQ6Zerk;;*9oU_ruWp!ooSc^$k;g%rf5r+ZKtg8e2jY}n8^BI>s0c$* zEhs$~m{M)=#lC{b@2^hD&v34(B`)>#ej)M$Dx9jybeA`SZyH=?^`KS|N zJhMM{e~?z4@i*{71G3BXCxx0b<5rgrlJ`Gki#d>rwk>M!4=|)=^c;{7n|lOc?&@*o zFIR-(j2eI1LNhAg61mHVJkM+%yx&EAja0T2a-=zAv;We0`0m#*J$w*!V`*mY_Y^?8 z9a*ubiPQ#6*c2%SrH`(j=GKm(D816f*=09q%?6x-@xNkPmv)&Myajyw;2jVMAe}ja z-aWW4dn|EW8~HAM(KocELxFIek=7%C>5cwmU-^Cm-fDrGZ>3&{u>HKPM zis+mGK^!_#FTJnbq)|xrDuOSBh8F{cGE8mlSJ0VeYcAI>1pr%t!K9)7X^p?VFHSo} z9!fV~OK9V}Z&Z|OUosZQyoP9f^W`KDl~!4iPNB1Tl_gcbtnP3V<@DvI(s2N~E-gR# zBI)y}t$Qv9lcmMOEmptHRoAAA&?w}_uM;H_Xe(6T*Ov%KC~&~N^3MyrHv7JozZw>kWjTEJ5OANP$-T|iY>G2FM8KIKhGbIhN91z(rNW4 zZ!zAsOl~AT)}J#HhDE>#t|Xavn#%1X7_PWIPjNm5s!bBOju1~;lEnyYDIEWadgM3v zAl9TzSM!<@zRx2jNC28vA&H#=iQ@ckOfY*L`Kk+Me{M1KB_<^ezMf9)GDEiU9=w(% ziBP08cWFmz|G$r3XZBWpBx{xgTBKh&io54(%NMI-e;{(MS`{~ z1J(gOs2dDmyF@ou-@|_ppu=*0f(8PdbzI`eW;2&aE0emS#}g@NH^p3#y3CZN)cU!;UR5rbL9P>e7__0qnwNICcEUfWG(u&hel|Cv<4!;?s9@gaK3k0=L}& zWu>HOb5{oZ#D--0f?U(;hS<9#)&LBB)rXxy8FBp-R~|RCpGlpfg^oRsBX&BN1A2s8zgo}Wl{z3>UXJ&1*bhACjzWl zxJ3lUI%$j_^0}48Z+JjJ*p{I>O7dH&XlQ}bL=4qLd|w&X7UNRwj$3}f%p$&kHH=6C zvGjz(%xl6$#t>ktorUOQyIWP4YRFf;TIZ$^Q9=zY)8ax8*vky6Z?JOqXyZLRyWJd| zPOVqDt??SY!7rck$rxm6sM+kUEELi*%tMz~eB2{VBX5Vms9QXjB% zbL}>TfCx?HlpIcoi$@T#CQI^B$K8U2|7MiR$e8VsA2NwzXOxcR!|C5f9YQKun}G;% z8Jqo^$@WZ%l(($#yt;R&?OaVdPHTIYvE6IY$*nfZCe|Xl+2=MOhbc$&YDEj0Ew#70 z0J6M{Ab$nT zxnar)jWL-AL>wwlWb8Eg{8@<9D=LLhk__=sBIN{SS_@u{c@+*v$139HW8Rg6KtcAp zfX>Z0wWh0|t^a!zdW=5DYDtg$C!Yk5n~noGNNq8v@Gw^paDu@_A2F$E4mo>$_3ka(8M0t3JRRDM<2Q zDvJ!IcPXZLf;q)}@_67AmmWQq41MGCjI^l4^MS7_*h{*lF(72Jf) z?c(h9$e0Vq0*0!ZcQz?Rc9U(fW6%Pu3)>)nf-db};+CMj@1NxL!gT4BR0o@C{0yi- zl)KYyGfQSj(?(NyD~9*`&BxD*(N>-z&^DTQ?#vx!#Aipp2=*g06R(dV6383jwBAn+ z1ZOC-zIJHTCJ_K=&4YicVA==u{%piDMt*NC*}1slwr|Iq$#UhIPtB7=k64M~Bgd$$ zix0?$AUuyWC2WjUzPm=H(+l!1)i5C;5-I2xmRh>x`gpCkPTH9&V}pA>9l>4 ziBB{0k3N8)_3dz9lWI6mu3g1|O8g`mwc^$Y$$Y^awYrjdUD+Pdz2mq4;@JZ>F34Rk zizzMxRA5SXs;*Dh(xNaULxt>{r#F1!PlY^JXRhTD&^1z4BRao8kW2D~eo9bnk|FY4 z!Na3@YjZ&6VXKirobOsFfi4*|1Pm>|jVEr~4{X|A79sF$A5abcEq2i1{Z1&|Rm%8E zYsB(-T48FuFNnbbm4%pbGzba-JGUZu`hH!N26R?y#P%;SBUS8VV?E#2YsNpvVjcQT zOd5s+0PlK%hc|8p<LXDd81bGAc0kH6y)hap zf)+?$=(Tjpi)FhjI6U4-FiO;%KOIuBQtoW$H{Tm zmnBvF5_$Ti`#CWF3&FZHZEVC4Ku=~*#Ey9MYw+eYMM(F53oQJAwmCTG1#G1Uh!heP z`oefdDXi1AJEMEE;!vm$E*Smq5;S+2E5y?^-eapiVa>nQZaB>aFt)F+Ms3>j`+;f! zQSH;$l+@7L{4hfZ_eOReU$Djj>*jK|JJVX+PWg&#axu{QT?9Kdi=2o)X?bSnjkz=_=LL> z)fHv4DF*-Pd@L5A=WLmCON-3wJCvP)%Vews=YESJ6oyQCR5=L+ac`YZ|0D5)qp`v0 z1Ps?qMp?$L+0HeU2rJWy^OO3LM>A|%VQ;U(9X*-P8kWzQxEq^Ekg{i9> z?#I`e7=vUejG4`N9$rKW=U0Z0b zXP;&zH?@5uEbWrXjG-{LE=5uVUI5@M9Y{B=w_zP@XiU~KK?6qY!7jYD_6c}P4oL$o zS(s9)Tls$RYr>2pvGz?5+-!gsazHyNK{-U7M{-sh^kcG^B-zIXhJ^_4Tfog~q*Bo< zrXHwC#wzd$T9DrY393mW{V4B!3X!QYU&kd?zI&Ph2hE9U73SF+ z+8WBim(BIfjGMKqnmlzf=r^}w_Ilujl-H47u^G+KqMeH#D{bAq^% z9yxeSs3~AI+3C8dSPX6pX_{1P36Rt)f-+jShn!8w!|8#XY;Qj>6nDOkLUm^^Gyh>c4ky<@j*hOoJ3V+#rI*dOE${zp2ZNyO+? zEXI^Yx7u?kQ9fD&IDt7TpX&s_hvOIdzQ0p$`+@0wEI~3v&(>i8HWp@ zyr`>PVB6A)3>97Cj{mpIU(WV17G-XKE>gp@gD{rJTJ+aBGQVV5>api9r+J=!eA+9v)^U6DVlQbsWM1^0r zJ2l+_l3quFhLoCNL^Xkn_EUKRAx?lv`uF1l_*hdZ&MYVE%TaFn=6q@fX zBcm5<;Cw`07QbV0@hC&IB^d5i9wa+=fBdXkz7W$$QUY*~%s8n}v@_4`)qvI};LKEH zr@x6(*bZZmxr~nrw4`ZwG`+2rfxg<8v0fl2KXdsnh4=rA%BkgI(PJk2!Ems88cBi; z3`$wq?PB{Y_DU?cUWg&I^=H-HIo$LKY;atC-hH#`s8{4%n2{djoK3!i#|iQI>7}qj z-kbeu=H9p5lCac0g2KC{hnk}>7P+mo28X7#O<7Q0`DCAcLoVt^v&&nu6Ck-JImQTw zeFlwiujwXjXZn6Cs$IOC;eAECjNGj`c$dR_S@EpVBN} z!$bJLiTD_C<#zEL-`djVEb@n28c5ag?C)5_a!_t6!Ks z{G{di3!~f>u&c(vAAIsR9J4JKb0aW9vj?CCRg>!GAS)rMg+YGhKWlA2A7qRinJ>}Z zOenju+~#k#3mJ0A!PBzu@9XX_bQRR$@38JNPt!udXR(lMP!^ORar}vatz%<`^Q{o9 zWvX*8)OQR0Oo#h1HtK^=d8GDI#>XH635T$#pD*<90cywj96rUdmFMgo=Bppqtj-|zlO&3l>K}eY~9H)rPcOd zj{uV5^Ok|^GeXi2_87WYNQg%PH#BIsRA}25CYu5{sRY^w;=-HpvAQ^q**UhXg%DJR zOzMc1wtZ~$_G3GeDMuUs2J19;BQy503bA?#o~#Q!n*P^4z5@51q8U$s+A2RcDJ@!= zLKToE!*-+jZX*fO6b(GK5cYSc5!4%19@KS1Hg*2A-o%hkd7dS2qvg}O8*xBE$)#~& z8-S1@-_2aui8O42xu^)Sc-Hwb^TYOA++3!nrL^>rmRqnTYT~OArpXK>HO0L0dYJ3o z9o+)V~*e9C9#SUnV&BES)ZCmSN z#NrD&o(}qmqxlp@)03)`@T8s&K7HY}^^#St>hUGAUfiXgdBAeeO@{LlGR1v8u?b2* zqRWB?^s~t0;!W1B_40$xt}|~{4_vUOC)C6aYZ2j89Ik8@vx$n6Y$Xm%xZA4Z3v)A$ zXV!`$ve#q^lRj&yY92+1hfqfyX9L(Y0n_t&%gn}+h$Yp%iS!N3hk9CWG35JM?Mdbr zk=iyLjgQ0LQJXF$ldTEOCU?S}8gh;4`3TMQ#!zKSQO&Eos7?&;Km!6aoK;lDxSPG-Q z^`IEn@2Q$F_F01bnk>kgVa4L8u=ckf{g&ZPJ;YPHEHt<_oqT9I@r)WwSNq=;8N>|q zTw2l267^Q9R9OHonSVV(UGivz9;_`V6GQ}zk5Qnypj%biQd0A|4ba^@DE{O#cxBYq zKs-NJvI{SXt8om!?OS5oM2dyya}s3n-N7j$kY!q-y4{Elml$6X-F` zLn41SSyAF=c^lNCXFGLIR9xTL^d`lEHq`@hN;AYNgnZHp9!-KPJxw487U2=)ElZ6} z&6#cS*iRv6{ za(573C2zH-H~M%;AC)92F-EY_ie}W&c&sCTT3?eKq$s zx{!uvJm?3%{Rk2#laB{W`Shs6tt4+u3c233(Qgx824f#(wvgToRJA038XqG(V%N*7 z20|ZG>%H5STj%+277$dSg z$S5Z21XN@Lx?;}47doQSiNHTUAIT;rgqi^iof zOC)G0H|6JVhH7EI*dLdKDWMsM(sD4A-qjQ4gO<>hSpEE#rTjEb0O+P=X@^kX!0Vy| zHC8*yNM_i4%BMZLi8l5K6fpbXJpzc$>I*YC{mB=zDHnbXus_3?nssy;Ra(w!ipE-~ zCiaODUSLH;3&Cd_W#inTKxCDTj&<$ZXhmFtpyoxhujTNTq4c17&sChd-SlXu;mE{M z$f4|K&_3-h)YnJrwybRt+7n!(1tA;!FdRA~^esLFn-%=Dz4MRrTfh)%wBI%N;5gX$ zO^bxYT)ewdd0iVfD^mj=)-TxNExOSLmI5&iKRgqDTVCRKllVyQrBq%y`;K?s*k-$? zV56u1{3!ByndFPBkN?JgbTKeYjKhan&flX?{4g!#>T?GdkY%;{est_pb`9#m2z!7zo^uv+KW4oZ5DBnd_)OyQz0kBJUavYWU?7>7FNYNEd+oKW(j~ zbVQW(1?Q*qBU?4JaZ~M!P+hVDUMS8-w|0|0Cl( zMhgOFTQ$w+Y+snrE(~rPrOvJs7Eeg95^;}~-Q?8h-M0+RXVM`*uBxqVl?lRRe1}44* znYn!nwY|W&j8xKZ9r>{Wmj%?zzH~%tOBX!w(UwtI7*mS%eu`%e(F=v*p55xcFyiNJ zatQuzGSrGBRr!nV#fsKPQFE>1d&2A+$D_R+=*@R6?uWzE(TyXA^=^3NVsYJBSL|$I z->Xy;WfB58I;Yq5!N~9t8Bcv>`%C|G|p(&MHdwxefH5u(TG?CKv_POGu2ZlCutChD#6 z6HB3gZ^-g*6RPE>eSC>aEr`62Tz}cq#}>2(3TA@I2WDJr-#<8V7{XlkzM+$djq@>C zb7F+z<6KlFWtuZ2R!15Nt_N<1W zUZm%%d+g}@D6#c<@u$gH?gI5U_mqRlM69pu6_C62zPaA<#n237iq7HjJaP$gHMXvK zs9|kvcC{?SFy8Qiv2UcSdFqB){S3Y3}8mf-SQ0i!nPxicso<#%vjalJYUf@V@;7%Jipa)wk0Q$cJIGKD>69M44 z8Q%+7-LpM7H(4|CPGBvb&KOps9^SE~w>ms#t=yLioY2jwN7Z!vpRw(A!4FiMt}8%P z13)-3t`}Ko1uxS{H5n3W#A8d;($A9Lm=d_8Kr7}a?-#*2^aL7T{`l%h*#e48u72qi2g@8)$uLw$H-^dvNsX*l6IyvEH^`duXQ)`A6XA~ zly~Kk4^nTDge^mCF}nh0{xWdC8SH}`f*qf0wG{_XJK@J2z7greW+`_)sz>Eol#Cyr z)8SCmL61}iKr)I4FpfeL8J9+n03LXUi7c~#0PI0m8jMbqcM*aT8OANx??uhYbAW?~ z+4rB6n!=Z9<6%5H8VM7q(R+PxB9cH-!tul;w1i2VfT$m}n)!ZL;0ucm!7#J6U{|Pa$?}fNxkOcKz zXJ}m(LhKWu^B-O+OCH%c$+;B*rBATI8rI$1pDA8HcO|jt3Rd00Zn2Fin zewb=PmTg9%M{#OyPX+8(lcR;;;e+F*zoV|y6GLQf1_uXJaoW7Vz;@O{5)ulv8D>VQ zI<@o0@S6B?5L?AS;&Kl5q_)NDp1-gIBYn^97i2u)O9i-H7-K|U*p{S79+;^ty!7<_ zC_p0lskUZK4n-5TC0$ZY3UKbwzjkx8(_wg-JZy&Yx118$@noiZ^oybYpC1jX%%KFN zRyc3P*|UwD&=sr$v={Jksm0cII47^Fd!X>gSA({TVNuhM!!>D@bVaan_Y-w4e-jSU&{3PJn??NSADIc zp)@u&qdr-Q(jwO(6VU{7wN-udxs2H11bw?nSvp7vAc4o4Yf2Mbf8S~O4u=-tcoOU2aBZt0E!z4mq%7XZ(l znwcZeprd=^7&Mu@-xM&)yXDm4jjoTGi{jXAS?RHqd7Q(b-4rbMpOoIC)s+EL`x=xAF^2Z6HZgop##v3iP!@kdlJ5{cf6!(lf1Un{Ob!4BPzeBH2Gp^t<{b^ZH0|CM+wS|xStr&pVmM@7? zly-orNBGxVNB`ZJpZK4$1Dgs{8*x5x=GcMWeumqO9=cPHcQ3%Ysf7znL(AUkoJFQa znZv*2Rrji;#d>*_K6LP0Q@C(W!V=CrTIZ!AKh>t(aDJ}8(y}ji#`^F2d)?R6kh=kb z@$5;Zd~93jyqpq#Y~aQEaQ1faP@VCQV9}a8$`6rdZmTsMPJ#R_V@i=wjY|Yt??{4~ zYe1;<@&T89-Voj;)Q4b-J**&sZy@voyf99*x6AO7$%6K9O^DDiVcvvy1!Lvr;Nncs z<2~kf%j%yllsU1NzMgp(=72gu2o&^0dwiD{UCXAMx{+UvXnjM#NM)R}ri9|VUW-nX z@_=umX!XRBZnz&GzC9w@0YylGhppWeW5Eu7E1o%SL++nE zV(Fv*Qh31Ku1^QZ*<6GHt_pq^hJ0&LFjrA%uyxt26LzKl7AZki{y-2zAbvh;ssv^a-?6L{{{}CL7LT-xF5#8%PfbuR2U5 z8geds6%|&_xYmhfVh!yyD+>Z{_iFJ$kn5R%-o?+@SA383tXH{~j5 zB<>?8?hCPz38cf_JXaa{yYA{6!#>Y7yc{{6YNkJYU{+?@1k+PfcG_FXOTw)LNAG#c z=SZjMO`D7Ur~^b^C14bKyOCB{+Rp+N!XpjQu#yY;c2y|-K|mo?>|z`*Lb8EnjQL@4 z!^VUDpr7OWB-2_KC#ixDWsqKvL=NVJUy%_IMYT$-CAz*J}ge**@69offj zv1`8+2AnPel0jMd9oWXi)+p0p{kD1N6e6iX${>>bA9;W%rQO zWh@6l9lRZ=FFyTq^l5hsvp%_Pj(@I*ebU>)g7U>mNE;ht@Gc-dp3r(GaAu`I>JFCY zlWif#ONodkz+-|```CNeF{-_1V`l*7Lzqo*7oBdxrDge2rbt$s4yLes)Y_Fg&?{|+ zX_yh=y2Y*#(Z=8oUiaouqwlh|_xh+(Rsjq!b3f#mp%z*mS>j;)r;SWB-ro!BR$o|)dlsrjo zw$qGao(@LOv_X;+f6EujjuF>akWOW^Dkv`EI}%mu`ErDBJ9R-l}4CYJTEx1O7j!gBB(!?Z*8! zGB_vgie?a3M>QfW&d%~#&dhXd&36RxFX;&9Z0%^|A>+Q)V`cvcG2E!VFO13-?Yx!o z^@ea@ClX3Z;mHTGhz<0+qy-AFDV)ePGVCWc?h&!`E;epE?=1C6VP)E>BP4Y1?L|r% zk#QM8+q-*BZe>&?g@9j)h+hR)<~4fNw0Ah|coX2?{w?+QP&Czj(Y!6&V^*sSVsJM5Inm8d&Jk` zvB;D&b8F&pRtXtWWRXpm^-1It$R0sDFbf$I`&G)9cd(W*6V-I6=jOE2k@(A_o9lM` zK^sIPUpHG?ZIpoY1Zgsh`II;GgRzC?8}aSaEKOjW>EAh8-o0ogU?SR0h&${QYr&Zv z?hD0K`@g{}q{=jj<}gY(&*hXALIMZUL<dK;B{m9eix}Luwq+qBx!<(3P>EGi>_F=8Sjj zH^nc;MI;jJadwvXSYG5=-75dOIedvH|KRY=lfhO40_}YKD_h`v&@yU7WyfCV{4>(6 z5jg2riUyLM&Cxr8C%U)t@@8#Ni5EUE1`^{ur(a?&J|*X5HES(~O_&^K$XD}j*+yA1 zJGlA6O+Zr5nh>@p!11e(u1^oa?+t~u{WjZU!DU0PASl^0s2F=@w(R$upUAso7hmpP zuDHbj_yp((&kzB7toouA=)oW&a8h2!HTbDQZ4GaRs74OQhX%l6rzEZ>$Xmkluwn^G za$~y~4X%gF>X8pYVHe*8=-PzJ(Ey0TI&+BdeWxMv!7ws~n9A-@75%>fnR)g3wFi0fcIAu*P zmBwR#y&2#}2mZd-`}INYz(OLVE_pH=A8V*U)I!R%#?Kjdb6rFVB~H<@mgjy;o@y%U5}BpeBB14*wbw)Xr+eyn@aiCudux z2boUuF3#;VyXi+fCq$L~%y1byGUPs@gtw zFlQlq&`;f+29@++$X_d?y`#0mz+-YJi;LC@LC>m&4g$xGx^4L$L`2KFskCbh5f3>)~328s9J6u>rV4=y#M#g=1IArUmQ$0>hzw;w7DqShs zrt?te{9P`%cXWNxurV}YPn+0qUz+Fh(?F57smeM#ZjLF02%H+?VGQ&0I;OOba7VUYk!jWZ48Gh81D-#27=fnZ&a80|GG8do@w>;fH4zJj>K?AM$9pg_ zD?F%vFaeTP3BIbCK#l9DIW7Cu#^3?O-z$^!}J7#dJ%75DFVcr+KIueA?yDSoBfMj+}I>iLJS$0EUhq0Y8 za=Ak$Hm;57hrBgNBJt2OFCQe(wF_0x3_|G93ts*fKpqtIs=72&v8m2+3RnO*uT5!n zRGuuAwE}yp<Y*qlAa%mh(3aC<9%Hmr*fbxj3}Dnetz1P?}X`k zCvNW^32YH^iaYi6d5}dvy>mS?Qo^xN(ire1A0rb;(oPOqd(Mdu;ku@l-0z|OXN7!W z-F`t!;+XHMgTge*n_fzrlfmdB$|QrHj4U_V(W*@LxYXBxsw&jy;a(DHp)He`4|0+@ zUte9JuMm;th7sO|SY-gSGMwzStvMo=xKnhW6!cQ5y3GPs9x=!n10dN_77h&w{@^Z= z&z`LJot)Rl+5xDh(GODcV6fa-Ch#4nqFy^#&`cccf1k(x zNXgLC%_p>+Ys5q8QQ=z5MIo}f!;#SNR1+$%o+l}aE!n#?%hxZC^jRpoPpldnGa9!b zpP7gY1NJ$@Q#6lR|ALyFS~8qUDdrLEMd^KMO=-sk<&2aGPK^B3r=sG~%{Ni!(I+n) zhZt5`P{f~+1y6d*l3ysqksjqbZ5TT?SUgIh7*MGvU7sf~ zfcBsY>{JoW%8AA_pIg)rIw<-a2kHM+V9K%_Ia2oI+J}o?iN`u5c`xXec8Z+$?o!O> zvu!xTz$wZJOZQhcYK(rR;%C>dAO_h0gn;(-z#Aagevg4yqA1lvwAim2naGPemDk(O z1ZU92%z`oX;+Fk~!i@2n6vWR={3sY2xKm{A$2Rq*;Xbawqmf&OrY9zJd-xvVZZng8 z`Tm?F!M9bmwm#A}Z}!75sy{0Y+H?XEYYVQ86pq^#LNEG$)j$bX8@4wzO(wC~C-ASb z%mtX6LY3GYJ+;hgOESh0IfaU-|8JP-ytXx`Y`R@T9c4n@y9rB&RCq_BM=bH!BEgIT zT(Fq)@_F!%X$S3uGSA)n6YD0(N;pCP8eXl&zd&36C>1fiA25gh;nW*v&^JmaH+^ov z7)0~4+t+gL*an%#HV4`~@1YK#|M}2Sg+~Ul!ua@WCb55SMl+5{@@`GzyhnoGw5O2B z&XKz4f6qiOSPjrMlpnueLe}+I^&9t5xO~o2cxIzbAI5mNr4RlILUF*JK{Jtns1zyA z6m1UV<2h-|A4s5S<9Fg1!%^v?3d{;`EG_8`=) zTsD^#QrU%Q8eLjG>4Wc5W06clI6Uxdq^rb|6q2frL_cm^sI>UvY=N=}K5z3dB^@Sa zu;-zXB`un>eHDtmg95-VbXj|d>1^9tl2^{SP4-`bN~5q7W(-u$8WitobN@YD>eE~` zj?S-#aKw0wZ>IP1Ks%;2JcbSH z6mV(qMuLCvuj{X0U)T0ag~*l9-1xIs@ByRY2~2~YQ}^0jD_l1@Gt=Jd3&4^EUc?K6 zwC3CoqKtOwyYH3IT#i6%YqInyWLqraSU?O)k3#dFG3&Ch0{WPVu~0Ff@i8_8leseh z4c(D{!CZwJBq4vRsm)J94R~-wkZz|T6ytnSUmW9FETxLren7zT@au2qSD1Gxh}KKI zy>1g%%b&_pIi{j{zjE{irxOs%UkF~7=@Qf9H?&g^?7dkI2 zZ&r5iz~3WE^Iv^pw7=)in4ejugI^@8isH zGMHj7T>&;WPGjjIfrT5Zj^zQ_O5wG_kxcM>-fAxmmIM>IZ0NS5jXIB2dFcn5KP;hxM_Z|G@@Rd z{>as zUQL!)9bv+(Lr`a?ZMkF}+kzF_ImjuG_o}P>Tij zym%_Z5nvYY8mw|58A$PQ0BE$xX|xS)u_##XNB(AUuKA=k$h$Z@rG(kyFFPFA+~Wqs*~?o?&TmCs(xe}r?#ONy;U>#wYKBHtv3ig zUywh_OEg%j$w3nEeTY-A*QomOrXAEE6x7@Gs}81>+pg;W|Nh;nqJ#>K_E~Y)_BRZvhdlgp@BureO`D+V5;qp5kD=K#FU#NZDPEaV1SwP0 z3cD;BL+Ahjo+h=P)m#@8X{jkU`+nZ}@`c5B9$tFSukgs~Ecw!<&m-Uz0RFgVT3C{C zN`ZW7qH4YLS;Q#?u()14wC8Y~m`HpMSr+LM2EqRogTRmZS~*ya^)q$9LLTgNMM~Fj z_;@M;|8#Ok!l%R4l#u!Hrh!+RbXnpc2c;U%gKCsQ6AkKtZ}yW^VR4(wFuYKC`mT`} zn_FqWlo1;XK)KqqRuHnqng)LGs**WChfM7^x)M^l7$vVzv>&$=^>l zZx2x8H6=4cgXot`WsZ0(Om=?T!rBv4r$D4(QWqy@H2fCHWrl-yLaUfa;6{tFQVt=x zn|ZuK8n&{~7mGj-)=aCZrTuV!NDNWM4y!RG`?ux~TT)v3_eIF&Rg=HtIv-cCipX}3 z*$n74Evj;7p+~@h8XBH~JCMvS9N>1y+ZhY{um(w!O5#l+wIX>I1~-;d*}949&i1<;C1Ts#pjso}$`z zPkIo}Z_ba)k-rua4~wgMLx^yawMF>><45xSg6a4B=Yh4Umc=y#Fz6dg_E=`QSOcUz zeFsm8b$I%n(>aLIn*WFP&BsFGriWh;ipWQGsSrWDsV2^tH=-q{kztE0inI3nO_v;+ zHDs-8^ZdXrC$KX6nj;#lNu0}_zb6br$fJDI0es!g-y<{2zTLB+hbCiQ8oWp42%Mgu zXh{&{!SGgx5hT0TzC!_!)VG9*mIMoC=NgKX^1<8+OBl0l$;56AowjH?W2n^*$XZNb ztFXB;v|0}IO28dpk(zGTsVPFywV(?Op-If|)~jgybdua1nv9Av+eu(o>aMS41h%0; z`i6a*;TBe0!IQ`CfhiC5H*!ZFpAznSk#1S0x+3Vz8(CRq2#`rnt4|csg>|$E66GtU zto3}aUlC?2Yd6nZf*oW;wnwEYsUcyq{@`23LP8yAj*dfClPcoY@^l@mGca3^jAHE% z7wRP^uSQoRUYg}CWO71St7BmtB%T!U~O7FZ%iEiKOZt`^Dr1KD2$?eV(_&{AS9jTnJU zxDz^0JK)xeE%N9(*D+;~j(gTTYx%Kaool(9{fds2_7U|5J11^@CkKASslZ zzv|E%%FidTS0%T%hFB~WoFafR#fH)z@kRED9atPiy8NC>%Zza{%8rK?G|2%Ro_J8C zx*l(ZrZ8))K?`n}ana&;ON^%5qoT`_rOYN{>NEq=M`qlWUbsGcV*aCU2_4$Zn5lB{ zL8*}?W=3!cy6l)N-k-T>YlswaXW_2|?)+$C4%O&I8Tszl;K>l$4IZM79J%6-HXnqY zWl8h|)xObG@s#lu!wcO4`+d`TTgK;L+h{HZbNu$&DsWal_{BtA zMrnh3K|5hffhKq^W@Dd_@Ec`hf<8l17yj+ z!AIK%g!7SxGP=+;Gn4f4DVA%;9am&O%lD~EBaN38&q(m2q4%Ia#k}AcetXw%Baue~ zW+)aumq3N~$nF)u!r&bu8D`|@0%}+ulVIHe9n9%;+Q{NW|24qLp3o=Ugh;4CIjL63 zRGLM)hnsob)s@@-9w^QnGNj&Lgs<XQc&tlN?RFgGGjElOhT|1U}tsd?Ezmu2Yy8mv8ZI|%FWrV_PPOO^B6b|mZq zrnk4C!>paIj`&!XRI5+vuBqLo6l3-<#XPzm)h#92`_zmM!5{W^b(cPt6?+FoF&fQ6 zW|9MgsU88JT|X=NQr-d!t&}zBh!vu;gTVl9VlSNz+$h_t18$`gvU1v@C1BC3tN(q7Ww?7_! z`ClcS31Q;d&J(U6h00banS&1JukMj{xMC%xH(2gUcm5`zx;Mj2J1`7Hc5U4bojdgH z=CTBBipg@QR1M6~m|q|Zzhilz|1mVWZ2FjZK#exvL24?H&@}{%RL`UWiSMinNNlOC z4x(jg@@V5Y5W3uE7&F0i=N@fNf$Wtl#y8?lKKuJpR;hKW{tJu6l*7(B!d;;Aq3kMe zn^J-8Q8ZP;$?te{0Da^-cScuokWFHDzxH#^0Ye&pK;%b)2leKORNcz#3Y*-9KZ?2A z2|eaHEw!SF5b6_J8P|6TReXpgk3*t|XIT7o3JiU?W-@YmgVf-R6a)y~?T{huAdey0 zJ#KwmdY=Fibexs^l=3zfHNb41q}A9s`~74y@&Hc;A@zsQB;LQ5*Tbk=+0Q8H36B02 zRB!HpaK{P5GE!aQaQOs_m}UxaRi<7GgRSxFK<`^=F@V^*?;3jVs|bxyQXfNQ#{SEe z$4QG(dOJPWuenZPdum9ATV$v=jA_QO&fu1aCKmkLCOG2q75OEv=Jui}X`=Q-{$LV4 z1M{bLwE1XSrp?An_a4rKPaO+>AT>0k<}D#InIR}mgamXCqeh@&X^=RU%K!tw0Q>jW z?H!}A>x5LsxA3GCpP6Q6V{;iM(P>PQEn2BpN2HjhEDA!_)GU;Rl%e;l{J2*4g~QF$&D^QbXCW7>0y6;F-c{@pRNA0E3=@b2$m7N`sT&(ne>} zSQ=+MGqP|&E;P-89LR|AKhf5eE%R9&BIQmEA7LdZDLz^|O}`oGOIzA*Z&RZv*dfZ! zo|aGK=0w@&nTaaM5mj<5;(_p_*EX8Q;lTFQH;xjDs1umbCur6--MVw9U8fvu{q%en z`Ytw7OI&S%Ie!PQLKS`}iqZLAr1o0>TIU)rQ0yj~#RMhOUyu%KF#quJ83m4Ij7L(= z18x_}TiW7*Q!@(gOsLT4!x`>2aWb;&5?h#h1rSEq4=z=p0HcLXU5NhnRMCES?8KE> zA0zf&SLp^q{W))uo@Z&(@>s6`mB6jrDxZi_A5W=Ui14)J2?1TAdL4Z9DS$zmQTMgs|C+D6~`ikN=eeS_8?w=M33fOyPR#*$b z@AHvjy2&yD?OF7qb6e-7{v3;2F%hGEseEuGbfAxM7mtr)PU?)%)hpt#i&55Z`P%+J8-Ko!C=_5Neq~l60U}% z%ZBBKkD2iBi}hP&E{7r}aOxsyn*!-xId06-aH#KgFx>)%EW0Okys_`ad z+eUVM+(*0}?}Go3uO`!%W>@jH*{k`-2LJxcsu8VRN%G#49dHDV zwbrv|sRTDaQ&GbpzOO|NF6YR5WG~QfvbwhsIRRUAElfLSvClWcn}Og_AepaYH&w(} zKASJUx2l`s)rh6oX7$?59Yj@QQY@1@r+r|~zS74agI%4|izUvxfP)E=kVjCDwGZaPPcXx1=(`2+Sl=YUPVh z@>*f!Ipar_oIu>G2qjT6v3rXMvAES)_WxIs{=pAG5Saed$y8IyPuHRvxO#kmxkDQt zGg3nv0wdmXR*4NjupO8Lpg&D4t3JWvS-M7GdM$ad!=|m&7O|8D@kdb@jtpsz9Syk| zTAn_W!TGiHD>>4^w&}XaEu;%ybe8XFk3z!1B8h~10-k6Zt@)oc6vB7Pi@gCN)+hPai zHMwJytfr!-4n_FhP7a=Ma-NDoh25>i9%s=NE@|99<+RjYWsoEA8`*jDs09)Ap30D| z$S304I};M_lH4{X7Gm{qb0$}-$EHQb6X2a>m@=hedIyXCd}|YBR5Bb*gVzM9w+ob( zoFeNX=w?8;Hs~DVGW`6{Y)GY?r|wCu%hTee@_^X}RmV>`q$VNOuZt(z$X8H`0)T}0Dyz7P87+WAgN1D@@3(ZcGZg%J2==O3Dw09ROYFS zm!ll?JjVg?jfDLoPV_l|7DTeOgIE(pXz7NN4X|rh64b`k9wQ`VvWAx4U=99T65uga}CiMml*g zzv{sCT!^`4K{^V)-W-#yVNJLwF_l$cm+15`b^ErXBA!fGJb^6@K$q|$On+fU+lO{D zNtt*Bj);`_fUVS->s%^G-_LV}?I;ld3*EN!TK#?4OLnbhlIO>dwr9Ou_i6iH?Kq-T zW1xkTf^Dt}kYdH@cUk}E_AJdQ5Uhb#Ec;@EGoIumhL=ngAXa|opPh8qr*Pade4Bck z{>nB0q|f?RF;YhNmq|2XW!7N8`|vqU!iMZN2!zoPOI~(_-yFw!dFVLJw7#j??X+0@ z<6IaO8^A}fE6x+7QX=1_$I-LX-XOHdJLkCeOJzB?d0V?L^$w24lj|**v}RxQ#k&c< zol2N8{Q`j~>XCuYo4KiwUS$dqW@T0&Ac+L2NdaoS*1fxI(h}Rn6lf^ZyX&8hZ5*W6 z`Rw8A4N%xdJbG%mmzw|Vm@PTd=>WfJx^=-amw^y@iTrQ(z zvxI;V0UJ_?rKD7cT!&O|3MsQb@)|kyn1aqF2aX!B+;M+QOJ*j5C6cSbc#rMzD<9?- zguflaB2*J{Z@(n&Rcp=-3L0+hRb%u9bk1d%$tjfzFQvT;8gUIt>T$5tWyR{%i~}{~ znCP{F>T?vajcAq-ltbmhe4^+_3(ay$BNaHVe^HY$nW((0DqvP#S)|ezn)sw(Wr|Bq)IEuxiIQf z=UKsH`j~Bf*XuQ{#T8B&5{W=Rnf!-?`y()S!fkOZ00JKwWPNS9_zyQkRTZ4A1?H4yIl*+s39>6Z!_i9usoruph ziH~|$8s;C!T8tJ|WxyGYMZ~QKbr77~9?OmNT7<>U8cAJ5u+Ki~EXS?X0(Xco_;l(D zwrggXvn1kRb@wx!89ZaRw9uX7X*5=8S-0_FJxm7_SiXKln|*He7r4%EaZh~FF1NpC zd|v#jc*9xa)N26%P$AI!P;+Jc^PYSQm}9+FhL^-vdq)e6^b8q|gdgPZDVeoZmfST1 zX9AdC{uP2jiWPyPk>F=f!AlL+yLTO%74jO0d;nJwKibuJv?kW~Gu60mVj)4kNIwJ9 zoct-oKa3x3_1&tQorU5QHB9K|@}Z$^qz{&~YTADdHKLNB+$(QuYC-5Ry%iIVqZDaN zoY~K;@Ae+1Xu)?I-=3-u!7^!&;vv}f70_KsHo?{|_S9);V`*zFB0^Dxm6+Xp%J^b(znFNMs$1hlF;rqjdy` zaex|s4}{YCrNn^S7i5}ni*za7zTiZn>jfZWHx5TxE9p{%o4~4pR2s+WFg1$R<0AYe zRpY|n3Xt(5g=EagBhBh+wcIf+TYuOW*6-zWQ%Hv3*$367rIq8m-``k^A*IAa(|>dh zpQg(9G#Ni4mAu~@NR#}d`BvvBW~~8o3YjbD;-21~q^kZA6}$;^^Ra1u!RhLCcR5vi zO6^pQPd<~|4aRK99&Z7@G6?dXUD3KG)+MZ31F*k;mq+J>Y5gtefb2P8MB!V;kNg={ zru&@4l2|N7gMHyiC-(=b^e{nFE^%3s5v9*4g?MB%t^C6KIbfyh;zGmUCIaiBT*2Wc z?8;PD0Wgb4Q=C&4Sj1ezYc00HT6y*C?pl_#@YG4#0JqElJ#Hgr!5n@rgxBXo3OaZt z{b(ohIi*;IBx$6ns@RZHIgwe> zw^)hm4&LWzp?6eepDe+HYWM|nRY0M~w@2n+xvX%)+LITqLtA5*h%DB0FZ4I)9z;5} zrxM;$*uiJNPE&SGYJ<&9k9p!~;BEX^^91t(tzM|vPiVR#n@8#bY%p03ZJ*XnGjltZ z0gqKl2EmMJ0slK!snG&o3qTbSKE)ftsx?&v%8UR1fH6Aqq{|#EROyP-Isc__y(0ky zqalNY*ilXItw~5dYHncLZqmQbeU9ZpsjGxfNVS*?E|!JTAfY;7$5t?ltO&c#B~;>e zw>;|0gZYZ537|7f)!cFsE&=`ncxYlb;$LmQemu6CQ@|R)oC=BG=>62SALL3ID zA$V?qD2AK{E^mO15(D2>f$xIoMM$|h3Zra?Nyi;l+7w`RVrKGQfxdV)D2Y-=hu%D z6{vz-$77enu=*x&ipM&yY=mtIIvKMXxUw&=pRI$pEXxw={|cH1S_N80SY{iT z=E=FW3A=S#h0i1kEJPn}>nWaZ)g4f9Dx!pt+K{v{Z_iyFJ} z2f+DZhne2qC0fMjYG%%q6Y#nB#TB>*`8yoM0o?On=B|v~(D1&>2-26Frzm0KCcr%Wk|3JByo#&Ibt7as(ZygW=n^uAD*bbL63 z6nna=Z{}1;M$Sm2mo=VQ$!--EHJw3S$+p@C$ojy?-R9y0AY2Kq!ryoaA5EJAn){1R zZ>Gdn5EdIuEW*(;JR(E8Yjm&G+lUL*G|5Wz5YAS}?VOC98((@g^h=X6G(Y+XI_W