diff --git a/.gitignore b/.gitignore index 758b312c..24f6d319 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,10 @@ logs/ .idea tools/ .vscode/ +.venv/ +.sessions/ convert_* *.pt -*.pth \ No newline at end of file +*.pth +.pla*/ +ref*/ diff --git a/MTV/nlf_bbox.py b/MTV/nlf_bbox.py new file mode 100644 index 00000000..603994de --- /dev/null +++ b/MTV/nlf_bbox.py @@ -0,0 +1,38 @@ +"""NLF bbox formatting helpers.""" + +from __future__ import annotations + + +def format_nlf_detected_boxes(all_boxes): + """Format detector xyxy rows for the public BBOX output.""" + + formatted_boxes = [] + for box in all_boxes: + if hasattr(box, "detach"): + box = box.detach() + if hasattr(box, "cpu"): + box = box.cpu() + if hasattr(box, "numel") and box.numel() == 0: + formatted_boxes.append([0.0, 0.0, 0.0, 0.0]) + continue + rows = box.tolist() if hasattr(box, "tolist") else box + if not rows: + formatted_boxes.append([0.0, 0.0, 0.0, 0.0]) + continue + if isinstance(rows[0], (bool, int, float)): + rows = [rows] + candidates = [] + for row in rows: + if len(row) < 4: + continue + x_min, y_min, x_max, y_max = (float(row[index]) for index in range(4)) + if x_max <= x_min or y_max <= y_min: + continue + candidates.append([x_min, y_min, x_max, y_max]) + if not candidates: + formatted_boxes.append([0.0, 0.0, 0.0, 0.0]) + elif len(candidates) == 1: + formatted_boxes.append(candidates[0]) + else: + formatted_boxes.append(candidates) + return formatted_boxes diff --git a/MTV/nodes.py b/MTV/nodes.py index b12539d9..ee4625bf 100644 --- a/MTV/nodes.py +++ b/MTV/nodes.py @@ -15,6 +15,7 @@ folder_paths.add_model_folder_path("nlf", os.path.join(folder_paths.models_dir, "nlf")) from .motion4d import SMPL_VQVAE, VectorQuantizer, Encoder, Decoder +from .nlf_bbox import format_nlf_detected_boxes def check_jit_script_function(): if torch.jit.script.__name__ != "script": @@ -276,17 +277,9 @@ def predict(self, model, images, per_batch=-1): 'joints3d_nonparam': [all_joints3d_nonparam], } - # Convert bboxes to list format: [x_min, y_min, x_max, y_max] for each detection - # Each box tensor is shape (1, 5) with [x_min, y_min, x_max, y_max, confidence] - formatted_boxes = [] - for box in all_boxes: - # Handle empty detections (no person detected in frame) - if box.numel() == 0 or box.shape[0] == 0: - formatted_boxes.append([0.0, 0.0, 0.0, 0.0]) - else: - # Extract first 4 values (x_min, y_min, x_max, y_max), drop confidence - bbox_values = box[0, :4].cpu().tolist() - formatted_boxes.append(bbox_values) + # Keep single-person frames backward compatible, but preserve all + # per-frame people for downstream multi-identity geometry matching. + formatted_boxes = format_nlf_detected_boxes(all_boxes) return (pose_results, formatted_boxes) diff --git a/SCAIL/nodes.py b/SCAIL/nodes.py index eb669aec..9bc745d3 100644 --- a/SCAIL/nodes.py +++ b/SCAIL/nodes.py @@ -1,9 +1,195 @@ import torch -from ..utils import log +import torch.nn.functional as F +import logging import comfy.model_management as mm device = mm.get_torch_device() offload_device = mm.unet_offload_device() +# IMPORTANT: keep SCAIL nodes off wrapper utils; utils pulls broad Comfy runtime at import time. +log = logging.getLogger(__name__) + +SCAIL2_PAYLOAD_KIND = "wanvideo_scail2_condition_adapter" +SCAIL2_PAYLOAD_SCHEMA_NAME = "scail_pose2.wanvideo_scail2_payload" +SCAIL2_PAYLOAD_VERSION = 1 +SCAIL2_EMBEDS_KEY = "scail2_embeds" +SCAIL_V1_EMBEDS_KEY = "scail_embeds" +BACKGROUND_INDEX = -1 +SCAIL2_STRENGTH_KEYS = ( + "ref_image", + "ref_mask", + "condition_video", + "driving_mask", +) + + +def _field(value, name, default=None): + if isinstance(value, dict): + return value.get(name, default) + return getattr(value, name, default) + + +def _require_scail2_payload(condition): + if not isinstance(condition, dict): + raise ValueError("SCAIL-2 condition must be a SCAIL2_WANVIDEO_PAYLOAD dict") + if condition.get("kind") != SCAIL2_PAYLOAD_KIND: + raise ValueError("Unsupported SCAIL-2 payload kind") + if condition.get("version") != SCAIL2_PAYLOAD_VERSION: + raise ValueError("Unsupported SCAIL-2 payload version") + schema = condition.get("schema") + if not isinstance(schema, dict): + raise ValueError("SCAIL-2 payload is missing schema metadata") + if schema.get("name") != SCAIL2_PAYLOAD_SCHEMA_NAME: + raise ValueError("Unsupported SCAIL-2 payload schema") + if schema.get("version") != SCAIL2_PAYLOAD_VERSION: + raise ValueError("Unsupported SCAIL-2 payload schema version") + native = schema.get("native_wrapper", {}) + if native.get("embeds_key") != SCAIL2_EMBEDS_KEY: + raise ValueError("SCAIL-2 payload targets an unsupported wrapper embeds key") + return condition + + +def _scail2_strength(name, value): + if isinstance(value, bool): + raise ValueError(f"{name} must be a number") + strength = float(value) + if strength < 0.0 or strength > 10.0: + raise ValueError(f"{name} must be between 0.0 and 10.0") + return strength + + +def _scail2_strengths( + *, + ref_image_strength, + ref_mask_strength, + condition_video_strength, + driving_mask_strength, +): + return { + "ref_image": _scail2_strength("ref_image_strength", ref_image_strength), + "ref_mask": _scail2_strength("ref_mask_strength", ref_mask_strength), + "condition_video": _scail2_strength( + "condition_video_strength", + condition_video_strength, + ), + "driving_mask": _scail2_strength("driving_mask_strength", driving_mask_strength), + } + + +def _latent_frame_count(num_frames): + return (int(num_frames) - 1) // 4 + 1 + + +def _validate_target_shape(embeds, payload): + target_shape = embeds.get("target_shape") + if target_shape is None: + return + if len(target_shape) != 4: + raise ValueError("WANVIDIMAGE_EMBEDS target_shape must be (channels, frames, height, width)") + dimensions = payload.get("dimensions", {}) + width = int(dimensions["width"]) + height = int(dimensions["height"]) + num_frames = int(dimensions["num_frames"]) + _, latent_frames, latent_h, latent_w = target_shape + if int(latent_w) * 8 != width or int(latent_h) * 8 != height: + raise ValueError("SCAIL-2 payload dimensions do not match image_embeds target_shape") + if int(latent_frames) != _latent_frame_count(num_frames): + raise ValueError("SCAIL-2 payload frame count does not match image_embeds target_shape") + + +def _image_to_cthw(image): + return (image[..., :3].permute(3, 0, 1, 2) * 2 - 1).to(device) + + +def _resize_cthw_spatial(image_cthw, height, width): + frames = image_cthw.permute(1, 0, 2, 3) + resized = F.interpolate(frames, size=(height, width), mode="bilinear", align_corners=False) + return resized.permute(1, 0, 2, 3) + + +def _resize_bhwc_spatial(image, height, width): + image = image[..., :3] + if int(image.shape[1]) == int(height) and int(image.shape[2]) == int(width): + return image + frames = image.permute(0, 3, 1, 2) + resized = F.interpolate(frames, size=(int(height), int(width)), mode="bilinear", align_corners=False) + return resized.permute(0, 2, 3, 1).contiguous() + + +def _mask_indices_to_reference_alpha(mask_indices, *, height, width, name): + if mask_indices is None: + return None + if hasattr(mask_indices, "detach"): + indices = mask_indices.detach().to(device) + else: + indices = torch.as_tensor(mask_indices, device=device) + if indices.ndim == 2: + indices = indices.unsqueeze(0) + if indices.ndim != 3: + raise ValueError(f"{name} mask indices must have shape [frames, height, width]") + if int(indices.shape[0]) <= 0 or int(indices.shape[1]) <= 0 or int(indices.shape[2]) <= 0: + raise ValueError(f"{name} mask indices must be non-empty") + alpha = (indices[:1] != BACKGROUND_INDEX).to(dtype=torch.float32).unsqueeze(1) + if int(alpha.shape[-2]) != int(height) or int(alpha.shape[-1]) != int(width): + alpha = F.interpolate(alpha, size=(int(height), int(width)), mode="nearest") + return alpha.permute(0, 2, 3, 1).contiguous() + + +def _prepare_reference_image(image, mask_indices, *, replace_flag, height, width, name): + if image is None: + raise ValueError(f"{name} is required for SCAIL-2 condition embeds") + resized = _resize_bhwc_spatial(image, height, width) + if replace_flag and mask_indices is not None: + alpha = _mask_indices_to_reference_alpha( + mask_indices, + height=height, + width=width, + name=name, + ) + alpha = alpha.to(resized.device, resized.dtype) + if int(alpha.shape[0]) != int(resized.shape[0]): + alpha = alpha[:1].expand(int(resized.shape[0]), -1, -1, -1) + resized = resized * alpha + return resized + + +def _prepare_condition_video(image, mask_indices, *, replace_flag, height, width): + if image is None: + raise ValueError("pose is required for SCAIL-2 condition embeds") + return _resize_bhwc_spatial(image, height, width) + + +def _encode_image_batch(vae, image, *, name, spatial_size=None): + if image is None: + raise ValueError(f"{name} is required for SCAIL-2 condition embeds") + image_cthw = _image_to_cthw(image).to(device, vae.dtype) + if spatial_size is not None: + image_cthw = _resize_cthw_spatial(image_cthw, spatial_size[0], spatial_size[1]) + latent = vae.encode([image_cthw], device, tiled=False)[0] + log.info(f"SCAIL-2 {name} latent shape: {latent.shape}") + return latent + + +def _runtime_mask_to_scail2_tensor(mask, *, name): + data = _field(mask, "data") + if data is None: + raise ValueError(f"{name} runtime mask is missing data") + tensor = torch.as_tensor(data, dtype=torch.float32) + if tensor.ndim == 5: + return tensor[0].permute(1, 0, 2, 3).contiguous() + if tensor.ndim == 4: + return tensor.contiguous() + raise ValueError(f"{name} runtime mask must be 4D or 5D") + + +def _additional_ref_image(additional_ref): + image = _field(additional_ref, "image") + if image is None: + raise ValueError("additional reference is missing image") + return image + + +def _additional_ref_mask_indices(additional_ref): + return _field(additional_ref, "mask_indices") class WanVideoAddSCAILReferenceEmbeds: @classmethod @@ -86,11 +272,145 @@ def add(self, embeds, vae, pose_images, strength, start_percent=0.0, end_percent return (updated,) +class WanVideoAddSCAIL2ConditionEmbeds: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "embeds": ("WANVIDIMAGE_EMBEDS",), + "condition": ("SCAIL2_WANVIDEO_PAYLOAD",), + "vae": ("WANVAE", {"tooltip": "VAE model"}), + "ref_image_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01, "tooltip": "Strength multiplier for SCAIL-2 reference image latents"}), + "ref_mask_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01, "tooltip": "Strength multiplier for SCAIL-2 reference mask embeddings"}), + "condition_video_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01, "tooltip": "Strength multiplier for SCAIL-2 condition video latents"}), + "driving_mask_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01, "tooltip": "Strength multiplier for SCAIL-2 driving mask embeddings"}), + }, + "optional": { + "clip_embeds": ("WANVIDIMAGE_CLIPEMBEDS", {"tooltip": "Clip vision encoded image"}), + } + } + + RETURN_TYPES = ("WANVIDIMAGE_EMBEDS",) + RETURN_NAMES = ("image_embeds",) + FUNCTION = "add" + CATEGORY = "WanVideoWrapper" + + def add( + self, + embeds, + condition, + vae, + ref_image_strength=1.0, + ref_mask_strength=1.0, + condition_video_strength=1.0, + driving_mask_strength=1.0, + clip_embeds=None, + ): + payload = _require_scail2_payload(condition) + updated = dict(embeds) + if updated.get(SCAIL_V1_EMBEDS_KEY) is not None: + raise ValueError("SCAIL-2 native embeds cannot be combined with v1 scail_embeds") + if updated.get(SCAIL2_EMBEDS_KEY) is not None: + raise ValueError("image_embeds already contains scail2_embeds") + _validate_target_shape(updated, payload) + + source_condition = payload.get("condition") + dimensions = payload["dimensions"] + width = int(dimensions["width"]) + height = int(dimensions["height"]) + replace_flag = bool(payload["replace_flag"]) + clip_context = clip_embeds.get("clip_embeds", None) if clip_embeds is not None else None + strengths = _scail2_strengths( + ref_image_strength=ref_image_strength, + ref_mask_strength=ref_mask_strength, + condition_video_strength=condition_video_strength, + driving_mask_strength=driving_mask_strength, + ) + + vae.to(device) + ref_image = _prepare_reference_image( + _field(source_condition, "ref_image"), + _field(source_condition, "ref_mask_indices"), + replace_flag=replace_flag, + height=height, + width=width, + name="reference", + ) + ref_latent = _encode_image_batch( + vae, + ref_image, + name="reference", + ) + pose_latent = _encode_image_batch( + vae, + _prepare_condition_video( + _field(source_condition, "pose_video"), + _field(source_condition, "driving_mask_indices"), + replace_flag=replace_flag, + height=height, + width=width, + ), + name="pose", + spatial_size=(height // 2, width // 2), + ) + additional_ref_latents = [] + for index, additional_ref in enumerate(payload.get("additional_references") or ()): + additional_ref_image = _prepare_reference_image( + _additional_ref_image(additional_ref), + _additional_ref_mask_indices(additional_ref), + replace_flag=replace_flag, + height=height, + width=width, + name=f"additional_reference_{index}", + ) + additional_ref_latents.append( + _encode_image_batch( + vae, + additional_ref_image, + name=f"additional_reference_{index}", + ) + ) + vae.to(offload_device) + + runtime_masks = payload["runtime_masks"] + additional_ref_masks = [ + _runtime_mask_to_scail2_tensor(mask, name=f"additional_reference_{index}") + for index, mask in enumerate(runtime_masks.get("additional_references") or ()) + ] + scail2_embeds = { + "schema": payload["schema"], + "mode": payload["mode"], + "replace_flag": payload["replace_flag"], + "dimensions": payload["dimensions"], + "segment": payload.get("segment"), + "source": payload["source"], + "strengths": strengths, + "ref_latents": [ref_latent], + "ref_masks": [ + _runtime_mask_to_scail2_tensor(runtime_masks["reference"], name="reference") + ], + "pose_latents": [pose_latent], + "driving_masks": [ + _runtime_mask_to_scail2_tensor(runtime_masks["driving"], name="driving") + ], + "additional_ref_latents": additional_ref_latents or None, + "additional_ref_masks": additional_ref_masks or None, + "clip_context": clip_context, + } + if clip_context is not None: + # IMPORTANT: WanVideoSampler reads CLIP image conditioning from the + # top-level image_embeds key, not from the nested SCAIL-2 payload. + updated["clip_context"] = clip_context + updated[SCAIL2_EMBEDS_KEY] = scail2_embeds + return (updated,) + + NODE_CLASS_MAPPINGS = { "WanVideoAddSCAILPoseEmbeds": WanVideoAddSCAILPoseEmbeds, "WanVideoAddSCAILReferenceEmbeds": WanVideoAddSCAILReferenceEmbeds, + "WanVideoAddSCAIL2ConditionEmbeds": WanVideoAddSCAIL2ConditionEmbeds, } NODE_DISPLAY_NAME_MAPPINGS = { "WanVideoAddSCAILReferenceEmbeds": "WanVideo Add SCAIL Reference Embeds", "WanVideoAddSCAILPoseEmbeds": "WanVideo Add SCAIL Pose Embeds", - } \ No newline at end of file + "WanVideoAddSCAIL2ConditionEmbeds": "WanVideo Add SCAIL-2 Condition Embeds", + } diff --git a/SCAIL/scail2_forward.py b/SCAIL/scail2_forward.py new file mode 100644 index 00000000..426c4734 --- /dev/null +++ b/SCAIL/scail2_forward.py @@ -0,0 +1,286 @@ +from __future__ import annotations + +from typing import Any + + +REF_SPATIAL_SHIFT = 120.0 +POSE_SPATIAL_SHIFT = 120.0 +SCAIL2_HISTORY_CHANNELS = 4 +SCAIL2_STRENGTH_DEFAULTS = { + "ref_image": 1.0, + "ref_mask": 1.0, + "condition_video": 1.0, + "driving_mask": 1.0, +} + + +def as_scail2_list(value: Any) -> list[Any]: + if value is None: + return [] + if isinstance(value, list): + return value + return [value] + + +def _scail2_strength(name: str, value: Any) -> float: + if isinstance(value, bool): + raise ValueError(f"SCAIL-2 strength {name} must be a number") + strength = float(value) + if strength < 0.0 or strength > 10.0: + raise ValueError(f"SCAIL-2 strength {name} must be between 0.0 and 10.0") + return strength + + +def scail2_strengths(scail2_input: dict[str, Any] | None) -> dict[str, float]: + if scail2_input is None: + return dict(SCAIL2_STRENGTH_DEFAULTS) + raw = scail2_input.get("strengths") or {} + if not isinstance(raw, dict): + raise ValueError("SCAIL-2 strengths must be a dict") + return { + name: _scail2_strength(name, raw.get(name, default)) + for name, default in SCAIL2_STRENGTH_DEFAULTS.items() + } + + +def scale_scail2_items(items: list[Any], strength: float) -> list[Any]: + if float(strength) == 1.0: + return items + return [item * float(strength) for item in items] + + +def shape_of(value: Any, *, name: str) -> tuple[int, ...]: + shape = getattr(value, "shape", None) + if shape is None: + raise ValueError(f"{name} must expose a shape") + return tuple(int(part) for part in shape) + + +def patch_embedding_input_channels(patch_embedding: Any) -> int | None: + weight = getattr(patch_embedding, "weight", None) + shape = getattr(weight, "shape", None) + if shape is None or len(tuple(shape)) < 2: + return None + return int(shape[1]) + + +def scail2_history_channels_needed( + *, + latent_channels: int, + patch_channels: int | None, +) -> int: + if patch_channels is None or patch_channels == latent_channels: + return 0 + if patch_channels - latent_channels == SCAIL2_HISTORY_CHANNELS: + return SCAIL2_HISTORY_CHANNELS + raise ValueError( + "SCAIL-2 main latent channel mismatch: " + f"patch embedding expects {patch_channels}, got {latent_channels}" + ) + + +def append_scail2_history_channels( + latent: Any, + *, + patch_embedding: Any, + name: str = "SCAIL-2 main latent", + fill_value: float = 0.0, +) -> Any: + latent_shape = shape_of(latent, name=name) + if len(latent_shape) != 4: + raise ValueError(f"{name} must be CTHW, got {latent_shape}") + needed = scail2_history_channels_needed( + latent_channels=latent_shape[0], + patch_channels=patch_embedding_input_channels(patch_embedding), + ) + if needed == 0: + return latent + + # Official ComfyUI SCAIL-2 supplies these 4 history-mask channels before + # the 20-channel patch embedding; zero is the official fallback without a mask. + try: + import torch + except ModuleNotFoundError as exc: # pragma: no cover - runtime requires torch + raise RuntimeError("torch is required to append SCAIL-2 history channels") from exc + history_shape = (needed, *latent_shape[1:]) + if float(fill_value) == 0.0: + history = latent.new_zeros(history_shape) + else: + history = latent.new_full(history_shape, float(fill_value)) + return torch.cat([latent, history], dim=0) + + +def mark_scail2_prefix_history_channels( + latent: Any, + *, + prefix_frames: int, + patch_embedding: Any, + name: str = "SCAIL-2 main latent", + fill_value: float = 1.0, +) -> Any: + latent_shape = shape_of(latent, name=name) + if len(latent_shape) != 4: + raise ValueError(f"{name} must be CTHW, got {latent_shape}") + prefix_frames = int(prefix_frames) + if prefix_frames <= 0: + return latent + if prefix_frames > latent_shape[1]: + raise ValueError( + f"{name} prefix_frames={prefix_frames} exceeds latent frame count {latent_shape[1]}" + ) + + patch_channels = patch_embedding_input_channels(patch_embedding) + if patch_channels is not None and patch_channels != latent_shape[0]: + raise ValueError( + "SCAIL-2 main latent channel mismatch after history append: " + f"patch embedding expects {patch_channels}, got {latent_shape[0]}" + ) + needed = SCAIL2_HISTORY_CHANNELS + if latent_shape[0] < needed: + raise ValueError(f"{name} has fewer channels than SCAIL-2 history channels") + + marked = latent.clone() + marked[-needed:, :prefix_frames] = float(fill_value) + return marked + + +def latent_frames(items: list[Any], *, name: str) -> int: + return sum(shape_of(item, name=name)[1] for item in items) + + +def patch_count(frames: int, height: int, width: int, patch_size: tuple[int, int, int]) -> int: + patch_t, patch_h, patch_w = patch_size + t = (frames + (patch_t // 2)) // patch_t + h = (height + (patch_h // 2)) // patch_h + w = (width + (patch_w // 2)) // patch_w + return t * h * w + + +def scail2_rope_shifts(*, replace_flag: bool, additional_ref_count: int) -> dict[str, dict[str, float]]: + base_video_shift = 0 if replace_flag else 1 + return { + "t": { + "additional_ref": 0, + "ref": additional_ref_count, + "pose": base_video_shift + additional_ref_count, + "video": base_video_shift + additional_ref_count, + }, + "h": { + "additional_ref": REF_SPATIAL_SHIFT if replace_flag else 0.0, + "ref": REF_SPATIAL_SHIFT if replace_flag else 0.0, + "pose": 0.0, + "video": 0.0, + }, + "w": { + "additional_ref": 0.0, + "ref": 0.0, + "pose": POSE_SPATIAL_SHIFT, + "video": 0.0, + }, + } + + +def _require_pair(left: list[Any], right: list[Any], *, left_name: str, right_name: str) -> None: + if left and not right: + raise ValueError(f"{right_name} is required when {left_name} is provided") + if right and not left: + raise ValueError(f"{right_name} requires {left_name}") + if left and right and len(left) != len(right): + raise ValueError(f"{left_name} and {right_name} must have the same item count") + + +def build_scail2_forward_plan( + scail2_input: dict[str, Any], + *, + video_shape: tuple[int, int, int, int], + patch_size: tuple[int, int, int], +) -> dict[str, Any]: + if scail2_input is None: + raise ValueError("scail2_input is required") + + video_shape = tuple(int(part) for part in video_shape) + if len(video_shape) != 4: + raise ValueError(f"video_shape must be (C, F, H, W), got {video_shape}") + + _channels, video_frames, height, width = video_shape + ref_latents = as_scail2_list(scail2_input.get("ref_latents")) + ref_masks = as_scail2_list(scail2_input.get("ref_masks")) + pose_latents = as_scail2_list(scail2_input.get("pose_latents")) + driving_masks = as_scail2_list(scail2_input.get("driving_masks")) + additional_ref_latents = as_scail2_list(scail2_input.get("additional_ref_latents")) + additional_ref_masks = as_scail2_list(scail2_input.get("additional_ref_masks")) + + _require_pair(ref_latents, ref_masks, left_name="ref_latents", right_name="ref_masks") + _require_pair( + additional_ref_latents, + additional_ref_masks, + left_name="additional_ref_latents", + right_name="additional_ref_masks", + ) + + if pose_latents and driving_masks: + pose_shape = shape_of(pose_latents[0], name="pose_latents") + mask_shape = shape_of(driving_masks[0], name="driving_masks") + if pose_shape[1:] != mask_shape[1:]: + raise ValueError( + "pose_latents and driving_masks must share temporal/spatial shape, " + f"got {pose_shape[1:]} and {mask_shape[1:]}" + ) + + control_shape = None + if pose_latents: + control_shape = shape_of(pose_latents[0], name="pose_latents") + elif driving_masks: + control_shape = shape_of(driving_masks[0], name="driving_masks") + + additional_ref_frames = latent_frames(additional_ref_latents, name="additional_ref_latents") + ref_frames = latent_frames(ref_latents, name="ref_latents") + prefix_frames = additional_ref_frames + ref_frames + main_length = patch_count(video_frames + prefix_frames, height, width, patch_size) + additional_ref_length = patch_count(additional_ref_frames, height, width, patch_size) if additional_ref_frames else 0 + ref_length = patch_count(ref_frames, height, width, patch_size) if ref_frames else 0 + control_length = 0 + if control_shape is not None: + control_length = patch_count(control_shape[1], control_shape[2], control_shape[3], patch_size) + + additional_ref_count = (additional_ref_frames + (patch_size[0] // 2)) // patch_size[0] + replace_flag = bool(scail2_input.get("replace_flag", False)) + rope_shifts = scail2_rope_shifts( + replace_flag=replace_flag, + additional_ref_count=additional_ref_count, + ) + + return { + "replace_flag": replace_flag, + "video_shape": video_shape, + "video_frames": video_frames, + "height": height, + "width": width, + "prefix_frames": prefix_frames, + "additional_ref_frames": additional_ref_frames, + "ref_frames": ref_frames, + "additional_ref_count": additional_ref_count, + "main_length": main_length, + "additional_ref_length": additional_ref_length, + "ref_length": ref_length, + "control_length": control_length, + "total_length": main_length + control_length, + "has_pose": bool(pose_latents), + "has_control_mask": bool(driving_masks), + "has_ref_mask_stream": bool(ref_masks or additional_ref_masks), + "control_shape": control_shape, + "rope_shifts": rope_shifts, + "cache_key": ( + replace_flag, + video_shape, + tuple(patch_size), + prefix_frames, + additional_ref_frames, + ref_frames, + control_shape, + tuple( + (axis, tuple(sorted(values.items()))) + for axis, values in sorted(rope_shifts.items()) + ), + ), + } diff --git a/SCAIL/scail2_loader.py b/SCAIL/scail2_loader.py new file mode 100644 index 00000000..a3b38821 --- /dev/null +++ b/SCAIL/scail2_loader.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from typing import Any + + +SCAIL2_MASK_CHANNELS = 28 + + +def _conv3d_shape(sd: dict[str, Any], key: str, *, dim: int, patch_size: tuple[int, int, int]) -> tuple[int, ...]: + weight = sd[key] + shape = getattr(weight, "shape", None) + if shape is None: + raise ValueError(f"{key} must expose a Conv3d weight shape") + + shape = tuple(int(value) for value in shape) + if len(shape) != 5: + raise ValueError(f"{key} must be a 5D Conv3d weight, got shape {shape}") + if shape[0] != dim: + raise ValueError(f"{key} output channels {shape[0]} do not match model dim {dim}") + if shape[2:] != tuple(patch_size): + raise ValueError(f"{key} kernel shape {shape[2:]} does not match patch_size {patch_size}") + return shape + + +def apply_scail_loader_patches( + transformer: Any, + sd: dict[str, Any], + nn: Any, + *, + dim: int, + patch_size: tuple[int, int, int], + log: Any, +) -> None: + transformer.scail2_enabled = False + transformer.scail2_mask_dim = None + + if "patch_embedding_pose.weight" in sd: + log.info("SCAIL model detected, patching model...") + pose_shape = _conv3d_shape( + sd, + "patch_embedding_pose.weight", + dim=dim, + patch_size=patch_size, + ) + pose_dim = pose_shape[1] + transformer.patch_embedding_pose = nn.Conv3d( + pose_dim, + dim, + kernel_size=patch_size, + stride=patch_size, + ) + + if "patch_embedding_mask.weight" in sd: + log.info("SCAIL-2 mask embedding detected, patching model...") + mask_shape = _conv3d_shape( + sd, + "patch_embedding_mask.weight", + dim=dim, + patch_size=patch_size, + ) + mask_dim = mask_shape[1] + if mask_dim != SCAIL2_MASK_CHANNELS: + raise ValueError( + "patch_embedding_mask.weight must use 28 input channels for SCAIL-2, " + f"got {mask_dim}" + ) + transformer.patch_embedding_mask = nn.Conv3d( + mask_dim, + dim, + kernel_size=patch_size, + stride=patch_size, + ) + transformer.scail2_enabled = True + transformer.scail2_mask_dim = mask_dim diff --git a/SCAIL/scail2_routing.py b/SCAIL/scail2_routing.py new file mode 100644 index 00000000..23286b31 --- /dev/null +++ b/SCAIL/scail2_routing.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +from typing import Any, Callable, MutableMapping + + +SCAIL_V1_EMBEDS_KEY = "scail_embeds" +SCAIL2_EMBEDS_KEY = "scail2_embeds" +SCAIL2_MODEL_ARG = "scail2_input" + + +def prepare_scail2_data( + image_embeds: MutableMapping[str, Any], + *, + dict_to_device: Callable[[dict[str, Any], Any, Any], dict[str, Any]], + device: Any, + dtype: Any, +) -> dict[str, Any] | None: + """Extract native SCAIL-2 embeds and move copied data to sampler device.""" + + scail2_embeds = image_embeds.get(SCAIL2_EMBEDS_KEY) + if scail2_embeds is None: + return None + + if image_embeds.get(SCAIL_V1_EMBEDS_KEY) is not None: + raise ValueError( + "SCAIL-2 native scail2_embeds cannot be combined with v1 scail_embeds" + ) + + if not isinstance(scail2_embeds, dict): + raise TypeError("image_embeds['scail2_embeds'] must be a dict") + + return dict_to_device(scail2_embeds.copy(), device, dtype) + + +def scail2_context_window_input( + scail2_data: dict[str, Any] | None, + context_window: Any, +) -> dict[str, Any] | None: + if scail2_data is None: + return None + + if context_window is None: + return scail2_data + + validate_scail2_context_frame_map(scail2_data, context_window) + + sliced = scail2_data.copy() + sliced["pose_latents"] = _slice_temporal_field( + scail2_data.get("pose_latents"), + context_window, + field_name="pose_latents", + ) + sliced["driving_masks"] = _slice_temporal_field( + scail2_data.get("driving_masks"), + context_window, + field_name="driving_masks", + ) + return sliced + + +def validate_scail2_context_frame_map( + scail2_data: dict[str, Any], + context_window: Any, +) -> None: + pose_counts = _temporal_frame_counts( + scail2_data.get("pose_latents"), + field_name="pose_latents", + ) + mask_counts = _temporal_frame_counts( + scail2_data.get("driving_masks"), + field_name="driving_masks", + ) + _validate_field_counts_consistent("pose_latents", pose_counts) + _validate_field_counts_consistent("driving_masks", mask_counts) + if pose_counts and mask_counts and sorted(set(pose_counts)) != sorted(set(mask_counts)): + raise ValueError( + "SCAIL-2 context frame map mismatch: " + f"pose_latents frames={list(pose_counts)} " + f"driving_masks frames={list(mask_counts)}" + ) + for field_name, counts in ( + ("pose_latents", pose_counts), + ("driving_masks", mask_counts), + ): + for frame_count in sorted(set(counts)): + _validate_context_indices( + context_window, + frame_count=frame_count, + field_name=field_name, + ) + + +def _temporal_frame_counts(value: Any, *, field_name: str) -> tuple[int, ...]: + if value is None: + return () + if isinstance(value, list): + return tuple( + count + for item in value + for count in _temporal_frame_counts(item, field_name=field_name) + ) + if isinstance(value, tuple): + return tuple( + count + for item in value + for count in _temporal_frame_counts(item, field_name=field_name) + ) + shape = getattr(value, "shape", None) + if shape is None or len(shape) < 2: + raise ValueError(f"SCAIL-2 {field_name} tensors must expose CTHW-like shape") + return (int(shape[1]),) + + +def _validate_field_counts_consistent(field_name: str, counts: tuple[int, ...]) -> None: + if len(set(counts)) > 1: + raise ValueError( + f"SCAIL-2 context frame map mismatch within {field_name}: " + f"frames={list(counts)}" + ) + + +def _validate_context_indices( + context_window: Any, + *, + frame_count: int, + field_name: str, +) -> None: + if isinstance(context_window, slice): + _context_window_length(context_window, frame_count) + return + if isinstance(context_window, (str, bytes)): + raise TypeError("SCAIL-2 context_window must be a sequence of frame indices") + length = _context_window_length(context_window, frame_count) + for raw_index in context_window: + index = int(raw_index) + if index < -frame_count or index >= frame_count: + raise ValueError( + "SCAIL-2 context_window index out of range: " + f"field={field_name} index={index} " + f"frame_count={frame_count} context_length={length}" + ) + + +def _slice_temporal_field( + value: Any, + context_window: Any, + *, + field_name: str, +) -> Any: + if value is None: + return None + if isinstance(value, list): + return [ + _slice_temporal_tensor(item, context_window, field_name=field_name) + for item in value + ] + if isinstance(value, tuple): + return tuple( + _slice_temporal_tensor(item, context_window, field_name=field_name) + for item in value + ) + return _slice_temporal_tensor(value, context_window, field_name=field_name) + + +def _slice_temporal_tensor(tensor: Any, context_window: Any, *, field_name: str) -> Any: + shape = getattr(tensor, "shape", None) + if shape is None or len(shape) < 2: + raise ValueError(f"SCAIL-2 {field_name} tensors must expose CTHW-like shape") + + expected_frames = _context_window_length(context_window, int(shape[1])) + try: + sliced = tensor[:, context_window] + except Exception as exc: + raise TypeError( + f"SCAIL-2 {field_name} cannot be sliced by wrapper context_window" + ) from exc + + sliced_shape = getattr(sliced, "shape", None) + if sliced_shape is None or len(sliced_shape) < 2: + raise ValueError( + f"SCAIL-2 {field_name} context slicing must preserve time dimension" + ) + if int(sliced_shape[1]) != expected_frames: + raise ValueError( + f"SCAIL-2 {field_name} context slice has {sliced_shape[1]} frames, " + f"expected {expected_frames}" + ) + return sliced + + +def _context_window_length(context_window: Any, frame_count: int) -> int: + if isinstance(context_window, slice): + return len(range(*context_window.indices(frame_count))) + if isinstance(context_window, (str, bytes)): + raise TypeError("SCAIL-2 context_window must be a sequence of frame indices") + try: + length = len(context_window) + except TypeError as exc: + raise TypeError( + "SCAIL-2 context_window must be a sequence of frame indices" + ) from exc + if length <= 0: + raise ValueError("SCAIL-2 context_window must contain at least one frame") + return int(length) + + +def add_scail2_model_param( + base_params: dict[str, Any], + scail2_data_in: dict[str, Any] | None, +) -> dict[str, Any]: + if scail2_data_in is not None: + base_params[SCAIL2_MODEL_ARG] = scail2_data_in + return base_params diff --git a/example_workflows/wanvideo_2_1_14B_SCAIL2_replacement_and_animate_dual_mode_example_01.json b/example_workflows/wanvideo_2_1_14B_SCAIL2_replacement_and_animate_dual_mode_example_01.json new file mode 100644 index 00000000..53dfa155 --- /dev/null +++ b/example_workflows/wanvideo_2_1_14B_SCAIL2_replacement_and_animate_dual_mode_example_01.json @@ -0,0 +1,5645 @@ +{ + "id": "8e186cd6-9111-4570-aa94-fccd0efefb3d", + "revision": 0, + "last_node_id": 486, + "last_link_id": 861, + "nodes": [ + { + "id": 22, + "type": "WanVideoModelLoader", + "pos": [ + 406.75567915179795, + -2634.510622288314 + ], + "size": [ + 433.3047848341723, + 338 + ], + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "label": "compile_args", + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS" + }, + { + "label": "block_swap_args", + "name": "block_swap_args", + "shape": 7, + "type": "BLOCKSWAPARGS", + "link": 828 + }, + { + "label": "lora", + "name": "lora", + "shape": 7, + "type": "WANVIDLORA" + }, + { + "label": "vram_management_args", + "name": "vram_management_args", + "shape": 7, + "type": "VRAM_MANAGEMENTARGS" + }, + { + "label": "extra_model", + "name": "extra_model", + "shape": 7, + "type": "VACEPATH" + }, + { + "label": "fantasytalking_model", + "name": "fantasytalking_model", + "shape": 7, + "type": "FANTASYTALKINGMODEL" + }, + { + "label": "multitalk_model", + "name": "multitalk_model", + "shape": 7, + "type": "MULTITALKMODEL" + }, + { + "label": "fantasyportrait_model", + "name": "fantasyportrait_model", + "shape": 7, + "type": "FANTASYPORTRAITMODEL" + }, + { + "label": "model", + "name": "model", + "type": "COMBO", + "widget": { + "name": "model" + } + }, + { + "label": "base_precision", + "name": "base_precision", + "type": "COMBO", + "widget": { + "name": "base_precision" + } + }, + { + "label": "quantization", + "name": "quantization", + "type": "COMBO", + "widget": { + "name": "quantization" + } + }, + { + "label": "load_device", + "name": "load_device", + "type": "COMBO", + "widget": { + "name": "load_device" + } + }, + { + "label": "attention_mode", + "name": "attention_mode", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "attention_mode" + } + }, + { + "label": "rms_norm_function", + "name": "rms_norm_function", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "rms_norm_function" + } + }, + { + "label": "vace_model", + "name": "vace_model", + "shape": 7, + "type": "VACEPATH" + } + ], + "outputs": [ + { + "label": "model", + "name": "model", + "type": "WANVIDEOMODEL", + "slot_index": 0, + "links": [ + 655 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "998a69cc0acbec503001b8b0ce0a5d5404420e1e", + "Node name for S&R": "WanVideoModelLoader", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Wan\\wan2.1_14B_SCAIL_2_fp8_scaled.safetensors", + "fp16", + "disabled", + "main_device", + "sageattn", + "default" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 28, + "type": "WanVideoDecode", + "pos": [ + 2156.8703818815693, + -2558.1337267059066 + ], + "size": [ + 210, + 198 + ], + "flags": {}, + "order": 67, + "mode": 0, + "inputs": [ + { + "label": "vae", + "name": "vae", + "type": "WANVAE", + "link": 831 + }, + { + "label": "samples", + "name": "samples", + "type": "LATENT", + "link": 596 + }, + { + "label": "enable_vae_tiling", + "name": "enable_vae_tiling", + "type": "BOOLEAN", + "widget": { + "name": "enable_vae_tiling" + } + }, + { + "label": "tile_x", + "name": "tile_x", + "type": "INT", + "widget": { + "name": "tile_x" + } + }, + { + "label": "tile_y", + "name": "tile_y", + "type": "INT", + "widget": { + "name": "tile_y" + } + }, + { + "label": "tile_stride_x", + "name": "tile_stride_x", + "type": "INT", + "widget": { + "name": "tile_stride_x" + } + }, + { + "label": "tile_stride_y", + "name": "tile_stride_y", + "type": "INT", + "widget": { + "name": "tile_stride_y" + } + }, + { + "label": "normalization", + "name": "normalization", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "normalization" + } + } + ], + "outputs": [ + { + "label": "images", + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 656, + 657 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "998a69cc0acbec503001b8b0ce0a5d5404420e1e", + "Node name for S&R": "WanVideoDecode", + "widget_ue_connectable": {} + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + "default" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 38, + "type": "WanVideoVAELoader", + "pos": [ + 37.226264142663304, + -2232.913330685107 + ], + "size": [ + 319.12502108487877, + 130 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "label": "compile_args", + "name": "compile_args", + "shape": 7, + "type": "WANCOMPILEARGS" + }, + { + "label": "model_name", + "name": "model_name", + "type": "COMBO", + "widget": { + "name": "model_name" + } + }, + { + "label": "precision", + "name": "precision", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "precision" + } + }, + { + "label": "use_cpu_cache", + "name": "use_cpu_cache", + "shape": 7, + "type": "BOOLEAN", + "widget": { + "name": "use_cpu_cache" + } + } + ], + "outputs": [ + { + "label": "vae", + "name": "vae", + "type": "WANVAE", + "slot_index": 0, + "links": [ + 700 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "998a69cc0acbec503001b8b0ce0a5d5404420e1e", + "Node name for S&R": "WanVideoVAELoader", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Wan21_vae.pth", + "fp32", + false, + false + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 80, + "type": "WanVideoSetLoRAs", + "pos": [ + 1188.1296785853535, + -2588.1317341767235 + ], + "size": [ + 222.27981567382812, + 46 + ], + "flags": { + "collapsed": true + }, + "order": 39, + "mode": 0, + "inputs": [ + { + "label": "model", + "name": "model", + "type": "WANVIDEOMODEL", + "link": 655 + }, + { + "label": "lora", + "name": "lora", + "shape": 7, + "type": "WANVIDLORA", + "link": 664 + } + ], + "outputs": [ + { + "label": "model", + "name": "model", + "type": "WANVIDEOMODEL", + "links": [ + 696 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "998a69cc0acbec503001b8b0ce0a5d5404420e1e", + "Node name for S&R": "WanVideoSetLoRAs", + "widget_ue_connectable": {} + }, + "widgets_values": [], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 99, + "type": "WanVideoEmptyEmbeds", + "pos": [ + 778.7827323504215, + -1560.1613228747765 + ], + "size": [ + 310.44073486328125, + 130.36167907714844 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "label": "control_embeds", + "name": "control_embeds", + "shape": 7, + "type": "WANVIDIMAGE_EMBEDS" + }, + { + "label": "extra_latents", + "name": "extra_latents", + "shape": 7, + "type": "LATENT" + }, + { + "label": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 727 + }, + { + "label": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 728 + }, + { + "label": "num_frames", + "name": "num_frames", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "link": 650 + } + ], + "outputs": [ + { + "label": "image_embeds", + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 787 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "0f217be4d8742741b0f89db50138214302a58dc3", + "Node name for S&R": "WanVideoEmptyEmbeds", + "widget_ue_connectable": { + "width": true, + "num_frames": true, + "height": true + } + }, + "widgets_values": [ + 480, + 1216, + 65 + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 106, + "type": "LoadImage", + "pos": [ + -1281.9405617212324, + -1842.4062280499497 + ], + "size": [ + 269.19939507501977, + 440 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "label": "image", + "name": "image", + "type": "COMBO", + "widget": { + "name": "image" + } + }, + { + "label": "upload", + "name": "upload", + "type": "IMAGEUPLOAD", + "widget": { + "name": "upload" + } + } + ], + "outputs": [ + { + "label": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 764 + ] + }, + { + "label": "MASK", + "name": "MASK", + "type": "MASK" + } + ], + "title": "Load Image: Reference", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.76", + "Node name for S&R": "LoadImage", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "ChatGPT Image 2026年6月25日 上午02_11_29.png", + "image" + ] + }, + { + "id": 137, + "type": "VHS_VideoCombine", + "pos": [ + -345.5823647384095, + -2251.7048696521515 + ], + "size": [ + 296.91351932288023, + 334 + ], + "flags": {}, + "order": 63, + "mode": 0, + "inputs": [ + { + "label": "images", + "name": "images", + "type": "IMAGE", + "link": 826 + }, + { + "label": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO" + }, + { + "label": "meta_batch", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager" + }, + { + "label": "vae", + "name": "vae", + "shape": 7, + "type": "VAE" + }, + { + "label": "frame_rate", + "name": "frame_rate", + "type": "FLOAT", + "widget": { + "name": "frame_rate" + }, + "link": 660 + }, + { + "label": "loop_count", + "name": "loop_count", + "type": "INT", + "widget": { + "name": "loop_count" + } + }, + { + "label": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + } + }, + { + "label": "format", + "name": "format", + "type": "COMBO", + "widget": { + "name": "format" + } + }, + { + "label": "pingpong", + "name": "pingpong", + "type": "BOOLEAN", + "widget": { + "name": "pingpong" + } + }, + { + "label": "save_output", + "name": "save_output", + "type": "BOOLEAN", + "widget": { + "name": "save_output" + } + } + ], + "outputs": [ + { + "label": "Filenames", + "name": "Filenames", + "type": "VHS_FILENAMES" + } + ], + "properties": { + "aux_id": "rookiestar28/ComfyUI-VideoHelper_Adv", + "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", + "Node name for S&R": "VHS_VideoCombine", + "cnr_id": "comfyui-videohelpersuite", + "widget_ue_connectable": {} + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "onetotall_pose", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": false, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "paused": false, + "hidden": false, + "params": { + "score": "0", + "filename": "onetotall_pose_00002.mp4", + "workflow": "onetotall_pose_00001_p80.png", + "fullpath": "A:\\ComfyUI\\temp\\onetotall_pose_00002.mp4", + "format": "video/h264-mp4", + "subfolder": "", + "label": "Normal", + "type": "temp", + "frame_rate": 24 + }, + "error": "Preview unavailable. Check VHS debug logs for details." + } + } + }, + { + "id": 139, + "type": "VHS_VideoCombine", + "pos": [ + 2410.972606155146, + -2600.57513218037 + ], + "size": [ + 434.2347106933594, + 952.0922864278158 + ], + "flags": {}, + "order": 68, + "mode": 0, + "inputs": [ + { + "label": "images", + "name": "images", + "type": "IMAGE", + "link": 656 + }, + { + "label": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 722 + }, + { + "label": "meta_batch", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager" + }, + { + "label": "vae", + "name": "vae", + "shape": 7, + "type": "VAE" + }, + { + "label": "frame_rate", + "name": "frame_rate", + "type": "FLOAT", + "widget": { + "name": "frame_rate" + }, + "link": 661 + }, + { + "label": "loop_count", + "name": "loop_count", + "type": "INT", + "widget": { + "name": "loop_count" + } + }, + { + "label": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + } + }, + { + "label": "format", + "name": "format", + "type": "COMBO", + "widget": { + "name": "format" + } + }, + { + "label": "pingpong", + "name": "pingpong", + "type": "BOOLEAN", + "widget": { + "name": "pingpong" + } + }, + { + "label": "save_output", + "name": "save_output", + "type": "BOOLEAN", + "widget": { + "name": "save_output" + } + } + ], + "outputs": [ + { + "label": "Filenames", + "name": "Filenames", + "type": "VHS_FILENAMES" + } + ], + "properties": { + "aux_id": "rookiestar28/ComfyUI-VideoHelper_Adv", + "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", + "Node name for S&R": "VHS_VideoCombine", + "cnr_id": "comfyui-videohelpersuite", + "widget_ue_connectable": {} + }, + "widgets_values": { + "frame_rate": 30, + "loop_count": 0, + "filename_prefix": "SCAIL", + "format": "video/nvenc_av1-mp4", + "pix_fmt": "p010le", + "bitrate": 15, + "megabit": true, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "paused": false, + "hidden": false, + "params": { + "score": "0", + "filename": "SCAIL_00068-audio.mp4", + "workflow": "WanVideo_SCAIL_00001_p80.png", + "fullpath": "A:\\ComfyUI\\output\\SCAIL_00068-audio.mp4", + "format": "video/nvenc_av1-mp4", + "subfolder": "", + "label": "Normal", + "type": "output", + "frame_rate": 24 + }, + "error": "" + } + } + }, + { + "id": 184, + "type": "SetNode", + "pos": [ + -687.3950295605496, + -1889.0762891866077 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 49, + "mode": 0, + "inputs": [ + { + "label": "INT", + "name": "INT", + "type": "INT", + "link": 716 + } + ], + "outputs": [ + { + "label": "*", + "name": "INT", + "type": "INT", + "links": [] + } + ], + "title": "Set_gen_width", + "properties": { + "Node name for S&R": "SetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "previousName": "gen_width", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "gen_width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 185, + "type": "SetNode", + "pos": [ + -517.2623759030691, + -1890.2335036272173 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 50, + "mode": 0, + "inputs": [ + { + "label": "INT", + "name": "INT", + "type": "INT", + "link": 717 + } + ], + "outputs": [ + { + "label": "*", + "name": "INT", + "type": "INT", + "links": [] + } + ], + "title": "Set_gen_height", + "properties": { + "Node name for S&R": "SetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "previousName": "gen_height", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "gen_height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 186, + "type": "GetNode", + "pos": [ + 541.641530903563, + -1508.2545140325485 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "INT", + "name": "INT", + "type": "INT", + "links": [ + 727 + ] + } + ], + "title": "Get_gen_width", + "properties": { + "Node name for S&R": "GetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "gen_width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 187, + "type": "GetNode", + "pos": [ + 539.2201194800551, + -1556.2213787211176 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "INT", + "name": "INT", + "type": "INT", + "links": [ + 728 + ] + } + ], + "title": "Get_gen_height", + "properties": { + "Node name for S&R": "GetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "gen_height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 206, + "type": "SetNode", + "pos": [ + -517.8576659112829, + -2604.8886310508574 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 43, + "mode": 0, + "inputs": [ + { + "label": "INT", + "name": "INT", + "type": "INT", + "link": 707 + } + ], + "outputs": [ + { + "label": "*", + "name": "INT", + "type": "INT", + "links": [ + 859 + ] + } + ], + "title": "Set_width", + "properties": { + "Node name for S&R": "SetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "previousName": "width", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 207, + "type": "SetNode", + "pos": [ + -511.55354550178043, + -2555.382112187544 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 44, + "mode": 0, + "inputs": [ + { + "label": "INT", + "name": "INT", + "type": "INT", + "link": 708 + } + ], + "outputs": [ + { + "label": "*", + "name": "INT", + "type": "INT", + "links": [ + 860 + ] + } + ], + "title": "Set_height", + "properties": { + "Node name for S&R": "SetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "previousName": "height", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 208, + "type": "GetNode", + "pos": [ + -852.9039379940787, + -1894.2511893170927 + ], + "size": [ + 210, + 50 + ], + "flags": { + "collapsed": true + }, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "INT", + "name": "INT", + "type": "INT", + "links": [ + 741 + ] + } + ], + "title": "Get_width", + "properties": { + "Node name for S&R": "GetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 209, + "type": "GetNode", + "pos": [ + -987.6082497012932, + -1893.6771553561034 + ], + "size": [ + 210, + 50 + ], + "flags": { + "collapsed": true + }, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "INT", + "name": "INT", + "type": "INT", + "links": [ + 740 + ] + } + ], + "title": "Get_height", + "properties": { + "Node name for S&R": "GetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 238, + "type": "FloatConstant", + "pos": [ + -1622.6524420933536, + -2208.491096747343 + ], + "size": [ + 210, + 72.86605072021484 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "label": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + } + } + ], + "outputs": [ + { + "label": "value", + "name": "value", + "type": "FLOAT", + "links": [ + 414 + ] + } + ], + "title": "CFG", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "50e7dd34d3b6e6bbab1d41e8068e1ddd19bd4d1b", + "Node name for S&R": "FloatConstant", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 1 + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 239, + "type": "SetNode", + "pos": [ + -1389.0156179588353, + -2180.8781944458206 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 30, + "mode": 0, + "inputs": [ + { + "label": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "link": 414 + } + ], + "outputs": [ + { + "label": "*", + "name": "*", + "type": "*" + } + ], + "title": "Set_cfg", + "properties": { + "Node name for S&R": "SetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "previousName": "cfg", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "cfg" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 240, + "type": "GetNode", + "pos": [ + 1722.7091122024583, + -2617.5769979052225 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 601 + ] + } + ], + "title": "Get_cfg", + "properties": { + "Node name for S&R": "GetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "cfg" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 318, + "type": "ImageConcatMulti", + "pos": [ + 2886.9486823435186, + -2399.844515024753 + ], + "size": [ + 270, + 150 + ], + "flags": {}, + "order": 69, + "mode": 0, + "inputs": [ + { + "label": "image_1", + "name": "image_1", + "type": "IMAGE,MASK", + "link": 657 + }, + { + "label": "image_2", + "name": "image_2", + "shape": 7, + "type": "IMAGE,MASK", + "link": 559 + }, + { + "label": "inputcount", + "name": "inputcount", + "type": "INT", + "widget": { + "name": "inputcount" + } + }, + { + "label": "direction", + "name": "direction", + "type": "COMBO", + "widget": { + "name": "direction" + } + }, + { + "label": "match_image_size", + "name": "match_image_size", + "type": "BOOLEAN", + "widget": { + "name": "match_image_size" + } + } + ], + "outputs": [ + { + "label": "images", + "name": "output", + "type": "IMAGE", + "links": [ + 651 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "ff7f876c09ff5a53be051073c3c1731b96ee31a3", + "widget_ue_connectable": {}, + "Node name for S&R": "ImageConcatMulti" + }, + "widgets_values": [ + 2, + "left", + true, + null + ] + }, + { + "id": 319, + "type": "VHS_VideoCombine", + "pos": [ + 3196.9588303569194, + -2592.0002689036155 + ], + "size": [ + 768.5992088421362, + 1075.804786374299 + ], + "flags": {}, + "order": 70, + "mode": 0, + "inputs": [ + { + "label": "images", + "name": "images", + "type": "IMAGE", + "link": 651 + }, + { + "label": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 723 + }, + { + "label": "meta_batch", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager" + }, + { + "label": "vae", + "name": "vae", + "shape": 7, + "type": "VAE" + }, + { + "label": "frame_rate", + "name": "frame_rate", + "type": "FLOAT", + "widget": { + "name": "frame_rate" + }, + "link": 671 + }, + { + "label": "loop_count", + "name": "loop_count", + "type": "INT", + "widget": { + "name": "loop_count" + } + }, + { + "label": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + } + }, + { + "label": "format", + "name": "format", + "type": "COMBO", + "widget": { + "name": "format" + } + }, + { + "label": "pingpong", + "name": "pingpong", + "type": "BOOLEAN", + "widget": { + "name": "pingpong" + } + }, + { + "label": "save_output", + "name": "save_output", + "type": "BOOLEAN", + "widget": { + "name": "save_output" + } + } + ], + "outputs": [ + { + "label": "Filenames", + "name": "Filenames", + "type": "VHS_FILENAMES" + } + ], + "properties": { + "aux_id": "rookiestar28/ComfyUI-VideoHelper_Adv", + "ver": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", + "Node name for S&R": "VHS_VideoCombine", + "cnr_id": "comfyui-videohelpersuite", + "widget_ue_connectable": {} + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "WanVideo_SCAIL", + "format": "video/nvenc_av1-mp4", + "pix_fmt": "yuv420p", + "bitrate": 15, + "megabit": true, + "save_metadata": false, + "pingpong": false, + "save_output": true, + "videopreview": { + "paused": false, + "hidden": false, + "params": { + "filename": "WanVideo_SCAIL_00045-audio.mp4", + "workflow": "WanVideo_SCAIL_00002.png", + "fullpath": "A:\\ComfyUI\\output\\WanVideo_SCAIL_00045-audio.mp4", + "format": "video/nvenc_av1-mp4", + "subfolder": "", + "type": "output", + "frame_rate": 24 + }, + "error": "" + } + } + }, + { + "id": 326, + "type": "CLIPVisionLoader", + "pos": [ + 81.79898735699742, + -1778.3026383710176 + ], + "size": [ + 380.8835245631712, + 58 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "label": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + } + } + ], + "outputs": [ + { + "label": "CLIP_VISION", + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 554 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.4.0", + "Node name for S&R": "CLIPVisionLoader", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "clip_vision_h.safetensors" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 327, + "type": "WanVideoClipVisionEncode", + "pos": [ + 89.45562957846558, + -1668.8931135125877 + ], + "size": [ + 367.40594893902175, + 262 + ], + "flags": {}, + "order": 52, + "mode": 0, + "inputs": [ + { + "label": "clip_vision", + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 554 + }, + { + "label": "image_1", + "name": "image_1", + "type": "IMAGE", + "link": 611 + }, + { + "label": "image_2", + "name": "image_2", + "shape": 7, + "type": "IMAGE" + }, + { + "label": "negative_image", + "name": "negative_image", + "shape": 7, + "type": "IMAGE" + }, + { + "label": "strength_1", + "name": "strength_1", + "type": "FLOAT", + "widget": { + "name": "strength_1" + } + }, + { + "label": "strength_2", + "name": "strength_2", + "type": "FLOAT", + "widget": { + "name": "strength_2" + } + }, + { + "label": "crop", + "name": "crop", + "type": "COMBO", + "widget": { + "name": "crop" + } + }, + { + "label": "combine_embeds", + "name": "combine_embeds", + "type": "COMBO", + "widget": { + "name": "combine_embeds" + } + }, + { + "label": "force_offload", + "name": "force_offload", + "type": "BOOLEAN", + "widget": { + "name": "force_offload" + } + }, + { + "label": "tiles", + "name": "tiles", + "shape": 7, + "type": "INT", + "widget": { + "name": "tiles" + } + }, + { + "label": "ratio", + "name": "ratio", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "ratio" + } + } + ], + "outputs": [ + { + "label": "image_embeds", + "name": "image_embeds", + "type": "WANVIDIMAGE_CLIPEMBEDS", + "links": [ + 791 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "4a3ab6958a5d9650eb4e3c2eb1753c2d7c5b77c6", + "Node name for S&R": "WanVideoClipVisionEncode", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 1.2, + 1, + "disabled", + "average", + false, + 0, + 0.5 + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 328, + "type": "ImageConcatMulti", + "pos": [ + 2882.8212968846383, + -2591.2294974073434 + ], + "size": [ + 270, + 150 + ], + "flags": {}, + "order": 54, + "mode": 0, + "inputs": [ + { + "label": "image_1", + "name": "image_1", + "type": "IMAGE,MASK", + "link": 851 + }, + { + "label": "image_2", + "name": "image_2", + "shape": 7, + "type": "IMAGE,MASK", + "link": 710 + }, + { + "label": "inputcount", + "name": "inputcount", + "type": "INT", + "widget": { + "name": "inputcount" + } + }, + { + "label": "direction", + "name": "direction", + "type": "COMBO", + "widget": { + "name": "direction" + } + }, + { + "label": "match_image_size", + "name": "match_image_size", + "type": "BOOLEAN", + "widget": { + "name": "match_image_size" + } + } + ], + "outputs": [ + { + "label": "images", + "name": "output", + "type": "IMAGE", + "links": [ + 559 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "ff7f876c09ff5a53be051073c3c1731b96ee31a3", + "widget_ue_connectable": {}, + "Node name for S&R": "ImageConcatMulti" + }, + "widgets_values": [ + 2, + "down", + true, + null + ] + }, + { + "id": 334, + "type": "NLFPredict", + "pos": [ + -667.2843405337521, + -2467.5700328297803 + ], + "size": [ + 286.90301513671875, + 78 + ], + "flags": {}, + "order": 51, + "mode": 0, + "inputs": [ + { + "label": "model", + "name": "model", + "type": "NLFMODEL", + "link": 570 + }, + { + "label": "images", + "name": "images", + "type": "IMAGE", + "link": 793 + }, + { + "label": "per_batch", + "name": "per_batch", + "shape": 7, + "type": "INT", + "widget": { + "name": "per_batch" + } + } + ], + "outputs": [ + { + "label": "pose_results", + "name": "pose_results", + "type": "NLFPRED", + "links": [ + 621 + ] + }, + { + "label": "bboxes", + "name": "bboxes", + "type": "BBOX" + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "4a3ab6958a5d9650eb4e3c2eb1753c2d7c5b77c6", + "Node name for S&R": "NLFPredict", + "widget_ue_connectable": {} + }, + "widgets_values": [ + -1 + ] + }, + { + "id": 335, + "type": "DownloadAndLoadNLFModel", + "pos": [ + -1028.5779421391826, + -2172.564084622057 + ], + "size": [ + 390, + 82 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "label": "url", + "name": "url", + "type": "COMBO", + "widget": { + "name": "url" + } + }, + { + "label": "warmup", + "name": "warmup", + "shape": 7, + "type": "BOOLEAN", + "widget": { + "name": "warmup" + } + } + ], + "outputs": [ + { + "label": "nlf_model", + "name": "nlf_model", + "type": "NLFMODEL", + "links": [ + 570 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "4a3ab6958a5d9650eb4e3c2eb1753c2d7c5b77c6", + "Node name for S&R": "DownloadAndLoadNLFModel", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "https://github.com/isarandi/nlf/releases/download/v0.3.2/nlf_l_multi_0.3.2.torchscript", + true + ] + }, + { + "id": 348, + "type": "WanVideoSamplerv2", + "pos": [ + 1830.6902649743722, + -2554.2758776644364 + ], + "size": [ + 292.26393847730924, + 620.9347350631786 + ], + "flags": {}, + "order": 66, + "mode": 0, + "inputs": [ + { + "label": "model", + "name": "model", + "type": "WANVIDEOMODEL", + "link": 696 + }, + { + "label": "image_embeds", + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 789 + }, + { + "label": "scheduler", + "name": "scheduler", + "type": "WANVIDEOSCHEDULER", + "link": 598 + }, + { + "label": "text_embeds", + "name": "text_embeds", + "shape": 7, + "type": "WANVIDEOTEXTEMBEDS", + "link": 703 + }, + { + "label": "samples", + "name": "samples", + "shape": 7, + "type": "LATENT", + "link": 834 + }, + { + "label": "extra_args", + "name": "extra_args", + "shape": 7, + "type": "WANVIDSAMPLEREXTRAARGS", + "link": 652 + }, + { + "label": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 601 + }, + { + "label": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + } + }, + { + "label": "force_offload", + "name": "force_offload", + "type": "BOOLEAN", + "widget": { + "name": "force_offload" + } + }, + { + "label": "add_noise_to_samples", + "name": "add_noise_to_samples", + "shape": 7, + "type": "BOOLEAN", + "widget": { + "name": "add_noise_to_samples" + } + } + ], + "outputs": [ + { + "label": "samples", + "name": "samples", + "type": "LATENT", + "links": [ + 596 + ] + }, + { + "label": "denoised_samples", + "name": "denoised_samples", + "type": "LATENT" + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "98f8e56bcacfc07e12cbb4b26555b2b28d9db92f", + "Node name for S&R": "WanVideoSamplerv2", + "widget_ue_connectable": { + "cfg": true + } + }, + "widgets_values": [ + 6, + 343301095742529, + "randomize", + true, + true + ] + }, + { + "id": 349, + "type": "WanVideoSchedulerv2", + "pos": [ + 1507.088713520658, + -2462.2588918327515 + ], + "size": [ + 255, + 228 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "label": "sigmas", + "name": "sigmas", + "shape": 7, + "type": "SIGMAS" + }, + { + "label": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + } + }, + { + "label": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + } + }, + { + "label": "shift", + "name": "shift", + "type": "FLOAT", + "widget": { + "name": "shift" + } + }, + { + "label": "start_step", + "name": "start_step", + "type": "INT", + "widget": { + "name": "start_step" + } + }, + { + "label": "end_step", + "name": "end_step", + "type": "INT", + "widget": { + "name": "end_step" + } + } + ], + "outputs": [ + { + "label": "scheduler", + "name": "scheduler", + "type": "WANVIDEOSCHEDULER", + "links": [ + 598 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "98f8e56bcacfc07e12cbb4b26555b2b28d9db92f", + "Node name for S&R": "WanVideoSchedulerv2", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "unipc/beta", + 4, + 7, + 0, + -1, + false + ] + }, + { + "id": 351, + "type": "Reroute", + "pos": [ + -390.5152635467508, + -1385.1320832758138 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 48, + "mode": 0, + "inputs": [ + { + "label": "", + "name": "", + "type": "*", + "link": 850 + } + ], + "outputs": [ + { + "label": "", + "name": "", + "type": "IMAGE", + "links": [ + 611, + 770, + 798, + 851 + ] + } + ], + "properties": { + "showOutputText": false, + "horizontal": false, + "widget_ue_connectable": {} + } + }, + { + "id": 354, + "type": "WanVideoSamplerExtraArgs", + "pos": [ + 1507.088713520658, + -2157.654441411218 + ], + "size": [ + 255, + 262 + ], + "flags": { + "collapsed": false + }, + "order": 38, + "mode": 0, + "inputs": [ + { + "label": "feta_args", + "name": "feta_args", + "shape": 7, + "type": "FETAARGS", + "link": null + }, + { + "label": "context_options", + "name": "context_options", + "shape": 7, + "type": "WANVIDCONTEXT", + "link": 614 + }, + { + "label": "cache_args", + "name": "cache_args", + "shape": 7, + "type": "CACHEARGS" + }, + { + "label": "slg_args", + "name": "slg_args", + "shape": 7, + "type": "SLGARGS", + "link": null + }, + { + "label": "loop_args", + "name": "loop_args", + "shape": 7, + "type": "LOOPARGS", + "link": null + }, + { + "label": "experimental_args", + "name": "experimental_args", + "shape": 7, + "type": "EXPERIMENTALARGS", + "link": null + }, + { + "label": "unianimate_poses", + "name": "unianimate_poses", + "shape": 7, + "type": "UNIANIMATE_POSE", + "link": null + }, + { + "label": "fantasytalking_embeds", + "name": "fantasytalking_embeds", + "shape": 7, + "type": "FANTASYTALKING_EMBEDS" + }, + { + "label": "uni3c_embeds", + "name": "uni3c_embeds", + "shape": 7, + "type": "UNI3C_EMBEDS" + }, + { + "label": "multitalk_embeds", + "name": "multitalk_embeds", + "shape": 7, + "type": "MULTITALK_EMBEDS" + }, + { + "label": "riflex_freq_index", + "name": "riflex_freq_index", + "shape": 7, + "type": "INT", + "widget": { + "name": "riflex_freq_index" + } + }, + { + "label": "rope_function", + "name": "rope_function", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "rope_function" + } + } + ], + "outputs": [ + { + "label": "extra_args", + "name": "extra_args", + "type": "WANVIDSAMPLEREXTRAARGS", + "links": [ + 652 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "98f8e56bcacfc07e12cbb4b26555b2b28d9db92f", + "Node name for S&R": "WanVideoSamplerExtraArgs", + "widget_ue_connectable": {} + }, + "widgets_values": [ + 0, + "comfy" + ] + }, + { + "id": 355, + "type": "WanVideoContextOptions", + "pos": [ + 1190.0248450577226, + -2141.1694928551838 + ], + "size": [ + 290, + 202 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "label": "reference_latent", + "name": "reference_latent", + "shape": 7, + "type": "LATENT" + }, + { + "label": "context_schedule", + "name": "context_schedule", + "type": "COMBO", + "widget": { + "name": "context_schedule" + } + }, + { + "label": "context_frames", + "name": "context_frames", + "type": "INT", + "widget": { + "name": "context_frames" + }, + "link": null + }, + { + "label": "context_stride", + "name": "context_stride", + "type": "INT", + "widget": { + "name": "context_stride" + } + }, + { + "label": "context_overlap", + "name": "context_overlap", + "type": "INT", + "widget": { + "name": "context_overlap" + }, + "link": null + }, + { + "label": "freenoise", + "name": "freenoise", + "type": "BOOLEAN", + "widget": { + "name": "freenoise" + } + }, + { + "label": "verbose", + "name": "verbose", + "type": "BOOLEAN", + "widget": { + "name": "verbose" + } + }, + { + "label": "fuse_method", + "name": "fuse_method", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "fuse_method" + } + } + ], + "outputs": [ + { + "label": "context_options", + "name": "context_options", + "type": "WANVIDCONTEXT", + "links": [ + 614 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "98f8e56bcacfc07e12cbb4b26555b2b28d9db92f", + "Node name for S&R": "WanVideoContextOptions", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "uniform_standard", + 81, + 4, + 48, + true, + false, + "linear" + ] + }, + { + "id": 362, + "type": "RenderNLFPoses", + "pos": [ + -350.33560634357934, + -2626.770868387454 + ], + "size": [ + 304.48026401138816, + 282 + ], + "flags": {}, + "order": 57, + "mode": 0, + "inputs": [ + { + "label": "nlf_poses", + "name": "nlf_poses", + "type": "NLFPRED", + "link": 621 + }, + { + "label": "dw_poses", + "name": "dw_poses", + "shape": 7, + "type": "DWPOSES", + "link": 626 + }, + { + "label": "ref_dw_pose", + "name": "ref_dw_pose", + "shape": 7, + "type": "DWPOSES", + "link": 633 + }, + { + "name": "bboxes", + "shape": 7, + "type": "BBOX", + "link": null + }, + { + "name": "pose_video_mask", + "shape": 7, + "type": "IMAGE", + "link": 844 + }, + { + "name": "render_width", + "type": "INT", + "widget": { + "name": "render_width" + }, + "link": 859 + }, + { + "name": "render_height", + "type": "INT", + "widget": { + "name": "render_height" + }, + "link": 860 + }, + { + "label": "draw_face", + "name": "draw_face", + "shape": 7, + "type": "BOOLEAN", + "widget": { + "name": "draw_face" + } + }, + { + "label": "draw_hands", + "name": "draw_hands", + "shape": 7, + "type": "BOOLEAN", + "widget": { + "name": "draw_hands" + } + } + ], + "outputs": [ + { + "label": "image", + "name": "image", + "type": "IMAGE", + "links": [ + 843 + ] + }, + { + "label": "mask", + "name": "mask", + "type": "MASK", + "links": [] + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-SCAIL-Pose", + "Node name for S&R": "RenderNLFPoses", + "cnr_id": "ComfyUI-SCAIL-Pose", + "ver": "93059f0fe4ea3881c0d6ec0e327e5d920554277d", + "widget_ue_connectable": { + "width": true, + "height": true + } + }, + "widgets_values": [ + 512, + 896, + true, + true, + "gpu", + false, + "taichi" + ] + }, + { + "id": 363, + "type": "PoseDetectionVitPoseToDWPose", + "pos": [ + -646.8590010135549, + -2316.056477871831 + ], + "size": [ + 280, + 46.75394058227539 + ], + "flags": { + "collapsed": true + }, + "order": 40, + "mode": 0, + "inputs": [ + { + "label": "vitpose_model", + "name": "vitpose_model", + "type": "POSEMODEL", + "link": 627 + }, + { + "label": "images", + "name": "images", + "type": "IMAGE", + "link": 706 + } + ], + "outputs": [ + { + "label": "dw_poses", + "name": "dw_poses", + "type": "DWPOSES", + "links": [ + 626 + ] + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-SCAIL-Pose", + "Node name for S&R": "PoseDetectionVitPoseToDWPose", + "cnr_id": "ComfyUI-SCAIL-Pose", + "ver": "93059f0fe4ea3881c0d6ec0e327e5d920554277d", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 364, + "type": "OnnxDetectionModelLoader", + "pos": [ + -1028.8480414197982, + -2044.6152865047302 + ], + "size": [ + 390, + 106 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "label": "vitpose_model", + "name": "vitpose_model", + "type": "COMBO", + "widget": { + "name": "vitpose_model" + } + }, + { + "label": "yolo_model", + "name": "yolo_model", + "type": "COMBO", + "widget": { + "name": "yolo_model" + } + }, + { + "label": "onnx_device", + "name": "onnx_device", + "type": "COMBO", + "widget": { + "name": "onnx_device" + } + } + ], + "outputs": [ + { + "label": "model", + "name": "model", + "type": "POSEMODEL", + "links": [ + 627, + 629 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanAnimatePreprocess", + "ver": "74f834b9ecec0e9f48535ab2ec95219f5288018a", + "Node name for S&R": "OnnxDetectionModelLoader", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "vitpose-l-wholebody.onnx", + "yolov10m.onnx", + "CUDAExecutionProvider" + ] + }, + { + "id": 365, + "type": "PoseDetectionVitPoseToDWPose", + "pos": [ + -641.8246696449892, + -2261.262160437339 + ], + "size": [ + 280, + 46.75394058227539 + ], + "flags": { + "collapsed": true + }, + "order": 46, + "mode": 0, + "inputs": [ + { + "label": "vitpose_model", + "name": "vitpose_model", + "type": "POSEMODEL", + "link": 629 + }, + { + "label": "images", + "name": "images", + "type": "IMAGE", + "link": 761 + } + ], + "outputs": [ + { + "label": "dw_poses", + "name": "dw_poses", + "type": "DWPOSES", + "links": [ + 633 + ] + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-SCAIL-Pose", + "Node name for S&R": "PoseDetectionVitPoseToDWPose", + "cnr_id": "ComfyUI-SCAIL-Pose", + "ver": "93059f0fe4ea3881c0d6ec0e327e5d920554277d", + "widget_ue_connectable": {} + }, + "widgets_values": [] + }, + { + "id": 368, + "type": "WanVideoTextEncodeCached", + "pos": [ + 1191.8249120373632, + -1828.0122546223386 + ], + "size": [ + 576.7184212265045, + 302 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "label": "extender_args", + "name": "extender_args", + "shape": 7, + "type": "WANVIDEOPROMPTEXTENDER_ARGS", + "link": null + }, + { + "label": "model_name", + "name": "model_name", + "type": "COMBO", + "widget": { + "name": "model_name" + } + }, + { + "label": "precision", + "name": "precision", + "type": "COMBO", + "widget": { + "name": "precision" + } + }, + { + "label": "positive_prompt", + "name": "positive_prompt", + "type": "STRING", + "widget": { + "name": "positive_prompt" + } + }, + { + "label": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + } + }, + { + "label": "quantization", + "name": "quantization", + "type": "COMBO", + "widget": { + "name": "quantization" + } + }, + { + "label": "use_disk_cache", + "name": "use_disk_cache", + "type": "BOOLEAN", + "widget": { + "name": "use_disk_cache" + } + }, + { + "label": "device", + "name": "device", + "type": "COMBO", + "widget": { + "name": "device" + } + } + ], + "outputs": [ + { + "label": "text_embeds", + "name": "text_embeds", + "type": "WANVIDEOTEXTEMBEDS", + "links": [ + 703 + ] + }, + { + "label": "negative_text_embeds", + "name": "negative_text_embeds", + "type": "WANVIDEOTEXTEMBEDS" + }, + { + "label": "positive_prompt", + "name": "positive_prompt", + "type": "STRING" + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "bb0c55da4d8f8bca4968704e877fd057a90a1eeb", + "Node name for S&R": "WanVideoTextEncodeCached", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "umt5-xxl-enc-bf16.safetensors", + "bf16", + "A dancing video of a young man, the man presents a highly muscular and athletic physique, standing confidently with his prominently vascular arms firmly crossed over a bare, glossy chest and a well-defined six-pack abdomen. His sharp, striking facial features include a chiseled jawline, high cheekbones, and a piercing gaze, all framed by dark, slightly wavy hair styled in a messy middle part with loose strands falling effortlessly over his forehead. A distinctive, stylized black cross is prominently marked in the center of his forehead, and he wears dangling earrings ending in dark, round beads. While predominantly shirtless, thick black tactical suspenders stretch over his shoulders, attaching securely to heavy-duty, faded black cargo pants. These rugged trousers feature multiple utility pockets, visible seam stitching, and reinforced knee patches, held up by a dark leather belt equipped with a heavy brass, double-pronged buckle. A bright red cloth strap hangs conspicuously from the waistband on his left hip near a small utility pouch, and his utilitarian look is anchored by a pair of well-worn, black lace-up combat boots with thick soles and reinforced toes. highly details, dynamic angle, Dutch angle, best quality.", + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + "disabled", + false, + "gpu" + ], + "color": "#2b582c", + "bgcolor": "#174418" + }, + { + "id": 377, + "type": "GetNode", + "pos": [ + 537.8502901355928, + -1457.53410470671 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "INT", + "name": "INT", + "type": "INT", + "links": [ + 650 + ] + } + ], + "title": "Get_frame_count", + "properties": { + "Node name for S&R": "GetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "frame_count" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 380, + "type": "MarkdownNote", + "pos": [ + -1667.9894260933672, + -1916.9097131777476 + ], + "size": [ + 331.5011901855469, + 513.44091796875 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Preprocessor links", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "Nodes:\n\n[https://github.com/kijai/ComfyUI-Wanvideowrapper](https://github.com/kijai/ComfyUI-Wanvideowrapper)\n\n[https://github.com/rookiestar28/ComfyUI-SCAIL-Pose2](https://github.com/rookiestar28/ComfyUI-SCAIL-Pose2)\n\nModels:\n\nYOLO:\n\n[https://huggingface.co/Wan-AI/Wan2.2-Animate-14B/blob/main/process_checkpoint/det/yolov10m.onnx](https://huggingface.co/Wan-AI/Wan2.2-Animate-14B/blob/main/process_checkpoint/det/yolov10m.onnx)\n\nViTPose\n\nLarge:\n\n[https://huggingface.co/JunkyByte/easy_ViTPose/blob/main/onnx/wholebody/vitpose-l-wholebody.onnx](https://huggingface.co/JunkyByte/easy_ViTPose/blob/main/onnx/wholebody/vitpose-l-wholebody.onnx)\n\nHuge (needs both files):\n\n[https://huggingface.co/Kijai/vitpose_comfy/blob/main/onnx/vitpose_h_wholebody_model.onnx](https://huggingface.co/Kijai/vitpose_comfy/blob/main/onnx/vitpose_h_wholebody_model.onnx)\n\n[https://huggingface.co/Kijai/vitpose_comfy/blob/main/onnx/vitpose_h_wholebody_data.bin](https://huggingface.co/Kijai/vitpose_comfy/blob/main/onnx/vitpose_h_wholebody_data.bin)\n\nNLF\n\n[https://huggingface.co/Kijai/WanVideo_comfy/blob/main/SCAIL/nlf_l_multi_0.3.2_fp16.safetensors](https://huggingface.co/Kijai/WanVideo_comfy/blob/main/SCAIL/nlf_l_multi_0.3.2_fp16.safetensors)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 394, + "type": "VHS_VideoInfo", + "pos": [ + -707.4837123224096, + -2598.4538081209143 + ], + "size": [ + 234.931640625, + 206 + ], + "flags": { + "collapsed": true + }, + "order": 36, + "mode": 0, + "inputs": [ + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "link": 724 + } + ], + "outputs": [ + { + "name": "source_fps🟨", + "type": "FLOAT", + "links": null + }, + { + "name": "source_frame_count🟨", + "type": "INT", + "links": null + }, + { + "name": "source_duration🟨", + "type": "FLOAT", + "links": null + }, + { + "name": "source_width🟨", + "type": "INT", + "links": null + }, + { + "name": "source_height🟨", + "type": "INT", + "links": null + }, + { + "name": "loaded_fps🟦", + "type": "FLOAT", + "links": [ + 660, + 661, + 671, + 802 + ] + }, + { + "name": "loaded_frame_count🟦", + "type": "INT", + "links": [ + 665, + 805 + ] + }, + { + "name": "loaded_duration🟦", + "type": "FLOAT", + "links": null + }, + { + "name": "loaded_width🟦", + "type": "INT", + "links": null + }, + { + "name": "loaded_height🟦", + "type": "INT", + "links": null + } + ], + "properties": { + "aux_id": "rookiestar28/ComfyUI-VideoHelper_Adv", + "ver": "d7d2f47eafc3c0b7831290d68ef7f3847a180ae9", + "Node name for S&R": "VHS_VideoInfo" + }, + "widgets_values": {} + }, + { + "id": 396, + "type": "WanVideoLoraSelectMulti", + "pos": [ + 405.79433386259245, + -2248.5966171763275 + ], + "size": [ + 661.337571307046, + 342 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "prev_lora", + "shape": 7, + "type": "WANVIDLORA", + "link": null + }, + { + "name": "blocks", + "shape": 7, + "type": "SELECTEDBLOCKS", + "link": null + } + ], + "outputs": [ + { + "name": "lora", + "type": "WANVIDLORA", + "links": [ + 664 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "b2c8cf969fcf60a38884ea2c29af177ae1f28b29", + "Node name for S&R": "WanVideoLoraSelectMulti", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.0.1" + } + }, + "widgets_values": [ + "加速與功能性\\Wan21-KJ-Lightx2v_I2V_14B_480p_cfg_step_distill_rank128_bf16.safetensors", + 1, + "none", + 0, + "none", + 0, + "none", + 0, + "none", + 0, + false, + false + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 398, + "type": "SetNode", + "pos": [ + -510.94617322369913, + -2656.515871273684 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 45, + "mode": 0, + "inputs": [ + { + "name": "INT", + "type": "INT", + "link": 665 + } + ], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": null + } + ], + "title": "Set_frame_count", + "properties": { + "Node name for S&R": "SetNode", + "aux_id": "SetNode", + "previousName": "frame_count", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.0.1" + } + }, + "widgets_values": [ + "frame_count" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 402, + "type": "INTConstant", + "pos": [ + -1622.6524420933536, + -2336.435781289098 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": false + }, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "value", + "type": "INT", + "links": [ + 704 + ] + } + ], + "title": "Longest Side", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "37659859825cea55940a58110525795ce5deb8be", + "Node name for S&R": "INTConstant", + "ue_properties": { + "widget_ue_connectable": {}, + "version": "7.0.1" + } + }, + "widgets_values": [ + 1120 + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 414, + "type": "Reroute", + "pos": [ + 689.1590969348648, + -1806.0599915407788 + ], + "size": [ + 75, + 26 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "label": "", + "name": "", + "type": "*", + "link": 700 + } + ], + "outputs": [ + { + "name": "", + "type": "WANVAE", + "links": [ + 790, + 831, + 837 + ] + } + ], + "properties": { + "showOutputText": false, + "horizontal": false, + "widget_ue_connectable": {} + } + }, + { + "id": 415, + "type": "ResizeImageAdvanced", + "pos": [ + -1011.0176294942933, + -2645.563561166475 + ], + "size": [ + 270, + 430 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 721 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "scale_to_length", + "type": "INT", + "widget": { + "name": "scale_to_length" + }, + "link": 704 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 706, + 710, + 769, + 792 + ] + }, + { + "name": "width", + "type": "INT", + "links": [ + 707 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 708 + ] + }, + { + "name": "mask", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "text_processor", + "ver": "cf29f2574efce7078a6b28dc9b2f33087142b6f5", + "Node name for S&R": "ResizeImageAdvanced" + }, + "widgets_values": [ + "aspect_ratio", + 512, + 512, + "3:4", + 1, + 1, + "crop", + "nvidia_rtx_vsr", + "longest", + 1024, + "#000000", + "center", + 32, + "gpu" + ], + "color": "#6a2e2e", + "bgcolor": "#561a1a" + }, + { + "id": 416, + "type": "ResizeImageAdvanced", + "pos": [ + -992.4892078602486, + -1835.6415165402009 + ], + "size": [ + 270, + 440 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 764 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 741 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 740 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 761, + 849, + 850 + ] + }, + { + "name": "width", + "type": "INT", + "links": [ + 716 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 717 + ] + }, + { + "name": "mask", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "text_processor", + "ver": "cf29f2574efce7078a6b28dc9b2f33087142b6f5", + "Node name for S&R": "ResizeImageAdvanced" + }, + "widgets_values": [ + "explicit", + 512, + 512, + "original", + 1, + 1, + "crop", + "lanczos", + "longest", + 1, + "#000000", + "center", + 32, + "cpu" + ], + "color": "#6a2e2e", + "bgcolor": "#561a1a" + }, + { + "id": 417, + "type": "VHS_LoadVideo", + "pos": [ + -1283.1349636032705, + -2622.928476363794 + ], + "size": [ + 241.84130859375, + 716.9496775922557 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "label": "meta_batch", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "label": "vae", + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + }, + { + "label": "video", + "name": "video", + "type": "COMBO", + "widget": { + "name": "video" + }, + "link": null + }, + { + "label": "force_rate", + "name": "force_rate", + "type": "FLOAT", + "widget": { + "name": "force_rate" + }, + "link": null + }, + { + "label": "custom_width", + "name": "custom_width", + "type": "INT", + "widget": { + "name": "custom_width" + }, + "link": null + }, + { + "label": "custom_height", + "name": "custom_height", + "type": "INT", + "widget": { + "name": "custom_height" + }, + "link": null + }, + { + "label": "frame_load_cap", + "name": "frame_load_cap", + "type": "INT", + "widget": { + "name": "frame_load_cap" + }, + "link": null + }, + { + "label": "skip_first_frames", + "name": "skip_first_frames", + "type": "INT", + "widget": { + "name": "skip_first_frames" + }, + "link": null + }, + { + "label": "select_every_nth", + "name": "select_every_nth", + "type": "INT", + "widget": { + "name": "select_every_nth" + }, + "link": null + }, + { + "label": "format", + "name": "format", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "label": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 721 + ] + }, + { + "label": "frame_count", + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "label": "audio", + "name": "audio", + "type": "AUDIO", + "links": [ + 722, + 723 + ] + }, + { + "label": "video_info", + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": [ + 724 + ] + } + ], + "properties": { + "aux_id": "rookiestar28/ComfyUI-VideoHelper_Adv", + "ver": "d7d2f47eafc3c0b7831290d68ef7f3847a180ae9", + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "NO BATIDÃO Dance Challenge #jennie #xlong [WlqVTF6VdfA].mp4", + "force_rate": 24, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "format": "Wan", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "NO BATIDÃO Dance Challenge #jennie #xlong [WlqVTF6VdfA].mp4", + "type": "input", + "format": "video/mp4", + "force_rate": 24, + "custom_width": 0, + "custom_height": 0, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1 + }, + "muted": false, + "error": "" + } + } + }, + { + "id": 418, + "type": "PreviewImage", + "pos": [ + -681.0912186615604, + -1813.9492916783802 + ], + "size": [ + 285.55889454699104, + 416.8700270864564 + ], + "flags": {}, + "order": 47, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 849 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.24.0", + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 431, + "type": "SCAILPose2ColoredMask", + "pos": [ + -460.04150569569424, + -823.6238038081243 + ], + "size": [ + 340, + 122 + ], + "flags": {}, + "order": 55, + "mode": 0, + "inputs": [ + { + "name": "driving_track_data", + "type": "SAM3_TRACK_DATA", + "link": 767 + }, + { + "name": "ref_track_data", + "shape": 7, + "type": "SAM3_TRACK_DATA", + "link": 768 + }, + { + "name": "ref_mask", + "shape": 7, + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "pose_video_mask", + "type": "IMAGE", + "links": [ + 801, + 809, + 812, + 844 + ] + }, + { + "name": "reference_image_mask", + "type": "IMAGE", + "links": [ + 785, + 823 + ] + } + ], + "properties": { + "Node name for S&R": "SCAILPose2ColoredMask" + }, + "widgets_values": [ + "", + "left_to_right" + ] + }, + { + "id": 438, + "type": "SAM3_VideoTrack", + "pos": [ + -464.2367783401059, + -1262.4568494024707 + ], + "size": [ + 340, + 166 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "label": "images", + "name": "images", + "type": "IMAGE", + "link": 769 + }, + { + "label": "model", + "name": "model", + "type": "MODEL", + "link": 776 + }, + { + "label": "initial_mask", + "name": "initial_mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "label": "conditioning", + "name": "conditioning", + "shape": 7, + "type": "CONDITIONING", + "link": 775 + }, + { + "label": "detection_threshold", + "name": "detection_threshold", + "type": "FLOAT", + "widget": { + "name": "detection_threshold" + }, + "link": null + }, + { + "label": "max_objects", + "name": "max_objects", + "type": "INT", + "widget": { + "name": "max_objects" + }, + "link": null + }, + { + "label": "detect_interval", + "name": "detect_interval", + "type": "INT", + "widget": { + "name": "detect_interval" + }, + "link": null + } + ], + "outputs": [ + { + "label": "track_data", + "name": "track_data", + "type": "SAM3_TRACK_DATA", + "links": [ + 767 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.24.0", + "Node name for S&R": "SAM3_VideoTrack", + "ue_properties": { + "widget_ue_connectable": { + "max_objects": true, + "detection_threshold": true, + "detect_interval": true + }, + "version": "7.0.1" + }, + "widget_ue_connectable": {} + }, + "widgets_values": [ + 0.4, + 1, + 2 + ] + }, + { + "id": 439, + "type": "SAM3_VideoTrack", + "pos": [ + -464.2367783401059, + -1043.8793811341798 + ], + "size": [ + 340, + 166 + ], + "flags": {}, + "order": 53, + "mode": 0, + "inputs": [ + { + "label": "images", + "name": "images", + "type": "IMAGE", + "link": 770 + }, + { + "label": "model", + "name": "model", + "type": "MODEL", + "link": 777 + }, + { + "label": "initial_mask", + "name": "initial_mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "label": "conditioning", + "name": "conditioning", + "shape": 7, + "type": "CONDITIONING", + "link": 778 + }, + { + "label": "detection_threshold", + "name": "detection_threshold", + "type": "FLOAT", + "widget": { + "name": "detection_threshold" + }, + "link": null + }, + { + "label": "max_objects", + "name": "max_objects", + "type": "INT", + "widget": { + "name": "max_objects" + }, + "link": null + }, + { + "label": "detect_interval", + "name": "detect_interval", + "type": "INT", + "widget": { + "name": "detect_interval" + }, + "link": null + } + ], + "outputs": [ + { + "label": "track_data", + "name": "track_data", + "type": "SAM3_TRACK_DATA", + "links": [ + 768 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.24.0", + "Node name for S&R": "SAM3_VideoTrack", + "ue_properties": { + "widget_ue_connectable": { + "max_objects": true, + "detection_threshold": true, + "detect_interval": true + }, + "version": "7.0.1" + }, + "widget_ue_connectable": {} + }, + "widgets_values": [ + 0.4, + 1, + 2 + ] + }, + { + "id": 440, + "type": "CheckpointLoaderSimple", + "pos": [ + -1228.3979288291118, + -1138.301693254964 + ], + "size": [ + 393.1847264404213, + 98 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "label": "ckpt_name", + "name": "ckpt_name", + "type": "COMBO", + "widget": { + "name": "ckpt_name" + }, + "link": null + } + ], + "outputs": [ + { + "label": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 776, + 777 + ] + }, + { + "label": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 771, + 773 + ] + }, + { + "label": "VAE", + "name": "VAE", + "type": "VAE" + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.24.0", + "Node name for S&R": "CheckpointLoaderSimple", + "ue_properties": { + "widget_ue_connectable": { + "ckpt_name": true + }, + "version": "7.0.1" + }, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "sam3.1_multiplex_fp16.safetensors" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 443, + "type": "CLIPTextEncode", + "pos": [ + -765.9766290984006, + -1215.740618631673 + ], + "size": [ + 235, + 115.328125 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "label": "clip", + "name": "clip", + "type": "CLIP", + "link": 771 + }, + { + "label": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "label": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 775 + ] + } + ], + "title": "Video seg prompt", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.24.0", + "Node name for S&R": "CLIPTextEncode", + "ue_properties": { + "widget_ue_connectable": { + "text": true + }, + "version": "7.0.1" + }, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "all person in the pcture" + ] + }, + { + "id": 444, + "type": "CLIPTextEncode", + "pos": [ + -765.9766290984006, + -1014.1271054820609 + ], + "size": [ + 235, + 115.328125 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "label": "clip", + "name": "clip", + "type": "CLIP", + "link": 773 + }, + { + "label": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "label": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 778 + ] + } + ], + "title": "Ref Image seg prompt", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.24.0", + "Node name for S&R": "CLIPTextEncode", + "ue_properties": { + "widget_ue_connectable": { + "text": true + }, + "version": "7.0.1" + }, + "widget_ue_connectable": {} + }, + "widgets_values": [ + "all person in the pcture" + ] + }, + { + "id": 446, + "type": "WanVideoAddSCAIL2ConditionEmbeds", + "pos": [ + 741.893587538584, + -1065.4061329448316 + ], + "size": [ + 350, + 190 + ], + "flags": {}, + "order": 65, + "mode": 0, + "inputs": [ + { + "name": "embeds", + "type": "WANVIDIMAGE_EMBEDS", + "link": 787 + }, + { + "name": "condition", + "type": "SCAIL2_WANVIDEO_PAYLOAD", + "link": 822 + }, + { + "name": "vae", + "type": "WANVAE", + "link": 790 + }, + { + "name": "clip_embeds", + "shape": 7, + "type": "WANVIDIMAGE_CLIPEMBEDS", + "link": 791 + } + ], + "outputs": [ + { + "name": "image_embeds", + "type": "WANVIDIMAGE_EMBEDS", + "links": [ + 789 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "088128b224242e110d3906c6750e9a3a348a659b", + "Node name for S&R": "WanVideoAddSCAIL2ConditionEmbeds" + }, + "widgets_values": [ + 1.25, + 1.1, + 1, + 1 + ] + }, + { + "id": 447, + "type": "SCAILPose2SCAIL2Condition", + "pos": [ + 308.7916385775617, + -1197.720974487722 + ], + "size": [ + 307.0129995528073, + 418 + ], + "flags": {}, + "order": 58, + "mode": 0, + "inputs": [ + { + "name": "pose_video_mask", + "type": "IMAGE", + "link": 812 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 798 + }, + { + "name": "ref_mask", + "type": "IMAGE", + "link": 785 + }, + { + "name": "pose_video", + "shape": 7, + "type": "IMAGE", + "link": 855 + }, + { + "name": "driving_video", + "shape": 7, + "type": "IMAGE", + "link": 856 + }, + { + "name": "additional_ref_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "additional_ref_mask", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 833 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 832 + }, + { + "name": "num_frames", + "type": "INT", + "widget": { + "name": "num_frames" + }, + "link": 805 + } + ], + "outputs": [ + { + "name": "condition", + "type": "SCAIL2_CONDITION", + "links": [ + 808, + 821 + ] + } + ], + "properties": { + "Node name for S&R": "SCAILPose2SCAIL2Condition" + }, + "widgets_values": [ + "replacement", + 512, + 512, + 81, + "contain", + "bottom_center", + "median_bbox", + "subject", + 0, + 2, + 0.0005 + ] + }, + { + "id": 448, + "type": "SetNode", + "pos": [ + -701.7993601847684, + -2551.565719387848 + ], + "size": [ + 272.5436891741035, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 42, + "mode": 0, + "inputs": [ + { + "label": "INT", + "name": "IMAGE", + "type": "IMAGE", + "link": 792 + } + ], + "outputs": [ + { + "label": "*", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 793 + ] + } + ], + "title": "Set_driving_video", + "properties": { + "Node name for S&R": "SetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "previousName": "driving_video", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "driving_video" + ], + "color": "#006691", + "bgcolor": "rgba(24,24,27,.9)" + }, + { + "id": 449, + "type": "GetNode", + "pos": [ + 1207.7176304574753, + -2523.6656937665916 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 835 + ] + } + ], + "title": "Get_driving_video", + "properties": { + "Node name for S&R": "GetNode", + "aux_id": "kijai/ComfyUI-KJNodes" + }, + "widgets_values": [ + "driving_video" + ], + "color": "#006691", + "bgcolor": "rgba(24,24,27,.9)" + }, + { + "id": 452, + "type": "VHS_VideoCombine", + "pos": [ + -87.67240336761344, + -1255.2538435913552 + ], + "size": [ + 290.651983241861, + 334 + ], + "flags": {}, + "order": 56, + "mode": 0, + "inputs": [ + { + "label": "images", + "name": "images", + "type": "IMAGE", + "link": 801 + }, + { + "label": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "meta_batch", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "label": "vae", + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + }, + { + "label": "frame_rate", + "name": "frame_rate", + "type": "FLOAT", + "widget": { + "name": "frame_rate" + }, + "link": 802 + }, + { + "label": "loop_count", + "name": "loop_count", + "type": "INT", + "widget": { + "name": "loop_count" + }, + "link": null + }, + { + "label": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "label": "format", + "name": "format", + "type": "COMBO", + "widget": { + "name": "format" + }, + "link": null + }, + { + "label": "pingpong", + "name": "pingpong", + "type": "BOOLEAN", + "widget": { + "name": "pingpong" + }, + "link": null + }, + { + "label": "save_output", + "name": "save_output", + "type": "BOOLEAN", + "widget": { + "name": "save_output" + }, + "link": null + } + ], + "outputs": [ + { + "label": "Filenames", + "name": "Filenames", + "type": "VHS_FILENAMES" + } + ], + "properties": { + "aux_id": "rookiestar28/ComfyUI-VideoHelper_Adv", + "ver": "8923bd836bdab8b7bbdf4ed104b7d045e70c66e2", + "Node name for S&R": "VHS_VideoCombine", + "cnr_id": "comfyui-videohelpersuite", + "ue_properties": { + "widget_ue_connectable": { + "save_output": true, + "filename_prefix": true, + "loop_count": true, + "pix_fmt": true, + "save_metadata": true, + "crf": true, + "trim_to_audio": true, + "format": true, + "frame_rate": true, + "pingpong": true + }, + "version": "7.0.1" + }, + "widget_ue_connectable": {} + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "SCAIL2_pose", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": false, + "trim_to_audio": false, + "pingpong": false, + "save_output": false, + "videopreview": { + "paused": false, + "hidden": false, + "params": { + "score": "0", + "filename": "SCAIL2_pose_00001.mp4", + "cos_url": "https://rh-images.xiaoyaoyou.com//8696605c7318b5f10dd8c8741df992bb/temp/Wan21_SCAIL2_00002_p80_fgiis_1781098496.mp4", + "workflow": "Wan21_SCAIL2_00001.png", + "fullpath": "A:\\ComfyUI\\temp\\SCAIL2_pose_00001.mp4", + "format": "video/h264-mp4", + "subfolder": "", + "label": "Normal", + "type": "temp", + "frame_rate": 24 + }, + "error": "Preview unavailable. Check VHS debug logs for details." + } + } + }, + { + "id": 454, + "type": "SCAILPose2ReplacementDenoiseMask", + "pos": [ + 738.8101836064761, + -818.1358930973687 + ], + "size": [ + 350, + 126 + ], + "flags": {}, + "order": 61, + "mode": 0, + "inputs": [ + { + "name": "condition", + "type": "SCAIL2_CONDITION", + "link": 808 + }, + { + "name": "pose_video_mask", + "type": "IMAGE", + "link": 809 + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 836 + ] + }, + { + "name": "summary", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "SCAILPose2ReplacementDenoiseMask" + }, + "widgets_values": [ + "custom", + 0, + 0 + ] + }, + { + "id": 459, + "type": "SCAILPose2WanVideoSCAIL2Adapter", + "pos": [ + 735.4328676661912, + -1202.071967749464 + ], + "size": [ + 350, + 82 + ], + "flags": {}, + "order": 62, + "mode": 0, + "inputs": [ + { + "name": "condition", + "type": "SCAIL2_CONDITION", + "link": 821 + } + ], + "outputs": [ + { + "name": "condition", + "type": "SCAIL2_WANVIDEO_PAYLOAD", + "links": [ + 822 + ] + } + ], + "properties": { + "Node name for S&R": "SCAILPose2WanVideoSCAIL2Adapter" + }, + "widgets_values": [ + false, + false + ] + }, + { + "id": 460, + "type": "PreviewImage", + "pos": [ + -429.25155566012836, + -650.5239416956003 + ], + "size": [ + 272.69543259293107, + 268.5838377089523 + ], + "flags": {}, + "order": 59, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 823 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.25.0", + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 461, + "type": "SetNode", + "pos": [ + -263.91882858544426, + -2303.6960039414676 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 60, + "mode": 0, + "inputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "link": 843 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 826 + ] + } + ], + "title": "Set_pose_video", + "properties": { + "Node name for S&R": "SetNode", + "aux_id": "kijai/ComfyUI-KJNodes", + "previousName": "pose_video" + }, + "widgets_values": [ + "pose_video" + ], + "color": "#006691", + "bgcolor": "rgba(24,24,27,.9)" + }, + { + "id": 462, + "type": "GetNode", + "pos": [ + 398.14777352004893, + -1259.0807029947598 + ], + "size": [ + 210, + 50 + ], + "flags": { + "collapsed": true + }, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 855 + ] + } + ], + "title": "Get_pose_video", + "properties": { + "Node name for S&R": "GetNode", + "aux_id": "kijai/ComfyUI-KJNodes" + }, + "widgets_values": [ + "pose_video" + ], + "color": "#006691", + "bgcolor": "rgba(24,24,27,.9)" + }, + { + "id": 466, + "type": "WanVideoBlockSwap", + "pos": [ + 53.354684669316384, + -2506.6850844453174 + ], + "size": [ + 286.56640625, + 202 + ], + "flags": {}, + "order": 12, + "mode": 4, + "inputs": [], + "outputs": [ + { + "name": "block_swap_args", + "type": "BLOCKSWAPARGS", + "links": [ + 828 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "088128b224242e110d3906c6750e9a3a348a659b", + "Node name for S&R": "WanVideoBlockSwap" + }, + "widgets_values": [ + 20, + false, + false, + false, + 0, + 0, + false + ] + }, + { + "id": 470, + "type": "MarkdownNote", + "pos": [ + -1678.6298646508444, + -2606.5411840807355 + ], + "size": [ + 361.08051227646297, + 198.4283944640215 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Markdown Note: Pose input info", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "# There's no need to calculate the aspect ratio; just input the longest side of the output dimensions you want. \n\n## (Dimensions should be divisible by 32)" + ], + "color": "#9b2121", + "bgcolor": "#870d0d" + }, + { + "id": 471, + "type": "MarkdownNote", + "pos": [ + 856.918018628063, + -2404.1057704089126 + ], + "size": [ + 251.773046875, + 88 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Markdown Note: Pose input info", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "## IF you have sage attention installed. enable to speed up." + ], + "color": "#912020", + "bgcolor": "#7d0c0c" + }, + { + "id": 472, + "type": "MarkdownNote", + "pos": [ + 315.4250762203824, + -732.8494140762087 + ], + "size": [ + 305.8454055325185, + 107.65535577178764 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Markdown Note: Pose input info", + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "# Use this node to switch between Animation and Replacement modes." + ], + "color": "#9b2121", + "bgcolor": "#870d0d" + }, + { + "id": 473, + "type": "GetNode", + "pos": [ + 577.0872843127523, + -1259.7367363164335 + ], + "size": [ + 210, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 21, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 832 + ] + } + ], + "title": "Get_height", + "properties": { + "Node name for S&R": "GetNode", + "aux_id": "kijai/ComfyUI-KJNodes" + }, + "widgets_values": [ + "height" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 474, + "type": "GetNode", + "pos": [ + 227.48518264566627, + -1256.9418212882667 + ], + "size": [ + 210, + 58 + ], + "flags": { + "collapsed": true + }, + "order": 20, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 833 + ] + } + ], + "title": "Get_width", + "properties": { + "Node name for S&R": "GetNode", + "aux_id": "kijai/ComfyUI-KJNodes" + }, + "widgets_values": [ + "width" + ], + "color": "#1b4669", + "bgcolor": "#29699c" + }, + { + "id": 475, + "type": "WanVideoEncode", + "pos": [ + 1186.5698763073565, + -2462.3668764111007 + ], + "size": [ + 290, + 242 + ], + "flags": {}, + "order": 64, + "mode": 0, + "inputs": [ + { + "name": "vae", + "type": "WANVAE", + "link": 837 + }, + { + "name": "driving_video", + "type": "IMAGE", + "link": 835 + }, + { + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 836 + } + ], + "outputs": [ + { + "name": "samples", + "type": "LATENT", + "links": [ + 834 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-WanVideoWrapper", + "ver": "088128b224242e110d3906c6750e9a3a348a659b", + "Node name for S&R": "WanVideoEncode" + }, + "widgets_values": [ + false, + 272, + 272, + 144, + 128, + 0, + 1 + ] + }, + { + "id": 478, + "type": "GetNode", + "pos": [ + 359.1487507666821, + -511.90727421559035 + ], + "size": [ + 210, + 50 + ], + "flags": { + "collapsed": true + }, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 856 + ] + } + ], + "title": "Get_driving_video", + "properties": { + "Node name for S&R": "GetNode", + "aux_id": "kijai/ComfyUI-KJNodes" + }, + "widgets_values": [ + "driving_video" + ], + "color": "#006691", + "bgcolor": "rgba(24,24,27,.9)" + } + ], + "links": [ + [ + 414, + 238, + 0, + 239, + 0, + "FLOAT" + ], + [ + 554, + 326, + 0, + 327, + 0, + "CLIP_VISION" + ], + [ + 559, + 328, + 0, + 318, + 1, + "IMAGE" + ], + [ + 570, + 335, + 0, + 334, + 0, + "NLFMODEL" + ], + [ + 596, + 348, + 0, + 28, + 1, + "LATENT" + ], + [ + 598, + 349, + 0, + 348, + 2, + "WANVIDEOSCHEDULER" + ], + [ + 601, + 240, + 0, + 348, + 6, + "FLOAT" + ], + [ + 611, + 351, + 0, + 327, + 1, + "IMAGE" + ], + [ + 614, + 355, + 0, + 354, + 1, + "WANVIDCONTEXT" + ], + [ + 621, + 334, + 0, + 362, + 0, + "NLFPRED" + ], + [ + 626, + 363, + 0, + 362, + 1, + "DWPOSES" + ], + [ + 627, + 364, + 0, + 363, + 0, + "POSEMODEL" + ], + [ + 629, + 364, + 0, + 365, + 0, + "POSEMODEL" + ], + [ + 633, + 365, + 0, + 362, + 2, + "DWPOSES" + ], + [ + 650, + 377, + 0, + 99, + 4, + "INT" + ], + [ + 651, + 318, + 0, + 319, + 0, + "IMAGE" + ], + [ + 652, + 354, + 0, + 348, + 5, + "WANVIDSAMPLEREXTRAARGS" + ], + [ + 655, + 22, + 0, + 80, + 0, + "WANVIDEOMODEL" + ], + [ + 656, + 28, + 0, + 139, + 0, + "IMAGE" + ], + [ + 657, + 28, + 0, + 318, + 0, + "IMAGE" + ], + [ + 660, + 394, + 5, + 137, + 4, + "FLOAT" + ], + [ + 661, + 394, + 5, + 139, + 4, + "FLOAT" + ], + [ + 664, + 396, + 0, + 80, + 1, + "WANVIDLORA" + ], + [ + 665, + 394, + 6, + 398, + 0, + "INT" + ], + [ + 671, + 394, + 5, + 319, + 4, + "FLOAT" + ], + [ + 696, + 80, + 0, + 348, + 0, + "WANVIDEOMODEL" + ], + [ + 700, + 38, + 0, + 414, + 0, + "WANVAE" + ], + [ + 703, + 368, + 0, + 348, + 3, + "WANVIDEOTEXTEMBEDS" + ], + [ + 704, + 402, + 0, + 415, + 2, + "INT" + ], + [ + 706, + 415, + 0, + 363, + 1, + "IMAGE" + ], + [ + 707, + 415, + 1, + 206, + 0, + "INT" + ], + [ + 708, + 415, + 2, + 207, + 0, + "INT" + ], + [ + 710, + 415, + 0, + 328, + 1, + "IMAGE" + ], + [ + 716, + 416, + 1, + 184, + 0, + "INT" + ], + [ + 717, + 416, + 2, + 185, + 0, + "INT" + ], + [ + 721, + 417, + 0, + 415, + 0, + "IMAGE" + ], + [ + 722, + 417, + 2, + 139, + 1, + "AUDIO" + ], + [ + 723, + 417, + 2, + 319, + 1, + "AUDIO" + ], + [ + 724, + 417, + 3, + 394, + 0, + "VHS_VIDEOINFO" + ], + [ + 727, + 186, + 0, + 99, + 2, + "INT" + ], + [ + 728, + 187, + 0, + 99, + 3, + "INT" + ], + [ + 740, + 209, + 0, + 416, + 3, + "INT" + ], + [ + 741, + 208, + 0, + 416, + 2, + "INT" + ], + [ + 761, + 416, + 0, + 365, + 1, + "IMAGE" + ], + [ + 764, + 106, + 0, + 416, + 0, + "IMAGE" + ], + [ + 767, + 438, + 0, + 431, + 0, + "SAM3_TRACK_DATA" + ], + [ + 768, + 439, + 0, + 431, + 1, + "SAM3_TRACK_DATA" + ], + [ + 769, + 415, + 0, + 438, + 0, + "IMAGE" + ], + [ + 770, + 351, + 0, + 439, + 0, + "IMAGE" + ], + [ + 771, + 440, + 1, + 443, + 0, + "CLIP" + ], + [ + 773, + 440, + 1, + 444, + 0, + "CLIP" + ], + [ + 775, + 443, + 0, + 438, + 3, + "CONDITIONING" + ], + [ + 776, + 440, + 0, + 438, + 1, + "MODEL" + ], + [ + 777, + 440, + 0, + 439, + 1, + "MODEL" + ], + [ + 778, + 444, + 0, + 439, + 3, + "CONDITIONING" + ], + [ + 785, + 431, + 1, + 447, + 2, + "IMAGE" + ], + [ + 787, + 99, + 0, + 446, + 0, + "WANVIDIMAGE_EMBEDS" + ], + [ + 789, + 446, + 0, + 348, + 1, + "WANVIDIMAGE_EMBEDS" + ], + [ + 790, + 414, + 0, + 446, + 2, + "WANVAE" + ], + [ + 791, + 327, + 0, + 446, + 3, + "WANVIDIMAGE_CLIPEMBEDS" + ], + [ + 792, + 415, + 0, + 448, + 0, + "IMAGE" + ], + [ + 793, + 448, + 0, + 334, + 1, + "IMAGE" + ], + [ + 798, + 351, + 0, + 447, + 1, + "IMAGE" + ], + [ + 801, + 431, + 0, + 452, + 0, + "IMAGE" + ], + [ + 802, + 394, + 5, + 452, + 4, + "FLOAT" + ], + [ + 805, + 394, + 6, + 447, + 9, + "INT" + ], + [ + 808, + 447, + 0, + 454, + 0, + "SCAIL2_CONDITION" + ], + [ + 809, + 431, + 0, + 454, + 1, + "IMAGE" + ], + [ + 812, + 431, + 0, + 447, + 0, + "IMAGE" + ], + [ + 821, + 447, + 0, + 459, + 0, + "SCAIL2_CONDITION" + ], + [ + 822, + 459, + 0, + 446, + 1, + "SCAIL2_WANVIDEO_PAYLOAD" + ], + [ + 823, + 431, + 1, + 460, + 0, + "IMAGE" + ], + [ + 826, + 461, + 0, + 137, + 0, + "IMAGE" + ], + [ + 828, + 466, + 0, + 22, + 1, + "BLOCKSWAPARGS" + ], + [ + 831, + 414, + 0, + 28, + 0, + "WANVAE" + ], + [ + 832, + 473, + 0, + 447, + 8, + "INT" + ], + [ + 833, + 474, + 0, + 447, + 7, + "INT" + ], + [ + 834, + 475, + 0, + 348, + 4, + "LATENT" + ], + [ + 835, + 449, + 0, + 475, + 1, + "IMAGE" + ], + [ + 836, + 454, + 0, + 475, + 2, + "MASK" + ], + [ + 837, + 414, + 0, + 475, + 0, + "WANVAE" + ], + [ + 843, + 362, + 0, + 461, + 0, + "IMAGE" + ], + [ + 844, + 431, + 0, + 362, + 4, + "IMAGE" + ], + [ + 849, + 416, + 0, + 418, + 0, + "IMAGE" + ], + [ + 850, + 416, + 0, + 351, + 0, + "IMAGE" + ], + [ + 851, + 351, + 0, + 328, + 0, + "IMAGE" + ], + [ + 855, + 462, + 0, + 447, + 3, + "IMAGE" + ], + [ + 856, + 478, + 0, + 447, + 4, + "IMAGE" + ], + [ + 859, + 206, + 0, + 362, + 5, + "INT" + ], + [ + 860, + 207, + 0, + 362, + 6, + "INT" + ] + ], + "groups": [ + { + "id": 1, + "title": "Pose extraction", + "bounding": [ + -1285.5413558301416, + -2695.201049244218, + 1248.8343505859375, + 1342.2572776692714 + ], + "color": "#3f789e", + "flags": { + "pinned": true + } + }, + { + "id": 3, + "title": "Models", + "bounding": [ + -23.485916137695312, + -2698.06884765625, + 1159.849609375, + 796.1673583984375 + ], + "color": "#88A", + "flags": { + "pinned": true + } + }, + { + "id": 4, + "title": "Group", + "bounding": [ + -1712.3123909030492, + -2693.5797235907676, + 416.61063757323814, + 596.9255594611086 + ], + "color": "#3f789e", + "flags": { + "pinned": true + } + }, + { + "id": 5, + "title": "Group", + "bounding": [ + -17.941054450142218, + -1885.1217335870915, + 1148.7906251940994, + 522.35645880005 + ], + "color": "#3f789e", + "flags": { + "pinned": true + } + }, + { + "id": 6, + "title": "Group", + "bounding": [ + 1145.8213107638906, + -2691.9847493489583, + 1245.4771740451386, + 1326.5878380533604 + ], + "color": "#3f789e", + "flags": { + "pinned": true + } + }, + { + "id": 7, + "title": "Group", + "bounding": [ + 2869.099278428825, + -2686.8496636284763, + 1323.107379557288, + 1311.1459240722213 + ], + "color": "#3f789e", + "flags": { + "pinned": true + } + }, + { + "id": 8, + "title": "Group", + "bounding": [ + 2403.30593948848, + -2689.5084655137, + 454.3203320306993, + 1317.3437312749368 + ], + "color": "#3f789e", + "flags": { + "pinned": true + } + }, + { + "id": 9, + "title": "SCAIL-POSE2", + "bounding": [ + -1281.890344330005, + -1332.4568494024707, + 2993.757181494024, + 985.0140876593911 + ], + "color": "#3f789e", + "flags": { + "pinned": true + } + } + ], + "config": {}, + "extra": { + "VHS_KeepIntermediate": true, + "links_added_by_ue": [], + "VHS_MetadataImage": false, + "ue_links": [], + "0246.VERSION": [ + 0, + 0, + 4 + ], + "workflowRendererVersion": "LG", + "VHS_latentpreviewrate": 16, + "frontendVersion": "1.45.19", + "VHS_latentpreview": true, + "node_versions": { + "ComfyUI-WanVideoWrapper": "5a2383621a05825d0d0437781afcb8552d9590fd", + "ComfyUI-VideoHelperSuite": "0a75c7958fe320efcb052f1d9f8451fd20c730a8", + "comfy-core": "0.3.26" + }, + "ds": { + "scale": 0.25937424601005543, + "offset": [ + 2383.2645033587746, + 3191.3634580530975 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/nodes.py b/nodes.py index f0b0e84d..5f1bcbf9 100644 --- a/nodes.py +++ b/nodes.py @@ -11,6 +11,10 @@ from comfy.utils import ProgressBar, common_upscale from comfy.clip_vision import clip_preprocess, ClipVisionModel import folder_paths +from .scail_pose2_mask_contract import ( + build_disabled_samples_payload, + scail_pose2_mask_disables_samples, +) script_directory = os.path.dirname(os.path.abspath(__file__)) @@ -19,8 +23,6 @@ VAE_STRIDE = (4, 8, 8) PATCH_SIZE = (1, 2, 2) - - class WanVideoEnhanceAVideo: @classmethod def INPUT_TYPES(s): @@ -2236,7 +2238,7 @@ class WanVideoEncode: def INPUT_TYPES(s): return {"required": { "vae": ("WANVAE",), - "image": ("IMAGE",), + "driving_video": ("IMAGE",), "enable_vae_tiling": ("BOOLEAN", {"default": False, "tooltip": "Drastically reduces memory use but may introduce seams"}), "tile_x": ("INT", {"default": 272, "min": 64, "max": 2048, "step": 1, "tooltip": "Tile size in pixels, smaller values use less VRAM, may introduce more seams"}), "tile_y": ("INT", {"default": 272, "min": 64, "max": 2048, "step": 1, "tooltip": "Tile size in pixels, smaller values use less VRAM, may introduce more seams"}), @@ -2255,10 +2257,19 @@ def INPUT_TYPES(s): FUNCTION = "encode" CATEGORY = "WanVideoWrapper" - def encode(self, vae, image, enable_vae_tiling, tile_x, tile_y, tile_stride_x, tile_stride_y, noise_aug_strength=0.0, latent_strength=1.0, mask=None): + def encode(self, vae, driving_video, enable_vae_tiling, tile_x, tile_y, tile_stride_x, tile_stride_y, noise_aug_strength=0.0, latent_strength=1.0, mask=None): + if scail_pose2_mask_disables_samples(mask): + payload = build_disabled_samples_payload(mask) + log.info( + "WanVideoEncode: SCAIL-Pose2 disabled samples path " + f"condition_mode={payload['scail_pose2_condition_mode']} " + f"reason={payload['scail_pose2_disable_reason']}" + ) + return (payload,) + vae.to(device) - image = image.clone() + image = driving_video.clone() B, H, W, C = image.shape if W % 16 != 0 or H % 16 != 0: diff --git a/nodes_model_loading.py b/nodes_model_loading.py index f5b7558e..1068cc30 100644 --- a/nodes_model_loading.py +++ b/nodes_model_loading.py @@ -11,6 +11,7 @@ from .wanvideo.modules.clip import CLIPModel from .wanvideo.wan_video_vae import WanVideoVAE, WanVideoVAE38 from .custom_linear import _replace_linear +from .SCAIL.scail2_loader import apply_scail_loader_patches from accelerate import init_empty_weights from .utils import set_module_tensor_to_device, get_module_memory_mb_per_device @@ -1637,11 +1638,8 @@ def loadmodel(self, model, base_precision, load_device, quantization, FactorConv3d(in_channels=in_dim_c, out_channels=in_dim_c, kernel_size=(3, 3, 3), stride=1), nn.SiLU()) transformer.condition_embedding_align = PoseRefNetNoBNV3(in_channels_x=16, in_channels_c=16, hidden_dim=128, num_heads=8) # Frame-wise Attention Alignment Unit - # SCAIL - if "patch_embedding_pose.weight" in sd: - log.info("SCAIL model detected, patching model...") - pose_dim = sd["patch_embedding_pose.weight"].shape[1] - transformer.patch_embedding_pose = nn.Conv3d(pose_dim, dim, kernel_size=patch_size, stride=patch_size) + # SCAIL / SCAIL-2 + apply_scail_loader_patches(transformer, sd, nn, dim=dim, patch_size=patch_size, log=log) if "image_to_cond.conv_in.bias" in sd: # One-to-all diff --git a/nodes_sampler.py b/nodes_sampler.py index 32b51c6a..4d25c259 100644 --- a/nodes_sampler.py +++ b/nodes_sampler.py @@ -13,6 +13,17 @@ from .multitalk.multitalk_loop import multitalk_loop from .cache_methods.cache_methods import cache_report from .nodes_model_loading import load_weights +from .SCAIL.scail2_routing import ( + add_scail2_model_param, + prepare_scail2_data, + scail2_context_window_input, +) +from .scail_pose2_mask_contract import ( + align_samples_to_latent_window, + apply_samples_to_noise, + normalize_samples_payload_for_sampler, + resize_noise_mask_for_latents, +) from .enhance_a_video.globals import set_enhance_weight, set_num_frames from .WanMove.trajectory import replace_feature from contextlib import nullcontext @@ -83,6 +94,12 @@ def process(self, model, image_embeds, shift, steps, cfg, seed, scheduler, rifle experimental_args=None, sigmas=None, unianimate_poses=None, fantasytalking_embeds=None, uni3c_embeds=None, multitalk_embeds=None, freeinit_args=None, start_step=0, end_step=-1, add_noise_to_samples=False): if flowedit_args is not None: raise Exception("FlowEdit support has been deprecated and removed due to lack of use and code maintainability") + samples, disabled_samples_context = normalize_samples_payload_for_sampler(samples) + if disabled_samples_context is not None: + log.info( + "WanVideoSampler: ignoring disabled SCAIL-Pose2 samples path " + f"{disabled_samples_context.to_log_fragment()}" + ) patcher = model model = model.model transformer = model.diffusion_model @@ -726,29 +743,35 @@ def process(self, model, image_embeds, shift, steps, cfg, seed, scheduler, rifle if input_samples.shape[1] != noise.shape[1]: input_samples = torch.cat([input_samples[:, :1].repeat(1, noise.shape[1] - input_samples.shape[1], 1, 1), input_samples], dim=1) - if add_noise_to_samples: - latent_timestep = timesteps[:1].to(noise) - noise = noise * latent_timestep / 1000 + (1 - latent_timestep / 1000) * input_samples - else: - noise = input_samples - noise_mask = samples.get("noise_mask", None) + noise_mask_contract = None if noise_mask is not None: log.info(f"Latent noise_mask shape: {noise_mask.shape}") original_image = samples.get("original_image", None) if original_image is None: original_image = input_samples - if len(noise_mask.shape) == 4: - noise_mask = noise_mask.squeeze(1) - if noise_mask.shape[0] < noise.shape[1]: - noise_mask = noise_mask.repeat(noise.shape[1] // noise_mask.shape[0], 1, 1) - - noise_mask = torch.nn.functional.interpolate( - noise_mask.unsqueeze(0).unsqueeze(0), # Add batch and channel dims [1,1,T,H,W] - size=(noise.shape[1], noise.shape[2], noise.shape[3]), - mode='trilinear', - align_corners=False - ).repeat(1, noise.shape[0], 1, 1, 1) + noise_mask, noise_mask_contract = resize_noise_mask_for_latents( + noise_mask, + latent_shape=(noise.shape[1], noise.shape[2], noise.shape[3]), + channel_count=noise.shape[0], + latent_grow_pixels=1, + latent_temporal_grow_frames=1, + ) + log.info(f"WanVideoSampler: {noise_mask_contract.to_log_string()}") + + noise, samples_init_contract = apply_samples_to_noise( + noise, + input_samples, + noise_mask=noise_mask, + timestep=timesteps[:1].to(noise), + add_noise_to_samples=add_noise_to_samples, + scail_pose2_replacement=( + noise_mask_contract.scail_pose2_replacement + if noise_mask_contract is not None + else False + ), + ) + log.info(f"WanVideoSampler: {samples_init_contract.to_log_string()}") # extra latents (Pusa) and 5b latents_to_insert = add_index = noise_multipliers = None @@ -1144,6 +1167,14 @@ def process(self, model, image_embeds, shift, steps, cfg, seed, scheduler, rifle scail_data = scail_embeds.copy() scail_data = dict_to_device(scail_data, device, dtype) + scail2_data = prepare_scail2_data( + image_embeds, + dict_to_device=dict_to_device, + device=device, + dtype=dtype, + ) + if scail2_data is not None: + log.info("Using SCAIL-2 native embeddings") # WanMove wanmove_embeds = None @@ -1417,6 +1448,8 @@ def predict_with_cfg(z, cfg_scale, positive_embeds, negative_embeds, timestep, i else: scail_data_in = scail_data + scail2_data_in = scail2_context_window_input(scail2_data, context_window) + if wanmove_embeds is not None and context_window is not None: image_cond_input = replace_feature(image_cond_input.unsqueeze(0), track_pos[:, context_window].unsqueeze(0), wanmove_embeds.get("strength", 1.0))[0] @@ -1500,6 +1533,7 @@ def predict_with_cfg(z, cfg_scale, positive_embeds, negative_embeds, timestep, i "rope_negative_offset": image_embeds.get("rope_negative_offset_frames", 0), # StoryMem rope negative offset "num_memory_frames": story_mem_latents.shape[1] if story_mem_latents is not None else 0, # StoryMem memory frames } + base_params = add_scail2_model_param(base_params, scail2_data_in) batch_size = 1 @@ -2317,45 +2351,58 @@ def predict_with_cfg(z, cfg_scale, positive_embeds, negative_embeds, timestep, i if samples is not None: input_samples = samples["samples"] + source_latent_frame_count = None + mask_start_latent = None + mask_end_latent = None if input_samples is not None: input_samples = input_samples.squeeze(0).to(noise) - # Check if we have enough frames in input_samples - # if latent_end_idx > input_samples.shape[1]: - # # We need more frames than available - pad the input_samples at the end - # pad_length = latent_end_idx - input_samples.shape[1] - # last_frame = input_samples[:, -1:].repeat(1, pad_length, 1, 1) - # input_samples = torch.cat([input_samples, last_frame], dim=1) - # input_samples = input_samples[:, latent_start_idx:latent_end_idx] - if noise_mask is not None: - original_image = input_samples.to(device) - - assert input_samples.shape[1] == noise.shape[1], f"Slice mismatch: {input_samples.shape[1]} vs {noise.shape[1]}" - - if add_noise_to_samples: - latent_timestep = timesteps[0] - noise = noise * latent_timestep / 1000 + (1 - latent_timestep / 1000) * input_samples - else: - noise = input_samples + input_samples, samples_window_contract = align_samples_to_latent_window( + input_samples, + target_frame_count=noise.shape[1], + start_latent=start_latent, + end_latent=end_latent, + ) + log.info(f"WanVideoSampler: {samples_window_contract.to_log_string()}") + if samples_window_contract.frame_policy.startswith("slice_"): + source_latent_frame_count = samples_window_contract.source_latent_frame_count + mask_start_latent = start_latent + mask_end_latent = end_latent # diff diff prep noise_mask = samples.get("noise_mask", None) + noise_mask_contract = None if noise_mask is not None: - if len(noise_mask.shape) == 4: - noise_mask = noise_mask.squeeze(1) - if noise_mask.shape[0] < noise.shape[1]: - noise_mask = noise_mask.repeat(noise.shape[1] // noise_mask.shape[0], 1, 1) - else: - noise_mask = noise_mask[start_latent:end_latent] - noise_mask = torch.nn.functional.interpolate( - noise_mask.unsqueeze(0).unsqueeze(0), # Add batch and channel dims [1,1,T,H,W] - size=(noise.shape[1], noise.shape[2], noise.shape[3]), - mode='trilinear', - align_corners=False - ).repeat(1, noise.shape[0], 1, 1, 1) + if input_samples is not None: + original_image = input_samples.to(device) + noise_mask, noise_mask_contract = resize_noise_mask_for_latents( + noise_mask, + latent_shape=(noise.shape[1], noise.shape[2], noise.shape[3]), + channel_count=noise.shape[0], + start_latent=mask_start_latent, + end_latent=mask_end_latent, + source_latent_frame_count=source_latent_frame_count, + latent_grow_pixels=1, + latent_temporal_grow_frames=1, + ) + log.info(f"WanVideoSampler: {noise_mask_contract.to_log_string()}") thresholds = torch.arange(len(timesteps), dtype=original_image.dtype) / len(timesteps) thresholds = thresholds.reshape(-1, 1, 1, 1, 1).to(device) masks = (1-noise_mask.repeat(len(timesteps), 1, 1, 1, 1).to(device)) > thresholds + if input_samples is not None: + noise, samples_init_contract = apply_samples_to_noise( + noise, + input_samples, + noise_mask=noise_mask, + timestep=timesteps[0], + add_noise_to_samples=add_noise_to_samples, + scail_pose2_replacement=( + noise_mask_contract.scail_pose2_replacement + if noise_mask_contract is not None + else False + ), + ) + log.info(f"WanVideoSampler: {samples_init_contract.to_log_string()}") if isinstance(scheduler, dict): sample_scheduler = copy.deepcopy(scheduler["sample_scheduler"]) diff --git a/readme.md b/readme.md index 018c2941..9aaca564 100644 --- a/readme.md +++ b/readme.md @@ -120,6 +120,18 @@ One-to-all-Animation: https://github.com/ssj9596/One-to-All-Animation SCAIL: https://github.com/zai-org/SCAIL +### SCAIL-Pose2 dual-mode samples + +The SCAIL-Pose2 dual-mode example can keep `WanVideoEncode.samples` connected +to `WanVideoSampler.samples`. `replacement` keeps the samples path active so the +sampler can use the encoded driving-video latents with the replacement denoise +mask. `animation` and other non-replacement modes mark that samples payload as +disabled, and the sampler ignores it before video-to-video samples handling. + +`add_noise_to_samples` only matters when the samples path is active. When +SCAIL-Pose2 disables samples for non-replacement mode, that sampler option has +no effect on the disabled payload. + Not exactly Wan model, but close enough to work with the code base: diff --git a/scail_pose2_mask_contract.py b/scail_pose2_mask_contract.py new file mode 100644 index 00000000..a50b5bf1 --- /dev/null +++ b/scail_pose2_mask_contract.py @@ -0,0 +1,495 @@ +"""SCAIL-Pose2 replacement noise-mask conversion helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +SCAIL_POSE2_CONDITION_MODE_ATTR = "scail_pose2_condition_mode" +SCAIL_POSE2_MASK_ROLE_ATTR = "scail_pose2_mask_role" +SCAIL_POSE2_REPLACEMENT_DENOISE_MASK_ROLE = "replacement_denoise_mask" +SCAIL_POSE2_DISABLE_SAMPLES_ATTR = "scail_pose2_disable_samples" +SCAIL_POSE2_DISABLE_SAMPLES_REASON_ATTR = "scail_pose2_disable_samples_reason" +SCAIL_POSE2_SAMPLES_DISABLED_KEY = "scail_pose2_samples_disabled" +SCAIL_POSE2_DISABLE_REASON_KEY = "scail_pose2_disable_reason" +SCAIL_POSE2_CONDITION_MODE_KEY = "scail_pose2_condition_mode" +SCAIL_POSE2_DISABLED_REASON_UNSPECIFIED = "unspecified" +SCAIL_POSE2_CONDITION_MODE_UNKNOWN = "unknown" + + +@dataclass(frozen=True) +class NoiseMaskLatentContract: + original_shape: tuple[int, ...] + prepared_shape: tuple[int, ...] + latent_shape: tuple[int, int, int] + channel_count: int + interpolation_mode: str + scail_pose2_replacement: bool + frame_policy: str + subject_ratio: float + preserve_ratio: float + pre_grow_subject_ratio: float + latent_grow_pixels: int + latent_temporal_grow_frames: int + + def to_log_string(self) -> str: + return ( + "noise_mask_latent_contract " + f"original_shape={self.original_shape} " + f"prepared_shape={self.prepared_shape} " + f"latent_shape={self.latent_shape} " + f"channels={self.channel_count} " + f"mode={self.interpolation_mode} " + f"scail_pose2_replacement={self.scail_pose2_replacement} " + f"frame_policy={self.frame_policy} " + f"latent_grow_pixels={self.latent_grow_pixels} " + f"latent_temporal_grow_frames={self.latent_temporal_grow_frames} " + f"pre_grow_subject_ratio={self.pre_grow_subject_ratio:.6f} " + f"subject_ratio={self.subject_ratio:.6f} " + f"preserve_ratio={self.preserve_ratio:.6f}" + ) + + +@dataclass(frozen=True) +class SamplesInitializationContract: + add_noise_to_samples: bool + scail_pose2_replacement: bool + mask_aware: bool + subject_ratio: float + preserve_ratio: float + subject_source: str + preserve_source: str + + def to_log_string(self) -> str: + return ( + "samples_initialization_contract " + f"add_noise_to_samples={self.add_noise_to_samples} " + f"scail_pose2_replacement={self.scail_pose2_replacement} " + f"mask_aware={self.mask_aware} " + f"subject_source={self.subject_source} " + f"preserve_source={self.preserve_source} " + f"subject_ratio={self.subject_ratio:.6f} " + f"preserve_ratio={self.preserve_ratio:.6f}" + ) + + +@dataclass(frozen=True) +class SamplesWindowAlignmentContract: + original_shape: tuple[int, ...] + output_shape: tuple[int, ...] + source_latent_frame_count: int + output_frame_count: int + frame_policy: str + start_latent: int | None + end_latent: int | None + + def to_log_string(self) -> str: + return ( + "samples_window_alignment_contract " + f"original_shape={self.original_shape} " + f"output_shape={self.output_shape} " + f"source_latent_frame_count={self.source_latent_frame_count} " + f"output_frame_count={self.output_frame_count} " + f"frame_policy={self.frame_policy} " + f"start_latent={self.start_latent} " + f"end_latent={self.end_latent}" + ) + + +@dataclass(frozen=True) +class DisabledSamplesPayloadContext: + condition_mode: str + reason: str + + def to_log_fragment(self) -> str: + return f"condition_mode={self.condition_mode} reason={self.reason}" + + +def is_scail_pose2_replacement_noise_mask(noise_mask: Any) -> bool: + return ( + getattr(noise_mask, SCAIL_POSE2_CONDITION_MODE_ATTR, None) == "replacement" + and getattr(noise_mask, SCAIL_POSE2_MASK_ROLE_ATTR, None) + == SCAIL_POSE2_REPLACEMENT_DENOISE_MASK_ROLE + ) + + +def scail_pose2_mask_disables_samples(mask: Any) -> bool: + """Return whether SCAIL-Pose2 metadata disables the samples path.""" + + return bool(getattr(mask, SCAIL_POSE2_DISABLE_SAMPLES_ATTR, False)) + + +def build_disabled_samples_payload(mask: Any) -> dict[str, Any]: + """Build a LATENT payload that downstream samplers intentionally ignore.""" + + reason = getattr( + mask, + SCAIL_POSE2_DISABLE_SAMPLES_REASON_ATTR, + SCAIL_POSE2_DISABLED_REASON_UNSPECIFIED, + ) + condition_mode = getattr( + mask, + SCAIL_POSE2_CONDITION_MODE_ATTR, + SCAIL_POSE2_CONDITION_MODE_UNKNOWN, + ) + return { + "samples": None, + "noise_mask": None, + SCAIL_POSE2_SAMPLES_DISABLED_KEY: True, + SCAIL_POSE2_DISABLE_REASON_KEY: reason, + SCAIL_POSE2_CONDITION_MODE_KEY: condition_mode, + } + + +def samples_payload_is_disabled(samples: Any) -> bool: + """Return whether a LATENT samples payload is explicitly disabled.""" + + return bool( + isinstance(samples, dict) + and samples.get(SCAIL_POSE2_SAMPLES_DISABLED_KEY, False) + ) + + +def disabled_samples_log_context( + samples: Any, +) -> DisabledSamplesPayloadContext | None: + """Return public-safe log context for an intentionally disabled samples path.""" + + if not samples_payload_is_disabled(samples): + return None + return DisabledSamplesPayloadContext( + condition_mode=str( + samples.get( + SCAIL_POSE2_CONDITION_MODE_KEY, + SCAIL_POSE2_CONDITION_MODE_UNKNOWN, + ) + ), + reason=str( + samples.get( + SCAIL_POSE2_DISABLE_REASON_KEY, + SCAIL_POSE2_DISABLED_REASON_UNSPECIFIED, + ) + ), + ) + + +def normalize_samples_payload_for_sampler( + samples: Any, +) -> tuple[Any | None, DisabledSamplesPayloadContext | None]: + """Normalize the sampler input before any vid2vid samples branch runs.""" + + context = disabled_samples_log_context(samples) + if context is not None: + return None, context + return samples, None + + +def _shape_tuple(value: Any) -> tuple[int, ...]: + return tuple(int(part) for part in getattr(value, "shape", ())) + + +def _positive_latent_shape(latent_shape: tuple[int, int, int]) -> tuple[int, int, int]: + parsed = tuple(int(part) for part in latent_shape) + if len(parsed) != 3 or any(part <= 0 for part in parsed): + raise ValueError("latent_shape must be a positive (frames, height, width) tuple") + return parsed + + +def _positive_channel_count(channel_count: Any) -> int: + parsed = int(channel_count) + if parsed <= 0: + raise ValueError("channel_count must be positive") + return parsed + + +def _non_negative_grow_pixels(latent_grow_pixels: Any) -> int: + parsed = int(latent_grow_pixels) + if parsed < 0: + raise ValueError("latent_grow_pixels must be non-negative") + return parsed + + +def align_samples_to_latent_window( + input_samples: Any, + *, + target_frame_count: Any, + start_latent: int | None = None, + end_latent: int | None = None, +) -> tuple[Any, SamplesWindowAlignmentContract]: + """Align `[C,T,H,W]` samples to the active latent window.""" + + original_shape = _shape_tuple(input_samples) + if len(original_shape) != 4: + raise ValueError("input_samples must have shape [C,T,H,W]") + target_frames = int(target_frame_count) + if target_frames <= 0: + raise ValueError("target_frame_count must be positive") + + source_frames = int(input_samples.shape[1]) + output = input_samples + policy = "direct" + parsed_start = int(start_latent) if start_latent is not None else None + parsed_end = int(end_latent) if end_latent is not None else None + + if parsed_start is not None and parsed_end is not None: + if source_frames == target_frames: + policy = f"direct_already_windowed_{parsed_start}_{parsed_end}" + else: + if parsed_start < 0 or parsed_end <= parsed_start or parsed_end > source_frames: + raise ValueError( + "sample latent window is out of range, " + f"got {parsed_start}:{parsed_end} for {source_frames} frames" + ) + output = input_samples[:, parsed_start:parsed_end] + policy = f"slice_{parsed_start}_{parsed_end}" + + if int(output.shape[1]) != target_frames: + raise ValueError( + "sample latent window frame count mismatch, " + f"got {int(output.shape[1])} frames for target {target_frames}" + ) + + contract = SamplesWindowAlignmentContract( + original_shape=original_shape, + output_shape=_shape_tuple(output), + source_latent_frame_count=source_frames, + output_frame_count=int(output.shape[1]), + frame_policy=policy, + start_latent=parsed_start, + end_latent=parsed_end, + ) + return output, contract + + +def resize_noise_mask_for_latents( + noise_mask: Any, + *, + latent_shape: tuple[int, int, int], + channel_count: Any, + start_latent: int | None = None, + end_latent: int | None = None, + source_latent_frame_count: int | None = None, + latent_grow_pixels: Any = 0, + latent_temporal_grow_frames: Any = 0, +) -> tuple[Any, NoiseMaskLatentContract]: + """Resize a sampler `noise_mask` to `[1, C, T, H, W]` latent mask shape.""" + + target_frames, target_height, target_width = _positive_latent_shape(latent_shape) + channels = _positive_channel_count(channel_count) + grow_pixels = _non_negative_grow_pixels(latent_grow_pixels) + temporal_grow_frames = _non_negative_grow_pixels(latent_temporal_grow_frames) + original_shape = _shape_tuple(noise_mask) + scail_pose2_replacement = is_scail_pose2_replacement_noise_mask(noise_mask) + prepared = noise_mask + + if len(prepared.shape) == 4: + prepared = prepared.squeeze(1) + if len(prepared.shape) != 3: + raise ValueError("noise_mask must have shape [T,H,W] or [T,1,H,W]") + + frame_policy = "direct" + if ( + start_latent is not None + and end_latent is not None + and source_latent_frame_count is not None + and int(prepared.shape[0]) != int(source_latent_frame_count) + ): + source_frames = int(source_latent_frame_count) + if source_frames <= 0: + raise ValueError("source_latent_frame_count must be positive") + prepared = _resize_prepared_mask( + prepared, + target_frames=source_frames, + target_height=target_height, + target_width=target_width, + scail_pose2_replacement=scail_pose2_replacement, + ) + prepared = prepared[int(start_latent):int(end_latent)] + frame_policy = ( + f"resize_full_{source_frames}_then_slice_" + f"{int(start_latent)}_{int(end_latent)}" + ) + elif prepared.shape[0] < target_frames: + repeat_count = max(1, target_frames // int(prepared.shape[0])) + if repeat_count > 1: + prepared = prepared.repeat(repeat_count, 1, 1) + frame_policy = f"repeat_x{repeat_count}" + else: + frame_policy = "interpolate_time" + elif start_latent is not None and end_latent is not None: + prepared = prepared[int(start_latent):int(end_latent)] + frame_policy = f"slice_{int(start_latent)}_{int(end_latent)}" + + prepared_shape = _shape_tuple(prepared) + interpolation_mode = "conservative_area" if scail_pose2_replacement else "trilinear" + resized_3d = _resize_prepared_mask( + prepared, + target_frames=target_frames, + target_height=target_height, + target_width=target_width, + scail_pose2_replacement=scail_pose2_replacement, + ) + pre_grow_subject_ratio = float(resized_3d.float().mean().item()) + if scail_pose2_replacement and grow_pixels > 0: + resized_3d = _grow_spatial_binary_mask(resized_3d, grow_pixels) + if scail_pose2_replacement and temporal_grow_frames > 0: + resized_3d = _grow_temporal_binary_mask(resized_3d, temporal_grow_frames) + resized = resized_3d.unsqueeze(0).unsqueeze(0) + subject_ratio = float(resized.float().mean().item()) + contract = NoiseMaskLatentContract( + original_shape=original_shape, + prepared_shape=prepared_shape, + latent_shape=(target_frames, target_height, target_width), + channel_count=channels, + interpolation_mode=interpolation_mode, + scail_pose2_replacement=scail_pose2_replacement, + frame_policy=frame_policy, + subject_ratio=subject_ratio, + preserve_ratio=1.0 - subject_ratio, + pre_grow_subject_ratio=pre_grow_subject_ratio, + latent_grow_pixels=grow_pixels if scail_pose2_replacement else 0, + latent_temporal_grow_frames=( + temporal_grow_frames if scail_pose2_replacement else 0 + ), + ) + return resized.repeat(1, channels, 1, 1, 1), contract + + +def _resize_prepared_mask( + prepared: Any, + *, + target_frames: int, + target_height: int, + target_width: int, + scail_pose2_replacement: bool, +) -> Any: + import torch.nn.functional as F + + view = prepared.unsqueeze(0).unsqueeze(0) + if scail_pose2_replacement: + binary = (prepared >= 0.5).to(dtype=prepared.dtype) + resized = F.interpolate( + binary.unsqueeze(0).unsqueeze(0), + size=(target_frames, target_height, target_width), + mode="area", + ) + resized = (resized > 0.0).to(dtype=prepared.dtype) + else: + resized = F.interpolate( + view, + size=(target_frames, target_height, target_width), + mode="trilinear", + align_corners=False, + ) + return resized.squeeze(0).squeeze(0) + + +def _grow_spatial_binary_mask(mask: Any, grow_pixels: int) -> Any: + import torch.nn.functional as F + + kernel = grow_pixels * 2 + 1 + view = mask.unsqueeze(0).unsqueeze(0).float() + grown = F.max_pool3d( + view, + kernel_size=(1, kernel, kernel), + stride=1, + padding=(0, grow_pixels, grow_pixels), + ) + return (grown.squeeze(0).squeeze(0) >= 0.5).to(dtype=mask.dtype) + + +def _grow_temporal_binary_mask(mask: Any, grow_frames: int) -> Any: + import torch.nn.functional as F + + kernel = grow_frames * 2 + 1 + view = mask.unsqueeze(0).unsqueeze(0).float() + grown = F.max_pool3d( + view, + kernel_size=(kernel, 1, 1), + stride=1, + padding=(grow_frames, 0, 0), + ) + return (grown.squeeze(0).squeeze(0) >= 0.5).to(dtype=mask.dtype) + + +def apply_samples_to_noise( + noise: Any, + input_samples: Any, + *, + noise_mask: Any | None, + timestep: Any, + add_noise_to_samples: bool, + scail_pose2_replacement: bool, +) -> tuple[Any, SamplesInitializationContract]: + """Apply input samples to sampler noise with SCAIL-Pose2 mask awareness.""" + + if input_samples is None: + raise ValueError("input_samples must not be None") + if tuple(input_samples.shape) != tuple(noise.shape): + raise ValueError( + "input_samples and noise must share shape, " + f"got {tuple(input_samples.shape)} and {tuple(noise.shape)}" + ) + + if add_noise_to_samples: + scale = _noise_timestep_scale(timestep, noise) + initialized_from_samples = noise * scale + (1.0 - scale) * input_samples + sample_source = "noised_samples" + else: + initialized_from_samples = input_samples + sample_source = "samples" + + if scail_pose2_replacement and noise_mask is not None: + subject_mask = _mask_like_noise(noise_mask, noise) + preserve_mask = 1.0 - subject_mask + initialized = initialized_from_samples * preserve_mask + noise * subject_mask + subject_ratio = float(subject_mask.float().mean().item()) + return initialized, SamplesInitializationContract( + add_noise_to_samples=bool(add_noise_to_samples), + scail_pose2_replacement=True, + mask_aware=True, + subject_ratio=subject_ratio, + preserve_ratio=1.0 - subject_ratio, + subject_source="random_noise", + preserve_source=sample_source, + ) + + return initialized_from_samples, SamplesInitializationContract( + add_noise_to_samples=bool(add_noise_to_samples), + scail_pose2_replacement=bool(scail_pose2_replacement), + mask_aware=False, + subject_ratio=0.0, + preserve_ratio=1.0, + subject_source=sample_source, + preserve_source=sample_source, + ) + + +def _noise_timestep_scale(timestep: Any, noise: Any) -> Any: + import torch + + if torch.is_tensor(timestep): + timestep_value = timestep.to(device=noise.device, dtype=noise.dtype).reshape(-1)[0] + else: + timestep_value = torch.tensor(timestep, device=noise.device, dtype=noise.dtype) + return timestep_value / 1000.0 + + +def _mask_like_noise(noise_mask: Any, noise: Any) -> Any: + mask = noise_mask + if len(mask.shape) == len(noise.shape) + 1 and int(mask.shape[0]) == 1: + mask = mask.squeeze(0) + if len(mask.shape) != len(noise.shape): + raise ValueError( + "noise_mask must have shape [C,T,H,W] or [1,C,T,H,W], " + f"got {tuple(noise_mask.shape)} for noise {tuple(noise.shape)}" + ) + if int(mask.shape[0]) == 1 and int(noise.shape[0]) != 1: + mask = mask.repeat(int(noise.shape[0]), 1, 1, 1) + if tuple(mask.shape) != tuple(noise.shape): + raise ValueError( + "noise_mask and noise must share broadcasted shape, " + f"got {tuple(mask.shape)} and {tuple(noise.shape)}" + ) + return mask.to(device=noise.device, dtype=noise.dtype).clamp(0.0, 1.0) diff --git a/tests/test_mtv_nlf_predict.py b/tests/test_mtv_nlf_predict.py new file mode 100644 index 00000000..be85ca38 --- /dev/null +++ b/tests/test_mtv_nlf_predict.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import importlib.util +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def import_nlf_bbox_module(): + spec = importlib.util.spec_from_file_location( + "mtv_nlf_bbox_under_test", + ROOT / "MTV" / "nlf_bbox.py", + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class MTVNLFPredictBBoxTests(unittest.TestCase): + def test_formatter_preserves_multi_person_candidates(self) -> None: + module = import_nlf_bbox_module() + + formatted = module.format_nlf_detected_boxes( + [ + [[1, 2, 4, 6, 0.9], [10, 20, 15, 26, 0.8]], + [[7, 8, 8, 10, 0.7]], + [], + ] + ) + + self.assertEqual( + [[1.0, 2.0, 4.0, 6.0], [10.0, 20.0, 15.0, 26.0]], + formatted[0], + ) + self.assertEqual([7.0, 8.0, 8.0, 10.0], formatted[1]) + self.assertEqual([0.0, 0.0, 0.0, 0.0], formatted[2]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scail2_condition_embeds.py b/tests/test_scail2_condition_embeds.py new file mode 100644 index 00000000..a41531fb --- /dev/null +++ b/tests/test_scail2_condition_embeds.py @@ -0,0 +1,660 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +import types +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +class FakeTensor: + def __init__(self, shape): + self.shape = tuple(shape) + self.ndim = len(self.shape) + + def __getitem__(self, key): + if isinstance(key, tuple) and key and key[-1] == slice(None, 3, None): + return FakeTensor((*self.shape[:-1], 3)) + if key == 0: + return FakeTensor(self.shape[1:]) + raise TypeError(f"unsupported fake tensor index: {key!r}") + + def permute(self, *dims): + return FakeTensor(tuple(self.shape[index] for index in dims)) + + def to(self, *args, **kwargs): + return self + + def contiguous(self): + return self + + def __mul__(self, other): + return FakeTensor(self.shape) + + def __sub__(self, other): + return FakeTensor(self.shape) + + +def install_torch_stub() -> None: + torch = types.ModuleType("torch") + torch.float32 = "float32" + torch.device = lambda name: name + torch.as_tensor = lambda data, dtype=None: data if isinstance(data, FakeTensor) else FakeTensor(()) + + nn = types.ModuleType("torch.nn") + functional = types.ModuleType("torch.nn.functional") + + def interpolate(tensor, size, mode=None, align_corners=None): + frames, channels, _height, _width = tensor.shape + return FakeTensor((frames, channels, int(size[0]), int(size[1]))) + + functional.interpolate = interpolate + nn.functional = functional + torch.nn = nn + + sys.modules["torch"] = torch + sys.modules["torch.nn"] = nn + sys.modules["torch.nn.functional"] = functional + + +def install_comfy_stub() -> None: + install_torch_stub() + comfy = types.ModuleType("comfy") + comfy.__path__ = [] + model_management = types.ModuleType("comfy.model_management") + model_management.get_torch_device = lambda: "cpu" + model_management.unet_offload_device = lambda: "cpu" + sys.modules["comfy"] = comfy + sys.modules["comfy.model_management"] = model_management + + +def install_comfy_stub_with_real_torch() -> None: + comfy = types.ModuleType("comfy") + comfy.__path__ = [] + model_management = types.ModuleType("comfy.model_management") + model_management.get_torch_device = lambda: "cpu" + model_management.unet_offload_device = lambda: "cpu" + sys.modules["comfy"] = comfy + sys.modules["comfy.model_management"] = model_management + + +def import_scail_nodes(): + preserved_modules = { + name: module + for name, module in sys.modules.items() + if name == "torch" + or name.startswith("torch.") + or name == "comfy" + or name.startswith("comfy.") + } + install_comfy_stub() + module_name = "wan_scail_nodes_under_test" + sys.modules.pop(module_name, None) + try: + spec = importlib.util.spec_from_file_location( + module_name, + ROOT / "SCAIL" / "nodes.py", + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + finally: + for name in tuple(sys.modules): + if ( + name == "torch" + or name.startswith("torch.") + or name == "comfy" + or name.startswith("comfy.") + ): + sys.modules.pop(name, None) + sys.modules.update(preserved_modules) + + +def import_scail_nodes_with_real_torch(): + for name in ("comfy", "comfy.model_management"): + sys.modules.pop(name, None) + import torch # noqa: F401 + + install_comfy_stub_with_real_torch() + module_name = "wan_scail_nodes_real_torch_under_test" + sys.modules.pop(module_name, None) + spec = importlib.util.spec_from_file_location( + module_name, + ROOT / "SCAIL" / "nodes.py", + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +class FakeVAE: + dtype = "float32" + + def __init__(self) -> None: + self.to_calls = [] + self.encode_calls = [] + + def to(self, target): + self.to_calls.append(str(target)) + return self + + def encode(self, images, device, tiled=False): + image = images[0] + self.encode_calls.append( + { + "shape": tuple(image.shape), + "device": str(device), + "tiled": tiled, + } + ) + _channels, frames, height, width = image.shape + latent_h = max(height // 8, 1) + latent_w = max(width // 8, 1) + return [FakeTensor((16, frames, latent_h, latent_w))] + + +class RecordingVAE(FakeVAE): + def __init__(self) -> None: + super().__init__() + import torch + + self.dtype = torch.float32 + + def encode(self, images, device, tiled=False): + image = images[0] + self.encode_calls.append( + { + "shape": tuple(image.shape), + "device": str(device), + "tiled": tiled, + "image": image.detach().cpu().clone(), + } + ) + _channels, frames, height, width = image.shape + latent_h = max(height // 8, 1) + latent_w = max(width // 8, 1) + return [FakeTensor((16, frames, latent_h, latent_w))] + + +def image_batch(frames: int, height: int = 8, width: int = 8): + return FakeTensor((frames, height, width, 3)) + + +def runtime_mask(latent_frames: int): + data = FakeTensor((1, latent_frames, 28, 1, 1)) + return { + "data": data, + "comfy_shape": tuple(data.shape), + "scail2_shape": (28, latent_frames, 1, 1), + } + + +def payload( + *, + include_additional: bool = True, + ref_height: int = 8, + ref_width: int = 8, + additional_height: int = 8, + additional_width: int = 8, +): + additional_ref = {"image": image_batch(1, additional_height, additional_width)} + additional_refs = [additional_ref] if include_additional else [] + additional_masks = [runtime_mask(1)] if include_additional else [] + condition = { + "ref_image": image_batch(1, ref_height, ref_width), + "pose_video": image_batch(5), + "additional_references": additional_refs, + } + return { + "kind": "wanvideo_scail2_condition_adapter", + "version": 1, + "schema": { + "name": "scail_pose2.wanvideo_scail2_payload", + "version": 1, + "native_wrapper": { + "embeds_key": "scail2_embeds", + }, + }, + "condition": condition, + "mode": "replacement", + "replace_flag": True, + "dimensions": { + "width": 8, + "height": 8, + "num_frames": 5, + }, + "source": { + "source_kind": "unit_test", + }, + "runtime_masks": { + "reference": runtime_mask(1), + "driving": runtime_mask(2), + "additional_references": additional_masks, + }, + "additional_references": additional_refs, + } + + +def real_runtime_mask(latent_frames: int): + import torch + + data = torch.zeros((1, latent_frames, 28, 1, 1), dtype=torch.float32) + return { + "data": data, + "comfy_shape": tuple(data.shape), + "scail2_shape": (28, latent_frames, 1, 1), + } + + +def real_replacement_payload(): + import torch + + ref_mask_indices = torch.full((1, 8, 8), -1, dtype=torch.int8) + ref_mask_indices[:, :, :4] = 3 + driving_mask_indices = torch.zeros((5, 8, 8), dtype=torch.int8) + driving_mask_indices[:, :, :4] = 3 + add_mask_indices = torch.full((1, 8, 8), -1, dtype=torch.int8) + add_mask_indices[:, :4, :] = 3 + additional_ref = { + "image": torch.ones((1, 8, 8, 3), dtype=torch.float32), + "mask_indices": add_mask_indices, + } + condition = { + "ref_image": torch.ones((1, 8, 8, 3), dtype=torch.float32), + "ref_mask_indices": ref_mask_indices, + "pose_video": torch.ones((5, 8, 8, 3), dtype=torch.float32), + "driving_mask_indices": driving_mask_indices, + "additional_references": [additional_ref], + } + return { + "kind": "wanvideo_scail2_condition_adapter", + "version": 1, + "schema": { + "name": "scail_pose2.wanvideo_scail2_payload", + "version": 1, + "native_wrapper": { + "embeds_key": "scail2_embeds", + }, + }, + "condition": condition, + "mode": "replacement", + "replace_flag": True, + "dimensions": { + "width": 8, + "height": 8, + "num_frames": 5, + }, + "source": { + "source_kind": "unit_test", + }, + "runtime_masks": { + "reference": real_runtime_mask(1), + "driving": real_runtime_mask(2), + "additional_references": [real_runtime_mask(1)], + }, + "additional_references": [additional_ref], + } + + +def real_condition_video_leak_payload(*, replace_flag: bool = True): + import torch + + driving_mask_indices = torch.zeros((5, 8, 8), dtype=torch.int8) + driving_mask_indices[:, :, :4] = 3 + pose_video = torch.ones((5, 8, 8, 3), dtype=torch.float32) + pose_video[:, :, :4] = 0.0 + condition = { + "ref_image": torch.ones((1, 8, 8, 3), dtype=torch.float32), + "ref_mask_indices": torch.zeros((1, 8, 8), dtype=torch.int8), + "pose_video": pose_video, + "driving_mask_indices": driving_mask_indices, + "additional_references": [], + } + return { + "kind": "wanvideo_scail2_condition_adapter", + "version": 1, + "schema": { + "name": "scail_pose2.wanvideo_scail2_payload", + "version": 1, + "native_wrapper": { + "embeds_key": "scail2_embeds", + }, + }, + "condition": condition, + "mode": "replacement" if replace_flag else "animation", + "replace_flag": replace_flag, + "dimensions": { + "width": 8, + "height": 8, + "num_frames": 5, + }, + "source": { + "source_kind": "unit_test", + }, + "runtime_masks": { + "reference": real_runtime_mask(1), + "driving": real_runtime_mask(2), + "additional_references": [], + }, + "additional_references": [], + } + + +def real_condition_video_structure_payload(): + import torch + + driving_mask_indices = torch.zeros((5, 8, 8), dtype=torch.int8) + driving_mask_indices[:, :, :4] = 3 + pose_video = torch.ones((5, 8, 8, 3), dtype=torch.float32) + pose_video[:, :, :2] = 0.05 + pose_video[:, :, 2:4] = 0.95 + condition = { + "ref_image": torch.ones((1, 8, 8, 3), dtype=torch.float32), + "ref_mask_indices": torch.zeros((1, 8, 8), dtype=torch.int8), + "pose_video": pose_video, + "driving_mask_indices": driving_mask_indices, + "additional_references": [], + } + return { + "kind": "wanvideo_scail2_condition_adapter", + "version": 1, + "schema": { + "name": "scail_pose2.wanvideo_scail2_payload", + "version": 1, + "native_wrapper": { + "embeds_key": "scail2_embeds", + }, + }, + "condition": condition, + "mode": "replacement", + "replace_flag": True, + "dimensions": { + "width": 8, + "height": 8, + "num_frames": 5, + }, + "source": { + "source_kind": "unit_test", + }, + "runtime_masks": { + "reference": real_runtime_mask(1), + "driving": real_runtime_mask(2), + "additional_references": [], + }, + "additional_references": [], + } + + +class WanVideoAddSCAIL2ConditionEmbedsTests(unittest.TestCase): + def test_node_contract_is_registered(self) -> None: + module = import_scail_nodes() + + self.assertIn("WanVideoAddSCAIL2ConditionEmbeds", module.NODE_CLASS_MAPPINGS) + self.assertIn("WanVideoAddSCAILPoseEmbeds", module.NODE_CLASS_MAPPINGS) + self.assertIn("WanVideoAddSCAILReferenceEmbeds", module.NODE_CLASS_MAPPINGS) + + node_cls = module.NODE_CLASS_MAPPINGS["WanVideoAddSCAIL2ConditionEmbeds"] + self.assertEqual(("WANVIDIMAGE_EMBEDS",), node_cls.RETURN_TYPES) + self.assertEqual(("image_embeds",), node_cls.RETURN_NAMES) + self.assertEqual( + "SCAIL2_WANVIDEO_PAYLOAD", + node_cls.INPUT_TYPES()["required"]["condition"][0], + ) + required = node_cls.INPUT_TYPES()["required"] + self.assertIn("ref_image_strength", required) + self.assertIn("ref_mask_strength", required) + self.assertIn("condition_video_strength", required) + self.assertIn("driving_mask_strength", required) + self.assertEqual(1.0, required["ref_image_strength"][1]["default"]) + self.assertEqual(1.0, required["ref_mask_strength"][1]["default"]) + self.assertEqual(1.0, required["condition_video_strength"][1]["default"]) + self.assertEqual(1.0, required["driving_mask_strength"][1]["default"]) + + def test_add_materializes_scail2_embeds(self) -> None: + module = import_scail_nodes() + vae = FakeVAE() + embeds = { + "target_shape": (16, 2, 1, 1), + "num_frames": 5, + } + clip_embeds = {"clip_embeds": object()} + + result = module.WanVideoAddSCAIL2ConditionEmbeds().add( + embeds, + payload(), + vae, + clip_embeds=clip_embeds, + )[0] + + self.assertIsNot(result, embeds) + self.assertNotIn("scail_embeds", result) + self.assertIn("scail2_embeds", result) + + scail2 = result["scail2_embeds"] + self.assertTrue(scail2["replace_flag"]) + self.assertEqual("replacement", scail2["mode"]) + self.assertEqual([(16, 1, 1, 1)], [x.shape for x in scail2["ref_latents"]]) + self.assertEqual([(16, 5, 1, 1)], [x.shape for x in scail2["pose_latents"]]) + self.assertEqual([(28, 1, 1, 1)], [x.shape for x in scail2["ref_masks"]]) + self.assertEqual([(28, 2, 1, 1)], [x.shape for x in scail2["driving_masks"]]) + self.assertEqual([(16, 1, 1, 1)], [x.shape for x in scail2["additional_ref_latents"]]) + self.assertEqual([(28, 1, 1, 1)], [x.shape for x in scail2["additional_ref_masks"]]) + self.assertIs(scail2["clip_context"], clip_embeds["clip_embeds"]) + self.assertIs(result["clip_context"], clip_embeds["clip_embeds"]) + self.assertIsNone(scail2["segment"]) + self.assertEqual({"source_kind": "unit_test"}, scail2["source"]) + self.assertEqual( + { + "ref_image": 1.0, + "ref_mask": 1.0, + "condition_video": 1.0, + "driving_mask": 1.0, + }, + scail2["strengths"], + ) + self.assertEqual((3, 1, 8, 8), vae.encode_calls[0]["shape"]) + self.assertEqual((3, 5, 4, 4), vae.encode_calls[1]["shape"]) + + def test_add_stores_non_default_strength_metadata(self) -> None: + module = import_scail_nodes() + result = module.WanVideoAddSCAIL2ConditionEmbeds().add( + { + "target_shape": (16, 2, 1, 1), + "num_frames": 5, + }, + payload(), + FakeVAE(), + ref_image_strength=1.4, + ref_mask_strength=0.7, + condition_video_strength=0.5, + driving_mask_strength=1.8, + )[0] + + self.assertEqual( + { + "ref_image": 1.4, + "ref_mask": 0.7, + "condition_video": 0.5, + "driving_mask": 1.8, + }, + result["scail2_embeds"]["strengths"], + ) + + def test_rejects_invalid_strength_metadata(self) -> None: + module = import_scail_nodes() + + with self.assertRaisesRegex(ValueError, "ref_image_strength"): + module.WanVideoAddSCAIL2ConditionEmbeds().add( + {"target_shape": (16, 2, 1, 1), "num_frames": 5}, + payload(), + FakeVAE(), + ref_image_strength=-0.1, + ) + + def test_references_are_resized_to_payload_dimensions_before_encode(self) -> None: + module = import_scail_nodes() + vae = FakeVAE() + embeds = { + "target_shape": (16, 2, 1, 1), + "num_frames": 5, + } + + module.WanVideoAddSCAIL2ConditionEmbeds().add( + embeds, + payload(ref_height=16, ref_width=12, additional_height=10, additional_width=14), + vae, + ) + + self.assertEqual((3, 1, 8, 8), vae.encode_calls[0]["shape"]) + self.assertEqual((3, 5, 4, 4), vae.encode_calls[1]["shape"]) + self.assertEqual((3, 1, 8, 8), vae.encode_calls[2]["shape"]) + + @unittest.skipUnless(importlib.util.find_spec("torch"), "torch is unavailable") + def test_replacement_mode_composites_reference_images_before_encode(self) -> None: + module = import_scail_nodes_with_real_torch() + vae = RecordingVAE() + embeds = { + "target_shape": (16, 2, 1, 1), + "num_frames": 5, + } + + module.WanVideoAddSCAIL2ConditionEmbeds().add( + embeds, + real_replacement_payload(), + vae, + ) + + primary_ref = vae.encode_calls[0]["image"] + additional_ref = vae.encode_calls[2]["image"] + self.assertEqual(1.0, float(primary_ref[0, 0, 0, 0].item())) + self.assertEqual(-1.0, float(primary_ref[0, 0, 0, 7].item())) + self.assertEqual(1.0, float(additional_ref[0, 0, 0, 0].item())) + self.assertEqual(-1.0, float(additional_ref[0, 0, 7, 0].item())) + + @unittest.skipUnless(importlib.util.find_spec("torch"), "torch is unavailable") + def test_replacement_mode_preserves_raw_condition_video_before_pose_encode( + self, + ) -> None: + module = import_scail_nodes_with_real_torch() + vae = RecordingVAE() + + module.WanVideoAddSCAIL2ConditionEmbeds().add( + { + "target_shape": (16, 2, 1, 1), + "num_frames": 5, + }, + real_condition_video_leak_payload(replace_flag=True), + vae, + ) + + encoded_pose_input = vae.encode_calls[1]["image"] + subject_region = encoded_pose_input[:, :, :, :2] + background_region = encoded_pose_input[:, :, :, 2:] + self.assertLess(float(subject_region[0].max().item()), -0.9) + self.assertLess(float(subject_region[1].max().item()), -0.9) + self.assertLess(float(subject_region[2].max().item()), -0.9) + self.assertGreater(float(background_region[0].min().item()), 0.9) + + @unittest.skipUnless(importlib.util.find_spec("torch"), "torch is unavailable") + def test_replacement_mode_preserves_raw_subject_rgb_gradient(self) -> None: + module = import_scail_nodes_with_real_torch() + vae = RecordingVAE() + + module.WanVideoAddSCAIL2ConditionEmbeds().add( + { + "target_shape": (16, 2, 1, 1), + "num_frames": 5, + }, + real_condition_video_structure_payload(), + vae, + ) + + encoded_pose_input = vae.encode_calls[1]["image"] + subject_column_left = encoded_pose_input[:, :, :, 0] + subject_column_right = encoded_pose_input[:, :, :, 1] + self.assertLess(float(subject_column_left.max().item()), -0.85) + self.assertGreater(float(subject_column_right.min().item()), 0.85) + + @unittest.skipUnless(importlib.util.find_spec("torch"), "torch is unavailable") + def test_animation_mode_does_not_sanitize_condition_video(self) -> None: + module = import_scail_nodes_with_real_torch() + vae = RecordingVAE() + + module.WanVideoAddSCAIL2ConditionEmbeds().add( + { + "target_shape": (16, 2, 1, 1), + "num_frames": 5, + }, + real_condition_video_leak_payload(replace_flag=False), + vae, + ) + + encoded_pose_input = vae.encode_calls[1]["image"] + self.assertLess(float(encoded_pose_input[:, :, :, :2].min().item()), -0.9) + + def test_rejects_invalid_payload(self) -> None: + module = import_scail_nodes() + bad_payload = payload() + bad_payload["schema"] = {"name": "wrong", "version": 1, "native_wrapper": {"embeds_key": "scail2_embeds"}} + + with self.assertRaisesRegex(ValueError, "schema"): + module.WanVideoAddSCAIL2ConditionEmbeds().add( + {"target_shape": (16, 2, 1, 1)}, + bad_payload, + FakeVAE(), + ) + + def test_rejects_existing_scail_embeds(self) -> None: + module = import_scail_nodes() + + with self.assertRaisesRegex(ValueError, "v1 scail_embeds"): + module.WanVideoAddSCAIL2ConditionEmbeds().add( + {"target_shape": (16, 2, 1, 1), "scail_embeds": {}}, + payload(), + FakeVAE(), + ) + + def test_rejects_target_shape_mismatch(self) -> None: + module = import_scail_nodes() + + with self.assertRaisesRegex(ValueError, "dimensions"): + module.WanVideoAddSCAIL2ConditionEmbeds().add( + {"target_shape": (16, 2, 1, 2)}, + payload(), + FakeVAE(), + ) + + def test_scail2_example_workflow_contains_strength_widgets(self) -> None: + path = ( + ROOT + / "example_workflows" + / "wanvideo_2_1_14B_SCAIL2_replacement_and_animate_dual_mode_example_01.json" + ) + workflow = json.loads(path.read_text(encoding="utf-8")) + nodes = [ + node + for node in workflow["nodes"] + if node.get("type") == "WanVideoAddSCAIL2ConditionEmbeds" + ] + + self.assertTrue(nodes) + for node in nodes: + self.assertEqual(4, len(node["widgets_values"])) + for value in node["widgets_values"]: + self.assertIsInstance(value, (int, float)) + self.assertGreaterEqual(value, 0.0) + self.assertLessEqual(value, 10.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scail2_example_workflows.py b/tests/test_scail2_example_workflows.py new file mode 100644 index 00000000..ef93eb5a --- /dev/null +++ b/tests/test_scail2_example_workflows.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +README = ROOT / "readme.md" +EXAMPLE = ( + ROOT + / "example_workflows" + / "wanvideo_2_1_14B_SCAIL2_replacement_and_animate_dual_mode_example_01.json" +) + + +def _load_example() -> dict: + return json.loads(EXAMPLE.read_text(encoding="utf-8")) + + +class Scail2ExampleWorkflowTests(unittest.TestCase): + def test_readme_documents_scail_pose2_dual_mode_samples_behavior(self) -> None: + source = README.read_text(encoding="utf-8") + + self.assertIn("SCAIL-Pose2 dual-mode samples", source) + self.assertIn("`replacement` keeps", source) + self.assertIn("`animation`", source) + self.assertIn("`WanVideoEncode.samples`", source) + self.assertIn("`WanVideoSampler.samples`", source) + self.assertIn("`add_noise_to_samples` only matters", source) + self.assertNotIn(".planning/", source) + self.assertNotIn("reference/docs", source) + + def test_replacement_example_uses_raw_driving_video_condition_route(self) -> None: + data = _load_example() + nodes = {node["id"]: node for node in data["nodes"]} + links = {link[0]: link for link in data["links"]} + + self.assertNotIn( + "SCAILPose2ReplacementConditionVideo", + {node["type"] for node in data["nodes"]}, + ) + + condition = nodes[447] + driving_input = next( + item for item in condition["inputs"] if item["name"] == "driving_video" + ) + driving_link = links[driving_input["link"]] + + self.assertEqual([856, 478, 0, 447, 4, "IMAGE"], driving_link) + self.assertEqual("Get_driving_video", nodes[478]["title"]) + + def test_replacement_example_wires_denoise_mask_to_encode_samples_path(self) -> None: + data = _load_example() + nodes = {node["id"]: node for node in data["nodes"]} + links = {link[0]: link for link in data["links"]} + + mask_node = nodes[454] + mask_output = next(item for item in mask_node["outputs"] if item["name"] == "mask") + mask_link = links[mask_output["links"][0]] + + self.assertEqual("SCAILPose2ReplacementDenoiseMask", mask_node["type"]) + self.assertEqual([836, 454, 0, 475, 2, "MASK"], mask_link) + self.assertEqual("WanVideoEncode", nodes[475]["type"]) + + def test_replacement_example_keeps_encode_samples_connected_to_sampler(self) -> None: + data = _load_example() + nodes = {node["id"]: node for node in data["nodes"]} + links = {link[0]: link for link in data["links"]} + + condition = nodes[447] + encode = nodes[475] + sampler = nodes[348] + encode_samples_output = next( + item for item in encode["outputs"] if item["name"] == "samples" + ) + sampler_samples_input = next( + item for item in sampler["inputs"] if item["name"] == "samples" + ) + samples_link = links[encode_samples_output["links"][0]] + + self.assertEqual("replacement", condition["widgets_values"][0]) + self.assertEqual("WanVideoEncode", encode["type"]) + self.assertEqual("WanVideoSamplerv2", sampler["type"]) + self.assertEqual(sampler_samples_input["link"], encode_samples_output["links"][0]) + self.assertEqual([834, 475, 0, 348, 4, "LATENT"], samples_link) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scail2_forward.py b/tests/test_scail2_forward.py new file mode 100644 index 00000000..d474b3f8 --- /dev/null +++ b/tests/test_scail2_forward.py @@ -0,0 +1,354 @@ +from __future__ import annotations + +import importlib.util +import sys +import unittest +from contextlib import contextmanager +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def import_forward_module(): + spec = importlib.util.spec_from_file_location( + "scail2_forward_under_test", + ROOT / "SCAIL" / "scail2_forward.py", + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FakeTensor: + def __init__(self, shape): + self.shape = tuple(shape) + + +class FakePatchEmbedding: + def __init__(self, in_channels): + self.weight = FakeTensor((8, in_channels, 1, 2, 2)) + + +class FakeLatent: + def __init__(self, shape, *, fill=1.0, parts=(), writes=()): + self.shape = tuple(shape) + self.fill = fill + self.parts = tuple(parts) + self.writes = tuple(writes) + + def new_zeros(self, shape): + return FakeLatent(shape, fill=0.0) + + def new_full(self, shape, fill_value): + return FakeLatent(shape, fill=float(fill_value)) + + def clone(self): + return FakeLatent(self.shape, fill=self.fill, parts=self.parts, writes=self.writes) + + def __setitem__(self, key, value): + self.writes = (*self.writes, (key, value)) + + def __mul__(self, value): + return FakeLatent(self.shape, fill=self.fill * float(value), parts=(self,)) + + +class FakeTorchModule: + @staticmethod + def cat(items, dim=0): + items = tuple(items) + shape = list(items[0].shape) + shape[dim] = sum(item.shape[dim] for item in items) + return FakeLatent(tuple(shape), parts=items) + + +@contextmanager +def fake_torch_module(): + sentinel = object() + previous = sys.modules.get("torch", sentinel) + sys.modules["torch"] = FakeTorchModule() + try: + yield + finally: + if previous is sentinel: + sys.modules.pop("torch", None) + else: + sys.modules["torch"] = previous + + +def plan(scail2_input, video_shape=(16, 2, 4, 4)): + module = import_forward_module() + return module.build_scail2_forward_plan( + scail2_input, + video_shape=video_shape, + patch_size=(1, 2, 2), + ) + + +class SCAIL2ForwardPlanTests(unittest.TestCase): + def test_history_channels_are_appended_for_twenty_channel_patch_embedding(self) -> None: + module = import_forward_module() + embedding = FakePatchEmbedding(20) + latent = FakeLatent((16, 2, 4, 4)) + + with fake_torch_module(): + expanded = module.append_scail2_history_channels( + latent, + patch_embedding=embedding, + ) + + self.assertEqual((20, 2, 4, 4), tuple(expanded.shape)) + self.assertIs(latent, expanded.parts[0]) + self.assertEqual((4, 2, 4, 4), expanded.parts[1].shape) + self.assertEqual(0.0, expanded.parts[1].fill) + + def test_history_channels_can_be_filled_for_reference_or_pose_markers(self) -> None: + module = import_forward_module() + embedding = FakePatchEmbedding(20) + latent = FakeLatent((16, 2, 4, 4)) + + with fake_torch_module(): + expanded = module.append_scail2_history_channels( + latent, + patch_embedding=embedding, + fill_value=1.0, + ) + + self.assertEqual((20, 2, 4, 4), tuple(expanded.shape)) + self.assertEqual((4, 2, 4, 4), expanded.parts[1].shape) + self.assertEqual(1.0, expanded.parts[1].fill) + + def test_reference_prefix_history_channels_are_marked(self) -> None: + module = import_forward_module() + embedding = FakePatchEmbedding(20) + latent = FakeLatent((20, 5, 4, 4)) + + marked = module.mark_scail2_prefix_history_channels( + latent, + prefix_frames=2, + patch_embedding=embedding, + ) + + self.assertIsNot(latent, marked) + self.assertEqual((20, 5, 4, 4), marked.shape) + self.assertEqual(1, len(marked.writes)) + self.assertEqual((slice(-4, None, None), slice(None, 2, None)), marked.writes[0][0]) + self.assertEqual(1.0, marked.writes[0][1]) + + def test_history_channels_are_noop_when_channels_already_match(self) -> None: + module = import_forward_module() + embedding = FakePatchEmbedding(20) + latent = FakeLatent((20, 2, 4, 4)) + + self.assertIs( + latent, + module.append_scail2_history_channels( + latent, + patch_embedding=embedding, + ), + ) + + def test_history_channels_reject_unexpected_channel_gap(self) -> None: + module = import_forward_module() + embedding = FakePatchEmbedding(20) + latent = FakeLatent((15, 2, 4, 4)) + + with self.assertRaisesRegex(ValueError, "expects 20, got 15"): + module.append_scail2_history_channels( + latent, + patch_embedding=embedding, + ) + + def test_mask_only_control_is_not_dropped(self) -> None: + result = plan( + { + "ref_latents": [FakeTensor((16, 1, 4, 4))], + "ref_masks": [FakeTensor((28, 1, 4, 4))], + "driving_masks": [FakeTensor((28, 2, 2, 2))], + } + ) + + self.assertFalse(result["has_pose"]) + self.assertTrue(result["has_control_mask"]) + self.assertGreater(result["control_length"], 0) + self.assertEqual(result["main_length"] + result["control_length"], result["total_length"]) + + def test_pose_only_control_is_planned_without_mask_control(self) -> None: + result = plan( + { + "ref_latents": [FakeTensor((16, 1, 4, 4))], + "ref_masks": [FakeTensor((28, 1, 4, 4))], + "pose_latents": [FakeTensor((16, 2, 2, 2))], + } + ) + + self.assertTrue(result["has_pose"]) + self.assertFalse(result["has_control_mask"]) + self.assertEqual((16, 2, 2, 2), result["control_shape"]) + self.assertGreater(result["control_length"], 0) + + def test_pose_and_mask_share_control_shape(self) -> None: + result = plan( + { + "ref_latents": [FakeTensor((16, 1, 4, 4))], + "ref_masks": [FakeTensor((28, 1, 4, 4))], + "pose_latents": [FakeTensor((16, 2, 2, 2))], + "driving_masks": [FakeTensor((28, 2, 2, 2))], + } + ) + + self.assertTrue(result["has_pose"]) + self.assertTrue(result["has_control_mask"]) + self.assertEqual((16, 2, 2, 2), result["control_shape"]) + + with self.assertRaisesRegex(ValueError, "share temporal/spatial"): + plan( + { + "pose_latents": [FakeTensor((16, 2, 2, 2))], + "driving_masks": [FakeTensor((28, 3, 2, 2))], + } + ) + + def test_replace_flag_changes_rope_shifts_and_cache_key(self) -> None: + animation = plan( + { + "ref_latents": [FakeTensor((16, 1, 4, 4))], + "ref_masks": [FakeTensor((28, 1, 4, 4))], + "pose_latents": [FakeTensor((16, 2, 2, 2))], + "replace_flag": False, + } + ) + replacement = plan( + { + "ref_latents": [FakeTensor((16, 1, 4, 4))], + "ref_masks": [FakeTensor((28, 1, 4, 4))], + "pose_latents": [FakeTensor((16, 2, 2, 2))], + "replace_flag": True, + } + ) + + self.assertNotEqual(animation["cache_key"], replacement["cache_key"]) + self.assertEqual(1, animation["rope_shifts"]["t"]["video"]) + self.assertEqual(0, replacement["rope_shifts"]["t"]["video"]) + self.assertEqual(0.0, animation["rope_shifts"]["h"]["ref"]) + self.assertEqual(120.0, replacement["rope_shifts"]["h"]["ref"]) + + def test_additional_references_affect_order_and_lengths(self) -> None: + result = plan( + { + "additional_ref_latents": [ + FakeTensor((16, 1, 4, 4)), + FakeTensor((16, 1, 4, 4)), + ], + "additional_ref_masks": [ + FakeTensor((28, 1, 4, 4)), + FakeTensor((28, 1, 4, 4)), + ], + "ref_latents": [FakeTensor((16, 1, 4, 4))], + "ref_masks": [FakeTensor((28, 1, 4, 4))], + "driving_masks": [FakeTensor((28, 2, 2, 2))], + } + ) + + self.assertEqual(3, result["prefix_frames"]) + self.assertEqual(2, result["additional_ref_count"]) + self.assertGreater(result["additional_ref_length"], 0) + self.assertEqual(2, result["rope_shifts"]["t"]["ref"]) + self.assertEqual(3, result["rope_shifts"]["t"]["video"]) + + def test_additional_reference_masks_are_required(self) -> None: + with self.assertRaisesRegex(ValueError, "additional_ref_masks is required"): + plan({"additional_ref_latents": [FakeTensor((16, 1, 4, 4))]}) + + with self.assertRaisesRegex(ValueError, "additional_ref_masks requires"): + plan({"additional_ref_masks": [FakeTensor((28, 1, 4, 4))]}) + + def test_scail2_strengths_default_and_partial_override(self) -> None: + module = import_forward_module() + + self.assertEqual( + { + "ref_image": 1.0, + "ref_mask": 1.0, + "condition_video": 1.0, + "driving_mask": 1.0, + }, + module.scail2_strengths({}), + ) + self.assertEqual( + { + "ref_image": 1.5, + "ref_mask": 1.0, + "condition_video": 0.25, + "driving_mask": 1.0, + }, + module.scail2_strengths( + {"strengths": {"ref_image": 1.5, "condition_video": 0.25}} + ), + ) + + def test_scail2_strengths_validate_range_and_shape(self) -> None: + module = import_forward_module() + + with self.assertRaisesRegex(ValueError, "must be a dict"): + module.scail2_strengths({"strengths": 1.0}) + with self.assertRaisesRegex(ValueError, "ref_image"): + module.scail2_strengths({"strengths": {"ref_image": -0.01}}) + with self.assertRaisesRegex(ValueError, "driving_mask"): + module.scail2_strengths({"strengths": {"driving_mask": 10.01}}) + with self.assertRaisesRegex(ValueError, "ref_mask"): + module.scail2_strengths({"strengths": {"ref_mask": True}}) + + def test_scale_scail2_items_preserves_default_and_scales_non_default(self) -> None: + module = import_forward_module() + latent = FakeLatent((16, 1, 4, 4), fill=2.0) + + self.assertIs(module.scale_scail2_items([latent], 1.0)[0], latent) + + scaled = module.scale_scail2_items([latent], 1.5)[0] + self.assertIsNot(scaled, latent) + self.assertEqual(3.0, scaled.fill) + self.assertIs(scaled.parts[0], latent) + + def test_model_keeps_v1_and_v2_forward_branches_distinct(self) -> None: + model_source = (ROOT / "wanvideo" / "modules" / "model.py").read_text(encoding="utf-8") + + self.assertIn("scail_input=None, # SCAIL pose", model_source) + self.assertIn("scail2_input=None, # SCAIL-2 native pose/mask conditioning", model_source) + self.assertIn("# SCAIL pose", model_source) + self.assertIn("if scail_input is not None:", model_source) + self.assertIn("if scail2_input is not None:", model_source) + self.assertIn("append_scail2_history_channels", model_source) + self.assertIn("mark_scail2_prefix_history_channels", model_source) + self.assertIn("scail2_strengths", model_source) + self.assertIn("scale_scail2_items", model_source) + + def test_model_pads_scail2_pose_latents_before_pose_patch_embedding(self) -> None: + model_source = (ROOT / "wanvideo" / "modules" / "model.py").read_text(encoding="utf-8") + + self.assertIn("append_scail2_history_channels(\n scail2_pose_latents", model_source) + self.assertIn("patch_embedding=self.patch_embedding_pose", model_source) + self.assertIn('name="SCAIL-2 pose latent"', model_source) + self.assertIn("fill_value=1.0", model_source) + + def test_model_marks_scail2_reference_prefix_before_patch_embedding(self) -> None: + model_source = (ROOT / "wanvideo" / "modules" / "model.py").read_text(encoding="utf-8") + + self.assertIn("mark_scail2_prefix_history_channels(", model_source) + self.assertIn('prefix_frames=scail2_plan["prefix_frames"]', model_source) + self.assertIn('name="SCAIL-2 main latent"', model_source) + + def test_model_applies_scail2_strengths_to_native_groups(self) -> None: + model_source = (ROOT / "wanvideo" / "modules" / "model.py").read_text(encoding="utf-8") + + self.assertIn("strengths = scail2_strengths(scail2_input)", model_source) + self.assertIn('scale_scail2_items(ref_latents, strengths["ref_image"])', model_source) + self.assertIn('strengths["ref_mask"]', model_source) + self.assertIn('strengths["condition_video"]', model_source) + self.assertIn('strengths["driving_mask"]', model_source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scail2_loader.py b/tests/test_scail2_loader.py new file mode 100644 index 00000000..577043c0 --- /dev/null +++ b/tests/test_scail2_loader.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import importlib.util +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def import_loader_module(): + spec = importlib.util.spec_from_file_location( + "scail2_loader_under_test", + ROOT / "SCAIL" / "scail2_loader.py", + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Weight: + def __init__(self, shape): + self.shape = tuple(shape) + + +class FakeConv3d: + def __init__(self, in_channels, out_channels, kernel_size, stride): + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = tuple(kernel_size) + self.stride = tuple(stride) + + +class FakeNN: + Conv3d = FakeConv3d + + +class FakeLog: + def __init__(self) -> None: + self.messages = [] + + def info(self, message) -> None: + self.messages.append(str(message)) + + +class Transformer: + pass + + +class SCAIL2LoaderTests(unittest.TestCase): + def test_v1_pose_embedding_still_patches_without_scail2_mask(self) -> None: + module = import_loader_module() + transformer = Transformer() + + module.apply_scail_loader_patches( + transformer, + {"patch_embedding_pose.weight": Weight((5120, 16, 1, 2, 2))}, + FakeNN, + dim=5120, + patch_size=(1, 2, 2), + log=FakeLog(), + ) + + self.assertEqual(16, transformer.patch_embedding_pose.in_channels) + self.assertEqual(5120, transformer.patch_embedding_pose.out_channels) + self.assertFalse(transformer.scail2_enabled) + self.assertIsNone(transformer.scail2_mask_dim) + + def test_scail2_mask_embedding_patches_with_28_channels(self) -> None: + module = import_loader_module() + transformer = Transformer() + + module.apply_scail_loader_patches( + transformer, + { + "patch_embedding_pose.weight": Weight((5120, 16, 1, 2, 2)), + "patch_embedding_mask.weight": Weight((5120, 28, 1, 2, 2)), + }, + FakeNN, + dim=5120, + patch_size=(1, 2, 2), + log=FakeLog(), + ) + + self.assertEqual(28, transformer.patch_embedding_mask.in_channels) + self.assertEqual(5120, transformer.patch_embedding_mask.out_channels) + self.assertTrue(transformer.scail2_enabled) + self.assertEqual(28, transformer.scail2_mask_dim) + + def test_non_scail_model_sets_negative_capability_metadata(self) -> None: + module = import_loader_module() + transformer = Transformer() + + module.apply_scail_loader_patches( + transformer, + {}, + FakeNN, + dim=5120, + patch_size=(1, 2, 2), + log=FakeLog(), + ) + + self.assertFalse(transformer.scail2_enabled) + self.assertIsNone(transformer.scail2_mask_dim) + self.assertFalse(hasattr(transformer, "patch_embedding_pose")) + self.assertFalse(hasattr(transformer, "patch_embedding_mask")) + + def test_rejects_mask_embedding_with_wrong_channel_count(self) -> None: + module = import_loader_module() + + with self.assertRaisesRegex(ValueError, "28 input channels"): + module.apply_scail_loader_patches( + Transformer(), + {"patch_embedding_mask.weight": Weight((5120, 27, 1, 2, 2))}, + FakeNN, + dim=5120, + patch_size=(1, 2, 2), + log=FakeLog(), + ) + + def test_rejects_inconsistent_mask_embedding_shape(self) -> None: + module = import_loader_module() + + with self.assertRaisesRegex(ValueError, "5D Conv3d"): + module.apply_scail_loader_patches( + Transformer(), + {"patch_embedding_mask.weight": Weight((5120, 28))}, + FakeNN, + dim=5120, + patch_size=(1, 2, 2), + log=FakeLog(), + ) + + with self.assertRaisesRegex(ValueError, "patch_size"): + module.apply_scail_loader_patches( + Transformer(), + {"patch_embedding_mask.weight": Weight((5120, 28, 1, 1, 1))}, + FakeNN, + dim=5120, + patch_size=(1, 2, 2), + log=FakeLog(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scail2_routing.py b/tests/test_scail2_routing.py new file mode 100644 index 00000000..fb5c2059 --- /dev/null +++ b/tests/test_scail2_routing.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import importlib.util +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def import_routing_module(): + spec = importlib.util.spec_from_file_location( + "scail2_routing_under_test", + ROOT / "SCAIL" / "scail2_routing.py", + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FakeTemporalTensor: + def __init__(self, shape, name="tensor"): + self.shape = tuple(shape) + self.name = name + + def __getitem__(self, key): + if not isinstance(key, tuple) or len(key) != 2: + raise TypeError(f"unsupported fake tensor index: {key!r}") + channel_index, frame_index = key + if channel_index != slice(None): + raise TypeError(f"unsupported fake tensor channel index: {key!r}") + if isinstance(frame_index, slice): + frame_count = len(range(*frame_index.indices(self.shape[1]))) + else: + frame_count = len(frame_index) + return FakeTemporalTensor( + (self.shape[0], frame_count, *self.shape[2:]), + name=f"{self.name}_sliced", + ) + + +class SCAIL2RoutingTests(unittest.TestCase): + def test_prepare_returns_none_without_scail2_embeds(self) -> None: + module = import_routing_module() + + self.assertIsNone( + module.prepare_scail2_data( + {"target_shape": (16, 2, 1, 1)}, + dict_to_device=lambda data, device, dtype: data, + device="cuda", + dtype="float16", + ) + ) + + def test_prepare_copies_and_moves_scail2_embeds(self) -> None: + module = import_routing_module() + original = {"pose_latents": object()} + image_embeds = {"scail2_embeds": original} + calls = [] + + def fake_dict_to_device(data, device, dtype): + calls.append((data, device, dtype)) + data["moved"] = True + return data + + result = module.prepare_scail2_data( + image_embeds, + dict_to_device=fake_dict_to_device, + device="cuda:0", + dtype="bfloat16", + ) + + self.assertIsNot(result, original) + self.assertEqual(1, len(calls)) + self.assertEqual("cuda:0", calls[0][1]) + self.assertEqual("bfloat16", calls[0][2]) + self.assertNotIn("moved", original) + self.assertIs(result["pose_latents"], original["pose_latents"]) + + def test_prepare_rejects_v1_and_v2_together(self) -> None: + module = import_routing_module() + + with self.assertRaisesRegex(ValueError, "v1 scail_embeds"): + module.prepare_scail2_data( + {"scail_embeds": {}, "scail2_embeds": {}}, + dict_to_device=lambda data, device, dtype: data, + device="cuda", + dtype="float16", + ) + + def test_prepare_rejects_non_dict_scail2_embeds(self) -> None: + module = import_routing_module() + + with self.assertRaisesRegex(TypeError, "must be a dict"): + module.prepare_scail2_data( + {"scail2_embeds": object()}, + dict_to_device=lambda data, device, dtype: data, + device="cuda", + dtype="float16", + ) + + def test_context_window_none_preserves_scail2_native_payload(self) -> None: + module = import_routing_module() + scail2_data = {"pose_latents": object()} + + self.assertIs(scail2_data, module.scail2_context_window_input(scail2_data, None)) + + def test_context_window_slices_temporal_controls_only(self) -> None: + module = import_routing_module() + ref_latents = [FakeTemporalTensor((16, 1, 2, 2), name="ref_latents")] + ref_masks = [FakeTemporalTensor((28, 1, 2, 2), name="ref_masks")] + additional_ref_latents = [ + FakeTemporalTensor((16, 1, 2, 2), name="additional_ref_latents") + ] + additional_ref_masks = [ + FakeTemporalTensor((28, 1, 2, 2), name="additional_ref_masks") + ] + pose_latents = [FakeTemporalTensor((16, 5, 2, 2), name="pose_latents")] + driving_masks = [FakeTemporalTensor((28, 5, 2, 2), name="driving_masks")] + scail2_data = { + "schema": {"version": 1}, + "ref_latents": ref_latents, + "ref_masks": ref_masks, + "additional_ref_latents": additional_ref_latents, + "additional_ref_masks": additional_ref_masks, + "pose_latents": pose_latents, + "driving_masks": driving_masks, + } + + result = module.scail2_context_window_input(scail2_data, [0, 2, 4]) + + self.assertIsNot(result, scail2_data) + self.assertIs(result["schema"], scail2_data["schema"]) + self.assertIs(result["ref_latents"], ref_latents) + self.assertIs(result["ref_masks"], ref_masks) + self.assertIs(result["additional_ref_latents"], additional_ref_latents) + self.assertIs(result["additional_ref_masks"], additional_ref_masks) + self.assertIs(scail2_data["pose_latents"], pose_latents) + self.assertIs(scail2_data["driving_masks"], driving_masks) + self.assertEqual((16, 3, 2, 2), result["pose_latents"][0].shape) + self.assertEqual((28, 3, 2, 2), result["driving_masks"][0].shape) + self.assertEqual((16, 5, 2, 2), scail2_data["pose_latents"][0].shape) + self.assertEqual((28, 5, 2, 2), scail2_data["driving_masks"][0].shape) + + def test_context_window_accepts_slice_objects(self) -> None: + module = import_routing_module() + scail2_data = { + "pose_latents": [FakeTemporalTensor((16, 5, 2, 2))], + "driving_masks": [FakeTemporalTensor((28, 5, 2, 2))], + } + + result = module.scail2_context_window_input(scail2_data, slice(1, 5, 2)) + + self.assertEqual((16, 2, 2, 2), result["pose_latents"][0].shape) + self.assertEqual((28, 2, 2, 2), result["driving_masks"][0].shape) + + def test_context_window_rejects_pose_mask_frame_count_mismatch(self) -> None: + module = import_routing_module() + scail2_data = { + "pose_latents": [FakeTemporalTensor((16, 5, 2, 2), name="pose_latents")], + "driving_masks": [FakeTemporalTensor((28, 4, 2, 2), name="driving_masks")], + } + + with self.assertRaisesRegex(ValueError, "pose_latents frames=.*driving_masks"): + module.scail2_context_window_input(scail2_data, [0, 1, 2]) + + def test_context_window_rejects_out_of_range_indices_before_slicing(self) -> None: + module = import_routing_module() + scail2_data = { + "pose_latents": [FakeTemporalTensor((16, 5, 2, 2), name="pose_latents")], + "driving_masks": [FakeTemporalTensor((28, 5, 2, 2), name="driving_masks")], + } + + with self.assertRaisesRegex(ValueError, "field=pose_latents.*frame_count=5"): + module.scail2_context_window_input(scail2_data, [0, 4, 5]) + + def test_context_window_rejects_scalar_window(self) -> None: + module = import_routing_module() + scail2_data = {"pose_latents": [FakeTemporalTensor((16, 5, 2, 2))]} + + with self.assertRaisesRegex(TypeError, "sequence of frame indices"): + module.scail2_context_window_input(scail2_data, 0) + + def test_model_params_receive_scail2_input_only_when_present(self) -> None: + module = import_routing_module() + base_params = {"scail_input": None} + scail2_data = {"pose_latents": object()} + + result = module.add_scail2_model_param(base_params, scail2_data) + + self.assertIs(result, base_params) + self.assertIs(result["scail2_input"], scail2_data) + + no_scail2 = {"scail_input": None} + module.add_scail2_model_param(no_scail2, None) + self.assertNotIn("scail2_input", no_scail2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scail_pose2_mask_contract.py b/tests/test_scail_pose2_mask_contract.py new file mode 100644 index 00000000..efb6db00 --- /dev/null +++ b/tests/test_scail_pose2_mask_contract.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import importlib.util +import unittest + +from scail_pose2_mask_contract import ( + SCAIL_POSE2_CONDITION_MODE_ATTR, + SCAIL_POSE2_DISABLE_SAMPLES_ATTR, + SCAIL_POSE2_DISABLE_SAMPLES_REASON_ATTR, + SCAIL_POSE2_MASK_ROLE_ATTR, + SCAIL_POSE2_REPLACEMENT_DENOISE_MASK_ROLE, + build_disabled_samples_payload, + is_scail_pose2_replacement_noise_mask, + normalize_samples_payload_for_sampler, + resize_noise_mask_for_latents, + samples_payload_is_disabled, + scail_pose2_mask_disables_samples, +) + + +class _DummyMask: + pass + + +class ScailPose2SamplesDisableContractTests(unittest.TestCase): + def test_non_replacement_mask_metadata_builds_disabled_samples_payload(self) -> None: + mask = _DummyMask() + setattr(mask, SCAIL_POSE2_DISABLE_SAMPLES_ATTR, True) + setattr(mask, SCAIL_POSE2_DISABLE_SAMPLES_REASON_ATTR, "non_replacement_mode") + setattr(mask, SCAIL_POSE2_CONDITION_MODE_ATTR, "animation") + + payload = build_disabled_samples_payload(mask) + + self.assertTrue(scail_pose2_mask_disables_samples(mask)) + self.assertTrue(samples_payload_is_disabled(payload)) + self.assertIsNone(payload["samples"]) + self.assertIsNone(payload["noise_mask"]) + self.assertTrue(payload["scail_pose2_samples_disabled"]) + self.assertEqual("animation", payload["scail_pose2_condition_mode"]) + self.assertEqual("non_replacement_mode", payload["scail_pose2_disable_reason"]) + + def test_replacement_and_generic_payloads_are_not_disabled(self) -> None: + replacement_mask = _DummyMask() + setattr(replacement_mask, SCAIL_POSE2_CONDITION_MODE_ATTR, "replacement") + setattr( + replacement_mask, + SCAIL_POSE2_MASK_ROLE_ATTR, + SCAIL_POSE2_REPLACEMENT_DENOISE_MASK_ROLE, + ) + generic_mask = _DummyMask() + + self.assertFalse(scail_pose2_mask_disables_samples(replacement_mask)) + self.assertFalse(scail_pose2_mask_disables_samples(generic_mask)) + self.assertFalse(samples_payload_is_disabled({"samples": object()})) + self.assertFalse(samples_payload_is_disabled(None)) + + def test_sampler_normalizer_ignores_disabled_payload_with_log_context(self) -> None: + mask = _DummyMask() + setattr(mask, SCAIL_POSE2_DISABLE_SAMPLES_ATTR, True) + setattr(mask, SCAIL_POSE2_DISABLE_SAMPLES_REASON_ATTR, "non_replacement_mode") + setattr(mask, SCAIL_POSE2_CONDITION_MODE_ATTR, "animation") + payload = build_disabled_samples_payload(mask) + + normalized, context = normalize_samples_payload_for_sampler(payload) + + self.assertIsNone(normalized) + self.assertIsNotNone(context) + self.assertEqual("animation", context.condition_mode) + self.assertEqual("non_replacement_mode", context.reason) + self.assertIn("condition_mode=animation", context.to_log_fragment()) + self.assertIn("reason=non_replacement_mode", context.to_log_fragment()) + + def test_sampler_normalizer_preserves_normal_payload(self) -> None: + payload = {"samples": object(), "noise_mask": object()} + + normalized, context = normalize_samples_payload_for_sampler(payload) + + self.assertIs(normalized, payload) + self.assertIsNone(context) + + +@unittest.skipUnless(importlib.util.find_spec("torch"), "torch is unavailable") +class ScailPose2MaskContractTests(unittest.TestCase): + def test_tagged_replacement_mask_uses_conservative_binary_latent_conversion(self) -> None: + import torch + + mask = torch.tensor([[[0.0, 1.0], [0.49, 0.51]]], dtype=torch.float32) + setattr(mask, SCAIL_POSE2_CONDITION_MODE_ATTR, "replacement") + setattr(mask, SCAIL_POSE2_MASK_ROLE_ATTR, SCAIL_POSE2_REPLACEMENT_DENOISE_MASK_ROLE) + + latent_mask, contract = resize_noise_mask_for_latents( + mask, + latent_shape=(1, 2, 2), + channel_count=3, + latent_grow_pixels=0, + ) + + self.assertTrue(is_scail_pose2_replacement_noise_mask(mask)) + self.assertEqual((1, 3, 1, 2, 2), tuple(latent_mask.shape)) + self.assertEqual("conservative_area", contract.interpolation_mode) + self.assertTrue(contract.scail_pose2_replacement) + self.assertEqual(0.0, float(latent_mask[0, 0, 0, 0, 0].item())) + self.assertEqual(1.0, float(latent_mask[0, 0, 0, 0, 1].item())) + self.assertEqual(0.0, float(latent_mask[0, 0, 0, 1, 0].item())) + self.assertEqual(1.0, float(latent_mask[0, 0, 0, 1, 1].item())) + self.assertAlmostEqual(0.5, contract.subject_ratio) + self.assertAlmostEqual(0.5, contract.preserve_ratio) + self.assertEqual(0, contract.latent_grow_pixels) + self.assertAlmostEqual(0.5, contract.pre_grow_subject_ratio) + + def test_thin_replacement_subject_survives_latent_downsampling(self) -> None: + import torch + + mask = torch.zeros((1, 8, 8), dtype=torch.float32) + mask[:, :, 7] = 1.0 + setattr(mask, SCAIL_POSE2_CONDITION_MODE_ATTR, "replacement") + setattr(mask, SCAIL_POSE2_MASK_ROLE_ATTR, SCAIL_POSE2_REPLACEMENT_DENOISE_MASK_ROLE) + + latent_mask, contract = resize_noise_mask_for_latents( + mask, + latent_shape=(1, 1, 1), + channel_count=1, + latent_grow_pixels=0, + ) + + self.assertEqual("conservative_area", contract.interpolation_mode) + self.assertEqual(1.0, float(latent_mask[0, 0, 0, 0, 0].item())) + self.assertAlmostEqual(1.0, contract.subject_ratio) + self.assertAlmostEqual(0.0, contract.preserve_ratio) + + def test_untagged_mask_keeps_trilinear_policy(self) -> None: + import torch + + mask = torch.tensor([[[0.0, 1.0]]], dtype=torch.float32) + + latent_mask, contract = resize_noise_mask_for_latents( + mask, + latent_shape=(2, 1, 2), + channel_count=2, + ) + + self.assertFalse(is_scail_pose2_replacement_noise_mask(mask)) + self.assertEqual((1, 2, 2, 1, 2), tuple(latent_mask.shape)) + self.assertEqual("trilinear", contract.interpolation_mode) + self.assertFalse(contract.scail_pose2_replacement) + + def test_frame_slice_policy_is_reported(self) -> None: + import torch + + mask = torch.ones((4, 2, 2), dtype=torch.float32) + setattr(mask, SCAIL_POSE2_CONDITION_MODE_ATTR, "replacement") + setattr(mask, SCAIL_POSE2_MASK_ROLE_ATTR, SCAIL_POSE2_REPLACEMENT_DENOISE_MASK_ROLE) + + latent_mask, contract = resize_noise_mask_for_latents( + mask, + latent_shape=(2, 1, 1), + channel_count=1, + start_latent=1, + end_latent=3, + ) + + self.assertEqual((1, 1, 2, 1, 1), tuple(latent_mask.shape)) + self.assertEqual("slice_1_3", contract.frame_policy) + + def test_context_slice_resizes_full_source_timeline_before_slice(self) -> None: + import torch + + mask = torch.zeros((8, 2, 2), dtype=torch.float32) + mask[4:] = 1.0 + setattr(mask, SCAIL_POSE2_CONDITION_MODE_ATTR, "replacement") + setattr(mask, SCAIL_POSE2_MASK_ROLE_ATTR, SCAIL_POSE2_REPLACEMENT_DENOISE_MASK_ROLE) + + latent_mask, contract = resize_noise_mask_for_latents( + mask, + latent_shape=(2, 1, 1), + channel_count=1, + start_latent=1, + end_latent=3, + source_latent_frame_count=4, + ) + + self.assertEqual((1, 1, 2, 1, 1), tuple(latent_mask.shape)) + self.assertEqual("resize_full_4_then_slice_1_3", contract.frame_policy) + + def test_replacement_latent_grow_is_spatial_only(self) -> None: + import torch + + mask = torch.zeros((2, 5, 5), dtype=torch.float32) + mask[:, 2, 2] = 1.0 + setattr(mask, SCAIL_POSE2_CONDITION_MODE_ATTR, "replacement") + setattr(mask, SCAIL_POSE2_MASK_ROLE_ATTR, SCAIL_POSE2_REPLACEMENT_DENOISE_MASK_ROLE) + + latent_mask, contract = resize_noise_mask_for_latents( + mask, + latent_shape=(2, 5, 5), + channel_count=1, + latent_grow_pixels=1, + ) + + self.assertEqual((1, 1, 2, 5, 5), tuple(latent_mask.shape)) + self.assertEqual(1, contract.latent_grow_pixels) + self.assertAlmostEqual(1.0 / 25.0, contract.pre_grow_subject_ratio) + self.assertAlmostEqual(9.0 / 25.0, contract.subject_ratio) + self.assertEqual(9.0, float(latent_mask[0, 0, 0].sum().item())) + self.assertEqual(9.0, float(latent_mask[0, 0, 1].sum().item())) + + def test_replacement_temporal_grow_is_bounded_and_reported(self) -> None: + import torch + + mask = torch.zeros((5, 1, 1), dtype=torch.float32) + mask[2, 0, 0] = 1.0 + setattr(mask, SCAIL_POSE2_CONDITION_MODE_ATTR, "replacement") + setattr(mask, SCAIL_POSE2_MASK_ROLE_ATTR, SCAIL_POSE2_REPLACEMENT_DENOISE_MASK_ROLE) + + latent_mask, contract = resize_noise_mask_for_latents( + mask, + latent_shape=(5, 1, 1), + channel_count=1, + latent_grow_pixels=0, + latent_temporal_grow_frames=1, + ) + + self.assertEqual(1, contract.latent_temporal_grow_frames) + self.assertAlmostEqual(1.0 / 5.0, contract.pre_grow_subject_ratio) + self.assertAlmostEqual(3.0 / 5.0, contract.subject_ratio) + self.assertEqual([0.0, 1.0, 1.0, 1.0, 0.0], latent_mask[0, 0, :, 0, 0].tolist()) + self.assertIn("latent_temporal_grow_frames=1", contract.to_log_string()) + + def test_temporal_grow_is_ignored_for_untagged_masks(self) -> None: + import torch + + mask = torch.zeros((5, 1, 1), dtype=torch.float32) + mask[2, 0, 0] = 1.0 + + latent_mask, contract = resize_noise_mask_for_latents( + mask, + latent_shape=(5, 1, 1), + channel_count=1, + latent_grow_pixels=0, + latent_temporal_grow_frames=1, + ) + + self.assertFalse(contract.scail_pose2_replacement) + self.assertEqual(0, contract.latent_temporal_grow_frames) + self.assertEqual([0.0, 0.0, 1.0, 0.0, 0.0], latent_mask[0, 0, :, 0, 0].tolist()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scail_pose2_samples_initialization.py b/tests/test_scail_pose2_samples_initialization.py new file mode 100644 index 00000000..a33b9baa --- /dev/null +++ b/tests/test_scail_pose2_samples_initialization.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import importlib.util +import unittest + +from scail_pose2_mask_contract import ( + SCAIL_POSE2_CONDITION_MODE_ATTR, + SCAIL_POSE2_MASK_ROLE_ATTR, + SCAIL_POSE2_REPLACEMENT_DENOISE_MASK_ROLE, + align_samples_to_latent_window, + apply_samples_to_noise, + resize_noise_mask_for_latents, +) + + +def _tag_replacement_mask(mask): + setattr(mask, SCAIL_POSE2_CONDITION_MODE_ATTR, "replacement") + setattr(mask, SCAIL_POSE2_MASK_ROLE_ATTR, SCAIL_POSE2_REPLACEMENT_DENOISE_MASK_ROLE) + return mask + + +@unittest.skipUnless(importlib.util.find_spec("torch"), "torch is unavailable") +class ScailPose2SamplesInitializationTests(unittest.TestCase): + def test_align_samples_to_latent_window_slices_and_preserves_source_count(self) -> None: + import torch + + samples = torch.arange(6, dtype=torch.float32).view(1, 6, 1, 1) + + sliced, contract = align_samples_to_latent_window( + samples, + target_frame_count=2, + start_latent=2, + end_latent=4, + ) + + self.assertEqual((1, 2, 1, 1), tuple(sliced.shape)) + self.assertEqual([2.0, 3.0], sliced[0, :, 0, 0].tolist()) + self.assertEqual(6, contract.source_latent_frame_count) + self.assertEqual(2, contract.output_frame_count) + self.assertEqual("slice_2_4", contract.frame_policy) + self.assertIn("source_latent_frame_count=6", contract.to_log_string()) + + def test_align_samples_to_latent_window_accepts_already_windowed_samples(self) -> None: + import torch + + samples = torch.arange(2, dtype=torch.float32).view(1, 2, 1, 1) + + aligned, contract = align_samples_to_latent_window( + samples, + target_frame_count=2, + start_latent=2, + end_latent=4, + ) + + self.assertTrue(torch.equal(samples, aligned)) + self.assertEqual(2, contract.source_latent_frame_count) + self.assertEqual("direct_already_windowed_2_4", contract.frame_policy) + + def test_align_samples_to_latent_window_rejects_mismatched_slice(self) -> None: + import torch + + samples = torch.arange(6, dtype=torch.float32).view(1, 6, 1, 1) + + with self.assertRaisesRegex(ValueError, "sample latent window"): + align_samples_to_latent_window( + samples, + target_frame_count=3, + start_latent=2, + end_latent=4, + ) + + def test_context_window_subject_uses_noise_after_sample_and_mask_slicing(self) -> None: + import torch + + samples = torch.arange(6, dtype=torch.float32).view(1, 6, 1, 1) + noise = torch.full((1, 2, 1, 1), 10.0, dtype=torch.float32) + full_mask = _tag_replacement_mask(torch.zeros((12, 1, 1), dtype=torch.float32)) + full_mask[6:8, 0, 0] = 1.0 + + sliced_samples, sample_contract = align_samples_to_latent_window( + samples, + target_frame_count=2, + start_latent=2, + end_latent=4, + ) + latent_mask, mask_contract = resize_noise_mask_for_latents( + full_mask, + latent_shape=(2, 1, 1), + channel_count=1, + start_latent=2, + end_latent=4, + source_latent_frame_count=sample_contract.source_latent_frame_count, + ) + initialized, init_contract = apply_samples_to_noise( + noise, + sliced_samples, + noise_mask=latent_mask, + timestep=torch.tensor([0.0], dtype=torch.float32), + add_noise_to_samples=False, + scail_pose2_replacement=mask_contract.scail_pose2_replacement, + ) + + self.assertEqual("slice_2_4", sample_contract.frame_policy) + self.assertEqual("resize_full_6_then_slice_2_4", mask_contract.frame_policy) + self.assertEqual([0.0, 1.0], latent_mask[0, 0, :, 0, 0].tolist()) + self.assertEqual(2.0, float(initialized[0, 0, 0, 0].item())) + self.assertEqual(10.0, float(initialized[0, 1, 0, 0].item())) + self.assertEqual("random_noise", init_contract.subject_source) + + def test_replacement_subject_keeps_random_noise_when_add_noise_is_enabled(self) -> None: + import torch + + noise = torch.full((2, 1, 2, 2), 10.0, dtype=torch.float32) + input_samples = torch.full_like(noise, 100.0) + mask = _tag_replacement_mask( + torch.tensor([[[1.0, 0.0], [0.0, 0.0]]], dtype=torch.float32) + ) + latent_mask, _ = resize_noise_mask_for_latents( + mask, + latent_shape=(1, 2, 2), + channel_count=2, + latent_grow_pixels=0, + ) + + initialized, contract = apply_samples_to_noise( + noise, + input_samples, + noise_mask=latent_mask, + timestep=torch.tensor([500.0], dtype=torch.float32), + add_noise_to_samples=True, + scail_pose2_replacement=True, + ) + + self.assertTrue(contract.mask_aware) + self.assertTrue(contract.scail_pose2_replacement) + self.assertEqual("random_noise", contract.subject_source) + self.assertEqual("noised_samples", contract.preserve_source) + self.assertIn("subject_source=random_noise", contract.to_log_string()) + self.assertIn("preserve_source=noised_samples", contract.to_log_string()) + self.assertAlmostEqual(0.25, contract.subject_ratio) + self.assertEqual(10.0, float(initialized[0, 0, 0, 0].item())) + self.assertEqual(55.0, float(initialized[0, 0, 0, 1].item())) + self.assertEqual(55.0, float(initialized[1, 0, 1, 1].item())) + + def test_replacement_subject_keeps_random_noise_when_add_noise_is_disabled(self) -> None: + import torch + + noise = torch.full((1, 1, 2, 2), 7.0, dtype=torch.float32) + input_samples = torch.full_like(noise, 99.0) + mask = _tag_replacement_mask( + torch.tensor([[[0.0, 1.0], [0.0, 0.0]]], dtype=torch.float32) + ) + latent_mask, _ = resize_noise_mask_for_latents( + mask, + latent_shape=(1, 2, 2), + channel_count=1, + latent_grow_pixels=0, + ) + + initialized, contract = apply_samples_to_noise( + noise, + input_samples, + noise_mask=latent_mask, + timestep=torch.tensor([0.0], dtype=torch.float32), + add_noise_to_samples=False, + scail_pose2_replacement=True, + ) + + self.assertTrue(contract.mask_aware) + self.assertEqual("random_noise", contract.subject_source) + self.assertEqual("samples", contract.preserve_source) + self.assertEqual(7.0, float(initialized[0, 0, 0, 1].item())) + self.assertEqual(99.0, float(initialized[0, 0, 0, 0].item())) + self.assertEqual(99.0, float(initialized[0, 0, 1, 1].item())) + + def test_generic_masks_keep_existing_full_sample_initialization(self) -> None: + import torch + + noise = torch.full((1, 1, 2, 2), 10.0, dtype=torch.float32) + input_samples = torch.full_like(noise, 100.0) + latent_mask = torch.tensor( + [[[[[1.0, 0.0], [0.0, 0.0]]]]], + dtype=torch.float32, + ) + + initialized, contract = apply_samples_to_noise( + noise, + input_samples, + noise_mask=latent_mask, + timestep=torch.tensor([500.0], dtype=torch.float32), + add_noise_to_samples=True, + scail_pose2_replacement=False, + ) + + self.assertFalse(contract.mask_aware) + self.assertEqual("noised_samples", contract.subject_source) + self.assertEqual("noised_samples", contract.preserve_source) + self.assertEqual(55.0, float(initialized[0, 0, 0, 0].item())) + self.assertEqual(55.0, float(initialized[0, 0, 1, 1].item())) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_wanvideo_encode_socket_names.py b/tests/test_wanvideo_encode_socket_names.py new file mode 100644 index 00000000..88650804 --- /dev/null +++ b/tests/test_wanvideo_encode_socket_names.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +class WanVideoEncodeSocketNameTests(unittest.TestCase): + def test_original_video_socket_is_named_driving_video(self) -> None: + source = (ROOT / "nodes.py").read_text(encoding="utf-8") + start = source.index("class WanVideoEncode:") + end = source.index("NODE_CLASS_MAPPINGS", start) + class_source = source[start:end] + + self.assertIn('"driving_video": ("IMAGE",)', class_source) + self.assertIn("def encode(self, vae, driving_video,", class_source) + self.assertIn("image = driving_video.clone()", class_source) + self.assertNotIn('"pose_video": ("IMAGE",)', class_source) + self.assertNotIn("def encode(self, vae, pose_video,", class_source) + + def test_scail_pose2_animation_mask_disables_samples_payload(self) -> None: + source = (ROOT / "nodes.py").read_text(encoding="utf-8") + start = source.index("class WanVideoEncode:") + end = source.index("NODE_CLASS_MAPPINGS", start) + class_source = source[start:end] + + self.assertIn("scail_pose2_mask_disables_samples(mask)", class_source) + self.assertIn("build_disabled_samples_payload(mask)", class_source) + + def test_scail_pose2_samples_disable_contract_is_centralized(self) -> None: + source = (ROOT / "nodes.py").read_text(encoding="utf-8") + prefix = source[: source.index("class WanVideoEnhanceAVideo:")] + + self.assertIn("build_disabled_samples_payload", prefix) + self.assertIn("scail_pose2_mask_disables_samples", prefix) + self.assertNotIn("SCAIL_POSE2_DISABLE_SAMPLES_ATTR =", prefix) + self.assertNotIn("def scail_pose2_mask_disables_samples", prefix) + + def test_sampler_ignores_disabled_scail_pose2_samples_payload(self) -> None: + source = (ROOT / "nodes_sampler.py").read_text(encoding="utf-8") + + self.assertIn("normalize_samples_payload_for_sampler", source) + self.assertNotIn('samples.get("scail_pose2_samples_disabled", False)', source) + self.assertIn("disabled_samples_context.to_log_fragment()", source) + + def test_sampler_uses_scail_pose2_noise_mask_contract_helper(self) -> None: + source = (ROOT / "nodes_sampler.py").read_text(encoding="utf-8") + + self.assertIn("align_samples_to_latent_window", source) + self.assertIn("resize_noise_mask_for_latents", source) + self.assertIn("apply_samples_to_noise", source) + self.assertGreaterEqual(source.count("apply_samples_to_noise("), 2) + self.assertIn("samples_window_contract.to_log_string()", source) + self.assertIn("source_latent_frame_count=source_latent_frame_count", source) + self.assertIn("latent_grow_pixels=1", source) + self.assertGreaterEqual(source.count("latent_grow_pixels=1"), 2) + self.assertIn("latent_temporal_grow_frames=1", source) + self.assertGreaterEqual(source.count("latent_temporal_grow_frames=1"), 2) + self.assertIn("noise_mask_contract.to_log_string()", source) + self.assertIn("samples_init_contract.to_log_string()", source) + self.assertNotIn("mode='trilinear',\n align_corners=False\n ).repeat(1, noise.shape[0], 1, 1, 1)", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/wanvideo/modules/model.py b/wanvideo/modules/model.py index e2566ec8..18f9ae8d 100644 --- a/wanvideo/modules/model.py +++ b/wanvideo/modules/model.py @@ -23,6 +23,14 @@ from ...multitalk.multitalk import get_attn_map_with_target from ...echoshot.echoshot import rope_apply_z, rope_apply_c, rope_apply_echoshot from ...custom_linear import update_lora_step +from ...SCAIL.scail2_forward import ( + append_scail2_history_channels, + as_scail2_list, + build_scail2_forward_plan, + mark_scail2_prefix_history_channels, + scale_scail2_items, + scail2_strengths, +) from ...MTV.mtv import apply_rotary_emb from comfy.ldm.flux.math import apply_rope1 as apply_rope_comfy1 @@ -2206,7 +2214,8 @@ def wananimate_forward(self, block, x, motion_vec, strength=1.0, motion_masks=No def rope_encode_comfy(self, t, h, w, freq_offset=0, t_start=0, ref_frame_shape=None, pose_frame_shape=None, steps_t=None, steps_h=None, steps_w=None, ntk_alphas=[1,1,1], device=None, dtype=None, - ref_frame_index=10, longcat_num_ref_latents=0, num_memory_frames=3, rope_negative_offset=0): + ref_frame_index=10, longcat_num_ref_latents=0, num_memory_frames=3, rope_negative_offset=0, + scail2_rope_plan=None): patch_size = self.patch_size t_len = ((t + (patch_size[0] // 2)) // patch_size[0]) @@ -2220,6 +2229,118 @@ def rope_encode_comfy(self, t, h, w, freq_offset=0, t_start=0, ref_frame_shape=N if steps_w is None: steps_w = w_len + if scail2_rope_plan is not None: + def patched_steps(size, patch): + return ((size + (patch // 2)) // patch) + + def make_ids(frame_count, h_steps, w_steps, t_shift, h_shift, w_shift): + frame_steps = patched_steps(frame_count, patch_size[0]) + ids = torch.zeros((frame_steps, h_steps, w_steps, 3), device=device, dtype=dtype) + ids[:, :, :, 0] = ids[:, :, :, 0] + torch.linspace( + t_shift + freq_offset, + t_shift + freq_offset + frame_steps - 1, + steps=frame_steps, + device=device, + dtype=dtype, + ).reshape(-1, 1, 1) + ids[:, :, :, 1] = ids[:, :, :, 1] + torch.linspace( + h_shift + freq_offset, + h_shift + freq_offset + h_steps - 1, + steps=h_steps, + device=device, + dtype=dtype, + ).reshape(1, -1, 1) + ids[:, :, :, 2] = ids[:, :, :, 2] + torch.linspace( + w_shift + freq_offset, + w_shift + freq_offset + w_steps - 1, + steps=w_steps, + device=device, + dtype=dtype, + ).reshape(1, 1, -1) + return ids.reshape(1, -1, ids.shape[-1]) + + rope_shifts = scail2_rope_plan["rope_shifts"] + segments = [] + additional_ref_frames = scail2_rope_plan["additional_ref_frames"] + ref_frames = scail2_rope_plan["ref_frames"] + video_frames = scail2_rope_plan["video_frames"] + + if additional_ref_frames > 0: + segments.append(make_ids( + additional_ref_frames, + h_len, + w_len, + rope_shifts["t"]["additional_ref"], + rope_shifts["h"]["additional_ref"], + rope_shifts["w"]["additional_ref"], + )) + if ref_frames > 0: + segments.append(make_ids( + ref_frames, + h_len, + w_len, + rope_shifts["t"]["ref"], + rope_shifts["h"]["ref"], + rope_shifts["w"]["ref"], + )) + if video_frames > 0: + segments.append(make_ids( + video_frames, + h_len, + w_len, + rope_shifts["t"]["video"], + rope_shifts["h"]["video"], + rope_shifts["w"]["video"], + )) + + control_shape = scail2_rope_plan["control_shape"] + control_full_tokens = 0 + control_downscale = False + if control_shape is not None: + control_frames, control_h, control_w = control_shape[-3], control_shape[-2], control_shape[-1] + control_downscale = control_h != h + h_scale = h / control_h + w_scale = w / control_w + h_shift = rope_shifts["h"]["pose"] + ((h_scale - 1) / 2) + w_shift = rope_shifts["w"]["pose"] + ((w_scale - 1) / 2) + control_h_full = patched_steps(control_h * (2 if control_downscale else 1), patch_size[1]) + control_w_full = patched_steps(control_w * (2 if control_downscale else 1), patch_size[2]) + control_f_full = patched_steps(control_frames, patch_size[0]) + control_full_tokens = control_f_full * control_h_full * control_w_full + segments.append(make_ids( + control_frames, + control_h_full, + control_w_full, + rope_shifts["t"]["pose"], + h_shift, + w_shift, + )) + + combined_img_ids = torch.cat(segments, dim=1) + freqs = self.rope_embedder(combined_img_ids, ntk_alphas).movedim(1, 2) + + if control_shape is not None and control_downscale: + control_frames, control_h, control_w = control_shape[-3], control_shape[-2], control_shape[-1] + control_f_full = patched_steps(control_frames, patch_size[0]) + control_h_full = patched_steps(control_h * 2, patch_size[1]) + control_w_full = patched_steps(control_w * 2, patch_size[2]) + control_h_actual = patched_steps(control_h, patch_size[1]) + control_w_actual = patched_steps(control_w, patch_size[2]) + + control_start_idx = freqs.shape[1] - control_full_tokens + main_freqs, control_freqs = freqs[:, :control_start_idx], freqs[:, control_start_idx:] + + B, _, heads, dim, _, _ = control_freqs.shape + control_freqs = control_freqs.reshape(B, control_f_full, control_h_full, control_w_full, heads, dim, 2, 2) + control_freqs = control_freqs.permute(0, 1, 4, 5, 6, 7, 2, 3).reshape(-1, control_h_full, control_w_full) + control_freqs = F.avg_pool2d(control_freqs, kernel_size=2, stride=2) + control_freqs = control_freqs.reshape(B, control_f_full, heads, dim, 2, 2, control_h_actual, control_w_actual) + control_freqs = control_freqs.permute(0, 1, 6, 7, 2, 3, 4, 5).reshape(B, -1, heads, dim, 2, 2) + + freqs = torch.cat([main_freqs, control_freqs], dim=1) + + return freqs + # Main frames position IDs img_ids = torch.zeros((steps_t, steps_h, steps_w, 3), device=device, dtype=dtype) @@ -2346,6 +2467,7 @@ def forward( sdancer_input=None, # SteadyDancer one_to_all_input=None, one_to_all_controlnet_strength=0.0, # One-to-All scail_input=None, # SCAIL pose + scail2_input=None, # SCAIL-2 native pose/mask conditioning dual_control_input=None, # LongVie2 dual controlnet transformer_options={}, rope_negative_offset=0, @@ -2496,6 +2618,66 @@ def forward( prefix_frames = 1 suffix_frames += 1 + scail2_plan = None + scail2_ref_video_mask = None + scail2_pose_latents = None + scail2_driving_masks = None + if scail2_input is not None: + scail2_plan = build_scail2_forward_plan( + scail2_input, + video_shape=tuple(x[0].shape), + patch_size=self.patch_size, + ) + ref_latents = as_scail2_list(scail2_input.get("ref_latents")) + ref_masks = as_scail2_list(scail2_input.get("ref_masks")) + additional_ref_latents = as_scail2_list(scail2_input.get("additional_ref_latents")) + additional_ref_masks = as_scail2_list(scail2_input.get("additional_ref_masks")) + pose_latents = as_scail2_list(scail2_input.get("pose_latents")) + driving_masks = as_scail2_list(scail2_input.get("driving_masks")) + strengths = scail2_strengths(scail2_input) + + ref_latents = scale_scail2_items(ref_latents, strengths["ref_image"]) + additional_ref_latents = scale_scail2_items( + additional_ref_latents, + strengths["ref_image"], + ) + ref_masks = scale_scail2_items(ref_masks, strengths["ref_mask"]) + additional_ref_masks = scale_scail2_items( + additional_ref_masks, + strengths["ref_mask"], + ) + pose_latents = scale_scail2_items( + pose_latents, + strengths["condition_video"], + ) + driving_masks = scale_scail2_items( + driving_masks, + strengths["driving_mask"], + ) + + scail2_pose_latents = torch.cat(pose_latents, dim=1) if pose_latents else None + scail2_driving_masks = torch.cat(driving_masks, dim=1) if driving_masks else None + + ref_prefix_parts = additional_ref_latents + ref_latents + ref_mask_prefix_parts = additional_ref_masks + ref_masks + if ref_prefix_parts: + if not hasattr(self, "patch_embedding_mask"): + raise ValueError("SCAIL-2 scail2_input requires patch_embedding_mask to consume ref_masks") + ref_prefix = torch.cat(ref_prefix_parts, dim=1) + ref_mask_prefix = torch.cat(ref_mask_prefix_parts, dim=1) + video_mask = ref_mask_prefix.new_zeros( + ref_mask_prefix.shape[0], + x[0].shape[1], + x[0].shape[2], + x[0].shape[3], + ) + scail2_ref_video_mask = torch.cat([ref_mask_prefix, video_mask], dim=1) + x = [torch.cat([ref_prefix.to(u.device, dtype=u.dtype), u], dim=1) for u in x] + seq_len += scail2_plan["additional_ref_length"] + scail2_plan["ref_length"] + F += scail2_plan["prefix_frames"] + prefix_frames += scail2_plan["prefix_frames"] + suffix_frames += scail2_plan["prefix_frames"] + #uni3c controlnet if uni3c_data is not None: render_latent = uni3c_data["render_latent"].to(self.base_dtype) @@ -2521,10 +2703,32 @@ def forward( # patch embed if control_lora_enabled: self.expanded_patch_embedding.to(self.main_device) - x = [self.expanded_patch_embedding(u.unsqueeze(0).to(torch.float32)).to(x[0].dtype) for u in x] + patch_embedding = self.expanded_patch_embedding else: self.original_patch_embedding.to(self.main_device) - x = [self.original_patch_embedding(u.unsqueeze(0).to(torch.float32)).to(x[0].dtype) for u in x] + patch_embedding = self.original_patch_embedding + input_dtype = x[0].dtype + if scail2_input is not None: + x = [ + mark_scail2_prefix_history_channels( + append_scail2_history_channels( + u, + patch_embedding=patch_embedding, + name="SCAIL-2 main latent", + ), + prefix_frames=scail2_plan["prefix_frames"], + patch_embedding=patch_embedding, + name="SCAIL-2 main latent", + ) + for u in x + ] + x = [patch_embedding(u.unsqueeze(0).to(torch.float32)).to(input_dtype) for u in x] + + if scail2_ref_video_mask is not None: + scail2_ref_video_mask_emb = self.patch_embedding_mask( + scail2_ref_video_mask.unsqueeze(0).to(torch.float32) + ).to(x[0].dtype) + x = [u + scail2_ref_video_mask_emb.to(u.device, dtype=u.dtype) for u in x] # ovi audio model if self.audio_model is not None: @@ -2597,6 +2801,29 @@ def forward( del scail_x pose_frame_shape = scail_pose_latents.shape + if scail2_input is not None: + scail2_x = None + if scail2_pose_latents is not None: + if not hasattr(self, "patch_embedding_pose"): + raise ValueError("SCAIL-2 scail2_input requires patch_embedding_pose to consume pose_latents") + scail2_pose_latents = append_scail2_history_channels( + scail2_pose_latents, + patch_embedding=self.patch_embedding_pose, + name="SCAIL-2 pose latent", + fill_value=1.0, + ) + scail2_x = self.patch_embedding_pose(scail2_pose_latents.unsqueeze(0).to(torch.float32)).to(x[0].dtype) + if scail2_driving_masks is not None: + if not hasattr(self, "patch_embedding_mask"): + raise ValueError("SCAIL-2 scail2_input requires patch_embedding_mask to consume driving_masks") + scail2_mask_x = self.patch_embedding_mask(scail2_driving_masks.unsqueeze(0).to(torch.float32)).to(x[0].dtype) + scail2_x = scail2_mask_x if scail2_x is None else scail2_x + scail2_mask_x + if scail2_x is not None: + scail2_x = scail2_x.flatten(2).transpose(1, 2) + x = [torch.cat([u, v], dim=1) for u, v in zip(x, [scail2_x])] + seq_len += scail2_x.shape[1] + pose_frame_shape = scail2_plan["control_shape"] + seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.int32) assert seq_lens.max() <= seq_len, f"max seq len {seq_lens.max()} exceeds provided seq_len {seq_len}" @@ -2670,6 +2897,7 @@ def forward( attn_cond is not None, tuple(ref_frame_shape) if ref_frame_shape is not None else None, tuple(pose_frame_shape) if pose_frame_shape is not None else None, + scail2_plan["cache_key"] if scail2_plan is not None else None, self.rope_embedder.k, tuple(ntk_alphas), longcat_num_ref_latents, @@ -2692,6 +2920,7 @@ def forward( longcat_num_ref_latents=longcat_num_ref_latents, rope_negative_offset=rope_negative_offset, num_memory_frames=num_memory_frames, + scail2_rope_plan=scail2_plan, device=x.device, dtype=x.dtype )