diff --git a/packages/studio/src/components/sidebar/AssetContextMenu.tsx b/packages/studio/src/components/sidebar/AssetContextMenu.tsx new file mode 100644 index 0000000000..07b5417a80 --- /dev/null +++ b/packages/studio/src/components/sidebar/AssetContextMenu.tsx @@ -0,0 +1,97 @@ +export function ContextMenu({ + x, + y, + asset, + onClose, + onCopy, + onDelete, + onRename, +}: { + x: number; + y: number; + asset: string; + onClose: () => void; + onCopy: (path: string) => void; + onDelete?: (path: string) => void; + onRename?: (oldPath: string, newPath: string) => void; +}) { + return ( +
{ + e.preventDefault(); + onClose(); + }} + > +
+ + {onRename && ( + + )} + {onDelete && ( + + )} +
+
+ ); +} + +export function DeleteConfirm({ + name, + onConfirm, + onCancel, +}: { + name: string; + onConfirm: () => void; + onCancel: () => void; +}) { + return ( +
+ Delete {name}? +
+ + +
+
+ ); +} diff --git a/packages/studio/src/components/sidebar/AssetsTab.tsx b/packages/studio/src/components/sidebar/AssetsTab.tsx index 4272c543dc..c41d20f282 100644 --- a/packages/studio/src/components/sidebar/AssetsTab.tsx +++ b/packages/studio/src/components/sidebar/AssetsTab.tsx @@ -1,8 +1,19 @@ -import { memo, useState, useCallback, useRef } from "react"; +import { memo, useState, useCallback, useRef, useMemo, useEffect } from "react"; import { VideoFrameThumbnail } from "../ui/VideoFrameThumbnail"; -import { MEDIA_EXT, IMAGE_EXT, VIDEO_EXT, AUDIO_EXT } from "../../utils/mediaTypes"; +import { MEDIA_EXT, IMAGE_EXT, VIDEO_EXT, FONT_EXT } from "../../utils/mediaTypes"; import { TIMELINE_ASSET_MIME } from "../../utils/timelineAssetDrop"; import { copyTextToClipboard } from "../../utils/clipboard"; +import { ContextMenu, DeleteConfirm } from "./AssetContextMenu"; +import { usePlayerStore } from "../../player/store/playerStore"; +import { + type MediaCategory, + getCategory, + getAudioSubtype, + basename, + ext, + CATEGORY_LABELS, + FILTER_ORDER, +} from "./assetHelpers"; interface AssetsTabProps { projectId: string; @@ -12,98 +23,244 @@ interface AssetsTabProps { onRename?: (oldPath: string, newPath: string) => void; } -/** Inline thumbnail content — rendered inside the container div in AssetCard. */ -function AssetThumbnail({ - serveUrl, - name, - isImage, - isVideo, - isAudio, +function AudioRow({ + projectId, + asset, + used, + meta, + onCopy, + isCopied, + onDelete, + onRename, }: { - serveUrl: string; - name: string; - isImage: boolean; - isVideo: boolean; - isAudio: boolean; + projectId: string; + asset: string; + used: boolean; + meta?: { description?: string; duration?: number }; + onCopy: (path: string) => void; + isCopied: boolean; + onDelete?: (path: string) => void; + onRename?: (oldPath: string, newPath: string) => void; }) { + const [contextMenu, setContextMenu] = useState<{ x: number; y: number } | null>(null); + const [confirmDelete, setConfirmDelete] = useState(false); + const [playing, setPlaying] = useState(false); + const [bars, setBars] = useState([]); + const audioRef = useRef(null); + const actxRef = useRef(null); + const analyserRef = useRef(null); + const sourceRef = useRef(null); + const animRef = useRef(0); + const name = basename(asset); + const subtype = getAudioSubtype(asset); + const serveUrl = `/api/projects/${projectId}/preview/${asset}`; + + useEffect(() => { + return () => { + cancelAnimationFrame(animRef.current); + audioRef.current?.pause(); + actxRef.current?.close(); + }; + }, []); + + useEffect(() => { + if (playing) { + const barCount = 24; + const loop = () => { + const analyser = analyserRef.current; + if (!analyser) { + animRef.current = requestAnimationFrame(loop); + return; + } + const data = new Uint8Array(analyser.frequencyBinCount); + analyser.getByteFrequencyData(data); + const step = Math.floor(data.length / barCount); + const next: number[] = []; + for (let i = 0; i < barCount; i++) { + let sum = 0; + for (let j = 0; j < step; j++) sum += data[i * step + j]; + next.push(sum / step / 255); + } + setBars(next); + if (audioRef.current && !audioRef.current.paused) + animRef.current = requestAnimationFrame(loop); + }; + animRef.current = requestAnimationFrame(loop); + } else { + setBars([]); + } + return () => cancelAnimationFrame(animRef.current); + }, [playing]); + + const togglePlay = useCallback(async () => { + if (playing) { + audioRef.current?.pause(); + setPlaying(false); + cancelAnimationFrame(animRef.current); + return; + } + + if (!actxRef.current) { + actxRef.current = new AudioContext(); + analyserRef.current = actxRef.current.createAnalyser(); + analyserRef.current.fftSize = 256; + analyserRef.current.smoothingTimeConstant = 0.7; + } + + if (!audioRef.current) { + const el = new Audio(); + el.onended = () => { + setPlaying(false); + cancelAnimationFrame(animRef.current); + }; + audioRef.current = el; + sourceRef.current = actxRef.current.createMediaElementSource(el); + sourceRef.current.connect(analyserRef.current!); + analyserRef.current!.connect(actxRef.current.destination); + el.src = serveUrl; + } + + if (actxRef.current.state === "suspended") await actxRef.current.resume(); + audioRef.current.currentTime = 0; + await audioRef.current.play(); + setPlaying(true); + }, [serveUrl, playing]); + return ( <> - {isImage && ( - {name} { - (e.target as HTMLImageElement).style.display = "none"; +
onCopy(asset)} + onDragStart={(e) => { + e.dataTransfer.effectAllowed = "copy"; + e.dataTransfer.setData(TIMELINE_ASSET_MIME, JSON.stringify({ path: asset })); + e.dataTransfer.setData("text/plain", asset); + }} + onContextMenu={(e) => { + e.preventDefault(); + setContextMenu({ x: e.clientX, y: e.clientY }); + }} + className={`group w-full text-left px-4 py-1.5 flex items-center gap-2.5 transition-all cursor-pointer ${ + playing + ? "bg-panel-accent/[0.06]" + : isCopied + ? "bg-panel-accent/10" + : "hover:bg-panel-surface-hover" + }`} + > + +
+
+ + {name} + + {!playing && ( + + {meta?.duration ? `${meta.duration}s · ` : ""} + {subtype} + + )} + {used && ( + + in use + + )} +
+ {bars.length > 0 && ( +
+ {bars.map((v, i) => ( +
+ ))} +
+ )}
+
+ + {contextMenu && ( + setContextMenu(null)} + onCopy={onCopy} + onDelete={onDelete} + onRename={onRename} + /> )} - {!isImage && !isVideo && !isAudio && ( -
- - - - -
+ {confirmDelete && ( + { + onDelete?.(asset); + setConfirmDelete(false); + }} + onCancel={() => setConfirmDelete(false)} + /> )} ); } -function AssetCard({ +function ImageCard({ projectId, asset, + used, onCopy, isCopied, onDelete, onRename, + size, }: { projectId: string; asset: string; + used: boolean; onCopy: (path: string) => void; isCopied: boolean; onDelete?: (path: string) => void; onRename?: (oldPath: string, newPath: string) => void; + size: "large" | "small"; }) { - const [hovered, setHovered] = useState(false); const [contextMenu, setContextMenu] = useState<{ x: number; y: number } | null>(null); - const [renaming, setRenaming] = useState(false); - const [renameName, setRenameName] = useState(""); - const [confirmDelete, setConfirmDelete] = useState(false); - const name = asset.split("/").pop() ?? asset; + const [hovered, setHovered] = useState(false); + const name = basename(asset); + const extension = ext(asset); const serveUrl = `/api/projects/${projectId}/preview/${asset}`; const isVideo = VIDEO_EXT.test(asset); + const isImage = IMAGE_EXT.test(asset); + + const thumbW = size === "large" ? "w-full" : "w-[50px]"; + const thumbH = size === "large" ? "h-[100px]" : "h-[32px]"; return ( <> @@ -121,158 +278,103 @@ function AssetCard({ }} onPointerEnter={() => setHovered(true)} onPointerLeave={() => setHovered(false)} - className={`w-full text-left px-2 py-1.5 flex items-center gap-2.5 transition-colors cursor-pointer ${ - isCopied - ? "bg-studio-accent/10 border-l-2 border-studio-accent" - : "border-l-2 border-transparent hover:bg-neutral-800/50" + className={`transition-colors cursor-pointer ${ + size === "large" + ? `px-2.5 py-1 ${isCopied ? "bg-studio-accent/10" : "hover:bg-neutral-800/30"}` + : `px-2.5 py-1.5 flex items-center gap-2.5 ${ + isCopied + ? "bg-studio-accent/10 border-l-2 border-studio-accent" + : "border-l-2 border-transparent hover:bg-neutral-800/50" + }` }`} > -
- - {isVideo && hovered && ( -
-
- {renaming ? ( - setRenameName(e.target.value)} - onKeyDown={(e) => { - if (e.key === "Enter") { - e.preventDefault(); - const trimmed = renameName.trim(); - if (trimmed && trimmed !== name) { - const dir = asset.includes("/") - ? asset.slice(0, asset.lastIndexOf("/") + 1) - : ""; - onRename?.(asset, dir + trimmed); - } - setRenaming(false); - } else if (e.key === "Escape") { - setRenaming(false); - } - }} - onBlur={() => { - const trimmed = renameName.trim(); - if (trimmed && trimmed !== name) { - const dir = asset.includes("/") ? asset.slice(0, asset.lastIndexOf("/") + 1) : ""; - onRename?.(asset, dir + trimmed); - } - setRenaming(false); - }} - onClick={(e) => e.stopPropagation()} - className="w-full bg-neutral-800 text-neutral-200 text-[11px] px-1.5 py-0.5 rounded border border-neutral-600 outline-none focus:border-studio-accent" - spellCheck={false} - /> - ) : ( - <> - + {size === "large" ? ( +
+
+ {isImage && ( + {name} { + (e.target as HTMLImageElement).style.display = "none"; + }} + /> + )} + {isVideo && } + {isVideo && hovered && ( +
+
+ {name} - {isCopied ? ( - Copied! - ) : ( - {asset} + {extension} + {used && ( + + in use + )} - - )} -
+
+
+ ) : ( + <> +
+ {isImage && ( + {name} { + (e.target as HTMLImageElement).style.display = "none"; + }} + /> + )} + {!isImage && ( + {extension} + )} +
+
+ + {name} + +
+ {extension} + {used && ( + + in use + + )} +
+
+ + )}
- {/* Context menu */} {contextMenu && ( -
setContextMenu(null)} - onContextMenu={(e) => { - e.preventDefault(); - setContextMenu(null); - }} - > -
- - {onRename && ( - - )} - {onDelete && ( - - )} -
-
- )} - - {/* Delete confirmation */} - {confirmDelete && ( -
- Delete {name}? -
- - -
-
+ setContextMenu(null)} + onCopy={onCopy} + onDelete={onDelete} + onRename={onRename} + /> )} ); @@ -288,6 +390,33 @@ export const AssetsTab = memo(function AssetsTab({ const fileInputRef = useRef(null); const [dragOver, setDragOver] = useState(false); const [copiedPath, setCopiedPath] = useState(null); + const [activeFilter, setActiveFilter] = useState("all"); + const [searchQuery, setSearchQuery] = useState(""); + const [manifest, setManifest] = useState< + Map + >(new Map()); + + useEffect(() => { + fetch(`/api/projects/${projectId}/preview/.media/manifest.jsonl`) + .then((r) => (r.ok ? r.text() : "")) + .then((text) => { + const m = new Map< + string, + { description?: string; duration?: number; width?: number; height?: number } + >(); + for (const line of text.split("\n")) { + if (!line.trim()) continue; + try { + const rec = JSON.parse(line); + if (rec.path) m.set(rec.path, rec); + } catch { + /* skip */ + } + } + setManifest(m); + }) + .catch(() => {}); + }, [projectId, assets]); const handleDrop = useCallback( (e: React.DragEvent) => { @@ -306,7 +435,56 @@ export const AssetsTab = memo(function AssetsTab({ } }, []); - const mediaAssets = assets.filter((a) => MEDIA_EXT.test(a)); + const elements = usePlayerStore((s) => s.elements); + const usedPaths = useMemo(() => { + const paths = new Set(); + for (const el of elements) { + if (el.src) { + const src = el.src.replace(/^\/api\/projects\/[^/]+\/preview\//, ""); + paths.add(src); + } + } + return paths; + }, [elements]); + + const mediaAssets = useMemo(() => { + const all = assets.filter((a) => MEDIA_EXT.test(a) || FONT_EXT.test(a)); + if (!searchQuery) return all; + const q = searchQuery.toLowerCase(); + return all.filter((a) => { + if (basename(a).toLowerCase().includes(q)) return true; + const rec = manifest.get(a); + return rec?.description?.toLowerCase().includes(q); + }); + }, [assets, searchQuery, manifest]); + + const categorized = useMemo(() => { + const groups: Record = { audio: [], images: [], video: [], fonts: [] }; + for (const a of mediaAssets) { + const cat = getCategory(a); + if (cat) groups[cat].push(a); + } + // Sort: used assets first within each category + for (const cat of FILTER_ORDER) { + groups[cat].sort((a, b) => { + const aUsed = usedPaths.has(a) ? 0 : 1; + const bUsed = usedPaths.has(b) ? 0 : 1; + return aUsed - bUsed; + }); + } + return groups; + }, [mediaAssets, usedPaths]); + + const counts = useMemo(() => { + const c: Record = { all: mediaAssets.length }; + for (const cat of FILTER_ORDER) c[cat] = categorized[cat].length; + return c; + }, [mediaAssets, categorized]); + + const visibleCategories = + activeFilter === "all" + ? FILTER_ORDER.filter((c) => categorized[c].length > 0) + : [activeFilter as MediaCategory].filter((c) => categorized[c].length > 0); return (
setDragOver(false)} onDrop={handleDrop} > - {/* Import button */} - {onImport && ( -
- - { - if (e.target.files?.length) { - onImport(e.target.files); - e.target.value = ""; - } - }} - /> -
- )} + setSearchQuery(e.target.value)} + placeholder="Search assets..." + className="min-w-0 w-full bg-transparent text-[11px] text-panel-text-1 outline-none placeholder:text-panel-text-5" + /> +
+ )} + + {/* Filter chips — panel-input style */} + {mediaAssets.length > 0 && ( +
+ + {FILTER_ORDER.map((cat) => + counts[cat] > 0 ? ( + + ) : null, + )} +
+ )} + {/* Asset list */} -
+
{mediaAssets.length === 0 ? (
Drop media files here

) : ( - mediaAssets.map((asset) => ( - + visibleCategories.map((cat) => ( +
+ {activeFilter === "all" && ( +
+

+ {CATEGORY_LABELS[cat]} +

+ {categorized[cat].length} +
+ )} + {cat === "audio" && + categorized[cat].map((a) => ( + + ))} + {(cat === "images" || cat === "video") && + categorized[cat].map((a) => ( + + ))} + {cat === "fonts" && + categorized[cat].map((a) => ( + + ))} +
)) )}
diff --git a/packages/studio/src/components/sidebar/assetHelpers.ts b/packages/studio/src/components/sidebar/assetHelpers.ts new file mode 100644 index 0000000000..069c47491c --- /dev/null +++ b/packages/studio/src/components/sidebar/assetHelpers.ts @@ -0,0 +1,40 @@ +import { AUDIO_EXT, IMAGE_EXT, VIDEO_EXT, FONT_EXT } from "../../utils/mediaTypes"; + +export type MediaCategory = "audio" | "images" | "video" | "fonts"; + +export function getCategory(path: string): MediaCategory | null { + if (AUDIO_EXT.test(path)) return "audio"; + if (IMAGE_EXT.test(path)) return "images"; + if (VIDEO_EXT.test(path)) return "video"; + if (FONT_EXT.test(path)) return "fonts"; + return null; +} + +export function getAudioSubtype(path: string): string { + const lower = path.toLowerCase(); + if (lower.includes("/bgm/") || lower.includes("/music/")) return "BGM"; + if (lower.includes("/sfx/") || lower.includes("/sound")) return "SFX"; + if (lower.includes("/voice/") || lower.includes("/narrat")) return "Voice"; + return "Audio"; +} + +export function basename(path: string): string { + const name = path.split("/").pop() ?? path; + const dot = name.lastIndexOf("."); + return dot > 0 ? name.slice(0, dot) : name; +} + +export function ext(path: string): string { + const name = path.split("/").pop() ?? path; + const dot = name.lastIndexOf("."); + return dot > 0 ? name.slice(dot + 1).toUpperCase() : ""; +} + +export const CATEGORY_LABELS: Record = { + audio: "Audio", + images: "Images", + video: "Video", + fonts: "Fonts", +}; + +export const FILTER_ORDER: MediaCategory[] = ["audio", "images", "video", "fonts"]; diff --git a/packages/studio/src/hooks/useMusicBeatAnalysis.ts b/packages/studio/src/hooks/useMusicBeatAnalysis.ts index 73a143f700..da533487e5 100644 --- a/packages/studio/src/hooks/useMusicBeatAnalysis.ts +++ b/packages/studio/src/hooks/useMusicBeatAnalysis.ts @@ -92,33 +92,48 @@ export function useMusicBeatAnalysis(): void { return; } let cancelled = false; - - let promise = analysisCache.get(musicSrc); - if (!promise) { - promise = analyzeMusicFromUrl(musicSrc); - cacheAnalysis(musicSrc, promise); - } - const beatPath = beatFilePathForSrc(musicSrc); - promise - .then(async (analysis) => { + const io = ioRef.current; + + // Only run expensive audio decode + beat analysis when the user has an + // explicit beats file saved. Without one, skip entirely — no surprise + // green lines on the timeline after dragging unrelated assets. + (async () => { + if (!beatPath || !io) return; + let hasSavedBeats = false; + try { + const content = await io.readOptionalProjectFile(beatPath); + const parsed = content ? parseBeats(content) : null; + hasSavedBeats = !!(parsed && parsed.times.length > 0); + } catch { + /* no file */ + } + if (cancelled) return; + if (!hasSavedBeats) { + setBeatAnalysis(null); + return; + } + + let promise = analysisCache.get(musicSrc); + if (!promise) { + promise = analyzeMusicFromUrl(musicSrc); + cacheAnalysis(musicSrc, promise); + } + try { + const analysis = await promise; const detected = { times: analysis.beatTimes, strengths: analysis.beatStrengths }; - const io = ioRef.current; - if (!io) return; - const { times, strengths, hasFile } = await resolveBeats(beatPath, detected, io); + const { times, strengths } = await resolveBeats(beatPath, detected, io); if (cancelled) return; setBeatEdits(null); resetBeatHistory(); setBeatAnalysis({ ...analysis, beatTimes: times, beatStrengths: strengths }); - // Seed a missing file through the SAME debounced writer the edits use, so - // the initial write can't race a near-simultaneous edit's persist. - if (beatPath && !hasFile && times.length > 0) usePlayerStore.getState().beatPersist?.(); - }) - .catch(() => { - if (cancelled) return; - setBeatAnalysis(null); - analysisCache.delete(musicSrc); - }); + } catch { + if (!cancelled) { + setBeatAnalysis(null); + analysisCache.delete(musicSrc); + } + } + })(); return () => { cancelled = true; diff --git a/skills/faceless-explainer/SKILL.md b/skills/faceless-explainer/SKILL.md index 1e801fc4fd..ea76de2e9c 100644 --- a/skills/faceless-explainer/SKILL.md +++ b/skills/faceless-explainer/SKILL.md @@ -3,6 +3,8 @@ name: faceless-explainer description: "turn arbitrary text — an article, notes, a topic, a brief — into a faceless explainer video, up to ~3 min (sweet spot 30-90s), where every visual is invented (typography, abstract graphics, diagrams, data-viz) rather than captured. There is no URL, no website capture, and no real assets. Use this skill for topic explainers, concept breakdowns, how-tos, listicles, and narrative explainers. Do not use it for a product launch/promo (use /product-launch-video), a tour of a real website (use /website-to-video), a GitHub PR (use /pr-to-video), captions on existing footage (use /embedded-captions), or a short unnarrated motion graphic (use /motion-graphics). If the intent is unclear, route through /hyperframes first." --- +> **media-use**: Before sourcing audio/images, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog. Run `--adopt` first to register existing assets. See `/media-use` skill. + # Faceless Explainer to HyperFrames Use this skill to turn a body of text into an explainer video: pick a design system, plan a teaching story, and build it frame by frame in HyperFrames. **Faceless** means every visual is invented downstream — there is no capture step and no real asset inventory. diff --git a/skills/general-video/SKILL.md b/skills/general-video/SKILL.md index 6e38f3961b..037aa32108 100644 --- a/skills/general-video/SKILL.md +++ b/skills/general-video/SKILL.md @@ -11,6 +11,8 @@ description: > metadata: { "tags": "orchestrator, general-video, fallback, freeform, composition-authoring" } --- +> **media-use**: Before sourcing audio/images, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog. Run `--adopt` first to register existing assets. See `/media-use` skill. + # general-video — general video workflow > **Confirm the route before you build.** This is the **fallback** for custom composition authoring. If the input clearly fits a specialized workflow, prefer it: marketed product → `/product-launch-video`; general site → `/website-to-video`; topic explainer → `/faceless-explainer`; GitHub PR → `/pr-to-video`; existing footage → `/embedded-captions` · `/graphic-overlays`; short unnarrated motion graphic → `/motion-graphics`; Remotion port → `/remotion-to-hyperframes`. **Out of scope**: live / at-render-time data, NLE-style editing of a finished video, or producing footage HyperFrames can't capture. Unsure? **Read `/hyperframes` first.** diff --git a/skills/hyperframes/SKILL.md b/skills/hyperframes/SKILL.md index 897c1018ab..14b9e0200d 100644 --- a/skills/hyperframes/SKILL.md +++ b/skills/hyperframes/SKILL.md @@ -32,6 +32,7 @@ Atomic capabilities you load **on demand** — not full video workflows. For "ma | **Animate** — atomic motion, scene blueprints, transitions, runtime adapters (GSAP / Lottie / Three.js / Anime.js / CSS / WAAPI / TypeGPU) | `/hyperframes-animation` | | **Creative direction** — `frame.md` / `design.md`, palettes, typography, narration, beat planning, audio-reactive | `/hyperframes-creative` | | **Media** — TTS voiceover, background music, transcription, background removal, captions | `/hyperframes-media` | +| **Media resolve** — find + freeze BGM, SFX, images, icons from HeyGen catalog into `.media/` with manifest tracking | `/media-use` | | **CLI dev loop** — init, lint, validate, inspect, preview, render, publish, doctor | `/hyperframes-cli` | | **Install registry blocks / components** (`hyperframes add`) | `/hyperframes-registry` | diff --git a/skills/media-use/.gitignore b/skills/media-use/.gitignore new file mode 100644 index 0000000000..17df1e2dbc --- /dev/null +++ b/skills/media-use/.gitignore @@ -0,0 +1 @@ +eval-report.html diff --git a/skills/media-use/SKILL.md b/skills/media-use/SKILL.md new file mode 100644 index 0000000000..cbe99b8ea6 --- /dev/null +++ b/skills/media-use/SKILL.md @@ -0,0 +1,124 @@ +--- +name: media-use +description: Agent Media OS — resolve any media need (BGM, SFX, image, icon) into a frozen local file + ledger record. One verb (`resolve`) handles the full cascade: project cache, global cache, HeyGen catalog search, freeze, register. Keeps search noise on disk, hands the agent a path. Use when a composition needs background music, sound effects, images, or icons. +--- + +# media-use + +Resolve media needs into frozen local files. One verb, four types, zero context noise. + +## When to use + +Call `resolve` whenever a composition needs media — background music, sound effects, images, or icons. media-use searches the HeyGen catalog, downloads the best match, freezes it locally, and registers it in a manifest. The agent gets back one line; all search noise stays on disk. + +## Resolve + +```bash +node /scripts/resolve.mjs --type --intent "" --project +``` + +Returns one line: `resolved → (, )` + +### Types + +| Type | What it finds | Provider | +| ------- | ------------------- | ---------------------------------------- | +| `bgm` | Background music | HeyGen audio catalog (10k+ tracks) | +| `sfx` | Sound effects | Bundled 19-file library + HeyGen catalog | +| `image` | Photos, backgrounds | HeyGen asset search (75k+ vectors) | +| `icon` | Icons, logos | HeyGen asset search (type=icon) | + +### Examples + +```bash +# Background music +node /scripts/resolve.mjs --type bgm --intent "upbeat tech launch" --project . +# → resolved bgm_001 → .media/audio/bgm/bgm_001.mp3 (bgm, 25s) + +# Sound effect +node /scripts/resolve.mjs --type sfx --intent "whoosh" --project . +# → resolved sfx_001 → .media/audio/sfx/sfx_001.mp3 (sfx, 0.57s) + +# Image +node /scripts/resolve.mjs --type image --intent "gradient tech background" --project . +# → resolved image_001 → .media/images/image_001.jpg (image) + +# Icon +node /scripts/resolve.mjs --type icon --intent "rocket" --project . +# → resolved icon_001 → .media/images/icon_001.png (icon, transparent) +``` + +### Flags + +| Flag | Description | +| --------------- | ------------------------------------------ | +| `--type, -t` | Media type: bgm, sfx, image, icon | +| `--intent, -i` | What you need (natural language) | +| `--entity, -e` | Entity name for cache matching (optional) | +| `--project, -p` | Project directory (default: .) | +| `--adopt` | Bulk-import existing assets/ into manifest | +| `--json` | Output JSON instead of one-line result | + +## How it works + +1. Check project `.media/manifest.jsonl` for exact-prompt match +2. Scan existing `assets/` directory for unregistered files matching the need +3. Check global cache `~/.media/` for reusable asset +4. Search via provider (HeyGen audio catalog, HeyGen asset search) +5. Freeze file to `.media//`, register in manifest, regenerate `index.md` + +The agent gets back **one line**. Candidates, scores, provenance stay on disk. + +## Adopt existing projects + +Most HyperFrames projects already have assets in `assets/`. media-use adopts them: + +```bash +node /scripts/resolve.mjs --adopt --project . +# → adopted 9 assets from assets/ +# bgm_001 → assets/bgm/mango-fizz.mp3 (bgm, 146.6s) +# image_001 → assets/images/avatar.jpg (image, 400×400) +``` + +`ffprobe` extracts real duration and dimensions. During resolve, unregistered files in `assets/` matching the intent are adopted on the fly. + +## Reading the inventory + +After resolve or adopt, read `.media/index.md` for the full inventory: + +``` +# .media · 4 assets + +id type dur dims path description +bgm_001 bgm 25s — .media/audio/bgm/bgm_001.mp3 upbeat tech launch +sfx_001 sfx 0.6s — .media/audio/sfx/sfx_001.mp3 whoosh +image_001 image — 1920×1080 .media/images/image_001.jpg gradient tech background +icon_001 icon — 200×200 .media/images/icon_001.png rocket +``` + +## Cross-project reuse + +Assets are cached automatically on resolve. Subsequent resolves for the same prompt hit the global cache at `~/.media/` — no re-download, no provider call. Promote an asset explicitly with `organize --promote ` to make it reusable across all projects. + +## Files + +- `.media/manifest.jsonl` — machine SSOT, one JSON record per line +- `.media/index.md` — agent-readable table (id, type, dur, dims, path, description) +- `~/.media/` — global cross-project reuse cache (content-addressed, SHA-256) + +## CLI tools used + +| Tool | Purpose | Required? | +| --------- | ------------------------------------------ | ------------- | +| `ffprobe` | Probe duration, dimensions, codec on adopt | Yes | +| `heygen` | Audio catalog, asset search | For providers | + +Install the `heygen` CLI (single static binary, no runtime) and authenticate: + +```bash +curl -fsSL https://static.heygen.ai/cli/install.sh | bash # installs latest to ~/.local/bin +heygen update # if already installed: needs >= v0.1.6 +export HEYGEN_API_KEY= # or: heygen auth login --key +``` + +Requires **heygen >= v0.1.6** — the providers tag requests with the allowlisted `--headers 'X-HeyGen-Client-Source: media-use'` flag, added in v0.1.6. `asset search` is a pre-launch command hidden from `heygen --help`, but it runs. Without a `heygen` on PATH (or a valid key) the providers print a one-line diagnostic to stderr and resolve falls through to "no provider could resolve". diff --git a/skills/media-use/scripts/lib/adopt.mjs b/skills/media-use/scripts/lib/adopt.mjs new file mode 100644 index 0000000000..773714890a --- /dev/null +++ b/skills/media-use/scripts/lib/adopt.mjs @@ -0,0 +1,112 @@ +import { readdirSync, statSync, existsSync } from "node:fs"; +import { join, extname, basename } from "node:path"; +import { readManifest, appendRecord, nextId } from "./manifest.mjs"; +import { regenerateIndex } from "./index-gen.mjs"; +import { probe } from "./probe.mjs"; + +const AUDIO_EXT = new Set([".mp3", ".wav", ".ogg", ".m4a", ".aac"]); +const IMAGE_EXT = new Set([".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico"]); +const VIDEO_EXT = new Set([".mp4", ".webm", ".mov"]); + +function inferType(filePath) { + const ext = extname(filePath).toLowerCase(); + if (AUDIO_EXT.has(ext)) { + const lower = filePath.toLowerCase(); + if (lower.includes("/bgm/") || lower.includes("/music/") || lower.startsWith("bgm/")) + return "bgm"; + if (lower.includes("/sfx/") || lower.includes("/sound") || lower.startsWith("sfx/")) + return "sfx"; + if (lower.includes("/voice/") || lower.includes("/narrat") || lower.startsWith("voice/")) + return "voice"; + return "bgm"; + } + if (IMAGE_EXT.has(ext)) { + if (ext === ".svg" || ext === ".ico") return "icon"; + return "image"; + } + if (VIDEO_EXT.has(ext)) return "video"; + return null; +} + +function walkDir(dir, base = "") { + const files = []; + if (!existsSync(dir)) return files; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const rel = base ? `${base}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + files.push(...walkDir(join(dir, entry.name), rel)); + } else { + files.push(rel); + } + } + return files; +} + +export function scanExistingAssets(projectDir) { + const assetsDir = join(projectDir, "assets"); + if (!existsSync(assetsDir)) return []; + + const files = walkDir(assetsDir); + const found = []; + for (const rel of files) { + const type = inferType(rel); + if (!type) continue; + const fullPath = join(assetsDir, rel); + const stat = statSync(fullPath); + const meta = probe(fullPath); + found.push({ + relativePath: `assets/${rel}`, + type, + size: stat.size, + name: basename(rel, extname(rel)), + ...meta, + }); + } + return found; +} + +export function adoptExistingAssets(projectDir) { + const existing = scanExistingAssets(projectDir); + if (existing.length === 0) return []; + + const manifest = readManifest(projectDir); + const knownPaths = new Set(manifest.map((r) => r.path)); + + const adopted = []; + for (const asset of existing) { + if (knownPaths.has(asset.relativePath)) continue; + + const id = nextId(projectDir, asset.type); + const record = { + id, + type: asset.type, + path: asset.relativePath, + source: "existing", + description: asset.name.replace(/[-_]/g, " "), + ...(asset.duration != null && { duration: asset.duration }), + ...(asset.width != null && { width: asset.width }), + ...(asset.height != null && { height: asset.height }), + provenance: { provider: "local", adopted: true }, + }; + appendRecord(projectDir, record); + adopted.push(record); + } + + if (adopted.length > 0) regenerateIndex(projectDir); + return adopted; +} + +export function findExistingAsset(projectDir, intent, type) { + const assetsDir = join(projectDir, "assets"); + if (!existsSync(assetsDir)) return null; + const lower = intent.toLowerCase(); + for (const rel of walkDir(assetsDir)) { + const t = inferType(rel); + if (!t || (type && t !== type)) continue; + const name = basename(rel, extname(rel)).toLowerCase().replace(/[-_]/g, " "); + if (name.includes(lower) || lower.includes(name)) { + return { relativePath: `assets/${rel}`, type: t, name: basename(rel, extname(rel)) }; + } + } + return null; +} diff --git a/skills/media-use/scripts/lib/bgm-provider.mjs b/skills/media-use/scripts/lib/bgm-provider.mjs new file mode 100644 index 0000000000..462354d098 --- /dev/null +++ b/skills/media-use/scripts/lib/bgm-provider.mjs @@ -0,0 +1,20 @@ +import { heygenSearch } from "./heygen-search.mjs"; + +export const bgmProvider = { + async search(intent) { + const results = heygenSearch("audio sounds list", intent, { type: "music" }); + if (!results) return null; + const best = results[0]; + return { + url: best.audio_url, + source: "search", + // ext derived from audio_url by resolve.mjs — catalog tracks are .mp3 or .wav + metadata: { + description: best.description || intent, + duration: best.duration || null, + provider: "heygen.audio.sounds", + provenance: { track_id: best.id, score: best.score, query: intent }, + }, + }; + }, +}; diff --git a/skills/media-use/scripts/lib/brand-provider.mjs b/skills/media-use/scripts/lib/brand-provider.mjs new file mode 100644 index 0000000000..e05caf1e3e --- /dev/null +++ b/skills/media-use/scripts/lib/brand-provider.mjs @@ -0,0 +1,59 @@ +import { readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +function findDesignSpec(projectDir) { + for (const name of ["frame.md", "design.md", "DESIGN.md"]) { + const p = join(projectDir, name); + if (existsSync(p)) return { path: p, name }; + } + return null; +} + +function parseFrontmatter(content) { + const match = content.match(/^---\n([\s\S]*?)\n---/); + if (!match) return null; + const yaml = match[1]; + const tokens = {}; + for (const line of yaml.split("\n")) { + const m = line.match(/^\s*(\w[\w-]*):\s*(.+)/); + if (m) tokens[m[1]] = m[2].trim().replace(/^["']|["']$/g, ""); + } + return tokens; +} + +function extractColors(tokens) { + const colors = []; + for (const [k, v] of Object.entries(tokens)) { + if (typeof v === "string" && /^#[0-9a-fA-F]{3,8}$/.test(v)) { + colors.push({ name: k, hex: v }); + } + } + return colors; +} + +export const brandProvider = { + async search(intent, { projectDir } = {}) { + if (!projectDir) return null; + const spec = findDesignSpec(projectDir); + if (!spec) return null; + const content = readFileSync(spec.path, "utf8"); + const tokens = parseFrontmatter(content); + if (!tokens) return null; + const colors = extractColors(tokens); + return { + localPath: spec.path, + source: "local", + ext: ".md", + metadata: { + description: "Brand tokens from " + spec.name, + provider: "design_spec", + provenance: { + file: spec.name, + colors, + font: tokens.font || tokens.typography || null, + logo: tokens.logo || null, + }, + }, + }; + }, +}; diff --git a/skills/media-use/scripts/lib/cache.mjs b/skills/media-use/scripts/lib/cache.mjs new file mode 100644 index 0000000000..e138e2bc01 --- /dev/null +++ b/skills/media-use/scripts/lib/cache.mjs @@ -0,0 +1,114 @@ +import { readFileSync, writeFileSync, mkdirSync, existsSync, copyFileSync } from "node:fs"; +import { join, basename } from "node:path"; +import { createHash } from "node:crypto"; +import { homedir } from "node:os"; +import { readManifest, appendRecord } from "./manifest.mjs"; + +const SCHEMA_PREFIX = "mu-v1-"; +const KEY_HEX_CHARS = 16; +const COMPLETE_SENTINEL = ".hf-complete"; + +export function globalMediaDir() { + return join(homedir(), ".media"); +} + +export function contentHash(filePath) { + const bytes = readFileSync(filePath); + return createHash("sha256").update(bytes).digest("hex"); +} + +function cacheEntryDir(rootDir, sha) { + return join(rootDir, SCHEMA_PREFIX + sha.slice(0, KEY_HEX_CHARS)); +} + +function isComplete(entryDir) { + return existsSync(join(entryDir, COMPLETE_SENTINEL)); +} + +function markComplete(entryDir) { + writeFileSync(join(entryDir, COMPLETE_SENTINEL), "", "utf8"); +} + +function readGlobalManifest() { + return readManifest(globalMediaDir()); +} + +function validateCacheHit(match) { + if (!match?.sha) return null; + return isComplete(cacheEntryDir(globalMediaDir(), match.sha)) ? match : null; +} + +export function cacheGet(prompt, type) { + return validateCacheHit( + readGlobalManifest().find( + (r) => r.reusable && r.provenance?.prompt === prompt && (type == null || r.type === type), + ), + ); +} + +export function cacheGetByEntity(entity) { + const lower = entity.toLowerCase(); + return validateCacheHit( + readGlobalManifest().find((r) => r.reusable && r.entity && r.entity.toLowerCase() === lower), + ); +} + +export function cachePut(filePath, record) { + const sha = contentHash(filePath); + const dir = globalMediaDir(); + const entryDir = cacheEntryDir(dir, sha); + mkdirSync(entryDir, { recursive: true }); + + const dest = join(entryDir, basename(filePath)); + copyFileSync(filePath, dest); + markComplete(entryDir); + + const globalRecord = { + ...record, + sha, + reusable: true, + cached_path: dest, + }; + appendRecord(globalMediaDir(), globalRecord); + return { sha, cached_path: dest }; +} + +export function importFromCache(cacheRecord, projectDir, localId, localPath) { + const sha = cacheRecord.sha; + const entryDir = cacheEntryDir(globalMediaDir(), sha); + if (!isComplete(entryDir)) return null; + + const cachedFile = cacheRecord.cached_path; + if (!cachedFile || !existsSync(cachedFile)) return null; + + mkdirSync(join(projectDir, ".media"), { recursive: true }); + const fullDest = join(projectDir, localPath); + mkdirSync(join(fullDest, ".."), { recursive: true }); + copyFileSync(cachedFile, fullDest); + + const projectRecord = { + ...cacheRecord, + id: localId, + path: localPath, + provenance: { + ...cacheRecord.provenance, + imported_from: sha, + }, + }; + delete projectRecord.sha; + delete projectRecord.reusable; + delete projectRecord.cached_path; + + return projectRecord; +} + +export function promote(projectDir, id) { + const records = readManifest(projectDir); + const record = records.find((r) => r.id === id); + if (!record) throw new Error(`asset not found in project manifest: ${id}`); + + const filePath = join(projectDir, record.path); + if (!existsSync(filePath)) throw new Error(`asset file not found: ${filePath}`); + + return cachePut(filePath, record); +} diff --git a/skills/media-use/scripts/lib/freeze.mjs b/skills/media-use/scripts/lib/freeze.mjs new file mode 100644 index 0000000000..aa4704201d --- /dev/null +++ b/skills/media-use/scripts/lib/freeze.mjs @@ -0,0 +1,26 @@ +import { writeFileSync, copyFileSync, mkdirSync } from "node:fs"; +import { dirname } from "node:path"; + +// ponytail: bound the download so a hostile/runaway URL can't fill the disk. +// 256MB covers any real media asset; raise if 4K video sources ever exceed it. +const MAX_FREEZE_BYTES = 256 * 1024 * 1024; + +export async function freezeUrl(url, destPath) { + const res = await fetch(url); + if (!res.ok) throw new Error(`freeze failed: HTTP ${res.status} for ${String(url).slice(0, 80)}`); + const bytes = Buffer.from(await res.arrayBuffer()); + if (bytes.length === 0) + throw new Error(`freeze failed: empty response for ${String(url).slice(0, 80)}`); + if (bytes.length > MAX_FREEZE_BYTES) + throw new Error( + `freeze failed: ${bytes.length} bytes exceeds ${MAX_FREEZE_BYTES} cap for ${String(url).slice(0, 80)}`, + ); + mkdirSync(dirname(destPath), { recursive: true }); + writeFileSync(destPath, bytes); + return bytes.length; +} + +export function freezeLocalFile(srcPath, destPath) { + mkdirSync(dirname(destPath), { recursive: true }); + copyFileSync(srcPath, destPath); +} diff --git a/skills/media-use/scripts/lib/heygen-search.mjs b/skills/media-use/scripts/lib/heygen-search.mjs new file mode 100644 index 0000000000..60ef2f7499 --- /dev/null +++ b/skills/media-use/scripts/lib/heygen-search.mjs @@ -0,0 +1,45 @@ +import { execSync } from "node:child_process"; + +export function heygenSearch(subcommand, query, { type, limit = 5, minScore } = {}) { + const q = query.replace(/'/g, "'\\''"); + // Tag the caller via the CLI's allowlisted attribution header (heygen >= v0.1.6). + const parts = [ + `heygen --headers 'X-HeyGen-Client-Source: media-use' ${subcommand} --query '${q}'`, + ]; + if (type) parts.push(`--type ${type}`); + parts.push(`--limit ${limit}`); + // Server-side score floor. Honored by `audio sounds list`; the `asset search` + // backend rejects it, so only audio providers pass minScore (see image-provider). + if (minScore != null) parts.push(`--min-score ${minScore}`); + + let out; + try { + out = execSync(parts.join(" "), { + encoding: "utf8", + timeout: 15000, + stdio: ["pipe", "pipe", "pipe"], + }); + } catch (err) { + // Don't swallow a broken command / auth failure as "no results" — that turns + // a typo or expired key into a silent dead end. Surface it, then give up. + const detail = err.stderr?.toString().trim() || err.stdout?.toString().trim() || err.message; + console.error(`media-use: \`heygen ${subcommand}\` failed: ${detail}`); + return null; + } + + let parsed; + try { + parsed = JSON.parse(out); + } catch { + console.error(`media-use: \`heygen ${subcommand}\` returned non-JSON output`); + return null; + } + if (parsed?.error) { + const e = parsed.error; + console.error(`media-use: \`heygen ${subcommand}\` error: ${e.message ?? JSON.stringify(e)}`); + return null; + } + + const data = parsed?.data; + return Array.isArray(data) && data.length > 0 ? data : null; +} diff --git a/skills/media-use/scripts/lib/image-provider.mjs b/skills/media-use/scripts/lib/image-provider.mjs new file mode 100644 index 0000000000..69b26397e7 --- /dev/null +++ b/skills/media-use/scripts/lib/image-provider.mjs @@ -0,0 +1,44 @@ +import { heygenSearch } from "./heygen-search.mjs"; + +export const imageProvider = { + async search(intent) { + const results = heygenSearch("asset search", intent, { type: "image" }); + if (!results) return null; + const best = results[0]; + return { + url: best.url, + source: "search", + // ext derived from the asset URL by resolve.mjs (.jpg/.png/.webp) + metadata: { + description: intent, + width: best.width || null, + height: best.height || null, + transparent: best.is_transparent || false, + provider: "heygen.asset.search", + provenance: { asset_id: best.id, score: best.score }, + }, + }; + }, +}; + +export const iconProvider = { + async search(intent) { + // No minScore: the `asset search` backend rejects --min-score and returns no score field. + const results = heygenSearch("asset search", intent, { type: "icon" }); + if (!results) return null; + const best = results[0]; + return { + url: best.url, + source: "search", + // ext derived from the asset URL by resolve.mjs — catalog icons are .png, not .svg + metadata: { + description: intent, + width: best.width || null, + height: best.height || null, + transparent: best.is_transparent ?? true, + provider: "heygen.asset.search", + provenance: { asset_id: best.id, score: best.score, type: "icon" }, + }, + }; + }, +}; diff --git a/skills/media-use/scripts/lib/index-gen.mjs b/skills/media-use/scripts/lib/index-gen.mjs new file mode 100644 index 0000000000..65b019a92e --- /dev/null +++ b/skills/media-use/scripts/lib/index-gen.mjs @@ -0,0 +1,63 @@ +import { writeFileSync, mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import { readManifest, indexPath } from "./manifest.mjs"; + +function pad(str, len) { + return String(str ?? "").padEnd(len); +} + +function formatDur(record) { + if (record.duration == null) return "—"; + return `${record.duration}s`; +} + +function formatDims(record) { + if (record.width && record.height) return `${record.width}×${record.height}`; + if (record.type === "icon" && record.transparent) return "svg"; + return "—"; +} + +export function generateIndexContent(records) { + const count = records.length; + const header = `# .media · ${count} asset${count === 1 ? "" : "s"}\n`; + if (count === 0) return header; + + const cols = { id: 4, type: 5, dur: 4, dims: 5, path: 5, desc: 11 }; + for (const r of records) { + cols.id = Math.max(cols.id, (r.id ?? "").length); + cols.type = Math.max(cols.type, (r.type ?? "").length); + cols.dur = Math.max(cols.dur, formatDur(r).length); + cols.dims = Math.max(cols.dims, formatDims(r).length); + cols.path = Math.max(cols.path, (r.path ?? "").length); + } + + const heading = + pad("id", cols.id + 2) + + pad("type", cols.type + 2) + + pad("dur", cols.dur + 2) + + pad("dims", cols.dims + 2) + + pad("path", cols.path + 2) + + "description"; + + const lines = [header, heading]; + for (const r of records) { + lines.push( + pad(r.id, cols.id + 2) + + pad(r.type, cols.type + 2) + + pad(formatDur(r), cols.dur + 2) + + pad(formatDims(r), cols.dims + 2) + + pad(r.path, cols.path + 2) + + (r.description ?? ""), + ); + } + return lines.join("\n") + "\n"; +} + +export function regenerateIndex(projectDir) { + const records = readManifest(projectDir); + const content = generateIndexContent(records); + const p = indexPath(projectDir); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, content); + return content; +} diff --git a/skills/media-use/scripts/lib/manifest.mjs b/skills/media-use/scripts/lib/manifest.mjs new file mode 100644 index 0000000000..a32ea181bf --- /dev/null +++ b/skills/media-use/scripts/lib/manifest.mjs @@ -0,0 +1,91 @@ +import { readFileSync, appendFileSync, mkdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +const MANIFEST_FILE = "manifest.jsonl"; +const INDEX_FILE = "index.md"; + +const TYPE_DIRS = { + bgm: "audio/bgm", + sfx: "audio/sfx", + voice: "audio/voice", + image: "images", + icon: "images", + brand: "images", + video: "video", +}; + +export function mediaDir(projectDir) { + return join(projectDir, ".media"); +} + +export function manifestPath(projectDir) { + return join(mediaDir(projectDir), MANIFEST_FILE); +} + +export function indexPath(projectDir) { + return join(mediaDir(projectDir), INDEX_FILE); +} + +export function typeSubdir(type) { + const sub = TYPE_DIRS[type]; + if (!sub) throw new Error(`unknown media type: ${type}`); + return sub; +} + +export function typeDirPath(projectDir, type) { + return join(mediaDir(projectDir), typeSubdir(type)); +} + +export function readManifest(projectDir) { + const p = manifestPath(projectDir); + if (!existsSync(p)) return []; + const raw = readFileSync(p, "utf8"); + const records = []; + for (const line of raw.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed) continue; + try { + records.push(JSON.parse(trimmed)); + } catch { + // ponytail: skip malformed lines, don't crash + } + } + return records; +} + +export function appendRecord(projectDir, record) { + const dir = mediaDir(projectDir); + mkdirSync(dir, { recursive: true }); + const typeDir = typeDirPath(projectDir, record.type); + mkdirSync(typeDir, { recursive: true }); + + const p = manifestPath(projectDir); + const line = JSON.stringify(record) + "\n"; + appendFileSync(p, line); +} + +export function findByPrompt(projectDir, prompt, type) { + const records = readManifest(projectDir); + return ( + records.find((r) => r.provenance?.prompt === prompt && (type == null || r.type === type)) || + null + ); +} + +export function findByEntity(projectDir, entity) { + const lower = entity.toLowerCase(); + const records = readManifest(projectDir); + return records.find((r) => r.entity && r.entity.toLowerCase() === lower) || null; +} + +export function nextId(projectDir, type) { + const records = readManifest(projectDir); + const prefix = type; + let max = 0; + for (const r of records) { + if (r.type !== type) continue; + const m = r.id?.match(new RegExp(`^${prefix}_(\\d+)$`)); + if (m) max = Math.max(max, parseInt(m[1], 10)); + } + return `${prefix}_${String(max + 1).padStart(3, "0")}`; +} diff --git a/skills/media-use/scripts/lib/manifest.test.mjs b/skills/media-use/scripts/lib/manifest.test.mjs new file mode 100644 index 0000000000..227addf309 --- /dev/null +++ b/skills/media-use/scripts/lib/manifest.test.mjs @@ -0,0 +1,293 @@ +import { strict as assert } from "node:assert"; +import { mkdtempSync, rmSync, readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + readManifest, + appendRecord, + findByPrompt, + findByEntity, + nextId, + manifestPath, + mediaDir, + typeDirPath, +} from "./manifest.mjs"; +import { regenerateIndex, generateIndexContent } from "./index-gen.mjs"; +import { + contentHash, + cachePut, + cacheGet, + cacheGetByEntity, + importFromCache, + promote, +} from "./cache.mjs"; + +let tmp; + +function setup() { + tmp = mkdtempSync(join(tmpdir(), "mu-test-")); +} + +function cleanup() { + if (tmp) rmSync(tmp, { recursive: true, force: true }); +} + +function makeRecord(overrides = {}) { + return { + id: "bgm_001", + type: "bgm", + path: ".media/audio/bgm/bgm_001.wav", + source: "search", + description: "soft minimal ambient", + duration: 11, + provenance: { provider: "heygen.audio.sounds", prompt: "subtle tech" }, + ...overrides, + }; +} + +function runTests() { + const tests = []; + + function test(name, fn) { + tests.push({ name, fn }); + } + + // --- manifest.mjs --- + + test("readManifest returns empty array when no manifest exists", () => { + setup(); + const result = readManifest(tmp); + assert.deepStrictEqual(result, []); + cleanup(); + }); + + test("appendRecord writes valid JSONL and readManifest parses it back", () => { + setup(); + const record = makeRecord(); + appendRecord(tmp, record); + const records = readManifest(tmp); + assert.equal(records.length, 1); + assert.deepStrictEqual(records[0], record); + cleanup(); + }); + + test("appendRecord creates .media/ and type subdirs on first write", () => { + setup(); + appendRecord(tmp, makeRecord()); + assert.ok(existsSync(mediaDir(tmp))); + assert.ok(existsSync(typeDirPath(tmp, "bgm"))); + cleanup(); + }); + + test("appendRecord appends multiple records", () => { + setup(); + appendRecord(tmp, makeRecord({ id: "bgm_001" })); + appendRecord(tmp, makeRecord({ id: "bgm_002", provenance: { prompt: "energetic" } })); + const records = readManifest(tmp); + assert.equal(records.length, 2); + assert.equal(records[0].id, "bgm_001"); + assert.equal(records[1].id, "bgm_002"); + cleanup(); + }); + + test("findByPrompt returns exact-match record", () => { + setup(); + appendRecord(tmp, makeRecord()); + const found = findByPrompt(tmp, "subtle tech", "bgm"); + assert.ok(found); + assert.equal(found.id, "bgm_001"); + cleanup(); + }); + + test("findByPrompt returns null on miss", () => { + setup(); + appendRecord(tmp, makeRecord()); + assert.equal(findByPrompt(tmp, "nonexistent", "bgm"), null); + cleanup(); + }); + + test("findByPrompt filters by type", () => { + setup(); + appendRecord(tmp, makeRecord({ type: "sfx" })); + assert.equal(findByPrompt(tmp, "subtle tech", "bgm"), null); + assert.ok(findByPrompt(tmp, "subtle tech", "sfx")); + cleanup(); + }); + + test("findByEntity matches case-insensitively", () => { + setup(); + appendRecord(tmp, makeRecord({ entity: "GitHub", type: "icon" })); + assert.ok(findByEntity(tmp, "github")); + assert.ok(findByEntity(tmp, "GITHUB")); + assert.equal(findByEntity(tmp, "gitlab"), null); + cleanup(); + }); + + test("nextId generates sequential ids", () => { + setup(); + assert.equal(nextId(tmp, "bgm"), "bgm_001"); + appendRecord(tmp, makeRecord({ id: "bgm_001" })); + assert.equal(nextId(tmp, "bgm"), "bgm_002"); + appendRecord(tmp, makeRecord({ id: "bgm_002" })); + assert.equal(nextId(tmp, "bgm"), "bgm_003"); + cleanup(); + }); + + // --- index-gen.mjs --- + + test("regenerateIndex produces plain-column table", () => { + setup(); + appendRecord(tmp, makeRecord()); + regenerateIndex(tmp); + const content = readFileSync(join(tmp, ".media", "index.md"), "utf8"); + assert.ok(content.includes("# .media · 1 asset")); + assert.ok(content.includes("bgm_001")); + assert.ok(content.includes("soft minimal ambient")); + assert.ok(content.includes("11s")); + cleanup(); + }); + + test("regenerateIndex handles empty manifest", () => { + setup(); + mkdirSync(join(tmp, ".media"), { recursive: true }); + writeFileSync(manifestPath(tmp), ""); + regenerateIndex(tmp); + const content = readFileSync(join(tmp, ".media", "index.md"), "utf8"); + assert.ok(content.includes("# .media · 0 assets")); + cleanup(); + }); + + test("generateIndexContent includes dims for images", () => { + const records = [ + makeRecord({ id: "img_001", type: "image", width: 1920, height: 1080, duration: null }), + ]; + const content = generateIndexContent(records); + assert.ok(content.includes("1920×1080")); + assert.ok(content.includes("img_001")); + }); + + test("regenerateIndex matches manifest content after multiple writes", () => { + setup(); + appendRecord(tmp, makeRecord({ id: "bgm_001" })); + appendRecord( + tmp, + makeRecord({ id: "sfx_001", type: "sfx", description: "whoosh", duration: 3 }), + ); + regenerateIndex(tmp); + const content = readFileSync(join(tmp, ".media", "index.md"), "utf8"); + assert.ok(content.includes("# .media · 2 assets")); + assert.ok(content.includes("bgm_001")); + assert.ok(content.includes("sfx_001")); + assert.ok(content.includes("whoosh")); + cleanup(); + }); + + // --- cache.mjs --- + + test("cacheGet returns null when cache is empty", () => { + const result = cacheGet("nonexistent prompt", "bgm"); + assert.equal(result, null); + }); + + test("cachePut + cacheGet round-trip", () => { + setup(); + const filePath = join(tmp, "test.wav"); + writeFileSync(filePath, "fake audio bytes for testing"); + const record = makeRecord({ provenance: { prompt: "cache test" } }); + + const { sha } = cachePut(filePath, record); + assert.ok(sha); + assert.equal(sha.length, 64); + + const found = cacheGet("cache test", "bgm"); + assert.ok(found); + assert.equal(found.reusable, true); + assert.equal(found.sha, sha); + cleanup(); + }); + + test("cacheGetByEntity finds cached asset", () => { + setup(); + const filePath = join(tmp, "logo.png"); + writeFileSync(filePath, "fake png bytes"); + const record = makeRecord({ + type: "icon", + entity: "TestCorp", + provenance: { prompt: "TestCorp logo" }, + }); + + cachePut(filePath, record); + const found = cacheGetByEntity("testcorp"); + assert.ok(found); + assert.equal(found.entity, "TestCorp"); + cleanup(); + }); + + test("contentHash is deterministic", () => { + setup(); + const filePath = join(tmp, "det.bin"); + writeFileSync(filePath, "deterministic content"); + const h1 = contentHash(filePath); + const h2 = contentHash(filePath); + assert.equal(h1, h2); + cleanup(); + }); + + test("promote copies project asset to global cache", () => { + setup(); + const record = makeRecord(); + appendRecord(tmp, record); + const filePath = join(tmp, record.path); + mkdirSync(join(filePath, ".."), { recursive: true }); + writeFileSync(filePath, "promotable audio data"); + + const { sha } = promote(tmp, "bgm_001"); + assert.ok(sha); + + const cached = cacheGet("subtle tech", "bgm"); + assert.ok(cached); + assert.equal(cached.sha, sha); + cleanup(); + }); + + test("importFromCache copies cached file into project", () => { + setup(); + const filePath = join(tmp, "source.wav"); + writeFileSync(filePath, "importable audio"); + const record = makeRecord({ provenance: { prompt: "import test" } }); + const { sha } = cachePut(filePath, record); + + const cached = cacheGet("import test", "bgm"); + const projectDir = mkdtempSync(join(tmpdir(), "mu-import-")); + const imported = importFromCache(cached, projectDir, "bgm_001", ".media/audio/bgm/bgm_001.wav"); + + assert.ok(imported); + assert.equal(imported.id, "bgm_001"); + assert.equal(imported.provenance.imported_from, sha); + assert.ok(existsSync(join(projectDir, ".media/audio/bgm/bgm_001.wav"))); + + rmSync(projectDir, { recursive: true, force: true }); + cleanup(); + }); + + // --- run --- + + let passed = 0; + let failed = 0; + for (const { name, fn } of tests) { + try { + fn(); + passed++; + console.log(` \x1b[32m✓\x1b[0m ${name}`); + } catch (err) { + failed++; + console.log(` \x1b[31m✗\x1b[0m ${name}`); + console.log(` ${err.message}`); + } + } + console.log(`\n${passed} passed, ${failed} failed`); + if (failed > 0) process.exit(1); +} + +console.log("media-use · manifest/index/cache tests\n"); +runTests(); diff --git a/skills/media-use/scripts/lib/probe.mjs b/skills/media-use/scripts/lib/probe.mjs new file mode 100644 index 0000000000..7528ef0261 --- /dev/null +++ b/skills/media-use/scripts/lib/probe.mjs @@ -0,0 +1,36 @@ +import { execSync } from "node:child_process"; +import { extname } from "node:path"; + +const IMAGE_EXT = new Set([".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico"]); + +export function probe(filePath) { + const ext = extname(filePath).toLowerCase(); + if (ext === ".svg") return { width: null, height: null, duration: null, codec: "svg" }; + + try { + const raw = execSync( + `ffprobe -v quiet -print_format json -show_format -show_streams "${filePath}"`, + { encoding: "utf8", timeout: 5000 }, + ); + const info = JSON.parse(raw); + const stream = info.streams?.[0]; + const format = info.format; + + const isImage = IMAGE_EXT.has(ext); + const duration = isImage + ? null + : parseFloat(format?.duration) || parseFloat(stream?.duration) || null; + const width = parseInt(stream?.width, 10) || null; + const height = parseInt(stream?.height, 10) || null; + const codec = stream?.codec_name || null; + + return { + duration: duration != null ? Math.round(duration * 10) / 10 : null, + width, + height, + codec, + }; + } catch { + return { duration: null, width: null, height: null, codec: null }; + } +} diff --git a/skills/media-use/scripts/lib/providers.mjs b/skills/media-use/scripts/lib/providers.mjs new file mode 100644 index 0000000000..f8924410d2 --- /dev/null +++ b/skills/media-use/scripts/lib/providers.mjs @@ -0,0 +1,29 @@ +import { sfxProvider } from "./sfx-provider.mjs"; +import { imageProvider, iconProvider } from "./image-provider.mjs"; +import { bgmProvider } from "./bgm-provider.mjs"; +import { brandProvider } from "./brand-provider.mjs"; + +const STUB = { + async search() { + return null; + }, +}; + +const registry = { + bgm: { ...bgmProvider, type: "bgm" }, + sfx: { ...sfxProvider, type: "sfx" }, + voice: { ...STUB, type: "voice" }, + image: { ...imageProvider, type: "image" }, + icon: { ...iconProvider, type: "icon" }, + brand: { ...brandProvider, type: "brand" }, +}; + +export function getProvider(type) { + const p = registry[type]; + if (!p) throw new Error(`unknown media type: ${type}`); + return p; +} + +export function listTypes() { + return Object.keys(registry); +} diff --git a/skills/media-use/scripts/lib/sfx-provider.mjs b/skills/media-use/scripts/lib/sfx-provider.mjs new file mode 100644 index 0000000000..24c9e5398f --- /dev/null +++ b/skills/media-use/scripts/lib/sfx-provider.mjs @@ -0,0 +1,23 @@ +import { heygenSearch } from "./heygen-search.mjs"; + +export const sfxProvider = { + async search(intent) { + const results = heygenSearch("audio sounds list", intent, { + type: "sound_effects", + minScore: 0.4, + }); + if (!results) return null; + const best = results[0]; + return { + url: best.audio_url, + source: "search", + // ext derived from audio_url by resolve.mjs — catalog SFX are .mp3 or .wav + metadata: { + description: best.description || best.name || intent, + duration: best.duration || null, + provider: "heygen.audio.sounds", + provenance: { track_id: best.id, score: best.score, query: intent }, + }, + }; + }, +}; diff --git a/skills/media-use/scripts/resolve.mjs b/skills/media-use/scripts/resolve.mjs new file mode 100644 index 0000000000..0de752cdd1 --- /dev/null +++ b/skills/media-use/scripts/resolve.mjs @@ -0,0 +1,247 @@ +#!/usr/bin/env node + +import { existsSync } from "node:fs"; +import { resolve, join, extname } from "node:path"; +import { parseArgs } from "node:util"; +import { appendRecord, findByPrompt, findByEntity, nextId, typeSubdir } from "./lib/manifest.mjs"; +import { regenerateIndex } from "./lib/index-gen.mjs"; +import { cacheGet, cacheGetByEntity, importFromCache } from "./lib/cache.mjs"; +import { getProvider, listTypes } from "./lib/providers.mjs"; +import { freezeUrl, freezeLocalFile } from "./lib/freeze.mjs"; +import { findExistingAsset } from "./lib/adopt.mjs"; + +const { values: args } = parseArgs({ + options: { + type: { type: "string", short: "t" }, + intent: { type: "string", short: "i" }, + entity: { type: "string", short: "e" }, + project: { type: "string", short: "p", default: "." }, + adopt: { type: "boolean", default: false }, + json: { type: "boolean", default: false }, + help: { type: "boolean", short: "h", default: false }, + }, + strict: true, +}); + +if (args.help) { + console.log(`media-use resolve — turn a media need into a frozen local file + +Usage: + node resolve.mjs --type --intent "" [--project ] + +Types: ${listTypes().join(", ")} + +Options: + --type, -t Media type (required) + --intent, -i What you need (required) + --entity, -e Entity name for cache matching (optional) + --project, -p Project directory (default: .) + --adopt Adopt all existing assets/ files into the manifest + --json Output JSON instead of one-line result + --help, -h Show this help`); + process.exit(0); +} + +if (args.adopt) { + const { adoptExistingAssets } = await import("./lib/adopt.mjs"); + const projectDir = resolve(args.project); + const adopted = adoptExistingAssets(projectDir); + if (args.json) { + console.log(JSON.stringify({ ok: true, adopted: adopted.length, assets: adopted })); + } else if (adopted.length === 0) { + console.log("no new assets to adopt (assets/ empty or already registered)"); + } else { + console.log(`adopted ${adopted.length} asset${adopted.length === 1 ? "" : "s"} from assets/`); + for (const r of adopted) console.log(` ${r.id} → ${r.path} (${r.type})`); + } + process.exit(0); +} + +if (!args.type || !args.intent) { + console.error("error: --type and --intent are required"); + process.exit(2); +} + +const projectDir = resolve(args.project); +const type = args.type; +const intent = args.intent; +const entity = args.entity || null; + +async function run() { + // 1. project manifest — exact-prompt match + const projectHit = findByPrompt(projectDir, intent, type); + if (projectHit && existsSync(join(projectDir, projectHit.path))) { + return result(projectHit, "cached"); + } + + // 1b. entity match in project + if (entity) { + const entityHit = findByEntity(projectDir, entity); + if (entityHit && entityHit.type === type && existsSync(join(projectDir, entityHit.path))) { + return result(entityHit, "cached"); + } + } + + // 1c. scan existing assets/ directory for unregistered matches + const existingAsset = findExistingAsset(projectDir, intent, type); + if (existingAsset) { + const id = nextId(projectDir, type); + const record = { + id, + type: existingAsset.type, + path: existingAsset.relativePath, + source: "existing", + description: existingAsset.name.replace(/[-_]/g, " "), + provenance: { provider: "local", adopted: true, prompt: intent }, + }; + appendRecord(projectDir, record); + regenerateIndex(projectDir); + return result(record, "existing"); + } + + // 2. global cache — exact-prompt or entity match + const cacheHit = cacheGet(intent, type); + if (cacheHit) { + const id = nextId(projectDir, type); + const ext = extname(cacheHit.cached_path); + const localPath = `.media/${typeSubdir(type)}/${id}${ext}`; + const imported = importFromCache(cacheHit, projectDir, id, localPath); + if (imported) { + appendRecord(projectDir, imported); + regenerateIndex(projectDir); + return result(imported, "reused"); + } + } + + if (entity) { + const entityCacheHit = cacheGetByEntity(entity); + if (entityCacheHit && entityCacheHit.type === type) { + const id = nextId(projectDir, type); + const ext = extname(entityCacheHit.cached_path); + const localPath = `.media/${typeSubdir(type)}/${id}${ext}`; + const imported = importFromCache(entityCacheHit, projectDir, id, localPath); + if (imported) { + appendRecord(projectDir, imported); + regenerateIndex(projectDir); + return result(imported, "reused"); + } + } + } + + // 3. provider search + const provider = getProvider(type); + let searchResult = null; + try { + searchResult = await provider.search(intent, { entity, projectDir }); + } catch { + // search failed, try generate + } + + // 4. generate fallback + if (!searchResult && provider.generate) { + try { + searchResult = await provider.generate(intent, { entity, projectDir }); + } catch { + // generate failed too + } + } + + if (!searchResult) { + if (args.json) { + console.log( + JSON.stringify({ ok: false, error: `no provider could resolve ${type}: "${intent}"` }), + ); + } else { + console.error(`error: no provider could resolve ${type}: "${intent}"`); + } + process.exit(1); + } + + // 5. freeze + register + const id = nextId(projectDir, type); + const ext = searchResult.ext || extFromUrl(searchResult.url || "") || defaultExt(type); + const localPath = `.media/${typeSubdir(type)}/${id}${ext}`; + const fullPath = join(projectDir, localPath); + + if (searchResult.localPath) { + freezeLocalFile(searchResult.localPath, fullPath); + } else if (searchResult.url) { + await freezeUrl(searchResult.url, fullPath); + } else { + console.error("error: provider returned no url or localPath"); + process.exit(1); + } + + const record = { + id, + type, + path: localPath, + source: searchResult.source || "search", + description: searchResult.metadata?.description || intent, + ...(searchResult.metadata?.duration != null && { duration: searchResult.metadata.duration }), + ...(searchResult.metadata?.width != null && { width: searchResult.metadata.width }), + ...(searchResult.metadata?.height != null && { height: searchResult.metadata.height }), + ...(searchResult.metadata?.transparent != null && { + transparent: searchResult.metadata.transparent, + }), + ...(entity && { entity }), + provenance: { + provider: searchResult.metadata?.provider || "unknown", + prompt: intent, + ...searchResult.metadata?.provenance, + }, + }; + + appendRecord(projectDir, record); + regenerateIndex(projectDir); + return result(record, searchResult.source || "search"); +} + +function result(record, source) { + if (args.json) { + console.log(JSON.stringify({ ok: true, ...record, _source: source })); + } else { + const meta = formatMeta(record, source); + console.log(`resolved ${record.id} → ${record.path} (${meta})`); + } +} + +function formatMeta(record, source) { + const parts = [record.type]; + if (record.duration != null) parts.push(`${record.duration}s`); + if (record.width && record.height) parts.push(`${record.width}×${record.height}`); + if (record.transparent) parts.push("transparent"); + if (source === "reused") parts.push("reused"); + if (source === "generated") parts.push("generated"); + return parts.join(", "); +} + +function extFromUrl(url) { + try { + return extname(new URL(url).pathname) || null; + } catch { + return null; + } +} + +const DEFAULT_EXT = { + bgm: ".wav", + sfx: ".mp3", + voice: ".wav", + image: ".jpg", + icon: ".svg", + brand: ".png", +}; + +function defaultExt(type) { + return DEFAULT_EXT[type] || ".bin"; +} + +run().catch((err) => { + if (args.json) { + console.log(JSON.stringify({ ok: false, error: err.message })); + } else { + console.error(`error: ${err.message}`); + } + process.exit(1); +}); diff --git a/skills/pr-to-video/SKILL.md b/skills/pr-to-video/SKILL.md index e17699422a..fbc8fee916 100644 --- a/skills/pr-to-video/SKILL.md +++ b/skills/pr-to-video/SKILL.md @@ -3,6 +3,8 @@ name: pr-to-video description: "turn a GitHub pull request (a PR URL like github.com///pull/, an /# ref, or 'this PR' in a checked-out repo) into a code-change explainer video, up to ~3 min (sweet spot 30-90s) — changelog, feature reveal, fix, or refactor walkthrough, rendered from the diff / commits / files. The input is a CODE CHANGE read via the gh CLI; there is no website capture. Use this skill for a GitHub PR. Do not use it for a product launch/promo (use /product-launch-video), a tour of a real website (use /website-to-video), a topic explainer with no PR (use /faceless-explainer), captions on existing footage (use /embedded-captions), or a short unnarrated motion graphic (use /motion-graphics). If the intent is unclear, route through /hyperframes first." --- +> **media-use**: Before sourcing audio/images, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog. Run `--adopt` first to register existing assets. See `/media-use` skill. + # PR to HyperFrames Use this skill to ingest a GitHub pull request, understand the change, plan a code-change explainer, and build it frame by frame in HyperFrames. The input is a **code change** (read via `gh`), not a website — there is **no capture step and no real assets** beyond the contributors' avatars. diff --git a/skills/product-launch-video/SKILL.md b/skills/product-launch-video/SKILL.md index c097822894..35f1ba69ab 100644 --- a/skills/product-launch-video/SKILL.md +++ b/skills/product-launch-video/SKILL.md @@ -3,6 +3,8 @@ name: product-launch-video description: "turn a product or marketing URL, pasted script, or brief into a product launch video, including SaaS promos, feature reveals, app launches, company promos, and product marketing videos. Use this skill when the user wants to market, launch, promote, or reveal a product. Do not use it for general non-launch website tours, non-product topic explainers, GitHub pull requests, captioning existing footage, or short unnarrated motion graphics. If the intent is unclear, route through /hyperframes first." --- +> **media-use**: Before sourcing audio/images, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog. Run `--adopt` first to register existing assets. See `/media-use` skill. + # Product Launch to HyperFrames Use this skill to capture a product, understand its brand, plan a launch video, and build it frame by frame in HyperFrames. diff --git a/skills/website-to-video/SKILL.md b/skills/website-to-video/SKILL.md index 3d9c6d7285..2211f0ba39 100644 --- a/skills/website-to-video/SKILL.md +++ b/skills/website-to-video/SKILL.md @@ -3,6 +3,8 @@ name: website-to-video description: "Capture a general website/URL and turn it into a HyperFrames video (site tour, showcase, or social clip from the site's own visuals). Uses headless Chrome screenshots + brand assets. Use when intent is general — portfolio/blog/landing-page showcase or social clip from the site. NOT for: product/SaaS launch or promo (→ /product-launch-video, even from a URL); topic explainer with no site (→ /faceless-explainer); GitHub PR (→ /pr-to-video); adding captions to existing video (→ /embedded-captions); short unnarrated page-highlight motion graphic (→ /motion-graphics). Unclear launch-vs-general-site? Ask one question or start at /hyperframes." --- +> **media-use**: Before sourcing audio/images, call `/media-use` to resolve BGM/SFX/images from the HeyGen catalog. Run `--adopt` first to register existing assets. See `/media-use` skill. + # Website to HyperFrames Capture a website, then produce a professional video from it.