diff --git a/packages/react-native-executorch/src/extensions/cv/ops/boxes.ts b/packages/react-native-executorch/src/extensions/cv/ops/boxes.ts index 8360472980..3351721d0a 100644 --- a/packages/react-native-executorch/src/extensions/cv/ops/boxes.ts +++ b/packages/react-native-executorch/src/extensions/cv/ops/boxes.ts @@ -8,9 +8,24 @@ import { scalePoint } from './points'; * @category Types */ export type BoxMap = { - xyxy: { xmin: number; ymin: number; xmax: number; ymax: number }; - xywh: { xmin: number; ymin: number; w: number; h: number }; - cxcywh: { cx: number; cy: number; w: number; h: number }; + xyxy: { + readonly xmin: number; + readonly ymin: number; + readonly xmax: number; + readonly ymax: number; + }; + xywh: { + readonly xmin: number; + readonly ymin: number; + readonly w: number; + readonly h: number; + }; + cxcywh: { + readonly cx: number; + readonly cy: number; + readonly w: number; + readonly h: number; + }; }; /** @@ -60,8 +75,8 @@ export function decodeBox( * @param opts Options defining dimensions and resize modes. * @param opts.from The source bounds (e.g. model input dimensions). * @param opts.to The destination bounds (e.g. original image dimensions). - * @param opts.resizeMode The mode used to resize the image ('letterbox' or - * 'stretch'). + * @param opts.resizeMode The mode used to resize the image {@link ResizeMode} + * (excluding `'crop'`). * @returns The scaled BoundingBox object. */ export function scaleBox( @@ -130,9 +145,16 @@ export function scaleBox( * @category Types */ export type NmsOptions = { + /** How bounding box coordinates are interpreted {@link BoxFormat}. */ readonly boxFormat: BoxFormat; + /** Intersection over Union (IoU) threshold for suppressing overlapping boxes. */ readonly iouThreshold: number; + /** Minimum confidence score threshold for filtering candidate boxes. */ readonly confidenceThreshold: number; + /** + * NMS algorithm variant (`standard` for hard suppression, `weighted` for soft + * coordinate averaging). + */ readonly nmsType: 'standard' | 'weighted'; }; @@ -142,7 +164,13 @@ export type NmsOptions = { * @category Utils * @param boxes Bounding boxes coordinate tensor. * @param scores Bounding boxes confidence scores tensor. - * @param opts Options configure NMS thresholds and execution mode. + * @param opts Options configuring NMS thresholds and execution mode. + * @param opts.boxFormat The bounding box format {@link BoxFormat}. + * @param opts.iouThreshold Intersection over Union (IoU) threshold for + * suppression. + * @param opts.confidenceThreshold Minimum confidence score for candidate + * selection. + * @param opts.nmsType The NMS algorithm variant {@link NmsOptions.nmsType}. * @returns The resulting indices of the non-suppressed boxes: * - For `standard` NMS: A 1D array of indices (`number[]`) representing the * selected boxes. diff --git a/packages/react-native-executorch/src/extensions/cv/ops/image.ts b/packages/react-native-executorch/src/extensions/cv/ops/image.ts index 1d98b73c41..799f30e259 100644 --- a/packages/react-native-executorch/src/extensions/cv/ops/image.ts +++ b/packages/react-native-executorch/src/extensions/cv/ops/image.ts @@ -74,8 +74,11 @@ export type InterpolationMethod = 'nearest' | 'area' | 'cubic' | 'lanczos' | 'li * @category Types */ export type ResizeOptions = { + /** How the image is resized {@link ResizeMode}. */ readonly mode?: ResizeMode; + /** Background fill value used when letterboxing. */ readonly padValue?: number; + /** Pixel interpolation method {@link InterpolationMethod}. */ readonly interpolation?: InterpolationMethod; }; @@ -84,21 +87,30 @@ export type ResizeOptions = { * @category Types */ export type NormalizeOptions = { + /** + * Multiplicative coefficient applied as `pixel * alpha`. Single value for + * uniform scaling, array for per-channel. + */ readonly alpha?: number | readonly number[]; + /** Additive offset applied as `pixel * alpha + beta`. Single value or per-channel array. */ readonly beta?: number | readonly number[]; }; /** * Resizes an image tensor from a source dimension to a destination dimension. * - * Supports various resize modes (`stretch`, `letterbox`, `crop`) and - * interpolation algorithms (`linear`, `lanczos`, etc.). + * Supports various {@link ResizeMode} and {@link InterpolationMethod} options. * @category Typescript API * @param src The source image tensor in HWC layout. Shape [H,W,C]. * @param dst The pre-allocated destination tensor to write the resized image * to. `dst` must be in HWC layout and its number of channels must match `src`. * Shape [H',W',C]. * @param opts Configuration options for resizing. + * @param opts.mode The resize algorithm mode {@link ResizeMode}. Defaults to + * `'stretch'`. + * @param opts.interpolation The pixel interpolation method + * {@link InterpolationMethod}. Defaults to `'lanczos'`. + * @param opts.padValue Fill value for letterboxing. Defaults to `0`. * @returns The destination tensor containing the resized image. */ export function resize(src: Tensor, dst: Tensor, opts?: ResizeOptions): Tensor { @@ -173,6 +185,9 @@ export function toChannelsLast(src: Tensor, dst: Tensor): Tensor { * @param dst The pre-allocated destination tensor to write the normalized * values to. `dst` must have the same shape as `src`. Shape [C,H,W]. * @param opts Normalization scaling coefficients. + * @param opts.alpha Multiplicative scaling coefficient(s). Defaults to + * `1 / 255.0`. + * @param opts.beta Additive offset coefficient(s). Defaults to `0.0`. * @returns The destination tensor containing the normalized image. */ export function normalize(src: Tensor, dst: Tensor, opts?: NormalizeOptions): Tensor { diff --git a/packages/react-native-executorch/src/extensions/cv/ops/points.ts b/packages/react-native-executorch/src/extensions/cv/ops/points.ts index 4464061e18..5c12d6a165 100644 --- a/packages/react-native-executorch/src/extensions/cv/ops/points.ts +++ b/packages/react-native-executorch/src/extensions/cv/ops/points.ts @@ -17,8 +17,8 @@ export type Point = { * @param opts Options detailing the scaling factors and resize mode. * @param opts.from The source bounds (e.g. model input dimensions). * @param opts.to The destination bounds (e.g. original image dimensions). - * @param opts.resizeMode The mode used to resize the image ('letterbox' or - * 'stretch'). + * @param opts.resizeMode The mode used to resize the image {@link ResizeMode} + * (excluding `'crop'`). * @returns The scaled coordinate point. */ export function scalePoint( diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts b/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts index 3a8b2d244c..a3109a4dea 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts @@ -14,15 +14,24 @@ import { createImagePreprocessor, type ImagePreprocessorOptions } from './prepro * vocabulary. * @category Types */ -export type ClassifierOptions = ImagePreprocessorOptions & { readonly labels: readonly L[] }; +export type ClassifierOptions = ImagePreprocessorOptions & { + /** Array of class labels matching the model's output vocabulary. */ + readonly labels: readonly L[]; +}; /** * Model configuration required to instantiate a classifier task runner. * @category Types */ export type ClassifierModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; - readonly classifierOpts: ClassifierOptions; + /** + * Image preprocessing and label vocabulary + * {@link ClassifierOptions}. The `labels` array length must + * match the model's output dimension. + */ + readonly modelOpts: ClassifierOptions; }; /** @@ -30,7 +39,9 @@ export type ClassifierModel = { * @category Types */ export type Classification = { + /** Predicted class label. */ readonly label: L; + /** Confidence score of the prediction (between 0.0 and 1.0). */ readonly confidence: number; }; @@ -74,7 +85,7 @@ export async function createClassifier( */ classifyWorklet: (input: ImageBuffer, options?: { topk?: number }) => Classification[]; }> { - const { modelPath, classifierOpts } = config; + const { modelPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); const meta = validateModelSchema( @@ -87,9 +98,9 @@ export async function createClassifier( const outShape = meta.outputTensorMeta[0]!.shape; const numLabels = outShape[outShape.length - 1]!; - if (classifierOpts.labels.length !== numLabels) { + if (modelOpts.labels.length !== numLabels) { throw new Error( - `Classifier labels length (${classifierOpts.labels.length}) must match model output dimension (${numLabels}).` + `Classifier labels length (${modelOpts.labels.length}) must match model output dimension (${numLabels}).` ); } @@ -100,7 +111,7 @@ export async function createClassifier( ] as const; const [tLogits, tProbas] = tensors; - const preprocessor = createImagePreprocessor(classifierOpts, inpShape); + const preprocessor = createImagePreprocessor(modelOpts, inpShape); const dispose = () => { preprocessor.dispose(); @@ -125,7 +136,7 @@ export async function createClassifier( .getData(new Float32Array(tProbas.numel)); return Array.from(probas) - .map((confidence, index) => ({ confidence, label: classifierOpts.labels[index]! })) + .map((confidence, index) => ({ confidence, label: modelOpts.labels[index]! })) .sort((a, b) => b.confidence - a.confidence) .slice(0, options?.topk); }; diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts index afd4c63383..af45da3191 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts @@ -13,8 +13,13 @@ import { createImagePreprocessor, type ImagePreprocessorOptions } from './prepro * @category Types */ export type ImageEmbedderModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; - readonly opts: ImagePreprocessorOptions; + /** + * Image preprocessing (resize, color conversion, normalization) + * for embedding models {@link ImagePreprocessorOptions}. + */ + readonly modelOpts: ImagePreprocessorOptions; }; /** @@ -53,7 +58,7 @@ export async function createImageEmbedder( */ embedWorklet: (input: ImageBuffer) => Float32Array; }> { - const { modelPath, opts } = config; + const { modelPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); const meta = validateModelSchema( @@ -67,7 +72,7 @@ export async function createImageEmbedder( const tensors = [tensor('float32', outShape)] as const; const [tEmbedding] = tensors; - const preprocessor = createImagePreprocessor(opts, inpShape); + const preprocessor = createImagePreprocessor(modelOpts, inpShape); const dispose = () => { preprocessor.dispose(); diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts index d26407ea39..a60da8eb6d 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts @@ -31,11 +31,17 @@ export type InstanceSegmenterOptions = Omit< ImagePreprocessorOptions, 'resizeMode' > & { + /** Resize mode for input images. Must be `'stretch'`. */ readonly resizeMode: 'stretch'; + /** Array of class labels matching the model's output vocabulary. */ readonly labels: readonly L[]; + /** How bounding box coordinates are interpreted {@link BoxFormat}. */ readonly boxFormat: F; + /** Default Intersection over Union (IoU) threshold for Non-Maximum Suppression (NMS). */ readonly defaultIouThreshold: number; + /** Default probability threshold for mask values. */ readonly defaultMaskThreshold: number; + /** Default minimum confidence score threshold for detected instances. */ readonly defaultConfidenceThreshold: number; }; @@ -46,8 +52,14 @@ export type InstanceSegmenterOptions = Omit< * @typeParam L The label type. */ export type InstanceSegmenterModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; - readonly opts: InstanceSegmenterOptions; + /** + * Image preprocessing, label vocabulary, bounding box format, + * and default NMS/mask/confidence thresholds + * {@link InstanceSegmenterOptions}. + */ + readonly modelOpts: InstanceSegmenterOptions; }; /** @@ -58,9 +70,13 @@ export type InstanceSegmenterModel = { * @typeParam L The label type. */ export type InstanceSegmentationResult = { + /** Scaled bounding box coordinates matching the input image resolution. */ readonly box: BoundingBox; + /** Binary segmentation mask buffer cropped to the instance bounding box. */ readonly mask: ImageBuffer; + /** Predicted instance class label. */ readonly label: L; + /** Confidence score of the instance detection (between 0.0 and 1.0). */ readonly confidence: number; }; @@ -93,10 +109,12 @@ export async function createInstanceSegmenter( * Performs asynchronous instance segmentation on the given input image. * @param input The input image buffer. * @param options Execution override options. - * @param options.confidenceThreshold Override for the minimum confidence - * threshold. - * @param options.iouThreshold Override for the IoU threshold in NMS. - * @param options.maskThreshold Override for the mask binarization threshold. + * @param options.confidenceThreshold Minimum confidence threshold. If + * omitted, uses `modelOpts.defaultConfidenceThreshold`. + * @param options.iouThreshold Intersection over Union (IoU) threshold in NMS. If omitted, uses + * `modelOpts.defaultIouThreshold`. + * @param options.maskThreshold Mask binarization threshold. If omitted, + * uses `modelOpts.defaultMaskThreshold`. * @returns A promise resolving to a list of detected instances. */ segmentInstances: ( @@ -113,7 +131,7 @@ export async function createInstanceSegmenter( options?: { confidenceThreshold?: number; iouThreshold?: number; maskThreshold?: number } ) => InstanceSegmentationResult[]; }> { - const { modelPath, opts } = config; + const { modelPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); const meta = validateModelSchema( model, @@ -149,7 +167,7 @@ export async function createInstanceSegmenter( const [tBoxes, tScores, tClasses, tAllMasks, tMask] = tensors; - const preprocessor = createImagePreprocessor(opts, inpShape); + const preprocessor = createImagePreprocessor(modelOpts, inpShape); const dispose = () => { preprocessor.dispose(); @@ -165,16 +183,17 @@ export async function createInstanceSegmenter( const tInput = preprocessor.process(input); model.execute('forward', [tInput], [tBoxes, tScores, tClasses, tAllMasks]); - const iouThreshold = options?.iouThreshold ?? opts.defaultIouThreshold; - const maskThreshold = options?.maskThreshold ?? opts.defaultMaskThreshold; - const confidenceThreshold = options?.confidenceThreshold ?? opts.defaultConfidenceThreshold; + const iouThreshold = options?.iouThreshold ?? modelOpts.defaultIouThreshold; + const maskThreshold = options?.maskThreshold ?? modelOpts.defaultMaskThreshold; + const confidenceThreshold = + options?.confidenceThreshold ?? modelOpts.defaultConfidenceThreshold; const eps = 1e-7; const clampedMaskThreshold = Math.max(eps, Math.min(1 - eps, maskThreshold)); const logitMaskThreshold = Math.log(clampedMaskThreshold / (1 - clampedMaskThreshold)); const indices = nms(tBoxes, tScores, { - boxFormat: opts.boxFormat, + boxFormat: modelOpts.boxFormat, iouThreshold, confidenceThreshold, nmsType: 'standard', @@ -199,12 +218,12 @@ export async function createInstanceSegmenter( for (const idx of indices) { const confidence = scores[idx]!; const classIdx = Math.round(classes[idx]!); - const label = opts.labels[classIdx]; + const label = modelOpts.labels[classIdx]; if (label === undefined) { throw new Error( `InstanceSegmenter: Predicted class index ${classIdx} is ` + - `out of bounds for labels array of size ${opts.labels.length}.` + `out of bounds for labels array of size ${modelOpts.labels.length}.` ); } @@ -213,7 +232,7 @@ export async function createInstanceSegmenter( const c = boxes[idx * 4 + 2]!; const d = boxes[idx * 4 + 3]!; - const box = scaleBox(decodeBox([a, b, c, d], opts.boxFormat), { + const box = scaleBox(decodeBox([a, b, c, d], modelOpts.boxFormat), { from: { width: targetW, height: targetH }, to: { width: input.width, height: input.height }, resizeMode: 'stretch', diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts b/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts index d6ea03a294..e1ee580b6e 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts @@ -22,10 +22,15 @@ export type KeypointDetectorOptions ImagePreprocessorOptions, 'resizeMode' > & { + /** Resize mode for preprocessing input images {@link ResizeMode} (excluding `'crop'`). */ readonly resizeMode: Exclude; + /** How bounding box coordinates are interpreted {@link BoxFormat}. */ readonly boxFormat: F; + /** Array of landmark names matching the model output keypoint locations. */ readonly landmarks: readonly L[]; + /** Default Intersection over Union (IoU) threshold for Non-Maximum Suppression (NMS). */ readonly defaultIouThreshold: number; + /** Default minimum confidence score threshold for keypoint detections. */ readonly defaultConfidenceThreshold: number; }; @@ -34,8 +39,14 @@ export type KeypointDetectorOptions * @category Types */ export type KeypointDetectorModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; - readonly opts: KeypointDetectorOptions; + /** + * Image preprocessing, landmark names, bounding box format, + * and default NMS/confidence thresholds + * {@link KeypointDetectorOptions}. + */ + readonly modelOpts: KeypointDetectorOptions; }; /** @@ -51,8 +62,11 @@ export type Landmarks = Record = { + /** Scaled bounding box coordinates matching the input image resolution. */ readonly box: BoundingBox; + /** Overall confidence score of the detection (between 0.0 and 1.0). */ readonly confidence: number; + /** Map of landmark names to their scaled pixel coordinates and individual confidence scores. */ readonly landmarks: Landmarks; }; @@ -160,9 +174,9 @@ export async function createKeypointDetector KeypointDetection[]; }> { - const { modelPath, opts } = config; - const { landmarks } = opts; + const { modelPath, modelOpts } = config; + const { landmarks } = modelOpts; const model = await wrapAsync(loadModel, runtime)(modelPath); const meta = validateModelSchema( model, @@ -205,7 +219,7 @@ export async function createKeypointDetector { preprocessor.dispose(); @@ -221,11 +235,12 @@ export async function createKeypointDetector = Omit< ImagePreprocessorOptions, 'resizeMode' > & { + /** Resize mode for preprocessing input images {@link ResizeMode} (excluding `'crop'`). */ readonly resizeMode: Exclude; + /** Array of class labels matching the model's output vocabulary. */ readonly labels: readonly L[]; + /** How bounding box coordinates are interpreted {@link BoxFormat}. */ readonly boxFormat: F; + /** Default Intersection over Union (IoU) threshold for Non-Maximum Suppression (NMS). */ readonly defaultIouThreshold: number; + /** Default minimum confidence score threshold for detections. */ readonly defaultConfidenceThreshold: number; }; @@ -33,8 +38,14 @@ export type ObjectDetectorOptions = Omit< * @category Types */ export type ObjectDetectorModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; - readonly opts: ObjectDetectorOptions; + /** + * Image preprocessing, label vocabulary, and default + * NMS/confidence thresholds {@link ObjectDetectorOptions}. + * Used as fallbacks when per-call overrides are omitted. + */ + readonly modelOpts: ObjectDetectorOptions; }; /** @@ -42,8 +53,11 @@ export type ObjectDetectorModel = { * @category Types */ export type ObjectDetection = { + /** Scaled bounding box coordinates matching the input image resolution. */ readonly box: BoundingBox; + /** Predicted object class label. */ readonly label: L; + /** Confidence score of the detection (between 0.0 and 1.0). */ readonly confidence: number; }; @@ -72,12 +86,12 @@ export async function createObjectDetector( */ dispose: () => void; /** - * Performs asynchronous object detection on the given input image. * @param input The input image buffer. * @param options Configuration options for object detection. - * @param options.confidenceThreshold Minimum confidence score for returned - * detections. - * @param options.iouThreshold Non-maximum suppression IoU threshold. + * @param options.confidenceThreshold Minimum confidence score threshold. If + * omitted, uses `modelOpts.defaultConfidenceThreshold`. + * @param options.iouThreshold Intersection over Union (IoU) threshold. If + * omitted, uses `modelOpts.defaultIouThreshold`. * @returns A promise resolving to the list of object detections. */ detectObjects: ( @@ -93,7 +107,7 @@ export async function createObjectDetector( options?: { confidenceThreshold?: number; iouThreshold?: number } ) => ObjectDetection[]; }> { - const { modelPath, opts } = config; + const { modelPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); const meta = validateModelSchema( @@ -122,9 +136,9 @@ export async function createObjectDetector( ] as const; const [tBoxes, tScores, tClasses] = tensors; - const preprocessor = createImagePreprocessor(opts, inpShape); + const preprocessor = createImagePreprocessor(modelOpts, inpShape); - const { boxFormat } = opts; + const { boxFormat } = modelOpts; const dispose = () => { preprocessor.dispose(); @@ -144,8 +158,9 @@ export async function createObjectDetector( const scores = tScores.getData(new Float32Array(tScores.numel)); const classes = tClasses.getData(new Float32Array(tClasses.numel)); - const iouThreshold = options?.iouThreshold ?? opts.defaultIouThreshold; - const confidenceThreshold = options?.confidenceThreshold ?? opts.defaultConfidenceThreshold; + const iouThreshold = options?.iouThreshold ?? modelOpts.defaultIouThreshold; + const confidenceThreshold = + options?.confidenceThreshold ?? modelOpts.defaultConfidenceThreshold; const results: ObjectDetection[] = []; const indices = nms(tBoxes, tScores, { @@ -158,12 +173,12 @@ export async function createObjectDetector( for (const index of indices) { const confidence = scores[index]!; const classIdx = Math.round(classes[index]!); - const label = opts.labels[classIdx]; + const label = modelOpts.labels[classIdx]; if (label === undefined) { throw new Error( `ObjectDetector: Predicted class index ${classIdx} is out of bounds for` + - `labels array of size ${opts.labels.length}.` + `labels array of size ${modelOpts.labels.length}.` ); } @@ -178,7 +193,7 @@ export async function createObjectDetector( box: scaleBox(decodeBox([a, b, c, d], boxFormat), { from: { width: targetW, height: targetH }, to: { width: input.width, height: input.height }, - ...opts, + ...modelOpts, }), }); } diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts b/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts index 3569210481..7555252e4c 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts @@ -5,6 +5,7 @@ import type { ImageBuffer } from '../image'; import { type ResizeMode, type InterpolationMethod, + type NormalizeOptions, FORMAT_CONVERSION, FORMAT_CHANNELS, resize, @@ -18,10 +19,16 @@ import { * @category Types */ export type ImagePreprocessorOptions = { + /** + * How the input image is resized to match the model's expected + * dimensions {@link ResizeMode}. + */ readonly resizeMode: ResizeMode; + /** Algorithm used when resizing {@link InterpolationMethod}. `'linear'` is a good default. */ readonly interpolation: InterpolationMethod; - readonly alpha: number | readonly number[]; - readonly beta: number | readonly number[]; + /** Normalization scaling coefficients. */ + readonly normalizeOpts: NormalizeOptions; + /** Optional background fill value used when letterboxing. */ readonly padValue?: number; }; @@ -84,7 +91,7 @@ export function createImagePreprocessor( ] as const; const [tColor, tChanFirst, tNorm, tOutput] = tensors; - const { resizeMode, interpolation, alpha, beta, padValue } = opts; + const { resizeMode, interpolation, normalizeOpts, padValue } = opts; const dispose = () => tensors.forEach((t) => t.dispose()); const process = (input: ImageBuffer): Tensor => { @@ -105,7 +112,7 @@ export function createImagePreprocessor( }) .throughIf(colorCode !== null, cvtColor, tColor, colorCode!) .through(toChannelsFirst, tChanFirst) - .through(normalize, tNorm, { alpha, beta }) + .through(normalize, tNorm, normalizeOpts) .copyTo(tOutput); } finally { tInput.dispose(); diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts index 148cf9b267..15e2a632ce 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts @@ -22,9 +22,12 @@ import { sigmoid, argmax } from '../../math'; * vocabulary. * @category Types */ -export type SemanticSegmentationOptions = Omit & { +export type SemanticSegmenterOptions = Omit & { + /** Resize mode for input images. Must be `'stretch'`. */ readonly resizeMode: 'stretch'; + /** Interpolation method used when resizing output masks back to input image dimensions. */ readonly outInterpolation: InterpolationMethod; + /** Array of class labels matching the model's output vocabulary. */ readonly labels: readonly L[]; }; @@ -32,9 +35,15 @@ export type SemanticSegmentationOptions = Omit = { +export type SemanticSegmenterModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; - readonly opts: SemanticSegmentationOptions; + /** + * Image preprocessing, output mask interpolation, and label + * vocabulary {@link SemanticSegmenterOptions}. `resizeMode` + * is fixed to `'stretch'`. + */ + readonly modelOpts: SemanticSegmenterOptions; }; /** @@ -48,8 +57,10 @@ export type ColorMap = Record = { - buffer: ImageBuffer; - colormap?: ColorMap; + /** Generated output RGBA image buffer containing the colored segmentation mask. */ + readonly buffer: ImageBuffer; + /** Applied color map mapping each class label to its RGBA tuple. */ + readonly colormap?: ColorMap; }; function hslToRgb(h: number, s: number, l: number): [number, number, number] { @@ -77,7 +88,7 @@ function hslToRgb(h: number, s: number, l: number): [number, number, number] { * disposal controls. */ export async function createSemanticSegmenter( - config: SemanticSegmentationModel, + config: SemanticSegmenterModel, runtime?: WorkletRuntime ): Promise<{ /** @@ -118,7 +129,7 @@ export async function createSemanticSegmenter( colormap?: Partial> ) => SemanticSegmentationResult; }> { - const { modelPath, opts } = config; + const { modelPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); const meta = validateModelSchema( @@ -136,14 +147,14 @@ export async function createSemanticSegmenter( // Generate highly distinct, high-contrast colors, see: // https://martin.ankerl.com/2009/12/09/how-to-create-random-colors-programmatically/ - const defaultColormap = opts.labels.map((_, i) => { + const defaultColormap = modelOpts.labels.map((_, i) => { if (i === 0) return [0, 0, 0, 0] as const; return [...hslToRgb((i * 137.5) % 360, 95, 50), 255] as const; }); - if (nClasses > 1 && opts.labels.length !== nClasses) { + if (nClasses > 1 && modelOpts.labels.length !== nClasses) { throw new Error( - `Model outputs ${nClasses} classes, but ${opts.labels.length} labels were provided in the configuration.` + `Model outputs ${nClasses} classes, but ${modelOpts.labels.length} labels were provided in the configuration.` ); } @@ -157,7 +168,7 @@ export async function createSemanticSegmenter( ] as const; const [tOutput, tReshape, tSigmoid, tChanLast, tMask, tRgba] = tensors; - const preprocessor = createImagePreprocessor(opts, inpShape); + const preprocessor = createImagePreprocessor(modelOpts, inpShape); const dispose = () => { tensors.forEach((t) => t.dispose()); @@ -177,15 +188,15 @@ export async function createSemanticSegmenter( if (nClasses > 1) { if (colormap) { returnColormap = Object.fromEntries( - opts.labels.map((l) => [l, colormap[l] ?? [0, 0, 0, 0]]) + modelOpts.labels.map((l) => [l, colormap[l] ?? [0, 0, 0, 0]]) ) as ColorMap; } else { returnColormap = Object.fromEntries( - opts.labels.map((l, i) => [l, defaultColormap[i]!]) + modelOpts.labels.map((l, i) => [l, defaultColormap[i]!]) ) as ColorMap; } - const colormapData = opts.labels.map((l) => returnColormap![l]); + const colormapData = modelOpts.labels.map((l) => returnColormap![l]); tOutput .copyTo(tReshape) @@ -205,7 +216,7 @@ export async function createSemanticSegmenter( const tResize = tensor('uint8', [input.height, input.width, 4]); try { tRgba - .through(resize, tResize, { mode: 'stretch', interpolation: opts.outInterpolation }) + .through(resize, tResize, { mode: 'stretch', interpolation: modelOpts.outInterpolation }) .getData(data); } finally { tResize.dispose(); diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts b/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts index 1c1a89cbe5..ed4b6ca4fb 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts @@ -13,6 +13,7 @@ import { cvtColor, resize, type InterpolationMethod, + type NormalizeOptions, } from '../ops/image'; /** @@ -20,9 +21,11 @@ import { * @category Types */ export type StyleTransferOptions = Omit & { + /** Resize mode for input images. Must be `'stretch'`. */ readonly resizeMode: 'stretch'; - readonly outAlpha: number | number[]; - readonly outBeta: number | number[]; + /** Normalization options for postprocessing output tensors back to uint8 pixel values. */ + readonly outNormalizeOpts: NormalizeOptions; + /** Interpolation method used when resizing output styled images to input dimensions. */ readonly outInterpolation: InterpolationMethod; }; @@ -31,8 +34,14 @@ export type StyleTransferOptions = Omit * @category Types */ export type StyleTransferModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; - readonly opts: StyleTransferOptions; + /** + * Input preprocessing and output postprocessing + * {@link StyleTransferOptions} (normalization back to uint8, + * interpolation). `resizeMode` is fixed to `'stretch'`. + */ + readonly modelOpts: StyleTransferOptions; }; /** @@ -66,7 +75,7 @@ export async function createStyleTransfer( */ transferStyleWorklet: (input: ImageBuffer) => ImageBuffer; }> { - const { modelPath, opts } = config; + const { modelPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); const meta = validateModelSchema( @@ -90,7 +99,7 @@ export async function createStyleTransfer( ] as const; const [tOutput, tReshape, tUint8, tChanLast, tRgba] = tensors; - const preprocessor = createImagePreprocessor(opts, inpShape); + const preprocessor = createImagePreprocessor(modelOpts, inpShape); const dispose = () => { tensors.forEach((t) => t.dispose()); @@ -108,10 +117,10 @@ export async function createStyleTransfer( try { tOutput .copyTo(tReshape) - .through(normalize, tUint8, { alpha: opts.outAlpha, beta: opts.outBeta }) + .through(normalize, tUint8, modelOpts.outNormalizeOpts) .through(toChannelsLast, tChanLast) .through(cvtColor, tRgba, 'RGB2RGBA') - .through(resize, tResize, { mode: 'stretch', interpolation: opts.outInterpolation }) + .through(resize, tResize, { mode: 'stretch', interpolation: modelOpts.outInterpolation }) .getData(data); } finally { tResize.dispose(); diff --git a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts index 857457d89f..9b5859b540 100644 --- a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts +++ b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts @@ -12,8 +12,11 @@ import { loadTokenizer } from '../tokenizer'; * @category Types */ export type TextEmbedderModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; + /** Local path or remote URL of the tokenizer file. */ readonly tokenizerPath: string; + /** Optional default prompt prefix added to input text before embedding. */ readonly defaultPrompt?: string; }; diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts index 9225c743ff..8e1dc354c2 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts @@ -33,35 +33,35 @@ const DEFAULT_DETECTION_MARGIN_MS = 100; * Tunable thresholds controlling how per-frame speech probabilities are turned * into speech {@link Segment}s. * @category Types - * @property {number} [speechThreshold] - Minimum speech probability (0-1) for a - * frame to count as speech. Defaults to `0.6`. - * @property {number} [minSpeechDurationMs] - Minimum duration a region must stay - * above the threshold to open a segment. Defaults to `250`. - * @property {number} [minSilenceDurationMs] - Minimum duration below the - * threshold required to close a segment. Defaults to `100`. - * @property {number} [speechPadMs] - Padding added to both ends of every - * detected segment. Defaults to `30`. - * @property {number} [mergeGapMs] - Segments closer than this gap are merged - * into one. Defaults to `0`. */ export type VadOptions = { + /** Minimum speech probability (0-1) for a frame to count as speech. */ readonly speechThreshold?: number; + /** + * Minimum duration a region must stay above the threshold to open a + * segment. + */ readonly minSpeechDurationMs?: number; + /** Minimum duration below threshold required to close a segment. */ readonly minSilenceDurationMs?: number; + /** Padding added to both ends of every detected segment. */ readonly speechPadMs?: number; + /** Segments closer than this gap are merged into one. */ readonly mergeGapMs?: number; }; /** * Model configuration required to instantiate an FSMN-VAD task runner. * @category Types - * @property {string} modelPath - Local path or remote URL of the `.pte` model. - * @property {Required} defaultOptions - Detection thresholds tuned - * for this model, overridable per `detectVoice` call. Defined alongside the - * model in the `models` registry so the defaults are discoverable there. */ export type FsmnVadModel = { + /** Local path or remote URL of the `.pte` model. */ readonly modelPath: string; + /** + * Detection thresholds tuned for this model, overridable per `detectVoice` + * call. Defined alongside the model in the `models` registry so defaults + * are discoverable there. + */ readonly defaultOptions: Required; }; @@ -70,7 +70,9 @@ export type FsmnVadModel = { * @category Types */ export type Segment = { + /** Start time of the speech segment in seconds. */ readonly start: number; + /** End time of the speech segment in seconds. */ readonly end: number; }; @@ -78,11 +80,12 @@ export type Segment = { * Options controlling live detection via `detectVoiceOnStream`. Extends the * per-call detection thresholds ({@link VadOptions}). * @category Types - * @property {number} [detectionMargin] - How recent (in milliseconds) the last - * detected speech segment must reach toward the end of the window for speech to - * still be considered ongoing. Defaults to `100`. */ export type VadStreamOptions = VadOptions & { + /** + * How recent (in milliseconds) the last detected speech segment must reach + * toward the end of the window for speech to still be considered ongoing. + */ readonly detectionMargin?: number; }; diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts index dcd71f9bc4..f5d0db9bbe 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts @@ -55,11 +55,13 @@ export type WhisperLanguage = (typeof WHISPER_LANGUAGES)[number]; /** * Options passed to a single transcription call. * @category Types - * @property language - Whisper language code of the spoken audio. Must be one of - * the {@link WhisperLanguage} values declared in the model's - * `supportedLanguages` list. */ export type WhisperSttOptions = { + /** + * Whisper language code of the spoken audio. Must be one of + * the {@link WhisperLanguage} values declared in the model's + * `supportedLanguages` list. + */ readonly language: L; }; @@ -67,20 +69,28 @@ export type WhisperSttOptions = { * Options for the live-streaming transcription API. * Extends {@link WhisperSttOptions} with optional VAD tuning. * @category Types - * @property vadOptions - Fine-tuning knobs forwarded to the voice-activity - * detector. Omit to use the detector's built-in defaults. */ export type WhisperStreamOptions = - WhisperSttOptions & { readonly vadOptions?: VadStreamOptions }; + WhisperSttOptions & { + /** + * Fine-tuning knobs forwarded to the voice-activity detector. Omit to use + * built-in defaults. + */ + readonly vadOptions?: VadStreamOptions; + }; /** * Paths and metadata required to instantiate a Whisper speech-to-text model. * @category Types */ export type WhisperSttModel = { + /** Local path or remote URL of the `.pte` model. */ readonly modelPath: string; + /** Local path or remote URL of the tokenizer file. */ readonly tokenizerPath: string; + /** List of supported language codes for this model. */ readonly supportedLanguages: readonly L[]; + /** VAD model configuration used for speech segmentation. */ readonly vadModel: FsmnVadModel; }; diff --git a/packages/react-native-executorch/src/extensions/speech/utils/vadUtils.ts b/packages/react-native-executorch/src/extensions/speech/utils/vadUtils.ts index 7b39705d64..f89207f4e3 100644 --- a/packages/react-native-executorch/src/extensions/speech/utils/vadUtils.ts +++ b/packages/react-native-executorch/src/extensions/speech/utils/vadUtils.ts @@ -4,13 +4,13 @@ import { type Tensor } from '../../../core/tensor'; /** * Options controlling how {@link extractFrames} slices and filters the waveform. * @category Types - * @property {number} numFrames - Number of frames to write (must be `<= dst.shape[0]`). - * @property {number} hopLength - Samples between consecutive frames. - * @property {number} preemphasis - Pre-emphasis filter coefficient. */ export type ExtractFramesOptions = { + /** Number of audio frames to extract and write into the destination tensor. */ readonly numFrames: number; + /** Number of samples between consecutive frames. */ readonly hopLength: number; + /** Pre-emphasis filter coefficient. */ readonly preemphasis: number; }; @@ -27,6 +27,10 @@ export type ExtractFramesOptions = { * @param hann Precomputed Hann window, shape `[frameLength]`. * @param dst Pre-allocated destination, shape `[frames, fftLength]`. * @param options Framing options. + * @param options.numFrames Number of frames to write (must not exceed `dst` + * tensor's first dimension `dst.shape[0]`). + * @param options.hopLength Number of audio samples between consecutive frames. + * @param options.preemphasis Pre-emphasis filter coefficient. * @returns The `dst` tensor, for convenience. */ export function extractFrames( diff --git a/packages/react-native-executorch/src/hooks/useClassifier.ts b/packages/react-native-executorch/src/hooks/useClassifier.ts index 574b040ea8..afedaac569 100644 --- a/packages/react-native-executorch/src/hooks/useClassifier.ts +++ b/packages/react-native-executorch/src/hooks/useClassifier.ts @@ -34,7 +34,7 @@ export function useClassifier(config: ClassifierModel, options?: { prevent error: downloadError || error, downloadProgress, localPath, - labels: config.classifierOpts.labels, + labels: config.modelOpts.labels, classify: model?.classify, classifyWorklet: model?.classifyWorklet, }; diff --git a/packages/react-native-executorch/src/hooks/useInstanceSegmenter.ts b/packages/react-native-executorch/src/hooks/useInstanceSegmenter.ts index 364f2b32a7..463f57a0d2 100644 --- a/packages/react-native-executorch/src/hooks/useInstanceSegmenter.ts +++ b/packages/react-native-executorch/src/hooks/useInstanceSegmenter.ts @@ -44,6 +44,6 @@ export function useInstanceSegmenter( localPath, segmentInstances: model?.segmentInstances, segmentInstancesWorklet: model?.segmentInstancesWorklet, - labels: config.opts.labels, + labels: config.modelOpts.labels, }; } diff --git a/packages/react-native-executorch/src/hooks/useKeypointDetector.ts b/packages/react-native-executorch/src/hooks/useKeypointDetector.ts index 6da33eb842..dabe3d1b3c 100644 --- a/packages/react-native-executorch/src/hooks/useKeypointDetector.ts +++ b/packages/react-native-executorch/src/hooks/useKeypointDetector.ts @@ -42,7 +42,7 @@ export function useKeypointDetector( error: downloadError || error, downloadProgress, localPath, - landmarks: config.opts.landmarks, + landmarks: config.modelOpts.landmarks, detectKeypoints: model?.detectKeypoints, detectKeypointsWorklet: model?.detectKeypointsWorklet, }; diff --git a/packages/react-native-executorch/src/hooks/useObjectDetector.ts b/packages/react-native-executorch/src/hooks/useObjectDetector.ts index 18e3ace827..138d9023f2 100644 --- a/packages/react-native-executorch/src/hooks/useObjectDetector.ts +++ b/packages/react-native-executorch/src/hooks/useObjectDetector.ts @@ -42,7 +42,7 @@ export function useObjectDetector( error: downloadError || error, downloadProgress, localPath, - labels: config.opts.labels, + labels: config.modelOpts.labels, detectObjects: model?.detectObjects, detectObjectsWorklet: model?.detectObjectsWorklet, }; diff --git a/packages/react-native-executorch/src/hooks/useSemanticSegmenter.ts b/packages/react-native-executorch/src/hooks/useSemanticSegmenter.ts index 8140204e18..d12f9faa91 100644 --- a/packages/react-native-executorch/src/hooks/useSemanticSegmenter.ts +++ b/packages/react-native-executorch/src/hooks/useSemanticSegmenter.ts @@ -2,7 +2,7 @@ import { useModel } from './useModel'; import { useResourceDownload } from './useResourceDownload'; import { createSemanticSegmenter, - type SemanticSegmentationModel, + type SemanticSegmenterModel, } from '../extensions/cv/tasks/semanticSegmentation'; /** @@ -22,7 +22,7 @@ import { * progress, and segmentation functions. */ export function useSemanticSegmenter( - config: SemanticSegmentationModel, + config: SemanticSegmenterModel, options?: { preventLoad?: boolean } ) { const { localPath, downloadProgress, downloadError } = useResourceDownload( @@ -42,6 +42,6 @@ export function useSemanticSegmenter( localPath, segment: model?.segment, segmentWorklet: model?.segmentWorklet, - labels: config.opts.labels, + labels: config.modelOpts.labels, }; } diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index fc8e7a5e7d..a27b743ae4 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1,7 +1,7 @@ import type { ClassifierModel } from './extensions/cv/tasks/classification'; import type { ObjectDetectorModel } from './extensions/cv/tasks/objectDetection'; import type { StyleTransferModel } from './extensions/cv/tasks/styleTransfer'; -import type { SemanticSegmentationModel } from './extensions/cv/tasks/semanticSegmentation'; +import type { SemanticSegmenterModel } from './extensions/cv/tasks/semanticSegmentation'; import type { KeypointDetectorModel } from './extensions/cv/tasks/keypointDetection'; import type { InstanceSegmenterModel } from './extensions/cv/tasks/instanceSegmentation'; import type { ImageEmbedderModel } from './extensions/cv/tasks/imageEmbedding'; @@ -38,21 +38,20 @@ const NEXT_VERSION_TAG = 'resolve/v0.10.0'; const EFFICIENTNET_V2_S_OPTS = { resizeMode: 'stretch' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, labels: IMAGENET1K_LABELS, }; const EFFICIENTNET_V2_S_XNNPACK_INT8: ClassifierModel = { modelPath: `${BASE_URL}-efficientnet-v2-s/${VERSION_TAG}/xnnpack/efficientnet_v2_s_xnnpack_int8.pte`, - classifierOpts: EFFICIENTNET_V2_S_OPTS, + modelOpts: EFFICIENTNET_V2_S_OPTS, }; const EFFICIENTNET_V2_S_XNNPACK_FP32: ClassifierModel = { modelPath: `${BASE_URL}-efficientnet-v2-s/${VERSION_TAG}/xnnpack/efficientnet_v2_s_xnnpack_fp32.pte`, - classifierOpts: EFFICIENTNET_V2_S_OPTS, + modelOpts: EFFICIENTNET_V2_S_OPTS, }; const EFFICIENTNET_V2_S_COREML_FP16: ClassifierModel = { modelPath: `${BASE_URL}-efficientnet-v2-s/${VERSION_TAG}/coreml/efficientnet_v2_s_coreml_fp16.pte`, - classifierOpts: EFFICIENTNET_V2_S_OPTS, + modelOpts: EFFICIENTNET_V2_S_OPTS, }; // ============================================================================= @@ -61,88 +60,85 @@ const EFFICIENTNET_V2_S_COREML_FP16: ClassifierModel = { const STYLE_TRANSFER_OPTS = { resizeMode: 'stretch' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, - outAlpha: 255.0, - outBeta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, + outNormalizeOpts: { alpha: 255.0, beta: 0.0 }, outInterpolation: 'lanczos' as const, }; const STYLE_TRANSFER_CANDY_XNNPACK_FP32: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-candy/${VERSION_TAG}/xnnpack/style_transfer_candy_xnnpack_fp32.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_CANDY_XNNPACK_INT8: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-candy/${VERSION_TAG}/xnnpack/style_transfer_candy_xnnpack_int8.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_CANDY_COREML_FP16: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-candy/${VERSION_TAG}/coreml/style_transfer_candy_coreml_fp16.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_CANDY_COREML_FP32: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-candy/${VERSION_TAG}/coreml/style_transfer_candy_coreml_fp32.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_MOSAIC_XNNPACK_FP32: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-mosaic/${VERSION_TAG}/xnnpack/style_transfer_mosaic_xnnpack_fp32.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_MOSAIC_XNNPACK_INT8: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-mosaic/${VERSION_TAG}/xnnpack/style_transfer_mosaic_xnnpack_int8.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_MOSAIC_COREML_FP16: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-mosaic/${VERSION_TAG}/coreml/style_transfer_mosaic_coreml_fp16.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_MOSAIC_COREML_FP32: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-mosaic/${VERSION_TAG}/coreml/style_transfer_mosaic_coreml_fp32.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_RAIN_PRINCESS_XNNPACK_FP32: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-rain-princess/${VERSION_TAG}/xnnpack/style_transfer_rain_princess_xnnpack_fp32.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_RAIN_PRINCESS_XNNPACK_INT8: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-rain-princess/${VERSION_TAG}/xnnpack/style_transfer_rain_princess_xnnpack_int8.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_RAIN_PRINCESS_COREML_FP16: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-rain-princess/${VERSION_TAG}/coreml/style_transfer_rain_princess_coreml_fp16.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_RAIN_PRINCESS_COREML_FP32: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-rain-princess/${VERSION_TAG}/coreml/style_transfer_rain_princess_coreml_fp32.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_UDNIE_XNNPACK_FP32: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-udnie/${VERSION_TAG}/xnnpack/style_transfer_udnie_xnnpack_fp32.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_UDNIE_XNNPACK_INT8: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-udnie/${VERSION_TAG}/xnnpack/style_transfer_udnie_xnnpack_int8.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_UDNIE_COREML_FP16: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-udnie/${VERSION_TAG}/coreml/style_transfer_udnie_coreml_fp16.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; const STYLE_TRANSFER_UDNIE_COREML_FP32: StyleTransferModel = { modelPath: `${BASE_URL}-style-transfer-udnie/${VERSION_TAG}/coreml/style_transfer_udnie_coreml_fp32.pte`, - opts: STYLE_TRANSFER_OPTS, + modelOpts: STYLE_TRANSFER_OPTS, }; // ============================================================================= // Semantic Segmentation // ============================================================================= -const SELFIE_SEGMENTATION_XNNPACK_FP32: SemanticSegmentationModel<'background' | 'person'> = { +const SELFIE_SEGMENTATION_XNNPACK_FP32: SemanticSegmenterModel<'background' | 'person'> = { modelPath: `${BASE_URL}-selfie-segmentation/${VERSION_TAG}/xnnpack/selfie_segmentation_xnnpack_fp32.pte`, - opts: { + modelOpts: { labels: ['background', 'person'] as const, resizeMode: 'stretch', interpolation: 'linear', - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, outInterpolation: 'lanczos', }, }; @@ -152,15 +148,15 @@ const LRASPP_MOBILENET_V3_LARGE_OPTS = { resizeMode: 'stretch' as const, interpolation: 'linear' as const, outInterpolation: 'lanczos' as const, - ...IMAGENET_NORM, + normalizeOpts: IMAGENET_NORM, }; -const LRASPP_MOBILENET_V3_LARGE_XNNPACK_FP32: SemanticSegmentationModel = { +const LRASPP_MOBILENET_V3_LARGE_XNNPACK_FP32: SemanticSegmenterModel = { modelPath: `${BASE_URL}-lraspp/${VERSION_TAG}/xnnpack/lraspp_mobilenet_v3_large_xnnpack_fp32.pte`, - opts: LRASPP_MOBILENET_V3_LARGE_OPTS, + modelOpts: LRASPP_MOBILENET_V3_LARGE_OPTS, }; -const LRASPP_MOBILENET_V3_LARGE_XNNPACK_INT8: SemanticSegmentationModel = { +const LRASPP_MOBILENET_V3_LARGE_XNNPACK_INT8: SemanticSegmenterModel = { modelPath: `${BASE_URL}-lraspp/${VERSION_TAG}/xnnpack/lraspp_mobilenet_v3_large_xnnpack_int8.pte`, - opts: LRASPP_MOBILENET_V3_LARGE_OPTS, + modelOpts: LRASPP_MOBILENET_V3_LARGE_OPTS, }; const DEEPLAB_V3_OPTS = { @@ -168,31 +164,31 @@ const DEEPLAB_V3_OPTS = { resizeMode: 'stretch' as const, interpolation: 'linear' as const, outInterpolation: 'lanczos' as const, - ...IMAGENET_NORM, + normalizeOpts: IMAGENET_NORM, }; -const DEEPLAB_V3_RESNET50_XNNPACK_FP32: SemanticSegmentationModel = { +const DEEPLAB_V3_RESNET50_XNNPACK_FP32: SemanticSegmenterModel = { modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_resnet50_xnnpack_fp32.pte`, - opts: DEEPLAB_V3_OPTS, + modelOpts: DEEPLAB_V3_OPTS, }; -const DEEPLAB_V3_RESNET50_XNNPACK_INT8: SemanticSegmentationModel = { +const DEEPLAB_V3_RESNET50_XNNPACK_INT8: SemanticSegmenterModel = { modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_resnet50_xnnpack_int8.pte`, - opts: DEEPLAB_V3_OPTS, + modelOpts: DEEPLAB_V3_OPTS, }; -const DEEPLAB_V3_RESNET101_XNNPACK_FP32: SemanticSegmentationModel = { +const DEEPLAB_V3_RESNET101_XNNPACK_FP32: SemanticSegmenterModel = { modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_resnet101_xnnpack_fp32.pte`, - opts: DEEPLAB_V3_OPTS, + modelOpts: DEEPLAB_V3_OPTS, }; -const DEEPLAB_V3_RESNET101_XNNPACK_INT8: SemanticSegmentationModel = { +const DEEPLAB_V3_RESNET101_XNNPACK_INT8: SemanticSegmenterModel = { modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_resnet101_xnnpack_int8.pte`, - opts: DEEPLAB_V3_OPTS, + modelOpts: DEEPLAB_V3_OPTS, }; -const DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_FP32: SemanticSegmentationModel = { +const DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_FP32: SemanticSegmenterModel = { modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_mobilenet_v3_large_xnnpack_fp32.pte`, - opts: DEEPLAB_V3_OPTS, + modelOpts: DEEPLAB_V3_OPTS, }; -const DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_INT8: SemanticSegmentationModel = { +const DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_INT8: SemanticSegmenterModel = { modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_mobilenet_v3_large_xnnpack_int8.pte`, - opts: DEEPLAB_V3_OPTS, + modelOpts: DEEPLAB_V3_OPTS, }; const FCN_OPTS = { @@ -200,23 +196,23 @@ const FCN_OPTS = { resizeMode: 'stretch' as const, interpolation: 'linear' as const, outInterpolation: 'lanczos' as const, - ...IMAGENET_NORM, + normalizeOpts: IMAGENET_NORM, }; -const FCN_RESNET50_XNNPACK_FP32: SemanticSegmentationModel = { +const FCN_RESNET50_XNNPACK_FP32: SemanticSegmenterModel = { modelPath: `${BASE_URL}-fcn/${NEXT_VERSION_TAG}/xnnpack/fcn_resnet50_xnnpack_fp32.pte`, - opts: FCN_OPTS, + modelOpts: FCN_OPTS, }; -const FCN_RESNET50_XNNPACK_INT8: SemanticSegmentationModel = { +const FCN_RESNET50_XNNPACK_INT8: SemanticSegmenterModel = { modelPath: `${BASE_URL}-fcn/${NEXT_VERSION_TAG}/xnnpack/fcn_resnet50_xnnpack_int8.pte`, - opts: FCN_OPTS, + modelOpts: FCN_OPTS, }; -const FCN_RESNET101_XNNPACK_FP32: SemanticSegmentationModel = { +const FCN_RESNET101_XNNPACK_FP32: SemanticSegmenterModel = { modelPath: `${BASE_URL}-fcn/${NEXT_VERSION_TAG}/xnnpack/fcn_resnet101_xnnpack_fp32.pte`, - opts: FCN_OPTS, + modelOpts: FCN_OPTS, }; -const FCN_RESNET101_XNNPACK_INT8: SemanticSegmentationModel = { +const FCN_RESNET101_XNNPACK_INT8: SemanticSegmenterModel = { modelPath: `${BASE_URL}-fcn/${NEXT_VERSION_TAG}/xnnpack/fcn_resnet101_xnnpack_int8.pte`, - opts: FCN_OPTS, + modelOpts: FCN_OPTS, }; // ============================================================================= @@ -227,22 +223,21 @@ const SSDLITE320_MOBILENET_V3_LARGE_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'stretch' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, defaultConfidenceThreshold: 0.5, defaultIouThreshold: 0.55, }; const SSDLITE320_MOBILENET_V3_LARGE_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClass> = { modelPath: `${BASE_URL}-ssdlite320-mobilenet-v3-large/${VERSION_TAG}/xnnpack/ssdlite320_mobilenet_v3_large_xnnpack_fp32.pte`, - opts: SSDLITE320_MOBILENET_V3_LARGE_OPTS, + modelOpts: SSDLITE320_MOBILENET_V3_LARGE_OPTS, }; const SSDLITE320_MOBILENET_V3_LARGE_COREML_FP16: ObjectDetectorModel<'xyxy', CocoClass> = { modelPath: `${BASE_URL}-ssdlite320-mobilenet-v3-large/${VERSION_TAG}/coreml/ssdlite320_mobilenet_v3_large_coreml_fp16.pte`, - opts: SSDLITE320_MOBILENET_V3_LARGE_OPTS, + modelOpts: SSDLITE320_MOBILENET_V3_LARGE_OPTS, }; const SSDLITE320_MOBILENET_V3_LARGE_COREML_FP32: ObjectDetectorModel<'xyxy', CocoClass> = { modelPath: `${BASE_URL}-ssdlite320-mobilenet-v3-large/${VERSION_TAG}/coreml/ssdlite320_mobilenet_v3_large_coreml_fp32.pte`, - opts: SSDLITE320_MOBILENET_V3_LARGE_OPTS, + modelOpts: SSDLITE320_MOBILENET_V3_LARGE_OPTS, }; const RFDETR_NANO_DETECTOR_OPTS = { @@ -250,17 +245,17 @@ const RFDETR_NANO_DETECTOR_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'stretch' as const, interpolation: 'linear' as const, - ...IMAGENET_NORM, + normalizeOpts: IMAGENET_NORM, defaultConfidenceThreshold: 0.5, defaultIouThreshold: 0.55, }; const RFDETR_NANO_DETECTOR_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClass> = { modelPath: `${BASE_URL}-rfdetr-nano-detector/${VERSION_TAG}/xnnpack/rfdetr_nano_xnnpack_fp32.pte`, - opts: RFDETR_NANO_DETECTOR_OPTS, + modelOpts: RFDETR_NANO_DETECTOR_OPTS, }; const RFDETR_NANO_DETECTOR_COREML_INT8: ObjectDetectorModel<'xyxy', CocoClass> = { modelPath: `${BASE_URL}-rfdetr-nano-detector/${VERSION_TAG}/coreml/rfdetr_nano_coreml_int8.pte`, - opts: RFDETR_NANO_DETECTOR_OPTS, + modelOpts: RFDETR_NANO_DETECTOR_OPTS, }; const YOLO26_DETECTOR_OPTS = { @@ -268,75 +263,74 @@ const YOLO26_DETECTOR_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'letterbox' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, defaultConfidenceThreshold: 0.25, defaultIouThreshold: 0.7, }; const YOLO26_NANO_384_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/n/xnnpack/yolo26n_384_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_NANO_512_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/n/xnnpack/yolo26n_512_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_NANO_640_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/n/xnnpack/yolo26n_640_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_SMALL_384_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/s/xnnpack/yolo26s_384_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_SMALL_512_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/s/xnnpack/yolo26s_512_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_SMALL_640_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/s/xnnpack/yolo26s_640_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_MEDIUM_384_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/m/xnnpack/yolo26m_384_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_MEDIUM_512_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/m/xnnpack/yolo26m_512_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_MEDIUM_640_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/m/xnnpack/yolo26m_640_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_LARGE_384_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/l/xnnpack/yolo26l_384_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_LARGE_512_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/l/xnnpack/yolo26l_512_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_LARGE_640_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/l/xnnpack/yolo26l_640_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_XLARGE_384_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/x/xnnpack/yolo26x_384_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_XLARGE_512_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/x/xnnpack/yolo26x_512_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; const YOLO26_XLARGE_640_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26/${NEXT_VERSION_TAG}/x/xnnpack/yolo26x_640_xnnpack_fp32.pte`, - opts: YOLO26_DETECTOR_OPTS, + modelOpts: YOLO26_DETECTOR_OPTS, }; // ============================================================================= @@ -344,12 +338,11 @@ const YOLO26_XLARGE_640_XNNPACK_FP32: ObjectDetectorModel<'xyxy', CocoClassYolo> // ============================================================================= const BLAZEFACE_XNNPACK_FP32: KeypointDetectorModel<'xyxy', BlazeFaceLandmark> = { modelPath: `${BASE_URL}-blazeface/${NEXT_VERSION_TAG}/xnnpack/blazeface_xnnpack_fp32.pte`, - opts: { + modelOpts: { boxFormat: 'xyxy', resizeMode: 'letterbox', interpolation: 'linear', - alpha: 1 / 127.5, - beta: -1.0, + normalizeOpts: { alpha: 1 / 127.5, beta: -1.0 }, defaultIouThreshold: 0.3, defaultConfidenceThreshold: 0.75, landmarks: BLAZEFACE_LANDMARKS, @@ -360,45 +353,44 @@ const YOLO26_POSE_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'letterbox' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, defaultIouThreshold: 0.7, defaultConfidenceThreshold: 0.25, landmarks: COCO_LANDMARKS, }; const YOLO26_POSE_384_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/xnnpack/yolo26n_pose_384_xnnpack_fp32.pte`, - opts: YOLO26_POSE_OPTS, + modelOpts: YOLO26_POSE_OPTS, }; const YOLO26_POSE_512_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/xnnpack/yolo26n_pose_512_xnnpack_fp32.pte`, - opts: YOLO26_POSE_OPTS, + modelOpts: YOLO26_POSE_OPTS, }; const YOLO26_POSE_640_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-yolo26-pose/${NEXT_VERSION_TAG}/xnnpack/yolo26n_pose_640_xnnpack_fp32.pte`, - opts: YOLO26_POSE_OPTS, + modelOpts: YOLO26_POSE_OPTS, }; const RFDETR_KEYPOINT_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'stretch' as const, interpolation: 'linear' as const, - ...IMAGENET_NORM, + normalizeOpts: IMAGENET_NORM, defaultIouThreshold: 0.55, defaultConfidenceThreshold: 0.5, landmarks: COCO_LANDMARKS, }; const RFDETR_KEYPOINT_XNNPACK_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/xnnpack/rfdetr_keypoint_preview_xnnpack_fp32.pte`, - opts: RFDETR_KEYPOINT_OPTS, + modelOpts: RFDETR_KEYPOINT_OPTS, }; const RFDETR_KEYPOINT_COREML_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/coreml/rfdetr_keypoint_preview_coreml_fp32.pte`, - opts: RFDETR_KEYPOINT_OPTS, + modelOpts: RFDETR_KEYPOINT_OPTS, }; const RFDETR_KEYPOINT_MLX_FP32: KeypointDetectorModel<'xyxy', CocoLandmark> = { modelPath: `${BASE_URL}-rfdetr-keypoint/${VERSION_TAG}/preview/mlx/rfdetr_keypoint_preview_mlx_fp32.pte`, - opts: RFDETR_KEYPOINT_OPTS, + modelOpts: RFDETR_KEYPOINT_OPTS, }; // ============================================================================= @@ -409,35 +401,34 @@ const FASTSAM_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'stretch' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, defaultConfidenceThreshold: 0.5, defaultIouThreshold: 0.9, defaultMaskThreshold: 0.5, }; const FASTSAM_S_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', 'object'> = { modelPath: `${BASE_URL}-fast-sam/${NEXT_VERSION_TAG}/s/xnnpack/fast_sam_s_xnnpack_fp32.pte`, - opts: FASTSAM_OPTS, + modelOpts: FASTSAM_OPTS, }; const FASTSAM_S_COREML_FP32: InstanceSegmenterModel<'xyxy', 'object'> = { modelPath: `${BASE_URL}-fast-sam/${NEXT_VERSION_TAG}/s/coreml/fast_sam_s_coreml_fp32.pte`, - opts: FASTSAM_OPTS, + modelOpts: FASTSAM_OPTS, }; const FASTSAM_S_COREML_FP16: InstanceSegmenterModel<'xyxy', 'object'> = { modelPath: `${BASE_URL}-fast-sam/${NEXT_VERSION_TAG}/s/coreml/fast_sam_s_coreml_fp16.pte`, - opts: FASTSAM_OPTS, + modelOpts: FASTSAM_OPTS, }; const FASTSAM_X_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', 'object'> = { modelPath: `${BASE_URL}-fast-sam/${NEXT_VERSION_TAG}/x/xnnpack/fast_sam_x_xnnpack_fp32.pte`, - opts: FASTSAM_OPTS, + modelOpts: FASTSAM_OPTS, }; const FASTSAM_X_COREML_FP32: InstanceSegmenterModel<'xyxy', 'object'> = { modelPath: `${BASE_URL}-fast-sam/${NEXT_VERSION_TAG}/x/coreml/fast_sam_x_coreml_fp32.pte`, - opts: FASTSAM_OPTS, + modelOpts: FASTSAM_OPTS, }; const FASTSAM_X_COREML_FP16: InstanceSegmenterModel<'xyxy', 'object'> = { modelPath: `${BASE_URL}-fast-sam/${NEXT_VERSION_TAG}/x/coreml/fast_sam_x_coreml_fp16.pte`, - opts: FASTSAM_OPTS, + modelOpts: FASTSAM_OPTS, }; const RFDETR_NANO_SEG_OPTS = { @@ -445,18 +436,18 @@ const RFDETR_NANO_SEG_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'stretch' as const, interpolation: 'linear' as const, - ...IMAGENET_NORM, + normalizeOpts: IMAGENET_NORM, defaultConfidenceThreshold: 0.5, defaultIouThreshold: 0.55, defaultMaskThreshold: 0.5, }; const RFDETR_NANO_SEG_COREML_INT8: InstanceSegmenterModel<'xyxy', CocoClass> = { modelPath: `${BASE_URL}-rfdetr-nano-segmentation/${NEXT_VERSION_TAG}/coreml/rfdetr_nano_coreml_int8.pte`, - opts: RFDETR_NANO_SEG_OPTS, + modelOpts: RFDETR_NANO_SEG_OPTS, }; const RFDETR_NANO_SEG_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClass> = { modelPath: `${BASE_URL}-rfdetr-nano-segmentation/${NEXT_VERSION_TAG}/xnnpack/rfdetr_nano_xnnpack_fp32.pte`, - opts: RFDETR_NANO_SEG_OPTS, + modelOpts: RFDETR_NANO_SEG_OPTS, }; const YOLO26_SEG_OPTS = { @@ -464,8 +455,7 @@ const YOLO26_SEG_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'stretch' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, defaultConfidenceThreshold: 0.25, defaultIouThreshold: 0.7, defaultMaskThreshold: 0.5, @@ -473,67 +463,67 @@ const YOLO26_SEG_OPTS = { const YOLO26_NANO_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/xnnpack/yolo26_seg_n_384_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_NANO_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/xnnpack/yolo26_seg_n_512_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_NANO_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/n/xnnpack/yolo26_seg_n_640_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_SMALL_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/xnnpack/yolo26_seg_s_384_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_SMALL_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/xnnpack/yolo26_seg_s_512_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_SMALL_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/s/xnnpack/yolo26_seg_s_640_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_MEDIUM_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/xnnpack/yolo26_seg_m_384_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_MEDIUM_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/xnnpack/yolo26_seg_m_512_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_MEDIUM_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/m/xnnpack/yolo26_seg_m_640_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_LARGE_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/xnnpack/yolo26_seg_l_384_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_LARGE_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/xnnpack/yolo26_seg_l_512_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_LARGE_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/l/xnnpack/yolo26_seg_l_640_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_XLARGE_SEG_384_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/xnnpack/yolo26_seg_x_384_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_XLARGE_SEG_512_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/xnnpack/yolo26_seg_x_512_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; const YOLO26_XLARGE_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoClassYolo> = { modelPath: `${BASE_URL}-yolo26-seg/${NEXT_VERSION_TAG}/x/xnnpack/yolo26_seg_x_640_xnnpack_fp32.pte`, - opts: YOLO26_SEG_OPTS, + modelOpts: YOLO26_SEG_OPTS, }; // ============================================================================= @@ -584,16 +574,15 @@ const LFM2_5_EMBEDDING_350M_MLX_INT4: TextEmbedderModel = { const CLIP_IMAGE_EMBEDDINGS_OPTS = { resizeMode: 'stretch' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, }; const CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32: ImageEmbedderModel = { modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_image_xnnpack_fp32.pte`, - opts: CLIP_IMAGE_EMBEDDINGS_OPTS, + modelOpts: CLIP_IMAGE_EMBEDDINGS_OPTS, }; const CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_INT8: ImageEmbedderModel = { modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_image_xnnpack_int8.pte`, - opts: CLIP_IMAGE_EMBEDDINGS_OPTS, + modelOpts: CLIP_IMAGE_EMBEDDINGS_OPTS, }; // =============================================================================