Skip to content
40 changes: 34 additions & 6 deletions packages/react-native-executorch/src/extensions/cv/ops/boxes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,24 @@ import { scalePoint } from './points';
* @category Types
*/
export type BoxMap = {
xyxy: { xmin: number; ymin: number; xmax: number; ymax: number };
xywh: { xmin: number; ymin: number; w: number; h: number };
cxcywh: { cx: number; cy: number; w: number; h: number };
xyxy: {
readonly xmin: number;
readonly ymin: number;
readonly xmax: number;
readonly ymax: number;
};
xywh: {
readonly xmin: number;
readonly ymin: number;
readonly w: number;
readonly h: number;
};
cxcywh: {
readonly cx: number;
readonly cy: number;
readonly w: number;
readonly h: number;
};
};

/**
Expand Down Expand Up @@ -60,8 +75,8 @@ export function decodeBox<F extends BoxFormat>(
* @param opts Options defining dimensions and resize modes.
* @param opts.from The source bounds (e.g. model input dimensions).
* @param opts.to The destination bounds (e.g. original image dimensions).
* @param opts.resizeMode The mode used to resize the image ('letterbox' or
* 'stretch').
* @param opts.resizeMode The mode used to resize the image {@link ResizeMode}
* (excluding `'crop'`).
* @returns The scaled BoundingBox object.
*/
export function scaleBox<F extends BoxFormat>(
Expand Down Expand Up @@ -130,9 +145,16 @@ export function scaleBox<F extends BoxFormat>(
* @category Types
*/
export type NmsOptions = {
/** How bounding box coordinates are interpreted {@link BoxFormat}. */
readonly boxFormat: BoxFormat;
/** Intersection over Union (IoU) threshold for suppressing overlapping boxes. */
readonly iouThreshold: number;
/** Minimum confidence score threshold for filtering candidate boxes. */
readonly confidenceThreshold: number;
/**
* NMS algorithm variant (`standard` for hard suppression, `weighted` for soft
* coordinate averaging).
*/
readonly nmsType: 'standard' | 'weighted';
};

Expand All @@ -142,7 +164,13 @@ export type NmsOptions = {
* @category Utils
* @param boxes Bounding boxes coordinate tensor.
* @param scores Bounding boxes confidence scores tensor.
* @param opts Options configure NMS thresholds and execution mode.
* @param opts Options configuring NMS thresholds and execution mode.
* @param opts.boxFormat The bounding box format {@link BoxFormat}.
* @param opts.iouThreshold Intersection over Union (IoU) threshold for
* suppression.
* @param opts.confidenceThreshold Minimum confidence score for candidate
* selection.
* @param opts.nmsType The NMS algorithm variant {@link NmsOptions.nmsType}.
* @returns The resulting indices of the non-suppressed boxes:
* - For `standard` NMS: A 1D array of indices (`number[]`) representing the
* selected boxes.
Expand Down
19 changes: 17 additions & 2 deletions packages/react-native-executorch/src/extensions/cv/ops/image.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,11 @@ export type InterpolationMethod = 'nearest' | 'area' | 'cubic' | 'lanczos' | 'li
* @category Types
*/
export type ResizeOptions = {
/** How the image is resized {@link ResizeMode}. */
readonly mode?: ResizeMode;
/** Background fill value used when letterboxing. */
readonly padValue?: number;
/** Pixel interpolation method {@link InterpolationMethod}. */
readonly interpolation?: InterpolationMethod;
};

Expand All @@ -84,21 +87,30 @@ export type ResizeOptions = {
* @category Types
*/
export type NormalizeOptions = {
/**
* Multiplicative coefficient applied as `pixel * alpha`. Single value for
* uniform scaling, array for per-channel.
*/
readonly alpha?: number | readonly number[];
/** Additive offset applied as `pixel * alpha + beta`. Single value or per-channel array. */
readonly beta?: number | readonly number[];
};

/**
* Resizes an image tensor from a source dimension to a destination dimension.
*
* Supports various resize modes (`stretch`, `letterbox`, `crop`) and
* interpolation algorithms (`linear`, `lanczos`, etc.).
* Supports various {@link ResizeMode} and {@link InterpolationMethod} options.
* @category Typescript API
* @param src The source image tensor in HWC layout. Shape [H,W,C].
* @param dst The pre-allocated destination tensor to write the resized image
* to. `dst` must be in HWC layout and its number of channels must match `src`.
* Shape [H',W',C].
* @param opts Configuration options for resizing.
* @param opts.mode The resize algorithm mode {@link ResizeMode}. Defaults to
* `'stretch'`.
* @param opts.interpolation The pixel interpolation method
* {@link InterpolationMethod}. Defaults to `'lanczos'`.
* @param opts.padValue Fill value for letterboxing. Defaults to `0`.
* @returns The destination tensor containing the resized image.
*/
export function resize(src: Tensor, dst: Tensor, opts?: ResizeOptions): Tensor {
Expand Down Expand Up @@ -173,6 +185,9 @@ export function toChannelsLast(src: Tensor, dst: Tensor): Tensor {
* @param dst The pre-allocated destination tensor to write the normalized
* values to. `dst` must have the same shape as `src`. Shape [C,H,W].
* @param opts Normalization scaling coefficients.
* @param opts.alpha Multiplicative scaling coefficient(s). Defaults to
* `1 / 255.0`.
* @param opts.beta Additive offset coefficient(s). Defaults to `0.0`.
* @returns The destination tensor containing the normalized image.
*/
export function normalize(src: Tensor, dst: Tensor, opts?: NormalizeOptions): Tensor {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ export type Point = {
* @param opts Options detailing the scaling factors and resize mode.
* @param opts.from The source bounds (e.g. model input dimensions).
* @param opts.to The destination bounds (e.g. original image dimensions).
* @param opts.resizeMode The mode used to resize the image ('letterbox' or
* 'stretch').
* @param opts.resizeMode The mode used to resize the image {@link ResizeMode}
* (excluding `'crop'`).
* @returns The scaled coordinate point.
*/
export function scalePoint(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,34 @@ import { createImagePreprocessor, type ImagePreprocessorOptions } from './prepro
* vocabulary.
* @category Types
*/
export type ClassifierOptions<L> = ImagePreprocessorOptions & { readonly labels: readonly L[] };
export type ClassifierOptions<L> = ImagePreprocessorOptions & {
/** Array of class labels matching the model's output vocabulary. */
readonly labels: readonly L[];
};

/**
* Model configuration required to instantiate a classifier task runner.
* @category Types
*/
export type ClassifierModel<L> = {
/** Local path or remote URL of the `.pte` model file. */
readonly modelPath: string;
readonly classifierOpts: ClassifierOptions<L>;
/**
* Image preprocessing and label vocabulary
* {@link ClassifierOptions}. The `labels` array length must
* match the model's output dimension.
*/
readonly modelOpts: ClassifierOptions<L>;
};

/**
* Result structure representing a single classification prediction.
* @category Types
*/
export type Classification<L> = {
/** Predicted class label. */
readonly label: L;
/** Confidence score of the prediction (between 0.0 and 1.0). */
readonly confidence: number;
};

Expand Down Expand Up @@ -74,7 +85,7 @@ export async function createClassifier<L>(
*/
classifyWorklet: (input: ImageBuffer, options?: { topk?: number }) => Classification<L>[];
}> {
const { modelPath, classifierOpts } = config;
const { modelPath, modelOpts } = config;
const model = await wrapAsync(loadModel, runtime)(modelPath);

const meta = validateModelSchema(
Expand All @@ -87,9 +98,9 @@ export async function createClassifier<L>(
const outShape = meta.outputTensorMeta[0]!.shape;

const numLabels = outShape[outShape.length - 1]!;
if (classifierOpts.labels.length !== numLabels) {
if (modelOpts.labels.length !== numLabels) {
throw new Error(
`Classifier labels length (${classifierOpts.labels.length}) must match model output dimension (${numLabels}).`
`Classifier labels length (${modelOpts.labels.length}) must match model output dimension (${numLabels}).`
);
}

Expand All @@ -100,7 +111,7 @@ export async function createClassifier<L>(
] as const;

const [tLogits, tProbas] = tensors;
const preprocessor = createImagePreprocessor(classifierOpts, inpShape);
const preprocessor = createImagePreprocessor(modelOpts, inpShape);

const dispose = () => {
preprocessor.dispose();
Expand All @@ -125,7 +136,7 @@ export async function createClassifier<L>(
.getData(new Float32Array(tProbas.numel));

return Array.from(probas)
.map((confidence, index) => ({ confidence, label: classifierOpts.labels[index]! }))
.map((confidence, index) => ({ confidence, label: modelOpts.labels[index]! }))
.sort((a, b) => b.confidence - a.confidence)
.slice(0, options?.topk);
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,13 @@ import { createImagePreprocessor, type ImagePreprocessorOptions } from './prepro
* @category Types
*/
export type ImageEmbedderModel = {
/** Local path or remote URL of the `.pte` model file. */
readonly modelPath: string;
readonly opts: ImagePreprocessorOptions;
/**
* Image preprocessing (resize, color conversion, normalization)
* for embedding models {@link ImagePreprocessorOptions}.
*/
readonly modelOpts: ImagePreprocessorOptions;
};

/**
Expand Down Expand Up @@ -53,7 +58,7 @@ export async function createImageEmbedder(
*/
embedWorklet: (input: ImageBuffer) => Float32Array;
}> {
const { modelPath, opts } = config;
const { modelPath, modelOpts } = config;
const model = await wrapAsync(loadModel, runtime)(modelPath);

const meta = validateModelSchema(
Expand All @@ -67,7 +72,7 @@ export async function createImageEmbedder(

const tensors = [tensor('float32', outShape)] as const;
const [tEmbedding] = tensors;
const preprocessor = createImagePreprocessor(opts, inpShape);
const preprocessor = createImagePreprocessor(modelOpts, inpShape);

const dispose = () => {
preprocessor.dispose();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,17 @@ export type InstanceSegmenterOptions<F extends BoxFormat, L> = Omit<
ImagePreprocessorOptions,
'resizeMode'
> & {
/** Resize mode for input images. Must be `'stretch'`. */
readonly resizeMode: 'stretch';
/** Array of class labels matching the model's output vocabulary. */
readonly labels: readonly L[];
/** How bounding box coordinates are interpreted {@link BoxFormat}. */
readonly boxFormat: F;
/** Default Intersection over Union (IoU) threshold for Non-Maximum Suppression (NMS). */
readonly defaultIouThreshold: number;
/** Default probability threshold for mask values. */
readonly defaultMaskThreshold: number;
/** Default minimum confidence score threshold for detected instances. */
readonly defaultConfidenceThreshold: number;
};

Expand All @@ -46,8 +52,14 @@ export type InstanceSegmenterOptions<F extends BoxFormat, L> = Omit<
* @typeParam L The label type.
*/
export type InstanceSegmenterModel<F extends BoxFormat, L> = {
/** Local path or remote URL of the `.pte` model file. */
readonly modelPath: string;
readonly opts: InstanceSegmenterOptions<F, L>;
/**
* Image preprocessing, label vocabulary, bounding box format,
* and default NMS/mask/confidence thresholds
* {@link InstanceSegmenterOptions}.
*/
readonly modelOpts: InstanceSegmenterOptions<F, L>;
};

/**
Expand All @@ -58,9 +70,13 @@ export type InstanceSegmenterModel<F extends BoxFormat, L> = {
* @typeParam L The label type.
*/
export type InstanceSegmentationResult<F extends BoxFormat, L> = {
/** Scaled bounding box coordinates matching the input image resolution. */
readonly box: BoundingBox<F>;
/** Binary segmentation mask buffer cropped to the instance bounding box. */
readonly mask: ImageBuffer;
/** Predicted instance class label. */
readonly label: L;
/** Confidence score of the instance detection (between 0.0 and 1.0). */
readonly confidence: number;
};

Expand Down Expand Up @@ -93,10 +109,12 @@ export async function createInstanceSegmenter<F extends BoxFormat, L>(
* Performs asynchronous instance segmentation on the given input image.
* @param input The input image buffer.
* @param options Execution override options.
* @param options.confidenceThreshold Override for the minimum confidence
* threshold.
* @param options.iouThreshold Override for the IoU threshold in NMS.
* @param options.maskThreshold Override for the mask binarization threshold.
* @param options.confidenceThreshold Minimum confidence threshold. If
* omitted, uses `modelOpts.defaultConfidenceThreshold`.
* @param options.iouThreshold Intersection over Union (IoU) threshold in NMS. If omitted, uses
* `modelOpts.defaultIouThreshold`.
* @param options.maskThreshold Mask binarization threshold. If omitted,
* uses `modelOpts.defaultMaskThreshold`.
* @returns A promise resolving to a list of detected instances.
*/
segmentInstances: (
Expand All @@ -113,7 +131,7 @@ export async function createInstanceSegmenter<F extends BoxFormat, L>(
options?: { confidenceThreshold?: number; iouThreshold?: number; maskThreshold?: number }
) => InstanceSegmentationResult<F, L>[];
}> {
const { modelPath, opts } = config;
const { modelPath, modelOpts } = config;
const model = await wrapAsync(loadModel, runtime)(modelPath);
const meta = validateModelSchema(
model,
Expand Down Expand Up @@ -149,7 +167,7 @@ export async function createInstanceSegmenter<F extends BoxFormat, L>(

const [tBoxes, tScores, tClasses, tAllMasks, tMask] = tensors;

const preprocessor = createImagePreprocessor(opts, inpShape);
const preprocessor = createImagePreprocessor(modelOpts, inpShape);

const dispose = () => {
preprocessor.dispose();
Expand All @@ -165,16 +183,17 @@ export async function createInstanceSegmenter<F extends BoxFormat, L>(
const tInput = preprocessor.process(input);
model.execute('forward', [tInput], [tBoxes, tScores, tClasses, tAllMasks]);

const iouThreshold = options?.iouThreshold ?? opts.defaultIouThreshold;
const maskThreshold = options?.maskThreshold ?? opts.defaultMaskThreshold;
const confidenceThreshold = options?.confidenceThreshold ?? opts.defaultConfidenceThreshold;
const iouThreshold = options?.iouThreshold ?? modelOpts.defaultIouThreshold;
const maskThreshold = options?.maskThreshold ?? modelOpts.defaultMaskThreshold;
const confidenceThreshold =
options?.confidenceThreshold ?? modelOpts.defaultConfidenceThreshold;

const eps = 1e-7;
const clampedMaskThreshold = Math.max(eps, Math.min(1 - eps, maskThreshold));
const logitMaskThreshold = Math.log(clampedMaskThreshold / (1 - clampedMaskThreshold));

const indices = nms(tBoxes, tScores, {
boxFormat: opts.boxFormat,
boxFormat: modelOpts.boxFormat,
iouThreshold,
confidenceThreshold,
nmsType: 'standard',
Expand All @@ -199,12 +218,12 @@ export async function createInstanceSegmenter<F extends BoxFormat, L>(
for (const idx of indices) {
const confidence = scores[idx]!;
const classIdx = Math.round(classes[idx]!);
const label = opts.labels[classIdx];
const label = modelOpts.labels[classIdx];

if (label === undefined) {
throw new Error(
`InstanceSegmenter: Predicted class index ${classIdx} is ` +
`out of bounds for labels array of size ${opts.labels.length}.`
`out of bounds for labels array of size ${modelOpts.labels.length}.`
);
}

Expand All @@ -213,7 +232,7 @@ export async function createInstanceSegmenter<F extends BoxFormat, L>(
const c = boxes[idx * 4 + 2]!;
const d = boxes[idx * 4 + 3]!;

const box = scaleBox(decodeBox([a, b, c, d], opts.boxFormat), {
const box = scaleBox(decodeBox([a, b, c, d], modelOpts.boxFormat), {
from: { width: targetW, height: targetH },
to: { width: input.width, height: input.height },
resizeMode: 'stretch',
Expand Down
Loading