diff --git a/apps/computer-vision/app/_layout.tsx b/apps/computer-vision/app/_layout.tsx
index 91b231a771..f56a7b5da0 100644
--- a/apps/computer-vision/app/_layout.tsx
+++ b/apps/computer-vision/app/_layout.tsx
@@ -27,6 +27,13 @@ export default function Layout() {
title: 'Classification',
}}
/>
+
router.navigate('classification/')}>
Classification
+ router.navigate('segmentation/')}>
+ Semantic Segmentation
+
router.navigate('inspect/')}>
Model Inspector
diff --git a/apps/computer-vision/app/segmentation/index.tsx b/apps/computer-vision/app/segmentation/index.tsx
new file mode 100644
index 0000000000..1a820b2825
--- /dev/null
+++ b/apps/computer-vision/app/segmentation/index.tsx
@@ -0,0 +1,206 @@
+import React, { useState, useRef } from 'react';
+import { View, Text, ScrollView } from 'react-native';
+import { useSafeAreaInsets } from 'react-native-safe-area-context';
+import { commonStyles, theme } from '../../theme';
+import {
+ Skia,
+ ColorType,
+ AlphaType,
+ useImage,
+ type SkImage as SkiaImageType,
+} from '@shopify/react-native-skia';
+import { useSemanticSegmenter, models } from 'react-native-executorch';
+import ScreenWrapper from '../../components/ScreenWrapper';
+import { ModelPicker, type ModelOption } from '../../components/ModelPicker';
+import { getImage } from '../../utils';
+import { ImageViewport } from '../../components/ImageViewport';
+import { ModelStatus } from '../../components/ModelStatus';
+import { LatencyIndicator } from '../../components/LatencyIndicator';
+import { Button } from '../../components/Button';
+
+const SEGMENTATION_OPTIONS: ModelOption[] = [
+ {
+ label: 'Selfie Seg (FP32)',
+ value: models.semanticSegmentation.SELFIE_SEGMENTATION.XNNPACK_FP32,
+ },
+ {
+ label: 'LRASPP MobileNet V3 (INT8)',
+ value: models.semanticSegmentation.LRASPP_MOBILENET_V3_LARGE.XNNPACK_INT8,
+ },
+ {
+ label: 'DeepLab V3 ResNet50 (INT8)',
+ value: models.semanticSegmentation.DEEPLAB_V3_RESNET50.XNNPACK_INT8,
+ },
+ {
+ label: 'DeepLab V3 ResNet101 (INT8)',
+ value: models.semanticSegmentation.DEEPLAB_V3_RESNET101.XNNPACK_INT8,
+ },
+ {
+ label: 'DeepLab V3 MobileNet V3 (INT8)',
+ value: models.semanticSegmentation.DEEPLAB_V3_MOBILENET_V3_LARGE.XNNPACK_INT8,
+ },
+ {
+ label: 'FCN ResNet50 (INT8)',
+ value: models.semanticSegmentation.FCN_RESNET50.XNNPACK_INT8,
+ },
+ {
+ label: 'FCN ResNet101 (INT8)',
+ value: models.semanticSegmentation.FCN_RESNET101.XNNPACK_INT8,
+ },
+];
+
+function SegmentationContent() {
+ const insets = useSafeAreaInsets();
+ const [selectedModel, setSelectedModel] = useState(SEGMENTATION_OPTIONS[0].value);
+ const [imageUri, setImageUri] = useState(null);
+ const [isProcessing, setIsProcessing] = useState(false);
+ const [latency, setLatency] = useState(null);
+ const [segmentationImage, setSegmentationImage] = useState(null);
+ const [error, setError] = useState(null);
+
+ const isProcessingRef = useRef(false);
+
+ const skiaImage = useImage(imageUri, (err) => setError(err.message || String(err)));
+
+ const {
+ isReady,
+ downloadProgress,
+ error: loadError,
+ segment,
+ segmentWorklet,
+ } = useSemanticSegmenter(selectedModel);
+
+ const handlePickImage = async (useCamera: boolean) => {
+ setError(null);
+ try {
+ const uri = await getImage(useCamera);
+ if (uri) {
+ setImageUri(uri);
+ setLatency(null);
+ setSegmentationImage(null);
+ }
+ } catch (e: any) {
+ setError(e.message || String(e));
+ }
+ };
+
+ const runSegmentation = async (sync: boolean) => {
+ if (isProcessingRef.current) return;
+ if (!skiaImage || !segment || !segmentWorklet) return;
+
+ isProcessingRef.current = true;
+ setIsProcessing(true);
+ setError(null);
+
+ try {
+ const pixels = skiaImage.readPixels();
+ if (!pixels) {
+ throw new Error('Failed to read pixels from image');
+ }
+ if (!(pixels instanceof Uint8Array)) {
+ throw new Error('Expected Uint8Array from readPixels');
+ }
+ const buffer = {
+ data: pixels,
+ width: skiaImage.width(),
+ height: skiaImage.height(),
+ format: 'rgba' as const,
+ layout: 'hwc' as const,
+ };
+
+ const start = Date.now();
+ const { buffer: outBuffer } = sync ? segmentWorklet(buffer) : await segment(buffer);
+ setLatency(Date.now() - start);
+
+ const outData = Skia.Data.fromBytes(outBuffer.data);
+ const info = {
+ width: buffer.width,
+ height: buffer.height,
+ colorType: ColorType.RGBA_8888,
+ alphaType: AlphaType.Premul,
+ };
+ const nextImage = Skia.Image.MakeImage(info, outData, buffer.width * 4);
+ if (!nextImage) {
+ throw new Error('Failed to create overlay image from output data');
+ }
+ setSegmentationImage(nextImage);
+ } catch (e: any) {
+ setError(e.message || String(e));
+ } finally {
+ isProcessingRef.current = false;
+ setIsProcessing(false);
+ }
+ };
+
+ const activeError = loadError ? String(loadError) : error;
+
+ return (
+
+
+ Upload or capture an image to partition it into multiple segments using semantic
+ segmentation.
+
+
+ {
+ setSelectedModel(model);
+ setLatency(null);
+ setError(null);
+ setSegmentationImage(null);
+ }}
+ />
+
+
+
+ handlePickImage(false)}
+ />
+
+
+
+
+
+ runSegmentation(false)}
+ disabled={!skiaImage || !isReady || isProcessing}
+ loading={isProcessing}
+ />
+ runSegmentation(true)}
+ disabled={!skiaImage || !isReady || isProcessing}
+ variant="accent"
+ />
+
+
+
+
+ );
+}
+
+export default function SegmentationScreen() {
+ return (
+
+
+
+ );
+}
diff --git a/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp b/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp
index 7f5e4b06de..6121de3838 100644
--- a/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp
+++ b/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp
@@ -614,4 +614,93 @@ void install_normalize(jsi::Runtime &rt, jsi::Object &module) {
module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 3, fnBody));
}
+
+void install_applyColormap(jsi::Runtime &rt, jsi::Object &module) {
+ auto name = "applyColormap";
+ auto fnBody = [](jsi::Runtime &rt, const jsi::Value &thisVal, const jsi::Value *args, size_t count) -> jsi::Value {
+ if (count != 3) {
+ throw jsi::JSError(rt, "Usage: applyColormap(src, dst, colormap)");
+ }
+
+ if (!args[0].isObject() || !args[0].asObject(rt).isHostObject(rt)) {
+ throw jsi::JSError(rt, "applyColormap: src must be a Tensor");
+ }
+ if (!args[1].isObject() || !args[1].asObject(rt).isHostObject(rt)) {
+ throw jsi::JSError(rt, "applyColormap: dst must be a Tensor");
+ }
+
+ auto src = args[0].asObject(rt).getHostObject(rt);
+ auto dst = args[1].asObject(rt).getHostObject(rt);
+ constexpr size_t numRgbaChannels = 4;
+
+ if (src->dtype_ != rnexecutorch::core::types::DType::int32) {
+ throw jsi::JSError(rt, "applyColormap: src must be int32");
+ }
+ if (dst->dtype_ != rnexecutorch::core::types::DType::uint8) {
+ throw jsi::JSError(rt, "applyColormap: dst must be uint8");
+ }
+ if (dst->numel_ != src->numel_ * numRgbaChannels) {
+ throw jsi::JSError(rt, "applyColormap: dst must have exactly 4 times the number of elements as src (RGBA channels)");
+ }
+
+ if (!args[2].isObject() || !args[2].asObject(rt).isArray(rt)) {
+ throw jsi::JSError(rt, "applyColormap: colormap must be an array");
+ }
+
+ auto colormapArray = args[2].asObject(rt).asArray(rt);
+ size_t numColors = colormapArray.size(rt);
+ std::vector> lut(numColors);
+ for (size_t i = 0; i < numColors; ++i) {
+ auto colorVal = colormapArray.getValueAtIndex(rt, i);
+ if (!colorVal.isObject() || !colorVal.asObject(rt).isArray(rt)) {
+ throw jsi::JSError(rt, "applyColormap: colormap entry must be an array");
+ }
+ auto color = colorVal.asObject(rt).asArray(rt);
+ if (color.size(rt) != numRgbaChannels) {
+ throw jsi::JSError(rt, "applyColormap: colormap entry must be an RGBA color array of size 4");
+ }
+ for (size_t c = 0; c < numRgbaChannels; ++c) {
+ auto channelVal = color.getValueAtIndex(rt, c);
+ if (!channelVal.isNumber()) {
+ throw jsi::JSError(rt, "applyColormap: colormap channel value must be a number");
+ }
+ double val = channelVal.asNumber();
+ if (std::isnan(val) || val < 0.0 || val > 255.0) {
+ throw jsi::JSError(rt, "applyColormap: colormap channel value must be between 0 and 255");
+ }
+ lut[i][c] = static_cast(val);
+ }
+ }
+
+ std::shared_lock srcLock(src->mutex_, std::try_to_lock);
+ std::unique_lock dstLock(dst->mutex_, std::try_to_lock);
+ if (!srcLock.owns_lock() || !dstLock.owns_lock()) {
+ throw jsi::JSError(rt, "applyColormap: tensors in use");
+ }
+
+ if (!src->data_ || !dst->data_) {
+ throw jsi::JSError(rt, "applyColormap: tensor has been disposed");
+ }
+
+ size_t pixels = src->numel_;
+
+ const int32_t *srcData = reinterpret_cast(src->data_.get());
+ uint8_t *dstData = dst->data_.get();
+
+ for (size_t i = 0; i < pixels; ++i) {
+ int32_t idx = srcData[i];
+ if (idx < 0 || static_cast(idx) >= numColors) {
+ throw jsi::JSError(rt, "applyColormap: tensor contains class index (" +
+ std::to_string(idx) + ") that exceeds provided colormap size (" +
+ std::to_string(numColors) + ")");
+ }
+ for (size_t c = 0; c < numRgbaChannels; ++c) {
+ dstData[i * numRgbaChannels + c] = lut[idx][c];
+ }
+ }
+
+ return jsi::Value(rt, args[1]);
+ };
+ module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 3, fnBody));
+}
} // namespace rnexecutorch::extensions::cv::image_ops
diff --git a/packages/react-native-executorch/cpp/extensions/cv/image_ops.h b/packages/react-native-executorch/cpp/extensions/cv/image_ops.h
index fe6cfa546d..893c0957e5 100644
--- a/packages/react-native-executorch/cpp/extensions/cv/image_ops.h
+++ b/packages/react-native-executorch/cpp/extensions/cv/image_ops.h
@@ -8,4 +8,5 @@ void install_cvtColor(facebook::jsi::Runtime &rt, facebook::jsi::Object &module)
void install_toChannelsFirst(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
void install_toChannelsLast(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
void install_normalize(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
+void install_applyColormap(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
} // namespace rnexecutorch::extensions::cv::image_ops
diff --git a/packages/react-native-executorch/cpp/extensions/cv/install.cpp b/packages/react-native-executorch/cpp/extensions/cv/install.cpp
index 676fab3cc0..f559c68552 100644
--- a/packages/react-native-executorch/cpp/extensions/cv/install.cpp
+++ b/packages/react-native-executorch/cpp/extensions/cv/install.cpp
@@ -12,6 +12,7 @@ void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module) {
image_ops::install_toChannelsFirst(rt, cvModule);
image_ops::install_toChannelsLast(rt, cvModule);
image_ops::install_normalize(rt, cvModule);
+ image_ops::install_applyColormap(rt, cvModule);
module.setProperty(rt, "cv", cvModule);
}
diff --git a/packages/react-native-executorch/cpp/extensions/math/operations.cpp b/packages/react-native-executorch/cpp/extensions/math/operations.cpp
index b4c77a4adc..d02738ca45 100644
--- a/packages/react-native-executorch/cpp/extensions/math/operations.cpp
+++ b/packages/react-native-executorch/cpp/extensions/math/operations.cpp
@@ -277,26 +277,21 @@ void install_argmax(jsi::Runtime &rt, jsi::Object &module) {
}
int32_t *dstData = reinterpret_cast(dst->data_.get());
- std::vector maxVals(inner);
+ // DO NOT swap loop order. This structure intentionally prioritizes the
+ // most common case (axis = -1, inner = 1) for sequential access.
for (size_t o = 0; o < outer; ++o) {
- const float *srcSlab = srcData + o * axisDim * inner;
- int32_t *dstRow = dstData + o * inner;
-
for (size_t i = 0; i < inner; ++i) {
- maxVals[i] = -std::numeric_limits::infinity();
- dstRow[i] = 0;
- }
-
- for (size_t d = 0; d < axisDim; ++d) {
- const float *srcRow = srcSlab + d * inner;
- for (size_t i = 0; i < inner; ++i) {
- const float val = srcRow[i];
- if (val > maxVals[i]) {
- maxVals[i] = val;
- dstRow[i] = static_cast(d);
+ float maxVal = -std::numeric_limits::infinity();
+ int32_t maxIdx = 0;
+ for (size_t d = 0; d < axisDim; ++d) {
+ const float val = srcData[o * axisDim * inner + d * inner + i];
+ if (val > maxVal) {
+ maxVal = val;
+ maxIdx = static_cast(d);
}
}
+ dstData[o * inner + i] = maxIdx;
}
}
diff --git a/packages/react-native-executorch/src/constants.ts b/packages/react-native-executorch/src/constants.ts
index 789f7358e2..55ce8d9993 100644
--- a/packages/react-native-executorch/src/constants.ts
+++ b/packages/react-native-executorch/src/constants.ts
@@ -1005,8 +1005,53 @@ export const IMAGENET1K_LABELS = [
'toilet tissue, toilet paper, bathroom tissue',
] as const;
+/**
+ * Pascal VOC dataset label array containing the 21 categories.
+ * @category Constants
+ */
+export const PASCAL_VOC_LABELS = [
+ 'background',
+ 'aeroplane',
+ 'bicycle',
+ 'bird',
+ 'boat',
+ 'bottle',
+ 'bus',
+ 'car',
+ 'cat',
+ 'chair',
+ 'cow',
+ 'diningtable',
+ 'dog',
+ 'horse',
+ 'motorbike',
+ 'person',
+ 'pottedplant',
+ 'sheep',
+ 'sofa',
+ 'train',
+ 'tvmonitor',
+] as const;
+
/**
* Type representing a valid ImageNet 1K label string.
* @category Types
*/
export type ImageNet1KLabel = (typeof IMAGENET1K_LABELS)[number];
+
+/**
+ * Type representing a valid Pascal VOC label string.
+ * @category Types
+ */
+export type PascalVocLabel = (typeof PASCAL_VOC_LABELS)[number];
+
+/**
+ * ImageNet standard normalization options containing alpha and beta
+ * coefficients. Based on the standard ImageNet mean [0.485, 0.456, 0.406] and
+ * std [0.229, 0.224, 0.225].
+ * @category Constants
+ */
+export const IMAGENET_NORM = {
+ alpha: [1 / (255.0 * 0.229), 1 / (255.0 * 0.224), 1 / (255.0 * 0.225)],
+ beta: [-0.485 / 0.229, -0.456 / 0.224, -0.406 / 0.225],
+} as const;
diff --git a/packages/react-native-executorch/src/extensions/cv/ops/image.ts b/packages/react-native-executorch/src/extensions/cv/ops/image.ts
index 352ad7f484..b5bc22b746 100644
--- a/packages/react-native-executorch/src/extensions/cv/ops/image.ts
+++ b/packages/react-native-executorch/src/extensions/cv/ops/image.ts
@@ -76,8 +76,8 @@ export type ResizeOptions = {
* @category Types
*/
export type NormalizeOptions = {
- readonly alpha?: number | number[];
- readonly beta?: number | number[];
+ readonly alpha?: number | readonly number[];
+ readonly beta?: number | readonly number[];
};
/**
@@ -175,3 +175,29 @@ export function normalize(src: Tensor, dst: Tensor, opts?: NormalizeOptions): Te
} as const;
return rnexecutorchJsi.cv.normalize(src, dst, { ...defaultNormalizeOptions, ...opts });
}
+
+/**
+ * Applies a colormap to a single-channel image tensor, mapping class indices to
+ * RGBA colors.
+ *
+ * This operation iterates over each index/class ID in the source tensor, looks
+ * up its corresponding RGBA color in the provided colormap palette, and writes
+ * it to the destination tensor.
+ * @category Typescript API
+ * @param src The source index/mask tensor. Must be an integer tensor of `int32`
+ * dtype containing class indices. Shape `[H, W, 1]` (or `[H, W]`).
+ * @param dst The pre-allocated destination tensor to write the mapped RGBA
+ * values to. Must be a 3D image tensor in HWC layout and `uint8` dtype. Shape
+ * `[H, W, 4]`.
+ * @param colormap An array of RGBA color arrays `[R, G, B, A]` corresponding to each
+ * class index. The size of this list must cover all class indices present in `src`.
+ * @returns The destination tensor with the applied colormap.
+ */
+export function applyColormap(
+ src: Tensor,
+ dst: Tensor,
+ colormap: [number, number, number, number][]
+): Tensor {
+ 'worklet';
+ return rnexecutorchJsi.cv.applyColormap(src, dst, colormap);
+}
diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts b/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts
index 09eb617c07..679ad68446 100644
--- a/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts
+++ b/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts
@@ -20,8 +20,8 @@ import {
export type ImagePreprocessorOptions = {
readonly resizeMode: ResizeMode;
readonly interpolation: InterpolationMethod;
- readonly alpha: number | number[];
- readonly beta: number | number[];
+ readonly alpha: number | readonly number[];
+ readonly beta: number | readonly number[];
readonly padValue?: number;
};
diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts
new file mode 100644
index 0000000000..148cf9b267
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts
@@ -0,0 +1,223 @@
+import type { WorkletRuntime } from 'react-native-worklets';
+
+import { tensor } from '../../../core/tensor';
+import { loadModel } from '../../../core/model';
+import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
+import { wrapAsync } from '../../../core/runtime';
+
+import type { ImageBuffer } from '../image';
+import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing';
+import {
+ toChannelsLast,
+ normalize,
+ resize,
+ cvtColor,
+ applyColormap,
+ type InterpolationMethod,
+} from '../ops/image';
+import { sigmoid, argmax } from '../../math';
+
+/**
+ * Options for configuring a semantic segmenter preprocessor and label
+ * vocabulary.
+ * @category Types
+ */
+export type SemanticSegmentationOptions = Omit & {
+ readonly resizeMode: 'stretch';
+ readonly outInterpolation: InterpolationMethod;
+ readonly labels: readonly L[];
+};
+
+/**
+ * Model configuration required to instantiate a segmenter task runner.
+ * @category Types
+ */
+export type SemanticSegmentationModel = {
+ readonly modelPath: string;
+ readonly opts: SemanticSegmentationOptions;
+};
+
+/**
+ * Maps each class label to its assigned RGBA color.
+ * @category Types
+ */
+export type ColorMap = Record;
+
+/**
+ * Result structure representing the output of a semantic segmentation task.
+ * @category Types
+ */
+export type SemanticSegmentationResult = {
+ buffer: ImageBuffer;
+ colormap?: ColorMap;
+};
+
+function hslToRgb(h: number, s: number, l: number): [number, number, number] {
+ s /= 100;
+ l /= 100;
+ const k = (n: number) => (n + h / 30) % 12;
+ const a = s * Math.min(l, 1 - l);
+ const f = (n: number) => l - a * Math.max(-1, Math.min(k(n) - 3, 9 - k(n), 1));
+ return [Math.round(255 * f(0)), Math.round(255 * f(8)), Math.round(255 * f(4))];
+}
+
+/**
+ * Creates a semantic segmenter runner for executing local Semantic Segmentation
+ * models.
+ *
+ * It validates the model inputs and outputs, asserts that the labels array
+ * length matches the model's output vocabulary size, pre-allocates the
+ * necessary static execution tensors, sets up an image preprocessor, and
+ * registers clean disposal hooks to clear all native memory.
+ * @category Typescript API
+ * @typeParam L The type representing the segmentation labels.
+ * @param config Segmenter task configuration containing path and options.
+ * @param runtime Optional worklet runtime thread environment context.
+ * @returns A promise resolving to an object containing segmentation and
+ * disposal controls.
+ */
+export async function createSemanticSegmenter(
+ config: SemanticSegmentationModel,
+ runtime?: WorkletRuntime
+): Promise<{
+ /**
+ * Releases all allocated native resources.
+ */
+ dispose: () => void;
+ /**
+ * Runs semantic segmentation asynchronously.
+ *
+ * For models returning logits for all classes, this performs an argmax over
+ * the class dimensions to identify the highest-probability class per pixel,
+ * then maps each class index to an RGBA color from the colormap. The final
+ * mapped colormap is returned in the result.
+ *
+ * For models returning only a single logit value for the positive class, this
+ * applies a sigmoid activation, normalizes the probability values to
+ * grayscale (0-255), and converts the output to a grayscale RGBA mask. No
+ * colormap is applied, and the colormap in the result is undefined.
+ * @param input The input image buffer to segment.
+ * @param colormap Optional partial color mapping overrides for labels
+ * (applicable only for multi-class models). If not provided, a default
+ * colormap is automatically generated with distinct, high-contrast colors
+ * (with the first label defaulting to transparent). If a partial map is
+ * provided, any labels omitted from it will default to being rendered as
+ * fully transparent.
+ * @returns A promise resolving to the segmentation result.
+ */
+ segment: (
+ input: ImageBuffer,
+ colormap?: Partial>
+ ) => Promise>;
+ /**
+ * Runs semantic segmentation synchronously.
+ * @see {@link segment} for details.
+ */
+ segmentWorklet: (
+ input: ImageBuffer,
+ colormap?: Partial>
+ ) => SemanticSegmentationResult;
+}> {
+ const { modelPath, opts } = config;
+ const model = await wrapAsync(loadModel, runtime)(modelPath);
+
+ const meta = validateModelSchema(
+ model,
+ 'forward',
+ [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])],
+ [SymbolicTensor('float32', [1, 'K', 'H', 'W'], ['K', 'H', 'W'])]
+ );
+ const inpShape = meta.inputTensorMeta[0]!.shape;
+ const outShape = meta.outputTensorMeta[0]!.shape;
+
+ const nClasses = outShape.at(-3)!;
+ const targetH = outShape.at(-2)!;
+ const targetW = outShape.at(-1)!;
+
+ // Generate highly distinct, high-contrast colors, see:
+ // https://martin.ankerl.com/2009/12/09/how-to-create-random-colors-programmatically/
+ const defaultColormap = opts.labels.map((_, i) => {
+ if (i === 0) return [0, 0, 0, 0] as const;
+ return [...hslToRgb((i * 137.5) % 360, 95, 50), 255] as const;
+ });
+
+ if (nClasses > 1 && opts.labels.length !== nClasses) {
+ throw new Error(
+ `Model outputs ${nClasses} classes, but ${opts.labels.length} labels were provided in the configuration.`
+ );
+ }
+
+ const tensors = [
+ tensor('float32', outShape),
+ tensor('float32', [nClasses, targetH, targetW]),
+ tensor('float32', [nClasses, targetH, targetW]),
+ tensor('float32', [targetH, targetW, nClasses]),
+ tensor(nClasses > 1 ? 'int32' : 'uint8', [targetH, targetW, 1]),
+ tensor('uint8', [targetH, targetW, 4]),
+ ] as const;
+
+ const [tOutput, tReshape, tSigmoid, tChanLast, tMask, tRgba] = tensors;
+ const preprocessor = createImagePreprocessor(opts, inpShape);
+
+ const dispose = () => {
+ tensors.forEach((t) => t.dispose());
+ preprocessor.dispose();
+ model.dispose();
+ };
+
+ const segmentWorklet = (
+ input: ImageBuffer,
+ colormap?: Partial>
+ ): SemanticSegmentationResult => {
+ 'worklet';
+ const tInput = preprocessor.process(input);
+ model.execute('forward', [tInput], [tOutput]);
+
+ let returnColormap: ColorMap | undefined;
+ if (nClasses > 1) {
+ if (colormap) {
+ returnColormap = Object.fromEntries(
+ opts.labels.map((l) => [l, colormap[l] ?? [0, 0, 0, 0]])
+ ) as ColorMap;
+ } else {
+ returnColormap = Object.fromEntries(
+ opts.labels.map((l, i) => [l, defaultColormap[i]!])
+ ) as ColorMap;
+ }
+
+ const colormapData = opts.labels.map((l) => returnColormap![l]);
+
+ tOutput
+ .copyTo(tReshape)
+ .through(toChannelsLast, tChanLast)
+ .through(argmax, tMask, -1)
+ .through(applyColormap, tRgba, colormapData);
+ } else {
+ tOutput
+ .copyTo(tReshape)
+ .through(sigmoid, tSigmoid)
+ .through(toChannelsLast, tChanLast)
+ .through(normalize, tMask, { alpha: 255.0 })
+ .through(cvtColor, tRgba, 'GRAY2RGBA');
+ }
+
+ const data = new Uint8Array(input.width * input.height * 4);
+ const tResize = tensor('uint8', [input.height, input.width, 4]);
+ try {
+ tRgba
+ .through(resize, tResize, { mode: 'stretch', interpolation: opts.outInterpolation })
+ .getData(data);
+ } finally {
+ tResize.dispose();
+ }
+
+ return {
+ buffer: { data, width: input.width, height: input.height, format: 'rgba', layout: 'hwc' },
+ colormap: returnColormap,
+ };
+ };
+
+ const segment = wrapAsync(segmentWorklet, runtime);
+
+ return { segment, segmentWorklet, dispose };
+}
diff --git a/packages/react-native-executorch/src/hooks/useSemanticSegmenter.ts b/packages/react-native-executorch/src/hooks/useSemanticSegmenter.ts
new file mode 100644
index 0000000000..8140204e18
--- /dev/null
+++ b/packages/react-native-executorch/src/hooks/useSemanticSegmenter.ts
@@ -0,0 +1,47 @@
+import { useModel } from './useModel';
+import { useResourceDownload } from './useResourceDownload';
+import {
+ createSemanticSegmenter,
+ type SemanticSegmentationModel,
+} from '../extensions/cv/tasks/semanticSegmentation';
+
+/**
+ * React hook to load and run a semantic segmentation model.
+ *
+ * This hook manages downloading (if it's a remote URL) and loading the model
+ * file, compiling it, tracking download progress and compilation errors, and
+ * cleaning up native model memory when the component unmounts or configuration
+ * changes.
+ * @category Hooks
+ * @typeParam L The type representing the segmentation labels.
+ * @param config The semantic segmentation model configuration.
+ * @param options Hook options.
+ * @param options.preventLoad If true, prevents downloading and compiling the
+ * model.
+ * @returns An object containing the model's loading state, error, download
+ * progress, and segmentation functions.
+ */
+export function useSemanticSegmenter(
+ config: SemanticSegmentationModel,
+ options?: { preventLoad?: boolean }
+) {
+ const { localPath, downloadProgress, downloadError } = useResourceDownload(
+ config.modelPath,
+ options?.preventLoad
+ );
+ const { model, error } = useModel(
+ createSemanticSegmenter,
+ localPath ? { ...config, modelPath: localPath } : null,
+ [localPath]
+ );
+
+ return {
+ isReady: !!model,
+ error: downloadError || error,
+ downloadProgress,
+ localPath,
+ segment: model?.segment,
+ segmentWorklet: model?.segmentWorklet,
+ labels: config.opts.labels,
+ };
+}
diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts
index 563c5b22c0..62b7a6a67f 100644
--- a/packages/react-native-executorch/src/index.ts
+++ b/packages/react-native-executorch/src/index.ts
@@ -1,5 +1,6 @@
// Hooks — primary API for app developers
export * from './hooks/useClassifier';
+export * from './hooks/useSemanticSegmenter';
export * from './hooks/useTokenizer';
export * from './hooks/useResourceDownload';
export * from './hooks/useModel';
@@ -10,6 +11,7 @@ export * as constants from './constants';
// Task APIs — for developers needing manual lifetime/disposal control
export * from './extensions/cv/tasks/classification';
+export * from './extensions/cv/tasks/semanticSegmentation';
export * from './extensions/nlp/tasks/tokenization';
// Core primitives — for library builders and power users
diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts
index dfb0b177e1..351b610f8a 100644
--- a/packages/react-native-executorch/src/models.ts
+++ b/packages/react-native-executorch/src/models.ts
@@ -1,8 +1,16 @@
import type { ClassifierModel } from './extensions/cv/tasks/classification';
-import { IMAGENET1K_LABELS, type ImageNet1KLabel } from './constants';
+import type { SemanticSegmentationModel } from './extensions/cv/tasks/semanticSegmentation';
+import {
+ IMAGENET_NORM,
+ IMAGENET1K_LABELS,
+ PASCAL_VOC_LABELS,
+ type ImageNet1KLabel,
+ type PascalVocLabel,
+} from './constants';
const BASE_URL = 'https://huggingface.co/software-mansion/react-native-executorch';
const VERSION_TAG = 'resolve/v0.9.0';
+const NEXT_VERSION_TAG = 'resolve/v0.10.0';
// =============================================================================
// Classification
@@ -27,6 +35,93 @@ const EFFICIENTNET_V2_S_COREML_FP16: ClassifierModel = {
classifierOpts: EFFICIENTNET_V2_S_OPTS,
};
+// =============================================================================
+// Semantic Segmentation
+// =============================================================================
+const SELFIE_SEGMENTATION_XNNPACK_FP32: SemanticSegmentationModel<'background' | 'person'> = {
+ modelPath: `${BASE_URL}-selfie-segmentation/${VERSION_TAG}/xnnpack/selfie_segmentation_xnnpack_fp32.pte`,
+ opts: {
+ labels: ['background', 'person'] as const,
+ resizeMode: 'stretch',
+ interpolation: 'linear',
+ alpha: 1 / 255.0,
+ beta: 0.0,
+ outInterpolation: 'lanczos',
+ },
+};
+
+const LRASPP_MOBILENET_V3_LARGE_OPTS = {
+ labels: PASCAL_VOC_LABELS,
+ resizeMode: 'stretch' as const,
+ interpolation: 'linear' as const,
+ outInterpolation: 'lanczos' as const,
+ ...IMAGENET_NORM,
+};
+const LRASPP_MOBILENET_V3_LARGE_XNNPACK_FP32: SemanticSegmentationModel = {
+ modelPath: `${BASE_URL}-lraspp/${VERSION_TAG}/xnnpack/lraspp_mobilenet_v3_large_xnnpack_fp32.pte`,
+ opts: LRASPP_MOBILENET_V3_LARGE_OPTS,
+};
+const LRASPP_MOBILENET_V3_LARGE_XNNPACK_INT8: SemanticSegmentationModel = {
+ modelPath: `${BASE_URL}-lraspp/${VERSION_TAG}/xnnpack/lraspp_mobilenet_v3_large_xnnpack_int8.pte`,
+ opts: LRASPP_MOBILENET_V3_LARGE_OPTS,
+};
+
+const DEEPLAB_V3_OPTS = {
+ labels: PASCAL_VOC_LABELS,
+ resizeMode: 'stretch' as const,
+ interpolation: 'linear' as const,
+ outInterpolation: 'lanczos' as const,
+ ...IMAGENET_NORM,
+};
+const DEEPLAB_V3_RESNET50_XNNPACK_FP32: SemanticSegmentationModel = {
+ modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_resnet50_xnnpack_fp32.pte`,
+ opts: DEEPLAB_V3_OPTS,
+};
+const DEEPLAB_V3_RESNET50_XNNPACK_INT8: SemanticSegmentationModel = {
+ modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_resnet50_xnnpack_int8.pte`,
+ opts: DEEPLAB_V3_OPTS,
+};
+const DEEPLAB_V3_RESNET101_XNNPACK_FP32: SemanticSegmentationModel = {
+ modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_resnet101_xnnpack_fp32.pte`,
+ opts: DEEPLAB_V3_OPTS,
+};
+const DEEPLAB_V3_RESNET101_XNNPACK_INT8: SemanticSegmentationModel = {
+ modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_resnet101_xnnpack_int8.pte`,
+ opts: DEEPLAB_V3_OPTS,
+};
+const DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_FP32: SemanticSegmentationModel = {
+ modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_mobilenet_v3_large_xnnpack_fp32.pte`,
+ opts: DEEPLAB_V3_OPTS,
+};
+const DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_INT8: SemanticSegmentationModel = {
+ modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_mobilenet_v3_large_xnnpack_int8.pte`,
+ opts: DEEPLAB_V3_OPTS,
+};
+
+const FCN_OPTS = {
+ labels: PASCAL_VOC_LABELS,
+ resizeMode: 'stretch' as const,
+ interpolation: 'linear' as const,
+ outInterpolation: 'lanczos' as const,
+ ...IMAGENET_NORM,
+};
+const FCN_RESNET50_XNNPACK_FP32: SemanticSegmentationModel = {
+ modelPath: `${BASE_URL}-fcn/${NEXT_VERSION_TAG}/xnnpack/fcn_resnet50_xnnpack_fp32.pte`,
+ opts: FCN_OPTS,
+};
+const FCN_RESNET50_XNNPACK_INT8: SemanticSegmentationModel = {
+ modelPath: `${BASE_URL}-fcn/${NEXT_VERSION_TAG}/xnnpack/fcn_resnet50_xnnpack_int8.pte`,
+ opts: FCN_OPTS,
+};
+const FCN_RESNET101_XNNPACK_FP32: SemanticSegmentationModel = {
+ modelPath: `${BASE_URL}-fcn/${NEXT_VERSION_TAG}/xnnpack/fcn_resnet101_xnnpack_fp32.pte`,
+ opts: FCN_OPTS,
+};
+const FCN_RESNET101_XNNPACK_INT8: SemanticSegmentationModel = {
+ modelPath: `${BASE_URL}-fcn/${NEXT_VERSION_TAG}/xnnpack/fcn_resnet101_xnnpack_int8.pte`,
+ opts: FCN_OPTS,
+};
+
// =============================================================================
// Tokenizers
// =============================================================================
@@ -49,6 +144,42 @@ export const models = {
COREML_FP16: EFFICIENTNET_V2_S_COREML_FP16,
},
},
+ semanticSegmentation: {
+ SELFIE_SEGMENTATION: {
+ ...SELFIE_SEGMENTATION_XNNPACK_FP32,
+ XNNPACK_FP32: SELFIE_SEGMENTATION_XNNPACK_FP32,
+ },
+ LRASPP_MOBILENET_V3_LARGE: {
+ ...LRASPP_MOBILENET_V3_LARGE_XNNPACK_INT8,
+ XNNPACK_FP32: LRASPP_MOBILENET_V3_LARGE_XNNPACK_FP32,
+ XNNPACK_INT8: LRASPP_MOBILENET_V3_LARGE_XNNPACK_INT8,
+ },
+ DEEPLAB_V3_RESNET50: {
+ ...DEEPLAB_V3_RESNET50_XNNPACK_INT8,
+ XNNPACK_FP32: DEEPLAB_V3_RESNET50_XNNPACK_FP32,
+ XNNPACK_INT8: DEEPLAB_V3_RESNET50_XNNPACK_INT8,
+ },
+ DEEPLAB_V3_RESNET101: {
+ ...DEEPLAB_V3_RESNET101_XNNPACK_INT8,
+ XNNPACK_FP32: DEEPLAB_V3_RESNET101_XNNPACK_FP32,
+ XNNPACK_INT8: DEEPLAB_V3_RESNET101_XNNPACK_INT8,
+ },
+ DEEPLAB_V3_MOBILENET_V3_LARGE: {
+ ...DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_INT8,
+ XNNPACK_FP32: DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_FP32,
+ XNNPACK_INT8: DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_INT8,
+ },
+ FCN_RESNET50: {
+ ...FCN_RESNET50_XNNPACK_INT8,
+ XNNPACK_FP32: FCN_RESNET50_XNNPACK_FP32,
+ XNNPACK_INT8: FCN_RESNET50_XNNPACK_INT8,
+ },
+ FCN_RESNET101: {
+ ...FCN_RESNET101_XNNPACK_INT8,
+ XNNPACK_FP32: FCN_RESNET101_XNNPACK_FP32,
+ XNNPACK_INT8: FCN_RESNET101_XNNPACK_INT8,
+ },
+ },
tokenizer: {
ALL_MINILM_L6_V2: ALL_MINILM_L6_V2_TOKENIZER,
},