diff --git a/apps/computer-vision/app/_layout.tsx b/apps/computer-vision/app/_layout.tsx index f269b27aae..1a53311ad9 100644 --- a/apps/computer-vision/app/_layout.tsx +++ b/apps/computer-vision/app/_layout.tsx @@ -27,6 +27,13 @@ export default function Layout() { title: 'Classification', }} /> + (MODEL_OPTIONS[0].value); const [imageUri, setImageUri] = useState(null); const [isProcessing, setIsProcessing] = useState(false); @@ -97,7 +99,10 @@ function ClassificationContent() { return ( Upload or capture an image to identify objects using a classifier. diff --git a/apps/computer-vision/app/imageEmbeddings/index.tsx b/apps/computer-vision/app/imageEmbeddings/index.tsx new file mode 100644 index 0000000000..972c063bd6 --- /dev/null +++ b/apps/computer-vision/app/imageEmbeddings/index.tsx @@ -0,0 +1,250 @@ +import React, { useState } from 'react'; +import { View, Text, StyleSheet, ScrollView, TextInput, TouchableOpacity } from 'react-native'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; +import { commonStyles, ColorPalette } from '../../theme'; +import { useImage } from '@shopify/react-native-skia'; +import { useImageEmbedder, useTextEmbedder, models } from 'react-native-executorch'; +import ScreenWrapper from '../../components/ScreenWrapper'; +import { getImage, skImageToBuffer } from '../../utils'; +import { ModelPicker, type ModelOption } from '../../components/ModelPicker'; +import { ImageViewport } from '../../components/ImageViewport'; +import { ModelStatus } from '../../components/ModelStatus'; +import { LatencyIndicator } from '../../components/LatencyIndicator'; +import { Button } from '../../components/Button'; + +const IMAGE_MODEL_OPTIONS: ModelOption[] = [ + { + label: 'CLIP ViT-B/32 (INT8)', + value: models.imageEmbeddings.CLIP_VIT_BASE_PATCH32.XNNPACK_INT8, + }, + { + label: 'CLIP ViT-B/32 (FP32)', + value: models.imageEmbeddings.CLIP_VIT_BASE_PATCH32.XNNPACK_FP32, + }, +]; + +const DEFAULT_LABELS = [ + 'a photo of a dog', + 'a photo of a cat', + 'a landscape photo', + 'a photo of food', + 'a photo of people', +]; + +// CLIP text and image embeddings are L2-normalized, so their cosine similarity +// is the dot product. +const dot = (a: Float32Array, b: Float32Array) => { + let s = 0; + for (let i = 0; i < a.length; i++) { + s += a[i]! * b[i]!; + } + return s; +}; + +function ImageEmbeddingsContent() { + const [selectedImageModel, setSelectedImageModel] = useState(IMAGE_MODEL_OPTIONS[0].value); + const [imageUri, setImageUri] = useState(null); + const [labels, setLabels] = useState(DEFAULT_LABELS); + const [newLabel, setNewLabel] = useState(''); + const [results, setResults] = useState<{ label: string; score: number }[]>([]); + const [latency, setLatency] = useState(null); + const [isProcessing, setIsProcessing] = useState(false); + const [error, setError] = useState(null); + + const insets = useSafeAreaInsets(); + const skiaImage = useImage(imageUri, (err) => setError(err.message || String(err))); + + // Zero-shot classification pairs a CLIP image encoder with the CLIP text + // encoder and scores the image against each text label by embedding similarity. + const imageModel = useImageEmbedder(selectedImageModel); + const textModel = useTextEmbedder(models.textEmbeddings.CLIP_VIT_BASE_PATCH32_TEXT); + + const ready = imageModel.isReady && textModel.isReady; + + const pickImage = async () => { + setError(null); + try { + const uri = await getImage(false); + if (!uri) return; + setImageUri(uri); + setResults([]); + setLatency(null); + } catch (e: any) { + setError(e.message || String(e)); + } + }; + + const classify = async () => { + if (!skiaImage || !ready || !imageModel.embed || !textModel.embed) return; + setIsProcessing(true); + setError(null); + try { + const start = Date.now(); + const imageEmbedding = await imageModel.embed(skImageToBuffer(skiaImage)); + const scored: { label: string; score: number }[] = []; + for (const label of labels) { + const textEmbedding = await textModel.embed(label); + scored.push({ label, score: dot(imageEmbedding, textEmbedding) }); + } + scored.sort((a, b) => b.score - a.score); + setLatency(Date.now() - start); + setResults(scored); + } catch (e: any) { + setError(e.message || String(e)); + } finally { + setIsProcessing(false); + } + }; + + const addLabel = () => { + const trimmed = newLabel.trim(); + if (!trimmed || labels.includes(trimmed)) return; + setLabels((prev) => [...prev, trimmed]); + setNewLabel(''); + setResults([]); + }; + + const removeLabel = (label: string) => { + setLabels((prev) => prev.filter((l) => l !== label)); + setResults((prev) => prev.filter((r) => r.label !== label)); + }; + + const activeError = imageModel.error + ? String(imageModel.error) + : textModel.error + ? String(textModel.error) + : error; + + return ( + + + Pick an image, then rank text labels by how well CLIP matches them to it (zero-shot + classification). + + + { + setSelectedImageModel(model); + setResults([]); + setLatency(null); + }} + /> + + + + + + +