|
| 1 | +import React, { useEffect, useRef, useState } from 'react'; |
| 2 | +import { View, Text, TextInput, ScrollView, StyleSheet } from 'react-native'; |
| 3 | +import { useTokenizer, models } from 'react-native-executorch'; |
| 4 | +import ScreenWrapper from '../../components/ScreenWrapper'; |
| 5 | +import { ModelStatus } from '../../components/ModelStatus'; |
| 6 | +import { Button } from '../../components/Button'; |
| 7 | +import { theme } from '../../theme'; |
| 8 | + |
| 9 | +type Check = { label: string; detail: string; pass: boolean }; |
| 10 | + |
| 11 | +function TokenizerContent() { |
| 12 | + const { isReady, downloadProgress, error, encode, decode, getVocabSize, idToToken, tokenToId } = |
| 13 | + useTokenizer(models.tokenizer.ALL_MINILM_L6_V2); |
| 14 | + |
| 15 | + const [text, setText] = useState('Hello world'); |
| 16 | + const [running, setRunning] = useState(false); |
| 17 | + const [runError, setRunError] = useState<string | null>(null); |
| 18 | + const [ids, setIds] = useState<number[] | null>(null); |
| 19 | + const [roundTrip, setRoundTrip] = useState<string | null>(null); |
| 20 | + const [vocabSize, setVocabSize] = useState<number | null>(null); |
| 21 | + const [checks, setChecks] = useState<Check[]>([]); |
| 22 | + |
| 23 | + const ready = isReady && encode && decode && getVocabSize && idToToken && tokenToId; |
| 24 | + |
| 25 | + const run = async () => { |
| 26 | + if (!ready) return; |
| 27 | + setRunning(true); |
| 28 | + setRunError(null); |
| 29 | + setIds(null); |
| 30 | + setRoundTrip(null); |
| 31 | + setVocabSize(null); |
| 32 | + setChecks([]); |
| 33 | + try { |
| 34 | + const tokenIds = await encode(text); |
| 35 | + const decoded = await decode(tokenIds, true); |
| 36 | + const vocab = await getVocabSize(); |
| 37 | + |
| 38 | + // Self-consistent inverse check on a token from the actual output |
| 39 | + // (HFTokenizer adds special tokens per the tokenizer.json post_processor). |
| 40 | + const sampleId = tokenIds[Math.min(1, tokenIds.length - 1)]!; |
| 41 | + const sampleToken = await idToToken(sampleId); |
| 42 | + const sampleIdBack = await tokenToId(sampleToken); |
| 43 | + |
| 44 | + const nextChecks: Check[] = [ |
| 45 | + { |
| 46 | + label: 'Round-trip decode(encode(text))', |
| 47 | + detail: `"${decoded}" vs "${text.toLowerCase()}"`, |
| 48 | + // all-MiniLM-L6-v2 is an uncased BERT WordPiece tokenizer |
| 49 | + pass: decoded.trim() === text.trim().toLowerCase(), |
| 50 | + }, |
| 51 | + { |
| 52 | + label: 'Vocabulary size', |
| 53 | + detail: `${vocab} (expected 30522 for bert-base-uncased)`, |
| 54 | + pass: vocab === 30522, |
| 55 | + }, |
| 56 | + { |
| 57 | + label: 'Inverse tokenToId(idToToken(id))', |
| 58 | + detail: `${sampleId} → "${sampleToken}" → ${sampleIdBack}`, |
| 59 | + pass: sampleIdBack === sampleId, |
| 60 | + }, |
| 61 | + ]; |
| 62 | + |
| 63 | + setIds(tokenIds); |
| 64 | + setRoundTrip(decoded); |
| 65 | + setVocabSize(vocab); |
| 66 | + setChecks(nextChecks); |
| 67 | + |
| 68 | + // Structured log so the result is verifiable from device/Metro logs. |
| 69 | + console.log( |
| 70 | + '[TokenizerTest]', |
| 71 | + JSON.stringify({ |
| 72 | + allPass: nextChecks.every((c) => c.pass), |
| 73 | + input: text, |
| 74 | + ids: tokenIds, |
| 75 | + decoded, |
| 76 | + vocab, |
| 77 | + checks: nextChecks.map((c) => ({ label: c.label, pass: c.pass, detail: c.detail })), |
| 78 | + }) |
| 79 | + ); |
| 80 | + } catch (e: any) { |
| 81 | + console.log('[TokenizerTest] ERROR', e?.message ?? String(e)); |
| 82 | + setRunError(e?.message ?? String(e)); |
| 83 | + } finally { |
| 84 | + setRunning(false); |
| 85 | + } |
| 86 | + }; |
| 87 | + |
| 88 | + // Auto-run once as soon as the tokenizer is ready, so the demo doubles as a |
| 89 | + // self-checking smoke test (results logged under "[TokenizerTest]"). |
| 90 | + const autoRan = useRef(false); |
| 91 | + useEffect(() => { |
| 92 | + if (ready && !autoRan.current) { |
| 93 | + autoRan.current = true; |
| 94 | + run(); |
| 95 | + } |
| 96 | + // eslint-disable-next-line react-hooks/exhaustive-deps |
| 97 | + }, [ready]); |
| 98 | + |
| 99 | + return ( |
| 100 | + <ScrollView style={styles.container} contentContainerStyle={styles.content}> |
| 101 | + <View style={styles.card}> |
| 102 | + <Text style={styles.cardTitle}>Tokenizer</Text> |
| 103 | + <Text style={styles.cardDescription}> |
| 104 | + Loads the all-MiniLM-L6-v2 tokenizer and proves encode / decode / getVocabSize / |
| 105 | + idToToken / tokenToId work end-to-end against the native HFTokenizer. |
| 106 | + </Text> |
| 107 | + |
| 108 | + <ModelStatus |
| 109 | + isReady={isReady} |
| 110 | + downloadProgress={downloadProgress} |
| 111 | + error={error ? error.message : null} |
| 112 | + modelTypeLabel="tokenizer" |
| 113 | + /> |
| 114 | + |
| 115 | + <TextInput |
| 116 | + style={styles.input} |
| 117 | + value={text} |
| 118 | + onChangeText={setText} |
| 119 | + autoCapitalize="none" |
| 120 | + autoCorrect={false} |
| 121 | + placeholder="Type text to tokenize" |
| 122 | + placeholderTextColor="#999" |
| 123 | + /> |
| 124 | + |
| 125 | + <View style={styles.buttonRow}> |
| 126 | + <Button title="Run tokenizer" onPress={run} disabled={!ready} loading={running} /> |
| 127 | + </View> |
| 128 | + </View> |
| 129 | + |
| 130 | + {runError && ( |
| 131 | + <View style={styles.errorContainer}> |
| 132 | + <Text style={styles.errorText}>{runError}</Text> |
| 133 | + </View> |
| 134 | + )} |
| 135 | + |
| 136 | + {ids && ( |
| 137 | + <View style={styles.resultsCard}> |
| 138 | + <Text style={styles.resultsHeader}>Results</Text> |
| 139 | + |
| 140 | + <Text style={styles.fieldLabel}>Token IDs ({ids.length})</Text> |
| 141 | + <Text style={styles.mono}>[{ids.join(', ')}]</Text> |
| 142 | + |
| 143 | + <Text style={styles.fieldLabel}>Decoded (skipSpecialTokens)</Text> |
| 144 | + <Text style={styles.mono}>{roundTrip}</Text> |
| 145 | + |
| 146 | + <Text style={styles.fieldLabel}>Vocab size</Text> |
| 147 | + <Text style={styles.mono}>{vocabSize}</Text> |
| 148 | + |
| 149 | + <Text style={styles.checksHeader}>Assertions</Text> |
| 150 | + {checks.map((c, i) => ( |
| 151 | + <View key={i} style={styles.checkRow}> |
| 152 | + <Text style={[styles.checkBadge, c.pass ? styles.pass : styles.fail]}> |
| 153 | + {c.pass ? 'PASS' : 'FAIL'} |
| 154 | + </Text> |
| 155 | + <View style={styles.checkBody}> |
| 156 | + <Text style={styles.checkLabel}>{c.label}</Text> |
| 157 | + <Text style={styles.checkDetail}>{c.detail}</Text> |
| 158 | + </View> |
| 159 | + </View> |
| 160 | + ))} |
| 161 | + </View> |
| 162 | + )} |
| 163 | + </ScrollView> |
| 164 | + ); |
| 165 | +} |
| 166 | + |
| 167 | +export default function TokenizerScreen() { |
| 168 | + return ( |
| 169 | + <ScreenWrapper> |
| 170 | + <TokenizerContent /> |
| 171 | + </ScreenWrapper> |
| 172 | + ); |
| 173 | +} |
| 174 | + |
| 175 | +const styles = StyleSheet.create({ |
| 176 | + container: { flex: 1, backgroundColor: theme.colors.background }, |
| 177 | + content: { padding: theme.spacing.large, paddingBottom: 40 }, |
| 178 | + card: { |
| 179 | + backgroundColor: theme.colors.cardBackground, |
| 180 | + borderRadius: theme.radius.large, |
| 181 | + padding: 20, |
| 182 | + marginBottom: 20, |
| 183 | + borderWidth: 1, |
| 184 | + borderColor: theme.colors.lightBorder, |
| 185 | + }, |
| 186 | + cardTitle: { |
| 187 | + fontSize: theme.typography.title.fontSize, |
| 188 | + fontWeight: theme.typography.title.fontWeight, |
| 189 | + color: theme.colors.strongPrimary, |
| 190 | + marginBottom: 8, |
| 191 | + }, |
| 192 | + cardDescription: { fontSize: 14, color: theme.colors.textMuted, lineHeight: 20, marginBottom: 16 }, |
| 193 | + input: { |
| 194 | + backgroundColor: '#f1f3f5', |
| 195 | + borderRadius: theme.radius.small, |
| 196 | + padding: 12, |
| 197 | + fontSize: 15, |
| 198 | + color: '#212529', |
| 199 | + marginBottom: 16, |
| 200 | + borderWidth: 1, |
| 201 | + borderColor: theme.colors.lightBorder, |
| 202 | + }, |
| 203 | + buttonRow: { flexDirection: 'row', gap: theme.spacing.small }, |
| 204 | + errorContainer: { |
| 205 | + backgroundColor: theme.colors.errorBackground, |
| 206 | + padding: 12, |
| 207 | + borderRadius: theme.radius.small, |
| 208 | + marginBottom: 16, |
| 209 | + }, |
| 210 | + errorText: { color: theme.colors.errorText, fontSize: 14, textAlign: 'center' }, |
| 211 | + resultsCard: { |
| 212 | + backgroundColor: theme.colors.cardBackground, |
| 213 | + borderRadius: theme.radius.large, |
| 214 | + padding: 20, |
| 215 | + borderWidth: 1, |
| 216 | + borderColor: theme.colors.lightBorder, |
| 217 | + }, |
| 218 | + resultsHeader: { |
| 219 | + fontSize: 18, |
| 220 | + fontWeight: '700', |
| 221 | + color: '#212529', |
| 222 | + marginBottom: 16, |
| 223 | + borderBottomWidth: 1, |
| 224 | + borderBottomColor: '#f1f3f5', |
| 225 | + paddingBottom: 10, |
| 226 | + }, |
| 227 | + fieldLabel: { |
| 228 | + fontSize: 12, |
| 229 | + fontWeight: '600', |
| 230 | + color: theme.colors.textPlaceholder, |
| 231 | + textTransform: 'uppercase', |
| 232 | + letterSpacing: 0.5, |
| 233 | + marginTop: 12, |
| 234 | + marginBottom: 4, |
| 235 | + }, |
| 236 | + mono: { |
| 237 | + fontSize: 13, |
| 238 | + color: '#495057', |
| 239 | + backgroundColor: '#f8f9fa', |
| 240 | + padding: 8, |
| 241 | + borderRadius: 6, |
| 242 | + borderWidth: 1, |
| 243 | + borderColor: theme.colors.lightBorder, |
| 244 | + }, |
| 245 | + checksHeader: { fontSize: 16, fontWeight: '700', color: '#212529', marginTop: 20, marginBottom: 10 }, |
| 246 | + checkRow: { flexDirection: 'row', alignItems: 'flex-start', gap: 10, marginBottom: 10 }, |
| 247 | + checkBadge: { |
| 248 | + fontSize: 11, |
| 249 | + fontWeight: '700', |
| 250 | + color: '#fff', |
| 251 | + paddingHorizontal: 8, |
| 252 | + paddingVertical: 4, |
| 253 | + borderRadius: 4, |
| 254 | + overflow: 'hidden', |
| 255 | + }, |
| 256 | + pass: { backgroundColor: '#2b8a3e' }, |
| 257 | + fail: { backgroundColor: theme.colors.errorText }, |
| 258 | + checkBody: { flex: 1 }, |
| 259 | + checkLabel: { fontSize: 14, fontWeight: '600', color: '#212529' }, |
| 260 | + checkDetail: { fontSize: 12, color: theme.colors.textPlaceholder, marginTop: 2 }, |
| 261 | +}); |
0 commit comments