refactor(nlp): address review feedback on tokenizer pipeline

- install loadTokenizer under the nlp JSI submodule (rnexecutorchJsi.nlp)
- move ops/tokenizer.ts to nlp/tokenizer.ts; standard-template JSDoc, drop
  low/high-level coupling and Model/Tensor mention
- default skipSpecialTokens and empty-input handling in C++ decode
- minimal tokenization task (no worklet wrappers; sync lightweight lookups);
  inline single-field config to a path string
- readable tokenizer error strings; drop unnecessary build-config comments

Author: msluszniak

packages/react-native-executorch/android/CMakeLists.txt

@@ -37,9 +37,6 @@ target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE
 target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE
     ../cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/../third-party/include
-    # pytorch/tokenizers headers (and the third-party libs they pull in:
-    # nlohmann/json, re2 and its abseil dependency) ship inside the ExecuTorch
-    # llm extension bundle
     ${CMAKE_CURRENT_SOURCE_DIR}/../third-party/include/executorch/extension/llm/tokenizers/include
     ${CMAKE_CURRENT_SOURCE_DIR}/../third-party/include/executorch/extension/llm/tokenizers/third-party/json/include
     ${CMAKE_CURRENT_SOURCE_DIR}/../third-party/include/executorch/extension/llm/tokenizers/third-party/re2

packages/react-native-executorch/cpp/extensions/nlp/install.cpp

@@ -2,7 +2,13 @@
 #include "tokenizer.h"
 
 namespace rnexecutorch::extensions::nlp {
+namespace jsi = facebook::jsi;
+
 void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module) {
-    tokenizer::install_loadTokenizer(rt, module);
+    jsi::Object nlpModule = jsi::Object(rt);
+
+    tokenizer::install_loadTokenizer(rt, nlpModule);
+
+    module.setProperty(rt, "nlp", nlpModule);
 }
 } // namespace rnexecutorch::extensions::nlp

packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp

@@ -14,6 +14,34 @@ namespace {
 // (i.e. special tokens are added exactly as configured in tokenizer.json).
 constexpr uint64_t kNumAddedBosTokens = 0;
 constexpr uint64_t kNumAddedEosTokens = 0;
+
+// tokenizers::Error is its own enum (not executorch::runtime::Error), and the
+// tokenizers library ships no to_string for it, so map it to a readable name.
+std::string toString(tokenizers::Error error) {
+  switch (error) {
+  case tokenizers::Error::Ok:
+    return "Ok";
+  case tokenizers::Error::Internal:
+    return "Internal";
+  case tokenizers::Error::Uninitialized:
+    return "Uninitialized";
+  case tokenizers::Error::OutOfRange:
+    return "OutOfRange";
+  case tokenizers::Error::LoadFailure:
+    return "LoadFailure";
+  case tokenizers::Error::EncodeFailure:
+    return "EncodeFailure";
+  case tokenizers::Error::Base64DecodeFailure:
+    return "Base64DecodeFailure";
+  case tokenizers::Error::ParseFailure:
+    return "ParseFailure";
+  case tokenizers::Error::DecodeFailure:
+    return "DecodeFailure";
+  case tokenizers::Error::RegexFailure:
+    return "RegexFailure";
+  }
+  return "Unknown(" + std::to_string(static_cast<int32_t>(error)) + ")";
+}
 } // namespace
 
 TokenizerHostObject::TokenizerHostObject(const std::string &tokenizerPath)
@@ -22,7 +50,7 @@ TokenizerHostObject::TokenizerHostObject(const std::string &tokenizerPath)
     auto error = tokenizer_->load(tokenizerPath_);
     if (error != tokenizers::Error::Ok) {
         throw std::runtime_error("Failed to load tokenizer from '" + tokenizerPath_ +
-                                 "': error " + std::to_string(static_cast<int32_t>(error)));
+                                 "': " + toString(error));
     }
 }
 
@@ -56,8 +84,8 @@ jsi::Value TokenizerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam
             auto text = args[0].asString(rt).utf8(rt);
             auto result = self->tokenizer_->encode(text, kNumAddedBosTokens, kNumAddedEosTokens);
             if (!result.ok()) {
-                throw jsi::JSError(rt, "encode: Failed to encode input: error " +
-                                           std::to_string(static_cast<int32_t>(result.error())));
+                throw jsi::JSError(rt, "encode: Failed to encode input: " +
+                                           toString(result.error()));
             }
 
             const auto &ids = result.get();
@@ -74,16 +102,21 @@ jsi::Value TokenizerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam
     if (nameStr == "decode") {
         auto self = shared_from_this();
         auto fnBody = [self](jsi::Runtime &rt, const jsi::Value &thisVal, const jsi::Value *args, size_t count) -> jsi::Value {
-            if (count != 2) {
-                throw jsi::JSError(rt, "decode: Usage: decode(tokens, skipSpecialTokens)");
+            if (count < 1 || count > 2) {
+                throw jsi::JSError(rt, "decode: Usage: decode(tokens, skipSpecialTokens?)");
             }
 
             if (!args[0].isObject() || !args[0].asObject(rt).isArray(rt)) {
                 throw jsi::JSError(rt, "decode: Expected arg0 to be an array");
             }
 
-            if (!args[1].isBool()) {
-                throw jsi::JSError(rt, "decode: Expected arg1 to be a boolean");
+            // skipSpecialTokens is optional and defaults to true.
+            bool skipSpecialTokens = true;
+            if (count == 2 && !args[1].isUndefined()) {
+                if (!args[1].isBool()) {
+                    throw jsi::JSError(rt, "decode: Expected arg1 to be a boolean");
+                }
+                skipSpecialTokens = args[1].asBool();
             }
 
             std::unique_lock<std::mutex> lock(self->mutex_, std::try_to_lock);
@@ -96,7 +129,6 @@ jsi::Value TokenizerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam
             }
 
             auto tokensArray = args[0].asObject(rt).asArray(rt);
-            auto skipSpecialTokens = args[1].asBool();
 
             std::vector<uint64_t> tokens;
             tokens.reserve(tokensArray.size(rt));
@@ -108,15 +140,19 @@ jsi::Value TokenizerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam
                 tokens.push_back(static_cast<uint64_t>(val.asNumber()));
             }
 
+            if (tokens.empty()) {
+                return jsi::String::createFromUtf8(rt, "");
+            }
+
             auto result = self->tokenizer_->decode(tokens, skipSpecialTokens);
             if (!result.ok()) {
-                throw jsi::JSError(rt, "decode: Failed to decode tokens: error " +
-                                           std::to_string(static_cast<int32_t>(result.error())));
+                throw jsi::JSError(rt, "decode: Failed to decode tokens: " +
+                                           toString(result.error()));
             }
 
             return jsi::String::createFromUtf8(rt, result.get());
         };
-        return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "decode"), 2, fnBody);
+        return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "decode"), 1, fnBody);
     }
 
     if (nameStr == "getVocabSize") {
@@ -163,8 +199,8 @@ jsi::Value TokenizerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam
             auto tokenId = static_cast<uint64_t>(args[0].asNumber());
             auto result = self->tokenizer_->id_to_piece(tokenId);
             if (!result.ok()) {
-                throw jsi::JSError(rt, "idToToken: Failed to convert id to token: error " +
-                                           std::to_string(static_cast<int32_t>(result.error())));
+                throw jsi::JSError(rt, "idToToken: Failed to convert id to token: " +
+                                           toString(result.error()));
             }
 
             return jsi::String::createFromUtf8(rt, result.get());
@@ -195,8 +231,8 @@ jsi::Value TokenizerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam
             auto token = args[0].asString(rt).utf8(rt);
             auto result = self->tokenizer_->piece_to_id(token);
             if (!result.ok()) {
-                throw jsi::JSError(rt, "tokenToId: Failed to convert token to id: error " +
-                                           std::to_string(static_cast<int32_t>(result.error())));
+                throw jsi::JSError(rt, "tokenToId: Failed to convert token to id: " +
+                                           toString(result.error()));
             }
 
             return static_cast<double>(result.get());

packages/react-native-executorch/react-native-executorch.podspec

@@ -32,9 +32,6 @@ Pod::Spec.new do |s|
     "HEADER_SEARCH_PATHS" => [
       "\"$(PODS_TARGET_SRCROOT)/cpp\"",
       "\"$(PODS_TARGET_SRCROOT)/third-party/include\"",
-      # pytorch/tokenizers headers (and the third-party libs they pull in:
-      # nlohmann/json, re2 and its abseil dependency) ship inside the ExecuTorch
-      # llm extension bundle
       "\"$(PODS_TARGET_SRCROOT)/third-party/include/executorch/extension/llm/tokenizers/include\"",
       "\"$(PODS_TARGET_SRCROOT)/third-party/include/executorch/extension/llm/tokenizers/third-party/json/include\"",
       "\"$(PODS_TARGET_SRCROOT)/third-party/include/executorch/extension/llm/tokenizers/third-party/re2\"",

packages/react-native-executorch/src/extensions/nlp/index.ts

@@ -1,2 +1,2 @@
-export * from './ops/tokenizer';
-export * from './tasks/tokenizer';
+export * from './tokenizer';
+export * from './tasks/tokenization';

packages/react-native-executorch/src/extensions/nlp/ops/tokenizer.ts

@@ -0,0 +1,28 @@
+import type { WorkletRuntime } from 'react-native-worklets';
+
+import { wrapAsync } from '../../../core/runtime';
+import { loadTokenizer } from '../tokenizer';
+
+/**
+ * Loads a tokenizer and exposes its operations with lifetime management, for
+ * use by the `useTokenizer` hook and as a building block for higher-level NLP
+ * tasks (embeddings, privacy filter, ...).
+ * @category Typescript API
+ * @param tokenizerPath Absolute local path to a `tokenizer.json` file.
+ * @param runtime Optional worklet runtime thread to run the tokenizer on.
+ * @returns A promise resolving to the tokenizer operations and a `dispose`
+ * handle that releases the native tokenizer.
+ */
+export async function createTokenizer(tokenizerPath: string, runtime?: WorkletRuntime) {
+  const tokenizer = await wrapAsync(loadTokenizer, runtime)(tokenizerPath);
+  const dispose = () => tokenizer.dispose();
+
+  return {
+    dispose,
+    encode: wrapAsync(tokenizer.encode, runtime),
+    decode: wrapAsync(tokenizer.decode, runtime),
+    getVocabSize: tokenizer.getVocabSize,
+    idToToken: tokenizer.idToToken,
+    tokenToId: tokenizer.tokenToId,
+  };
+}

packages/react-native-executorch/src/extensions/nlp/tasks/tokenization.ts

@@ -0,0 +1,66 @@
+import { rnexecutorchJsi } from '../../native/bridge';
+
+declare const tokenizerBrand: unique symbol;
+
+/**
+ * A native HuggingFace-compatible tokenizer instance backed by a JSI host
+ * object. All methods are synchronous and worklet-compatible.
+ * @category Types
+ */
+export type Tokenizer = {
+  /** Absolute local path of the loaded `tokenizer.json`. */
+  readonly path: string;
+
+  /**
+   * Encodes a string into token ids (special tokens are added according to the
+   * tokenizer.json post_processor).
+   * @param text The input text to tokenize.
+   * @returns The encoded token ids.
+   */
+  encode(text: string): number[];
+
+  /**
+   * Decodes token ids back into a string.
+   * @param tokens The token ids to decode.
+   * @param skipSpecialTokens Whether to omit special tokens. Defaults to `true`.
+   * @returns The decoded text.
+   */
+  decode(tokens: number[], skipSpecialTokens?: boolean): string;
+
+  /**
+   * @returns The size of the tokenizer's vocabulary.
+   */
+  getVocabSize(): number;
+
+  /**
+   * @param id The token id to look up.
+   * @returns The token string for the given id.
+   */
+  idToToken(id: number): string;
+
+  /**
+   * @param token The token string to look up.
+   * @returns The id for the given token string.
+   */
+  tokenToId(token: string): number;
+
+  /** Releases the native tokenizer. The instance must not be used afterwards. */
+  dispose(): void;
+
+  /**
+   * Prevents plain JS objects from being cast as Tokenizers.
+   * @internal
+   */
+  readonly [tokenizerBrand]: never;
+};
+
+/**
+ * Loads a HuggingFace tokenizer from a local `tokenizer.json` file.
+ * @category Typescript API
+ * @param tokenizerPath Absolute local path to a `tokenizer.json` file.
+ * @returns The loaded tokenizer.
+ */
+export function loadTokenizer(tokenizerPath: string): Tokenizer {
+  'worklet';
+  return rnexecutorchJsi.nlp.loadTokenizer(tokenizerPath) as Tokenizer;
+}