From 4de1a86e3747d83e7f08636e03d1bea7cc2234ca Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 7 Apr 2026 11:09:39 +0200 Subject: [PATCH 1/4] wip --- .../common/rnexecutorch/models/llm/LLM.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp b/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp index 64e94c2ff0..98e083ea7b 100644 --- a/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp +++ b/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp @@ -20,7 +20,7 @@ using executorch::runtime::Error; LLM::LLM(const std::string &modelSource, const std::string &tokenizerSource, std::vector capabilities, std::shared_ptr callInvoker) - : BaseModel(modelSource, callInvoker, Module::LoadMode::File) { + : BaseModel(modelSource, callInvoker, Module::LoadMode::Mmap) { if (capabilities.empty()) { runner_ = @@ -194,9 +194,7 @@ int32_t LLM::countTextTokens(std::string text) const { return runner_->count_text_tokens(text); } -size_t LLM::getMemoryLowerBound() const noexcept { - return memorySizeLowerBound; -} +size_t LLM::getMemoryLowerBound() const noexcept { return 0; } void LLM::setCountInterval(size_t countInterval) { if (!runner_ || !runner_->is_loaded()) { From 1aac7aa72c052f6e95d36188596459f30c0fdeb9 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 8 Apr 2026 11:54:36 +0200 Subject: [PATCH 2/4] fix: update memoryLowerBound for llm --- .../common/rnexecutorch/models/llm/LLM.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp b/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp index 98e083ea7b..648ac0aee9 100644 --- a/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp +++ b/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp @@ -1,5 +1,6 @@ #include "LLM.h" +#include #include #include #include @@ -42,8 +43,12 @@ LLM::LLM(const std::string &modelSource, const std::string &tokenizerSource, throw RnExecutorchError(loadResult, "Failed to load LLM runner"); } - memorySizeLowerBound = fs::file_size(fs::path(modelSource)) + - fs::file_size(fs::path(tokenizerSource)); + // I am purposefully not adding file size of the model here. The reason is + // that Hermes would crash the app if we try to alloc too much memory here. + // Also, given we're using mmap, the true memory consumption of a model is not + // really equal to the size of the model. The size of the tokenizer file is a + // hint to the GC that this object might be worth getting rid of. + memorySizeLowerBound = fs::file_size(fs::path(tokenizerSource)); } std::string LLM::generate(std::string input, @@ -194,7 +199,9 @@ int32_t LLM::countTextTokens(std::string text) const { return runner_->count_text_tokens(text); } -size_t LLM::getMemoryLowerBound() const noexcept { return 0; } +size_t LLM::getMemoryLowerBound() const noexcept { + return memorySizeLowerBound; +}; void LLM::setCountInterval(size_t countInterval) { if (!runner_ || !runner_->is_loaded()) { From aeac24974d5a25455e77c6db83f9e07f571c832a Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 8 Apr 2026 12:02:40 +0200 Subject: [PATCH 3/4] chore: remove unused header --- .../common/rnexecutorch/models/llm/LLM.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp b/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp index 648ac0aee9..cf7f311cba 100644 --- a/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp +++ b/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp @@ -1,6 +1,5 @@ #include "LLM.h" -#include #include #include #include From e76e2cc35008e2408591e8ccd0703ff13224c4fb Mon Sep 17 00:00:00 2001 From: Jakub Chmura <92989966+chmjkb@users.noreply.github.com> Date: Wed, 8 Apr 2026 12:23:16 +0200 Subject: [PATCH 4/4] Update packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com> --- .../common/rnexecutorch/models/llm/LLM.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp b/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp index cf7f311cba..10aff5cc77 100644 --- a/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp +++ b/packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp @@ -200,7 +200,7 @@ int32_t LLM::countTextTokens(std::string text) const { size_t LLM::getMemoryLowerBound() const noexcept { return memorySizeLowerBound; -}; +} void LLM::setCountInterval(size_t countInterval) { if (!runner_ || !runner_->is_loaded()) {