feat(fts): add MeCab tokenizer for Japanese + tokenizer registry refactor

fts/CMakeLists.txt

@@ -1,10 +1,33 @@
+# Vendored dependencies live in the extensions repo root's third_party/.
+# Use CMAKE_CURRENT_LIST_DIR (not PROJECT_SOURCE_DIR) so this also resolves
+# when built under the engine's build system, where PROJECT_SOURCE_DIR points
+# at the engine checkout.
+set(FTS_EXTENSIONS_ROOT ${CMAKE_CURRENT_LIST_DIR}/../..)
+
 include_directories(
         ${PROJECT_SOURCE_DIR}/src/include
         ${CMAKE_BINARY_DIR}/src/include
         src/include
         third_party/snowball/libstemmer
-        ${PROJECT_SOURCE_DIR}/third_party/cppjieba/include
-        ${PROJECT_SOURCE_DIR}/third_party/cppjieba/deps/limonp/include)
+        ${FTS_EXTENSIONS_ROOT}/third_party/cppjieba/include
+        ${FTS_EXTENSIONS_ROOT}/third_party/cppjieba/deps/limonp/include
+        ${FTS_EXTENSIONS_ROOT}/third_party/mecab/src)
+
+# cppjieba (jieba tokenizer dependency) is a header-only interface library;
+# upstream referenced it but never vendored it, so define the target here.
+if(NOT TARGET cppjieba)
+    add_library(cppjieba INTERFACE)
+    target_include_directories(cppjieba INTERFACE
+            ${FTS_EXTENSIONS_ROOT}/third_party/cppjieba/include
+            ${FTS_EXTENSIONS_ROOT}/third_party/cppjieba/deps/limonp/include)
+endif()
+
+# MeCab (Japanese tokenizer): static lib + mecab-dict-index tool + build-time
+# ipadic dictionary compilation (downloads the dictionary CSV at configure).
+# The source dir is outside this extension's tree, so an explicit binary dir
+# is required (this also applies when built under the engine's build system).
+add_subdirectory(${FTS_EXTENSIONS_ROOT}/third_party/mecab
+        ${CMAKE_CURRENT_BINARY_DIR}/mecab)
 
 add_subdirectory(src/catalog)
 add_subdirectory(src/function)
@@ -15,16 +38,29 @@ add_subdirectory(src/utils)
 
 add_subdirectory(third_party/snowball)
 
+# tokenizer.cpp includes mecab.h, which requires DLL_EXPORT to declare the
+# MeCab API without __declspec(dllimport) on Windows.
+target_compile_definitions(lbug_fts_utils PRIVATE DLL_EXPORT)
+
 build_extension_lib(${BUILD_STATIC_EXTENSION} "fts")
 
 target_link_libraries(lbug_${EXTENSION_LIB_NAME}_extension
         PRIVATE
         snowball
         re2
-        cppjieba)
+        cppjieba
+        mecab)
 
 # Copy Jieba dictionaries next to the built extension so runtime can find a default path
 add_custom_command(TARGET lbug_${EXTENSION_LIB_NAME}_extension POST_BUILD
         COMMAND ${CMAKE_COMMAND} -E copy_directory
         ${PROJECT_SOURCE_DIR}/third_party/cppjieba/dict
         ${PROJECT_SOURCE_DIR}/extension/fts/build/dict)
+
+# Copy the compiled ipadic dictionary (UTF-8) next to the built extension so
+# the default mecab_dict_dir resolves at runtime.
+add_custom_command(TARGET lbug_${EXTENSION_LIB_NAME}_extension POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_directory
+        ${CMAKE_BINARY_DIR}/mecab-dict/ipadic
+        ${PROJECT_SOURCE_DIR}/extension/fts/build/dict-ipadic)
+add_dependencies(lbug_${EXTENSION_LIB_NAME}_extension mecab-ipadic-dict)

fts/src/function/create_fts_index.cpp

@@ -157,13 +157,22 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData&
     std::string query = "";
     if (!catalog::Catalog::Get(context)->containsMacro(transaction::Transaction::Get(context),
             FTSUtils::getTokenizeMacroName(tableID, indexName))) {
-        // TOKENIZE(text, tokenizer, extra_param)
+        // TOKENIZE(text, tokenizer, extra_param); the extra param is the
+        // dictionary directory for dictionary-based tokenizers. Pick it by
+        // tokenizer name (the params map holds defaults for all tokenizers).
+        auto& tokenizerParams = ftsBindData->createFTSConfig.tokenizerInfo.params;
+        auto& tokenizerName = ftsBindData->createFTSConfig.tokenizerInfo.tokenizer;
+        std::string extraParam = "";
+        if (tokenizerName == "jieba") {
+            extraParam = tokenizerParams.at("jieba_dict_dir");
+        } else if (tokenizerName == "mecab") {
+            extraParam = tokenizerParams.at("mecab_dict_dir");
+        }
         query += std::format(R"(CREATE MACRO `{}`(query) AS
                             TOKENIZE(lower(regexp_replace(CAST(query as STRING), '{}', ' ', 'g')), '{}', '{}');)",
             FTSUtils::getTokenizeMacroName(tableID, indexName),
             formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern),
-            ftsBindData->createFTSConfig.tokenizerInfo.tokenizer,
-            ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir);
+            ftsBindData->createFTSConfig.tokenizerInfo.tokenizer, extraParam);
     }
 
     // Create the stop words table if not exists, or the user is not using the default english
@@ -236,7 +245,17 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData&
     std::string params;
     params += std::format("stemmer := '{}', ", ftsBindData->createFTSConfig.stemmer);
     params +=
-        std::format("stopWords := '{}'", ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords);
+        std::format("stopWords := '{}', ", ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords);
+    // The internal index uses this config for incremental inserts/updates and
+    // queries, so the tokenizer (and its parameters) must be forwarded or it
+    // silently falls back to the default 'simple' tokenizer.
+    params += std::format("tokenizer := '{}'", ftsBindData->createFTSConfig.tokenizerInfo.tokenizer);
+    auto& ftsTokenizerParams = ftsBindData->createFTSConfig.tokenizerInfo.params;
+    if (ftsBindData->createFTSConfig.tokenizerInfo.tokenizer == "jieba") {
+        params += std::format(", jieba_dict_dir := '{}'", ftsTokenizerParams.at("jieba_dict_dir"));
+    } else if (ftsBindData->createFTSConfig.tokenizerInfo.tokenizer == "mecab") {
+        params += std::format(", mecab_dict_dir := '{}'", ftsTokenizerParams.at("mecab_dict_dir"));
+    }
     query += std::format("CALL _CREATE_FTS_INDEX('{}', '{}', {}, {});", tableName, indexName,
         properties, params);
     query += std::format("RETURN 'Index {} has been created.' as result;", ftsBindData->indexName);

fts/src/function/fts_config.cpp

@@ -156,8 +156,11 @@ CreateFTSConfig::CreateFTSConfig(main::ClientContext& context, common::table_id_
             Tokenizer::validate(tokenizerInfo.tokenizer);
         } else if (lowerCaseName == "jieba_dict_dir") {
             value.validateType(common::LogicalTypeID::STRING);
-            tokenizerInfo.jiebaDictDir =
-                common::StringUtils::getLower(value.getValue<std::string>());
+            // Note: the dict dir is a file path and must not be lower-cased.
+            tokenizerInfo.params["jieba_dict_dir"] = value.getValue<std::string>();
+        } else if (lowerCaseName == "mecab_dict_dir") {
+            value.validateType(common::LogicalTypeID::STRING);
+            tokenizerInfo.params["mecab_dict_dir"] = value.getValue<std::string>();
         } else {
             throw common::BinderException{"Unrecognized optional parameter: " + name};
         }
@@ -166,17 +169,25 @@ CreateFTSConfig::CreateFTSConfig(main::ClientContext& context, common::table_id_
 
 FTSConfig CreateFTSConfig::getFTSConfig() const {
     return FTSConfig{stemmer, stopWordsTableInfo.tableName, stopWordsTableInfo.stopWords,
-        ignorePattern, ignorePatternQuery, tokenizerInfo.tokenizer, tokenizerInfo.jiebaDictDir};
+        ignorePattern, ignorePatternQuery, tokenizerInfo.tokenizer, tokenizerInfo.params};
 }
 
+// Magic marker written before the tokenizer params so that deserialization can
+// tell apart new catalogs (marker + unordered map) from legacy ones (a single
+// "jiebaDictDir" string field).
+namespace {
+constexpr const char* TOKENIZER_PARAMS_MAGIC = "lbug_tokenizer_params_v1";
+} // namespace
+
 void FTSConfig::serialize(common::Serializer& serializer) const {
     serializer.serializeValue(stemmer);
     serializer.serializeValue(stopWordsTableName);
     serializer.serializeValue(stopWordsSource);
     serializer.serializeValue(ignorePattern);
     serializer.serializeValue(ignorePatternQuery);
     serializer.serializeValue(tokenizer);
-    serializer.serializeValue(jiebaDictDir);
+    serializer.serializeValue(std::string{TOKENIZER_PARAMS_MAGIC});
+    serializer.serializeUnorderedMap(tokenizerParams);
 }
 
 FTSConfig FTSConfig::deserialize(common::Deserializer& deserializer) {
@@ -187,7 +198,14 @@ FTSConfig FTSConfig::deserialize(common::Deserializer& deserializer) {
     deserializer.deserializeValue(config.ignorePattern);
     deserializer.deserializeValue(config.ignorePatternQuery);
     deserializer.deserializeValue(config.tokenizer);
-    deserializer.deserializeValue(config.jiebaDictDir);
+    std::string tokenizerParamsField;
+    deserializer.deserializeValue(tokenizerParamsField);
+    if (tokenizerParamsField == TOKENIZER_PARAMS_MAGIC) {
+        deserializer.deserializeUnorderedMap(config.tokenizerParams);
+    } else {
+        // Legacy catalog: the field after tokenizer was the jieba dict dir.
+        config.tokenizerParams["jieba_dict_dir"] = tokenizerParamsField;
+    }
     return config;
 }
 
@@ -214,7 +232,7 @@ void TopK::validate(uint64_t value) {
 }
 
 void Tokenizer::validate(const std::string& tokenizer) {
-    if (tokenizer == "simple" || tokenizer == "jieba") {
+    if (TokenizerRegistry::isSupported(tokenizer)) {
         return;
     }
     throw common::BinderException{

fts/src/function/tokenize.cpp

@@ -6,27 +6,28 @@
 #include "common/types/string_t.h"
 #include "common/types/types.h"
 #include "common/vector/value_vector.h"
-#include "cppjieba/Jieba.hpp"
 #include "expression_evaluator/expression_evaluator_utils.h"
 #include "function/scalar_function.h"
 #include "re2.h"
+#include "utils/tokenizer.h"
 
 namespace lbug {
 namespace fts_extension {
 
 using namespace function;
 using namespace common;
 
-struct JiebaBindData final : public FunctionBindData {
-    std::shared_ptr<cppjieba::Jieba> jieba;
+struct TokenizerBindData final : public FunctionBindData {
+    std::shared_ptr<const ITokenizer> tokenizer;
 
-    JiebaBindData(common::logical_type_vec_t paramTypes, std::shared_ptr<cppjieba::Jieba> jieba)
+    TokenizerBindData(common::logical_type_vec_t paramTypes,
+        std::shared_ptr<const ITokenizer> tokenizer)
         : FunctionBindData{std::move(paramTypes),
               common::LogicalType::LIST(common::LogicalType::STRING())},
-          jieba{std::move(jieba)} {}
+          tokenizer{std::move(tokenizer)} {}
 
     std::unique_ptr<FunctionBindData> copy() const override {
-        return std::make_unique<JiebaBindData>(copyVector(paramTypes), jieba);
+        return std::make_unique<TokenizerBindData>(copyVector(paramTypes), tokenizer);
     }
 };
 
@@ -38,21 +39,11 @@ static void addTokensToVector(const std::vector<std::string>& tokens, list_entry
     }
 }
 
-struct JiebaTokenizer {
+struct TokenizeOp {
     static void operation(string_t& text, string_t& /*tokenizerName*/, string_t& /*extraParam*/,
         list_entry_t& result, common::ValueVector& resultVector, void* dataPtr) {
-        std::vector<std::string> tokens;
-        auto bindData = reinterpret_cast<JiebaBindData*>(dataPtr);
-        bindData->jieba->CutForSearch(text.getAsString(), tokens);
-        addTokensToVector(tokens, result, resultVector);
-    }
-};
-
-struct SimpleTokenizer {
-    static void operation(string_t& text, string_t& /*tokenizerName*/, string_t& /*extraParam*/,
-        list_entry_t& result, common::ValueVector& resultVector, void* /*dataPtr*/) {
-        auto tokens =
-            StringUtils::split(text.getAsString(), " ", true /* ignoreEmptyStringParts */);
+        auto bindData = reinterpret_cast<TokenizerBindData*>(dataPtr);
+        auto tokens = bindData->tokenizer->tokenize(text.getAsString());
         addTokensToVector(tokens, result, resultVector);
     }
 };
@@ -62,37 +53,31 @@ static std::unique_ptr<FunctionBindData> bindFunc(const ScalarBindFuncInput& inp
         throw BinderException{"The tokenizer parameter must be a literal expression."};
     }
     if (input.arguments[2]->expressionType != ExpressionType::LITERAL) {
-        throw BinderException{"The path to the jieba dict directory must be a literal expression."};
+        throw BinderException{"The tokenizer parameter must be a literal expression."};
     }
     auto value = evaluator::ExpressionEvaluatorUtils::evaluateConstantExpression(input.arguments[1],
         input.context);
-    auto tokenizer = common::StringUtils::getLower(value.getValue<std::string>());
-    if (tokenizer == "jieba") {
-        std::string dictDir = evaluator::ExpressionEvaluatorUtils::evaluateConstantExpression(
-            input.arguments[2], input.context)
-                                  .getValue<std::string>();
-        std::string dict = dictDir + "/jieba.dict.utf8";
-        std::string hmm = dictDir + "/hmm_model.utf8";
-        std::string user = dictDir + "/user.dict.utf8"; // Contains custom AI/ML terms
-        std::string idf = dictDir + "/idf.utf8";
-        std::string stop = dictDir + "/stop_words.utf8";
-        auto jieba = std::make_unique<cppjieba::Jieba>(dict, hmm, user, idf, stop);
-        input.definition->ptrCast<ScalarFunction>()->execFunc =
-            ScalarFunction::TernaryRegexExecFunction<string_t, string_t, string_t, list_entry_t,
-                JiebaTokenizer>;
-        return std::make_unique<JiebaBindData>(
-            binder::ExpressionUtil::getDataTypes(input.arguments), std::move(jieba));
-    } else if (tokenizer == "simple" || tokenizer == "") {
-        input.definition->ptrCast<ScalarFunction>()->execFunc =
-            ScalarFunction::TernaryRegexExecFunction<string_t, string_t, string_t, list_entry_t,
-                SimpleTokenizer>;
-        return FunctionBindData::getSimpleBindData(input.arguments,
-            LogicalType::LIST(LogicalType::STRING()));
-    } else {
-        throw common::BinderException{
-            "Unsupported tokenizer: " + tokenizer +
-            ".\nSupported tokenizers: 'simple' (default), 'jieba' (advanced Chinese)"};
+    auto tokenizerName = common::StringUtils::getLower(value.getValue<std::string>());
+    if (tokenizerName.empty()) {
+        tokenizerName = "simple";
+    }
+    // The third argument is the tokenizer's extra parameter; for 'jieba' it is
+    // the dictionary directory.
+    auto extraParam = evaluator::ExpressionEvaluatorUtils::evaluateConstantExpression(
+        input.arguments[2], input.context)
+                          .getValue<std::string>();
+    TokenizerParams params;
+    if (tokenizerName == "jieba") {
+        params["jieba_dict_dir"] = extraParam;
+    } else if (tokenizerName == "mecab") {
+        params["mecab_dict_dir"] = extraParam;
     }
+    auto tokenizer = TokenizerPool::getOrCreate(tokenizerName, params);
+    input.definition->ptrCast<ScalarFunction>()->execFunc =
+        ScalarFunction::TernaryRegexExecFunction<string_t, string_t, string_t, list_entry_t,
+            TokenizeOp>;
+    return std::make_unique<TokenizerBindData>(
+        binder::ExpressionUtil::getDataTypes(input.arguments), std::move(tokenizer));
 }
 
 function::function_set TokenizeFunction::getFunctionSet() {

fts/src/include/function/fts_config.h

@@ -4,6 +4,7 @@
 
 #include "common/types/types.h"
 #include "function/table/bind_input.h"
+#include "utils/tokenizer.h"
 #include <format>
 
 namespace lbug {
@@ -65,9 +66,14 @@ struct Tokenizer {
 
 struct TokenizerInfo {
     std::string tokenizer = Tokenizer::DEFAULT_VALUE;
-    std::string jiebaDictDir = std::format("{}/extension/fts/build/dict", LBUG_ROOT_DIRECTORY);
-
-    TokenizerInfo() = default;
+    // Tokenizer-specific parameters (e.g. "jieba_dict_dir" for the jieba tokenizer).
+    TokenizerParams params;
+
+    TokenizerInfo() {
+        params["jieba_dict_dir"] = std::format("{}/extension/fts/build/dict", LBUG_ROOT_DIRECTORY);
+        params["mecab_dict_dir"] =
+            std::format("{}/extension/fts/build/dict-ipadic", LBUG_ROOT_DIRECTORY);
+    }
 };
 
 struct FTSConfig;
@@ -95,16 +101,19 @@ struct FTSConfig {
     std::string ignorePattern = "";
     std::string ignorePatternQuery = "";
     std::string tokenizer = "";
-    std::string jiebaDictDir = "";
+    // Serialized as an unordered map after a magic marker; entries of legacy
+    // catalogs (which stored a single "jiebaDictDir" string here) are folded
+    // into tokenizerParams["jieba_dict_dir"] at deserialization time.
+    TokenizerParams tokenizerParams = {};
 
     FTSConfig() = default;
     FTSConfig(std::string stemmer, std::string stopWordsTableName, std::string stopWordsSource,
         std::string ignorePattern, std::string ignorePatternQuery, std::string tokenizer,
-        std::string jiebaDictDir)
+        TokenizerParams tokenizerParams)
         : stemmer{std::move(stemmer)}, stopWordsTableName{std::move(stopWordsTableName)},
           stopWordsSource{std::move(stopWordsSource)}, ignorePattern{std::move(ignorePattern)},
           ignorePatternQuery{std::move(ignorePatternQuery)}, tokenizer{std::move(tokenizer)},
-          jiebaDictDir{std::move(jiebaDictDir)} {}
+          tokenizerParams{std::move(tokenizerParams)} {}
 
     void serialize(common::Serializer& serializer) const;
 

fts/src/include/utils/fts_utils.h

@@ -58,7 +58,8 @@ struct FTSUtils {
         return std::format("{}_tokenize", getInternalTablePrefix(tableID, indexName));
     }
 
-    static std::vector<std::string> tokenizeString(std::string& str, const FTSConfig& tokenizer);
+    static std::vector<std::string> tokenizeString(const std::string& str,
+        const FTSConfig& tokenizer);
 };
 
 } // namespace fts_extension