Match keywords typed in Arabic script, and stop splitting combining marks

__tests__/keyword-matcher-arabic-script.test.ts

@@ -0,0 +1,103 @@
+/**
+ * Keyword Matcher — Arabic-script (Persian / Arabic / Urdu) coverage.
+ *
+ * Every case below returned `matched: false` before normalizeArabicScript
+ * existed, which made keyword campaigns unusable for a Persian-speaking
+ * account: the comment and the keyword look identical on screen and differ
+ * only by codepoint.
+ */
+
+import { describe, it, expect } from "vitest";
+import {
+  matchKeywords,
+  normalizeArabicScript,
+  stripSpecialCharacters,
+} from "../lib/utils/keyword-matcher";
+
+const matches = (comment: string, keyword: string, wholeWord = true) =>
+  matchKeywords(comment, [keyword], wholeWord).matched;
+
+describe("normalizeArabicScript", () => {
+  it("folds Arabic yeh and kaf onto their Persian forms", () => {
+    // U+064A + U+0643 in, U+06CC + U+06A9 out.
+    expect(normalizeArabicScript("لينك")).toBe("لینک");
+  });
+
+  it("folds alef variants and teh marbuta", () => {
+    expect(normalizeArabicScript("آإأ")).toBe("ااا");
+    expect(normalizeArabicScript("هدية")).toBe("هدیه");
+  });
+
+  it("drops harakat, tatweel and ZWNJ", () => {
+    expect(normalizeArabicScript("لِینک")).toBe("لینک");
+    expect(normalizeArabicScript("لیــنک")).toBe("لینک");
+    expect(normalizeArabicScript("قیمت‌ها")).toBe("قیمتها");
+  });
+
+  it("converts Persian and Arabic-Indic digits to ASCII", () => {
+    expect(normalizeArabicScript("۰۱۲۳۴۵۶۷۸۹")).toBe("0123456789");
+    expect(normalizeArabicScript("٠١٢٣٤٥٦٧٨٩")).toBe("0123456789");
+  });
+
+  it("leaves Latin and other scripts untouched", () => {
+    expect(normalizeArabicScript("Клод link 链接")).toBe("Клод link 链接");
+  });
+});
+
+describe("matchKeywords — Persian comments", () => {
+  it("matches a Persian keyword against an Arabic-keyboard comment", () => {
+    expect(matches("لينك بده لطفا", "لینک")).toBe(true);
+  });
+
+  it("matches in the other direction too", () => {
+    expect(matches("لینک بده لطفا", "لينك")).toBe(true);
+  });
+
+  it("matches across the ZWNJ spelling split", () => {
+    expect(matches("قیمت‌ها چنده؟", "قیمتها")).toBe(true);
+    expect(matches("قیمتها چنده؟", "قیمت‌ها")).toBe(true);
+  });
+
+  it("matches Persian digits against an ASCII-digit keyword", () => {
+    expect(matches("کد۵ رو میخوام", "کد5")).toBe(true);
+  });
+
+  it("matches through kashida stretching and harakat", () => {
+    expect(matches("لیــنک", "لینک")).toBe(true);
+    expect(matches("لِینک", "لینک")).toBe(true);
+  });
+
+  it("still respects whole-word boundaries in Persian", () => {
+    expect(matches("لینکدونی", "لینک", true)).toBe(false);
+    expect(matches("لینکدونی", "لینک", false)).toBe(true);
+  });
+
+  it("does not match an unrelated Persian word", () => {
+    expect(matches("سلام خوبی", "لینک")).toBe(false);
+  });
+
+  it("strips Persian punctuation around the keyword", () => {
+    expect(matches("سلام، لینک؟", "لینک")).toBe(true);
+  });
+});
+
+describe("stripSpecialCharacters — combining marks survive", () => {
+  it("keeps a decomposed Latin diacritic joined to its base", () => {
+    // Was "sen or" before \p{M} was added to the keep-set.
+    expect(stripSpecialCharacters("señor".normalize("NFD"))).toBe(
+      "señor".normalize("NFD")
+    );
+  });
+
+  it("keeps Devanagari vowel signs", () => {
+    // Built by codepoint on purpose: an editor bidi-reorders this string,
+    // so a second hand-typed copy silently drifts by a character.
+    const kitab = "\u0915\u093f\u0924\u093e\u092c"; // kitab
+    expect(stripSpecialCharacters(kitab)).toBe(kitab);
+  });
+
+  it("matches a decomposed accented comment against a plain keyword", () => {
+    expect(matches("señor".normalize("NFD"), "senor")).toBe(true);
+    expect(matches("preço".normalize("NFD"), "preco")).toBe(true);
+  });
+});

lib/utils/keyword-matcher.ts

@@ -27,6 +27,55 @@ export interface KeywordMatchResult {
   matchedKeyword: string | null;
 }
 
+/**
+ * Canonicalise Arabic-script text (Persian, Arabic, Urdu) before matching.
+ *
+ * `foldDiacritics` deliberately leaves non-Latin marks alone because they are
+ * load bearing in Devanagari, Thai and Japanese. In the Arabic script they are
+ * not: the letter variants below are the *same letter* typed on a different
+ * keyboard, and harakat are optional vocalisation almost nobody types. Without
+ * this step an Iranian account keyed on "لینک" misses every commenter whose
+ * phone sends the Arabic yeh and kaf (U+064A / U+0643) instead of the Persian
+ * ones (U+06CC / U+06A9) — the two strings look identical on screen and never
+ * compare equal. The same holds for "کد۵" against a "کد5" keyword.
+ *
+ * Applied to both sides of the comparison, so it never matters which form the
+ * account owner typed into the campaign builder.
+ */
+// Read this table by codepoint, not by eye: several sources render identically
+// to their targets (U+064A vs U+06CC, U+0643 vs U+06A9) and the last rule
+// matches characters that render as nothing at all.
+const ARABIC_SCRIPT_FOLDING: Array<[RegExp, string]> = [
+  // Same letter, different keyboard layout.
+  [/[يىے]/gu, "ی"], // Arabic yeh, alef maksura, barree ye
+  [/ك/gu, "ک"], // Arabic kaf -> Persian keheh
+  [/ة/gu, "ه"], // teh marbuta -> heh
+  [/[آأإٱ]/gu, "ا"], // alef w/ madda or hamza -> alef
+  // Optional vocalisation and typographic padding: never semantic in Persian.
+  [/[ً-ْٰ]/gu, ""], // harakat, sukun, superscript alef
+  [/ـ/gu, ""], // tatweel / kashida stretching
+  // ZWNJ is a rendering hint and half of Instagram types it while half does
+  // not, so "قیمت‌ها" and "قیمتها" have to compare equal. Deleted rather than
+  // turned into a space, because the no-separator spelling is the fallback
+  // people actually type. Bidi marks go with it — they carry no meaning.
+  [/[‌‎‏]/gu, ""],
+];
+
+// Persian (U+06F0..) and Arabic-Indic (U+0660..) digit blocks, both ordered 0-9.
+const EASTERN_DIGITS = /[۰-۹٠-٩]/gu;
+
+export function normalizeArabicScript(text: string): string {
+  let out = text;
+  for (const [pattern, replacement] of ARABIC_SCRIPT_FOLDING) {
+    out = out.replace(pattern, replacement);
+  }
+  return out.replace(EASTERN_DIGITS, (digit) => {
+    const code = digit.codePointAt(0)!;
+    const zero = code >= 0x06f0 ? 0x06f0 : 0x0660;
+    return String(code - zero);
+  });
+}
+
 /**
  * Strip emojis and special characters from text, keeping only
  * letters (any script), numbers, and whitespace.
@@ -37,8 +86,14 @@ export function stripSpecialCharacters(text: string): string {
       /[\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F1E0}-\u{1F1FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}\u{FE00}-\u{FE0F}\u{1F900}-\u{1F9FF}\u{1FA00}-\u{1FA6F}\u{1FA70}-\u{1FAFF}\u{200D}\u{20E3}]/gu,
       ""
     )
-    // Keep letters (any script) and numbers; turn everything else into a space.
-    .replace(/[^\p{L}\p{N}\s]/gu, " ")
+    // Keep letters (any script), numbers, and combining marks; turn everything
+    // else into a space. `\p{M}` has to be kept here or this replace undoes the
+    // work foldDiacritics does further down the pipeline: a combining mark is
+    // neither a letter nor a number, so without it "señor" typed in NFD becomes
+    // "sen or", "किताब" becomes "क त ब", and Arabic harakat split every
+    // vocalised Persian word into fragments. Marks survive this step and
+    // foldDiacritics then decides, per script, which ones to drop.
+    .replace(/[^\p{L}\p{N}\p{M}\s]/gu, " ")
     .replace(/\s+/g, " ")
     .trim();
 }
@@ -100,7 +155,7 @@ export function matchKeywords(
   }
 
   const cleanedText = foldDiacritics(
-    stripSpecialCharacters(commentText)
+    stripSpecialCharacters(normalizeArabicScript(commentText))
   ).toLowerCase();
 
   if (!cleanedText) {
@@ -109,7 +164,7 @@ export function matchKeywords(
 
   for (const keyword of keywords) {
     const cleanedKeyword = foldDiacritics(
-      stripSpecialCharacters(keyword)
+      stripSpecialCharacters(normalizeArabicScript(keyword))
     ).toLowerCase();
 
     if (!cleanedKeyword) continue;