diff --git a/CHANGELOG.md b/CHANGELOG.md index cd3c11d97..a557a1d2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. ## [unreleased] ### Analyzer +#### Added +- Added `NgBvnRecognizer` for Nigerian Bank Verification Numbers (`NG_BVN`). The BVN is an 11-digit identifier issued by the CBN and managed by NIBSS. No public checksum is documented; confidence is context-driven via `LemmaContextAwareEnhancer`. Registered in `default_recognizers.yaml` (disabled by default). (#2169, Thanks @kingztech2019) +- Added `KeNationalIdRecognizer` for Kenyan National Identity Card numbers (`KE_NATIONAL_ID`). The card number is a 7- or 8-digit identifier issued by the National Registration Bureau. No public checksum is documented; confidence is context-driven. Registered in `default_recognizers.yaml` (disabled by default). (#2169, Thanks @kingztech2019) + #### Fixed - `PhoneRecognizer.DEFAULT_SUPPORTED_REGIONS` used `"UK"`, which is not a valid `phonenumbers` (libphonenumber) region code — region codes are ISO 3166-1 alpha-2, where the United Kingdom is `"GB"`. The `"UK"` entry was a no-op, so UK numbers in national/local format (e.g. `020 7946 0958`) were never detected by default; only international-format `+44 …` numbers matched, because they carry the country code and match under any region. Replaced `"UK"` with `"GB"`. - Language model recognizers (`BasicLangExtractRecognizer`, `AzureOpenAILangExtractRecognizer`) configured in a recognizer registry YAML now honour `config_path` (and other recognizer-specific kwargs). Previously these entries were validated by the strict `PredefinedRecognizerConfig` schema, which has no `config_path` field and does not allow extra keys, so `config_path` was silently dropped and the recognizer fell back to its bundled default model configuration. Added a `LangExtractRecognizerConfig` model (`extra="allow"`) and registered both recognizer class names in `CONFIG_MODEL_MAP`. diff --git a/docs/supported_entities.md b/docs/supported_entities.md index e04f770f0..4183931a2 100644 --- a/docs/supported_entities.md +++ b/docs/supported_entities.md @@ -118,8 +118,14 @@ For more information, refer to the [adding new recognizers documentation](analyz | FieldType | Description | Detection Method | |------------|---------------------------------------------------------------------------------------------------------|------------------------------------------| | NG_NIN | The Nigerian National Identification Number (NIN) is a unique 11-digit number issued by the National Identity Management Commission (NIMC). | Pattern match, context, and checksum | +| NG_BVN | The Nigerian Bank Verification Number (BVN) is an 11-digit identifier issued by the Central Bank of Nigeria (CBN) and managed by NIBSS. It links every bank account held by an individual to a single biometric record. No public checksum algorithm is documented; confidence is context-driven. | Pattern match and context | | NG_VEHICLE_REGISTRATION | Nigerian vehicle registration plate number in the current format (2011+): 3 letters (LGA code), 3 digits (serial), 2 letters (year/batch). | Pattern match and context | +### Kenya +| FieldType | Description | Detection Method | +|------------|---------------------------------------------------------------------------------------------------------|------------------------------------------| +| KE_NATIONAL_ID | The Kenyan National Identity Card number is a 7- or 8-digit sequential identifier issued by the National Registration Bureau under the Registration of Persons Act (Cap. 107). No public checksum algorithm is documented; confidence is context-driven. | Pattern match and context | + ### Philippines | FieldType | Description | Detection Method | |------------|---------------------------------------------------------------------------------------------------------|------------------------------------------| diff --git a/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml b/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml index a82c62dda..a38b10363 100644 --- a/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml +++ b/presidio-analyzer/presidio_analyzer/conf/default_recognizers.yaml @@ -181,6 +181,13 @@ recognizers: enabled: false country_code: ng + - name: NgBvnRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: ng + - name: InPanRecognizer supported_languages: - en @@ -188,6 +195,13 @@ recognizers: enabled: false country_code: in + - name: KeNationalIdRecognizer + supported_languages: + - en + type: predefined + enabled: false + country_code: ke + - name: InAadhaarRecognizer supported_languages: - en diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py index 3715da7c3..6c310966e 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/__init__.py @@ -70,11 +70,15 @@ from .country_specific.korea.kr_rrn_recognizer import KrRrnRecognizer # Nigeria recognizers +from .country_specific.nigeria.ng_bvn_recognizer import NgBvnRecognizer from .country_specific.nigeria.ng_nin_recognizer import NgNinRecognizer from .country_specific.nigeria.ng_vehicle_registration_recognizer import ( NgVehicleRegistrationRecognizer, ) +# Kenya recognizers +from .country_specific.kenya.ke_national_id_recognizer import KeNationalIdRecognizer + # Philippines recognizers from .country_specific.philippines.ph_tin_recognizer import PhTinRecognizer from .country_specific.philippines.ph_umid_recognizer import PhUmidRecognizer @@ -255,6 +259,8 @@ "AzureOpenAILangExtractRecognizer", "BasicLangExtractRecognizer", "KrPassportRecognizer", + "KeNationalIdRecognizer", + "NgBvnRecognizer", "NgNinRecognizer", "NgVehicleRegistrationRecognizer", "MedicalNERRecognizer", diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/kenya/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/kenya/__init__.py new file mode 100644 index 000000000..5063b3f91 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/kenya/__init__.py @@ -0,0 +1,8 @@ +"""Kenya-specific recognizers.""" + +from .ke_national_id_recognizer import KeNationalIdRecognizer + +__all__ = [ + "KeNationalIdRecognizer", +] + diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/kenya/ke_national_id_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/kenya/ke_national_id_recognizer.py new file mode 100644 index 000000000..08c0a9426 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/kenya/ke_national_id_recognizer.py @@ -0,0 +1,91 @@ +from typing import List, Optional + +from presidio_analyzer import Pattern, PatternRecognizer + + +class KeNationalIdRecognizer(PatternRecognizer): + """ + Recognizes Kenyan National Identity Card numbers. + + The Kenyan National Identity Card (ID card) is issued by the National + Registration Bureau under the Registration of Persons Act (Cap. 107). + The card number is a 7- or 8-digit sequential identifier unique to each + cardholder. Cards issued since the late 1990s typically carry an 8-digit + number; earlier cards may have 7 digits. + + No checksum algorithm is publicly documented for Kenyan national ID + numbers. Confidence is therefore driven primarily by context words. + Without context the base score (0.01) is intentionally low enough to be + filtered by callers applying a reasonable minimum threshold (e.g. 0.35). + + Reference: https://www.ecitizen.go.ke/ + + :param patterns: List of patterns to be used by this recognizer + :param context: List of context words to increase confidence in detection + :param supported_language: Language this recognizer supports + :param supported_entity: The entity this recognizer can detect + """ + + COUNTRY_CODE = "ke" + + # 7- or 8-digit national ID number, not embedded in a longer number. + PATTERNS = [ + Pattern( + "KE National ID (Very Weak)", + r"\b\d{7,8}\b", + 0.01, + ), + ] + + CONTEXT = [ + # Single-token entries — matched against individual spaCy lemmas by + # LemmaContextAwareEnhancer (substring mode). Multi-word phrases alone + # are never matched because the enhancer compares against single tokens. + "kenya", + "kenyan", + "national", + "nid", + "registration", + # Multi-word entries — only effective when the enhancer is given + # pre-tokenised context via the `context` parameter. + "national id", + "national identity", + "national identity card", + "id number", + "kenyan id", + "kenya national id", + "registration number", + "id card", + "national registration", + ] + + def __init__( + self, + patterns: Optional[List[Pattern]] = None, + context: Optional[List[str]] = None, + supported_language: str = "en", + supported_entity: str = "KE_NATIONAL_ID", + name: Optional[str] = None, + ) -> None: + patterns = patterns if patterns else self.PATTERNS + context = context if context else self.CONTEXT + super().__init__( + supported_entity=supported_entity, + patterns=patterns, + context=context, + supported_language=supported_language, + name=name, + ) + + def validate_result(self, pattern_text: str) -> Optional[bool]: + """ + Validate basic Kenyan national ID format. + + Returns False (and removes the result) if the match is not 7 or 8 + numeric digits. Returns None to preserve the context-boosted score + rather than unconditionally promoting to MAX_SCORE, since no checksum + is available to justify that level of certainty. + """ + if not pattern_text.isnumeric() or len(pattern_text) not in (7, 8): + return False + return None diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/nigeria/__init__.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/nigeria/__init__.py index 711e94eff..c868ae34f 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/nigeria/__init__.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/nigeria/__init__.py @@ -1,9 +1,12 @@ """Nigeria-specific recognizers.""" +from .ng_bvn_recognizer import NgBvnRecognizer from .ng_nin_recognizer import NgNinRecognizer from .ng_vehicle_registration_recognizer import NgVehicleRegistrationRecognizer __all__ = [ + "NgBvnRecognizer", "NgNinRecognizer", "NgVehicleRegistrationRecognizer", ] + diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/nigeria/ng_bvn_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/nigeria/ng_bvn_recognizer.py new file mode 100644 index 000000000..cdb896f0e --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/country_specific/nigeria/ng_bvn_recognizer.py @@ -0,0 +1,82 @@ +from typing import List, Optional + +from presidio_analyzer import Pattern, PatternRecognizer + + +class NgBvnRecognizer(PatternRecognizer): + """ + Recognizes Nigerian Bank Verification Number (BVN). + + The BVN is an 11-digit identifier issued by the Central Bank of Nigeria + (CBN) and managed by the Nigerian Interbank Settlement System (NIBSS). + It links every bank account held by an individual to a single biometric + identity record. Unauthorized access to, use of, or disclosure of a BVN + is an offence under the CBN BVN Regulatory Framework (CBN Circular + Ref: FPR/DIR/GEN/CIR/01/009, 2014). + + Unlike the Nigerian NIN, the BVN does not have a publicly documented + checksum algorithm. Confidence is therefore driven primarily by context + words rather than structural validation. Without context the base pattern + score (0.01) is low enough to be filtered by callers applying a reasonable + minimum score threshold (e.g. 0.35). + + Reference: https://www.cbn.gov.ng/bvn/ + + :param patterns: List of patterns to be used by this recognizer + :param context: List of context words to increase confidence in detection + :param supported_language: Language this recognizer supports + :param supported_entity: The entity this recognizer can detect + """ + + COUNTRY_CODE = "ng" + + PATTERNS = [ + Pattern( + "BVN (Very Weak)", + r"\b\d{11}\b", + 0.01, + ), + ] + + CONTEXT = [ + "bvn", + "bank verification number", + "bank verification no", + "bank verification", + "nibss", + "nigeria bank id", + "bank identity number", + ] + + def __init__( + self, + patterns: Optional[List[Pattern]] = None, + context: Optional[List[str]] = None, + supported_language: str = "en", + supported_entity: str = "NG_BVN", + name: Optional[str] = None, + ) -> None: + patterns = patterns if patterns else self.PATTERNS + context = context if context else self.CONTEXT + super().__init__( + supported_entity=supported_entity, + patterns=patterns, + context=context, + supported_language=supported_language, + name=name, + ) + + def validate_result(self, pattern_text: str) -> Optional[bool]: + """ + Validate basic BVN format. + + Returns False (and removes the result) if the match is not exactly 11 + numeric digits. Returns None — rather than True — to preserve the + context-boosted score instead of unconditionally promoting it to + MAX_SCORE, since there is no checksum to provide that level of + certainty. + """ + if len(pattern_text) != 11 or not pattern_text.isnumeric(): + return False + # Valid format; let context enrichment determine final confidence. + return None diff --git a/presidio-analyzer/tests/test_ke_national_id_recognizer.py b/presidio-analyzer/tests/test_ke_national_id_recognizer.py new file mode 100644 index 000000000..e3e20f67b --- /dev/null +++ b/presidio-analyzer/tests/test_ke_national_id_recognizer.py @@ -0,0 +1,203 @@ +import pytest + +from presidio_analyzer.predefined_recognizers import KeNationalIdRecognizer +from presidio_analyzer.context_aware_enhancers import LemmaContextAwareEnhancer +from tests.assertions import assert_result_within_score_range + +# Synthetic Kenyan national ID numbers used throughout the tests. +VALID_ID_8 = "12345678" # 8-digit (current format) +VALID_ID_7 = "1234567" # 7-digit (older cards) + + +@pytest.fixture(scope="module") +def recognizer(): + return KeNationalIdRecognizer() + + +@pytest.fixture(scope="module") +def entities(): + return ["KE_NATIONAL_ID"] + + +@pytest.fixture(scope="module") +def enhancer(): + return LemmaContextAwareEnhancer() + + +# ── Tests that call analyze() directly (no context enhancement) ─────────────── +# PatternRecognizer does not apply LemmaContextAwareEnhancer internally, so +# scores remain at the base pattern score (0.01) regardless of surrounding text. + +@pytest.mark.parametrize( + "text, expected_len, expected_positions, expected_score_ranges", + [ + # fmt: off + + # Bare 8-digit ID, no context + ( + VALID_ID_8, + 1, + ((0, 8),), + ((0.0, 0.29),), + ), + # Bare 7-digit ID, no context + ( + VALID_ID_7, + 1, + ((0, 7),), + ((0.0, 0.29),), + ), + + # --- Invalid inputs --- + + # 6 digits: below minimum length + ( + "123456", + 0, + (), + (), + ), + # 9 digits: above maximum length + ( + "123456789", + 0, + (), + (), + ), + # Non-numeric + ( + "1234567a", + 0, + (), + (), + ), + # Embedded in a longer number (word boundary) + ( + f"99{VALID_ID_8}88", + 0, + (), + (), + ), + # fmt: on + ], +) +def test_when_ke_national_id_without_context_then_expected_results( + text, + expected_len, + expected_positions, + expected_score_ranges, + recognizer, + entities, +): + results = recognizer.analyze(text, entities) + assert len(results) == expected_len + + for res, (st_pos, fn_pos), (st_score, fn_score) in zip( + results, expected_positions, expected_score_ranges + ): + assert_result_within_score_range( + res, entities[0], st_pos, fn_pos, st_score, fn_score + ) + + +# ── Tests that verify context-driven score boosts ──────────────────────────── +# LemmaContextAwareEnhancer compares individual spaCy token lemmas against the +# recognizer's CONTEXT list (substring mode). Every case below includes a +# single-token context word that is present in KeNationalIdRecognizer.CONTEXT +# ("national", "kenya", "kenyan", "nid", "registration") so the boost is +# deterministic. + +@pytest.mark.parametrize( + "text, expected_len, expected_positions, expected_score_ranges", + [ + # fmt: off + + # "national" token — single-token context entry + ( + f"national id: {VALID_ID_8}", + 1, + ((13, 21),), + ((0.3, 1.0),), + ), + # "kenya" token — single-token context entry + ( + f"Kenya national id number: {VALID_ID_7}", + 1, + ((26, 33),), + ((0.3, 1.0),), + ), + # "nid" token — single-token context entry + ( + f"nid: {VALID_ID_8}", + 1, + ((5, 13),), + ((0.3, 1.0),), + ), + # "national" token in multi-word phrase + ( + f"national identity card: {VALID_ID_8}", + 1, + ((24, 32),), + ((0.3, 1.0),), + ), + # "kenyan" token — single-token context entry + ( + f"Kenyan id {VALID_ID_7} was reported", + 1, + ((10, 17),), + ((0.3, 1.0),), + ), + # fmt: on + ], +) +def test_when_ke_national_id_with_context_then_score_boosted( + text, + expected_len, + expected_positions, + expected_score_ranges, + recognizer, + entities, + enhancer, + spacy_nlp_engine, +): + if spacy_nlp_engine is None: + pytest.skip("spaCy NLP engine not available in this test run") + + nlp_artifacts = spacy_nlp_engine.process_text(text, "en") + raw_results = recognizer.analyze(text, entities) + results = enhancer.enhance_using_context( + text, raw_results, nlp_artifacts, [recognizer] + ) + + assert len(results) == expected_len + + for res, (st_pos, fn_pos), (st_score, fn_score) in zip( + results, expected_positions, expected_score_ranges + ): + assert_result_within_score_range( + res, entities[0], st_pos, fn_pos, st_score, fn_score + ) + + +class TestKeNationalIdValidation: + """Unit tests for KeNationalIdRecognizer.validate_result.""" + + def test_8_digit_id_returns_none(self): + recognizer = KeNationalIdRecognizer() + assert recognizer.validate_result(VALID_ID_8) is None + + def test_7_digit_id_returns_none(self): + recognizer = KeNationalIdRecognizer() + assert recognizer.validate_result(VALID_ID_7) is None + + def test_6_digit_returns_false(self): + recognizer = KeNationalIdRecognizer() + assert recognizer.validate_result("123456") is False + + def test_9_digit_returns_false(self): + recognizer = KeNationalIdRecognizer() + assert recognizer.validate_result("123456789") is False + + def test_non_numeric_returns_false(self): + recognizer = KeNationalIdRecognizer() + assert recognizer.validate_result("1234567a") is False diff --git a/presidio-analyzer/tests/test_ng_bvn_recognizer.py b/presidio-analyzer/tests/test_ng_bvn_recognizer.py new file mode 100644 index 000000000..394064ad9 --- /dev/null +++ b/presidio-analyzer/tests/test_ng_bvn_recognizer.py @@ -0,0 +1,201 @@ +import pytest + +from presidio_analyzer.predefined_recognizers import NgBvnRecognizer +from presidio_analyzer.context_aware_enhancers import LemmaContextAwareEnhancer +from tests.assertions import assert_result_within_score_range + +# Representative 11-digit BVN values used throughout the tests. +# These are synthetic — they follow the 11-digit format but are not real BVNs. +VALID_BVN_1 = "22345678901" +VALID_BVN_2 = "98765432101" +VALID_BVN_3 = "55512345678" + + +@pytest.fixture(scope="module") +def recognizer(): + return NgBvnRecognizer() + + +@pytest.fixture(scope="module") +def entities(): + return ["NG_BVN"] + + +@pytest.fixture(scope="module") +def enhancer(): + return LemmaContextAwareEnhancer() + + +# ── Tests that call analyze() directly (no context enhancement) ─────────────── +# PatternRecognizer does not apply LemmaContextAwareEnhancer internally, so +# scores remain at the base pattern score (0.01) regardless of surrounding text. +# These cases cover bare matches and invalid inputs. + +@pytest.mark.parametrize( + "text, expected_len, expected_positions, expected_score_ranges", + [ + # fmt: off + + # Bare BVN, no context — very weak signal + ( + VALID_BVN_1, + 1, + ((0, 11),), + ((0.0, 0.29),), + ), + # Two bare BVNs, no context + ( + f"{VALID_BVN_1} and {VALID_BVN_2}", + 2, + ((0, 11), (16, 27)), + ((0.0, 0.29), (0.0, 0.29)), + ), + + # --- Invalid inputs: must not be detected --- + + # Wrong length: 10 digits + ( + "2234567890", + 0, + (), + (), + ), + # Wrong length: 12 digits + ( + "223456789012", + 0, + (), + (), + ), + # Non-numeric characters + ( + "2234567890a", + 0, + (), + (), + ), + # Embedded in a longer number: word boundary must not match + ( + f"99{VALID_BVN_1}88", + 0, + (), + (), + ), + # fmt: on + ], +) +def test_when_bvn_without_context_then_expected_results( + text, + expected_len, + expected_positions, + expected_score_ranges, + recognizer, + entities, +): + results = recognizer.analyze(text, entities) + assert len(results) == expected_len + + for res, (st_pos, fn_pos), (st_score, fn_score) in zip( + results, expected_positions, expected_score_ranges + ): + assert_result_within_score_range( + res, entities[0], st_pos, fn_pos, st_score, fn_score + ) + + +# ── Tests that verify context-driven score boosts ──────────────────────────── +# LemmaContextAwareEnhancer compares individual spaCy token lemmas against the +# recognizer's CONTEXT list. Only single-token context entries (e.g. "bvn", +# "nibss") trigger a boost; multi-word phrases are never matched against a +# single lemma. Every test case below includes at least one such single-token +# context word so the boost is deterministic. + +@pytest.mark.parametrize( + "text, expected_len, expected_positions, expected_score_ranges", + [ + # fmt: off + + # Exact context word "bvn" + ( + f"bvn: {VALID_BVN_1}", + 1, + ((5, 16),), + ((0.3, 1.0),), + ), + # "BVN" token alongside the full phrase — "bvn" lemma triggers the boost + ( + f"BVN bank verification number: {VALID_BVN_2}", + 1, + ((30, 41),), + ((0.3, 1.0),), + ), + # Context word "BVN" appears earlier in sentence + ( + f"Please provide your BVN for KYC. Your BVN is {VALID_BVN_3}.", + 1, + ((45, 56),), + ((0.3, 1.0),), + ), + # "nibss" single-token context word + ( + f"NIBSS BVN record: {VALID_BVN_1}", + 1, + ((18, 29),), + ((0.3, 1.0),), + ), + # fmt: on + ], +) +def test_when_bvn_with_context_then_score_boosted( + text, + expected_len, + expected_positions, + expected_score_ranges, + recognizer, + entities, + enhancer, + spacy_nlp_engine, +): + if spacy_nlp_engine is None: + pytest.skip("spaCy NLP engine not available in this test run") + + nlp_artifacts = spacy_nlp_engine.process_text(text, "en") + raw_results = recognizer.analyze(text, entities) + results = enhancer.enhance_using_context( + text, raw_results, nlp_artifacts, [recognizer] + ) + + assert len(results) == expected_len + + for res, (st_pos, fn_pos), (st_score, fn_score) in zip( + results, expected_positions, expected_score_ranges + ): + assert_result_within_score_range( + res, entities[0], st_pos, fn_pos, st_score, fn_score + ) + + +class TestBvnValidation: + """Unit tests for NgBvnRecognizer.validate_result.""" + + def test_valid_11_digit_bvn_returns_none(self): + recognizer = NgBvnRecognizer() + # Returns None (not True) to preserve context-driven score. + assert recognizer.validate_result(VALID_BVN_1) is None + + def test_10_digit_string_returns_false(self): + recognizer = NgBvnRecognizer() + assert recognizer.validate_result("2234567890") is False + + def test_12_digit_string_returns_false(self): + recognizer = NgBvnRecognizer() + assert recognizer.validate_result("223456789012") is False + + def test_non_numeric_returns_false(self): + recognizer = NgBvnRecognizer() + assert recognizer.validate_result("2234567890a") is False + + def test_all_zeros_11_digits_returns_none(self): + # All-zeros is numerically valid format (11 digits, all numeric). + recognizer = NgBvnRecognizer() + assert recognizer.validate_result("00000000000") is None