Upgrade Transformers to resolve six Dependabot alerts

docs/analyzer/nlp_engines/transformers.md

@@ -215,7 +215,7 @@ The `ner_model_configuration` section contains the following parameters:
     For example, for `bert-base-NER-uncased`, it can be found here: <https://huggingface.co/dslim/bert-base-NER-uncased/blob/main/config.json>.
     Note that most NER models add a prefix to the class (e.g. `B-PER` for class `PER`). When creating the mapping, do not add the prefix.
 
-See more information on parameters on the [spacy-huggingface-pipelines Github repo](https://github.com/explosion/spacy-huggingface-pipelines#token-classification).
+See more information on parameters in the [Hugging Face token-classification pipeline documentation](https://huggingface.co/docs/transformers/main_classes/pipelines#transformers.TokenClassificationPipeline).
 
 Once created, see [the NLP configuration documentation](../customizing_nlp_models.md#Configure-Presidio-to-use-the-new-model) for more information.
 

presidio-analyzer/presidio_analyzer/nlp_engine/transformers_nlp_engine.py

@@ -1,23 +1,150 @@
 import logging
-from typing import Dict, List, Optional
+from typing import Any, Dict, Iterable, Iterator, List, Optional
 
 import spacy
-from spacy.tokens import Doc, Span
+from spacy.language import Language
+from spacy.pipeline import Pipe
+from spacy.tokens import Doc, Span, SpanGroup
 
 try:
-    import spacy_huggingface_pipelines
-    import transformers
+    from transformers import pipeline as hf_pipeline
 except ImportError:
-    spacy_huggingface_pipelines = None
-    transformers = None
+    hf_pipeline = None
 
 from presidio_analyzer.nlp_engine import (
     NerModelConfiguration,
     SpacyNlpEngine,
+    device_detector,
 )
 
 logger = logging.getLogger("presidio-analyzer")
 
+_PIPE_NAME = "presidio_transformers_ner"
+
+
+@Language.factory(
+    _PIPE_NAME,
+    assigns=[],
+    default_config={
+        "model": "",
+        "stride": 14,
+        "aggregation_strategy": "max",
+        "alignment_mode": "expand",
+        "spans_key": "bert-base-ner",
+    },
+)
+def _create_transformers_ner_pipe(
+    nlp: Language,
+    name: str,
+    model: str,
+    stride: Optional[int],
+    aggregation_strategy: str,
+    alignment_mode: str,
+    spans_key: str,
+) -> "_TransformersEntityPipe":
+    if hf_pipeline is None:
+        raise ImportError(
+            "transformers is not installed. Install presidio-analyzer[transformers] "
+            "to use TransformersNlpEngine."
+        )
+    if not model:
+        raise ValueError("A Hugging Face token-classification model is required")
+
+    pipeline_kwargs = {
+        "task": "token-classification",
+        "model": model,
+        "aggregation_strategy": aggregation_strategy,
+        "device": device_detector.get_device(),
+    }
+    if stride is not None:
+        pipeline_kwargs["stride"] = stride
+
+    pipeline = hf_pipeline(**pipeline_kwargs)
+    return _TransformersEntityPipe(
+        name=name,
+        pipeline=pipeline,
+        alignment_mode=alignment_mode,
+        spans_key=spans_key,
+    )
+
+
+class _TransformersEntityPipe(Pipe):
+    """Add Hugging Face token-classification predictions to a spaCy document."""
+
+    def __init__(
+        self,
+        name: str,
+        pipeline: Any,
+        alignment_mode: str,
+        spans_key: str,
+    ) -> None:
+        self.name = name
+        self.pipeline = pipeline
+        self.alignment_mode = alignment_mode
+        self.spans_key = spans_key
+
+    def __call__(self, doc: Doc) -> Doc:
+        return self._add_predictions(doc, self.pipeline(doc.text))
+
+    def pipe(
+        self, stream: Iterable[Doc], *, batch_size: int = 128
+    ) -> Iterator[Doc]:
+        for docs in spacy.util.minibatch(stream, size=batch_size):
+            docs = list(docs)
+            predictions = self.pipeline(
+                [doc.text for doc in docs], batch_size=batch_size
+            )
+            for doc, doc_predictions in zip(docs, predictions, strict=True):
+                yield self._add_predictions(doc, doc_predictions)
+
+    def _add_predictions(
+        self, doc: Doc, predictions: Iterable[Dict[str, Any]]
+    ) -> Doc:
+        spans = SpanGroup(doc, name=self.spans_key, attrs={"scores": []})
+        previous_end = 0
+
+        for prediction in predictions:
+            label = prediction.get("entity_group") or prediction.get("entity")
+            start = prediction.get("start")
+            end = prediction.get("end")
+            if not label or not isinstance(start, int) or not isinstance(end, int):
+                logger.warning(
+                    "Skipping malformed Transformers prediction: %s", prediction
+                )
+                continue
+
+            span = doc.char_span(
+                start,
+                end,
+                label=str(label),
+                alignment_mode=self.alignment_mode,
+            )
+            if span is None or span.start_char < previous_end:
+                logger.warning(
+                    "Skipping unaligned or overlapping Transformers prediction: %s",
+                    prediction,
+                )
+                continue
+
+            spans.append(span)
+            spans.attrs["scores"].append(float(prediction.get("score", 0.0)))
+            previous_end = span.end_char
+
+        doc.spans[self.spans_key] = spans
+        return doc
+
+    def to_bytes(self, **kwargs: Any) -> bytes:
+        return b""
+
+    def from_bytes(self, bytes_data: bytes, **kwargs: Any) -> "_TransformersEntityPipe":
+        return self
+
+    def to_disk(self, path: Any, **kwargs: Any) -> None:
+        return None
+
+    def from_disk(self, path: Any, **kwargs: Any) -> "_TransformersEntityPipe":
+        return self
+
 
 class TransformersNlpEngine(SpacyNlpEngine):
     """
@@ -50,7 +177,7 @@ class TransformersNlpEngine(SpacyNlpEngine):
     """
 
     engine_name = "transformers"
-    is_available = bool(spacy_huggingface_pipelines)
+    is_available = bool(hf_pipeline)
 
     def __init__(
         self,
@@ -89,14 +216,13 @@ def load(self) -> None:
 
             pipe_config = {
                 "model": transformers_model,
-                "annotate": "spans",
                 "stride": self.ner_model_configuration.stride,
                 "alignment_mode": self.ner_model_configuration.alignment_mode,
                 "aggregation_strategy": self.ner_model_configuration.aggregation_strategy,  # noqa: E501
-                "annotate_spans_key": self.entity_key,
+                "spans_key": self.entity_key,
             }
 
-            nlp.add_pipe("hf_token_pipe", config=pipe_config)
+            nlp.add_pipe(_PIPE_NAME, config=pipe_config)
             self.nlp[model["lang_code"]] = nlp
 
     @staticmethod

presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/transformers_recognizer.py

@@ -16,7 +16,7 @@ class TransformersRecognizer(SpacyRecognizer):
     but loads the output from the NlpArtifacts
     See:
      - https://huggingface.co/docs/transformers/main/en/index for transformer models
-     - https://github.com/explosion/spacy-huggingface-pipelines on the spaCy wrapper to transformers
+     - https://huggingface.co/docs/transformers/main_classes/pipelines for the pipeline API
     """  # noqa: E501
 
     ENTITIES = [

presidio-analyzer/pyproject.toml

@@ -42,10 +42,9 @@ server = [
     "waitress (>=2.0.0,<4.0.0); platform_system == 'Windows'"
 ]
 transformers = [
-    "transformers (>=4.0.0,<6.0.0)",
+    "transformers (>=5.5.0,<6.0.0)",
     "accelerate (>=0.20.0,<2.0.0)",
-    "huggingface_hub (>=0.20.0,<2.0.0)",
-    "spacy_huggingface_pipelines (>=0.0.4,<1.0.0)"]
+    "huggingface_hub (>=0.20.0,<2.0.0)"]
 
 stanza = [
     "stanza (>=1.11.1,<2.0.0)",
@@ -59,7 +58,7 @@ ahds = [
     "azure-health-deidentification (>=1.1.0b1,<2.0.0)"
 ]
 gliner = [
-    "transformers",
+    "transformers (>=5.5.0,<5.7.0)",
     "huggingface_hub",
     "gliner (>=0.2.26,<1.0.0)",
     "onnxruntime (>=1.19, <1.24.1) ; python_version == '3.10'",

presidio-analyzer/tests/test_transformers_nlp_engine.py

@@ -1,5 +1,5 @@
 import pytest
-
+import spacy
 from presidio_analyzer.nlp_engine import TransformersNlpEngine
 
 
@@ -22,6 +22,68 @@ def test_validate_model_params_happy_path():
     TransformersNlpEngine._validate_model_params(model)
 
 
+def test_transformers_pipe_adds_spans_and_scores(mocker):
+    """Verify single-document predictions are stored as scored spaCy spans."""
+    predictions = [
+        {
+            "entity_group": "PER",
+            "start": 11,
+            "end": 14,
+            "score": 0.98,
+        }
+    ]
+    pipeline = mocker.Mock(return_value=predictions)
+    mocker.patch(
+        "presidio_analyzer.nlp_engine.transformers_nlp_engine.hf_pipeline",
+        return_value=pipeline,
+    )
+
+    nlp = spacy.blank("en")
+    nlp.add_pipe(
+        "presidio_transformers_ner",
+        config={
+            "model": "test-model",
+            "alignment_mode": "strict",
+            "spans_key": "test-entities",
+        },
+    )
+
+    doc = nlp("my name is Dan")
+
+    assert [(span.text, span.label_) for span in doc.spans["test-entities"]] == [
+        ("Dan", "PER")
+    ]
+    assert doc.spans["test-entities"].attrs["scores"] == [0.98]
+
+
+def test_transformers_pipe_batches_documents(mocker):
+    """Verify batch predictions remain associated with their source documents."""
+    pipeline = mocker.Mock(
+        return_value=[
+            [{"entity_group": "PER", "start": 0, "end": 3, "score": 0.98}],
+            [{"entity_group": "ORG", "start": 0, "end": 6, "score": 0.92}],
+        ]
+    )
+    mocker.patch(
+        "presidio_analyzer.nlp_engine.transformers_nlp_engine.hf_pipeline",
+        return_value=pipeline,
+    )
+
+    nlp = spacy.blank("en")
+    nlp.add_pipe(
+        "presidio_transformers_ner",
+        config={"model": "test-model", "spans_key": "test-entities"},
+    )
+
+    docs = list(nlp.pipe(["Dan", "GitHub"], batch_size=2))
+
+    assert [doc.spans["test-entities"][0].text for doc in docs] == ["Dan", "GitHub"]
+    assert [doc.spans["test-entities"].attrs["scores"] for doc in docs] == [
+        [0.98],
+        [0.92],
+    ]
+
+
 @pytest.mark.parametrize(
     "key",
     [("lang_code"), ("model_name"), ("model_name.spacy"), ("model_name.transformers")],