PII and HayStack

Author: afizs

ai-apps/haystack_tonic.ipynb

@@ -0,0 +1,270 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "authorship_tag": "ABX9TyPTApHdeA109Za10Eckvgnh",
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/afizs/python-notes/blob/main/ai-apps/haystack_tonic.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## PII detection and extraction in Haystack pipelines"
+      ],
+      "metadata": {
+        "id": "skFZ8NKLZR08"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Installation"
+      ],
+      "metadata": {
+        "id": "KIpZyP_GcQNr"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 11,
+      "metadata": {
+        "id": "OaRdp0C9Yxsy"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install textual-haystack -q"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Setup\n",
+        "First thing first get your textual FREE API Key.\n",
+        "https://docs.tonic.ai/textual/tonic-textual-getting-started"
+      ],
+      "metadata": {
+        "id": "Q9q6XTh_btGr"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import os\n",
+        "from google.colab import userdata\n",
+        "\n",
+        "# Load the TONIC_TEXTUAL_API_KEY from Colab secrets\n",
+        "os.environ[\"TONIC_TEXTUAL_API_KEY\"] = userdata.get('TONIC_TEXTUAL_API_KEY')"
+      ],
+      "metadata": {
+        "id": "eyqkUKPfaCDG"
+      },
+      "execution_count": 12,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## 1. Document cleaning\n",
+        "\n",
+        "**Clean up documents before replacing PII with realistic fake data**"
+      ],
+      "metadata": {
+        "id": "_gDSEoRQaTzQ"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from haystack.dataclasses import Document\n",
+        "from haystack_integrations.components.tonic_textual import TonicTextualDocumentCleaner\n",
+        "\n",
+        "cleaner = TonicTextualDocumentCleaner(generator_default=\"Synthesis\")\n",
+        "result = cleaner.run(documents=[\n",
+        "    Document(content=\"Patient John Smith, DOB 03/15/1982, was admitted for chest pain.\")\n",
+        "])\n",
+        "print(result[\"documents\"][0].content)\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "o_gGC5o6Zosb",
+        "outputId": "0bb4b1b6-ee17-4203-ed95-51b6a0b70d40"
+      },
+      "execution_count": 13,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Patient Alfonzo Uva, DOB 03/20/1982, was admitted for chest pain.\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "**Tokenize PII**"
+      ],
+      "metadata": {
+        "id": "HuX870z_a9HF"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "cleaner = TonicTextualDocumentCleaner(generator_default=\"Redaction\")\n",
+        "result = cleaner.run(documents=[\n",
+        "    Document(content=\"Contact Jane Doe at jane@example.com.\")\n",
+        "])\n",
+        "print(result[\"documents\"][0].content)\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "SuBT0lKla8z4",
+        "outputId": "6776a34f-f711-4338-d71c-32d641872057"
+      },
+      "execution_count": 14,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Contact [NAME_GIVEN_iKpB3] [NAME_FAMILY_Z3W2] at [EMAIL_ADDRESS_QEgToEQ8FO6hC16fJG].\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Entity Extraction"
+      ],
+      "metadata": {
+        "id": "W_6L_KoccBGc"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from haystack.dataclasses import Document\n",
+        "from haystack_integrations.components.tonic_textual import TonicTextualEntityExtractor\n",
+        "\n",
+        "extractor = TonicTextualEntityExtractor()\n",
+        "result = extractor.run(documents=[\n",
+        "    Document(content=\"My name is Afiz Shaik and my email is afiz@example.com.\")\n",
+        "])\n",
+        "\n",
+        "for entity in TonicTextualEntityExtractor.get_stored_annotations(result[\"documents\"][0]):\n",
+        "    print(f\"{entity.entity}: {entity.text} (confidence: {entity.score:.2f})\")\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "yZ6I4B9obCYa",
+        "outputId": "c0ecec1f-f1b6-48a1-df1d-4ceed020e1ff"
+      },
+      "execution_count": 15,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "NAME_GIVEN: Afiz (confidence: 0.95)\n",
+            "NAME_FAMILY: Shaik (confidence: 0.93)\n",
+            "EMAIL_ADDRESS: afiz@example.com (confidence: 0.99)\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Complete Haystack Pipeline"
+      ],
+      "metadata": {
+        "id": "Jduqp5y1cy-p"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from haystack import Pipeline\n",
+        "from haystack.dataclasses import Document\n",
+        "from haystack_integrations.components.tonic_textual import (\n",
+        "    TonicTextualDocumentCleaner,\n",
+        "    TonicTextualEntityExtractor,\n",
+        ")\n",
+        "\n",
+        "pipeline = Pipeline()\n",
+        "pipeline.add_component(\"cleaner\", TonicTextualDocumentCleaner(generator_default=\"Synthesis\"))\n",
+        "pipeline.add_component(\"extractor\", TonicTextualEntityExtractor())\n",
+        "pipeline.connect(\"cleaner\", \"extractor\")\n",
+        "\n",
+        "result = pipeline.run({\n",
+        "    \"cleaner\": {\n",
+        "        \"documents\": [\n",
+        "            Document(content=\"Contact Jane Doe at jane@example.com or (555) 867-5309.\"),\n",
+        "        ]\n",
+        "    }\n",
+        "})\n",
+        "\n",
+        "for doc in result[\"extractor\"][\"documents\"]:\n",
+        "    entities = TonicTextualEntityExtractor.get_stored_annotations(doc)\n",
+        "    print(f\"Cleaned: {doc.content}\")\n",
+        "    print(f\"Entities: {[(e.entity, e.text) for e in entities]}\")\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "CtmPwcwscLqo",
+        "outputId": "20e05741-2dba-459d-d80f-fb0b1dae0135"
+      },
+      "execution_count": 16,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Cleaned: Contact Antonina Crummitt at pacv@pvjivyj.lfq or (851) 452-9397.\n",
+            "Entities: [('NAME_GIVEN', 'Antonina'), ('NAME_FAMILY', 'Crummitt'), ('EMAIL_ADDRESS', 'pacv@pvjivyj.lfq'), ('PHONE_NUMBER', '(851) 452-9397')]\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [],
+      "metadata": {
+        "id": "TBgcSEt-dc_n"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}