CoNLL to spacy pkl format hindi ner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open(\"train.txt\", \"r\",encoding = \"utf-8\") \n",
    "trainList = []\n",
    "for line in file:\n",
    "    trainList.append(line[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['अरे O',\n",
       " 'सारथी O',\n",
       " 'जम्मू U-location',\n",
       " 'कैसा O',\n",
       " 'मौसम O',\n",
       " 'है O',\n",
       " '',\n",
       " 'अभी U-date',\n",
       " 'दिल्ली U-location',\n",
       " 'में O']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainList[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentList = []\n",
    "temp = []\n",
    "for word in trainList[:]:\n",
    "    if word == \"\":\n",
    "        sent = \" \".join(temp)\n",
    "        sentList.append(sent)\n",
    "        temp = []\n",
    "    else:\n",
    "        temp.append(word)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['अरे O सारथी O जम्मू U-location कैसा O मौसम O है O',\n",
       " 'अभी U-date दिल्ली U-location में O कैसा O मौसम O है O',\n",
       " 'आज U-date के O लिए O मौसम O रिपोर्ट O बताओ O',\n",
       " 'दार्जिलिंग U-location अभी U-date कैसा O है O',\n",
       " 'आज U-date हम O मौसम O के O तरीके O कैसे O हैं O',\n",
       " 'अभी U-date तापमान U-weather_type क्या O है O',\n",
       " 'कल U-date मौसम O के O लिए O पूर्वानुमान O क्या O हैं O',\n",
       " 'मुंबई U-location में O मौसम O कैसा O है O',\n",
       " 'जयपुर U-location में O मौसम O की O स्थिति O क्या O है O',\n",
       " 'मुंबई U-location में O मौसम O की O स्थिति O क्या O है O']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentList[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'आर्द्रता U-weather_type की O स्थिति O क्या O है O ? O'"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sent.split(\" \")[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'आर्द्रता'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent.split(\" \")[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'आर्द्रता U-weather_type की स्थिति क्या है ?'"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = re.sub(\" O\",\"\",sent)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = []\n",
    "tup = (0, 13, 'Person')\n",
    "a.append(tup) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 13, 'Person')]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('अरे सारथी जम्मू कैसा मौसम है',\n",
       " {'entities': (0, 13, 'Person', 0, 13, 'Person', 0, 13, 'Person')})"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(sentList[0],e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "a.append(tup) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 13, 'Person'), (0, 13, 'Person')]"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('अरे सारथी जम्मू कैसा मौसम है',\n",
       " {'entities': [(0, 13, 'Person'), (0, 13, 'Person')]})"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "e = {'entities':a }\n",
    "(sentList[0],e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentList = []\n",
    "temp = []\n",
    "startloc = 0\n",
    "endloc = 0\n",
    "entity = []\n",
    "\n",
    "for word in trainList:\n",
    "    if word == \"\":# new sent\n",
    "        sent = \" \".join(temp)\n",
    "        e = {'entities':entity }\n",
    "        finalSent = (sent,e)\n",
    "        sentList.append(finalSent)\n",
    "        temp = []\n",
    "        entity = []\n",
    "        startloc = 0\n",
    "        endloc = 0\n",
    "        \n",
    "    else: # old sent\n",
    "        w = word.split(\" \")\n",
    "        wordlen = len(w[0])\n",
    "        if w[1] == 'O': # if normal increase both loc\n",
    "            startloc += wordlen\n",
    "            endloc += wordlen\n",
    "        else:# if not normal then start and end diff\n",
    "            endloc += wordlen\n",
    "            tup = (startloc,endloc,w[1])\n",
    "            entity.append(tup)\n",
    "            startloc += wordlen\n",
    "        temp.append(w[0])\n",
    "        # including space\n",
    "        startloc += 1\n",
    "        endloc += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('अरे सारथी जम्मू कैसा मौसम है', {'entities': [(10, 15, 'जम्मू')]}),\n",
       " ('अभी दिल्ली में कैसा मौसम है',\n",
       "  {'entities': [(0, 3, 'अभी'), (4, 10, 'दिल्ली')]}),\n",
       " ('आज के लिए मौसम रिपोर्ट बताओ', {'entities': [(0, 2, 'आज')]}),\n",
       " ('दार्जिलिंग अभी कैसा है',\n",
       "  {'entities': [(0, 10, 'दार्जिलिंग'), (11, 14, 'अभी')]}),\n",
       " ('आज हम मौसम के तरीके कैसे हैं', {'entities': [(0, 2, 'आज')]}),\n",
       " ('अभी तापमान क्या है', {'entities': [(0, 3, 'अभी'), (4, 10, 'तापमान')]}),\n",
       " ('कल मौसम के लिए पूर्वानुमान क्या हैं', {'entities': [(0, 2, 'कल')]}),\n",
       " ('मुंबई में मौसम कैसा है', {'entities': [(0, 5, 'मुंबई')]}),\n",
       " ('जयपुर में मौसम की स्थिति क्या है', {'entities': [(0, 5, 'जयपुर')]}),\n",
       " ('मुंबई में मौसम की स्थिति क्या है', {'entities': [(0, 5, 'मुंबई')]})]"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentList[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n",
      "6\n"
     ]
    }
   ],
   "source": [
    "sentList1 = []\n",
    "temp = []\n",
    "for word in trainList[:14]:\n",
    "    if word == \"\":\n",
    "        sent = \" \".join(temp)\n",
    "        sentList1.append(sent)\n",
    "        print(len(temp))\n",
    "        temp = []\n",
    "    else:\n",
    "        temp.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['अरे O सारथी O जम्मू U-location कैसा O मौसम O है O',\n",
       " 'अभी U-date दिल्ली U-location में O कैसा O मौसम O है O']"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentList1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(\"दार्जिलिंग\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(\"आज\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "203"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = open('train.pkl', 'wb')\n",
    "\n",
    "# Pickle dictionary using protocol 0.\n",
    "pickle.dump(sentList, output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "def savepkl(data,filename):\n",
    "    output = open(filename + '.pkl', 'wb')\n",
    "\n",
    "    # Pickle dictionary using protocol 0.\n",
    "    pickle.dump(data, output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conll2spacy(data):\n",
    "    sentList = []\n",
    "    temp = []\n",
    "    startloc = 0\n",
    "    endloc = 0\n",
    "    entity = []\n",
    "\n",
    "    for word in data:\n",
    "        if word == \"\":# new sent\n",
    "            sent = \" \".join(temp)\n",
    "            e = {'entities':entity }\n",
    "            finalSent = (sent,e)\n",
    "            sentList.append(finalSent)\n",
    "            temp = []\n",
    "            entity = []\n",
    "            startloc = 0\n",
    "            endloc = 0\n",
    "\n",
    "        else: # old sent\n",
    "            w = word.split(\" \")\n",
    "            wordlen = len(w[0])\n",
    "            if w[1] == 'O': # if normal increase both loc\n",
    "                startloc += wordlen\n",
    "                endloc += wordlen\n",
    "            else:# if not normal then start and end diff\n",
    "                endloc += wordlen\n",
    "                tup = (startloc,endloc,w[1])\n",
    "                entity.append(tup)\n",
    "                startloc += wordlen\n",
    "            temp.append(w[0])\n",
    "            # including space\n",
    "            startloc += 1\n",
    "            endloc += 1\n",
    "    return sentList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['आज U-date',\n",
       " 'मौसम O',\n",
       " 'कैसा O',\n",
       " 'है O',\n",
       " '',\n",
       " 'क्या O',\n",
       " 'हरिद्वार U-location',\n",
       " 'में O',\n",
       " 'बहुत O',\n",
       " 'ठंड U-weather_type',\n",
       " 'है O',\n",
       " 'अभी U-date',\n",
       " '',\n",
       " 'आज U-date',\n",
       " 'इतनी O']"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = open(\"val.txt\", \"r\",encoding = \"utf-8\") \n",
    "trainList = []\n",
    "for line in file:\n",
    "    trainList.append(line[:-1])\n",
    "trainList[:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "valSentList = conll2spacy(trainList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('आज मौसम कैसा है', {'entities': [(0, 2, 'U-date')]}),\n",
       " ('क्या हरिद्वार में बहुत ठंड है अभी',\n",
       "  {'entities': [(5, 13, 'U-location'),\n",
       "    (23, 26, 'U-weather_type'),\n",
       "    (30, 33, 'U-date')]}),\n",
       " ('आज इतनी गर्मी क्यों है',\n",
       "  {'entities': [(0, 2, 'U-date'), (8, 13, 'U-weather_type')]}),\n",
       " ('क्या मुंबई में ठंडी है',\n",
       "  {'entities': [(5, 10, 'U-location'), (15, 19, 'U-weather_type')]}),\n",
       " ('क्या पाली में सर्दी आ गयी',\n",
       "  {'entities': [(5, 9, 'U-location'), (14, 19, 'U-weather_type')]})]"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valSentList[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(\"क्या \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(\"हरिद्वार\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(\"है\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(\"क्या हरिद्वार में बहुत ठंड है \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "savepkl(valSentList,\"val\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['अरे O',\n",
       " 'सारथी O',\n",
       " 'जम्मू U-location',\n",
       " 'कैसा O',\n",
       " 'मौसम O',\n",
       " 'है O',\n",
       " '',\n",
       " 'अभी U-date',\n",
       " 'दिल्ली U-location',\n",
       " 'में O',\n",
       " 'कैसा O',\n",
       " 'मौसम O',\n",
       " 'है O',\n",
       " '',\n",
       " 'आज U-date']"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = open(\"train.txt\", \"r\",encoding = \"utf-8\") \n",
    "trainList = []\n",
    "for line in file:\n",
    "    trainList.append(line[:-1])\n",
    "trainList[:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainSentList = conll2spacy(trainList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('अरे सारथी जम्मू कैसा मौसम है', {'entities': [(10, 15, 'U-location')]}),\n",
       " ('अभी दिल्ली में कैसा मौसम है',\n",
       "  {'entities': [(0, 3, 'U-date'), (4, 10, 'U-location')]}),\n",
       " ('आज के लिए मौसम रिपोर्ट बताओ', {'entities': [(0, 2, 'U-date')]}),\n",
       " ('दार्जिलिंग अभी कैसा है',\n",
       "  {'entities': [(0, 10, 'U-location'), (11, 14, 'U-date')]}),\n",
       " ('आज हम मौसम के तरीके कैसे हैं', {'entities': [(0, 2, 'U-date')]})]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainSentList[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "savepkl(trainSentList,\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}