00.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5738d9f3-3de4-4d2f-b26a-1efadb213ab2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import wandb\n",
    "import nltk\n",
    "from nltk.stem.porter import *\n",
    "from torch.nn import *\n",
    "from torch.optim import *\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch,torchvision\n",
    "import random\n",
    "from tqdm import *\n",
    "from torch.utils.data import Dataset,DataLoader\n",
    "stemmer = PorterStemmer()\n",
    "PROJECT_NAME = 'kickstarter-NLP-v3'\n",
    "device = 'cuda'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3220171b-4bb9-4fb2-a9f1-1273943876cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(sentence):\n",
    "    return nltk.word_tokenize(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3296630f-1347-4070-a6f8-d9cc3745e711",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['$', '100']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenize('$100')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a5a8f95f-3485-4f77-b9d9-a46d91a2e65d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def stem(word):\n",
    "    return stemmer.stem(word.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e85af1cf-7496-4037-a142-fe17167b4550",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'organ'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stem('organic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b5d76d09-1d2f-48fe-baae-5ac1f97b0c43",
   "metadata": {},
   "outputs": [],
   "source": [
    "def bag_of_words(tokenized_words,words):\n",
    "    tokenized_words = [stem(w) for w in tokenized_words]\n",
    "    bag = np.zeros(len(words))\n",
    "    for idx,w in enumerate(words):\n",
    "        if w in tokenized_words:\n",
    "            bag[idx] = 1.0\n",
    "    return bag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "88d39b08-8163-4431-aa31-279feca72534",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 0., 1.])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bag_of_words(['hi'],['hi','how','hi'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fc47b31b-30f4-49eb-a465-39146859b962",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('./data.csv').dropna().sample(frac=1)[:5000]\n",
    "X = data['text']\n",
    "y = data['subject']\n",
    "words = []\n",
    "data = []\n",
    "idx = 0\n",
    "labels = {}\n",
    "labels_r = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5c0eeb54-bb81-4f1d-a8c2-d4b181fe7f77",
   "metadata": {},
   "outputs": [],
   "source": [
    "for label in list(y):\n",
    "    if label not in list(labels.keys()):\n",
    "        idx += 1\n",
    "        labels[label] = idx\n",
    "        labels_r[idx] = label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b95a17ab-01da-48d7-ac3a-f54c192c9487",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'worldnews': 1, 'politicsNews': 2},\n",
       " {1: 'worldnews', 2: 'politicsNews'},\n",
       " 'worldnews',\n",
       " 2)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels,labels_r,label,idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ef5b376f-9bff-492c-8227-7ca45200e49c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5000it [00:32, 154.80it/s]\n"
     ]
    }
   ],
   "source": [
    "for X_batch,y_batch in tqdm(zip(X,y)):\n",
    "    X_batch = tokenize(X_batch)\n",
    "    new_X = []\n",
    "    for Xb in X_batch:\n",
    "        new_X.append(stem(Xb))\n",
    "    words.extend(new_X)\n",
    "    data.append([new_X,np.eye(labels[y_batch],len(labels))[labels[y_batch]-1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "95eaa893-bdaf-46e2-ad73-bc1e1336cc5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "words = sorted(set(words))\n",
    "np.random.shuffle(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "def51a2c-000d-4be7-9d95-b0335bf9779f",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.shuffle(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "67be4110-a2d4-469b-8a5f-ee47ad2f3f51",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "y = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d7597f62-ee16-4be1-8426-d74505c631b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████| 5000/5000 [08:34<00:00,  9.72it/s]\n"
     ]
    }
   ],
   "source": [
    "for sentence,tag in tqdm(data):\n",
    "    X.append(bag_of_words(sentence,words))\n",
    "    y.append(tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2764af1d-c2c0-44a3-b002-f71963060489",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import *\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.125,shuffle=False)\n",
    "X_train = torch.from_numpy(np.array(X_train)).to(device).float()\n",
    "y_train = torch.from_numpy(np.array(y_train)).to(device).float()\n",
    "X_test = torch.from_numpy(np.array(X_test)).to(device).float()\n",
    "y_test = torch.from_numpy(np.array(y_test)).to(device).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "349251c3-ed7a-4a45-8cfd-1a7bd937f17f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_loss(model,X,y,criterion):\n",
    "    preds = model(X)\n",
    "    loss = criterion(preds,y)\n",
    "    return loss.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f11453c5-0e28-4fbd-a270-88bd9aa861bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_accuracy(model,X,y):\n",
    "    preds = model(X)\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for pred,yb in zip(preds,y):\n",
    "        pred = int(torch.argmax(pred))\n",
    "        yb = int(torch.argmax(yb))\n",
    "        if pred == yb:\n",
    "            correct += 1\n",
    "        total += 1\n",
    "    acc = round(correct/total,3)*100\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3c5e8220-768f-495b-bb9b-3ad2dceae341",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.activation = ReLU()\n",
    "        self.iters = 10\n",
    "        self.hidden = 1024\n",
    "        self.linear1 = Linear(len(words),self.hidden)\n",
    "        self.linear2 = Linear(self.hidden,self.hidden)\n",
    "        self.linear2bn = BatchNorm1d(self.hidden)\n",
    "        self.output = Linear(self.hidden,len(labels))\n",
    "    \n",
    "    def forward(self,X):\n",
    "        preds = self.linear1(X)\n",
    "        for _ in range(self.iters):\n",
    "            preds = self.activation(self.linear2bn(self.linear2(preds)))\n",
    "        preds = self.output(preds)\n",
    "        return preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5edd890b-dd0a-4ff2-92b3-6cbfa9653c38",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model().to(device)\n",
    "criterion = MSELoss()\n",
    "optimizer = Adam(model.parameters(),lr=0.001)\n",
    "epochs = 500\n",
    "batch_size = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1e5d149d-6596-452e-91d0-a368253ff2e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: W&B API key is configured (use `wandb login --relogin` to force relogin)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                Tracking run with wandb version 0.12.1<br/>\n",
       "                Syncing run <strong style=\"color:#cdcd00\">baseline</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
       "                Project page: <a href=\"https://wandb.ai/ranuga-d/kickstarter-NLP-v3\" target=\"_blank\">https://wandb.ai/ranuga-d/kickstarter-NLP-v3</a><br/>\n",
       "                Run page: <a href=\"https://wandb.ai/ranuga-d/kickstarter-NLP-v3/runs/1b25xq4h\" target=\"_blank\">https://wandb.ai/ranuga-d/kickstarter-NLP-v3/runs/1b25xq4h</a><br/>\n",
       "                Run data is saved locally in <code>/home/indika/Programming/Projects/Python/Artifical-Intelligence/PyTorch/NLP/Subject-Cassification-NLP/wandb/run-20211013_195101-1b25xq4h</code><br/><br/>\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███████████████▊                         | 193/500 [07:11<11:27,  2.24s/it]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_251115/3062525215.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    253\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    254\u001b[0m                 inputs=inputs)\n\u001b[0;32m--> 255\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    256\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    257\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    147\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m    148\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 149\u001b[0;31m         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    150\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    151\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "wandb.init(project=PROJECT_NAME,name='baseline')\n",
    "for _ in tqdm(range(epochs)):\n",
    "    for i in range(0,len(X_train),batch_size):\n",
    "        X_batch = X_train[i:i+batch_size]\n",
    "        y_batch = y_train[i:i+batch_size]\n",
    "        preds = model(X_batch)\n",
    "        loss = criterion(preds,y_batch)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    model.eval()\n",
    "    torch.cuda.empty_cache()\n",
    "    wandb.log({'Loss':(get_loss(model,X_train,y_train,criterion)+get_loss(model,X_batch,y_batch,criterion)/2)})\n",
    "    torch.cuda.empty_cache()\n",
    "    wandb.log({'Val Loss':get_loss(model,X_test,y_test,criterion)})\n",
    "    torch.cuda.empty_cache()\n",
    "    wandb.log({'Acc':(get_accuracy(model,X_train,y_train)+get_accuracy(model,X_batch,y_batch))/2})\n",
    "    torch.cuda.empty_cache()\n",
    "    wandb.log({'Val Acc':get_accuracy(model,X_test,y_test)})\n",
    "    torch.cuda.empty_cache()\n",
    "    model.train()\n",
    "wandb.finish()\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "986bd483-97b1-4515-b5bc-223f920d9903",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model,'model.pt')\n",
    "torch.save(model,'model.pth')\n",
    "torch.save(model.state_dict(),'model-sd.pt')\n",
    "torch.save(model.state_dict(),'model-sd.pth')\n",
    "torch.save(words,'words.pt')\n",
    "torch.save(words,'words.pth')\n",
    "torch.save(data,'data.pt')\n",
    "torch.save(data,'data.pth')\n",
    "torch.save(labels,'labels.pt')\n",
    "torch.save(labels,'labels.pth')\n",
    "torch.save(idx,'idx.pt')\n",
    "torch.save(idx,'idx.pth')\n",
    "torch.save(y_train,'y_train.pt')\n",
    "torch.save(y_test,'y_test.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "86eb6015-3a8d-41d7-8757-38ceaf960411",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(y,'y.pt')\n",
    "torch.save(y,'y.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3143430-4c7e-4844-aa63-8c9f1cc7b075",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.11 64-bit ('base': conda)",
   "language": "python",
   "name": "python3711jvsc74a57bd0210f9608a45c0278a93c9e0b10db32a427986ab48cfc0d20c139811eb78c4bbc"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}