Untitled24.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "import torchvision\n",
    "import numpy as np\n",
    "from torch.autograd import Variable\n",
    "import nltk\n",
    "import pymorphy2\n",
    "import torch.nn as nn\n",
    "from torch.utils.data.dataloader import DataLoader\n",
    "from torch.utils.data.dataset import Dataset\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import os\n",
    "import math\n",
    "from torch.utils.data.sampler import RandomSampler\n",
    "from string import punctuation\n",
    "from string import digits\n",
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "import sys\n",
    "from sklearn.metrics import f1_score\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from sklearn.model_selection import KFold\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpu_avaliable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepr_french(input):\n",
    "    input = input.lower()\n",
    "    input = re.sub(':', '', input)\n",
    "    input = re.sub(\"#\", '', input)\n",
    "    delreg = \"(\\S+(\\.|\\/)+\\S+)+\"\n",
    "    input = re.sub(delreg, '', input)\n",
    "    stop = punctuation + digits + '»«'\n",
    "    french = 'qwertyuiopasdfghjklzxcvbnmàâçéèêëîïôûùüÿñæœ '\n",
    "    return ''.join([s for s in input if s not in stop and s in french])\n",
    "\n",
    "def clean_text_french(file, plain_text):\n",
    "    french_stemmer = SnowballStemmer('french', ignore_stopwords=True)\n",
    "    file = [[french_stemmer.stem(word) for word in prepr_french(plain_text[i]).split()] for i in range(len(file))]\n",
    "    return file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepr_ger(input):\n",
    "    input = input.lower()\n",
    "    input = re.sub(':', '', input)\n",
    "    input = re.sub(\"#\", '', input)\n",
    "    delreg = \"(\\S+(\\.|\\/)+\\S+)+\"\n",
    "    input = re.sub(delreg, '', input)\n",
    "    stop = punctuation + digits + '»«'\n",
    "    ger = 'ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÜß '.lower()\n",
    "    return ''.join([s for s in input if s not in stop and s in ger])\n",
    "\n",
    "def clean_text_ger(file, plain_text):\n",
    "    ger_stemmer = SnowballStemmer('german', ignore_stopwords=True)\n",
    "    file = [[ger_stemmer.stem(word) for word in prepr_ger(plain_text[i]).split()] for i in range(len(file))]\n",
    "    return file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepr_eng(input):\n",
    "    input = input.lower()\n",
    "    input = re.sub(':', '', input)\n",
    "    input = re.sub(\"#\", '', input)\n",
    "    delreg = \"(\\S+(\\.|\\/)+\\S+)+\"\n",
    "    input = re.sub(delreg, '', input)\n",
    "    stop = punctuation + digits + '»«'\n",
    "    eng = 'abcdefghijklmnopqrstuvwxyz '\n",
    "    return ''.join([s for s in input if s not in stop and s in eng])\n",
    "\n",
    "def clean_text_eng(file, plain_text):\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    file = [[lemmatizer.lemmatize(word) for word in prepr_eng(plain_text[i]).split()] for i in range(len(file))]\n",
    "    return file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepr_ru(input):\n",
    "    input = input.lower()\n",
    "    input = re.sub(':', '', input)\n",
    "    input = re.sub(\"#\", '', input)\n",
    "    delreg = \"(\\S+(\\.|\\/)+\\S+)+\"\n",
    "    input = re.sub(delreg, '', input)\n",
    "    stop = punctuation + digits + '»«'\n",
    "    ru = 'йцукенгшщзхъфывапролджэячсмитьбюЙЦУКЕНГШЩЗХЪФЫВАПРОЛДЖЭЯЧСМИТЬБЮ '\n",
    "    return ''.join([s for s in input if s not in stop and s in ru])\n",
    "\n",
    "def clean_text_ru(file, plain_text):\n",
    "    morph = pymorphy2.MorphAnalyzer()\n",
    "    file = [[morph.parse(word)[0].normal_form for word in prepr_ru(plain_text[i]).split()] for i in range(len(file))]\n",
    "    return file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence(array, seq_len):\n",
    "    if len(array) < seq_len:\n",
    "        array = [0 for i in range((seq_len - len(array)))] + array\n",
    "    elif len(array) > seq_len:\n",
    "        array = array[:seq_len]\n",
    "    return array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Textset(Dataset):\n",
    "    \n",
    "    def __init__(self, excel_file, file_path, clean_transform = None):\n",
    "        self.file = pd.read_excel(os.path.join(file_path, excel_file))\n",
    "        self.file = self.file.loc[:, 'tweet':'SENTIMENT']\n",
    "        self.file = self.file[self.file['SENTIMENT'] != 'IRR']\n",
    "        self.file = self.file[self.file['SENTIMENT'] != 'mixed']\n",
    "        self.file = self.file[self.file['SENTIMENT'] != 99]\n",
    "        self.file = self.file[self.file['SENTIMENT'] != '99']\n",
    "        self.file.dropna(inplace = True)\n",
    "        self.file = self.file.reset_index(drop = True)\n",
    "        plain_text = self.file['tweet']\n",
    "        self.prep_data = clean_transform(self.file, plain_text)\n",
    "        self.clean_transform = clean_transform\n",
    "        unique = set([word for tweet in self.prep_data for word in tweet])\n",
    "        self.vocab = {word:idx + 1 for idx, word in enumerate(unique)}\n",
    "        self.labels = self.file['SENTIMENT'].to_numpy()\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.file)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        sentence = self.prep_data[idx]\n",
    "        sentence = [self.vocab[word] for word in sentence]\n",
    "        label = self.labels[idx]\n",
    "        if self.clean_transform == clean_text_ger or self.clean_transform == clean_text_french:\n",
    "            return torch.Tensor(pad_sentence(sentence, 15)), torch.Tensor([int(label)])\n",
    "        else:\n",
    "            return torch.Tensor(pad_sentence(sentence, 15)), torch.Tensor([int(label) - 1])\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_set = Textset('jesuischarlie_infl_coded_26.11.2018.xlsx', '/home/max/sentiment_analysis/', clean_transform = clean_text_french)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('birulevo_sentiment_coded.xlsx',\n",
       " 'Ferguson_sentiment_CODED.xlsx',\n",
       " 'Сентимент_КЁЛЬН.xlsx',\n",
       " 'jesuischarlie_infl_coded_26.11.2018.xlsx')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'birulevo_sentiment_coded.xlsx', 'Ferguson_sentiment_CODED.xlsx', 'Сентимент_КЁЛЬН.xlsx', 'jesuischarlie_infl_coded_26.11.2018.xlsx'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5456\n"
     ]
    }
   ],
   "source": [
    "print(len(train_set.vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'divers': 1,\n",
       " 'continuerquot': 2,\n",
       " 'dorénav': 3,\n",
       " 'sexprim': 4,\n",
       " 'julien': 5,\n",
       " 'lâch': 6,\n",
       " 'bleu': 7,\n",
       " 'réseauislamistemarocain': 8,\n",
       " 'terrainb': 9,\n",
       " 'pas': 10,\n",
       " 'mrapofficiel': 11,\n",
       " 'tik': 12,\n",
       " 'a': 13,\n",
       " 'aiglon': 14,\n",
       " 'réveillent': 15,\n",
       " 'multiplient': 16,\n",
       " 'hi': 17,\n",
       " 'déguis': 18,\n",
       " 'kiosqu': 19,\n",
       " 'plein': 20,\n",
       " 'mélenchon': 21,\n",
       " 'ewen': 22,\n",
       " 'quotjour': 23,\n",
       " 'quatrem': 24,\n",
       " 'jbertholon': 25,\n",
       " 'dsl': 26,\n",
       " 'divis': 27,\n",
       " 'friteux': 28,\n",
       " 'quotsoyon': 29,\n",
       " 'san': 30,\n",
       " 'onclebernard': 31,\n",
       " 'allez': 32,\n",
       " 'publi': 33,\n",
       " 'stat': 34,\n",
       " 'saïd': 35,\n",
       " 'quotléconom': 36,\n",
       " 'tatidaniel': 37,\n",
       " 'reg': 38,\n",
       " 'onfaitlapaix': 39,\n",
       " 'exempl': 40,\n",
       " 'lhain': 41,\n",
       " 'débil': 42,\n",
       " 'secour': 43,\n",
       " 'gé': 44,\n",
       " 'naur': 45,\n",
       " 'partout': 46,\n",
       " 'bordeau': 47,\n",
       " 'reviv': 48,\n",
       " 'valeryhach': 49,\n",
       " 'newspap': 50,\n",
       " 'religieusesquot': 51,\n",
       " 'phillyd': 52,\n",
       " 'por': 53,\n",
       " 'inde': 54,\n",
       " 'cloudbreak': 55,\n",
       " 'album': 56,\n",
       " 'dapplaud': 57,\n",
       " 'julpellici': 58,\n",
       " 'brunobeschizz': 59,\n",
       " 'modem': 60,\n",
       " 'icib': 61,\n",
       " 'lordr': 62,\n",
       " 'osi': 63,\n",
       " 'interpel': 64,\n",
       " 'tribut': 65,\n",
       " 'hommagecharliehebdo': 66,\n",
       " 'lefigaro': 67,\n",
       " 'faitquot': 68,\n",
       " 'envi': 69,\n",
       " 'asshol': 70,\n",
       " 'cré': 71,\n",
       " 'confitur': 72,\n",
       " 'mouill': 73,\n",
       " 'affect': 74,\n",
       " 'stooop': 75,\n",
       " 'dellitali': 76,\n",
       " 'auss': 77,\n",
       " 'téléphon': 78,\n",
       " 'ruejesuischarl': 79,\n",
       " 'bcq': 80,\n",
       " 'dorléan': 81,\n",
       " 'scoignard': 82,\n",
       " 'lomofficiel': 83,\n",
       " 'rural': 84,\n",
       " 'photos': 85,\n",
       " 'barb': 86,\n",
       " 'arrêt': 87,\n",
       " 'goel': 88,\n",
       " 'autocensurequot': 89,\n",
       " 'velo': 90,\n",
       " 'wantz': 91,\n",
       " 'isnt': 92,\n",
       " 'collect': 93,\n",
       " 'prismamedi': 94,\n",
       " 'rempl': 95,\n",
       " 'irrespons': 96,\n",
       " 'pourt': 97,\n",
       " 'fej': 98,\n",
       " 'dotagequot': 99,\n",
       " 'soient': 100,\n",
       " 'ca': 101,\n",
       " 'hainen': 102,\n",
       " 'janvierlaïc': 103,\n",
       " 'richess': 104,\n",
       " 'jesuisyoav': 105,\n",
       " 'marseillais': 106,\n",
       " 'brico': 107,\n",
       " 'assez': 108,\n",
       " 'cavousregard': 109,\n",
       " 'nant': 110,\n",
       " 'veng': 111,\n",
       " 'fdelign': 112,\n",
       " 'nikosaliag': 113,\n",
       " 'plq': 114,\n",
       " 'jesuischagrin': 115,\n",
       " 'jesuisdaccordaveclui': 116,\n",
       " 'risquent': 117,\n",
       " 'réfléchirquot': 118,\n",
       " 'ahahahahah': 119,\n",
       " 'leg': 120,\n",
       " 'hélicopter': 121,\n",
       " 'devenu': 122,\n",
       " 'mar': 123,\n",
       " 'micro': 124,\n",
       " 'aux': 125,\n",
       " 'paulinezero': 126,\n",
       " 'deniscosnard': 127,\n",
       " 'sylviedlcroistj': 128,\n",
       " 'paraitil': 129,\n",
       " 'mrobertson': 130,\n",
       " 'boulevard': 131,\n",
       " 'amédy': 132,\n",
       " 'sarkozy': 133,\n",
       " 'sujet': 134,\n",
       " 'pdt': 135,\n",
       " 'observ': 136,\n",
       " 'kosovo': 137,\n",
       " 'lauredlr': 138,\n",
       " 'toit': 139,\n",
       " 'surtout': 140,\n",
       " 'anim': 141,\n",
       " 'morbid': 142,\n",
       " 'épous': 143,\n",
       " 'lécrivain': 144,\n",
       " 'ce': 145,\n",
       " 'deven': 146,\n",
       " 'cdeloir': 147,\n",
       " 'limog': 148,\n",
       " 'quotzemmour': 149,\n",
       " 'dordur': 150,\n",
       " 'lkebch': 151,\n",
       " 'présum': 152,\n",
       " 'maintenu': 153,\n",
       " 'lagglo': 154,\n",
       " 'venu': 155,\n",
       " 'bau': 156,\n",
       " 'dexpressionquot': 157,\n",
       " 'idéal': 158,\n",
       " 'notion': 159,\n",
       " 'douvr': 160,\n",
       " 'demain': 161,\n",
       " 'vol': 162,\n",
       " 'diplomat': 163,\n",
       " 'fb': 164,\n",
       " 'réfutent': 165,\n",
       " 'philo': 166,\n",
       " 'tjr': 167,\n",
       " 'solidaridad': 168,\n",
       " 'envoyeznous': 169,\n",
       " 'luimêm': 170,\n",
       " 'morganelaurent': 171,\n",
       " 'zam': 172,\n",
       " 'reven': 173,\n",
       " 'geluckofficiel': 174,\n",
       " 'mtn': 175,\n",
       " 'confianc': 176,\n",
       " 'quotc': 177,\n",
       " 'taisonsnous': 178,\n",
       " 'eureetloirrassembl': 179,\n",
       " 'benyamin': 180,\n",
       " 'altereco': 181,\n",
       " 'ème': 182,\n",
       " 'lhomm': 183,\n",
       " 'musulmanequot': 184,\n",
       " 'clariss': 185,\n",
       " 'rirequot': 186,\n",
       " 'trans': 187,\n",
       " 'lindécent': 188,\n",
       " 'rayclid': 189,\n",
       " 'nabed': 190,\n",
       " 'croi': 191,\n",
       " 'rochedy': 192,\n",
       " 'défilerquot': 193,\n",
       " 'islamofascismeno': 194,\n",
       " 'ny': 195,\n",
       " 'fier': 196,\n",
       " 'marseillaisequot': 197,\n",
       " 'deuil': 198,\n",
       " 'ceuxquiveulentdétruirelafr': 199,\n",
       " 'lafriqu': 200,\n",
       " 'manifestationquot': 201,\n",
       " 'fdg': 202,\n",
       " 'popolo': 203,\n",
       " 'lexprim': 204,\n",
       " 'autocensur': 205,\n",
       " 'dêtr': 206,\n",
       " 'toulousequot': 207,\n",
       " 'terror': 208,\n",
       " 'devient': 209,\n",
       " 'résoudr': 210,\n",
       " 'sgp': 211,\n",
       " 'prier': 212,\n",
       " 'religionsquot': 213,\n",
       " 'culturel': 214,\n",
       " 'estce': 215,\n",
       " 'sophiaaram': 216,\n",
       " 'jir': 217,\n",
       " 'ta': 218,\n",
       " 'poussin': 219,\n",
       " 'travail': 220,\n",
       " 'cf': 221,\n",
       " 'car': 222,\n",
       " 'quotnonquot': 223,\n",
       " 'tapi': 224,\n",
       " 'googl': 225,\n",
       " 'sappellent': 226,\n",
       " 'deurop': 227,\n",
       " 'vit': 228,\n",
       " 'revolu': 229,\n",
       " 'direct': 230,\n",
       " 'div': 231,\n",
       " 'cazenavetous': 232,\n",
       " 'laiss': 233,\n",
       " 'charliechaplin': 234,\n",
       " 'plussomm': 235,\n",
       " 'francetv': 236,\n",
       " 'business': 237,\n",
       " 'mendorm': 238,\n",
       " 'quotsuspend': 239,\n",
       " 'dal': 240,\n",
       " 'castellan': 241,\n",
       " 'éveil': 242,\n",
       " 'dimam': 243,\n",
       " 'dintegr': 244,\n",
       " 'tair': 245,\n",
       " 'indien': 246,\n",
       " 'varvel': 247,\n",
       " 'saint': 248,\n",
       " 'civil': 249,\n",
       " 'jeandormesson': 250,\n",
       " 'lign': 251,\n",
       " 'poursuiv': 252,\n",
       " 'pasaran': 253,\n",
       " 'autor': 254,\n",
       " 'jullien': 255,\n",
       " 'grov': 256,\n",
       " 'islamophobequot': 257,\n",
       " 'grandcorpsmalad': 258,\n",
       " 'plutôt': 259,\n",
       " 'great': 260,\n",
       " 'indigen': 261,\n",
       " 'obsequ': 262,\n",
       " 'délit': 263,\n",
       " 'jp': 264,\n",
       " 'mehdiyan': 265,\n",
       " 'clé': 266,\n",
       " 'colon': 267,\n",
       " 'g': 268,\n",
       " 'lislam': 269,\n",
       " 'laisn': 270,\n",
       " 'arismessin': 271,\n",
       " 'peu': 272,\n",
       " 'amusent': 273,\n",
       " 'payet': 274,\n",
       " 'lacteur': 275,\n",
       " 'jabberwork': 276,\n",
       " 'interview': 277,\n",
       " 'mettent': 278,\n",
       " 'quotminoritairesquot': 279,\n",
       " 'lcp': 280,\n",
       " 'william': 281,\n",
       " 'immobil': 282,\n",
       " 'rt': 283,\n",
       " 'gign': 284,\n",
       " 'mes': 285,\n",
       " 'sécuris': 286,\n",
       " 'colloqu': 287,\n",
       " 'quel': 288,\n",
       " 'list': 289,\n",
       " 'rpim': 290,\n",
       " 'support': 291,\n",
       " 'benedict': 292,\n",
       " 'livreur': 293,\n",
       " 'décis': 294,\n",
       " 'huffpostquebec': 295,\n",
       " 'leur': 296,\n",
       " 'ignorent': 297,\n",
       " 'yeux': 298,\n",
       " 'seineetmarn': 299,\n",
       " 'interrompu': 300,\n",
       " 'colomb': 301,\n",
       " 'symp': 302,\n",
       " 'cert': 303,\n",
       " 'sappropri': 304,\n",
       " 'chariahebdo': 305,\n",
       " 'biiiiiiiiiim': 306,\n",
       " 'ameliegoursaud': 307,\n",
       " 'nytim': 308,\n",
       " 'toulouseestcharl': 309,\n",
       " 'visibl': 310,\n",
       " 'blzonlin': 311,\n",
       " 'choc': 312,\n",
       " 'lencr': 313,\n",
       " 'mondieu': 314,\n",
       " 'grandios': 315,\n",
       " 'tchekhov': 316,\n",
       " 'applaud': 317,\n",
       " 'courtis': 318,\n",
       " 'souchon': 319,\n",
       " 'lesconsequent': 320,\n",
       " 'actuel': 321,\n",
       " 'chimulus': 322,\n",
       " 'jesuischarliechérifkouach': 323,\n",
       " 'joursson': 324,\n",
       " 'bel': 325,\n",
       " 'quotrésisterquotjesuischarl': 326,\n",
       " 'vidberg': 327,\n",
       " 'formid': 328,\n",
       " 'patrick': 329,\n",
       " 'pavois': 330,\n",
       " 'term': 331,\n",
       " 'version': 332,\n",
       " 'inquiet': 333,\n",
       " 'râleur': 334,\n",
       " 'sinvit': 335,\n",
       " 'avoir': 336,\n",
       " 'llligiouxxx': 337,\n",
       " 'prit': 338,\n",
       " 'dépêch': 339,\n",
       " 'montron': 340,\n",
       " 'libertéquot': 341,\n",
       " 'daccueil': 342,\n",
       " 'pasjenesuispascharl': 343,\n",
       " 'néerland': 344,\n",
       " 'dexemplair': 345,\n",
       " 'danielleduch': 346,\n",
       " 'l': 347,\n",
       " 'traquequot': 348,\n",
       " 'en': 349,\n",
       " 'dangoiss': 350,\n",
       " 'viennequot': 351,\n",
       " 'trait': 352,\n",
       " 'maitr': 353,\n",
       " 'mediapollcp': 354,\n",
       " 'trist': 355,\n",
       " 'dencr': 356,\n",
       " 'jesuischarliequand': 357,\n",
       " 'rentr': 358,\n",
       " 'quotcet': 359,\n",
       " 'québécois': 360,\n",
       " 'dessinon': 361,\n",
       " 'enfantin': 362,\n",
       " 'quotcaricatur': 363,\n",
       " 'solidairesquot': 364,\n",
       " 'évit': 365,\n",
       " 'grandeur': 366,\n",
       " 'audreygoutard': 367,\n",
       " 'saur': 368,\n",
       " 'est': 369,\n",
       " 'ggrmc': 370,\n",
       " 'désorm': 371,\n",
       " 'beau': 372,\n",
       " 'doù': 373,\n",
       " 'fric': 374,\n",
       " 'fbid': 375,\n",
       " 'fait': 376,\n",
       " 'jaid': 377,\n",
       " 'lintérieur': 378,\n",
       " 'direquot': 379,\n",
       " 'pouvoir': 380,\n",
       " 'sachon': 381,\n",
       " 'chrisdegr': 382,\n",
       " 'orleansestcharl': 383,\n",
       " 'offrent': 384,\n",
       " 'crach': 385,\n",
       " 'jeanbriault': 386,\n",
       " 'palestinien': 387,\n",
       " 'of': 388,\n",
       " 'recenséequot': 389,\n",
       " 'rochesuryon': 390,\n",
       " 'laccord': 391,\n",
       " 'accept': 392,\n",
       " 'social': 393,\n",
       " 'nogentleroiarriv': 394,\n",
       " 'offre': 395,\n",
       " 'lecteur': 396,\n",
       " 'vient': 397,\n",
       " 'éton': 398,\n",
       " 'raison': 399,\n",
       " 'comprisquot': 400,\n",
       " 'jmgermain': 401,\n",
       " 'tient': 402,\n",
       " 'programm': 403,\n",
       " 'svp': 404,\n",
       " 'rois': 405,\n",
       " 'geueledebois': 406,\n",
       " 'quotlà': 407,\n",
       " 'author': 408,\n",
       " 'jeannedarcestcharl': 409,\n",
       " 'pupuch': 410,\n",
       " 'jenesuispasterroe': 411,\n",
       " 'prefpolic': 412,\n",
       " 'jregard': 413,\n",
       " 'avion': 414,\n",
       " 'géhennequot': 415,\n",
       " 'israël': 416,\n",
       " 'manqu': 417,\n",
       " 'frisson': 418,\n",
       " 'lenfantquot': 419,\n",
       " 'arrest': 420,\n",
       " 'an': 421,\n",
       " 'reçoit': 422,\n",
       " 'quotlorsqu': 423,\n",
       " 'midilibr': 424,\n",
       " 'vouiiii': 425,\n",
       " 'sorganisent': 426,\n",
       " 'donilem': 427,\n",
       " 'show': 428,\n",
       " 'orang': 429,\n",
       " 'solidair': 430,\n",
       " 'terrequot': 431,\n",
       " 'sappel': 432,\n",
       " 'denisbaupin': 433,\n",
       " 'vaincr': 434,\n",
       " 'éven': 435,\n",
       " 'cecileduflot': 436,\n",
       " 'prisedotagebravoaugignetauraid': 437,\n",
       " 'lois': 438,\n",
       " 'touchéquot': 439,\n",
       " 'instaur': 440,\n",
       " 'démarr': 441,\n",
       " 'taisezvous': 442,\n",
       " 'quotvoir': 443,\n",
       " 'complais': 444,\n",
       " 'b': 445,\n",
       " 'profil': 446,\n",
       " 'janet': 447,\n",
       " 'branchequot': 448,\n",
       " 'abrezet': 449,\n",
       " 'sbeaugeafp': 450,\n",
       " 'citat': 451,\n",
       " 'remis': 452,\n",
       " 'parol': 453,\n",
       " 'loloric': 454,\n",
       " 'kaganmcleod': 455,\n",
       " 'akbar': 456,\n",
       " 'aliciafall': 457,\n",
       " 'hebdo': 458,\n",
       " 'retr': 459,\n",
       " 'yan': 460,\n",
       " 'ailleur': 461,\n",
       " 'quotdes': 462,\n",
       " 'timesjesuischarl': 463,\n",
       " 'lath': 464,\n",
       " 'world': 465,\n",
       " 'larg': 466,\n",
       " 'nin': 467,\n",
       " 'las': 468,\n",
       " 'day': 469,\n",
       " 'planqu': 470,\n",
       " 'denec': 471,\n",
       " 'peupl': 472,\n",
       " 'john': 473,\n",
       " 'auront': 474,\n",
       " 'noussommestousdescharl': 475,\n",
       " 'beignet': 476,\n",
       " 'vin': 477,\n",
       " 'londr': 478,\n",
       " 'quotn': 479,\n",
       " 'detruir': 480,\n",
       " 'extrait': 481,\n",
       " 'relat': 482,\n",
       " 'sourir': 483,\n",
       " 'nomquot': 484,\n",
       " 'auditeur': 485,\n",
       " 'antonio': 486,\n",
       " 'liniti': 487,\n",
       " 'journ': 488,\n",
       " 'bild': 489,\n",
       " 'battu': 490,\n",
       " 'jesuischarlieallon': 491,\n",
       " 'illustr': 492,\n",
       " 'yve': 493,\n",
       " 'chaussur': 494,\n",
       " 'marcheur': 495,\n",
       " 'raif': 496,\n",
       " 'jesuisfranc': 497,\n",
       " 'dentr': 498,\n",
       " 'detien': 499,\n",
       " 'quotjodelb': 500,\n",
       " 'dunion': 501,\n",
       " 'c': 502,\n",
       " 'stationnair': 503,\n",
       " 'cultur': 504,\n",
       " 'bourgoinjallieu': 505,\n",
       " 'dictator': 506,\n",
       " 'censur': 507,\n",
       " 'ahmedmerabet': 508,\n",
       " 'chrétien': 509,\n",
       " 'orléan': 510,\n",
       " 'nation': 511,\n",
       " 'blandinek': 512,\n",
       " 'écrivon': 513,\n",
       " 'particuli': 514,\n",
       " 'obscurant': 515,\n",
       " 'foi': 516,\n",
       " 'quaux': 517,\n",
       " 'chi': 518,\n",
       " 'blanch': 519,\n",
       " 'retranch': 520,\n",
       " 'devenus': 521,\n",
       " 'abim': 522,\n",
       " 'afpfr': 523,\n",
       " 'matern': 524,\n",
       " 'charliehedbo': 525,\n",
       " 'suis': 526,\n",
       " 'kissprisesdotag': 527,\n",
       " 'publicquot': 528,\n",
       " 'islamoracaill': 529,\n",
       " 'espoir': 530,\n",
       " 'christienicor': 531,\n",
       " 'couvr': 532,\n",
       " 'troisiem': 533,\n",
       " 'soud': 534,\n",
       " 'lhabituel': 535,\n",
       " 'barak': 536,\n",
       " 'dormesson': 537,\n",
       " 'guerr': 538,\n",
       " 'rag': 539,\n",
       " 'cassel': 540,\n",
       " 'détat': 541,\n",
       " 'lequip': 542,\n",
       " 'again': 543,\n",
       " 'simultan': 544,\n",
       " 'lallemand': 545,\n",
       " 'minimum': 546,\n",
       " 'pnational': 547,\n",
       " 'pav': 548,\n",
       " 'poursuit': 549,\n",
       " 'avocat': 550,\n",
       " 'banni': 551,\n",
       " 'linvit': 552,\n",
       " 'ecaban': 553,\n",
       " 'lémoticôn': 554,\n",
       " 'attentat': 555,\n",
       " 'royan': 556,\n",
       " 'pareil': 557,\n",
       " 'ouvert': 558,\n",
       " 'cerveau': 559,\n",
       " 'mettr': 560,\n",
       " 'franzolivierg': 561,\n",
       " 'beezoo': 562,\n",
       " 'rouden': 563,\n",
       " 'teamenomalagr': 564,\n",
       " 'suspect': 565,\n",
       " 'introduit': 566,\n",
       " 'étriqu': 567,\n",
       " 'ogcn': 568,\n",
       " 'euxmêm': 569,\n",
       " 'quotpolitikenquot': 570,\n",
       " 'procureur': 571,\n",
       " 'vous': 572,\n",
       " 'rud': 573,\n",
       " 'résum': 574,\n",
       " 'english': 575,\n",
       " 'lexpress': 576,\n",
       " 'ménilmont': 577,\n",
       " 'répons': 578,\n",
       " 'hélen': 579,\n",
       " 'lhumeur': 580,\n",
       " 'guesd': 581,\n",
       " 'dalilboubakeur': 582,\n",
       " 'inévit': 583,\n",
       " 'newsthissecond': 584,\n",
       " 'n': 585,\n",
       " 'vichy': 586,\n",
       " 'sag': 587,\n",
       " 'postur': 588,\n",
       " 'répondr': 589,\n",
       " 'empêch': 590,\n",
       " 'national': 591,\n",
       " 'douz': 592,\n",
       " 'tte': 593,\n",
       " 'jfkahn': 594,\n",
       " 'lenf': 595,\n",
       " 'quotvoisin': 596,\n",
       " 'poursuivr': 597,\n",
       " 'quotdieu': 598,\n",
       " 'manipul': 599,\n",
       " 'orlean': 600,\n",
       " 'lilou': 601,\n",
       " 'dexpressionjenesuispascharl': 602,\n",
       " 'cliquon': 603,\n",
       " 'fidoroyal': 604,\n",
       " 'chari': 605,\n",
       " 'gouvern': 606,\n",
       " 'zadigetmoi': 607,\n",
       " 'quotl': 608,\n",
       " 'pis': 609,\n",
       " 'maréchaltous': 610,\n",
       " 'nentendr': 611,\n",
       " 'courag': 612,\n",
       " 'linquiétud': 613,\n",
       " 'hât': 614,\n",
       " 'musical': 615,\n",
       " 'quotcabu': 616,\n",
       " 'robin': 617,\n",
       " 'racism': 618,\n",
       " 'tout': 619,\n",
       " 'janv': 620,\n",
       " 'effondr': 621,\n",
       " 'crayonsjesuischarl': 622,\n",
       " 'connaisson': 623,\n",
       " 'safiaotokor': 624,\n",
       " 'vraisembl': 625,\n",
       " 'sal': 626,\n",
       " 'test': 627,\n",
       " 'interpol': 628,\n",
       " 'retweeet': 629,\n",
       " 'camill': 630,\n",
       " 'caricatur': 631,\n",
       " 'ministr': 632,\n",
       " 'dir': 633,\n",
       " 'visit': 634,\n",
       " 'lamyf': 635,\n",
       " 'bandeau': 636,\n",
       " 'pellegrin': 637,\n",
       " 'matabiau': 638,\n",
       " 'istanbul': 639,\n",
       " 'fris': 640,\n",
       " 'maltrait': 641,\n",
       " 'torchon': 642,\n",
       " 'augmenton': 643,\n",
       " 'assembl': 644,\n",
       " 'simpl': 645,\n",
       " 'quotsymboliquesquot': 646,\n",
       " 'is': 647,\n",
       " 'commerc': 648,\n",
       " 'profess': 649,\n",
       " 'crumb': 650,\n",
       " 'expir': 651,\n",
       " 'sinstall': 652,\n",
       " 'rajout': 653,\n",
       " 'quotmoi': 654,\n",
       " 'franck': 655,\n",
       " 'offic': 656,\n",
       " 'quotjaur': 657,\n",
       " 'manudansl': 658,\n",
       " 'isahorlan': 659,\n",
       " 'francemidipy': 660,\n",
       " 'pp': 661,\n",
       " 'pot': 662,\n",
       " 'jeanclaud': 663,\n",
       " 'fond': 664,\n",
       " 'cathyrochon': 665,\n",
       " 'cerdan': 666,\n",
       " 'vend': 667,\n",
       " 'chateaubriand': 668,\n",
       " 'français': 669,\n",
       " 'jusquà': 670,\n",
       " 'var': 671,\n",
       " 'relig': 672,\n",
       " 'errorisml': 673,\n",
       " 'tueursquot': 674,\n",
       " 'actionquot': 675,\n",
       " 'effroyablequot': 676,\n",
       " 'vot': 677,\n",
       " 'daccord': 678,\n",
       " 'perdr': 679,\n",
       " 'basket': 680,\n",
       " 'lintégrationquot': 681,\n",
       " 'pathet': 682,\n",
       " 'cambadel': 683,\n",
       " 'conlieg': 684,\n",
       " 'pressequot': 685,\n",
       " 'pipeau': 686,\n",
       " 'tourn': 687,\n",
       " 'roomquot': 688,\n",
       " 'sharing': 689,\n",
       " 'voir': 690,\n",
       " 'malgr': 691,\n",
       " 'dteyssou': 692,\n",
       " 'bridg': 693,\n",
       " 'mdr': 694,\n",
       " 'thomaswied': 695,\n",
       " 'americain': 696,\n",
       " 'sourc': 697,\n",
       " 'laquel': 698,\n",
       " 'vinciautorout': 699,\n",
       " 'adult': 700,\n",
       " 't': 701,\n",
       " 'chtit': 702,\n",
       " 'petit': 703,\n",
       " 'blog': 704,\n",
       " 'conten': 705,\n",
       " 'éteint': 706,\n",
       " 'apolog': 707,\n",
       " 'dimens': 708,\n",
       " 'sandrap': 709,\n",
       " 'helenefontanaud': 710,\n",
       " 'rteefufkin': 711,\n",
       " 'religion': 712,\n",
       " 'iranian': 713,\n",
       " 'dur': 714,\n",
       " 'mari': 715,\n",
       " 'sinon': 716,\n",
       " 'capabl': 717,\n",
       " 'ampproudtob': 718,\n",
       " 'salam': 719,\n",
       " 'linsult': 720,\n",
       " 'quarti': 721,\n",
       " 'ridicul': 722,\n",
       " 'létat': 723,\n",
       " 'vall': 724,\n",
       " 'rer': 725,\n",
       " 'athen': 726,\n",
       " 'précis': 727,\n",
       " 'elsa': 728,\n",
       " 'dix': 729,\n",
       " 'copenhagu': 730,\n",
       " 'le': 731,\n",
       " 'sombr': 732,\n",
       " 'nordest': 733,\n",
       " 'afp': 734,\n",
       " 'chef': 735,\n",
       " 'dégoulin': 736,\n",
       " 'alqaed': 737,\n",
       " 'erwan': 738,\n",
       " 'blasphem': 739,\n",
       " 'fkgirard': 740,\n",
       " 'ump': 741,\n",
       " 'rend': 742,\n",
       " 'chaqu': 743,\n",
       " 'rat': 744,\n",
       " 'gommon': 745,\n",
       " 'marcheump': 746,\n",
       " 'extrêm': 747,\n",
       " 'breakingzero': 748,\n",
       " 'tous': 749,\n",
       " 'antisystem': 750,\n",
       " 'accueil': 751,\n",
       " 'paru': 752,\n",
       " 'daily': 753,\n",
       " 'letap': 754,\n",
       " 'jai': 755,\n",
       " 'ni': 756,\n",
       " 'buzz': 757,\n",
       " 'cri': 758,\n",
       " 'réun': 759,\n",
       " 'taub': 760,\n",
       " 'pans': 761,\n",
       " 'ciblequot': 762,\n",
       " 'crois': 763,\n",
       " 'émeut': 764,\n",
       " 'rassemblent': 765,\n",
       " 'medh': 766,\n",
       " 'contrôlequot': 767,\n",
       " 'surfeur': 768,\n",
       " 'nonos': 769,\n",
       " 'sudarsan': 770,\n",
       " 'ai': 771,\n",
       " 'montpellierom': 772,\n",
       " 'ali': 773,\n",
       " 'vieuxport': 774,\n",
       " 'pres': 775,\n",
       " 'émigrent': 776,\n",
       " 'bourrag': 777,\n",
       " 'produit': 778,\n",
       " 'didi': 779,\n",
       " 'jesuisflic': 780,\n",
       " 'fendr': 781,\n",
       " 'retient': 782,\n",
       " 'combattr': 783,\n",
       " 'seul': 784,\n",
       " 'sifongtous': 785,\n",
       " 'sobr': 786,\n",
       " 'liberteexpress': 787,\n",
       " 'sao': 788,\n",
       " 'attack': 789,\n",
       " 'ne': 790,\n",
       " 'respect': 791,\n",
       " 'mid': 792,\n",
       " 'jesuisbluff': 793,\n",
       " 'ains': 794,\n",
       " 'europ': 795,\n",
       " 'exprim': 796,\n",
       " 'linsécur': 797,\n",
       " 'lieuxquot': 798,\n",
       " 'loutranc': 799,\n",
       " 'johnplissken': 800,\n",
       " 'chiffr': 801,\n",
       " 'chow': 802,\n",
       " 'abandon': 803,\n",
       " 'croyanc': 804,\n",
       " 'nicolasbatum': 805,\n",
       " 'gendarm': 806,\n",
       " 'courrierpicard': 807,\n",
       " 'conseildepar': 808,\n",
       " 'sain': 809,\n",
       " 'chois': 810,\n",
       " 'tes': 811,\n",
       " 'yvepain': 812,\n",
       " 'equip': 813,\n",
       " 'q': 814,\n",
       " 'jour': 815,\n",
       " 'tais': 816,\n",
       " 'pleas': 817,\n",
       " 'dquotentr': 818,\n",
       " 'rebond': 819,\n",
       " 'lafranceselev': 820,\n",
       " 'notre': 821,\n",
       " 'weekend': 822,\n",
       " 'titr': 823,\n",
       " 'mustaph': 824,\n",
       " 'politiken': 825,\n",
       " 'dévast': 826,\n",
       " 'initial': 827,\n",
       " 'tonvoisin': 828,\n",
       " 'cot': 829,\n",
       " 'homepag': 830,\n",
       " 'traqu': 831,\n",
       " 'peuvent': 832,\n",
       " 'négoci': 833,\n",
       " 'normalis': 834,\n",
       " 'subliminaljesuischarliejesuisisraël': 835,\n",
       " 'semainesquot': 836,\n",
       " 'déjà': 837,\n",
       " 'contrair': 838,\n",
       " 'indécent': 839,\n",
       " 'lieux': 840,\n",
       " 'questionsquot': 841,\n",
       " 'aka': 842,\n",
       " 'islamo': 843,\n",
       " 'algérien': 844,\n",
       " 'froid': 845,\n",
       " 'loper': 846,\n",
       " 'mour': 847,\n",
       " 'recherch': 848,\n",
       " 'cœur': 849,\n",
       " 'suff': 850,\n",
       " 'redescendr': 851,\n",
       " 'hor': 852,\n",
       " 'benjamin': 853,\n",
       " 'libertéquotl': 854,\n",
       " 'hauteur': 855,\n",
       " 'partenair': 856,\n",
       " 'tt': 857,\n",
       " 'désatr': 858,\n",
       " 'lol': 859,\n",
       " 'dr': 860,\n",
       " 'propagand': 861,\n",
       " 'militair': 862,\n",
       " 'ahmedelsafranckfrédéricmichelmustaphahonorétignousbernardcharbwolinskicabujesuischarl': 863,\n",
       " 'urb': 864,\n",
       " 'radiocheriefm': 865,\n",
       " 'arrier': 866,\n",
       " 'bombard': 867,\n",
       " 'te': 868,\n",
       " 'leman': 869,\n",
       " 'bafou': 870,\n",
       " 'femen': 871,\n",
       " 'odeur': 872,\n",
       " 'wellis': 873,\n",
       " 'parl': 874,\n",
       " 'cc': 875,\n",
       " 'davantag': 876,\n",
       " 'meurtriequot': 877,\n",
       " 'angiemin': 878,\n",
       " 'alarm': 879,\n",
       " 'légit': 880,\n",
       " 'foufoun': 881,\n",
       " 'orchestr': 882,\n",
       " 'tousjesuischarl': 883,\n",
       " 'diffus': 884,\n",
       " 'curieux': 885,\n",
       " 'disqualifi': 886,\n",
       " 'plui': 887,\n",
       " 'redact': 888,\n",
       " 'personnesquot': 889,\n",
       " 'newyork': 890,\n",
       " 'daor': 891,\n",
       " 'summum': 892,\n",
       " 'pellisard': 893,\n",
       " 'lignesquot': 894,\n",
       " 'thom': 895,\n",
       " 'respectent': 896,\n",
       " 'unionsquar': 897,\n",
       " 'revendiquent': 898,\n",
       " 'serieux': 899,\n",
       " 'quotcarolinefourest': 900,\n",
       " 'sacrifi': 901,\n",
       " 'dénonc': 902,\n",
       " 'plongent': 903,\n",
       " 'croientjesuischarl': 904,\n",
       " 'peux': 905,\n",
       " 'déjeun': 906,\n",
       " 'enlev': 907,\n",
       " 'quottous': 908,\n",
       " 'exact': 909,\n",
       " 'glac': 910,\n",
       " 'rath': 911,\n",
       " 'lebonlebontous': 912,\n",
       " 'tranquill': 913,\n",
       " 'lambassad': 914,\n",
       " 'joindr': 915,\n",
       " 'augment': 916,\n",
       " 'be': 917,\n",
       " 'malcolm': 918,\n",
       " 'jvoul': 919,\n",
       " 'pessin': 920,\n",
       " 'afif': 921,\n",
       " 'aymericcaron': 922,\n",
       " 'suppport': 923,\n",
       " 'scotch': 924,\n",
       " 'mediapart': 925,\n",
       " 'schulz': 926,\n",
       " 'ruthelkrief': 927,\n",
       " 'our': 928,\n",
       " 'brav': 929,\n",
       " 'dhab': 930,\n",
       " 'luz': 931,\n",
       " 'pierrottous': 932,\n",
       " 'quothérosquot': 933,\n",
       " 'pist': 934,\n",
       " 'soutien': 935,\n",
       " 'dessousquot': 936,\n",
       " 'rugby': 937,\n",
       " 'jeuquot': 938,\n",
       " 'brefjetweet': 939,\n",
       " 'messagefort': 940,\n",
       " 'interact': 941,\n",
       " 'airfranc': 942,\n",
       " 'sauvent': 943,\n",
       " 'quotmesur': 944,\n",
       " 'ghil': 945,\n",
       " 'hold': 946,\n",
       " 'quotlecho': 947,\n",
       " 'maison': 948,\n",
       " 'exceptionnel': 949,\n",
       " 'croit': 950,\n",
       " 'bain': 951,\n",
       " 'ya': 952,\n",
       " 'saintgermain': 953,\n",
       " 'autr': 954,\n",
       " 'choix': 955,\n",
       " 'vant': 956,\n",
       " 'pizz': 957,\n",
       " 'regard': 958,\n",
       " 'toureiffel': 959,\n",
       " 'attendu': 960,\n",
       " 'just': 961,\n",
       " 'libertair': 962,\n",
       " 'vigipirat': 963,\n",
       " 'mrtthomasragon': 964,\n",
       " 'indemn': 965,\n",
       " 'dessin': 966,\n",
       " 'nom': 967,\n",
       " 'légend': 968,\n",
       " 'expriméequot': 969,\n",
       " 'youp': 970,\n",
       " 'kong': 971,\n",
       " 'séteindront': 972,\n",
       " 'mai': 973,\n",
       " 'sandyjoy': 974,\n",
       " 'navon': 975,\n",
       " 'nicematin': 976,\n",
       " 'illumin': 977,\n",
       " 'import': 978,\n",
       " 'quotarrêt': 979,\n",
       " 'puquot': 980,\n",
       " 'périmetr': 981,\n",
       " 'rebel': 982,\n",
       " 'soudain': 983,\n",
       " 'ded': 984,\n",
       " 'incroi': 985,\n",
       " 'difficil': 986,\n",
       " 'lepoint': 987,\n",
       " 'fphilippot': 988,\n",
       " 'vérit': 989,\n",
       " 'pqr': 990,\n",
       " 'divanquot': 991,\n",
       " 'bory': 992,\n",
       " 'honneur': 993,\n",
       " 'châteaurenard': 994,\n",
       " 'has': 995,\n",
       " 'quotbildquot': 996,\n",
       " 'alfred': 997,\n",
       " 'coin': 998,\n",
       " 'préfer': 999,\n",
       " 'lovato': 1000,\n",
       " ...}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([263., 338., 426., 633., 337., 464., 345., 207.,  66.,  12.]),\n",
       " array([ 0. ,  2.8,  5.6,  8.4, 11.2, 14. , 16.8, 19.6, 22.4, 25.2, 28. ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAD4VJREFUeJzt3W2MHVd9x/HvrwmhVaA4IZvIst0uFKstqkQSrdJUqRAlLcpDVacSroiqxkSW3BehAlGpuLyBSq1kqpZApCqSi2mdiqcoQGORiGKZINoXSVmHNA8Yajdy461de2keII0oCvz7Ys+KxV5773rv+voevh9pNTP/OffOORr5t6PjmdlUFZKkfv3UqDsgSVpdBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z6CWpcxeOugMAl112WU1OTo66G5I0Vvbv3//tqppYqt15EfSTk5NMT0+PuhuSNFaS/Ocg7Zy6kaTOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z6CWpcwa9JHXOoJekzp0XT8ZqfExuf2Akxz284+aRHFfqgVf0ktQ5g16SOmfQS1LnDHpJ6pxBL0mdM+glqXMDBX2SNUnuS/LNJAeS/FqSS5PsTXKwLS9pbZPkriSHkjye5OrVHYIk6UwGvaL/KPDFqvol4E3AAWA7sK+qNgL72jbAjcDG9rMNuHuoPZYkLcuSQZ/kZ4E3A7sAqur7VfU8sAnY3ZrtBm5p65uAe2rOw8CaJGuH3nNJ0kAGuaJ/PTAL/F2Sryf5WJKLgSuq6hhAW17e2q8Djiz4/EyrSZJGYJCgvxC4Gri7qq4C/pcfTdMsJovU6pRGybYk00mmZ2dnB+qsJGn5Bgn6GWCmqh5p2/cxF/zH56dk2vLEgvYbFnx+PXD05C+tqp1VNVVVUxMTE2fbf0nSEpYM+qr6b+BIkl9speuBbwB7gC2ttgW4v63vAW5rd99cC7wwP8UjSTr3Bn175R8Bn0hyEfA0cDtzvyTuTbIVeAbY3No+CNwEHAJeam0lSSMyUNBX1WPA1CK7rl+kbQF3rLBfkqQh8clYSeqcQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5g16SOmfQS1LnDHpJ6pxBL0mdM+glqXMGvSR1zqCXpM4Z9JLUOYNekjpn0EtS5wYK+iSHkzyR5LEk0612aZK9SQ625SWtniR3JTmU5PEkV6/mACRJZ7acK/rfqKorq2qqbW8H9lXVRmBf2wa4EdjYfrYBdw+rs5Kk5VvJ1M0mYHdb3w3csqB+T815GFiTZO0KjiNJWoFBg76ALyXZn2Rbq11RVccA2vLyVl8HHFnw2ZlW+zFJtiWZTjI9Ozt7dr2XJC3pwgHbXVdVR5NcDuxN8s0ztM0itTqlULUT2AkwNTV1yn5J0nAMdEVfVUfb8gTweeAa4Pj8lExbnmjNZ4ANCz6+Hjg6rA5LkpZnyaBPcnGSV8+vA28DngT2AFtasy3A/W19D3Bbu/vmWuCF+SkeSdK5N8jUzRXA55PMt/9kVX0xydeAe5NsBZ4BNrf2DwI3AYeAl4Dbh95rSdLAlgz6qnoaeNMi9f8Brl+kXsAdQ+mdJGnFfDJWkjpn0EtS5wx6SeqcQS9JnTPoJalzBr0kdW7QVyBIP5Emtz8wsmMf3nHzyI6tvnhFL0mdM+glqXMGvSR1zqCXpM4Z9JLUOYNekjpn0EtS5wx6SeqcQS9JnTPoJalzBr0kdc533YyhUb5/RdL48Ypekjpn0EtS5wx6SeqcQS9JnRs46JNckOTrSb7Qtl+X5JEkB5N8JslFrf7Ktn2o7Z9cna5LkgaxnCv6dwMHFmx/CLizqjYCzwFbW30r8FxVvQG4s7WTJI3IQEGfZD1wM/Cxth3grcB9rclu4Ja2vqlt0/Zf39pLkkZg0Cv6jwB/Avywbb8WeL6qXm7bM8C6tr4OOALQ9r/Q2kuSRmDJoE/y28CJqtq/sLxI0xpg38Lv3ZZkOsn07OzsQJ2VJC3fIFf01wG/k+Qw8Gnmpmw+AqxJMv9k7XrgaFufATYAtP2vAZ49+UuramdVTVXV1MTExIoGIUk6vSWDvqr+tKrWV9Uk8A7gy1X1+8BDwNtbsy3A/W19T9um7f9yVZ1yRS9JOjdWch/9+4D3JjnE3Bz8rlbfBby21d8LbF9ZFyVJK7Gsl5pV1VeAr7T1p4FrFmnzPWDzEPomSRoCn4yVpM4Z9JLUOYNekjpn0EtS5wx6SeqcQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknq3LLeXinp3Jnc/sBIjnt4x80jOa5Wj1f0ktQ5r+hXYFRXXJK0HAa9xoK/VKWz59SNJHXOoJekzhn0ktQ5g16SOmfQS1LnDHpJ6pxBL0mdWzLok/x0kn9N8m9JnkryZ63+uiSPJDmY5DNJLmr1V7btQ23/5OoOQZJ0JoNc0f8f8NaqehNwJXBDkmuBDwF3VtVG4Dlga2u/FXiuqt4A3NnaSZJGZMmgrzkvts1XtJ8C3grc1+q7gVva+qa2Tdt/fZIMrceSpGUZaI4+yQVJHgNOAHuB/wCer6qXW5MZYF1bXwccAWj7XwBeO8xOS5IGN1DQV9UPqupKYD1wDfDLizVry8Wu3uvkQpJtSaaTTM/Ozg7aX0nSMi3rrpuqeh74CnAtsCbJ/EvR1gNH2/oMsAGg7X8N8Owi37WzqqaqampiYuLsei9JWtIgd91MJFnT1n8G+E3gAPAQ8PbWbAtwf1vf07Zp+79cVadc0UuSzo1BXlO8Ftid5ALmfjHcW1VfSPIN4NNJ/hz4OrCrtd8F/EOSQ8xdyb9jFfotSRrQkkFfVY8DVy1Sf5q5+fqT698DNg+ld5KkFfPJWEnqnEEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5g16SOjfIA1PntcntD4y6C5J0XvOKXpI6Z9BLUucMeknqnEEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5g16SOmfQS1LnDHpJ6pxBL0mdM+glqXMGvSR1zqCXpM4Z9JLUuSWDPsmGJA8lOZDkqSTvbvVLk+xNcrAtL2n1JLkryaEkjye5erUHIUk6vUH+wtTLwB9X1aNJXg3sT7IXeCewr6p2JNkObAfeB9wIbGw/vwrc3ZaSxsAo/2rb4R03j+zYPVvyir6qjlXVo239u8ABYB2wCdjdmu0Gbmnrm4B7as7DwJoka4fec0nSQJY1R59kErgKeAS4oqqOwdwvA+Dy1mwdcGTBx2Za7eTv2pZkOsn07Ozs8nsuSRrIwEGf5FXAZ4H3VNV3ztR0kVqdUqjaWVVTVTU1MTExaDckScs0UNAneQVzIf+JqvpcKx+fn5JpyxOtPgNsWPDx9cDR4XRXkrRcg9x1E2AXcKCqPrxg1x5gS1vfAty/oH5bu/vmWuCF+SkeSdK5N8hdN9cBfwA8keSxVns/sAO4N8lW4Blgc9v3IHATcAh4Cbh9qD2WJC3LkkFfVf/C4vPuANcv0r6AO1bYL0nSkPhkrCR1zqCXpM4Z9JLUOYNekjpn0EtS5wx6SeqcQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknqnEEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5g16SOmfQS1LnDHpJ6pxBL0mdWzLok3w8yYkkTy6oXZpkb5KDbXlJqyfJXUkOJXk8ydWr2XlJ0tIGuaL/e+CGk2rbgX1VtRHY17YBbgQ2tp9twN3D6aYk6WwtGfRV9VXg2ZPKm4DdbX03cMuC+j0152FgTZK1w+qsJGn5znaO/oqqOgbQlpe3+jrgyIJ2M60mSRqRYf9nbBap1aINk21JppNMz87ODrkbkqR5Zxv0x+enZNryRKvPABsWtFsPHF3sC6pqZ1VNVdXUxMTEWXZDkrSUsw36PcCWtr4FuH9B/bZ29821wAvzUzySpNG4cKkGST4FvAW4LMkM8AFgB3Bvkq3AM8Dm1vxB4CbgEPAScPsq9FmStAxLBn1V3XqaXdcv0raAO1baKUnS8PhkrCR1zqCXpM4Z9JLUOYNekjpn0EtS5wx6SerckrdXStK5Mrn9gZEc9/COm0dy3HPFK3pJ6pxBL0mdM+glqXMGvSR1zqCXpM4Z9JLUOYNekjpn0EtS5wx6SeqcQS9JnTPoJalzBr0kdc6gl6TOGfSS1DmDXpI6Z9BLUucMeknq3Kr8hakkNwAfBS4APlZVO1bjOJI0DKP6y1Zwbv661dCv6JNcAPwNcCPwRuDWJG8c9nEkSYNZjamba4BDVfV0VX0f+DSwaRWOI0kawGoE/TrgyILtmVaTJI3AaszRZ5FandIo2QZsa5svJvnWWR7vMuDbZ/nZ812vY3Nc46fXsY18XPnQij7+84M0Wo2gnwE2LNheDxw9uVFV7QR2rvRgSaaramql33M+6nVsjmv89Dq2Xsd1stWYuvkasDHJ65JcBLwD2LMKx5EkDWDoV/RV9XKSdwH/xNztlR+vqqeGfRxJ0mBW5T76qnoQeHA1vnsRK57+OY/1OjbHNX56HVuv4/oxqTrl/0klSR3xFQiS1LmxDvokNyT5VpJDSbaPuj/DkuRwkieSPJZketT9WYkkH09yIsmTC2qXJtmb5GBbXjLKPp6N04zrg0n+q523x5LcNMo+no0kG5I8lORAkqeSvLvVx/qcnWFcY3/OBjG2UzftVQv/DvwWc7d0fg24taq+MdKODUGSw8BUVY39fctJ3gy8CNxTVb/San8JPFtVO9ov6Euq6n2j7OdynWZcHwRerKq/GmXfViLJWmBtVT2a5NXAfuAW4J2M8Tk7w7h+jzE/Z4MY5yt6X7UwBqrqq8CzJ5U3Abvb+m7m/sGNldOMa+xV1bGqerStfxc4wNyT7WN9zs4wrp8I4xz0Pb9qoYAvJdnfniDuzRVVdQzm/gECl4+4P8P0riSPt6mdsZreOFmSSeAq4BE6OmcnjQs6OmenM85BP9CrFsbUdVV1NXNvAL2jTRPo/Hc38AvAlcAx4K9H252zl+RVwGeB91TVd0bdn2FZZFzdnLMzGeegH+hVC+Ooqo625Qng88xNU/XkeJsznZ87PTHi/gxFVR2vqh9U1Q+Bv2VMz1uSVzAXhp+oqs+18tifs8XG1cs5W8o4B32Xr1pIcnH7zyKSXAy8DXjyzJ8aO3uALW19C3D/CPsyNPNB2PwuY3jekgTYBRyoqg8v2DXW5+x04+rhnA1ibO+6AWi3Qn2EH71q4S9G3KUVS/J65q7iYe7J5U+O87iSfAp4C3NvCTwOfAD4R+Be4OeAZ4DNVTVW/7F5mnG9hbkpgAIOA384P689LpL8OvDPwBPAD1v5/czNZ4/tOTvDuG5lzM/ZIMY66CVJSxvnqRtJ0gAMeknqnEEvSZ0z6CWpcwa9JHXOoJekzhn0ktQ5g16SOvf/KKw939PHYzcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "arr = []\n",
    "for sentence in train_set.prep_data:\n",
    "    arr.append(len(sentence))\n",
    "plt.hist(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "class SentimentLSTM(nn.Module):\n",
    "\n",
    "    def __init__(self, vocab_size, output_size, embedding_dim, hidden_dim, n_layers, seq_len, drop_prob = 0):\n",
    "        super().__init__()\n",
    "\n",
    "        self.output_size = output_size\n",
    "        self.n_layers = n_layers\n",
    "        self.hidden_dim = hidden_dim\n",
    "        \n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, \n",
    "                            dropout=drop_prob, batch_first=True, bidirectional = True)\n",
    "        self.seq_len = seq_len\n",
    "        self.fc = nn.Linear(hidden_dim * self.seq_len * 2, output_size)\n",
    "        \n",
    "\n",
    "    def forward(self, x, hidden):\n",
    "\n",
    "        batch_size = x.size(0)\n",
    "\n",
    "        embeds = self.embedding(x.long())\n",
    "        lstm_out, hidden = self.lstm(embeds, hidden)\n",
    "    \n",
    "        lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim * self.seq_len * 2)\n",
    "        \n",
    "        out = self.fc(lstm_out)\n",
    "        return out, hidden\n",
    "    \n",
    "    \n",
    "    def init_hidden(self, batch_size):\n",
    "        ''' Initializes hidden state '''\n",
    "        # Create two new tensors with sizes n_layers x batch_size x hidden_dim,\n",
    "        # initialized to zero, for hidden state and cell state of LSTM\n",
    "        weight = next(self.parameters()).data\n",
    "        \n",
    "        if (gpu_avaliable):\n",
    "            hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda(),\n",
    "                  weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda())\n",
    "        else:\n",
    "            hidden = (weight.new(self.n_layers * 2, batch_size, self.hidden_dim).zero_(),\n",
    "                      weight.new(self.n_layers * 2, batch_size, self.hidden_dim).zero_())\n",
    "        \n",
    "        return hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_len = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epochs, train_loader, val_loader):\n",
    "    vocab_size = len(train_set.vocab) + 1\n",
    "    output_size = 3\n",
    "    embedding_dim = 128\n",
    "    hidden_dim = 64\n",
    "    n_layers = 2\n",
    "    batch_size = 16\n",
    "    seq_len = 15\n",
    "\n",
    "    net = SentimentLSTM(vocab_size, output_size, embedding_dim, hidden_dim, n_layers,seq_len)\n",
    "    if gpu_avaliable:\n",
    "        net.cuda()\n",
    "\n",
    "    lr=0.001\n",
    "\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(net.parameters(), lr=lr)\n",
    "\n",
    "    counter = 0\n",
    "    print_every = 100\n",
    "    clip=5 # gradient clipping\n",
    "\n",
    "    net.train()\n",
    "    # train for some number of epochs\n",
    "    for e in range(epochs):\n",
    "        # initialize hidden state\n",
    "        h = net.init_hidden(batch_size)\n",
    "\n",
    "        # batch loop\n",
    "        for inputs, labels in train_loader:\n",
    "            counter += 1\n",
    "\n",
    "        # Creating new variables for the hidden state, otherwise\n",
    "        # we'd backprop through the entire training history\n",
    "            h = tuple([each.data for each in h])\n",
    "\n",
    "            net.zero_grad()\n",
    "\n",
    "            inputs = inputs.float()\n",
    "            if gpu_avaliable:\n",
    "                inputs = inputs.cuda()\n",
    "                labels = labels.cuda()\n",
    "            output, h = net(inputs, h)\n",
    "\n",
    "\n",
    "            loss = criterion(output, labels.long().squeeze())\n",
    "            loss.backward()\n",
    "        # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.\n",
    "            nn.utils.clip_grad_norm_(net.parameters(), clip)\n",
    "            optimizer.step()\n",
    "\n",
    "            if counter % print_every == 0:\n",
    "                \n",
    "                val_h = net.init_hidden(batch_size)\n",
    "                val_losses = []\n",
    "                net.eval()\n",
    "                correct = 0\n",
    "                y_pred = []\n",
    "                y_test = []\n",
    "                for inputs, labels in val_loader:\n",
    "\n",
    "                # Creating new variables for the hidden state, otherwise\n",
    "                # we'd backprop through the entire training history\n",
    "                    val_h = tuple([each.data for each in val_h])\n",
    "                    \n",
    "                    \n",
    "                    inputs = inputs.float()\n",
    "                    if gpu_avaliable:\n",
    "                        inputs = inputs.cuda()\n",
    "                        labels = labels.cuda()\n",
    "                    \n",
    "                    output, val_h = net(inputs, val_h)\n",
    "                    val_loss = criterion(output, labels.long().squeeze())\n",
    "                    _, ans = torch.max(output, dim = 1)\n",
    "                    correct += torch.sum(torch.eq(ans.data, labels.long().squeeze().data)).item()\n",
    "                    y_pred.append(ans.data)\n",
    "                    y_test.append(labels)\n",
    "                    val_losses.append(val_loss.item())\n",
    "                y_pred = torch.stack(y_pred).view(-1).numpy()\n",
    "                y_test = torch.stack(y_test).view(-1).numpy()\n",
    "                f1 = f1_score(y_test, y_pred, average = 'macro')\n",
    "                acc = correct /(len(val_loader) * val_loader.batch_size)\n",
    "                    \n",
    "                    \n",
    "\n",
    "\n",
    "\n",
    "                net.train()\n",
    "                print(\"Epoch: {}/{}...\".format(e+1, epochs),\n",
    "                      \"Step: {}...\".format(counter),\n",
    "                      \"Loss: {:.6f}...\".format(loss.item()),\n",
    "                \"Val Loss: {:.6f}\".format(np.mean(val_losses)))\n",
    "    return f1, acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1/7... Step: 100... Loss: 0.647496... Val Loss: 0.955356\n",
      "Epoch: 2/7... Step: 200... Loss: 0.407285... Val Loss: 0.633546\n",
      "Epoch: 2/7... Step: 300... Loss: 0.343375... Val Loss: 0.571400\n",
      "Epoch: 3/7... Step: 400... Loss: 0.323662... Val Loss: 0.419238\n",
      "Epoch: 3/7... Step: 500... Loss: 0.291844... Val Loss: 0.225609\n",
      "Epoch: 4/7... Step: 600... Loss: 0.151361... Val Loss: 0.136355\n",
      "Epoch: 5/7... Step: 700... Loss: 0.146166... Val Loss: 0.087538\n",
      "Epoch: 5/7... Step: 800... Loss: 0.064135... Val Loss: 0.048075\n",
      "Epoch: 6/7... Step: 900... Loss: 0.008059... Val Loss: 0.020815\n",
      "Epoch: 6/7... Step: 1000... Loss: 0.000614... Val Loss: 0.025089\n",
      "Epoch: 7/7... Step: 1100... Loss: 0.000824... Val Loss: 0.044117\n",
      "Epoch: 1/7... Step: 100... Loss: 0.511604... Val Loss: 1.063780\n",
      "Epoch: 2/7... Step: 200... Loss: 0.637589... Val Loss: 0.871774\n",
      "Epoch: 2/7... Step: 300... Loss: 0.587542... Val Loss: 0.545168\n",
      "Epoch: 3/7... Step: 400... Loss: 0.440436... Val Loss: 0.449877\n",
      "Epoch: 3/7... Step: 500... Loss: 0.226574... Val Loss: 0.250004\n",
      "Epoch: 4/7... Step: 600... Loss: 0.487570... Val Loss: 0.143084\n",
      "Epoch: 5/7... Step: 700... Loss: 0.196488... Val Loss: 0.101650\n",
      "Epoch: 5/7... Step: 800... Loss: 0.009249... Val Loss: 0.026154\n",
      "Epoch: 6/7... Step: 900... Loss: 0.022558... Val Loss: 0.034112\n",
      "Epoch: 6/7... Step: 1000... Loss: 0.006362... Val Loss: 0.025108\n",
      "Epoch: 7/7... Step: 1100... Loss: 0.028122... Val Loss: 0.005588\n",
      "Epoch: 1/7... Step: 100... Loss: 0.884226... Val Loss: 1.022384\n",
      "Epoch: 2/7... Step: 200... Loss: 0.518340... Val Loss: 0.720469\n",
      "Epoch: 2/7... Step: 300... Loss: 0.454941... Val Loss: 0.621711\n",
      "Epoch: 3/7... Step: 400... Loss: 0.276739... Val Loss: 0.331259\n",
      "Epoch: 3/7... Step: 500... Loss: 0.178237... Val Loss: 0.243225\n",
      "Epoch: 4/7... Step: 600... Loss: 0.231290... Val Loss: 0.138364\n",
      "Epoch: 5/7... Step: 700... Loss: 0.049398... Val Loss: 0.053891\n",
      "Epoch: 5/7... Step: 800... Loss: 0.250975... Val Loss: 0.030118\n",
      "Epoch: 6/7... Step: 900... Loss: 0.034929... Val Loss: 0.020728\n",
      "Epoch: 6/7... Step: 1000... Loss: 0.014930... Val Loss: 0.030022\n",
      "Epoch: 7/7... Step: 1100... Loss: 0.013745... Val Loss: 0.042523\n",
      "Epoch: 1/7... Step: 100... Loss: 0.899100... Val Loss: 1.109522\n",
      "Epoch: 2/7... Step: 200... Loss: 0.568757... Val Loss: 0.822212\n",
      "Epoch: 2/7... Step: 300... Loss: 0.592738... Val Loss: 0.670664\n",
      "Epoch: 3/7... Step: 400... Loss: 0.537778... Val Loss: 0.377548\n",
      "Epoch: 3/7... Step: 500... Loss: 0.394242... Val Loss: 0.285590\n",
      "Epoch: 4/7... Step: 600... Loss: 0.128904... Val Loss: 0.138353\n",
      "Epoch: 5/7... Step: 700... Loss: 0.115236... Val Loss: 0.117092\n",
      "Epoch: 5/7... Step: 800... Loss: 0.048735... Val Loss: 0.064570\n",
      "Epoch: 6/7... Step: 900... Loss: 0.012992... Val Loss: 0.049659\n",
      "Epoch: 6/7... Step: 1000... Loss: 0.015600... Val Loss: 0.027865\n",
      "Epoch: 7/7... Step: 1100... Loss: 0.023272... Val Loss: 0.016532\n",
      "Epoch: 1/7... Step: 100... Loss: 0.851884... Val Loss: 0.983771\n",
      "Epoch: 2/7... Step: 200... Loss: 0.592073... Val Loss: 0.758277\n",
      "Epoch: 2/7... Step: 300... Loss: 0.747572... Val Loss: 0.539109\n",
      "Epoch: 3/7... Step: 400... Loss: 0.352914... Val Loss: 0.383307\n",
      "Epoch: 3/7... Step: 500... Loss: 0.239383... Val Loss: 0.322183\n",
      "Epoch: 4/7... Step: 600... Loss: 0.207884... Val Loss: 0.153422\n",
      "Epoch: 5/7... Step: 700... Loss: 0.095848... Val Loss: 0.095516\n",
      "Epoch: 5/7... Step: 800... Loss: 0.060500... Val Loss: 0.054619\n",
      "Epoch: 6/7... Step: 900... Loss: 0.572122... Val Loss: 0.025677\n",
      "Epoch: 6/7... Step: 1000... Loss: 0.023544... Val Loss: 0.037922\n",
      "Epoch: 7/7... Step: 1100... Loss: 0.020284... Val Loss: 0.013141\n",
      "Epoch: 1/7... Step: 100... Loss: 0.811914... Val Loss: 1.010018\n",
      "Epoch: 2/7... Step: 200... Loss: 0.560769... Val Loss: 0.741428\n",
      "Epoch: 2/7... Step: 300... Loss: 0.615759... Val Loss: 0.568622\n",
      "Epoch: 3/7... Step: 400... Loss: 0.169290... Val Loss: 0.351110\n",
      "Epoch: 3/7... Step: 500... Loss: 1.201058... Val Loss: 0.281994\n",
      "Epoch: 4/7... Step: 600... Loss: 0.163824... Val Loss: 0.127368\n",
      "Epoch: 5/7... Step: 700... Loss: 0.196147... Val Loss: 0.076112\n",
      "Epoch: 5/7... Step: 800... Loss: 0.037948... Val Loss: 0.042112\n",
      "Epoch: 6/7... Step: 900... Loss: 0.034093... Val Loss: 0.029928\n",
      "Epoch: 6/7... Step: 1000... Loss: 0.003020... Val Loss: 0.013547\n",
      "Epoch: 7/7... Step: 1100... Loss: 0.000751... Val Loss: 0.041435\n",
      "Epoch: 1/7... Step: 100... Loss: 0.826254... Val Loss: 1.225570\n",
      "Epoch: 2/7... Step: 200... Loss: 0.282106... Val Loss: 0.633013\n",
      "Epoch: 2/7... Step: 300... Loss: 0.526553... Val Loss: 0.609123\n",
      "Epoch: 3/7... Step: 400... Loss: 0.716411... Val Loss: 0.391387\n",
      "Epoch: 3/7... Step: 500... Loss: 0.206767... Val Loss: 0.230808\n",
      "Epoch: 4/7... Step: 600... Loss: 0.195873... Val Loss: 0.111202\n",
      "Epoch: 5/7... Step: 700... Loss: 0.014290... Val Loss: 0.111846\n",
      "Epoch: 5/7... Step: 800... Loss: 0.578244... Val Loss: 0.055462\n",
      "Epoch: 6/7... Step: 900... Loss: 0.001644... Val Loss: 0.042333\n",
      "Epoch: 6/7... Step: 1000... Loss: 0.007662... Val Loss: 0.018535\n",
      "Epoch: 7/7... Step: 1100... Loss: 0.001534... Val Loss: 0.024192\n",
      "Epoch: 1/7... Step: 100... Loss: 0.821046... Val Loss: 0.948754\n",
      "Epoch: 2/7... Step: 200... Loss: 0.386877... Val Loss: 0.786029\n",
      "Epoch: 2/7... Step: 300... Loss: 0.398970... Val Loss: 0.524389\n",
      "Epoch: 3/7... Step: 400... Loss: 0.303028... Val Loss: 0.308212\n",
      "Epoch: 3/7... Step: 500... Loss: 0.487272... Val Loss: 0.230518\n",
      "Epoch: 4/7... Step: 600... Loss: 0.093815... Val Loss: 0.146962\n",
      "Epoch: 5/7... Step: 700... Loss: 0.032293... Val Loss: 0.050786\n",
      "Epoch: 5/7... Step: 800... Loss: 0.335122... Val Loss: 0.058295\n",
      "Epoch: 6/7... Step: 900... Loss: 0.018474... Val Loss: 0.025832\n",
      "Epoch: 6/7... Step: 1000... Loss: 0.004278... Val Loss: 0.010100\n",
      "Epoch: 7/7... Step: 1100... Loss: 0.024094... Val Loss: 0.080685\n",
      "f1 score is: 0.9910791526513272\n",
      "accuracy is: 0.9925130208333333\n"
     ]
    }
   ],
   "source": [
    "kf = KFold(n_splits=8)\n",
    "f1_cross = []\n",
    "accuracy = []\n",
    "for train_idx, val_idx in kf.split(train_set):\n",
    "    train_sampler = RandomSampler(train_idx)\n",
    "    val_sampler = RandomSampler(val_idx)\n",
    "    train_loader = DataLoader(train_set, batch_size=16, sampler = train_sampler, num_workers=2, drop_last=True)\n",
    "    val_loader = DataLoader(train_set, batch_size=16, sampler = val_sampler, num_workers=2, drop_last=True)\n",
    "    f1, acc = train(7, train_loader, val_loader)\n",
    "    f1_cross.append(f1)\n",
    "    accuracy.append(acc)\n",
    "print('f1 score is:',np.array(f1_cross).mean())\n",
    "print('accuracy is:', np.array(accuracy).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.9947916666666666, 1.0, 0.9869791666666666, 0.9947916666666666, 1.0, 0.9869791666666666, 0.9921875, 0.984375]\n"
     ]
    }
   ],
   "source": [
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}