TopicModeling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /home/vignesh/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "from gensim import matutils, models\n",
    "import scipy.sparse\n",
    "from nltk import word_tokenize, pos_tag\n",
    "\n",
    "import nltk\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "\n",
    "from sklearn.feature_extraction import text\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aaaaah</th>\n",
       "      <th>aaaaahhhhhhh</th>\n",
       "      <th>aaaaauuugghhhhhh</th>\n",
       "      <th>aaaahhhhh</th>\n",
       "      <th>aaah</th>\n",
       "      <th>aah</th>\n",
       "      <th>abc</th>\n",
       "      <th>abcs</th>\n",
       "      <th>ability</th>\n",
       "      <th>abject</th>\n",
       "      <th>...</th>\n",
       "      <th>zealand</th>\n",
       "      <th>zee</th>\n",
       "      <th>zen</th>\n",
       "      <th>zeppelin</th>\n",
       "      <th>zero</th>\n",
       "      <th>zillion</th>\n",
       "      <th>zombie</th>\n",
       "      <th>zombies</th>\n",
       "      <th>zoning</th>\n",
       "      <th>zoo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>louis</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>dave</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ricky</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bo</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bill</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 7583 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       aaaaah  aaaaahhhhhhh  aaaaauuugghhhhhh  aaaahhhhh  aaah  aah  abc  \\\n",
       "louis       0             0                 0          0     0    3    0   \n",
       "dave        0             0                 0          0     1    0    0   \n",
       "ricky       0             0                 0          0     0    0    0   \n",
       "bo          0             1                 1          1     0    0    0   \n",
       "bill        1             0                 0          0     0    0    0   \n",
       "\n",
       "       abcs  ability  abject  ...  zealand  zee  zen  zeppelin  zero  zillion  \\\n",
       "louis     0        0       0  ...        0    0    0         0     2        0   \n",
       "dave      0        0       0  ...        0    0    0         0     0        0   \n",
       "ricky     0        1       1  ...        0    0    0         0     0        0   \n",
       "bo        0        1       0  ...        0    0    0         0     1        0   \n",
       "bill      1        0       0  ...        0    0    0         0     1        1   \n",
       "\n",
       "       zombie  zombies  zoning  zoo  \n",
       "louis       0        0       0    0  \n",
       "dave        0        0       0    0  \n",
       "ricky       0        0       0    1  \n",
       "bo          0        0       0    0  \n",
       "bill        1        1       1    0  \n",
       "\n",
       "[5 rows x 7583 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_pickle('dtm.pkl')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>louis</th>\n",
       "      <th>dave</th>\n",
       "      <th>ricky</th>\n",
       "      <th>bo</th>\n",
       "      <th>bill</th>\n",
       "      <th>jim</th>\n",
       "      <th>john</th>\n",
       "      <th>hasan</th>\n",
       "      <th>ali</th>\n",
       "      <th>anthony</th>\n",
       "      <th>mike</th>\n",
       "      <th>joe</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>aaaaah</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>aaaaahhhhhhh</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>aaaaauuugghhhhhh</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>aaaahhhhh</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>aaah</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  louis  dave  ricky  bo  bill  jim  john  hasan  ali  \\\n",
       "aaaaah                0     0      0   0     1    0     0      0    0   \n",
       "aaaaahhhhhhh          0     0      0   1     0    0     0      0    0   \n",
       "aaaaauuugghhhhhh      0     0      0   1     0    0     0      0    0   \n",
       "aaaahhhhh             0     0      0   1     0    0     0      0    0   \n",
       "aaah                  0     1      0   0     0    0     0      0    0   \n",
       "\n",
       "                  anthony  mike  joe  \n",
       "aaaaah                  0     0    0  \n",
       "aaaaahhhhhhh            0     0    0  \n",
       "aaaaauuugghhhhhh        0     0    0  \n",
       "aaaahhhhh               0     0    0  \n",
       "aaah                    0     0    0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tdm = data.transpose()\n",
    "tdm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sparse_counts = scipy.sparse.csr_matrix(tdm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<7583x12 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 17668 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = matutils.Sparse2Corpus(sparse_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<gensim.matutils.Sparse2Corpus at 0x7fb9697b38d0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = pickle.load(open(\"cv.pkl\", \"rb\"))\n",
    "id2word = dict((v, k) for k, v in cv.vocabulary_.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# topic modeling - with all text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.009*\"fucking\" + 0.006*\"fuck\" + 0.006*\"yeah\" + 0.005*\"shit\" + 0.005*\"voice\" + 0.005*\"thing\" + 0.005*\"guy\" + 0.005*\"say\" + 0.005*\"the\" + 0.004*\"going\"'),\n",
       " (1,\n",
       "  '0.005*\"she\" + 0.005*\"time\" + 0.005*\"the\" + 0.005*\"no\" + 0.004*\"oh\" + 0.004*\"shit\" + 0.004*\"dad\" + 0.004*\"say\" + 0.004*\"well\" + 0.004*\"going\"')]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = models.LdaModel(corpus=corpus, id2word=id2word, num_topics=2, passes=10)\n",
    "lda.print_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.007*\"voice\" + 0.006*\"she\" + 0.005*\"no\" + 0.005*\"say\" + 0.005*\"time\" + 0.005*\"dad\" + 0.005*\"love\" + 0.004*\"the\" + 0.004*\"well\" + 0.004*\"want\"'),\n",
       " (1,\n",
       "  '0.010*\"fucking\" + 0.007*\"shit\" + 0.007*\"yeah\" + 0.007*\"fuck\" + 0.005*\"say\" + 0.005*\"the\" + 0.005*\"no\" + 0.004*\"she\" + 0.004*\"going\" + 0.004*\"little\"'),\n",
       " (2,\n",
       "  '0.008*\"fucking\" + 0.006*\"shit\" + 0.006*\"fuck\" + 0.006*\"the\" + 0.005*\"went\" + 0.005*\"good\" + 0.005*\"thing\" + 0.005*\"time\" + 0.004*\"man\" + 0.004*\"cause\"')]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = models.LdaModel(corpus=corpus, id2word=id2word, num_topics=3, passes=10)\n",
    "lda.print_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# topic modelling - with noun only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nouns(text):\n",
    "    \"\"\" given a string of text, tokenize the text and pull out only the nouns.\"\"\"\n",
    "    is_noun = lambda pos: pos[:2] == 'NN'\n",
    "    tokenized = word_tokenize(text)\n",
    "    all_nouns = [word for (word, pos) in pos_tag(tokenized) if is_noun(pos)]\n",
    "    return ' '.join(all_nouns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>transcript</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>louis</td>\n",
       "      <td>intro fade music  let  roll  hold  lights  do ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>dave</td>\n",
       "      <td>this dave  he tells dirty jokes living  that s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ricky</td>\n",
       "      <td>hello  hello  how  great  thank  wow  calm  sh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bo</td>\n",
       "      <td>bo what  old macdonald farm e i e i o and farm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bill</td>\n",
       "      <td>cheers applause  all right  thank  thank much...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              transcript\n",
       "louis  intro fade music  let  roll  hold  lights  do ...\n",
       "dave   this dave  he tells dirty jokes living  that s...\n",
       "ricky  hello  hello  how  great  thank  wow  calm  sh...\n",
       "bo     bo what  old macdonald farm e i e i o and farm...\n",
       "bill    cheers applause  all right  thank  thank much..."
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_clean = pd.read_pickle('cleaned_corpus.pkl')\n",
    "data_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>transcript</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>louis</td>\n",
       "      <td>intro fade music let roll lights lights i i pl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>dave</td>\n",
       "      <td>dave jokes work train alchemist fire transform...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ricky</td>\n",
       "      <td>hello wow calm thank i gon tonight money guy r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bo</td>\n",
       "      <td>macdonald farm e i o farm pig e i i macdonald ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bill</td>\n",
       "      <td>cheers thank thank thank pleasure georgia area...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              transcript\n",
       "louis  intro fade music let roll lights lights i i pl...\n",
       "dave   dave jokes work train alchemist fire transform...\n",
       "ricky  hello wow calm thank i gon tonight money guy r...\n",
       "bo     macdonald farm e i o farm pig e i i macdonald ...\n",
       "bill   cheers thank thank thank pleasure georgia area..."
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_nouns = pd.DataFrame(data_clean.transcript.apply(nouns))\n",
    "data_nouns.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "add_stop_words = ['like', 'im', 'know', 'just', 'dont', 'thats', 'right', 'people',\n",
    "                  'youre', 'got', 'gonna', 'time', 'think', 'yeah', 'said']\n",
    "stop_words = text.ENGLISH_STOP_WORDS.union(add_stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aaaaah</th>\n",
       "      <th>aaaahhhhh</th>\n",
       "      <th>aah</th>\n",
       "      <th>abc</th>\n",
       "      <th>ability</th>\n",
       "      <th>abortion</th>\n",
       "      <th>abortions</th>\n",
       "      <th>abuse</th>\n",
       "      <th>accent</th>\n",
       "      <th>accents</th>\n",
       "      <th>...</th>\n",
       "      <th>youth</th>\n",
       "      <th>youtube</th>\n",
       "      <th>yulin</th>\n",
       "      <th>zealand</th>\n",
       "      <th>zee</th>\n",
       "      <th>zeppelin</th>\n",
       "      <th>zero</th>\n",
       "      <th>zillion</th>\n",
       "      <th>zombie</th>\n",
       "      <th>zombies</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>louis</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>dave</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ricky</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bo</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bill</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 4280 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       aaaaah  aaaahhhhh  aah  abc  ability  abortion  abortions  abuse  \\\n",
       "louis       0          0    2    0        0         0          0      0   \n",
       "dave        0          0    0    0        0         0          1      0   \n",
       "ricky       0          0    0    0        1         0          0      0   \n",
       "bo          0          1    0    0        1         0          0      0   \n",
       "bill        1          0    0    0        0         0          0      0   \n",
       "\n",
       "       accent  accents  ...  youth  youtube  yulin  zealand  zee  zeppelin  \\\n",
       "louis       0        0  ...      1        1      0        0    0         0   \n",
       "dave        0        0  ...      0        0      0        0    0         0   \n",
       "ricky       0        0  ...      0        0      1        0    0         0   \n",
       "bo          0        0  ...      0        0      0        0    0         0   \n",
       "bill        0        0  ...      0        0      0        0    0         0   \n",
       "\n",
       "       zero  zillion  zombie  zombies  \n",
       "louis     0        0       0        0  \n",
       "dave      0        0       0        0  \n",
       "ricky     0        0       0        0  \n",
       "bo        1        0       0        0  \n",
       "bill      0        1       1        1  \n",
       "\n",
       "[5 rows x 4280 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cvn = CountVectorizer(stop_words = add_stop_words)\n",
    "data_cvn = cvn.fit_transform(data_nouns.transcript)\n",
    "data_dtmn = pd.DataFrame(data_cvn.toarray(), columns=cvn.get_feature_names())\n",
    "data_dtmn.index = data_nouns.index\n",
    "data_dtmn.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating the gensim corpus\n",
    "corpusn = matutils.Sparse2Corpus(scipy.sparse.csr_matrix(data_dtmn.transpose()))\n",
    "id2wordn = dict((v, k) for k, v in cvn.vocabulary_.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.028*\"voice\" + 0.009*\"laughter\" + 0.007*\"dad\" + 0.006*\"life\" + 0.006*\"bo\" + 0.006*\"guy\" + 0.006*\"man\" + 0.005*\"repeat\" + 0.005*\"stuff\" + 0.004*\"show\"'),\n",
       " (1,\n",
       "  '0.011*\"thing\" + 0.010*\"day\" + 0.009*\"man\" + 0.008*\"cause\" + 0.008*\"life\" + 0.007*\"way\" + 0.007*\"guy\" + 0.007*\"shit\" + 0.006*\"gon\" + 0.006*\"women\"')]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ldan = models.LdaModel(corpus=corpusn, id2word=id2wordn, num_topics=2, passes=10)\n",
    "ldan.print_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.013*\"day\" + 0.010*\"thing\" + 0.009*\"man\" + 0.009*\"cause\" + 0.007*\"house\" + 0.007*\"women\" + 0.007*\"way\" + 0.006*\"life\" + 0.006*\"things\" + 0.006*\"lot\"'),\n",
       " (1,\n",
       "  '0.016*\"voice\" + 0.009*\"guy\" + 0.008*\"man\" + 0.008*\"thing\" + 0.007*\"way\" + 0.007*\"day\" + 0.006*\"shit\" + 0.006*\"years\" + 0.006*\"gon\" + 0.006*\"something\"'),\n",
       " (2,\n",
       "  '0.014*\"life\" + 0.008*\"dad\" + 0.008*\"thing\" + 0.007*\"laughter\" + 0.006*\"kids\" + 0.006*\"way\" + 0.005*\"mom\" + 0.005*\"man\" + 0.005*\"school\" + 0.005*\"parents\"')]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ldan = models.LdaModel(corpus=corpusn, id2word=id2wordn, num_topics=3, passes=10)\n",
    "ldan.print_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.035*\"voice\" + 0.012*\"laughter\" + 0.008*\"dad\" + 0.008*\"life\" + 0.007*\"bo\" + 0.007*\"guy\" + 0.007*\"man\" + 0.006*\"repeat\" + 0.006*\"stuff\" + 0.005*\"show\"'),\n",
       " (1,\n",
       "  '0.001*\"man\" + 0.000*\"voice\" + 0.000*\"thing\" + 0.000*\"day\" + 0.000*\"life\" + 0.000*\"cause\" + 0.000*\"way\" + 0.000*\"things\" + 0.000*\"something\" + 0.000*\"years\"'),\n",
       " (2,\n",
       "  '0.012*\"thing\" + 0.009*\"life\" + 0.009*\"day\" + 0.009*\"cause\" + 0.008*\"way\" + 0.008*\"guy\" + 0.007*\"years\" + 0.007*\"gon\" + 0.006*\"man\" + 0.006*\"kind\"'),\n",
       " (3,\n",
       "  '0.012*\"man\" + 0.011*\"day\" + 0.009*\"thing\" + 0.009*\"women\" + 0.009*\"shit\" + 0.008*\"cause\" + 0.007*\"lot\" + 0.006*\"fuck\" + 0.006*\"things\" + 0.006*\"way\"')]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ldan = models.LdaModel(corpus=corpusn, id2word=id2wordn, num_topics=4, passes=10)\n",
    "ldan.print_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# topic modeling - with nouns and adjectives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nouns_adj(text):\n",
    "    \"\"\"given a string of text, tokenize the text and pull out only the nouns and adjectives\"\"\"\n",
    "    is_noun_adj = lambda pos: pos[:2] == 'NN' or pos[:2] == 'JJ'\n",
    "    tokenized = word_tokenize(text)\n",
    "    nouns_adj = [word for (word, pos) in pos_tag(tokenized) if is_noun_adj(pos)]\n",
    "    return ' '.join(nouns_adj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>transcript</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>louis</td>\n",
       "      <td>intro fade music let roll lights lights much i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>dave</td>\n",
       "      <td>dave dirty jokes stare hard work train thought...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ricky</td>\n",
       "      <td>hello great thank wow calm fuck thank welcome ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bo</td>\n",
       "      <td>old macdonald farm e i i o farm pig e i i old ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bill</td>\n",
       "      <td>cheers right thank much thank thank pleasure g...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              transcript\n",
       "louis  intro fade music let roll lights lights much i...\n",
       "dave   dave dirty jokes stare hard work train thought...\n",
       "ricky  hello great thank wow calm fuck thank welcome ...\n",
       "bo     old macdonald farm e i i o farm pig e i i old ...\n",
       "bill   cheers right thank much thank thank pleasure g..."
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_nouns_adj = pd.DataFrame(data_clean.transcript.apply(nouns_adj))\n",
    "data_nouns_adj.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aaaaah</th>\n",
       "      <th>aaaahhhhh</th>\n",
       "      <th>aah</th>\n",
       "      <th>abc</th>\n",
       "      <th>ability</th>\n",
       "      <th>abject</th>\n",
       "      <th>able</th>\n",
       "      <th>ablebodied</th>\n",
       "      <th>abortion</th>\n",
       "      <th>abortions</th>\n",
       "      <th>...</th>\n",
       "      <th>yummy</th>\n",
       "      <th>ze</th>\n",
       "      <th>zealand</th>\n",
       "      <th>zee</th>\n",
       "      <th>zen</th>\n",
       "      <th>zeppelin</th>\n",
       "      <th>zero</th>\n",
       "      <th>zillion</th>\n",
       "      <th>zombie</th>\n",
       "      <th>zombies</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>louis</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>dave</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ricky</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bo</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bill</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5372 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       aaaaah  aaaahhhhh  aah  abc  ability  abject  able  ablebodied  \\\n",
       "louis       0          0    2    0        0       0     1           0   \n",
       "dave        0          0    0    0        0       0     0           0   \n",
       "ricky       0          0    0    0        1       1     2           0   \n",
       "bo          0          1    0    0        1       0     0           0   \n",
       "bill        1          0    0    0        0       0     1           0   \n",
       "\n",
       "       abortion  abortions  ...  yummy  ze  zealand  zee  zen  zeppelin  zero  \\\n",
       "louis         0          0  ...      0   0        0    0    0         0     0   \n",
       "dave          0          1  ...      0   0        0    0    0         0     0   \n",
       "ricky         0          0  ...      0   0        0    0    0         0     0   \n",
       "bo            0          0  ...      0   0        0    0    0         0     1   \n",
       "bill          0          0  ...      1   1        0    0    0         0     0   \n",
       "\n",
       "       zillion  zombie  zombies  \n",
       "louis        0       0        0  \n",
       "dave         0       0        0  \n",
       "ricky        0       0        0  \n",
       "bo           0       0        0  \n",
       "bill         1       1        1  \n",
       "\n",
       "[5 rows x 5372 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cvna = CountVectorizer(stop_words=stop_words, max_df=.8)\n",
    "data_cvna = cvna.fit_transform(data_nouns_adj.transcript)\n",
    "data_dtmna = pd.DataFrame(data_cvna.toarray(), columns=cvna.get_feature_names())\n",
    "data_dtmna.index = data_nouns_adj.index\n",
    "data_dtmna.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpusna = matutils.Sparse2Corpus(scipy.sparse.csr_matrix(data_dtmna.transpose()))\n",
    "\n",
    "# creating the vocabulary dictionary\n",
    "id2wordna = dict((v, k) for k, v in cvna.vocabulary_.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.005*\"dude\" + 0.004*\"mom\" + 0.004*\"wife\" + 0.003*\"joke\" + 0.003*\"ahah\" + 0.003*\"jenny\" + 0.003*\"friend\" + 0.003*\"parents\" + 0.002*\"anthony\" + 0.002*\"son\"'),\n",
       " (1,\n",
       "  '0.014*\"voice\" + 0.005*\"audience\" + 0.004*\"laughter\" + 0.004*\"bro\" + 0.003*\"fucking\" + 0.003*\"joke\" + 0.003*\"bo\" + 0.003*\"robotic\" + 0.003*\"mom\" + 0.003*\"dude\"')]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ldana = models.LdaModel(corpus=corpusna, num_topics=2, id2word=id2wordna, passes=10)\n",
    "ldana.print_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.005*\"ahah\" + 0.004*\"jenny\" + 0.004*\"laughter\" + 0.004*\"parents\" + 0.003*\"hasan\" + 0.003*\"mom\" + 0.003*\"door\" + 0.002*\"minutes\" + 0.002*\"comedy\" + 0.002*\"friend\"'),\n",
       " (1,\n",
       "  '0.013*\"voice\" + 0.006*\"dude\" + 0.005*\"joke\" + 0.005*\"mom\" + 0.004*\"bro\" + 0.003*\"bo\" + 0.003*\"robotic\" + 0.003*\"fucking\" + 0.003*\"wife\" + 0.003*\"jokes\"'),\n",
       " (2,\n",
       "  '0.008*\"audience\" + 0.007*\"guns\" + 0.005*\"ass\" + 0.005*\"girlfriend\" + 0.005*\"gun\" + 0.004*\"accent\" + 0.004*\"cunt\" + 0.004*\"class\" + 0.004*\"fucking\" + 0.004*\"son\"')]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ldana = models.LdaModel(corpus=corpusna, num_topics=3, id2word=id2wordna, passes=10)\n",
    "ldana.print_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.000*\"voice\" + 0.000*\"dude\" + 0.000*\"na\" + 0.000*\"mom\" + 0.000*\"audience\" + 0.000*\"ass\" + 0.000*\"dead\" + 0.000*\"parents\" + 0.000*\"dog\" + 0.000*\"date\"'),\n",
       " (1,\n",
       "  '0.007*\"ahah\" + 0.007*\"joke\" + 0.005*\"son\" + 0.005*\"audience\" + 0.005*\"anthony\" + 0.005*\"guns\" + 0.004*\"american\" + 0.004*\"gun\" + 0.004*\"fucking\" + 0.004*\"party\"'),\n",
       " (2,\n",
       "  '0.015*\"voice\" + 0.005*\"mom\" + 0.005*\"laughter\" + 0.004*\"bro\" + 0.004*\"parents\" + 0.003*\"dude\" + 0.003*\"audience\" + 0.003*\"bo\" + 0.003*\"robotic\" + 0.003*\"joke\"'),\n",
       " (3,\n",
       "  '0.006*\"jenny\" + 0.006*\"dude\" + 0.004*\"morning\" + 0.003*\"gun\" + 0.003*\"jesus\" + 0.003*\"dog\" + 0.003*\"idea\" + 0.003*\"date\" + 0.003*\"parents\" + 0.003*\"andy\"')]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ldana = models.LdaModel(corpus=corpusna, num_topics=4, id2word=id2wordna, passes=10)\n",
    "ldana.print_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# out of all the above topic models , the noun and adjective, 4 topic one made the most sense, lets move on with\n",
    "# that model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.006*\"mom\" + 0.006*\"clinton\" + 0.005*\"wife\" + 0.005*\"president\" + 0.005*\"dude\" + 0.004*\"cow\" + 0.004*\"parents\" + 0.004*\"dick\" + 0.003*\"stupid\" + 0.003*\"office\"'),\n",
       " (1,\n",
       "  '0.007*\"mom\" + 0.006*\"laughter\" + 0.006*\"joke\" + 0.005*\"parents\" + 0.005*\"anthony\" + 0.005*\"hasan\" + 0.003*\"date\" + 0.003*\"brown\" + 0.003*\"york\" + 0.003*\"birthday\"'),\n",
       " (2,\n",
       "  '0.006*\"dude\" + 0.006*\"ahah\" + 0.006*\"jenny\" + 0.003*\"gay\" + 0.003*\"wife\" + 0.003*\"son\" + 0.003*\"gun\" + 0.003*\"nigga\" + 0.003*\"morning\" + 0.003*\"friend\"'),\n",
       " (3,\n",
       "  '0.021*\"voice\" + 0.006*\"audience\" + 0.005*\"bro\" + 0.005*\"joke\" + 0.005*\"bo\" + 0.004*\"robotic\" + 0.004*\"um\" + 0.004*\"repeat\" + 0.004*\"ass\" + 0.003*\"eye\"')]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ldana = models.LdaModel(corpus=corpusna, num_topics=4, id2word=id2wordna, passes=70)\n",
    "ldana.print_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we will settle with the above topics and try to interpret them.\n",
    "# topic 0: [conversation comedies]\n",
    "# topic 1: [accident, gun, spirituality]\n",
    "# topic 2: [teenager, profanity]\n",
    "# topic 3: [husband, wife, family]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, 'louis'),\n",
       " (2, 'dave'),\n",
       " (3, 'ricky'),\n",
       " (3, 'bo'),\n",
       " (2, 'bill'),\n",
       " (3, 'jim'),\n",
       " (0, 'john'),\n",
       " (1, 'hasan'),\n",
       " (3, 'ali'),\n",
       " (1, 'anthony'),\n",
       " (2, 'mike'),\n",
       " (0, 'joe')]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lets take a look at which topics each transcript contains\n",
    "corpus_transformed = ldana[corpusna]\n",
    "list(zip([a for [(a, b)] in corpus_transformed], data_dtmna.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from our first pass of LDA, we come to following conclusion\n",
    "# topic 0: [conversation comedies] -> (john, joe)\n",
    "# topic 1: [accident, gun, spirituality] -> (louis, hasan, anthony)\n",
    "# topic 2: [teenager, profanity] -> (dave, bill, mike)\n",
    "# topic 3: [husband, wife, family] -> (ricky, bo, jim, ali)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}