Federalist-Author-Identification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import nltk\n",
    "import glob\n",
    "import os\n",
    "import ntpath\n",
    "import re\n",
    "import pandas as pd\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.cluster import KMeans\n",
    "from scipy.cluster.vq import whiten\n",
    "%matplotlib inline\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import random\n",
    "import string\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "sentence_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "word_tokenizer = nltk.tokenize.RegexpTokenizer(r'\\w+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data\n",
    "papers = r\"/Users/jonlucadecaro/Documents/Other/Federalist-Papers-NLP/papers\"\n",
    "hamilton = sorted(glob.glob(os.path.join(papers, \"hamilton/*\")))\n",
    "madison = sorted(glob.glob(os.path.join(papers, \"madison/*\")))\n",
    "disputed = sorted(glob.glob(os.path.join(papers, \"disputed/*\")))\n",
    "\n",
    "hamilton_papers = []\n",
    "for fn in hamilton:\n",
    "    with open(fn) as f:\n",
    "        hamilton_papers.append(f.read().replace('\\n', ' ').replace('\\r',''))\n",
    "hamilton_papers_all = ' '.join(hamilton_papers)\n",
    "\n",
    "madison_papers = []\n",
    "for fn in madison:\n",
    "    with open(fn) as f:\n",
    "        madison_papers.append(f.read().replace('\\n', ' ').replace('\\r',''))\n",
    "madison_papers_all = ' '.join(madison_papers)\n",
    "\n",
    "disputed_papers = []\n",
    "disputed_papers_file_names = []\n",
    "for fn in disputed:\n",
    "    with open(fn) as f:\n",
    "        disputed_papers.append(f.read().replace('\\n', ' ').replace('\\r',''))\n",
    "        disputed_papers_file_names.append(ntpath.basename(fn))\n",
    "disputed_papers_all = ' '.join(disputed_papers)\n",
    "\n",
    "known_papers_all = hamilton_papers_all + \" \" + madison_papers_all\n",
    "known_papers = hamilton_papers + madison_papers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LexicalFeatures(papers, all_papers):\n",
    "    \"\"\"\n",
    "    Compute feature vectors for word and punctuation features\n",
    "    \"\"\"\n",
    "    num_papers = len(papers)\n",
    "    fvs_lexical = np.zeros((len(papers), 2), np.float64)\n",
    "    fvs_punct = np.zeros((len(papers), 3), np.float64)\n",
    "    for e, single_paper_text in enumerate(papers):\n",
    "        # note: the nltk.word_tokenize includes punctuation\n",
    "        tokens = nltk.word_tokenize(single_paper_text.lower())\n",
    "        words = word_tokenizer.tokenize(single_paper_text.lower())\n",
    "        sentences = sentence_tokenizer.tokenize(single_paper_text)\n",
    "        vocab = set(words)\n",
    "        words_per_sentence = np.array([len(word_tokenizer.tokenize(s)) for s in sentences])\n",
    "\n",
    "        # average number of words per sentence\n",
    "        fvs_lexical[e, 0] = words_per_sentence.mean()\n",
    "        # Lexical diversity\n",
    "        fvs_lexical[e, 1] = len(vocab) / float(len(words))\n",
    "\n",
    "        # Commas per sentence\n",
    "        fvs_punct[e, 0] = tokens.count(';') / float(len(sentences))\n",
    "        fvs_punct[e, 1] = tokens.count('\"') / float(len(sentences))\n",
    "        fvs_punct[e, 2] = tokens.count(',') / float(len(sentences))\n",
    "        \n",
    "\n",
    "    # apply whitening to decorrelate the features\n",
    "    fvs_lexical = whiten(fvs_lexical)\n",
    "    fvs_punct = whiten(fvs_punct)\n",
    "\n",
    "    return fvs_lexical, fvs_punct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SyntacticFeatures(papers, all_papers):\n",
    "    \"\"\"\n",
    "    Extract feature vector for part of speech frequencies\n",
    "    \"\"\"\n",
    "    def token_to_pos(paper):\n",
    "        tokens = nltk.word_tokenize(paper)\n",
    "        return [p[1] for p in nltk.pos_tag(tokens)]\n",
    "\n",
    "    paper_pos = [token_to_pos(paper) for paper in papers]\n",
    "    pos_list = ['NN', 'NNP', 'DT', 'IN', 'JJ', 'NNS']\n",
    "    fvs_syntax = np.array([[paper.count(pos) for pos in pos_list] for paper in paper_pos]).astype(np.float64)\n",
    "\n",
    "    # normalise by dividing each row by number of tokens in the paper\n",
    "    fvs_syntax /= np.c_[np.array([len(paper) for paper in paper_pos])]\n",
    "\n",
    "    return fvs_syntax\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "def PredictAuthors(fvs):\n",
    "    km = KMeans(n_clusters=2, init='k-means++', n_init=100, max_iter=300, verbose=0)\n",
    "    km.fit(fvs)\n",
    "    return km\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jonlucadecaro/.virtualenvs/Federalist-Papers-NLP-3/lib/python3.7/site-packages/scipy/cluster/vq.py:141: RuntimeWarning: Some columns have standard deviation zero. The values of these columns will not change.\n",
      "  RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "known_set = list(LexicalFeatures(known_papers, known_papers_all))\n",
    "known_set.append(SyntacticFeatures(known_papers, known_papers_all))\n",
    "\n",
    "classifications = [PredictAuthors(fvs) for fvs in known_set]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jonlucadecaro/.virtualenvs/Federalist-Papers-NLP-3/lib/python3.7/site-packages/scipy/cluster/vq.py:141: RuntimeWarning: Some columns have standard deviation zero. The values of these columns will not change.\n",
      "  RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "# Get FVS/normalized data to predict from disputed papers\n",
    "disputed_set = list(LexicalFeatures(disputed_papers, disputed_papers_all))\n",
    "disputed_set.append(SyntacticFeatures(disputed_papers, disputed_papers_all))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = list()\n",
    "results.append([classifications[0].predict(disputed_set[0]),\"Lexical Features\"]) # Predict results of Lexical Features\n",
    "results.append([classifications[1].predict(disputed_set[1]),\"Lexical Features - Punctuation\"]) # Predict results of Lexical Features, Punctuation\n",
    "results.append([classifications[2].predict(disputed_set[2]),\"Syntactic Features\"]) # Predict results of their syntactic feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hamilton', 'Hamilton', 'Madison', 'Madison', 'Madison', 'Madison', 'Madison', 'Madison', 'Madison', 'Madison', 'Hamilton', 'Madison']\n",
      "['Hamilton', 'Hamilton', 'Hamilton', 'Hamilton', 'Madison', 'Madison', 'Madison', 'Hamilton', 'Hamilton', 'Hamilton', 'Madison', 'Madison']\n",
      "['Madison', 'Madison', 'Madison', 'Madison', 'Madison', 'Hamilton', 'Madison', 'Madison', 'Madison', 'Madison', 'Madison', 'Madison']\n"
     ]
    }
   ],
   "source": [
    "all_results = []\n",
    "for i in range(len(classifications)):\n",
    "    hamilton = classifications[i].labels_[0] # Extract first\n",
    "    individual_classifier_results = []\n",
    "    for j in range(len(results[i][0])):\n",
    "        if results[i][0][j] == hamilton: # We know for a fact Hamilton wrote the first paper, so use that as an index\n",
    "            individual_classifier_results.append(\"Hamilton\")\n",
    "        else:\n",
    "            individual_classifier_results.append(\"Madison\")\n",
    "    print(individual_classifier_results)\n",
    "    all_results.append(individual_classifier_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ", 49.txt,50.txt,51.txt,52.txt,53.txt,54.txt,55.txt,56.txt,57.txt,58.txt,62.txt,63.txt\n",
      "Lexical Features , Hamilton,Hamilton,Madison,Madison,Madison,Madison,Madison,Madison,Madison,Madison,Hamilton,Madison\n",
      "Lexical Features - Punctuation , Hamilton,Hamilton,Hamilton,Hamilton,Madison,Madison,Madison,Hamilton,Hamilton,Hamilton,Madison,Madison\n",
      "Syntactic Features , Madison,Madison,Madison,Madison,Madison,Hamilton,Madison,Madison,Madison,Madison,Madison,Madison\n"
     ]
    }
   ],
   "source": [
    "# Print csv results\n",
    "print(\",\",','.join(disputed_papers_file_names))\n",
    "for i in range(len(results)):\n",
    "    print(results[i][1],\",\",','.join(all_results[i]))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### OLD WORK ### - saved just in case, but plays no part in above work\n",
    "# original work with attribution from main dataset, not used in rest of notebook\n",
    "files = open(\"The-Federalist-Papers.txt\")\n",
    "\n",
    "lines = files.read()\n",
    "files.close()\n",
    "\n",
    "elem = re.split(r'FEDERALIST\\.? No\\.?', lines)\n",
    "dataset = pd.DataFrame(columns=['author', 'paper'])\n",
    "\n",
    "for i in range(len(elem)):\n",
    "    paper = elem[i]  # get paper\n",
    "    paper = paper.replace(\"\\n\", \" \")  # strip newlines\n",
    "    paper = paper.replace(\"\\r\", \" \")  # strip windows return\n",
    "    paper_body = paper[paper.find(\"To the People of the State of New York: \") + 40:]  # get body\n",
    "\n",
    "    tokens = word_tokenize(paper_body)  # tokenize\n",
    "    tokens = [w.lower() for w in tokens]  # convert to lower case\n",
    "    table = str.maketrans('', '', string.punctuation)  # remove punctuation from each word\n",
    "    stripped = [w.translate(table) for w in tokens]\n",
    "    # remove remaining tokens that are not alphabetic\n",
    "    words = [word for word in stripped if word.isalpha()]\n",
    "    # filter out stop words\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    words = [w for w in words if not w in stop_words]\n",
    "    if \"MADISON\" in paper and \"JAY\" in paper:\n",
    "        dataset = dataset.append({\"author\": \"Madison and Jay\", \"paper\": paper_body}, ignore_index=True)\n",
    "    elif \"MADISON\" in paper and \"HAMILTON\" in paper:\n",
    "        dataset = dataset.append({\"author\": \"Madison and Hamilton\", \"paper\": paper_body}, ignore_index=True)\n",
    "    elif \"HAMILTON\" in paper and \"JAY\" in paper:\n",
    "        dataset = dataset.append({\"author\": \"Hamilton and Jay\", \"paper\": paper_body}, ignore_index=True)\n",
    "    elif \"MADISON\" in paper:\n",
    "        dataset = dataset.append({\"author\": \"Madison\", \"paper\": paper_body}, ignore_index=True)\n",
    "    elif \"JAY\" in paper:\n",
    "        dataset = dataset.append({\"author\": \"Jay\", \"paper\": paper_body}, ignore_index=True)\n",
    "    elif \"HAMILTON\" in paper:\n",
    "        dataset = dataset.append({\"author\": \"Hamilton\", \"paper\": paper_body}, ignore_index=True)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}