detecting_loanwords/make_frequency_distribution_greek_transliterated.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making a Frequency Distribution for Transliterated Greek"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import numpy as np\n",
    "import pickle\n",
    "\n",
    "from tqdm import tqdm\n",
    "from cltk.corpus.readers import get_corpus_reader\n",
    "from cltk.prosody.latin.string_utils import remove_punctuation_dict\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "from joblib import load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "from cltk.corpus.readers import get_corpus_reader\n",
    "from cltk.prosody.latin.string_utils import remove_punctuation_dict\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "from joblib import load"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the Transliterate Greek Pipeline built in another notebook `loanwords_problems_solutions`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys, inspect\n",
    "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n",
    "parentdir = os.path.dirname(currentdir)\n",
    "sys.path.insert(0, parentdir) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "process_greek_pipeline = load('process_greek_text_pipeline.0.22.1.joblib')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Perseus Greek Corpus Reader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "perseus_greek = get_corpus_reader(language='greek', corpus_name='greek_text_perseus')\n",
    "greek_texts = perseus_greek.fileids() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 222/222 [04:45<00:00,  1.29s/files]\n"
     ]
    }
   ],
   "source": [
    "word_counter = Counter()\n",
    "\n",
    "for file in tqdm(greek_texts, total=len(greek_texts), unit='files'):\n",
    "    X_greek_transliterated = process_greek_pipeline.fit_transform([list(perseus_greek.words(file))])\n",
    "    for word in X_greek_transliterated[0]:\n",
    "        word_counter.update({word: 1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('kai', 96043),\n",
       " ('de', 49241),\n",
       " ('men', 23247),\n",
       " ('tōn', 21936),\n",
       " ('to', 19808),\n",
       " ('tēn', 19392),\n",
       " ('en', 19085),\n",
       " ('ho', 18969),\n",
       " ('d', 17139),\n",
       " ('tou', 16483)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_counter.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.05150631420658943"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_words = sum(word_counter.values())\n",
    "word_probabilities = {key : val/total_words for key,val in word_counter.items()}\n",
    "word_probabilities['kai']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's save the counter for resuse."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('freq_dist.greek.transliterated.pkl', 'wb') as writer:\n",
    "    pickle.dump(word_probabilities, writer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}