yemen_rainfall_data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compile and process yemeni rainfall data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nA script to compile multiple netCDF4 files into one,\\nmaster file, which can then be excel-ized and processed\\nfor a machine learning algorithm\\n'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make imports\n",
    "\n",
    "#import netCDF4 as nc\n",
    "#import geopy as gp\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "#import xarray as xr\n",
    "import time\n",
    "#from geopy.exc import GeocoderTimedOut\n",
    "\n",
    "\"\"\"\n",
    "A script to compile multiple netCDF4 files into one,\n",
    "master file, which can then be excel-ized and processed\n",
    "for a machine learning algorithm\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ncfilesdir = '/Users/Rohil/Documents/iGEM/nc4_rainfall_files'\n",
    "#pandasfilesdir = r'C:\\Users\\Victor\\Desktop\\Pandas Rainfall Files'\n",
    "ultimatedir = '/Users/Rohil/Documents/iGEM/The Holy File'\n",
    "testingdir = '/Users/Rohil/Documents/iGEM/Testing File.nc4'\n",
    "pandasfiles = []\n",
    "points_location = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def date_extract(ncfile):\n",
    "    \"\"\" THIS SECTION COMPLETE \"\"\"\n",
    "    date = ncfile[11:19]\n",
    "    date = pd.to_datetime(date, format='%Y%m%d', errors='ignore')\n",
    "    return (date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "master_df = pd.DataFrame(columns = ['lat', 'lon', 'IRprecipitation_cnt', 'precipitation', 'HQprecipitation',\n",
    "       'randomError', 'randomError_cnt', 'precipitation_cnt',\n",
    "       'IRprecipitation', 'HQprecipitation_cnt', 'date'])\n",
    "for ncfile_original in os.listdir(ncfilesdir):\n",
    "    \"\"\" adding the date \"\"\"\n",
    "    ncfile = xr.open_dataset(ncfilesdir + '/' + ncfile_original)\n",
    "    ncfile = ncfile.to_dataframe().reset_index()\n",
    "    ncfile['date'] = date_extract(ncfile_original)\n",
    "    master_df = master_df.append(ncfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lat_long_unique = master_df[['lat', 'lon']].drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Location(Gombi, West Wellega, Oromia, ኢ.ፌ.ዲ.ሪ., (9.88333, 35.11667, 0.0))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "geolocator = gp.Nominatim()\n",
    "geolocator.reverse(('9.875', '35.125'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_coords(pair):\n",
    "    try:\n",
    "        return(geolocator.reverse((pair.lat, pair.lon)))\n",
    "    except:\n",
    "        time.sleep(60)\n",
    "        return(geolocator.reverse((pair.lat, pair.lon)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "master_location_df = pd.Series()\n",
    "for i in range(0,6500,500):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    (Gombi, West Wellega, Oromia, ኢ.ፌ.ዲ.ሪ., (9.883...\n",
       "1    (Sirba, Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ...\n",
       "2    (Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ.ሪ., (9...\n",
       "3    (Oco, Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ.ሪ...\n",
       "4    (Daleti, Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ....\n",
       "dtype: object"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9.875</td>\n",
       "      <td>35.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.875</td>\n",
       "      <td>35.375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9.875</td>\n",
       "      <td>35.625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9.875</td>\n",
       "      <td>35.875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9.875</td>\n",
       "      <td>36.125</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     lat     lon\n",
       "0  9.875  35.125\n",
       "1  9.875  35.375\n",
       "2  9.875  35.625\n",
       "3  9.875  35.875\n",
       "4  9.875  36.125"
      ]
     },
     "execution_count": 253,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lat_long_unique.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "locations = lat_long_unique.iloc[5500:].apply(get_coords, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "locations = locations.apply(pd.Series)\n",
    "locations.columns = ['address', 'coords']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lat_long_locs_5500_end = pd.concat([lat_long_unique.iloc[5500:], locations], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "locations_4000_to_5500 = lat_long_unique.iloc[4000:5500].apply(get_coords, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "locations_4000_to_5500 = locations_4000_to_5500.apply(pd.Series)\n",
    "locations_4000_to_5500.columns = ['address', 'coords']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lat_long_locs_4000_to_5500 = pd.concat([lat_long_unique.iloc[4000:5500], locations_4000_to_5500], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "locations_start_to_999 = lat_long_unique.iloc[:1000].apply(get_coords, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "locations_start_to_999 = locations_start_to_999.apply(pd.Series)\n",
    "locations_start_to_999.columns = ['address', 'coords']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "locations_1000_to_1999 = lat_long_unique.iloc[1000:2000].apply(get_coords, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "locations_1000_to_1999 = locations_1000_to_1999.apply(pd.Series)\n",
    "locations_1000_to_1999.columns = ['address', 'coords']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "locations_2000_to_2999 = lat_long_unique.iloc[2000:3000].apply(get_coords, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "locations_2000_to_2999 = locations_2000_to_2999.apply(pd.Series)\n",
    "locations_2000_to_2999.columns = ['address', 'coords']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "locations_3000_to_3999 = lat_long_unique.iloc[3000:4000].apply(get_coords, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "locations_3000_to_3999 = locations_3000_to_3999.apply(pd.Series)\n",
    "locations_3000_to_3999.columns = ['address', 'coords']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_locations = pd.concat([locations_start_to_999, locations_1000_to_1999, locations_2000_to_2999, locations_3000_to_3999, locations_4000_to_5500, locations])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>address</th>\n",
       "      <th>coords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Gombi, West Wellega, Oromia, ኢ.ፌ.ዲ.ሪ.</td>\n",
       "      <td>(9.88333, 35.11667)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sirba, Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ.ሪ.</td>\n",
       "      <td>(9.9, 35.38333)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ.ሪ.</td>\n",
       "      <td>(9.87505204088444, 35.6250050336125)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Oco, Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ.ሪ.</td>\n",
       "      <td>(9.91667, 35.91667)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Daleti, Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ.ሪ.</td>\n",
       "      <td>(9.86667, 36.1)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             address  \\\n",
       "0              Gombi, West Wellega, Oromia, ኢ.ፌ.ዲ.ሪ.   \n",
       "1   Sirba, Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ.ሪ.   \n",
       "2          Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ.ሪ.   \n",
       "3     Oco, Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ.ሪ.   \n",
       "4  Daleti, Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ.ሪ.   \n",
       "\n",
       "                                 coords  \n",
       "0                   (9.88333, 35.11667)  \n",
       "1                       (9.9, 35.38333)  \n",
       "2  (9.87505204088444, 35.6250050336125)  \n",
       "3                   (9.91667, 35.91667)  \n",
       "4                       (9.86667, 36.1)  "
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_locations.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "unique_location_df = pd.concat([lat_long_unique, all_locations], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_unique_location_df = unique_location_df[~unique_location_df.address.isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4395, 4)"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_unique_location_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "final_unique_location_df['untranslated_address'] = final_unique_location_df.address.str.split(', ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "untranslated_addresses = final_unique_location_df.address.drop_duplicates().str.split(', ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0              [Gombi, West Wellega, Oromia, ኢ.ፌ.ዲ.ሪ.]\n",
       "1    [Sirba, Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ...\n",
       "2          [Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ.ሪ.]\n",
       "3     [Oco, Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ.ዲ.ሪ.]\n",
       "4    [Daleti, Kemashi, Bīnshangul-Gumuz Kilil, ኢ.ፌ....\n",
       "Name: address, dtype: object"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "untranslated_addresses.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from googletrans import Translator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "translator = Translator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "def is_non_english_alphanum(s):\n",
    "    try:\n",
    "        s.encode(encoding='utf-8').decode('ascii')\n",
    "    except UnicodeDecodeError:\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Sayh as Saqlah', 'Ras Al Khaimah', 'The United Arab Emirates']"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[translator.translate(s).text if is_non_english_alphanum(s) else (s) for s in ['Sayh as Saqlah', '\\u200fرأس الخيمة\\u200e', 'الإمارات العربية المتحدة']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def translate_address(untranslated):\n",
    "    try:\n",
    "        translator = Translator()\n",
    "        return ([translator.translate(s).text if is_non_english_alphanum(s) else (s) for s in untranslated])\n",
    "    except:\n",
    "        time.sleep(60)\n",
    "        translator = Translator()\n",
    "        return ([translator.translate(s).text if is_non_english_alphanum(s) else (s) for s in untranslated])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "ename": "JSONDecodeError",
     "evalue": "Expecting value: line 1 column 1 (char 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mJSONDecodeError\u001b[0m                           Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-251-041fd0eea95c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m     \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m60\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mtranslator\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTranslator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m     \u001b[0mtranslated_unique_addresses_list\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muntranslated_addresses\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;36m50\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtranslate_address\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m     \u001b[0mtranslator\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36mapply\u001b[1;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[0;32m   2549\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2550\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masobject\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2551\u001b[1;33m                 \u001b[0mmapped\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2552\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2553\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/src/inference.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32m<ipython-input-237-734af079de1e>\u001b[0m in \u001b[0;36mtranslate_address\u001b[1;34m(untranslated)\u001b[0m\n\u001b[0;32m      6\u001b[0m         \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m60\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m         \u001b[0mtranslator\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTranslator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtranslator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtranslate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mis_non_english_alphanum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[1;32min\u001b[0m \u001b[0muntranslated\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-237-734af079de1e>\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m      6\u001b[0m         \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m60\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m         \u001b[0mtranslator\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTranslator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtranslator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtranslate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mis_non_english_alphanum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[1;32min\u001b[0m \u001b[0muntranslated\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\googletrans\\client.py\u001b[0m in \u001b[0;36mtranslate\u001b[1;34m(self, text, dest, src)\u001b[0m\n\u001b[0;32m    170\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    171\u001b[0m         \u001b[0morigin\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 172\u001b[1;33m         \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_translate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdest\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msrc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    173\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    174\u001b[0m         \u001b[1;31m# this code will be updated when the format is changed.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\googletrans\\client.py\u001b[0m in \u001b[0;36m_translate\u001b[1;34m(self, text, dest, src)\u001b[0m\n\u001b[0;32m     79\u001b[0m         \u001b[0mr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     80\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 81\u001b[1;33m         \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mutils\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat_json\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     82\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     83\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\googletrans\\utils.py\u001b[0m in \u001b[0;36mformat_json\u001b[1;34m(original)\u001b[0m\n\u001b[0;32m     60\u001b[0m         \u001b[0mconverted\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moriginal\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     61\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 62\u001b[1;33m         \u001b[0mconverted\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlegacy_format_json\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moriginal\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     63\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     64\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mconverted\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\googletrans\\utils.py\u001b[0m in \u001b[0;36mlegacy_format_json\u001b[1;34m(original)\u001b[0m\n\u001b[0;32m     52\u001b[0m             \u001b[0mtext\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mstates\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnxt\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 54\u001b[1;33m     \u001b[0mconverted\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     55\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mconverted\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     56\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Rohil\\Anaconda3\\lib\\json\\__init__.py\u001b[0m in \u001b[0;36mloads\u001b[1;34m(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[0;32m    352\u001b[0m             \u001b[0mparse_int\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mparse_float\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    353\u001b[0m             parse_constant is None and object_pairs_hook is None and not kw):\n\u001b[1;32m--> 354\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_default_decoder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    355\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    356\u001b[0m         \u001b[0mcls\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mJSONDecoder\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Rohil\\Anaconda3\\lib\\json\\decoder.py\u001b[0m in \u001b[0;36mdecode\u001b[1;34m(self, s, _w)\u001b[0m\n\u001b[0;32m    337\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    338\u001b[0m         \"\"\"\n\u001b[1;32m--> 339\u001b[1;33m         \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraw_decode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    340\u001b[0m         \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    341\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Rohil\\Anaconda3\\lib\\json\\decoder.py\u001b[0m in \u001b[0;36mraw_decode\u001b[1;34m(self, s, idx)\u001b[0m\n\u001b[0;32m    355\u001b[0m             \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscan_once\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    356\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 357\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mJSONDecodeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Expecting value\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    358\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)"
     ]
    }
   ],
   "source": [
    "translated_unique_addresses_list = []\n",
    "for i in range(0,2500,50):\n",
    "    time.sleep(60)\n",
    "    translator = Translator()\n",
    "    translated_unique_addresses_list.append(untranslated_addresses.iloc[i:i+50].apply(translate_address))\n",
    "    translator = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_unique_location_df.to_csv('/Users/Rohil/Documents/iGEM/yemen_rainfall_unique_locations.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "master_rainfall_df.drop(columns = ['city', 'admin', 'country'], inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "master_rainfall_df = pd.read_csv('/Users/Rohil/Documents/iGEM/yemen/rainfall/master_rainfall_file.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_unique_location_df = pd.read_csv('/Users/Rohil/Documents/iGEM/yemen/rainfall/yemen_rainfall_unique_locations.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lat_lon_translation = pd.read_csv('/Users/Rohil/Documents/iGEM/yemen/rainfall/yemen_rainfall_lat_lon_to_address.csv', encoding = \"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4395, 6)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lat_lon_translation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4395, 6)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_unique_location_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>city</th>\n",
       "      <th>admin2</th>\n",
       "      <th>admin1</th>\n",
       "      <th>country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9.875</td>\n",
       "      <td>35.125</td>\n",
       "      <td>NaN</td>\n",
       "      <td>West Wellega</td>\n",
       "      <td>Oromia</td>\n",
       "      <td>Ethiopia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.875</td>\n",
       "      <td>35.375</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Kamashi</td>\n",
       "      <td>Benishangul-Gumuz</td>\n",
       "      <td>Ethiopia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9.875</td>\n",
       "      <td>35.625</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Kamashi</td>\n",
       "      <td>Benishangul-Gumuz</td>\n",
       "      <td>Ethiopia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9.875</td>\n",
       "      <td>35.875</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Kamashi</td>\n",
       "      <td>Benishangul-Gumuz</td>\n",
       "      <td>Ethiopia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9.875</td>\n",
       "      <td>36.125</td>\n",
       "      <td>NaN</td>\n",
       "      <td>East Wellega</td>\n",
       "      <td>Oromia</td>\n",
       "      <td>Ethiopia</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     lat     lon city        admin2             admin1   country\n",
       "0  9.875  35.125  NaN  West Wellega             Oromia  Ethiopia\n",
       "1  9.875  35.375  NaN       Kamashi  Benishangul-Gumuz  Ethiopia\n",
       "2  9.875  35.625  NaN       Kamashi  Benishangul-Gumuz  Ethiopia\n",
       "3  9.875  35.875  NaN       Kamashi  Benishangul-Gumuz  Ethiopia\n",
       "4  9.875  36.125  NaN  East Wellega             Oromia  Ethiopia"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lat_lon_translation.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "master_rainfall_df = master_rainfall_df.merge(lat_lon_translation, how = 'left', on = ['lat', 'lon'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "master_rainfall_df.to_csv('/Users/Rohil/Documents/iGEM/yemen/rainfall/master_rainfall_file.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Najran is a border city between Saudi and Yemen - should not be included in Yemen (google maps sometimes accidentally includes it)\n",
    "yemen_master_rainfall = master_rainfall_df[(master_rainfall_df.country == 'Yemen')  & (master_rainfall_df.admin1 != 'Najran')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "yemen_master_rainfall[yemen_master_rainfall.admin1.isnull()].to_csv('/Users/Rohil/Documents/iGEM/yemen/rainfall/yemen_rainfall_na_to_rerun.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>IRprecipitation_cnt</th>\n",
       "      <th>precipitation</th>\n",
       "      <th>HQprecipitation</th>\n",
       "      <th>randomError</th>\n",
       "      <th>randomError_cnt</th>\n",
       "      <th>precipitation_cnt</th>\n",
       "      <th>IRprecipitation</th>\n",
       "      <th>HQprecipitation_cnt</th>\n",
       "      <th>date</th>\n",
       "      <th>city</th>\n",
       "      <th>admin2</th>\n",
       "      <th>admin1</th>\n",
       "      <th>country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>977</th>\n",
       "      <td>977</td>\n",
       "      <td>12.125</td>\n",
       "      <td>52.125</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.305515</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hadhramaut Governorate</td>\n",
       "      <td>Yemen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>978</th>\n",
       "      <td>978</td>\n",
       "      <td>12.125</td>\n",
       "      <td>52.375</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.305515</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hadhramaut Governorate</td>\n",
       "      <td>Yemen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1084</th>\n",
       "      <td>1084</td>\n",
       "      <td>12.375</td>\n",
       "      <td>53.625</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.305515</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hadhramaut Governorate</td>\n",
       "      <td>Yemen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1085</th>\n",
       "      <td>1085</td>\n",
       "      <td>12.375</td>\n",
       "      <td>53.875</td>\n",
       "      <td>8</td>\n",
       "      <td>2.1</td>\n",
       "      <td>2.1</td>\n",
       "      <td>13.232515</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hadhramaut Governorate</td>\n",
       "      <td>Yemen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1086</th>\n",
       "      <td>1086</td>\n",
       "      <td>12.375</td>\n",
       "      <td>54.125</td>\n",
       "      <td>8</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>7.650176</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hadhramaut Governorate</td>\n",
       "      <td>Yemen</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0     lat     lon  IRprecipitation_cnt  precipitation  \\\n",
       "977          977  12.125  52.125                    8            0.0   \n",
       "978          978  12.125  52.375                    8            0.0   \n",
       "1084        1084  12.375  53.625                    8            0.0   \n",
       "1085        1085  12.375  53.875                    8            2.1   \n",
       "1086        1086  12.375  54.125                    8            0.9   \n",
       "\n",
       "      HQprecipitation  randomError  randomError_cnt  precipitation_cnt  \\\n",
       "977               0.0     2.305515                8                  8   \n",
       "978               0.0     2.305515                8                  8   \n",
       "1084              0.0     2.305515                8                  8   \n",
       "1085              2.1    13.232515                8                  8   \n",
       "1086              0.9     7.650176                8                  8   \n",
       "\n",
       "      IRprecipitation  HQprecipitation_cnt        date city admin2  \\\n",
       "977               0.0                    6  2017-01-01  NaN    NaN   \n",
       "978               0.0                    6  2017-01-01  NaN    NaN   \n",
       "1084              0.0                    6  2017-01-01  NaN    NaN   \n",
       "1085              0.0                    6  2017-01-01  NaN    NaN   \n",
       "1086              0.0                    5  2017-01-01  NaN    NaN   \n",
       "\n",
       "                      admin1 country  \n",
       "977   Hadhramaut Governorate   Yemen  \n",
       "978   Hadhramaut Governorate   Yemen  \n",
       "1084  Hadhramaut Governorate   Yemen  \n",
       "1085  Hadhramaut Governorate   Yemen  \n",
       "1086  Hadhramaut Governorate   Yemen  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yemen_master_rainfall.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'Al Buraiqeh', 'Khormaksar', 'Al Maqatirah District',\n",
       "       'Al Anad', 'Mocha', ' ', 'Al Qabbaytah District', 'Ba Tays', '  ',\n",
       "       'Jiblah', 'Al Bayda', 'Lawdar', 'Al Jarrahi', 'Wosab AL-Alsafel',\n",
       "       'Ash Sharyah District', 'Wosab Al-Aali', 'Magreb Anns', 'Anns',\n",
       "       \"Rada'a District\", 'Habbn', 'Algabin', 'Alsalfiah', 'Almanar',\n",
       "       'Mangathah', 'Mefaat Anns', 'Kaninah', 'Ash Shihr', 'Al Marawi`ah',\n",
       "       'Jabal Alsharq', 'Dhoran Aans', 'Zarajah', 'Alhada', \"Qusay'ir\",\n",
       "       'Urj', 'Bait Alahmar', 'Kamaran', 'Al Luhayyah', 'Hiswah', 'Qasam',\n",
       "       'Alamashya', 'Sahwa', 'Rumah'], dtype=object)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yemen_master_rainfall.admin2.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Hadhramaut Governorate', 'Lahij Governorate', 'Taizz Governorate',\n",
       "       \"'Adan Governorate\", 'Abyan Governorate', \"Ad Dali' Governorate\",\n",
       "       'Shabwah Governorate', 'Al Hudaydah Governorate',\n",
       "       'Ibb Governorate', \"Al Bayda' Governorate\", 'Dhamar Governorate',\n",
       "       'Raymah Governorate', \"Ma'rib Governorate\", \"Sana'a Governorate\",\n",
       "       'Al Mahrah Governorate', 'Al Mahwit Governorate',\n",
       "       'Hajjah Governorate', \"'Amran Governorate\", 'Al Jawf Governorate',\n",
       "       'Saada Governorate'], dtype=object)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yemen_master_rainfall.admin1.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan], dtype=object)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yemen_master_rainfall.city.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "governorate_to_iso_dict = {'Hadhramaut Governorate':'YE-HD', 'Lahij Governorate':'YE-LA', 'Taizz Governorate':'YE-TA',\n",
    "       \"'Adan Governorate\":'YE-AD', 'Abyan Governorate':'YE-AB', \"Ad Dali' Governorate\":'YE-DA',\n",
    "       'Shabwah Governorate':'YE-SH', 'Al Hudaydah Governorate':'YE-HU',\n",
    "       'Ibb Governorate':'YE-IB', \"Al Bayda' Governorate\":'YE-BA', 'Dhamar Governorate':'YE-DH',\n",
    "       'Raymah Governorate':'YE-RA', \"Ma'rib Governorate\":'YE-MA',\n",
    "       \"Sana'a Governorate\":'YE-SN', 'Al Mahrah Governorate':'YE-MR',\n",
    "       'Al Mahwit Governorate':'YE-MW', 'Hajjah Governorate':'YE-HJ',\n",
    "       \"'Amran Governorate\":'YE-AM', 'Al Jawf Governorate':'YE-JA', 'Saada Governorate':'YE-SD'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "yemen_master_rainfall['governorate_isocode'] = yemen_master_rainfall.admin1.map(governorate_to_iso_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coord_iso_to_replace = pd.read_csv('/Users/Rohil/Documents/iGEM/yemen/rainfall/iso_coords_to_reassign.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>governorate_isocode</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>YE-HD-AL</th>\n",
       "      <td>14.375</td>\n",
       "      <td>49.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>YE-HD-AL</th>\n",
       "      <td>14.625</td>\n",
       "      <td>49.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>YE-SA</th>\n",
       "      <td>15.375</td>\n",
       "      <td>44.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>YE-SA</th>\n",
       "      <td>15.375</td>\n",
       "      <td>44.375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>YE-SA</th>\n",
       "      <td>15.625</td>\n",
       "      <td>44.125</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        lat     lon\n",
       "governorate_isocode                \n",
       "YE-HD-AL             14.375  49.125\n",
       "YE-HD-AL             14.625  49.125\n",
       "YE-SA                15.375  44.125\n",
       "YE-SA                15.375  44.375\n",
       "YE-SA                15.625  44.125"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coord_iso_to_replace[['governorate_isocode', 'lat', 'lon']].set_index('governorate_isocode')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>IRprecipitation_cnt</th>\n",
       "      <th>precipitation</th>\n",
       "      <th>HQprecipitation</th>\n",
       "      <th>randomError</th>\n",
       "      <th>randomError_cnt</th>\n",
       "      <th>precipitation_cnt</th>\n",
       "      <th>IRprecipitation</th>\n",
       "      <th>HQprecipitation_cnt</th>\n",
       "      <th>date</th>\n",
       "      <th>city</th>\n",
       "      <th>admin2</th>\n",
       "      <th>admin1</th>\n",
       "      <th>country</th>\n",
       "      <th>governorate_isocode</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>977</td>\n",
       "      <td>12.125</td>\n",
       "      <td>52.125</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.305515</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hadhramaut Governorate</td>\n",
       "      <td>Yemen</td>\n",
       "      <td>YE-HD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>978</td>\n",
       "      <td>12.125</td>\n",
       "      <td>52.375</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.305515</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hadhramaut Governorate</td>\n",
       "      <td>Yemen</td>\n",
       "      <td>YE-HD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1084</td>\n",
       "      <td>12.375</td>\n",
       "      <td>53.625</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.305515</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hadhramaut Governorate</td>\n",
       "      <td>Yemen</td>\n",
       "      <td>YE-HD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1085</td>\n",
       "      <td>12.375</td>\n",
       "      <td>53.875</td>\n",
       "      <td>8</td>\n",
       "      <td>2.1</td>\n",
       "      <td>2.1</td>\n",
       "      <td>13.232515</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hadhramaut Governorate</td>\n",
       "      <td>Yemen</td>\n",
       "      <td>YE-HD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1086</td>\n",
       "      <td>12.375</td>\n",
       "      <td>54.125</td>\n",
       "      <td>8</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>7.650176</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hadhramaut Governorate</td>\n",
       "      <td>Yemen</td>\n",
       "      <td>YE-HD</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0     lat     lon  IRprecipitation_cnt  precipitation  \\\n",
       "0         977  12.125  52.125                    8            0.0   \n",
       "1         978  12.125  52.375                    8            0.0   \n",
       "2        1084  12.375  53.625                    8            0.0   \n",
       "3        1085  12.375  53.875                    8            2.1   \n",
       "4        1086  12.375  54.125                    8            0.9   \n",
       "\n",
       "   HQprecipitation  randomError  randomError_cnt  precipitation_cnt  \\\n",
       "0              0.0     2.305515                8                  8   \n",
       "1              0.0     2.305515                8                  8   \n",
       "2              0.0     2.305515                8                  8   \n",
       "3              2.1    13.232515                8                  8   \n",
       "4              0.9     7.650176                8                  8   \n",
       "\n",
       "   IRprecipitation  HQprecipitation_cnt        date  city admin2  \\\n",
       "0              0.0                    6  2017-01-01   NaN    NaN   \n",
       "1              0.0                    6  2017-01-01   NaN    NaN   \n",
       "2              0.0                    6  2017-01-01   NaN    NaN   \n",
       "3              0.0                    6  2017-01-01   NaN    NaN   \n",
       "4              0.0                    5  2017-01-01   NaN    NaN   \n",
       "\n",
       "                   admin1 country governorate_isocode  \n",
       "0  Hadhramaut Governorate   Yemen               YE-HD  \n",
       "1  Hadhramaut Governorate   Yemen               YE-HD  \n",
       "2  Hadhramaut Governorate   Yemen               YE-HD  \n",
       "3  Hadhramaut Governorate   Yemen               YE-HD  \n",
       "4  Hadhramaut Governorate   Yemen               YE-HD  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yemen_master_rainfall.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "iso_cord_replace_dict = {\n",
    "    'YE-HD-AL':(14.375,49.125),\n",
    "    'YE-HD-AL':(14.625,49.125),\n",
    "    'YE-SA':(15.375,44.125),\n",
    "    'YE-SA':(15.375,44.375),\n",
    "    'YE-SA':(15.625,44.125)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py:3643: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self[name] = value\n"
     ]
    }
   ],
   "source": [
    "for key, val in iso_cord_replace_dict.items():\n",
    "    rainfall_to_reassign = yemen_master_rainfall[(yemen_master_rainfall.lat == val[0]) & (yemen_master_rainfall.lon == val[1])]\n",
    "    rainfall_to_reassign.governorate_isocode = key\n",
    "    rainfall_to_reassign.admin2 = key\n",
    "    yemen_master_rainfall = yemen_master_rainfall.append(rainfall_to_reassign)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "yemen_master_rainfall = yemen_master_rainfall.sort_values('date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "yemen_master_rainfall.to_csv('/Users/Rohil/Documents/iGEM/yemen/rainfall/yemen_master_rainfall.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['YE-HD', 'YE-SD', 'YE-JA', 'YE-MR', 'YE-HJ', 'YE-AM', 'YE-MA',\n",
       "       'YE-HD-AL', 'YE-SA', 'YE-SH', 'YE-HU', 'YE-RA', 'YE-DH', 'YE-BA',\n",
       "       'YE-AB', 'YE-IB', 'YE-DA', 'YE-AD', 'YE-TA', 'YE-LA', 'YE-SN',\n",
       "       'YE-MW'], dtype=object)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yemen_master_rainfall.governorate_isocode.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "daily_max_rainfall_crosstab = yemen_master_rainfall.pivot_table(index = 'date', columns = 'governorate_isocode', values = 'precipitation', aggfunc = 'max')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "daily_mean_rainfall_crosstab = yemen_master_rainfall.pivot_table(index = 'date', columns = 'governorate_isocode', values = 'precipitation', aggfunc = 'mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "daily_min_rainfall_crosstab = yemen_master_rainfall.pivot_table(index = 'date', columns = 'governorate_isocode', values = 'precipitation', aggfunc = 'min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "daily_max_rainfall_crosstab.to_csv('/Users/Rohil/Documents/iGEM/yemen/rainfall/yemen_daily_max_rainfall_crosstab.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "daily_mean_rainfall_crosstab.to_csv('/Users/Rohil/Documents/iGEM/yemen/rainfall/yemen_daily_mean_rainfall_crosstab.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "daily_min_rainfall_crosstab.to_csv('/Users/Rohil/Documents/iGEM/yemen/rainfall/yemen_daily_min_rainfall_crosstab.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultiIndex(levels=[['amax', 'mean', 'amin'], ['YE-AB', 'YE-AD', 'YE-AM', 'YE-BA', 'YE-DA', 'YE-DH', 'YE-HD', 'YE-HD-AL', 'YE-HJ', 'YE-HU', 'YE-IB', 'YE-JA', 'YE-LA', 'YE-MA', 'YE-MR', 'YE-MW', 'YE-RA', 'YE-SA', 'YE-SD', 'YE-SH', 'YE-SN', 'YE-TA']],\n",
       "           labels=[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]],\n",
       "           names=[None, 'governorate_isocode'])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yemen_master_rainfall.pivot_table(index = 'date', columns = 'governorate_isocode', values = 'precipitation', aggfunc = [np.max, np.mean, np.min]).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.19519311818181817"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yemen_master_rainfall[(yemen_master_rainfall.governorate_isocode == 'YE-HJ') & (yemen_master_rainfall.date == '2017-01-02')].precipitation.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "yemen_master_rainfall = pd.read_csv('/Users/Rohil/Documents/iGEM/yemen/rainfall/yemen_master_rainfall.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "rainfall_timeseries = yemen_master_rainfall[['governorate_isocode', 'date', 'precipitation']].rename(columns = {'governorate_isocode':'gov_iso', 'precipitation':\n",
    "                                                                                         'rainfall'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "rainfall_timeseries.to_csv('/Users/Rohil/Documents/iGEM/yemen/rainfall/yemen_rainfall_timeseries.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}