22millionRowdata-LightGBM model-Run different year expeiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Author: Chongzheng Zhao\n",
    "# Kaggle Competition - New York City Taxi Fare Prediction\n",
    "# FINAL BEAT 1454 TEAMS!\n",
    "# We team NYCTAXI located at 30 out of 1484 teams, Top 2%!\n",
    "# Final Score 2.86770\n",
    "# Competition Official Website: https://www.kaggle.com/c/new-york-city-taxi-fare-prediction\n",
    "# To see the leaderboard(competition result): https://www.kaggle.com/c/new-york-city-taxi-fare-prediction/leaderboard\n",
    "# Welcome to my Github profile: https://github.com/ChongzhengZhao/\n",
    "# Welcome to my Kaggle Profile: https://www.kaggle.com/chongzhengzhao\n",
    "# Welcome to my Linkedin Profile: https://www.linkedin.com/in/chongzhengzhao/\n",
    "# Last updated: 30/11/2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "import scipy as scipy\n",
    "import datetime as dt\n",
    "from sklearn.model_selection import train_test_split\n",
    "import lightgbm as lgbm\n",
    "import os\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reading Data\n",
    "train_df_raw =  pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  ！NOTE： 下一行修改年份,从2009~2015 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# N O T E: 修改年份,从2009~2015 ####\n",
    "####\n",
    "####\n",
    "train_df = train_df_raw[train_df_raw['year']==2009]\n",
    "####\n",
    "####\n",
    "#### N O T E: 修改年份,从2009~2015 ####"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Drop rows with null values\n",
    "train_df = train_df.dropna(how = 'any', axis = 'rows')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_df(df):\n",
    "    return df[(df.fare_amount > 0)  & (df.fare_amount <= 500) &\n",
    "          # (df.passenger_count >= 0) & (df.passenger_count <= 8)  &\n",
    "           ((df.pickup_longitude != 0) & (df.pickup_latitude != 0) & (df.dropoff_longitude != 0) & (df.dropoff_latitude != 0) )]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = clean_df(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To Compute Haversine distance\n",
    "def sphere_dist(pickup_lat, pickup_lon, dropoff_lat, dropoff_lon):\n",
    "    \"\"\"\n",
    "    Return distance along great radius between pickup and dropoff coordinates.\n",
    "    \"\"\"\n",
    "    #Define earth radius (km)\n",
    "    R_earth = 6371\n",
    "    #Convert degrees to radians\n",
    "    pickup_lat, pickup_lon, dropoff_lat, dropoff_lon = map(np.radians,\n",
    "                                                             [pickup_lat, pickup_lon, \n",
    "                                                              dropoff_lat, dropoff_lon])\n",
    "    #Compute distances along lat, lon dimensions\n",
    "    dlat = dropoff_lat - pickup_lat\n",
    "    dlon = dropoff_lon - pickup_lon\n",
    "    \n",
    "    #Compute haversine distance\n",
    "    a = np.sin(dlat/2.0)**2 + np.cos(pickup_lat) * np.cos(dropoff_lat) * np.sin(dlon/2.0)**2\n",
    "    return 2 * R_earth * np.arcsin(np.sqrt(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sphere_dist_bear(pickup_lat, pickup_lon, dropoff_lat, dropoff_lon):\n",
    "    \"\"\"\n",
    "    Return distance along great radius between pickup and dropoff coordinates.\n",
    "    \"\"\"\n",
    "    #Define earth radius (km)\n",
    "    R_earth = 6371\n",
    "    #Convert degrees to radians\n",
    "    pickup_lat, pickup_lon, dropoff_lat, dropoff_lon = map(np.radians,\n",
    "                                                             [pickup_lat, pickup_lon, \n",
    "                                                              dropoff_lat, dropoff_lon])\n",
    "    #Compute distances along lat, lon dimensions\n",
    "    dlat = dropoff_lat - pickup_lat\n",
    "    dlon = pickup_lon - dropoff_lon\n",
    "    \n",
    "    #Compute bearing distance\n",
    "    a = np.arctan2(np.sin(dlon * np.cos(dropoff_lat)),np.cos(pickup_lat) * np.sin(dropoff_lat) - np.sin(pickup_lat) * np.cos(dropoff_lat) * np.cos(dlon))\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def radian_conv(degree):\n",
    "    \"\"\"\n",
    "    Return radian.\n",
    "    \"\"\"\n",
    "    return  np.radians(degree)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_datetime_info(dataset):\n",
    "    #Convert to datetime format\n",
    "    dataset['pickup_datetime'] = pd.to_datetime(dataset['pickup_datetime'],format=\"%Y-%m-%d %H:%M:%S UTC\")\n",
    "    \n",
    "    dataset['hour'] = dataset.pickup_datetime.dt.hour\n",
    "    dataset['day'] = dataset.pickup_datetime.dt.day\n",
    "    dataset['month'] = dataset.pickup_datetime.dt.month\n",
    "    dataset['weekday'] = dataset.pickup_datetime.dt.weekday\n",
    "    dataset['year'] = dataset.pickup_datetime.dt.year\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_airport_dist(dataset):\n",
    "    \"\"\"\n",
    "    Return minumum distance from pickup or dropoff coordinates to each airport.\n",
    "    JFK: John F. Kennedy International Airport\n",
    "    EWR: Newark Liberty International Airport\n",
    "    LGA: LaGuardia Airport\n",
    "    SOL: Statue of Liberty \n",
    "    NYC: Newyork Central\n",
    "    \"\"\"\n",
    "    jfk_coord = (40.639722, -73.778889)\n",
    "    ewr_coord = (40.6925, -74.168611)\n",
    "    lga_coord = (40.77725, -73.872611)\n",
    "    sol_coord = (40.6892,-74.0445) # Statue of Liberty\n",
    "    nyc_coord = (40.7141667,-74.0063889) \n",
    "    \n",
    "    \n",
    "    pickup_lat = dataset['pickup_latitude']\n",
    "    dropoff_lat = dataset['dropoff_latitude']\n",
    "    pickup_lon = dataset['pickup_longitude']\n",
    "    dropoff_lon = dataset['dropoff_longitude']\n",
    "    \n",
    "    pickup_jfk = sphere_dist(pickup_lat, pickup_lon, jfk_coord[0], jfk_coord[1]) \n",
    "    dropoff_jfk = sphere_dist(jfk_coord[0], jfk_coord[1], dropoff_lat, dropoff_lon) \n",
    "    pickup_ewr = sphere_dist(pickup_lat, pickup_lon, ewr_coord[0], ewr_coord[1])\n",
    "    dropoff_ewr = sphere_dist(ewr_coord[0], ewr_coord[1], dropoff_lat, dropoff_lon) \n",
    "    pickup_lga = sphere_dist(pickup_lat, pickup_lon, lga_coord[0], lga_coord[1]) \n",
    "    dropoff_lga = sphere_dist(lga_coord[0], lga_coord[1], dropoff_lat, dropoff_lon)\n",
    "    pickup_sol = sphere_dist(pickup_lat, pickup_lon, sol_coord[0], sol_coord[1]) \n",
    "    dropoff_sol = sphere_dist(sol_coord[0], sol_coord[1], dropoff_lat, dropoff_lon)\n",
    "    pickup_nyc = sphere_dist(pickup_lat, pickup_lon, nyc_coord[0], nyc_coord[1]) \n",
    "    dropoff_nyc = sphere_dist(nyc_coord[0], nyc_coord[1], dropoff_lat, dropoff_lon)\n",
    "    \n",
    "    \n",
    "    \n",
    "    dataset['jfk_dist'] = pickup_jfk + dropoff_jfk\n",
    "    dataset['ewr_dist'] = pickup_ewr + dropoff_ewr\n",
    "    dataset['lga_dist'] = pickup_lga + dropoff_lga\n",
    "    dataset['sol_dist'] = pickup_sol + dropoff_sol\n",
    "    dataset['nyc_dist'] = pickup_nyc + dropoff_nyc\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = add_datetime_info(train_df)\n",
    "train_df = add_airport_dist(train_df)\n",
    "train_df['distance'] = sphere_dist(train_df['pickup_latitude'], train_df['pickup_longitude'], \n",
    "                                   train_df['dropoff_latitude'] , train_df['dropoff_longitude']) \n",
    "\n",
    "train_df['bearing'] = sphere_dist_bear(train_df['pickup_latitude'], train_df['pickup_longitude'], \n",
    "                                   train_df['dropoff_latitude'] , train_df['dropoff_longitude'])                                    \n",
    "train_df['pickup_latitude'] = radian_conv(train_df['pickup_latitude'])\n",
    "train_df['pickup_longitude'] = radian_conv(train_df['pickup_longitude'])\n",
    "train_df['dropoff_latitude'] = radian_conv(train_df['dropoff_latitude'])\n",
    "train_df['dropoff_longitude'] = radian_conv(train_df['dropoff_longitude'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.drop(['key', 'pickup_datetime'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ！NOTE：跑其他年份时，记得更改下一行导出的文件名，以免覆盖上次导出文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.to_csv('train_22m_python_2009.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = train_df['fare_amount']\n",
    "train_df = train_df.drop(['fare_amount'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test = train_test_split(train_df,y,random_state=123,test_size=0.10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ！NOTE：跑其他年份时，记得更改下一行导出的文件名，以免覆盖上次导出文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_list = x_train.index.tolist()\n",
    "x_test_list = x_test.index.tolist()\n",
    "temp_train = pd.DataFrame(columns=['x_train_index'],data=x_train_list)\n",
    "temp_train.head()\n",
    "temp_train.to_csv('x_train_index_2009.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ！NOTE：跑其他年份时，记得更改下一行导出的文件名，以免覆盖上次导出文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_test = pd.DataFrame(columns=['x_test_index'],data=x_test_list)\n",
    "temp_test.to_csv(\"x_test_index_2009.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train_df\n",
    "del y\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train1=pd.DataFrame(np.array(y_train), columns=['y_train'])\n",
    "y_valid1=pd.DataFrame(np.array(y_test), columns=['y_valid'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ！NOTE：跑其他年份时，记得更改下一行导出的文件名，以免覆盖上次导出文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train1.to_csv('y_train_2009.csv',index=False)\n",
    "y_valid1.to_csv('y_valid_2009.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "        'boosting_type':'gbdt',\n",
    "        'objective': 'regression',\n",
    "        'nthread': 4,\n",
    "        'num_leaves': 31,\n",
    "        'learning_rate': 0.029,\n",
    "        'max_depth': -1,\n",
    "        'subsample': 0.8,\n",
    "        'colsample_bytree': 0.6,\n",
    "        'metric': 'rmse',\n",
    "        'min_split_gain': 0.5,\n",
    "        'min_child_weight': 1,\n",
    "        'min_child_samples': 10,\n",
    "        'scale_pos_weight':1,\n",
    "        'zero_as_missing': True,\n",
    "        'seed':0,\n",
    "        'num_rounds':50000\n",
    "    }\n",
    "    \n",
    "train_set = lgbm.Dataset(x_train, y_train, silent=False,categorical_feature=['year','month','day','weekday'])\n",
    "valid_set = lgbm.Dataset(x_test, y_test, silent=False,categorical_feature=['year','month','day','weekday'])\n",
    "model = lgbm.train(params, train_set = train_set, num_boost_round=10000,early_stopping_rounds=500,verbose_eval=500, valid_sets=valid_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred=model.predict(x_train, num_iteration = model.best_iteration)  \n",
    "y_valid_pred=model.predict(x_test, num_iteration = model.best_iteration)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred1=pd.DataFrame(np.array(y_train_pred), columns=['y_train_pred'])\n",
    "y_valid_pred1=pd.DataFrame(np.array(y_valid_pred), columns=['y_valid_pred'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ！NOTE：跑其他年份时，记得更改下一行导出的文件名，以免覆盖上次导出文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred1.to_csv('y_train_pred_2009.csv', index=False)\n",
    "y_valid_pred1.to_csv('y_valid_pred_2009.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del x_train\n",
    "del y_train\n",
    "del x_test\n",
    "del y_test\n",
    "del y_train1\n",
    "del y_valid1\n",
    "del y_train_pred\n",
    "del y_valid_pred\n",
    "del y_train_pred1\n",
    "del y_valid_pred1\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df =  pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = add_datetime_info(test_df)\n",
    "test_df = add_airport_dist(test_df)\n",
    "test_df['distance'] = sphere_dist(test_df['pickup_latitude'], test_df['pickup_longitude'], \n",
    "                                   test_df['dropoff_latitude'] , test_df['dropoff_longitude'])\n",
    "\n",
    "test_df['bearing'] = sphere_dist_bear(test_df['pickup_latitude'], test_df['pickup_longitude'], \n",
    "                                    test_df['dropoff_latitude'] , test_df['dropoff_longitude'])  \n",
    "test_df['pickup_latitude'] = radian_conv(test_df['pickup_latitude'])\n",
    "test_df['pickup_longitude'] = radian_conv(test_df['pickup_longitude'])\n",
    "test_df['dropoff_latitude'] = radian_conv(test_df['dropoff_latitude'])\n",
    "test_df['dropoff_longitude'] = radian_conv(test_df['dropoff_longitude'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ！NOTE：跑其他年份时，记得更改下一行导出的文件名，以免覆盖上次导出文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_key = test_df['key']\n",
    "test_df = test_df.drop(['key', 'pickup_datetime'],axis=1)\n",
    "test_df.to_csv('test_22m_python.csv_2009', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ！NOTE：跑其他年份时，记得更改下一行导出的文件名，以免覆盖上次导出文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Predict from test set\n",
    "prediction = model.predict(test_df, num_iteration = model.best_iteration)      \n",
    "submission = pd.DataFrame({\n",
    "        \"key\": test_key,\n",
    "        \"fare_amount\": prediction\n",
    "})\n",
    "\n",
    "submission.to_csv('submission_22m_python_2009.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importance_df = pd.DataFrame()\n",
    "importance_df[\"feature\"] = x_train.columns\n",
    "importance_df[\"importance\"] = model.feature_importance(importance_type='gain')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "% matplotlib inline\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(8, 10))\n",
    "sns.barplot(x=\"importance\", y=\"feature\", data=importance_df.sort_values(by=\"importance\", ascending=False))\n",
    "plt.title('LightGBM Features (avg over folds)')\n",
    "plt.tight_layout\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}