prediction_template.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import preprocess_tools as tools\n",
    "import itertools\n",
    "from sklearn.metrics import precision_score,recall_score,accuracy_score,confusion_matrix, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load pretrained model\n",
    "rf = pickle.load(open('final_model.sav', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load data to be predicted\n",
    "df_measurement = pd.read_csv('/Users/grandia/Downloads/safety/features/part-00000-e6120af0-10c2-4248-97c4-81baf4304e5c-c000.csv')\n",
    "df_label = pd.read_csv('/Users/grandia/Downloads/safety/labels/part-00000-e9445087-aa0a-433b-a7f6-7f4c19d78ad6-c000.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# change data type of bookingID to str\n",
    "df_measurement['bookingID'] = df_measurement['bookingID'].astype(str)\n",
    "df_label['bookingID'] = df_label['bookingID'].astype(str)\n",
    "# combine the label and the measurement\n",
    "df_merge = df_measurement.merge(df_label, on='bookingID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# drop booking ID which has less than 30 readings\n",
    "count_booking = df_merge.groupby('bookingID').Speed.count()\n",
    "id_to_be_dropped = list(count_booking[count_booking > 60].index)\n",
    "df_merge = df_merge.drop(df_merge[df_merge['bookingID'].isin(id_to_be_dropped)].index, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# group dataframe by bookingId\n",
    "grouped = df_merge.groupby('bookingID')\n",
    "groups = dict(list(grouped))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# extract the g, this might take a while depending on how many data there is\n",
    "df_g = tools.create_gravity_adjustment_df(groups)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# clean and reorient data, this might take a while depending on how many data there is\n",
    "df_merge = tools.process_clean_and_reorient(df_merge, df_g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# group reoriented dataframe by bookingId\n",
    "grouped_feature = df_merge.groupby('bookingID')\n",
    "groups_feature = dict(list(grouped_feature))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# extract features, there might be some warnings if the bookingID only has few measurements\n",
    "df_features, df_label = tools.extract_features(groups_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# sometimes the features resulted in infinity or nan, replace them with 0\n",
    "df_features.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "df_features.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# prepare the data for prediction and result\n",
    "X_test = df_features\n",
    "y_test = df_label.values.ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# show confusion matrix and calculate score\n",
    "y_pred = rf.predict(X_test)\n",
    "print(\"Confusion Matrix \\n\", confusion_matrix(y_test,y_pred))\n",
    "print(\"Precision\", precision_score(y_test,y_pred))\n",
    "print(\"Recall\", recall_score(y_test,y_pred))\n",
    "print(\"Accuracy\", accuracy_score(y_test,y_pred))\n",
    "y_pred_proba = rf.predict_proba(X_test)\n",
    "print(\"ROC AUC\", roc_auc_score(y_test, y_pred_proba[:,1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}