8_Model_LightGBM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/bpc_msds2019/miniconda3/lib/python3.7/site-packages/lightgbm/__init__.py:46: UserWarning: Starting from version 2.2.1, the library file in distribution wheels for macOS is built by the Apple Clang (Xcode_8.3.1) compiler.\n",
      "This means that in case of installing LightGBM from PyPI via the ``pip install lightgbm`` command, you don't need to install the gcc compiler anymore.\n",
      "Instead of that, you need to install the OpenMP library, which is required for running LightGBM on the system with the Apple Clang compiler.\n",
      "You can install the OpenMP library by the following command: ``brew install libomp``.\n",
      "  \"You can install the OpenMP library by the following command: ``brew install libomp``.\", UserWarning)\n"
     ]
    }
   ],
   "source": [
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import explained_variance_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transaction Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/bpc_msds2019/miniconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3018: DtypeWarning: Columns (1) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "#Get data\n",
    "raw_trans_csv = pd.read_csv('../Data/train_trans.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "classification_labels = raw_trans_csv.iloc[:,1] #Unused here - regression only\n",
    "regression_labels = raw_trans_csv.iloc[:,2]\n",
    "transaction_data = raw_trans_csv.iloc[:,3:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Split test and train\n",
    "X_train_1, X_test, y_train_1, y_test = train_test_split(transaction_data, regression_labels, test_size=0.33)\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X_train_1, y_train_1, test_size=0.33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create lgb datasets\n",
    "train_data = lgb.Dataset(X_train, label=y_train)\n",
    "test_data = lgb.Dataset(X_test, label=y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Search Space:\n",
    "\n",
    "num_leaves : 31 (default), 50, 100\n",
    "\n",
    "learning_rate: 0.1 (default), 0.05\n",
    "\n",
    "max_bin: 255 (default), 300\n",
    "\n",
    "Total iterations required:  12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Manual Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#All params\n",
    "param1 = {'num_leaves':31, 'learning_rate':0.1, 'max_bin':255,'objective':'rmse'}\n",
    "param2 = {'num_leaves':50, 'learning_rate':0.1, 'max_bin':255,'objective':'rmse'}\n",
    "param3 = {'num_leaves':100, 'learning_rate':0.1, 'max_bin':255,'objective':'rmse'}\n",
    "param4 = {'num_leaves':31, 'learning_rate':0.05, 'max_bin':255,'objective':'rmse'}\n",
    "param5 = {'num_leaves':50, 'learning_rate':0.05, 'max_bin':255,'objective':'rmse'}\n",
    "param6 = {'num_leaves':100, 'learning_rate':0.05, 'max_bin':255,'objective':'rmse'}\n",
    "param7 = {'num_leaves':31, 'learning_rate':0.1, 'max_bin':300,'objective':'rmse'}\n",
    "param8 = {'num_leaves':50, 'learning_rate':0.1, 'max_bin':300,'objective':'rmse'}\n",
    "param9 = {'num_leaves':100, 'learning_rate':0.1, 'max_bin':300,'objective':'rmse'}\n",
    "param10 = {'num_leaves':31, 'learning_rate':0.05, 'max_bin':300,'objective':'rmse'}\n",
    "param11 = {'num_leaves':50, 'learning_rate':0.05, 'max_bin':300,'objective':'rmse'}\n",
    "param12 = {'num_leaves':100, 'learning_rate':0.05, 'max_bin':300,'objective':'rmse'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Custom code for grid search\n",
    "def train_model_print_metrics(param_dict, train_data, X_test, y_test):\n",
    "    model = lgb.train(param_dict, train_data)\n",
    "    ypred = model.predict(X_test)\n",
    "    \n",
    "    rmse = mean_squared_error(y_test,ypred)**(1/2)\n",
    "    psuedo_r2 = r2_score(y_test, ypred)\n",
    "    explained_variance = explained_variance_score(y_test,ypred)\n",
    "    print(\"RMSE: \"+str(rmse)+\"\\n\")\n",
    "    print(\"Psuedo R2: \"+str(psuedo_r2)+\"\\n\")\n",
    "    print(\"Explained Variance: \"+str(explained_variance)+\"\\n\")\n",
    "    \n",
    "    return [rmse, psuedo_r2, explained_variance]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_grid = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 50926370.31321675\n",
      "\n",
      "Psuedo R2: -0.047743702266721755\n",
      "\n",
      "Explained Variance: -0.047743459423267964\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid.append(train_model_print_metrics(param1, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 50957469.26743807\n",
      "\n",
      "Psuedo R2: -0.049023733892990196\n",
      "\n",
      "Explained Variance: -0.0490227561846075\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid.append(train_model_print_metrics(param2, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 51287110.642113\n",
      "\n",
      "Psuedo R2: -0.06263979911720452\n",
      "\n",
      "Explained Variance: -0.06263894423677474\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid.append(train_model_print_metrics(param3, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 49647262.613002636\n",
      "\n",
      "Psuedo R2: 0.004227272565601736\n",
      "\n",
      "Explained Variance: 0.004227317994338731\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid.append(train_model_print_metrics(param4, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 49552962.32314937\n",
      "\n",
      "Psuedo R2: 0.00800643271341217\n",
      "\n",
      "Explained Variance: 0.008006571298355691\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid.append(train_model_print_metrics(param5, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 49609265.143288925\n",
      "\n",
      "Psuedo R2: 0.005750916080727642\n",
      "\n",
      "Explained Variance: 0.005751306386997945\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid.append(train_model_print_metrics(param6, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 50926370.31321675\n",
      "\n",
      "Psuedo R2: -0.047743702266721755\n",
      "\n",
      "Explained Variance: -0.047743459423267964\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid.append(train_model_print_metrics(param7, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 50957469.26743807\n",
      "\n",
      "Psuedo R2: -0.049023733892990196\n",
      "\n",
      "Explained Variance: -0.0490227561846075\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid.append(train_model_print_metrics(param8, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 51287110.642113\n",
      "\n",
      "Psuedo R2: -0.06263979911720452\n",
      "\n",
      "Explained Variance: -0.06263894423677474\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid.append(train_model_print_metrics(param9, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 49647262.613002636\n",
      "\n",
      "Psuedo R2: 0.004227272565601736\n",
      "\n",
      "Explained Variance: 0.004227317994338731\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid.append(train_model_print_metrics(param10, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 49552962.32314937\n",
      "\n",
      "Psuedo R2: 0.00800643271341217\n",
      "\n",
      "Explained Variance: 0.008006571298355691\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid.append(train_model_print_metrics(param11, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 49609265.143288925\n",
      "\n",
      "Psuedo R2: 0.005750916080727642\n",
      "\n",
      "Explained Variance: 0.005751306386997945\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid.append(train_model_print_metrics(param12, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[50926370.31321675, -0.047743702266721755, -0.047743459423267964],\n",
       " [50957469.26743807, -0.049023733892990196, -0.0490227561846075],\n",
       " [51287110.642113, -0.06263979911720452, -0.06263894423677474],\n",
       " [49647262.613002636, 0.004227272565601736, 0.004227317994338731],\n",
       " [49552962.32314937, 0.00800643271341217, 0.008006571298355691],\n",
       " [49609265.143288925, 0.005750916080727642, 0.005751306386997945],\n",
       " [50926370.31321675, -0.047743702266721755, -0.047743459423267964],\n",
       " [50957469.26743807, -0.049023733892990196, -0.0490227561846075],\n",
       " [51287110.642113, -0.06263979911720452, -0.06263894423677474],\n",
       " [49647262.613002636, 0.004227272565601736, 0.004227317994338731],\n",
       " [49552962.32314937, 0.00800643271341217, 0.008006571298355691],\n",
       " [49609265.143288925, 0.005750916080727642, 0.005751306386997945]]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_grid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lowest RMSE: Parameters 5 and 11\n",
    "\n",
    "Highest Psuedo R2: Parameters 5 and 11\n",
    "\n",
    "Highest Explained Variance: Parameters 5 and 11"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I'm choosing p10 as my final model: \n",
    "\n",
    "num_leaves: 31\n",
    "\n",
    "learning_rate: 0.05\n",
    "\n",
    "max_bin: 300\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final model fit and metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 42661156.43881862\n",
      "\n",
      "Psuedo R2: -0.010275758822533287\n",
      "\n",
      "Explained Variance: -0.010273749115046638\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[42661156.43881862, -0.010275758822533287, -0.010273749115046638]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Fit final model, predict, metrics\n",
    "train_model_print_metrics(param11, train_data, X_test, y_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Overall, this model is struggling - the negative explained variance / r2 is an indicater that this is actually doing worse than a mean-prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# User Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get data, split test and train\n",
    "raw_user_csv = pd.read_csv('../Data/train_user.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_classification_labels = raw_user_csv.iloc[:,1] #Unused here - regression only\n",
    "user_regression_labels = raw_user_csv.iloc[:,2]\n",
    "user_data = raw_user_csv.iloc[:,3:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Split test and train\n",
    "X_train_1, X_test, y_train_1, y_test = train_test_split(user_data, user_regression_labels, test_size=0.33)\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X_train_1, y_train_1, test_size=0.33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create lgb datasets\n",
    "train_data = lgb.Dataset(X_train, label=y_train)\n",
    "test_data = lgb.Dataset(X_test, label=y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Search Space:\n",
    "\n",
    "num_leaves : 31 (default), 50, 100\n",
    "\n",
    "learning_rate: 0.1 (default), 0.05\n",
    "\n",
    "max_bin: 255 (default), 300\n",
    "\n",
    "Total iterations required:  12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I will be using the same param dicts and helper function from the transactions data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_grid_user = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.6348454970703414\n",
      "\n",
      "Psuedo R2: 0.3858928666909015\n",
      "\n",
      "Explained Variance: 0.3858936012634171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid_user.append(train_model_print_metrics(param1, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.6375186034645695\n",
      "\n",
      "Psuedo R2: 0.3838829938784353\n",
      "\n",
      "Explained Variance: 0.3838832845184944\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid_user.append(train_model_print_metrics(param2, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.653283751369526\n",
      "\n",
      "Psuedo R2: 0.3719626012525894\n",
      "\n",
      "Explained Variance: 0.3719635483979076\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid_user.append(train_model_print_metrics(param3, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.6345452544390613\n",
      "\n",
      "Psuedo R2: 0.3861184099742774\n",
      "\n",
      "Explained Variance: 0.38611924565219224\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid_user.append(train_model_print_metrics(param4, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.6315890296656872\n",
      "\n",
      "Psuedo R2: 0.38833692410692333\n",
      "\n",
      "Explained Variance: 0.38833778614872605\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid_user.append(train_model_print_metrics(param5, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.633182221286418\n",
      "\n",
      "Psuedo R2: 0.3871418041950321\n",
      "\n",
      "Explained Variance: 0.3871427432530763\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid_user.append(train_model_print_metrics(param6, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.6348454970703414\n",
      "\n",
      "Psuedo R2: 0.3858928666909015\n",
      "\n",
      "Explained Variance: 0.3858936012634171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid_user.append(train_model_print_metrics(param7, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.6375186034645695\n",
      "\n",
      "Psuedo R2: 0.3838829938784353\n",
      "\n",
      "Explained Variance: 0.3838832845184944\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid_user.append(train_model_print_metrics(param8, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.653283751369526\n",
      "\n",
      "Psuedo R2: 0.3719626012525894\n",
      "\n",
      "Explained Variance: 0.3719635483979076\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid_user.append(train_model_print_metrics(param9, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.6345452544390613\n",
      "\n",
      "Psuedo R2: 0.3861184099742774\n",
      "\n",
      "Explained Variance: 0.38611924565219224\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid_user.append(train_model_print_metrics(param10, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.6315890296656872\n",
      "\n",
      "Psuedo R2: 0.38833692410692333\n",
      "\n",
      "Explained Variance: 0.38833778614872605\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid_user.append(train_model_print_metrics(param11, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.633182221286418\n",
      "\n",
      "Psuedo R2: 0.3871418041950321\n",
      "\n",
      "Explained Variance: 0.3871427432530763\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics_grid_user.append(train_model_print_metrics(param12, train_data, X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[1.6348454970703414, 0.3858928666909015, 0.3858936012634171],\n",
       " [1.6375186034645695, 0.3838829938784353, 0.3838832845184944],\n",
       " [1.653283751369526, 0.3719626012525894, 0.3719635483979076],\n",
       " [1.6345452544390613, 0.3861184099742774, 0.38611924565219224],\n",
       " [1.6315890296656872, 0.38833692410692333, 0.38833778614872605],\n",
       " [1.633182221286418, 0.3871418041950321, 0.3871427432530763],\n",
       " [1.6348454970703414, 0.3858928666909015, 0.3858936012634171],\n",
       " [1.6375186034645695, 0.3838829938784353, 0.3838832845184944],\n",
       " [1.653283751369526, 0.3719626012525894, 0.3719635483979076],\n",
       " [1.6345452544390613, 0.3861184099742774, 0.38611924565219224],\n",
       " [1.6315890296656872, 0.38833692410692333, 0.38833778614872605],\n",
       " [1.633182221286418, 0.3871418041950321, 0.3871427432530763]]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_grid_user"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lowest RMSE: Parameters 5 and 11\n",
    "\n",
    "Highest Psuedo R2: Parameters 5 and 11\n",
    "\n",
    "Highest Explained Variance: Parameters 5 and 11"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final model fit and metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 1.6340376865887687\n",
      "\n",
      "Psuedo R2: 0.399714042176322\n",
      "\n",
      "Explained Variance: 0.3997148914892392\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.6340376865887687, 0.399714042176322, 0.3997148914892392]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Fit final model, predict, metrics\n",
    "train_model_print_metrics(param11, train_data, X_test, y_test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This model is much better - very low RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = lgb.train(param11, train_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('pageviews_mean', 893),\n",
       " ('hits_mean', 601),\n",
       " ('next_session_1_adjmean', 459),\n",
       " ('Americas_freq', 247),\n",
       " ('h_13_17_rate', 165),\n",
       " ('is_source_googleplex_freq', 136),\n",
       " ('h_18_23_rate', 134),\n",
       " ('is_mobile_freq', 121),\n",
       " ('visitNumber_total', 119),\n",
       " ('q2_rate', 112),\n",
       " ('h_0_6_rate', 112),\n",
       " ('q2_freq', 110),\n",
       " ('is_bayarea_freq', 106),\n",
       " ('browser_chrome_freq', 106),\n",
       " ('is_source_googleplex_rate', 103),\n",
       " ('weekday_freq', 96),\n",
       " ('h_13_17_freq', 91),\n",
       " ('weekday_rate', 90),\n",
       " ('q4_freq', 85),\n",
       " ('q3_rate', 80),\n",
       " ('bounces_mean', 75),\n",
       " ('is_source_direct_rate', 73),\n",
       " ('h_18_23_freq', 68),\n",
       " ('Americas_rate', 64),\n",
       " ('is_medium_organic_rate', 61),\n",
       " ('q3_freq', 59),\n",
       " ('q1_rate', 58),\n",
       " ('h_7_12_rate', 58),\n",
       " ('is_medium_organic_freq', 52),\n",
       " ('is_source_direct_freq', 50),\n",
       " ('is_bayarea_rate', 46),\n",
       " ('q4_rate', 46),\n",
       " ('is_medium_referral_freq', 45),\n",
       " ('q1_freq', 44),\n",
       " ('is_medium_referral_rate', 42),\n",
       " ('h_0_6_freq', 38),\n",
       " ('system_google_freq', 37),\n",
       " ('h_7_12_freq', 11),\n",
       " ('browser_chrome_rate', 3),\n",
       " ('Asia_freq', 2),\n",
       " ('Europe_freq', 1),\n",
       " ('is_mobile_rate', 1),\n",
       " ('is_source_google_freq', 0),\n",
       " ('is_source_google_rate', 0),\n",
       " ('is_medium_none_freq', 0),\n",
       " ('is_medium_none_rate', 0),\n",
       " ('Africa_freq', 0),\n",
       " ('Africa_rate', 0),\n",
       " ('Asia_rate', 0),\n",
       " ('Europe_rate', 0),\n",
       " ('Oceania_freq', 0),\n",
       " ('Oceania_rate', 0),\n",
       " ('last_session_1_adjmean', 0),\n",
       " ('system_google_rate', 0)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = dict(zip(X_train.columns.values,model.feature_importance()))\n",
    "sorted(d.items(), key=lambda kv: -kv[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}