14_optymalizacja_hiperparametrów.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "from warnings import filterwarnings\n",
    "filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "column_names = ['age', 'workclass', 'fnlwgt', 'education', 'education_num', \n",
    "                'marital_status', 'occupation', 'relationship', 'race', \n",
    "                'sex', 'capital_gain', 'capital_loss', 'hours_per_week', \n",
    "                'native_country', 'wage_class']\n",
    "\n",
    "data = pd.read_csv('data/adult.data', \n",
    "                    sep = \",\",\n",
    "                    skipinitialspace=True,  \n",
    "                    names = column_names).iloc[::3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education_num</th>\n",
       "      <th>marital_status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital_gain</th>\n",
       "      <th>capital_loss</th>\n",
       "      <th>hours_per_week</th>\n",
       "      <th>native_country</th>\n",
       "      <th>wage_class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>77516</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>2174</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>Private</td>\n",
       "      <td>234721</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>49</td>\n",
       "      <td>Private</td>\n",
       "      <td>160187</td>\n",
       "      <td>9th</td>\n",
       "      <td>5</td>\n",
       "      <td>Married-spouse-absent</td>\n",
       "      <td>Other-service</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>Black</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>Jamaica</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>42</td>\n",
       "      <td>Private</td>\n",
       "      <td>159449</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>5178</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>23</td>\n",
       "      <td>Private</td>\n",
       "      <td>122272</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age  workclass  fnlwgt  education  education_num         marital_status  \\\n",
       "0    39  State-gov   77516  Bachelors             13          Never-married   \n",
       "3    53    Private  234721       11th              7     Married-civ-spouse   \n",
       "6    49    Private  160187        9th              5  Married-spouse-absent   \n",
       "9    42    Private  159449  Bachelors             13     Married-civ-spouse   \n",
       "12   23    Private  122272  Bachelors             13          Never-married   \n",
       "\n",
       "           occupation   relationship   race     sex  capital_gain  \\\n",
       "0        Adm-clerical  Not-in-family  White    Male          2174   \n",
       "3   Handlers-cleaners        Husband  Black    Male             0   \n",
       "6       Other-service  Not-in-family  Black  Female             0   \n",
       "9     Exec-managerial        Husband  White    Male          5178   \n",
       "12       Adm-clerical      Own-child  White  Female             0   \n",
       "\n",
       "    capital_loss  hours_per_week native_country wage_class  \n",
       "0              0              40  United-States      <=50K  \n",
       "3              0              40  United-States      <=50K  \n",
       "6              0              16        Jamaica      <=50K  \n",
       "9              0              40  United-States       >50K  \n",
       "12             0              30  United-States      <=50K  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10854, 50)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = data.drop([\"fnlwgt\", \"education\", \"wage_class\"], axis=1)\n",
    "X[\"immigrant\"] = (X.native_country == \"United-States\").astype(int)\n",
    "X = X.drop([\"native_country\"], axis=1)\n",
    "X = pd.get_dummies(X)\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "y = (data.wage_class == \">50K\").astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.metrics import roc_auc_score, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, \n",
    "                                                    random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "from xgboost.sklearn import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "?RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "from scipy.stats import uniform, randint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "param_dists = {\"n_estimators\": randint(20, 100), # liczba calkowita z [20, 100]\n",
    "               \"learning_rate\": uniform(0.01, 0.3), # zakres [0.01, 0.31]\n",
    "               \"max_depth\": [5,6,7],  # losowa (jednostajnie) wartosc z listy\n",
    "               \"min_child_weight\": randint(1, 10),\n",
    "               \"subsample\": [0.7, 0.5, 0.3],\n",
    "               \"colsample_bytree\": [0.7, 0.5, 0.3],\n",
    "               \"gamma\": uniform(0.01, 0.2),\n",
    "               \"reg_lambda\": uniform(0.01, 0.2)\n",
    "               }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[11:51:56] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:51:57] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:51:58] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:51:58] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:51:59] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:51:59] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:51:59] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:00] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:00] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:00] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:01] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:01] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:02] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:02] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:03] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:03] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:04] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:04] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:04] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:05] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:05] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:05] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:06] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:07] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:08] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:08] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:08] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:08] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:09] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:10] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:11] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:11] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:12] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[11:52:12] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:13] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:13] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:13] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:14] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:14] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:14] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:15] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:15] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:15] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:16] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:16] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:17] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:18] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:18] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:19] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:19] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:20] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:20] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:21] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:21] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:22] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:22] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:23] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:23] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:23] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:24] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[11:52:24] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=3,\n",
       "                   estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                           colsample_bylevel=None,\n",
       "                                           colsample_bynode=None,\n",
       "                                           colsample_bytree=None,\n",
       "                                           enable_categorical=False, gamma=None,\n",
       "                                           gpu_id=None, importance_type=None,\n",
       "                                           interaction_constraints=None,\n",
       "                                           learning_rate=None,\n",
       "                                           max_delta_step=None, max_depth=None,\n",
       "                                           min_child_weight=None, missing=nan,\n",
       "                                           monotone_constraints...\n",
       "                                        'learning_rate': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7fa228219310>,\n",
       "                                        'max_depth': [5, 6, 7],\n",
       "                                        'min_child_weight': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7fa1a6359370>,\n",
       "                                        'n_estimators': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7fa1a6352f10>,\n",
       "                                        'reg_lambda': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7fa1a6346a60>,\n",
       "                                        'subsample': [0.7, 0.5, 0.3]},\n",
       "                   scoring='roc_auc')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = XGBClassifier()\n",
    "optimizer = RandomizedSearchCV(model, param_dists, \n",
    "                               n_iter=20, \n",
    "                               scoring=\"roc_auc\", \n",
    "                               cv=3)\n",
    "optimizer.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'colsample_bytree': 0.7,\n",
       "  'gamma': 0.1491700995250253,\n",
       "  'learning_rate': 0.06879220971662639,\n",
       "  'max_depth': 7,\n",
       "  'min_child_weight': 6,\n",
       "  'n_estimators': 88,\n",
       "  'reg_lambda': 0.1169789722877478,\n",
       "  'subsample': 0.7},\n",
       " {'colsample_bytree': 0.3,\n",
       "  'gamma': 0.18290713957656893,\n",
       "  'learning_rate': 0.05924442053700988,\n",
       "  'max_depth': 5,\n",
       "  'min_child_weight': 1,\n",
       "  'n_estimators': 48,\n",
       "  'reg_lambda': 0.08992823195482595,\n",
       "  'subsample': 0.7},\n",
       " {'colsample_bytree': 0.7,\n",
       "  'gamma': 0.023640190542817133,\n",
       "  'learning_rate': 0.21487878746051778,\n",
       "  'max_depth': 7,\n",
       "  'min_child_weight': 4,\n",
       "  'n_estimators': 36,\n",
       "  'reg_lambda': 0.12072941987718716,\n",
       "  'subsample': 0.3},\n",
       " {'colsample_bytree': 0.3,\n",
       "  'gamma': 0.04708443127386561,\n",
       "  'learning_rate': 0.06227904243308058,\n",
       "  'max_depth': 6,\n",
       "  'min_child_weight': 6,\n",
       "  'n_estimators': 94,\n",
       "  'reg_lambda': 0.17454354654949725,\n",
       "  'subsample': 0.5},\n",
       " {'colsample_bytree': 0.5,\n",
       "  'gamma': 0.03652695447453246,\n",
       "  'learning_rate': 0.0291260696132888,\n",
       "  'max_depth': 6,\n",
       "  'min_child_weight': 1,\n",
       "  'n_estimators': 59,\n",
       "  'reg_lambda': 0.1342909204147211,\n",
       "  'subsample': 0.5},\n",
       " {'colsample_bytree': 0.5,\n",
       "  'gamma': 0.08194150789431597,\n",
       "  'learning_rate': 0.07137275054636406,\n",
       "  'max_depth': 6,\n",
       "  'min_child_weight': 2,\n",
       "  'n_estimators': 30,\n",
       "  'reg_lambda': 0.038683693636920695,\n",
       "  'subsample': 0.3},\n",
       " {'colsample_bytree': 0.3,\n",
       "  'gamma': 0.06669172443112963,\n",
       "  'learning_rate': 0.13939985253392606,\n",
       "  'max_depth': 7,\n",
       "  'min_child_weight': 2,\n",
       "  'n_estimators': 60,\n",
       "  'reg_lambda': 0.0785023546582719,\n",
       "  'subsample': 0.7},\n",
       " {'colsample_bytree': 0.7,\n",
       "  'gamma': 0.1630383035454983,\n",
       "  'learning_rate': 0.15322024868067557,\n",
       "  'max_depth': 5,\n",
       "  'min_child_weight': 4,\n",
       "  'n_estimators': 89,\n",
       "  'reg_lambda': 0.02282428549388657,\n",
       "  'subsample': 0.7},\n",
       " {'colsample_bytree': 0.3,\n",
       "  'gamma': 0.09142809967698501,\n",
       "  'learning_rate': 0.11840975917649257,\n",
       "  'max_depth': 5,\n",
       "  'min_child_weight': 9,\n",
       "  'n_estimators': 40,\n",
       "  'reg_lambda': 0.15862672010915935,\n",
       "  'subsample': 0.5},\n",
       " {'colsample_bytree': 0.7,\n",
       "  'gamma': 0.08014835228944148,\n",
       "  'learning_rate': 0.2401340193165167,\n",
       "  'max_depth': 7,\n",
       "  'min_child_weight': 9,\n",
       "  'n_estimators': 91,\n",
       "  'reg_lambda': 0.045172784642837555,\n",
       "  'subsample': 0.5},\n",
       " {'colsample_bytree': 0.3,\n",
       "  'gamma': 0.14788999218399498,\n",
       "  'learning_rate': 0.04425825549314355,\n",
       "  'max_depth': 5,\n",
       "  'min_child_weight': 2,\n",
       "  'n_estimators': 72,\n",
       "  'reg_lambda': 0.1659460536055186,\n",
       "  'subsample': 0.3},\n",
       " {'colsample_bytree': 0.5,\n",
       "  'gamma': 0.09490698000052177,\n",
       "  'learning_rate': 0.05599591197226004,\n",
       "  'max_depth': 6,\n",
       "  'min_child_weight': 2,\n",
       "  'n_estimators': 39,\n",
       "  'reg_lambda': 0.1835962806275857,\n",
       "  'subsample': 0.5},\n",
       " {'colsample_bytree': 0.3,\n",
       "  'gamma': 0.023663287024658447,\n",
       "  'learning_rate': 0.2821900865771332,\n",
       "  'max_depth': 5,\n",
       "  'min_child_weight': 4,\n",
       "  'n_estimators': 63,\n",
       "  'reg_lambda': 0.0686768823336895,\n",
       "  'subsample': 0.5},\n",
       " {'colsample_bytree': 0.3,\n",
       "  'gamma': 0.09757971417156358,\n",
       "  'learning_rate': 0.1523863274419754,\n",
       "  'max_depth': 5,\n",
       "  'min_child_weight': 3,\n",
       "  'n_estimators': 49,\n",
       "  'reg_lambda': 0.06399588513412348,\n",
       "  'subsample': 0.5},\n",
       " {'colsample_bytree': 0.5,\n",
       "  'gamma': 0.029474875214659225,\n",
       "  'learning_rate': 0.16198139318651752,\n",
       "  'max_depth': 7,\n",
       "  'min_child_weight': 5,\n",
       "  'n_estimators': 84,\n",
       "  'reg_lambda': 0.012435186714769522,\n",
       "  'subsample': 0.3},\n",
       " {'colsample_bytree': 0.7,\n",
       "  'gamma': 0.0765420396132421,\n",
       "  'learning_rate': 0.048615371123999745,\n",
       "  'max_depth': 6,\n",
       "  'min_child_weight': 2,\n",
       "  'n_estimators': 82,\n",
       "  'reg_lambda': 0.1981416581524257,\n",
       "  'subsample': 0.5},\n",
       " {'colsample_bytree': 0.3,\n",
       "  'gamma': 0.17310515562336273,\n",
       "  'learning_rate': 0.058339730184185784,\n",
       "  'max_depth': 7,\n",
       "  'min_child_weight': 9,\n",
       "  'n_estimators': 51,\n",
       "  'reg_lambda': 0.20961259442383529,\n",
       "  'subsample': 0.5},\n",
       " {'colsample_bytree': 0.7,\n",
       "  'gamma': 0.15774514361991845,\n",
       "  'learning_rate': 0.21873823234384385,\n",
       "  'max_depth': 5,\n",
       "  'min_child_weight': 7,\n",
       "  'n_estimators': 73,\n",
       "  'reg_lambda': 0.10412100045159416,\n",
       "  'subsample': 0.7},\n",
       " {'colsample_bytree': 0.5,\n",
       "  'gamma': 0.06710017749305285,\n",
       "  'learning_rate': 0.057771707700586705,\n",
       "  'max_depth': 6,\n",
       "  'min_child_weight': 3,\n",
       "  'n_estimators': 43,\n",
       "  'reg_lambda': 0.1562655060967643,\n",
       "  'subsample': 0.5},\n",
       " {'colsample_bytree': 0.5,\n",
       "  'gamma': 0.06892668796384402,\n",
       "  'learning_rate': 0.0948119752936468,\n",
       "  'max_depth': 5,\n",
       "  'min_child_weight': 4,\n",
       "  'n_estimators': 64,\n",
       "  'reg_lambda': 0.1847436101371385,\n",
       "  'subsample': 0.3}]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.cv_results_[\"params\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.7,\n",
       " 'gamma': 0.1630383035454983,\n",
       " 'learning_rate': 0.15322024868067557,\n",
       " 'max_depth': 5,\n",
       " 'min_child_weight': 4,\n",
       " 'n_estimators': 89,\n",
       " 'reg_lambda': 0.02282428549388657,\n",
       " 'subsample': 0.7}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Część walidacyjna zamiast kroswalidacji."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(array([3, 1, 0]), array([2, 4]))]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import ShuffleSplit\n",
    "\n",
    "x = [1,2,3,4,5]\n",
    "cv = ShuffleSplit(1, test_size=0.4)\n",
    "list(cv.split(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "param_dists = {\"n_estimators\": randint(10, 20), \n",
    "               \"learning_rate\": uniform(0.01, 0.3),\n",
    "               \"max_depth\": [5,6,7],\n",
    "               \"min_child_weight\": randint(1, 10),\n",
    "               \"subsample\": [0.7, 0.5, 0.3],\n",
    "               \"colsample_bytree\": [0.7, 0.5, 0.3],\n",
    "               \"gamma\": uniform(0.01, 0.2),\n",
    "               \"reg_lambda\": uniform(0.01, 0.2)\n",
    "               }\n",
    "model = XGBClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "cv = ShuffleSplit(1, test_size=0.1)\n",
    "\n",
    "optimizer = RandomizedSearchCV(model, param_dists, \n",
    "                               n_iter=2, scoring=\"roc_auc\", \n",
    "                               cv=cv)\n",
    "optimizer.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}