Gradient_boosting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  \n",
       "0     15.3  396.90   4.98  \n",
       "1     17.8  396.90   9.14  \n",
       "2     17.8  392.83   4.03  \n",
       "3     18.7  394.63   2.94  \n",
       "4     18.7  396.90   5.33  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from numpy import mean\n",
    "from numpy import std\n",
    "\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, RepeatedStratifiedKFold ,RepeatedKFold\n",
    "from sklearn.metrics import mean_absolute_error,mean_squared_error,r2_score\n",
    "from sklearn.datasets import load_boston\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "import numpy as np\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2, f_regression\n",
    "\n",
    "bd=load_boston()\n",
    "X=pd.DataFrame(bd.data,columns=bd.feature_names)\n",
    "y=pd.Series(bd.target)\n",
    "\n",
    "X.head()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "https://www.youtube.com/watch?v=b7JuBsswDlo&ab_channel=AppliedAICourse \n",
    "corr_matrix = np.corrcoef(x.T)\n",
    "pd.DataFrame(corr_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAF1CAYAAADx1LGMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZKElEQVR4nO3dfZxcdXn38e+X7IaEJCTh4Q4PCQQVo4CKklYERCtYI6VAe1uBijdRMIoVULGU1GqpomC1FnvbVzQCBgrFB4RbsBCNSEopD5pERCAICOQBEhJACJBIsuTqH3MWx2V3Z3dmzhmuvT/v12tfO+fMmd91nTOz35z5zUzGESEAQD7bdLoBAEBzCHAASIoAB4CkCHAASIoAB4CkCHAASIoARxq2T7H9qO1nbO/Y6X562T7b9qUt3P4u229tX0edY/s62yd2uo//XxDgL3G2H7J9eJ91s23f1Kbxw/Yr2jFWmWx3S/qypD+OiPER8Xine2qG7QW2z6lfFxH7RsTiDrU0JEP9Ryoi3hkRF1fREwhw5DFF0hhJd5VZxHZXP+tGlVlzJHANeVIxDvgIYHs329+zvd72g7ZPq7vuD23fYvtJ22tsf9X26OK6G4vNflFMSxxr+622V9s+0/a64jbH2D7C9r22n7D9t0MZv7g+bJ9m+wHbj9n+4kB/6La3tX2+7UeKn/OLda+U9Ktisydt/2SA2x9i++ail1W2ZxfrJ9q+pDg+K2z/XW8PxbOZ/7b9z7Yfl3R2cZY8z/a1tp+V9EeDHeN++viu7bW2n7J9o+19i/VzJL1H0pnF8b6mWP/Cs6yBjkFxXe99c0bdffO+QfpYbPuc4pg8Y/sa2zvavsz2Bts/sz29bvuvFMdtg+2ltt9crJ8l6W8lHVuM84u68T9n+78lbZT0smLdycX182x/r278L9i+3rYH6hnDFBH8vIR/JD0k6fA+62ZLuqm4vI2kpZI+LWm0pJdJekDSO4rrD5B0oKQuSdMlLZf00bqxQtIr6pbfKqmnGK9b0gckrZf075ImSNpX0iZJew1j/Bsk7SBpD0n3Sjp5gH39jKRbJf0vSTtLulnSZ4vrphdjdQ1w2z0lPS3p+KLvHSXtX1x3iaTvF/1PL3o4qe5Y9kg6tdiHsZIWSHpK0sHF8d2uwTE+W9Kldb28v6i1raTzJd1ed90CSecMdB83OAa9981nin08QrXgnDzAMVks6X5JL5c0UdLdxb4fXuzrJZK+Wbf9CcVx65J0hqS1ksb0t491468sHhNdRU+Le+/f4rjdWxzjN0t6TNLUTv9NjaSfjjfAT4M7qPbH/YykJ+t+Nup3Af5GSSv73GZu/R9mn+s+KumquuX+AnyTpFHF8oRimzfWbbNU0jHDGH9W3fKHJV0/wG1/LemIuuV3SHqouDxdgwf43Pq6detHSdosaZ+6dR+UtLi4PLuf47dA0iV1y4Me4/7CrW67SUXfE+vGHizABzsGvfdNV9316yQdOEDtxZI+Wbf8T5Kuq1v+U9X949LP7X8j6XUD7WMx/mf6WXdy3fIbJT0haYWk46v++xnpPy+a78NL0jER8ePehWJq4ORicU9Ju9l+sm77UZL+q9j2laq9+DdTtTOiLtUCeDCPR8TzxeVNxe9H667fJGn8MMZfVXd5haTdBqi7W3H9ULbta5pq4dfXTqqdGfYdd/cB+utv3aDHuJ5r8+Wfk/QXqp1Bb63r46lB96Cm0TF4PCJ66pY3qrgvBtD3fuv3fix6/4Skk4p6IWn7ou/B9HfsXhARt9l+QLVnFN9pMBaGiTnw/FZJejAiJtX9TIiII4rr50m6R9LeEbG9anOZ7ZyDHMr40+ou7yHpkQHGekS1sBzKtn2tUm2qoK/HJG3pZ9yH65b7+y8569c1Osb1/lLS0apNU0xU7ZmD9Ltj0ui//2zlGDStmO8+U9K7VZuSmaTaPziN+h50f2z/lWpTSY8U46ONCPD8firpadt/Y3us7VG297P9B8X1EyRtkPSM7VdJOqXP7R9VbU63WY3Gl6S/tj3Z9jRJp0v69gBjXS7p72zvbHsn1each/r+6sskHW773ba7ihfr9i+eSXxH0udsT7C9p6SPD2NcqfExrjdB0nOSHlftGcnn+1zf6Hi3cgxaMUG1+fX1krpsf1q1M/Bej0qaPtAL0P0pnp2do9rc+ntVe/F2/7Z1DAI8uyKgjpS0v6QHVTvjvEC1sz9J+oRqZ4VPS/qGXhyeZ0u6uHjnxrubaKHR+FLtBcSlkm6X9B+SLhxgrHMkLZF0h6RfSlpWrGsoIlaq9qLeGarNud4u6XXF1adKela1Fx5vUu0F2YuGMm4xdqNjXO8S1aY9HlbtRcNb+1x/oaR9iuP9//q5fdPHoEU/lLRQtRcdV0j6rX5/euS7xe/HbS9rNJhrb8e8VNIXIuIXEXGfas/O/q33XTVonYsXGoBS2A7Vplfu73QvwEjDGTgAJEWAA0BSTKEAQFKcgQNAUgQ4ACRV6ScxuyeOjTG79PfOq/bxmmp2aczUTY03aoOnf7Nd6TXGT95Yeg1J2vzrau6bbV/e03ijNvjtijGl1/C0avaly1sbb9QGm3q6K6kzan3556Yz9nys9Bq9lt7x3GMRsXPf9ZUG+JhdJmrmvPeUWsPnvmgfS7HvF35ZSZ3rr5hZeo1D/3fDt/W2xYp3VXPfvOLSRxtv1Aa/+tCM0mt0f6makJi0bTUnJHeu36WSOpO+vn3jjVp0wze+UXqNXqN2vW9Ff+uZQgGApAhwAEiKAAeApAhwAEiKAAeApAhwAEiKAAeApAhwAEiKAAeApAhwAEiKAAeApAhwAEiKAAeApAhwAEiKAAeApAhwAEiKAAeApAhwAEiKAAeApBoGuO2LbK+zfWfdui/avsf2Hbavsj2p1C4BAC8ylDPwBZJm9Vm3SNJ+EfFaSfdKmtvmvgAADTQM8Ii4UdITfdb9KCJ6isVbJU0toTcAwCDaMQf+fknXtWEcAMAwdLVyY9uflNQj6bJBtpkjaY4kbTtmkuKLU1op2dCD76rmddmes/evpM6Wg8qvsfxTrym/iKSVHx9VSZ3f/NPuldRZf8ZvS6+x16d2Kb2GJK11JWU06tVjK6lzwzfmlV7jjz7wgdJr/M6Z/a5tOu1sz5Z0pKT3REQMtF1EzI+ImRExs3v0uGbLAQD6aOoM3PYs1f5JeEtEbGxvSwCAoRjK2wgvl3SLpBm2V9s+SdJXJU2QtMj27ba/VnKfAIA+Gp6BR8Tx/ay+sIReAADDwCcxASApAhwAkiLAASApAhwAkiLAASApAhwAkiLAASApAhwAkiLAASApAhwAkiLAASApAhwAkiLAASApAhwAkiLAASApAhwAkiLAASApAhwAkiLAASApAhwAkmr4pcbttHmn0KrZW0qtMf2b1ezSk6c+U0mdsT/eofQaK/7MpdeQpD2u2VpJnVXH9lRSZ6eFY0uvEWevLr2GJG2Nah4DS/a5ppI6M//hlNJrPP/BJ0qv8YIf9L+aM3AASIoAB4CkCHAASIoAB4CkCHAASIoAB4CkCHAASIoAB4CkCHAASIoAB4CkCHAASIoAB4CkCHAASIoAB4CkCHAASIoAB4CkCHAASIoAB4CkCHAASIoAB4CkGga47Ytsr7N9Z926HWwvsn1f8XtyuW0CAPoayhn4Akmz+qw7S9L1EbG3pOuLZQBAhRoGeETcKOmJPquPlnRxcfliSce0ty0AQCPNzoFPiYg1xeW1kqa0qR8AwBB1tTpARITtGOh623MkzZGk8buM02Evv7fVkoO6a/xrSh2/15xX3FRJnf9761Gl13j/m/6r9BqSdN1P3lJJndfu+XAldZ5bVv5LP1/77OWl15CkPbomVFLnsLv/tJI6k5dvKr3GLrPXll6j1x0DrG/2DPxR27tKUvF73UAbRsT8iJgZETPHTB7TZDkAQF/NBvjVkk4sLp8o6fvtaQcAMFRDeRvh5ZJukTTD9mrbJ0k6T9Lbbd8n6fBiGQBQoYZz4BFx/ABXHdbmXgAAw8AnMQEgKQIcAJIiwAEgKQIcAJIiwAEgKQIcAJIiwAEgKQIcAJIiwAEgKQIcAJIiwAEgKQIcAJIiwAEgKQIcAJIiwAEgKQIcAJIiwAEgKQIcAJIiwAEgKQIcAJJq+KXG7TTKW7V992/LLWKXO37hzmd3r6ROVHAPPb5lXPlFJG0ZV819s2HztpXU6d62u/Qae3RNKL2GJK3sebqSOts4KqlTRZknnxtbfpEGOAMHgKQIcABIigAHgKQIcABIigAHgKQIcABIigAHgKQIcABIigAHgKQIcABIigAHgKQIcABIigAHgKQIcABIigAHgKQIcABIigAHgKQIcABIigAHgKQIcABIigAHgKRaCnDbH7N9l+07bV9ue0y7GgMADK7pALe9u6TTJM2MiP0kjZJ0XLsaAwAMrtUplC5JY213SdpO0iOttwQAGIqmAzwiHpb0JUkrJa2R9FRE/Kjvdrbn2F5ie8nG3zzXfKcAgN/T1ewNbU+WdLSkvSQ9Kem7tk+IiEvrt4uI+ZLmS9KEiVPjZ3MPaL7bIXjHP/5nqeP3uvmgHSup0/2B8mvcfUBP+UUkrb+gmjo7zFpTSZ2FK68pvcaso04ovYYkbfPclkrqbHjD5ErqPPHhTaXXePkndiq9RiOtTKEcLunBiFgfEVskXSnpoPa0BQBopJUAXynpQNvb2bakwyQtb09bAIBGWpkDv03SFZKWSfplMdb8NvUFAGig6TlwSYqIv5f0923qBQAwDHwSEwCSIsABICkCHACSIsABICkCHACSIsABICkCHACSIsABICkCHACSIsABICkCHACSIsABICkCHACSIsABICkCHACSIsABICkCHACSIsABICkCHACSIsABIKmWvtR4uHp23qrHPrix1Bq3/MW+pY7f64CbflVJnfsWRek1jr1nbek1JGn+56t5uC1cuaSSOrP2mFl6jS0Lnyq9hiRtP/q5Suo8tmK7SupMuXpM6TVmfG1Z6TV6LTqg//WcgQNAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACTVUoDbnmT7Ctv32F5u+03tagwAMLhWvyb8K5IWRsS7bI+WVM1XTgMAmg9w2xMlHSpptiRFxGZJm9vTFgCgkVamUPaStF7SN23/3PYFtsf13cj2HNtLbC95fsPGFsoBAOo5Ipq7oT1T0q2SDo6I22x/RdKGiPjUQLcZt9O0ePXRH2uu0yHa8M5nSx2/124XjK6kzqo/7i69xpRbm3sMDNdN//L1SuoceNaHKqnz6KE9pdfY+eZWZzmHpvvZah4D3c9uraTOqreX//6MV3354dJr9Fr40D8vjYiZfde3sperJa2OiNuK5SskvaGF8QAAw9B0gEfEWkmrbM8oVh0m6e62dAUAaKjV52enSrqseAfKA5Le13pLAIChaCnAI+J2SS+alwEAlI9PYgJAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACTV6rfSD4t36FH3setKrbHX3G1LHb/Xid/+QSV1PnX1saXXuOlfvl56DUk65LQPVlJnzjlXVVLn26/apfQa+yyt5k90x+5nK6lz0S1vrqTOnlc9X36NK9aXXuMFA3x1PGfgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASbUc4LZH2f657Wq+5RcAIKk9Z+CnS1rehnEAAMPQUoDbnirpTyRd0J52AABD1eoZ+PmSzpS0daANbM+xvcT2ki1PbWyxHACglyOiuRvaR0o6IiI+bPutkj4REUcOdpuxu06Lvd738abqDVlFL8tu3r6aOvf+n3ml19j73z9Ueo1KNfeQHrbtHnHpNbZU9DhzTzV1up+pps6mKeXX6N5Qfo1ey8/9+NKImNl3fStxd7Cko2w/JOlbkt5m+9IWxgMADEPTAR4RcyNiakRMl3ScpJ9ExAlt6wwAMCjeBw4ASXW1Y5CIWCxpcTvGAgAMDWfgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASbXlS42HapsJPRp/6PpSa+xwVjW7dO3Cb1VS55WXnFJ6jc/+eTX7Mv8jf15JnYP/8bZK6iw9ZGLpNV62eEvpNSRpv3EPV1Jn/v2HVFJn169vX3qN1519e+k1ei0/t//1nIEDQFIEOAAkRYADQFIEOAAkRYADQFIEOAAkRYADQFIEOAAkRYADQFIEOAAkRYADQFIEOAAkRYADQFIEOAAkRYADQFIEOAAkRYADQFIEOAAkRYADQFIEOAAk1XSA255m+wbbd9u+y/bp7WwMADC4Vr7CvUfSGRGxzPYESUttL4qIu9vUGwBgEE2fgUfEmohYVlx+WtJySbu3qzEAwODaMgdue7qk10u6rR3jAQAaa2UKRZJke7yk70n6aERs6Of6OZLmSNLo8ZP1/JU7t1pyUNcunFfq+L2OmHVcJXVGH1l+jYuPnVV+EUmPzN1cSZ1lx86opM5BN99Veo0f/sNbSq8hSb+I/SupM+mZ5yup89CfufQaXX/zmtJrNNLSGbjtbtXC+7KIuLK/bSJifkTMjIiZXWPGtVIOAFCnlXehWNKFkpZHxJfb1xIAYChaOQM/WNJ7Jb3N9u3FzxFt6gsA0EDTc+ARcZOk8ieaAAD94pOYAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASTX9pcbNePVu6/XTz8wrtcYffPqUUsfvNeq89ZXU0Y3ll9hw7nPlF5G0/bcnVlJn5XnPV1Jn45mHlF7jTef+tPQakrRhy5hK6lz/61dWUmf6gvK/bz3mVpQBkvTj/ldzBg4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJAUAQ4ASRHgAJBUSwFue5btX9m+3/ZZ7WoKANBY0wFue5Skf5X0Tkn7SDre9j7tagwAMLhWzsD/UNL9EfFARGyW9C1JR7enLQBAI60E+O6SVtUtry7WAQAq4Iho7ob2uyTNioiTi+X3SnpjRHykz3ZzJM0pFveTdGfz7b6k7CTpsU430UYjaX9G0r5II2t/RtK+SNXtz54RsXPflV0tDPiwpGl1y1OLdb8nIuZLmi9JtpdExMwWar5kjKR9kUbW/oykfZFG1v6MpH2ROr8/rUyh/EzS3rb3sj1a0nGSrm5PWwCARpo+A4+IHtsfkfRDSaMkXRQRd7WtMwDAoFqZQlFEXCvp2mHcZH4r9V5iRtK+SCNrf0bSvkgja39G0r5IHd6fpl/EBAB0Fh+lB4CkKgnwkfSRe9vTbN9g+27bd9k+vdM9tcr2KNs/t/2DTvfSKtuTbF9h+x7by22/qdM9Ncv2x4rH2J22L7c9ptM9DYfti2yvs31n3bodbC+yfV/xe3InexyOAfbni8Vj7Q7bV9meVGVPpQf4CPzIfY+kMyJiH0kHSvqr5PsjSadLWt7pJtrkK5IWRsSrJL1OSffL9u6STpM0MyL2U+2NAsd1tqthWyBpVp91Z0m6PiL2lnR9sZzFAr14fxZJ2i8iXivpXklzq2yoijPwEfWR+4hYExHListPqxYQaT+BanuqpD+RdEGne2mV7YmSDpV0oSRFxOaIeLKjTbWmS9JY212StpP0SIf7GZaIuFHSE31WHy3p4uLyxZKOqbKnVvS3PxHxo4joKRZvVe3zMJWpIsBH7EfubU+X9HpJt3W4lVacL+lMSVs73Ec77CVpvaRvFlNCF9ge1+mmmhERD0v6kqSVktZIeioiftTZrtpiSkSsKS6vlTSlk8202fslXVdlQV7EbJLt8ZK+J+mjEbGh0/00w/aRktZFxNJO99ImXZLeIGleRLxe0rPK9RT9BcXc8NGq/aO0m6Rxtk/obFftFbW3wI2It8HZ/qRq06uXVVm3igAf0kfuM7HdrVp4XxYRV3a6nxYcLOko2w+pNrX1NtuXdrallqyWtDoiep8RXaFaoGd0uKQHI2J9RGyRdKWkgzrcUzs8antXSSp+r+twPy2zPVvSkZLeExW/L7uKAB9RH7m3bdXmWJdHxJc73U8rImJuREyNiOmq3S8/iYi0Z3kRsVbSKtszilWHSbq7gy21YqWkA21vVzzmDlPSF2T7uFrSicXlEyV9v4O9tMz2LNWmII+KiI1V1y89wIsJ/t6P3C+X9J3kH7k/WNJ7VTtbvb34OaLTTeEFp0q6zPYdkvaX9PnOttOc4lnEFZKWSfqlan+rqT7FaPtySbdImmF7te2TJJ0n6e2271PtWcZ5nexxOAbYn69KmiBpUZEFX6u0Jz6JCQA58SImACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUgQ4ACRFgANAUv8DTHifuyEqEMwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(6, 6))\n",
    "\n",
    "ax.pcolor(corr_matrix)\n",
    "\n",
    "ax.set_title('Heatmap of correlation matrix')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "select = SelectKBest(score_func=f_regression, k=8)\n",
    "z = select.fit_transform(X, y) \n",
    "filter = select.get_support()\n",
    "features = np.array(bd.feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['CRIM' 'INDUS' 'NOX' 'RM' 'RAD' 'TAX' 'PTRATIO' 'LSTAT']\n"
     ]
    }
   ],
   "source": [
    "print(features[filter])\n",
    "X=X[features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=RepeatedKFold(n_repeats=3, n_splits=10, random_state=1),\n",
       "             error_score=nan,\n",
       "             estimator=GradientBoostingRegressor(alpha=0.9, ccp_alpha=0.0,\n",
       "                                                 criterion='friedman_mse',\n",
       "                                                 init=None, learning_rate=0.1,\n",
       "                                                 loss='ls', max_depth=3,\n",
       "                                                 max_features=None,\n",
       "                                                 max_leaf_nodes=None,\n",
       "                                                 min_impurity_decrease=0.0,\n",
       "                                                 min_impurity_split=None,\n",
       "                                                 min_samples_leaf=1,\n",
       "                                                 min_samples_split=2,\n",
       "                                                 mi...\n",
       "                                                 presort='deprecated',\n",
       "                                                 random_state=None,\n",
       "                                                 subsample=1.0, tol=0.0001,\n",
       "                                                 validation_fraction=0.1,\n",
       "                                                 verbose=0, warm_start=False),\n",
       "             iid='deprecated', n_jobs=None,\n",
       "             param_grid={'alpha': [0.15, 0.1, 0.08, 0.05, 0.02],\n",
       "                         'max_depth': [15, 10, 7, 8, 3],\n",
       "                         'min_samples_split': [2, 3, 4, 5, 10],\n",
       "                         'n_estimators': [3, 5, 10, 20, 50]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='r2', verbose=0)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "#scale=StandardScaler()\n",
    "#scaled_data = scale.fit_transform(data) \n",
    "#test_predict = scaler.inverse_transform(test_predict)\n",
    "x_train,x_test,y_train,y_test=train_test_split(X,y, test_size=0.2)\n",
    "\n",
    "search_space={}\n",
    "search_space[\"alpha\"]=[0.15,0.10,0.08,0.05,0.02]\n",
    "search_space[\"n_estimators\"]=[3,5,10,20,50]\n",
    "search_space[\"max_depth\"]=[15,10,7,8,3]\n",
    "search_space[\"min_samples_split\"]=[2,3,4,5,10]\n",
    "cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "tuner=GridSearchCV(estimator=GradientBoostingRegressor(),param_grid=search_space, scoring='r2', cv=cv)\n",
    "tuner.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.05, 'max_depth': 3, 'min_samples_split': 3, 'n_estimators': 50} 0.8645467316224792\n",
      "0.9102513121294137 6.441957906854423\n"
     ]
    }
   ],
   "source": [
    "print(tuner.best_params_, tuner.best_score_)\n",
    "bp=tuner.best_params_\n",
    "model =GradientBoostingRegressor(n_estimators=bp[\"n_estimators\"],max_depth=bp[\"max_depth\"], min_samples_split=bp[\"min_samples_split\"], alpha=bp[\"alpha\"])\n",
    "model.fit(x_train,y_train)\n",
    "y_predicted=model.predict(x_test)\n",
    "\n",
    "#coefficient of determination\n",
    "score=r2_score(y_predicted,y_test)\n",
    "residuals=mean_squared_error(y_predicted,y_test)\n",
    "print(score,residuals)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Feature Importance')"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEWCAYAAACaBstRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAeo0lEQVR4nO3deZwcVbn/8c+XsEMWIAgYQuaCYFiCAQa4euEKguKCghcUoiLx6i+CCMjidvUquCF6NT8UlRc/BQGVsP3UiApEICqyTkhCCBA0ISwJAZIoawQJz/2jTkul6Z6pSbq6ejLf9+s1L6rqnKp6pqHn4dTpPo8iAjMzsyLWqToAMzMbOJw0zMysMCcNMzMrzEnDzMwKc9IwM7PCnDTMzKwwJw0zMyvMScM6jqSFklZIeib38+oWXPPgVsVY4H5nSPpJu+7XG0kTJd1UdRy2dnDSsE71zojYNPezuMpgJK1b5f1X10CN2zqXk4YNGJKGS/qRpEclLZL0FUlDUtsOkm6QtEzSUkk/lTQitV0CbAf8Ko1aPiXpAEmP1F3/n6ORNFK4UtJPJD0FTOzt/gViD0kfk/RnSU9L+nKK+WZJT0m6XNL6qe8Bkh6R9F/pd1ko6f11r8PFkp6Q9KCkz0taJ7VNlPQnSZMlLQMuA84DXp9+97+lfu+QNDPd+2FJZ+Su35XiPVbSQymGz+Xah6TY5qffZYak0altrKRpkpZLmifpvf36l2wdz0nDBpIfAy8CrwH2AN4CfCS1CTgLeDWwMzAaOAMgIo4BHuLl0cs3Ct7vMOBKYATw0z7uX8QhwF7AvwKfAs4HPpBi3Q2YkOu7NTASGAUcC5wv6bWp7bvAcGB74I3AB4EP5c7dF1gAbJWufxxwS/rdR6Q+z6bzRgDvAI6XdHhdvPsBrwUOAr4gaed0/NQU69uBYcB/As9J2gSYBvwMeBVwNPB9SbsUf4ms0zlpWKf6haS/pZ9fSNqK7I/UJyLi2Yh4HJhM9oeJiPhLREyLiOcj4gng22R/UNfELRHxi4h4ieyPY9P7F/SNiHgqIuYCdwPXRcSCiHgS+C1ZIsr77/T7/B74NfDeNLI5GvhsRDwdEQuBbwHH5M5bHBHfjYgXI2JFo0AiYnpEzImIlyLiLuBSXvl6nRkRKyJiNjAbeF06/hHg8xExLzKzI2IZcCiwMCIuTPeeCVwFvKcfr5F1OD/vtE51eET8rrYjaR9gPeBRSbXD6wAPp/atgHOA/YGhqe2vaxjDw7ntMb3dv6DHctsrGuxvndv/a0Q8m9t/kGwUNTLF8WBd26gmcTckaV/g62QjnPWBDYAr6rotyW0/B2yatkcD8xtcdgywb+0RWLIucElf8djA4ZGGDRQPA88DIyNiRPoZFhG7pvavAQGMi4hhZI9llDu/fjnnZ4GNazvp/+C3rOuTP6ev+7faZulxT812wGJgKfAPsj/Q+bZFTeJutA/ZI6SpwOiIGE4276EG/Rp5GNihyfHf516fEemR2PEFr2sDgJOGDQgR8ShwHfAtScMkrZMmkmuPVIYCzwBPShoFfLLuEo+RzQHU3A9smCaE1wM+T/Z/26t7/zKcKWl9SfuTPfq5IiJWApcDX5U0VNIYsjmG3j7e+xiwbW2iPRkKLI+Iv6dR3Pv6EdcPgS9L2lGZ3SVtAVwN7CTpGEnrpZ+9c3MhthZw0rCB5INkj1LuIXv0dCWwTWo7E9gTeJLs+f//rzv3LODzaY7k9DSP8DGyP4CLyEYej9C73u7fakvSPRaTTcIfFxH3pbYTyeJdANxENmq4oJdr3QDMBZZIWpqOfQz4kqSngS+QJaKivp36Xwc8BfwI2Cginib7cMDRKe4lwNn0koxt4JGLMJl1FkkHAD+JiG0rDsXsFTzSMDOzwpw0zMysMD+eMjOzwjzSMDOzwtbqL/eNHDkyurq6qg7DzGxAmTFjxtKIqP/eErCWJ42uri56enqqDsPMbECR9GCzNj+eMjOzwpw0zMysMCcNMzMrbK2e05iz6Em6PvPrqsMwM2urhV9/R2nXbvlIQ9IzDY69VtJ0SbMk3SvpfEmHpP1ZqaLYvLR9cTrn8FQ9bGzavy21P5QqltXO7Wr172BmZo21a6TxHWByRPwSQNK4iJgDXJv2pwOnR0T+o04TyBZjmwB8MSL2TX0nAt0R8fE2xW5mZkm75jS2IbeCaEoYTUnalKzU5IfpX2U0MzMrUbuSxmTgBkm/lXSKpBF99D8MuCYi7geWSdqr6I0kTZLUI6ln5XNPrkHIZmZWry1JIyIuBHYmKyd5AHCrpN7W2J8ATEnbU9J+0XudHxHdEdE9ZOPhqxmxmZk10rZPT0XEYrJCMRdIupusNvGM+n6SNgfeBIyTFMAQICR9Mry6oplZpdoy0pD01lRSE0lbA1uwak3jvCOBSyJiTER0RcRo4AFg/3bEamZmzZUx0thYUr5s5reBbYFzJP09HftkRCxpcv4EshKReVel43/oTyDjRg2np8TPK5uZDTZrdT2N7u7u8IKFZmb9I2lGRHQ3avMyImZmVpiThpmZFeakYWZmhTlpmJlZYU4aZmZWmJOGmZkV5noaZmYNlFmTYiDrmJGGpJWpPsbdkn5VW9RQUleqq/GVXN+Rkv4h6dzKAjYzG4Q6JmkAKyJifETsBiwHTsi1PQDk0/57gLntDM7MzDoraeTdAozK7T8H3Cup9g3Fo4DL2x6Vmdkg13FJQ9IQ4CBgal3TFOBoSaOBlcDiJue7noaZWUk6KWlsJGkWsATYCphW134N8GaySn6XNbuI62mYmZWnk5LGiogYD4wBxKpzGkTEC2T1N04Drmx7dGZm1lFJA4CIeA44CThNUv1Hgr8FfDoilrc/MjMz67ikARARM4G7qCvzGhFzI+KiaqIyMzPX0zAzs1W4noaZmbWEk4aZmRXmpGFmZoU5aZiZWWFOGmZmVpiThpmZFeakYWZmhbkI0yDiojJmtqY6YqSRK8A0V9JsSadJWie1HSDp6rS9laSrU597JP2m2sjNzAaXThlp1BYrRNKrgJ8Bw4Av1vX7EjAtIs5JfXdvZ5BmZoNdR4w08iLicWAS8HFJqmveBngk1/eudsZmZjbYdVzSAIiIBcAQ4FV1Td8DfiTpRkmfk/Tq+nNdhMnMrDwdmTSaiYhrge2B/weMBWZK2rKuj4swmZmVpCOThqTtyUq6Pl7fFhHLI+JnEXEMcAfw7+2Oz8xssOq4pJFGDucB50bduu2S3iRp47Q9FNgBeKj9UZqZDU6d8umpWn3w9YAXgUuAbzfotxdwrqQXyRLeDyPijmYXHTdqOD3+boKZWct0RNKIiCG9tE0HpqftbwLfbE9UZmZWr+MeT5mZWedy0jAzs8KcNMzMrDAnDTMzK8xJw8zMCnPSMDOzwpw0zMyssI74nkZZOqUIk4sfmdnaou0jDUkh6Vu5/dMlnZHbnyTpvvRzu6T90vFTJV2Q6/d+SdVnBDOzQaSKx1PPA/8haWR9g6RDgY8C+0XEWOA44GeStga+A+wp6d8kjQC+ApzYvrDNzKyKpPEicD5wSoO2TwOfjIilABFxJ3ARcEJEvAh8jKymxjeAC1LdDTMza5OqJsK/B7xfUn3Bi12BGXXHetJxIuJm4F7gYLLE8QouwmRmVp5KkkZEPAVcDJzUn/MkbQp0k62Gu2WjPi7CZGZWnio/cvt/gQ8Dm+SO3UO2/HneXsDctH0m8BPgq8DkkuMzM7M6lSWNiFgOXE6WOGq+AZwtaQsASeOBicD3JY0D3gGcTTYn0iXpze2M2cxssKv6exrfAj5e24mIqZJGATdLCuBp4APAEuAK4JSI+DuApOOBiyWNj4gXGl3cRZjMzFqr7UkjIjbNbT8GbFzX/gPgBw1O3a+uXw+wSxkxmplZY15GxMzMCnPSMDOzwpw0zMysMCcNMzMrzEnDzMwKc9IwM7PCqv6eRqnaVU/D9TLMbLDoM2lIWgnMSX3vBT4B1P4Sbw2sBJ5I+/sAK3L9HwCOiYi/5a43C7gvIo6W9CHg5NS0CzAvXe8a4D6gOyI+ns6bBJya+j4FnBoRN/X3FzYzs9VX5PHUiogYHxG7AS8AR6X98cB5wOTafvpmdr7/cuCE2oUk7QwMAfaXtElEXJi71mLgwLT/mXwAfdTZMDOzNunvnMYfgdf0o/8twKjc/gTgEuA64LB+XKdpnY1+XMPMzNZQ4aQhaV3gbWSPnor0HwIcBEzNHT4KmAJcSpZAiuq1zoaZmbVHkaSxUZqH6AEeAn5UsP8SYCtgGoCkbmBpRDwEXA/sIWnz1Yy7KRdhMjMrT3/mNMZHxInNVpSt7w+MAcTLj5AmAGMlLQTmA8OAIwrG2VedjX9yESYzs/KU9j2NiHiOrDLfaZLWB94LjIuIrojoIpvTKPqIqmmdjRaHbWZmvSj1exoRMVPSXcBngUURsTjX/AdgF0nbRMSjfVynYZ2Nvs4zM7PWUkRUHUNpuru7o6enp+owzMwGFEkzIqK7UZuXETEzs8KcNMzMrDAnDTMzK8xJw8zMCnPSMDOzwpw0zMysMCcNMzMrzEWYGnBRJTOzxtoy0pC0taQpkuZLmiHpN5J2krRC0ixJ90i6WNJ6qf8Bkq5O2xMlhaSDc9c7PB07sh3xm5lZpvSkIUnAz4HpEbFDROxFtqzIVsD8tLjhOGBbsvWpGpkDHJ3bnwDMLi1oMzNrqB0jjQOBf0TEebUDETEbeDi3vxK4nVULNuX9EdhH0nqSNiUrBDWrtIjNzKyhdiSN3XhlAaVVSNoQ2JesNngjAfwOOIRsddypTfq5noaZWYmq/vTUDqlg02PAoxFxVy99p5A9ojqarPJfQ66nYWZWnnYkjbm8soBSTW1OYwdgL0nvanaRiLidbO5jZETc3/IozcysT+1IGjcAG0iaVDsgaXdgdG0/IpYCnyGbIO/NZ4D/KiNIMzPrW+lJI7KCHe8GDk4fuZ0LnEVWQzzvF8DGkvbv5Vq/jYgbSwvWzMx65SJMZma2ChdhMjOzlnDSMDOzwpw0zMysMCcNMzMrzEnDzMwKc9IwM7PCnDTMzKywQV+EyQWXzMyKq2SkIWmLVHxplqQlkhbl9l8l6R+Sjsv1H5q+Tb5j2l9P0hxJ+1YRv5nZYFVJ0oiIZRExPi1WeB4wObd/BHArWaGlWv+nydalOjcdOh24OSJua2vgZmaDXCfOaUwATgNGSdq2djAiLgeQ9CngOPpe3NDMzFqso5KGpNHANmkZ9MuBo+q6nAycDXwlIpY3uYaLMJmZlaSjkgZZkrg8bU8h94gqeSvwKFk1wIZchMnMrDydljQmABMlLSQr6bp7bvL71cBJwD7A21NNDjMza6OOSRqSdgI2jYhREdEVEV1kdTdqo43JwNci4hHgVOB7klRNtGZmg1MnfU9jAvDzumNXAZdJugXYDvgRQET8StL/AT4IXNTsguNGDafH38MwM2uZypNGRJzRS9tdwM5pd1pdW9N64mZmVo6OeTxlZmadz0nDzMwKc9IwM7PCnDTMzKwwJw0zMyvMScPMzApz0jAzs8Iq/55GmZoVYXLhJTOz1VPZSEPS4ZJC0tjcsX0kTZf0Z0l3Svq1pHGp7Yy6Yk2zJI2oKn4zs8GoypHGBOCm9M8vStqKbIXb90XEzQCS9gN2AOakcyZHxP9UEayZmVWUNCRtCuwHHAj8Cvgi8HHgolrCAIiIm6qIz8zMGqvq8dRhwDURcT+wTNJewK7AnX2cd0ru0dSNjTq4CJOZWXmqShoTyIosQeNiS0i6TdK9ks7JHf5nLfGIOLDRhV2EycysPG1/PCVpc+BNwDhJAQwBgmyJ8z2BXwJExL6SjgQObXeMZmbWWBUjjSOBSyJiTCq2NBp4gGzp84mS3pDru3EF8ZmZWRNVTIRPAM6uO3ZVOn4UcLakUcDjwFLgS7l+p0j6QG7/8IhY2OxGLsJkZtZaioiqYyhNd3d39PT0VB2GmdmAImlGRHQ3avMyImZmVpiThpmZFeakYWZmhTlpmJlZYU4aZmZWmJOGmZkV5noaZmZW2IAaaUhamRYrnJ3qbbyh77PMzKxVBtpIY0VEjAeQdAhwFvDGSiMyMxtEBtRIo84w4K9VB2FmNpgMtJHGRpJmARsC25CtlmtmZm0y0JJG/vHU64GLJe0WuQW0JE0CJgEMGbZlJUGama2tBuzjqYi4BRgJbFl33EWYzMxKMmCThqSxZAWcllUdi5nZYDHQHk/V5jQABBwbESsrjMfMbFAZUEkjIob0p7+LMJmZtdaAfTxlZmbt56RhZmaFOWmYmVlhThpmZlaYk4aZmRXmpGFmZoU5aZiZWWED6nsa/VVfhMnFl8zM1kzbRxq5Qkp3S/qVpBF17bMkTak79mNJD6TiS/dLuljStm0N3MzMKnk8tSIixkfEbsBy4IRag6SdydaT2l/SJnXnfTIiXge8FpgJ3CBp/XYFbWZm1c9p3AKMyu1PAC4BrgMOa3RCZCYDS4C3lR6hmZn9U2VJQ9IQ4CBgau7wUcAU4FKyBNKbO4GxDa47SVKPpJ6Vzz3ZqnDNzIxqkkZtpdolwFbANABJ3cDSiHgIuB7YQ9LmvVxHjQ66noaZWXkqm9MAxpD94a/NaUwAxkpaCMwnqwF+RC/X2QO4t7wwzcysXmWPpyLiOeAk4LQ0of1eYFxEdEVEF9mcxiseUSlzElmN8GvaGLKZ2aBX6fc0ImKmpLuAzwKLImJxrvkPwC6Stkn735T038DGwK3AgRHxQm/Xdz0NM7PWanvSiIhN6/bfmTbPrDu+Etg67U4sPzIzM+tL1R+5NTOzAcRJw8zMCnPSMDOzwpw0zMysMCcNMzMrzEnDzMwKc9IwM7PC1uqkUV+EyczM1kzLkoakZ9I/uySFpBNzbedKmpi2ey2oVLtObn+ipHPT9mslTU+Fmu6VdH6r4jczs76VNdJ4HDi5lyJJq1tQ6TvA5FTEaWfgu60J18zMiigraTxBtrz5sb11Wo2CStsAj+TOn7MmQZqZWf+UOadxNnB6KrbUl4YFlRqYTDYq+a2kU+rri4OLMJmZlam0pBERC4DbgPcV6N6woFL+cumaFwI7A1cABwC3Stqg7r4uwmRmVpKyPz31NeDT9J0U8gWVVtTNb2wOLK3tRMTiiLggIg4DXgR2a2G8ZmbWi1KTRkTcB9wDvLNRe5OCSr8HPpDaNyIrznRj2n+rpPXS9tbAFsCiMn8HMzN7WTu+p/FVYNu6Y9+UNBu4H9ibVQsqnQz8R6ojfitwRUT8IbW9Bbg7nXst2aewljS78bhRw1noIkxmZi2jiKg6htJ0d3dHT09P1WGYmQ0okmZERHejtrX6G+FmZtZaThpmZlaYk4aZmRXmpGFmZoU5aZiZWWFOGmZmVpiThpmZFbZWJ405i7xgoZlZK5WWNCRtLWmKpPmSZkj6jaSdJN1d1+8MSafn9teV9ISkr9f1O1TSzFS86R5JHy0rdjMza2zdMi4qScDPgYsi4uh07HXAVgVOfzPZ8iLvkfTZiIi03tT5wD4R8Uha2barjNjNzKy5skYaBwL/iIjzagciYjbwcIFzJwDnAA8Br0/HhpIluGXpWs9HxLyWRmxmZn0qZaRBtlz5jCZtO6TFCGu2Bv4HQNKGwMHAR4ERZAnk5ohYLmkq8KCk64GrgUsj4qX6i0uaBEwCGDJsy5b8MmZmlqliInx+qvE9PiLGA+fl2g4FboyIFcBVwOG1yn8R8RHgIOB24HTggkYXdxEmM7PylJU05gJ7rcZ5E4CDJS0kG6lsAbyp1hgRc1JN8TcDR7QgTjMz64eyksYNwAbpUREAknYHRjc7QdIwYH9gu4joiogu4ARggqRNJR2Q6z4eeLD1YZuZWW9KSRqRFel4N9moYb6kucBZQNOCSan/DRHxfO7YL8mq/g0BPiVpXpoPOROY2Fcc40b58ZSZWSu5CJOZma3CRZjMzKwlnDTMzKwwJw0zMyvMScPMzApz0jAzs8KcNMzMrLC1Omm4noaZWWt1VNKQ9G5Js+p+XpJ0vKSQdGKu77mSJlYYrpnZoNNRSSMifl63mOH3gT8C1wKPAydLWr/KGM3MBrOOShp5knYCvgAcA7wEPAFcDxxbZVxmZoNZRyaNVKnvZ8BpEfFQruls4PTaculNzp0kqUdSz8rnPKdhZtZKHZk0gC8DcyPisvzBiFgA3Aa8r9mJrqdhZlaesir3rba0BPoRwJ5NunwNuBL4fZtCMjOzpKNGGpI2Ay4EPhgRTzfqExH3AfeQLZluZmZt1GkjjeOAVwE/kJQ/fmldv68CM9sVlJmZZVxPw8zMVuF6GmZm1hJOGmZmVpiThpmZFbZWz2lIehqYV3UcDYwEllYdRAOOq38cV3GdGBM4rmbGRMSWjRo67dNTrTav2WROlST1OK7iHFf/dGJcnRgTOK7V4cdTZmZWmJOGmZkVtrYnjfOrDqAJx9U/jqt/OjGuTowJHFe/rdUT4WZm1lpr+0jDzMxayEnDzMwKG7BJQ9JbJc2T9BdJn2nQvoGky1L7bZK6cm2fTcfnSTqkE+KS1CVpRa42+nltjuvfJd0p6UVJR9a1HSvpz+mnZZUT1zCmlbnXamqrYioY16mS7pF0l6TrJY3JtZXyWrUgripfr+MkzUn3vknSLrm2Kt+LDeOq+r2Y63eEpJDUnTtW2utVWEQMuB9gCDAf2B5YH5gN7FLX52PAeWn7aOCytL1L6r8B8C/pOkM6IK4u4O4KX68uYHfgYuDI3PHNgQXpn5ul7c2qjCm1PVPha3UgsHHaPj7377CU12pN4+qA12tYbvtdwDVpu+r3YrO4Kn0vpn5DgT8AtwLdZb9e/fkZqCONfYC/RMSCiHgBmAIcVtfnMOCitH0lcJCy9dYPA6ZExPMR8QDwl3S9quMqU59xRcTCiLiLrB573iHAtIhYHhF/BaYBb604pjIVievGiHgu7d4KbJu2y3qt1jSuMhWJ66nc7iZA7dM3lb4Xe4mrTEX+RkBWvfRs4O+5Y2W+XoUN1KQxCng4t/9IOtawT0S8CDwJbFHw3CriAvgXSTMl/V7S/i2KqWhcZZxb5nU3VFYL/lZJh7cgntWN68PAb1fz3HbFBRW/XpJOkDQf+AZwUn/OrSAuqPC9KGlPYHRE/Lq/57bD2r6MyEDyKLBdRCyTtBfwC0m71v3fkL1sTEQskrQ9cIOkORExv50BSPoA0A28sZ337UuTuCp9vSLie8D3JL0P+DzQ0vme1dUkrsrei5LWAb4NTCz7XqtroI40FgGjc/vbpmMN+0haFxgOLCt4btvjSkPOZQARMYPseeVObYyrjHNLu25ELEr/XABMB/ZoQUyF45J0MPA54F0R8Xx/zq0grspfr5wpwOGreW5b4qr4vTgU2A2YLmkh8K/A1DQZXubrVVy7J1Fa8UM2QlpANhlUm0zata7PCaw64Xx52t6VVSeTFtC6ybc1iWvLWhxkk2SLgM3bFVeu74955UT4A2QTu5ul7TWOaw1j2gzYIG2PBP5Mg8nEEv8d7kH2h2THuuOlvFYtiKvq12vH3PY7gZ60XfV7sVlcHfFeTP2n8/JEeGmvV79+h3bfsGWBw9uB+9Ob5HPp2JfI/g8LYEPgCrLJotuB7XPnfi6dNw94WyfEBRwBzAVmAXcC72xzXHuTPSN9lmxENjd37n+meP8CfKjqmIA3AHPSG2gO8OE2v1a/Ax5L/65mAVPLfq3WJK4OeL3Oyf23fSO5P5IVvxcbxlX1e7Gu73RS0ij79Sr642VEzMyssIE6p2FmZhVw0jAzs8KcNMzMrDAnDTMzK8xJw8zMCnPSMEsk3dzm+3WlbyKbDRhOGmZJRLyhXfdKqwF0AU4aNqA4aZglkp5J/zwgLVT3S0kLJH1d0vsl3Z7qL+yQ+v1Y0nlpIcD7JR2ajm8o6cLUd6akA9PxiZKmSroBuB74OrB/qtlwShp5/FFZDZE7Jb0hF890SVdKuk/ST2srI0vaW9LNkman+IZKGiLpm5LuUFZb46MVvJy2lvKChWaNvQ7YGVhOtlzDDyNiH0knAycCn0j9usiWp94BuFHSa8iWiomIGCdpLHCdpNraRXsCu0fEckkHAKdHRC3ZbAy8OSL+LmlH4FKyhQchWyJkV2Ax8Cfg3yTdDlwGHBURd0gaBqwgW+H2yYjYW9IGwJ8kXRfZctpma8RJw6yxOyLiUYC0dPZ16fgcsmJHNZdHxEvAnyUtAMYC+wHfBYiI+yQ9yMsL3k2LiOVN7rkecK6k8cBKVl0k7/aIeCTFM4ssWT0JPBoRd6R7PZXa3wLsrperHQ4HdiRbC8tsjThpmDX2fG77pdz+S6z6vqlfh6evdXme7aXtFLK1o15H9ug4X4AnH89Ken/vCjgxIq7tIxazfvOchtmaeY+kddI8x/ZkC8n9EXg/QHostV06Xu9psqWwa4aTjRxeAo4hKw3am3nANpL2TvcamibYrwWOl7ReLQZJm6zuL2iW55GG2Zp5iGy14mHAcWk+4vvADyTNAV4EJkbE8w2q+t4FrJQ0m2z59+8DV0n6IHANvY9KiIgXJB0FfFfSRmTzGQcDPyR7fHVnmjB/gpdrWJitEa9ya7aaJP0YuDoirqw6FrN28eMpMzMrzCMNMzMrzCMNMzMrzEnDzMwKc9IwM7PCnDTMzKwwJw0zMyvsfwE6Tt19ulp6NgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt \n",
    "feat_importances = pd.Series(model.feature_importances_, index=bd.feature_names)\n",
    "feat_importances.sort_values(ascending = True).plot(kind='barh')\n",
    "plt.xlabel('importance')\n",
    "plt.title('Feature Importance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'self' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-61-c2ee16e06f6b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mstatsmodels\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mstatsmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregression\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear_model\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mlm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mlasso\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOLS\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_regularized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mL1_wt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1.0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'self' is not defined"
     ]
    }
   ],
   "source": [
    "import statsmodels\n",
    "import statsmodels.regression.linear_model as lm\n",
    "lasso=lm.OLS.fit_regularized(L1_wt=1.0,alpha=1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score: -29.02809240024379\n",
      "Best Hyperparameters: {'alpha': 0.03064709917930863, 'fit_intercept': True, 'normalize': True, 'solver': 'sag'}\n"
     ]
    }
   ],
   "source": [
    "# random search linear regression model on the auto insurance dataset\n",
    "from scipy.stats import loguniform\n",
    "from pandas import read_csv\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.model_selection import RepeatedKFold\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from numpy import mean\n",
    "from numpy import std\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from matplotlib import pyplot\n",
    "\n",
    "\n",
    "model = Ridge()\n",
    "# define evaluation\n",
    "cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "# define search space\n",
    "space = dict()\n",
    "space['solver'] = ['svd', 'cholesky', 'lsqr', 'sag']\n",
    "space['alpha'] = loguniform(1e-5, 100)\n",
    "space['fit_intercept'] = [True, False]\n",
    "space['normalize'] = [True, False]\n",
    "# define search\n",
    "search = RandomizedSearchCV(model, space, n_iter=500, scoring='neg_mean_absolute_error', n_jobs=-1, cv=cv, random_state=1)\n",
    "# execute search\n",
    "result = search.fit(X, y)\n",
    "# summarize result\n",
    "print('Best Score: %s' % result.best_score_)\n",
    "print('Best Hyperparameters: %s' % result.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}