3_higgsboson_speed.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finish loading from csv \n",
      "weight statistics: wpos=1522.37, wneg=904200, ratio=593.94\n",
      "loading data end, start to boost trees\n",
      "training GBM from sklearn\n",
      "      Iter       Train Loss   Remaining Time \n",
      "         1           1.2069           28.07s\n",
      "         2           1.1437           24.22s\n",
      "         3           1.0909           20.81s\n",
      "         4           1.0471           17.85s\n",
      "         5           1.0096           14.84s\n",
      "         6           0.9775           11.81s\n",
      "         7           0.9505            8.80s\n",
      "         8           0.9264            5.84s\n",
      "         9           0.9058            2.91s\n",
      "        10           0.8878            0.00s\n",
      "sklearn.GBM costs: 29.9052710533 seconds\n",
      "training xgboost\n",
      "[0]\ttrain-auc:0.910911\ttrain-ams@0.15:3.69957\n",
      "[1]\ttrain-auc:0.917686\ttrain-ams@0.15:3.96347\n",
      "[2]\ttrain-auc:0.920749\ttrain-ams@0.15:4.25961\n",
      "[3]\ttrain-auc:0.922946\ttrain-ams@0.15:4.33056\n",
      "[4]\ttrain-auc:0.924997\ttrain-ams@0.15:4.41044\n",
      "[5]\ttrain-auc:0.927125\ttrain-ams@0.15:4.49308\n",
      "[6]\ttrain-auc:0.928785\ttrain-ams@0.15:4.64614\n",
      "[7]\ttrain-auc:0.929895\ttrain-ams@0.15:4.63706\n",
      "[8]\ttrain-auc:0.931058\ttrain-ams@0.15:4.73032\n",
      "[9]\ttrain-auc:0.932234\ttrain-ams@0.15:4.79447\n",
      "XGBoost with 1 thread costs: 9.41331887245 seconds\n",
      "[0]\ttrain-auc:0.910911\ttrain-ams@0.15:3.69957\n",
      "[1]\ttrain-auc:0.917686\ttrain-ams@0.15:3.96347\n",
      "[2]\ttrain-auc:0.920749\ttrain-ams@0.15:4.25961\n",
      "[3]\ttrain-auc:0.922946\ttrain-ams@0.15:4.33056\n",
      "[4]\ttrain-auc:0.924997\ttrain-ams@0.15:4.41044\n",
      "[5]\ttrain-auc:0.927125\ttrain-ams@0.15:4.49308\n",
      "[6]\ttrain-auc:0.928785\ttrain-ams@0.15:4.64614\n",
      "[7]\ttrain-auc:0.929895\ttrain-ams@0.15:4.63706\n",
      "[8]\ttrain-auc:0.931058\ttrain-ams@0.15:4.73032\n",
      "[9]\ttrain-auc:0.932234\ttrain-ams@0.15:4.79447\n",
      "XGBoost with 2 thread costs: 4.73568201065 seconds\n",
      "[0]\ttrain-auc:0.910911\ttrain-ams@0.15:3.69957\n",
      "[1]\ttrain-auc:0.917686\ttrain-ams@0.15:3.96347\n",
      "[2]\ttrain-auc:0.920749\ttrain-ams@0.15:4.25961\n",
      "[3]\ttrain-auc:0.922946\ttrain-ams@0.15:4.33056\n",
      "[4]\ttrain-auc:0.924997\ttrain-ams@0.15:4.41044\n",
      "[5]\ttrain-auc:0.927125\ttrain-ams@0.15:4.49308\n",
      "[6]\ttrain-auc:0.928785\ttrain-ams@0.15:4.64614\n",
      "[7]\ttrain-auc:0.929895\ttrain-ams@0.15:4.63706\n",
      "[8]\ttrain-auc:0.931058\ttrain-ams@0.15:4.73032\n",
      "[9]\ttrain-auc:0.932234\ttrain-ams@0.15:4.79447\n",
      "XGBoost with 4 thread costs: 2.7055978775 seconds\n",
      "[0]\ttrain-auc:0.910911\ttrain-ams@0.15:3.69957\n",
      "[1]\ttrain-auc:0.917686\ttrain-ams@0.15:3.96347\n",
      "[2]\ttrain-auc:0.920749\ttrain-ams@0.15:4.25961\n",
      "[3]\ttrain-auc:0.922946\ttrain-ams@0.15:4.33056\n",
      "[4]\ttrain-auc:0.924997\ttrain-ams@0.15:4.41044\n",
      "[5]\ttrain-auc:0.927125\ttrain-ams@0.15:4.49308\n",
      "[6]\ttrain-auc:0.928785\ttrain-ams@0.15:4.64614\n",
      "[7]\ttrain-auc:0.929895\ttrain-ams@0.15:4.63706\n",
      "[8]\ttrain-auc:0.931058\ttrain-ams@0.15:4.73032\n",
      "[9]\ttrain-auc:0.932234\ttrain-ams@0.15:4.79447\n",
      "XGBoost with 16 thread costs: 2.21966814995 seconds\n",
      "finish training\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/python\n",
    "# this is the example script to use xgboost to train\n",
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "import time\n",
    "\n",
    "\n",
    "test_size = 550000\n",
    "\n",
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "\n",
    "# load in training data, directly use numpy\n",
    "dtrain = np.loadtxt( dpath+'/higgsboson_training.csv', delimiter=',', skiprows=1, converters={32: lambda x:int(x=='s') } )\n",
    "print ('finish loading from csv ')\n",
    "\n",
    "label  = dtrain[:,32]\n",
    "data   = dtrain[:,1:31]\n",
    "# rescale weight to make it same as test set\n",
    "weight = dtrain[:,31] * float(test_size) / len(label)\n",
    "\n",
    "sum_wpos = sum( weight[i] for i in range(len(label)) if label[i] == 1.0  )\n",
    "sum_wneg = sum( weight[i] for i in range(len(label)) if label[i] == 0.0  )\n",
    "\n",
    "# print weight statistics\n",
    "print ('weight statistics: wpos=%g, wneg=%g, ratio=%g' % ( sum_wpos, sum_wneg, sum_wneg/sum_wpos ))\n",
    "\n",
    "# construct xgboost.DMatrix from numpy array, treat -999.0 as missing value\n",
    "xgmat = xgb.DMatrix( data, label=label, missing = -999.0, weight=weight )\n",
    "\n",
    "# setup parameters for xgboost\n",
    "param = {}\n",
    "# use logistic regression loss\n",
    "param['objective'] = 'binary:logitraw'\n",
    "# scale weight of positive examples\n",
    "param['scale_pos_weight'] = sum_wneg/sum_wpos\n",
    "param['bst:eta'] = 0.1\n",
    "param['bst:max_depth'] = 6\n",
    "param['eval_metric'] = 'auc'\n",
    "param['silent'] = 1\n",
    "param['nthread'] = 4\n",
    "\n",
    "plst = param.items()+[('eval_metric', 'ams@0.15')]\n",
    "\n",
    "watchlist = [ (xgmat,'train') ]\n",
    "# boost 10 trees\n",
    "num_round = 10\n",
    "print ('loading data end, start to boost trees')\n",
    "print (\"training GBM from sklearn\")\n",
    "tmp = time.time()\n",
    "gbm = GradientBoostingClassifier(n_estimators=num_round, max_depth=6, verbose=2)\n",
    "gbm.fit(data, label)\n",
    "print (\"sklearn.GBM costs: %s seconds\" % str(time.time() - tmp))\n",
    "#raw_input()\n",
    "print (\"training xgboost\")\n",
    "threads = [1, 2, 4, 8]\n",
    "for i in threads:\n",
    "    param['nthread'] = i\n",
    "    tmp = time.time()\n",
    "    plst = param.items()+[('eval_metric', 'ams@0.15')]\n",
    "    bst = xgb.train( plst, xgmat, num_round, watchlist );\n",
    "    print (\"XGBoost with %d thread costs: %s seconds\" % (i, str(time.time() - tmp)))\n",
    "\n",
    "print ('finish training')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}