02_allstate_stacking_baseline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Allstate Stacking Baseline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import. We will use package 'vecstack' which implenets stacking routines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\python27x64\\lib\\site-packages\\sklearn\\cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division\n",
    "import gc\n",
    "import numpy as np\n",
    "np.set_printoptions(suppress = True)\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from xgboost import XGBRegressor\n",
    "from vecstack import stacking"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dir_path = '../input/'\n",
    "train_df = pd.read_csv(dir_path + 'train.csv', sep = ',', header = 0, low_memory = False, nrows = None)\n",
    "test_df = pd.read_csv(dir_path + 'test.csv', sep = ',', header = 0, nrows = None)\n",
    "subm_df = pd.read_csv(dir_path + 'sample_submission.csv', sep = ',', header = 0, nrows = None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare column lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "id_col = 'id'\n",
    "y_col = 'loss'\n",
    "all_cols = train_df.columns.tolist()\n",
    "all_cols.remove(id_col)\n",
    "all_cols.remove(y_col)\n",
    "obj_cols = train_df.dtypes[train_df.dtypes == 'object'].index.tolist()\n",
    "num_cols = list(set(all_cols) - set(obj_cols))\n",
    "num_cols.sort(key = lambda x: '{0:0>6}'.format(x)) # sort"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Factorize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "r, c = train_df.shape\n",
    "test_df.loc[:, 'loss'] = 0\n",
    "z_df = pd.concat([train_df, test_df], ignore_index = True)\n",
    "\n",
    "for col in obj_cols:\n",
    "    z_df.loc[:, col] = pd.factorize(z_df[col], sort = True)[0]\n",
    "        \n",
    "train_df = z_df[:r]\n",
    "test_df = z_df[r:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get numpy arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = train_df[all_cols].values\n",
    "y_train = train_df[y_col].values\n",
    "X_test = test_df[all_cols].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Free RAM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "del train_df\n",
    "del test_df\n",
    "del z_df\n",
    "_ = gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize 1-st level models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "models = [\n",
    "    XGBRegressor(seed = 0, colsample_bytree = 0.7, subsample = 0.7, learning_rate = 0.075, \n",
    "                 max_depth = 7, min_child_weight = 1, n_estimators = 400),\n",
    "    ExtraTreesRegressor(random_state = 0, n_jobs = 4, n_estimators = 100, max_features = 0.5, \n",
    "                        max_depth = 12, min_samples_leaf = 2),\n",
    "    RandomForestRegressor(random_state = 0, n_jobs = 4, n_estimators = 100, max_features = 0.2, \n",
    "                          max_depth = 8, min_samples_leaf = 2),\n",
    "    ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get stacking features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "task:   [regression]\n",
      "metric: [mean_absolute_error]\n",
      "\n",
      "model 0: [XGBRegressor]\n",
      "    fold 0: [1149.39250935]\n",
      "    fold 1: [1156.79667203]\n",
      "    fold 2: [1141.18669243]\n",
      "    ----\n",
      "    MEAN:   [1149.12533343]\n",
      "\n",
      "model 1: [ExtraTreesRegressor]\n",
      "    fold 0: [1236.34859807]\n",
      "    fold 1: [1247.20592610]\n",
      "    fold 2: [1231.92712626]\n",
      "    ----\n",
      "    MEAN:   [1238.49391835]\n",
      "\n",
      "model 2: [RandomForestRegressor]\n",
      "    fold 0: [1286.40670066]\n",
      "    fold 1: [1298.92017554]\n",
      "    fold 2: [1282.92941276]\n",
      "    ----\n",
      "    MEAN:   [1289.41879744]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "S_train, S_test = stacking(models, X_train, y_train, X_test, \n",
    "    n_folds = 3, shuffle = True, transform_target = np.log, \n",
    "    transform_pred = np.exp, verbose = 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize and apply 2-nd level model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = XGBRegressor(seed = 0, colsample_bytree = 0.8, subsample = 0.6, \n",
    "                     learning_rate = 0.01, max_depth = 4, min_child_weight = 1, \n",
    "                     n_estimators = 1028)\n",
    "model = model.fit(S_train, np.log(y_train))\n",
    "subm_df.iloc[:, 1] = np.exp(model.predict(S_test))\n",
    "subm_df.to_csv(dir_path + 'submission.csv', sep = ',', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}