main_train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 工具包导入&数据读取\n",
    "## 工具包导入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda2/lib/python2.7/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import lightgbm as lgb\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import gc\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "%matplotlib inline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'/home/user/secury/scr'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据读取\n",
    "- 为了方便分析，我们读取3000万条数据进行处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path = '../final_input/'\n",
    "# train = pd.read_csv(path + 'final_train.csv',nrows=1000000)\n",
    "# test = pd.read_csv(path + 'final_test.csv',nrows=1000000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../input/'\n",
    "train = pd.read_csv(path + 'final_train.csv')\n",
    "#test = pd.read_csv(path + 'final_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 特征工程 & 验证结果(1-Gram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6786</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7602</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8065</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10111</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       file_id  label\n",
       "0            1      5\n",
       "6786         2      2\n",
       "7602         3      0\n",
       "8065         4      0\n",
       "10111        5      0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = train[['file_id','label']].drop_duplicates()\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    4978\n",
       "5    4289\n",
       "7    1487\n",
       "2    1196\n",
       "3     820\n",
       "6     515\n",
       "1     502\n",
       "4     100\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data['label'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 全局特征:\n",
    "- File_id (Api): count,nunique\n",
    "- File_id (Tid): count,nunique,max,min,quantile(20,40,50,60,80),std,range\n",
    "- File_id (Return Value): count,nunique,max,min,quantile(20,40,50,60,80),std,range\n",
    "- File_id (Index): count,nunique,max,min,quantile(20,40,50,60,80),std,range"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### File_id (Api): count,nunique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>label</th>\n",
       "      <th>api</th>\n",
       "      <th>tid</th>\n",
       "      <th>index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrLoadDll</td>\n",
       "      <td>2488</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   file_id  label                     api   tid  index\n",
       "0        1      5              LdrLoadDll  2488      0\n",
       "1        1      5  LdrGetProcedureAddress  2488      1\n",
       "2        1      5  LdrGetProcedureAddress  2488      2\n",
       "3        1      5  LdrGetProcedureAddress  2488      3\n",
       "4        1      5  LdrGetProcedureAddress  2488      4"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:4: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nunique\n"
     ]
    }
   ],
   "source": [
    "api_opt = ['count','nunique'] \n",
    "for opt in api_opt:\n",
    "    print(opt)\n",
    "    tmp = train.groupby(['file_id'])['api'].agg({'fileid_api_' + opt: opt}).reset_index() \n",
    "    train_data = pd.merge(train_data,tmp,how='left', on='file_id')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>label</th>\n",
       "      <th>fileid_api_count</th>\n",
       "      <th>fileid_api_nunique</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6786</td>\n",
       "      <td>116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>816</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>463</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2046</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>10002</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   file_id  label  fileid_api_count  fileid_api_nunique\n",
       "0        1      5              6786                 116\n",
       "1        2      2               816                  30\n",
       "2        3      0               463                  42\n",
       "3        4      0              2046                  51\n",
       "4        5      0             10002                  65"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### File_id (Tid): count,nunique,max,min,quantile(20,40,50,60,80),std,range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:4: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nunique\n",
      "max\n",
      "min\n",
      "median\n",
      "std\n"
     ]
    }
   ],
   "source": [
    "tid_opt = ['count','nunique','max','min','median','std'] \n",
    "for opt in tid_opt:\n",
    "    print(opt)\n",
    "    tmp = train.groupby(['file_id'])['tid'].agg({'fileid_tid_' + opt: opt}).reset_index() \n",
    "    train_data = pd.merge(train_data,tmp,how='left', on='file_id') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "secs = [0.2,0.4,0.6,0.8]\n",
    "for sec in secs: \n",
    "    train_data['fileid_tid_quantile_' + str(sec * 100)] = train.groupby(['file_id'])['tid'].quantile(sec).values\n",
    " \n",
    "train_data['fileid_tid_range'] = train.groupby(['file_id'])['tid'].quantile(0.975).values - train.groupby(['file_id'])['tid'].quantile(0.0125).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### File_id (Index): count,nunique,max,min,quantile(20,40,50,60,80),std,range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:4: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nunique\n",
      "max\n",
      "min\n",
      "median\n",
      "std\n"
     ]
    }
   ],
   "source": [
    "index_opt = ['count','nunique','max','min','median','std'] \n",
    "for opt in index_opt:\n",
    "    print(opt)\n",
    "    tmp = train.groupby(['file_id'])['index'].agg({'fileid_index_' + opt: opt}).reset_index() \n",
    "    train_data = pd.merge(train_data,tmp,how='left', on='file_id') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "secs = [0.2,0.4,0.6,0.8]\n",
    "for sec in secs: \n",
    "    train_data['fileid_index_quantile_' + str(sec * 100)] = train.groupby(['file_id'])['index'].quantile(sec).values\n",
    " \n",
    "train_data['fileid_index_range'] = train.groupby(['file_id'])['index'].quantile(0.975).values - train.groupby(['file_id'])['index'].quantile(0.0125).values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 全局特征的线下验证 <font color=red>( 0.0969482)</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 评估指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lgb_logloss(preds,data):\n",
    "    labels_ = data.get_label()\n",
    "    classes_ = np.unique(labels_) \n",
    "    preds_prob = []\n",
    "    for i in range(len(classes_)):\n",
    "        preds_prob.append(preds[i*len(labels_):(i+1) * len(labels_)])\n",
    "    preds_prob_ = np.vstack(preds_prob) \n",
    "    \n",
    "    loss = [] \n",
    "    for i in range(preds_prob_.shape[1]):  # 样本个数\n",
    "        sum_ = 0  \n",
    "        for j in range(preds_prob_.shape[0]): #类别个数\n",
    "            pred = preds_prob_[j,i] # 第i个样本预测为第j类的概率\n",
    "            if  j == labels_[i]:\n",
    "                sum_ += np.log(pred)\n",
    "            else:\n",
    "                sum_ += np.log(1 - pred) \n",
    "             \n",
    "        loss.append(sum_)  \n",
    "         \n",
    "    return 'loss is: ' ,-1 * (np.sum(loss) / preds_prob_.shape[1]),False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 训练特征 & 标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features = [col for col in train_data.columns if col!='label' and col!='file_id']\n",
    "train_label = 'label'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split( train_data[train_features],train_data[train_label].values, test_size = 0.33) \n",
    "del _\n",
    "gc.collect()\n",
    "\n",
    "train_ind = train_X.index\n",
    "test_ind = test_X.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# dtrain = lgb.Dataset(train_X,train_Y) \n",
    "# dval   = lgb.Dataset(test_X,test_Y, reference = dtrain) \n",
    "\n",
    "# params = {\n",
    "#         'task':'train', \n",
    "#         'num_leaves': 255,\n",
    "#         'objective': 'multiclass',\n",
    "#         'num_class':8,\n",
    "#         #'min_data_in_leaf': 40,\n",
    "#         'min_data_in_leaf': 1,\n",
    "#         'learning_rate': 0.05,\n",
    "#         'feature_fraction': 0.85,\n",
    "#         'bagging_fraction': 0.9,\n",
    "#         'bagging_freq': 5, \n",
    "#         'max_bin':128,\n",
    "#         'num_threads': 10,\n",
    "#         'random_state':100\n",
    "#     }  \n",
    "# lgb_model_0_order = lgb.train(params, dtrain, num_boost_round=500,valid_sets=[dtrain,dval], early_stopping_rounds=50, feval=lgb_logloss)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 全局特征扩充\n",
    "- File_id + return_value分段：计数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 局部组合特征(展开形式)\n",
    "### File_id + Api  \n",
    "- File_id + Api (tid): count,nunique\n",
    "- File_id + Api (return value): nunique, max, min, median, std\n",
    "- File_id + Api (index):  nunique, max, min, median, std\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### File_id + Api (tid): count,nunique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def groupby_pivot_features(data_merge, data_orig , groupby_features,col1 = None, col2 = None, opts = None):\n",
    "    for opt in opts:\n",
    "        print(opt)\n",
    "        train_split = data_orig.groupby(['file_id',col1])[col2].agg({'fileid_' + col1 + '_'+col2+'_'+ str(opt):opt}).reset_index() \n",
    "        \n",
    "        train_split_ =  pd.pivot_table(train_split, values = 'fileid_' + col1 + '_'+col2+'_'+ str(opt), index=['file_id'],columns=[col1])\n",
    "        new_cols = [ 'fileid_' + col1 + '_'+col2+  '_' + opt + '_' + str(col) for col in train_split_.columns]\n",
    "        \n",
    "        groupby_features.append(new_cols)\n",
    "        train_split_.columns = new_cols \n",
    "\n",
    "        train_split_.reset_index(inplace = True)\n",
    "        \n",
    "        data_merge = pd.merge(data_merge,train_split_,how='left', on='file_id') \n",
    "    return data_merge,groupby_features \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:4: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nunique\n"
     ]
    }
   ],
   "source": [
    "groupby_features = []\n",
    "api_opts = ['count', 'nunique']\n",
    "train_data_,groupby_features = groupby_pivot_features(train_data, train, groupby_features, col1 = 'api', col2 = 'tid', opts = api_opts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### File_id + Api (return value): nunique, max, min, median, std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# api_opts = ['nunique','max','min','median','std']\n",
    "# train_data_,groupby_features = groupby_pivot_features(train_data_, train, groupby_features, col1 = 'api', col2 = 'return_value', opts = api_opts) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  File_id + Api(index): nunique, max, min, median, std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nunique\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:4: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max\n",
      "min\n",
      "median\n",
      "std\n"
     ]
    }
   ],
   "source": [
    "api_opts = ['nunique','max','min','median','std']\n",
    "train_data_,groupby_features = groupby_pivot_features(train_data_, train, groupby_features, col1 = 'api', col2 = 'index', opts = api_opts) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>label</th>\n",
       "      <th>fileid_api_count</th>\n",
       "      <th>fileid_api_nunique</th>\n",
       "      <th>fileid_tid_count</th>\n",
       "      <th>fileid_tid_nunique</th>\n",
       "      <th>fileid_tid_max</th>\n",
       "      <th>fileid_tid_min</th>\n",
       "      <th>fileid_tid_median</th>\n",
       "      <th>fileid_tid_std</th>\n",
       "      <th>...</th>\n",
       "      <th>fileid_api_index_std_recv</th>\n",
       "      <th>fileid_api_index_std_recvfrom</th>\n",
       "      <th>fileid_api_index_std_select</th>\n",
       "      <th>fileid_api_index_std_send</th>\n",
       "      <th>fileid_api_index_std_sendto</th>\n",
       "      <th>fileid_api_index_std_setsockopt</th>\n",
       "      <th>fileid_api_index_std_shutdown</th>\n",
       "      <th>fileid_api_index_std_socket</th>\n",
       "      <th>fileid_api_index_std_system</th>\n",
       "      <th>fileid_api_index_std_timeGetTime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6786</td>\n",
       "      <td>116</td>\n",
       "      <td>6786</td>\n",
       "      <td>11</td>\n",
       "      <td>2812</td>\n",
       "      <td>2488</td>\n",
       "      <td>2488</td>\n",
       "      <td>83.881299</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>94.185986</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>816</td>\n",
       "      <td>30</td>\n",
       "      <td>816</td>\n",
       "      <td>4</td>\n",
       "      <td>2604</td>\n",
       "      <td>2320</td>\n",
       "      <td>2480</td>\n",
       "      <td>101.506783</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>55.055816</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>463</td>\n",
       "      <td>42</td>\n",
       "      <td>463</td>\n",
       "      <td>1</td>\n",
       "      <td>2208</td>\n",
       "      <td>2208</td>\n",
       "      <td>2208</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2046</td>\n",
       "      <td>51</td>\n",
       "      <td>2046</td>\n",
       "      <td>9</td>\n",
       "      <td>2980</td>\n",
       "      <td>2284</td>\n",
       "      <td>2340</td>\n",
       "      <td>150.460506</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>10002</td>\n",
       "      <td>65</td>\n",
       "      <td>10002</td>\n",
       "      <td>6</td>\n",
       "      <td>2676</td>\n",
       "      <td>2500</td>\n",
       "      <td>2596</td>\n",
       "      <td>49.556301</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2086 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   file_id  label  fileid_api_count  fileid_api_nunique  fileid_tid_count  \\\n",
       "0        1      5              6786                 116              6786   \n",
       "1        2      2               816                  30               816   \n",
       "2        3      0               463                  42               463   \n",
       "3        4      0              2046                  51              2046   \n",
       "4        5      0             10002                  65             10002   \n",
       "\n",
       "   fileid_tid_nunique  fileid_tid_max  fileid_tid_min  fileid_tid_median  \\\n",
       "0                  11            2812            2488               2488   \n",
       "1                   4            2604            2320               2480   \n",
       "2                   1            2208            2208               2208   \n",
       "3                   9            2980            2284               2340   \n",
       "4                   6            2676            2500               2596   \n",
       "\n",
       "   fileid_tid_std                ...                 \\\n",
       "0       83.881299                ...                  \n",
       "1      101.506783                ...                  \n",
       "2        0.000000                ...                  \n",
       "3      150.460506                ...                  \n",
       "4       49.556301                ...                  \n",
       "\n",
       "   fileid_api_index_std_recv  fileid_api_index_std_recvfrom  \\\n",
       "0                        NaN                            NaN   \n",
       "1                        NaN                            NaN   \n",
       "2                        NaN                            NaN   \n",
       "3                        NaN                            NaN   \n",
       "4                        NaN                            NaN   \n",
       "\n",
       "   fileid_api_index_std_select  fileid_api_index_std_send  \\\n",
       "0                          NaN                        NaN   \n",
       "1                          NaN                        NaN   \n",
       "2                          NaN                        NaN   \n",
       "3                          NaN                        NaN   \n",
       "4                          NaN                        NaN   \n",
       "\n",
       "   fileid_api_index_std_sendto  fileid_api_index_std_setsockopt  \\\n",
       "0                          NaN                              NaN   \n",
       "1                          NaN                              NaN   \n",
       "2                          NaN                              NaN   \n",
       "3                          NaN                              NaN   \n",
       "4                          NaN                              NaN   \n",
       "\n",
       "   fileid_api_index_std_shutdown  fileid_api_index_std_socket  \\\n",
       "0                            NaN                    94.185986   \n",
       "1                            NaN                    55.055816   \n",
       "2                            NaN                          NaN   \n",
       "3                            NaN                          NaN   \n",
       "4                            NaN                          NaN   \n",
       "\n",
       "   fileid_api_index_std_system  fileid_api_index_std_timeGetTime  \n",
       "0                          NaN                               NaN  \n",
       "1                          NaN                               NaN  \n",
       "2                          NaN                               NaN  \n",
       "3                          NaN                               NaN  \n",
       "4                          NaN                               NaN  \n",
       "\n",
       "[5 rows x 2086 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1阶特征的线下验证(File_id + Api)（<font color=red>0.0347293</font>）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### File_id + Index  \n",
    "- File_id + Index (api): count,nunique\n",
    "- File_id + Index (return value): nunique, max, min, median, std(暂时先搁置)\n",
    "- File_id + Index (tid):  nunique, max, min, median, std(暂时先搁置)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### File_id +Tid (api): count,nunique"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### File_id + Index特征过拟合，删除\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# delcol = []\n",
    "# for i in range(2):\n",
    "#     for item in groupby_features2[i]:\n",
    "#         delcol.append(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data_.drop(delcol,axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 特征补充（加入index的差值特征）\n",
    "- File_id + Api (index_diff): 'nunique','max','min','median','std'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_diff = train.groupby(['file_id','tid'])['index'].diff().fillna(-999).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['index_diff'] = train_diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_diff = train.loc[train.index_diff!=-999] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nunique\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:4: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max\n",
      "min\n",
      "median\n",
      "std\n"
     ]
    }
   ],
   "source": [
    "api_opts = ['nunique','max','min','median','std']\n",
    "train_data_,groupby_features = groupby_pivot_features(train_data_, train_diff, groupby_features, col1 = 'api', col2 = 'index_diff', opts = api_opts) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 线下验证(<font color=red>0.0346954</font>)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# train_features = [col for col in train_data_.columns if col!='label' and col!='file_id']\n",
    "# train_label = 'label'\n",
    "# print(len(train_features))\n",
    "# dtrain = lgb.Dataset(train_data_.loc[train_ind,train_features],train_data_.loc[train_ind,train_label].values) \n",
    "# dval   = lgb.Dataset(train_data_.loc[test_ind,train_features],train_data_.loc[test_ind,train_label].values, reference = dtrain) \n",
    "\n",
    "# params = {\n",
    "#         'task':'train', \n",
    "#         'num_leaves': 255,\n",
    "#         'objective': 'multiclass',\n",
    "#         'num_class':6,\n",
    "#         'min_data_in_leaf': 40,\n",
    "#         'learning_rate': 0.05,\n",
    "#         'feature_fraction': 0.85,\n",
    "#         'bagging_fraction': 0.9,\n",
    "#         'bagging_freq': 5, \n",
    "#         'max_bin':128,\n",
    "#         'num_threads': 64,\n",
    "#         'random_state':100\n",
    "#     }  \n",
    "# lgb_model_3_order = lgb.train(params, dtrain, num_boost_round=500,valid_sets=[dtrain,dval], early_stopping_rounds=50, feval=lgb_logloss)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 删除quantile,std统计变量之后的验证(<font color=red>0.0350054</font>)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# train_features = [col for col in train_data_.columns if col!='label' and col!='file_id' and 'std' not in col and 'quantile' not in col]\n",
    "# train_label = 'label'\n",
    "# print(len(train_features))\n",
    "# dtrain = lgb.Dataset(train_data_.loc[train_ind,train_features],train_data_.loc[train_ind,train_label].values) \n",
    "# dval   = lgb.Dataset(train_data_.loc[test_ind,train_features],train_data_.loc[test_ind,train_label].values, reference = dtrain) \n",
    "\n",
    "# params = {\n",
    "#         'task':'train', \n",
    "#         'num_leaves': 255,\n",
    "#         'objective': 'multiclass',\n",
    "#         'num_class':6,\n",
    "#         'min_data_in_leaf': 40,\n",
    "#         'learning_rate': 0.05,\n",
    "#         'feature_fraction': 0.85,\n",
    "#         'bagging_fraction': 0.9,\n",
    "#         'bagging_freq': 5, \n",
    "#         'max_bin':128,\n",
    "#         'num_threads': 64,\n",
    "#         'random_state':100\n",
    "#     }  \n",
    "# lgb_model_3_order = lgb.train(params, dtrain, num_boost_round=500,valid_sets=[dtrain,dval], early_stopping_rounds=50, feval=lgb_logloss)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data_.to_csv('/data/Data_JieZhang/TC_SAFE/train_val/train_data.csv',index = None) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 特征工程& 验证结果 2-Gram\n",
    "## 全局特征\n",
    "### File_id（Api_2）:count,nunique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['api_shift'] = train['api'].shift(-1)\n",
    "train['api_2'] = train['api'] +'_' + train['api_shift']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.drop(['api_shift'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "api_count = train['api_2'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:4: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nunique\n"
     ]
    }
   ],
   "source": [
    "api_opt = ['count','nunique'] \n",
    "for opt in api_opt:\n",
    "    print(opt)\n",
    "    tmp = train.groupby(['file_id'])['api_2'].agg({'fileid_api_2_' + opt: opt}).reset_index() \n",
    "    train_data_ = pd.merge(train_data_,tmp,how='left', on='file_id')  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 局部特征\n",
    "### File_id + tid (Api_2): count特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "api_value_counts = pd.DataFrame(api_count).reset_index()\n",
    "api_value_counts.columns = ['api_2','api_2_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.merge(train, api_value_counts, on ='api_2' , how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:4: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "api_opts = ['count']\n",
    "groupby_features =  []\n",
    "train_data_,groupby_features = groupby_pivot_features(train_data_, train.loc[train.api_2_count>=20], groupby_features, col1 = 'api_2', col2 = 'tid', opts = api_opts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 线下验证(<font color=red> 0.0330886</font>)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_features = [col for col in train_data_.columns if col!='label' and col!='file_id']\n",
    "# train_label = 'label'\n",
    "# print(len(train_features))\n",
    "# dtrain = lgb.Dataset(train_data_.loc[train_ind,train_features],train_data_.loc[train_ind,train_label].values) \n",
    "# dval   = lgb.Dataset(train_data_.loc[test_ind,train_features],train_data_.loc[test_ind,train_label].values, reference = dtrain) \n",
    "\n",
    "# params = {\n",
    "#         'task':'train', \n",
    "#         'num_leaves': 255,\n",
    "#         'objective': 'multiclass',\n",
    "#         'num_class':6,\n",
    "#         'min_data_in_leaf': 40,\n",
    "#         'learning_rate': 0.05,\n",
    "#         'feature_fraction': 0.85,\n",
    "#         'bagging_fraction': 0.9,\n",
    "#         'bagging_freq': 5, \n",
    "#         'max_bin':128,\n",
    "#         'num_threads': 64,\n",
    "#         'random_state':100\n",
    "#     }  \n",
    "# lgb_model_3_order = lgb.train(params, dtrain, num_boost_round=500,valid_sets=[dtrain,dval], early_stopping_rounds=50, feval=lgb_logloss)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### File_id + index (Api_2): max,min特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12236\n",
      "[1]\ttraining's multi_logloss: 1.90825\ttraining's loss is: : 2.81617\tvalid_1's multi_logloss: 1.92056\tvalid_1's loss is: : 2.83041\n",
      "Training until validation scores don't improve for 50 rounds.\n",
      "[2]\ttraining's multi_logloss: 1.7664\ttraining's loss is: : 2.64826\tvalid_1's multi_logloss: 1.78989\tvalid_1's loss is: : 2.67577\n",
      "[3]\ttraining's multi_logloss: 1.64441\ttraining's loss is: : 2.50066\tvalid_1's multi_logloss: 1.67822\tvalid_1's loss is: : 2.54072\n",
      "[4]\ttraining's multi_logloss: 1.53808\ttraining's loss is: : 2.36933\tvalid_1's multi_logloss: 1.58129\tvalid_1's loss is: : 2.42106\n",
      "[5]\ttraining's multi_logloss: 1.44341\ttraining's loss is: : 2.25009\tvalid_1's multi_logloss: 1.49486\tvalid_1's loss is: : 2.31237\n",
      "[6]\ttraining's multi_logloss: 1.35794\ttraining's loss is: : 2.14047\tvalid_1's multi_logloss: 1.41736\tvalid_1's loss is: : 2.21313\n",
      "[7]\ttraining's multi_logloss: 1.28042\ttraining's loss is: : 2.03927\tvalid_1's multi_logloss: 1.3472\tvalid_1's loss is: : 2.12168\n",
      "[8]\ttraining's multi_logloss: 1.20948\ttraining's loss is: : 1.94505\tvalid_1's multi_logloss: 1.28358\tvalid_1's loss is: : 2.03733\n",
      "[9]\ttraining's multi_logloss: 1.14453\ttraining's loss is: : 1.85734\tvalid_1's multi_logloss: 1.22492\tvalid_1's loss is: : 1.95836\n",
      "[10]\ttraining's multi_logloss: 1.08453\ttraining's loss is: : 1.77504\tvalid_1's multi_logloss: 1.17135\tvalid_1's loss is: : 1.8851\n",
      "[11]\ttraining's multi_logloss: 1.02856\ttraining's loss is: : 1.69716\tvalid_1's multi_logloss: 1.12133\tvalid_1's loss is: : 1.81578\n",
      "[12]\ttraining's multi_logloss: 0.976554\ttraining's loss is: : 1.62376\tvalid_1's multi_logloss: 1.0752\tvalid_1's loss is: : 1.75093\n",
      "[13]\ttraining's multi_logloss: 0.928142\ttraining's loss is: : 1.55447\tvalid_1's multi_logloss: 1.03248\tvalid_1's loss is: : 1.69002\n",
      "[14]\ttraining's multi_logloss: 0.883126\ttraining's loss is: : 1.48916\tvalid_1's multi_logloss: 0.992714\tvalid_1's loss is: : 1.63258\n",
      "[15]\ttraining's multi_logloss: 0.840695\ttraining's loss is: : 1.42686\tvalid_1's multi_logloss: 0.955498\tvalid_1's loss is: : 1.57816\n",
      "[16]\ttraining's multi_logloss: 0.800954\ttraining's loss is: : 1.3678\tvalid_1's multi_logloss: 0.920533\tvalid_1's loss is: : 1.52646\n",
      "[17]\ttraining's multi_logloss: 0.763765\ttraining's loss is: : 1.31188\tvalid_1's multi_logloss: 0.887673\tvalid_1's loss is: : 1.47736\n",
      "[18]\ttraining's multi_logloss: 0.728787\ttraining's loss is: : 1.2587\tvalid_1's multi_logloss: 0.856913\tvalid_1's loss is: : 1.4309\n",
      "[19]\ttraining's multi_logloss: 0.695569\ttraining's loss is: : 1.20769\tvalid_1's multi_logloss: 0.828211\tvalid_1's loss is: : 1.38708\n",
      "[20]\ttraining's multi_logloss: 0.664199\ttraining's loss is: : 1.15899\tvalid_1's multi_logloss: 0.800872\tvalid_1's loss is: : 1.34494\n",
      "[21]\ttraining's multi_logloss: 0.634357\ttraining's loss is: : 1.11231\tvalid_1's multi_logloss: 0.775091\tvalid_1's loss is: : 1.30487\n",
      "[22]\ttraining's multi_logloss: 0.605941\ttraining's loss is: : 1.06746\tvalid_1's multi_logloss: 0.750618\tvalid_1's loss is: : 1.26651\n",
      "[23]\ttraining's multi_logloss: 0.579217\ttraining's loss is: : 1.02488\tvalid_1's multi_logloss: 0.727735\tvalid_1's loss is: : 1.23031\n",
      "[24]\ttraining's multi_logloss: 0.553998\ttraining's loss is: : 0.984349\tvalid_1's multi_logloss: 0.706673\tvalid_1's loss is: : 1.19664\n",
      "[25]\ttraining's multi_logloss: 0.529941\ttraining's loss is: : 0.945384\tvalid_1's multi_logloss: 0.68617\tvalid_1's loss is: : 1.16368\n",
      "[26]\ttraining's multi_logloss: 0.506906\ttraining's loss is: : 0.907815\tvalid_1's multi_logloss: 0.666587\tvalid_1's loss is: : 1.13209\n",
      "[27]\ttraining's multi_logloss: 0.485085\ttraining's loss is: : 0.871957\tvalid_1's multi_logloss: 0.64791\tvalid_1's loss is: : 1.10182\n",
      "[28]\ttraining's multi_logloss: 0.464389\ttraining's loss is: : 0.837699\tvalid_1's multi_logloss: 0.630219\tvalid_1's loss is: : 1.07294\n",
      "[29]\ttraining's multi_logloss: 0.44473\ttraining's loss is: : 0.804927\tvalid_1's multi_logloss: 0.61377\tvalid_1's loss is: : 1.04588\n",
      "[30]\ttraining's multi_logloss: 0.4259\ttraining's loss is: : 0.773345\tvalid_1's multi_logloss: 0.598212\tvalid_1's loss is: : 1.02011\n",
      "[31]\ttraining's multi_logloss: 0.40794\ttraining's loss is: : 0.743069\tvalid_1's multi_logloss: 0.583315\tvalid_1's loss is: : 0.99528\n",
      "[32]\ttraining's multi_logloss: 0.39069\ttraining's loss is: : 0.713822\tvalid_1's multi_logloss: 0.569091\tvalid_1's loss is: : 0.971497\n",
      "[33]\ttraining's multi_logloss: 0.374381\ttraining's loss is: : 0.686018\tvalid_1's multi_logloss: 0.555696\tvalid_1's loss is: : 0.948995\n",
      "[34]\ttraining's multi_logloss: 0.35876\ttraining's loss is: : 0.659245\tvalid_1's multi_logloss: 0.543135\tvalid_1's loss is: : 0.927734\n",
      "[35]\ttraining's multi_logloss: 0.343904\ttraining's loss is: : 0.633634\tvalid_1's multi_logloss: 0.531209\tvalid_1's loss is: : 0.907459\n",
      "[36]\ttraining's multi_logloss: 0.32964\ttraining's loss is: : 0.608957\tvalid_1's multi_logloss: 0.519762\tvalid_1's loss is: : 0.887926\n",
      "[37]\ttraining's multi_logloss: 0.316108\ttraining's loss is: : 0.585414\tvalid_1's multi_logloss: 0.508903\tvalid_1's loss is: : 0.869326\n",
      "[38]\ttraining's multi_logloss: 0.303214\ttraining's loss is: : 0.562877\tvalid_1's multi_logloss: 0.498482\tvalid_1's loss is: : 0.851482\n",
      "[39]\ttraining's multi_logloss: 0.290812\ttraining's loss is: : 0.54111\tvalid_1's multi_logloss: 0.48849\tvalid_1's loss is: : 0.834326\n",
      "[40]\ttraining's multi_logloss: 0.279049\ttraining's loss is: : 0.520365\tvalid_1's multi_logloss: 0.479091\tvalid_1's loss is: : 0.81804\n",
      "[41]\ttraining's multi_logloss: 0.267642\ttraining's loss is: : 0.500194\tvalid_1's multi_logloss: 0.469912\tvalid_1's loss is: : 0.802232\n",
      "[42]\ttraining's multi_logloss: 0.256756\ttraining's loss is: : 0.480875\tvalid_1's multi_logloss: 0.461132\tvalid_1's loss is: : 0.787072\n",
      "[43]\ttraining's multi_logloss: 0.246276\ttraining's loss is: : 0.462203\tvalid_1's multi_logloss: 0.452835\tvalid_1's loss is: : 0.772721\n",
      "[44]\ttraining's multi_logloss: 0.236308\ttraining's loss is: : 0.444381\tvalid_1's multi_logloss: 0.445202\tvalid_1's loss is: : 0.759409\n",
      "[45]\ttraining's multi_logloss: 0.226883\ttraining's loss is: : 0.42744\tvalid_1's multi_logloss: 0.437831\tvalid_1's loss is: : 0.746604\n",
      "[46]\ttraining's multi_logloss: 0.217787\ttraining's loss is: : 0.411061\tvalid_1's multi_logloss: 0.430787\tvalid_1's loss is: : 0.73432\n",
      "[47]\ttraining's multi_logloss: 0.209078\ttraining's loss is: : 0.395321\tvalid_1's multi_logloss: 0.42435\tvalid_1's loss is: : 0.723004\n",
      "[48]\ttraining's multi_logloss: 0.200714\ttraining's loss is: : 0.38016\tvalid_1's multi_logloss: 0.418208\tvalid_1's loss is: : 0.712147\n",
      "[49]\ttraining's multi_logloss: 0.192732\ttraining's loss is: : 0.365651\tvalid_1's multi_logloss: 0.412177\tvalid_1's loss is: : 0.701588\n",
      "[50]\ttraining's multi_logloss: 0.185053\ttraining's loss is: : 0.351648\tvalid_1's multi_logloss: 0.406677\tvalid_1's loss is: : 0.691801\n",
      "[51]\ttraining's multi_logloss: 0.177728\ttraining's loss is: : 0.338248\tvalid_1's multi_logloss: 0.401446\tvalid_1's loss is: : 0.682569\n",
      "[52]\ttraining's multi_logloss: 0.170654\ttraining's loss is: : 0.325288\tvalid_1's multi_logloss: 0.396174\tvalid_1's loss is: : 0.673259\n",
      "[53]\ttraining's multi_logloss: 0.163948\ttraining's loss is: : 0.312951\tvalid_1's multi_logloss: 0.391198\tvalid_1's loss is: : 0.664466\n",
      "[54]\ttraining's multi_logloss: 0.157468\ttraining's loss is: : 0.301008\tvalid_1's multi_logloss: 0.38649\tvalid_1's loss is: : 0.656172\n",
      "[55]\ttraining's multi_logloss: 0.151274\ttraining's loss is: : 0.289559\tvalid_1's multi_logloss: 0.382204\tvalid_1's loss is: : 0.648541\n",
      "[56]\ttraining's multi_logloss: 0.14527\ttraining's loss is: : 0.278452\tvalid_1's multi_logloss: 0.377978\tvalid_1's loss is: : 0.641073\n",
      "[57]\ttraining's multi_logloss: 0.139582\ttraining's loss is: : 0.267897\tvalid_1's multi_logloss: 0.373886\tvalid_1's loss is: : 0.633879\n",
      "[58]\ttraining's multi_logloss: 0.134135\ttraining's loss is: : 0.257766\tvalid_1's multi_logloss: 0.370216\tvalid_1's loss is: : 0.627353\n",
      "[59]\ttraining's multi_logloss: 0.128886\ttraining's loss is: : 0.247989\tvalid_1's multi_logloss: 0.366885\tvalid_1's loss is: : 0.621431\n",
      "[60]\ttraining's multi_logloss: 0.12385\ttraining's loss is: : 0.238582\tvalid_1's multi_logloss: 0.363389\tvalid_1's loss is: : 0.615355\n",
      "[61]\ttraining's multi_logloss: 0.118987\ttraining's loss is: : 0.229488\tvalid_1's multi_logloss: 0.360251\tvalid_1's loss is: : 0.609829\n",
      "[62]\ttraining's multi_logloss: 0.114318\ttraining's loss is: : 0.220738\tvalid_1's multi_logloss: 0.357381\tvalid_1's loss is: : 0.604729\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[63]\ttraining's multi_logloss: 0.109875\ttraining's loss is: : 0.212393\tvalid_1's multi_logloss: 0.354111\tvalid_1's loss is: : 0.599118\n",
      "[64]\ttraining's multi_logloss: 0.105601\ttraining's loss is: : 0.204351\tvalid_1's multi_logloss: 0.351429\tvalid_1's loss is: : 0.594374\n",
      "[65]\ttraining's multi_logloss: 0.101489\ttraining's loss is: : 0.196593\tvalid_1's multi_logloss: 0.348967\tvalid_1's loss is: : 0.590063\n",
      "[66]\ttraining's multi_logloss: 0.0975043\ttraining's loss is: : 0.189071\tvalid_1's multi_logloss: 0.346486\tvalid_1's loss is: : 0.585778\n",
      "[67]\ttraining's multi_logloss: 0.0936928\ttraining's loss is: : 0.181861\tvalid_1's multi_logloss: 0.344046\tvalid_1's loss is: : 0.581524\n",
      "[68]\ttraining's multi_logloss: 0.0900635\ttraining's loss is: : 0.17498\tvalid_1's multi_logloss: 0.341734\tvalid_1's loss is: : 0.577578\n",
      "[69]\ttraining's multi_logloss: 0.086594\ttraining's loss is: : 0.16839\tvalid_1's multi_logloss: 0.33952\tvalid_1's loss is: : 0.573784\n",
      "[70]\ttraining's multi_logloss: 0.0833025\ttraining's loss is: : 0.162128\tvalid_1's multi_logloss: 0.337602\tvalid_1's loss is: : 0.570458\n",
      "[71]\ttraining's multi_logloss: 0.0800773\ttraining's loss is: : 0.155987\tvalid_1's multi_logloss: 0.335933\tvalid_1's loss is: : 0.567505\n",
      "[72]\ttraining's multi_logloss: 0.076983\ttraining's loss is: : 0.150086\tvalid_1's multi_logloss: 0.334096\tvalid_1's loss is: : 0.564415\n",
      "[73]\ttraining's multi_logloss: 0.0740294\ttraining's loss is: : 0.144443\tvalid_1's multi_logloss: 0.332228\tvalid_1's loss is: : 0.561257\n",
      "[74]\ttraining's multi_logloss: 0.0712099\ttraining's loss is: : 0.139045\tvalid_1's multi_logloss: 0.330842\tvalid_1's loss is: : 0.558803\n",
      "[75]\ttraining's multi_logloss: 0.0684856\ttraining's loss is: : 0.133822\tvalid_1's multi_logloss: 0.329386\tvalid_1's loss is: : 0.556336\n",
      "[76]\ttraining's multi_logloss: 0.0658205\ttraining's loss is: : 0.128712\tvalid_1's multi_logloss: 0.327849\tvalid_1's loss is: : 0.553826\n",
      "[77]\ttraining's multi_logloss: 0.0633134\ttraining's loss is: : 0.123896\tvalid_1's multi_logloss: 0.326368\tvalid_1's loss is: : 0.551398\n",
      "[78]\ttraining's multi_logloss: 0.0609074\ttraining's loss is: : 0.119266\tvalid_1's multi_logloss: 0.325233\tvalid_1's loss is: : 0.549503\n",
      "[79]\ttraining's multi_logloss: 0.0585798\ttraining's loss is: : 0.114782\tvalid_1's multi_logloss: 0.324104\tvalid_1's loss is: : 0.547633\n",
      "[80]\ttraining's multi_logloss: 0.0563709\ttraining's loss is: : 0.110522\tvalid_1's multi_logloss: 0.323144\tvalid_1's loss is: : 0.546122\n",
      "[81]\ttraining's multi_logloss: 0.0542073\ttraining's loss is: : 0.106353\tvalid_1's multi_logloss: 0.322383\tvalid_1's loss is: : 0.544899\n",
      "[82]\ttraining's multi_logloss: 0.0521054\ttraining's loss is: : 0.102297\tvalid_1's multi_logloss: 0.321488\tvalid_1's loss is: : 0.543505\n",
      "[83]\ttraining's multi_logloss: 0.0500874\ttraining's loss is: : 0.0983978\tvalid_1's multi_logloss: 0.320799\tvalid_1's loss is: : 0.542458\n",
      "[84]\ttraining's multi_logloss: 0.0481415\ttraining's loss is: : 0.0946342\tvalid_1's multi_logloss: 0.320204\tvalid_1's loss is: : 0.541552\n",
      "[85]\ttraining's multi_logloss: 0.0462877\ttraining's loss is: : 0.0910436\tvalid_1's multi_logloss: 0.319564\tvalid_1's loss is: : 0.540575\n",
      "[86]\ttraining's multi_logloss: 0.0444902\ttraining's loss is: : 0.0875609\tvalid_1's multi_logloss: 0.318937\tvalid_1's loss is: : 0.539661\n",
      "[87]\ttraining's multi_logloss: 0.0427774\ttraining's loss is: : 0.0842371\tvalid_1's multi_logloss: 0.318292\tvalid_1's loss is: : 0.538696\n",
      "[88]\ttraining's multi_logloss: 0.0411403\ttraining's loss is: : 0.0810573\tvalid_1's multi_logloss: 0.317676\tvalid_1's loss is: : 0.53781\n",
      "[89]\ttraining's multi_logloss: 0.0395835\ttraining's loss is: : 0.0780297\tvalid_1's multi_logloss: 0.317388\tvalid_1's loss is: : 0.537468\n",
      "[90]\ttraining's multi_logloss: 0.0380927\ttraining's loss is: : 0.0751272\tvalid_1's multi_logloss: 0.317162\tvalid_1's loss is: : 0.537266\n",
      "[91]\ttraining's multi_logloss: 0.0366448\ttraining's loss is: : 0.072304\tvalid_1's multi_logloss: 0.316999\tvalid_1's loss is: : 0.53711\n",
      "[92]\ttraining's multi_logloss: 0.0352705\ttraining's loss is: : 0.0696215\tvalid_1's multi_logloss: 0.316857\tvalid_1's loss is: : 0.537084\n",
      "[93]\ttraining's multi_logloss: 0.0339552\ttraining's loss is: : 0.0670518\tvalid_1's multi_logloss: 0.316638\tvalid_1's loss is: : 0.536914\n",
      "[94]\ttraining's multi_logloss: 0.0326861\ttraining's loss is: : 0.0645707\tvalid_1's multi_logloss: 0.316545\tvalid_1's loss is: : 0.536977\n",
      "[95]\ttraining's multi_logloss: 0.0314704\ttraining's loss is: : 0.0621919\tvalid_1's multi_logloss: 0.316538\tvalid_1's loss is: : 0.537064\n",
      "[96]\ttraining's multi_logloss: 0.0302971\ttraining's loss is: : 0.0598966\tvalid_1's multi_logloss: 0.316489\tvalid_1's loss is: : 0.537225\n",
      "[97]\ttraining's multi_logloss: 0.0291617\ttraining's loss is: : 0.0576743\tvalid_1's multi_logloss: 0.316366\tvalid_1's loss is: : 0.53719\n",
      "[98]\ttraining's multi_logloss: 0.028085\ttraining's loss is: : 0.0555642\tvalid_1's multi_logloss: 0.316227\tvalid_1's loss is: : 0.537176\n",
      "[99]\ttraining's multi_logloss: 0.0270546\ttraining's loss is: : 0.0535439\tvalid_1's multi_logloss: 0.316357\tvalid_1's loss is: : 0.53762\n",
      "[100]\ttraining's multi_logloss: 0.0260603\ttraining's loss is: : 0.0515926\tvalid_1's multi_logloss: 0.316364\tvalid_1's loss is: : 0.537916\n",
      "[101]\ttraining's multi_logloss: 0.0250948\ttraining's loss is: : 0.0496975\tvalid_1's multi_logloss: 0.316562\tvalid_1's loss is: : 0.538526\n",
      "[102]\ttraining's multi_logloss: 0.0241782\ttraining's loss is: : 0.0478967\tvalid_1's multi_logloss: 0.316849\tvalid_1's loss is: : 0.539264\n",
      "[103]\ttraining's multi_logloss: 0.0233006\ttraining's loss is: : 0.0461713\tvalid_1's multi_logloss: 0.316902\tvalid_1's loss is: : 0.539698\n",
      "[104]\ttraining's multi_logloss: 0.0224614\ttraining's loss is: : 0.0445202\tvalid_1's multi_logloss: 0.317095\tvalid_1's loss is: : 0.540255\n",
      "[105]\ttraining's multi_logloss: 0.0216475\ttraining's loss is: : 0.0429183\tvalid_1's multi_logloss: 0.317374\tvalid_1's loss is: : 0.541047\n",
      "[106]\ttraining's multi_logloss: 0.0208167\ttraining's loss is: : 0.0412842\tvalid_1's multi_logloss: 0.317574\tvalid_1's loss is: : 0.541732\n",
      "[107]\ttraining's multi_logloss: 0.0200159\ttraining's loss is: : 0.0397085\tvalid_1's multi_logloss: 0.31765\tvalid_1's loss is: : 0.542132\n",
      "[108]\ttraining's multi_logloss: 0.0192522\ttraining's loss is: : 0.0382044\tvalid_1's multi_logloss: 0.317923\tvalid_1's loss is: : 0.542864\n",
      "[109]\ttraining's multi_logloss: 0.0185108\ttraining's loss is: : 0.0367436\tvalid_1's multi_logloss: 0.318155\tvalid_1's loss is: : 0.543552\n",
      "[110]\ttraining's multi_logloss: 0.0178154\ttraining's loss is: : 0.0353723\tvalid_1's multi_logloss: 0.318435\tvalid_1's loss is: : 0.544264\n",
      "[111]\ttraining's multi_logloss: 0.0171383\ttraining's loss is: : 0.0340366\tvalid_1's multi_logloss: 0.318857\tvalid_1's loss is: : 0.545266\n",
      "[112]\ttraining's multi_logloss: 0.0164915\ttraining's loss is: : 0.0327601\tvalid_1's multi_logloss: 0.31927\tvalid_1's loss is: : 0.546311\n",
      "[113]\ttraining's multi_logloss: 0.0158643\ttraining's loss is: : 0.0315216\tvalid_1's multi_logloss: 0.319763\tvalid_1's loss is: : 0.547494\n",
      "[114]\ttraining's multi_logloss: 0.0152653\ttraining's loss is: : 0.0303382\tvalid_1's multi_logloss: 0.320388\tvalid_1's loss is: : 0.548872\n",
      "[115]\ttraining's multi_logloss: 0.0146866\ttraining's loss is: : 0.0291942\tvalid_1's multi_logloss: 0.320772\tvalid_1's loss is: : 0.549795\n",
      "[116]\ttraining's multi_logloss: 0.0141331\ttraining's loss is: : 0.0280997\tvalid_1's multi_logloss: 0.32136\tvalid_1's loss is: : 0.551088\n",
      "[117]\ttraining's multi_logloss: 0.0136076\ttraining's loss is: : 0.02706\tvalid_1's multi_logloss: 0.322171\tvalid_1's loss is: : 0.552673\n",
      "[118]\ttraining's multi_logloss: 0.0131105\ttraining's loss is: : 0.0260758\tvalid_1's multi_logloss: 0.322776\tvalid_1's loss is: : 0.55387\n",
      "[119]\ttraining's multi_logloss: 0.0126347\ttraining's loss is: : 0.0251336\tvalid_1's multi_logloss: 0.323436\tvalid_1's loss is: : 0.555166\n",
      "[120]\ttraining's multi_logloss: 0.0121683\ttraining's loss is: : 0.0242095\tvalid_1's multi_logloss: 0.324263\tvalid_1's loss is: : 0.556835\n",
      "[121]\ttraining's multi_logloss: 0.0117035\ttraining's loss is: : 0.0232891\tvalid_1's multi_logloss: 0.32488\tvalid_1's loss is: : 0.558246\n",
      "[122]\ttraining's multi_logloss: 0.0112584\ttraining's loss is: : 0.0224074\tvalid_1's multi_logloss: 0.325591\tvalid_1's loss is: : 0.55978\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[123]\ttraining's multi_logloss: 0.0108364\ttraining's loss is: : 0.0215714\tvalid_1's multi_logloss: 0.326212\tvalid_1's loss is: : 0.561192\n",
      "[124]\ttraining's multi_logloss: 0.0104299\ttraining's loss is: : 0.0207655\tvalid_1's multi_logloss: 0.326855\tvalid_1's loss is: : 0.56255\n",
      "[125]\ttraining's multi_logloss: 0.0100383\ttraining's loss is: : 0.0199888\tvalid_1's multi_logloss: 0.327589\tvalid_1's loss is: : 0.564102\n",
      "[126]\ttraining's multi_logloss: 0.00966387\ttraining's loss is: : 0.0192463\tvalid_1's multi_logloss: 0.328251\tvalid_1's loss is: : 0.565496\n",
      "[127]\ttraining's multi_logloss: 0.00930306\ttraining's loss is: : 0.0185304\tvalid_1's multi_logloss: 0.329061\tvalid_1's loss is: : 0.567172\n",
      "[128]\ttraining's multi_logloss: 0.00895435\ttraining's loss is: : 0.0178384\tvalid_1's multi_logloss: 0.329773\tvalid_1's loss is: : 0.568687\n",
      "[129]\ttraining's multi_logloss: 0.00862021\ttraining's loss is: : 0.0171752\tvalid_1's multi_logloss: 0.330612\tvalid_1's loss is: : 0.570442\n",
      "[130]\ttraining's multi_logloss: 0.00830182\ttraining's loss is: : 0.016543\tvalid_1's multi_logloss: 0.331456\tvalid_1's loss is: : 0.572168\n",
      "[131]\ttraining's multi_logloss: 0.00799225\ttraining's loss is: : 0.0159282\tvalid_1's multi_logloss: 0.332171\tvalid_1's loss is: : 0.573713\n",
      "[132]\ttraining's multi_logloss: 0.00769811\ttraining's loss is: : 0.0153438\tvalid_1's multi_logloss: 0.333124\tvalid_1's loss is: : 0.57568\n",
      "[133]\ttraining's multi_logloss: 0.00741437\ttraining's loss is: : 0.0147798\tvalid_1's multi_logloss: 0.333953\tvalid_1's loss is: : 0.577409\n",
      "[134]\ttraining's multi_logloss: 0.00714643\ttraining's loss is: : 0.0142472\tvalid_1's multi_logloss: 0.334653\tvalid_1's loss is: : 0.578877\n",
      "[135]\ttraining's multi_logloss: 0.00688838\ttraining's loss is: : 0.0137341\tvalid_1's multi_logloss: 0.335446\tvalid_1's loss is: : 0.580554\n",
      "[136]\ttraining's multi_logloss: 0.00663641\ttraining's loss is: : 0.0132331\tvalid_1's multi_logloss: 0.336295\tvalid_1's loss is: : 0.58228\n",
      "[137]\ttraining's multi_logloss: 0.00639192\ttraining's loss is: : 0.0127469\tvalid_1's multi_logloss: 0.337232\tvalid_1's loss is: : 0.58419\n",
      "[138]\ttraining's multi_logloss: 0.00615912\ttraining's loss is: : 0.0122837\tvalid_1's multi_logloss: 0.33828\tvalid_1's loss is: : 0.58626\n",
      "[139]\ttraining's multi_logloss: 0.00593774\ttraining's loss is: : 0.0118433\tvalid_1's multi_logloss: 0.339247\tvalid_1's loss is: : 0.588201\n",
      "[140]\ttraining's multi_logloss: 0.00572555\ttraining's loss is: : 0.011421\tvalid_1's multi_logloss: 0.339976\tvalid_1's loss is: : 0.589764\n",
      "[141]\ttraining's multi_logloss: 0.00551842\ttraining's loss is: : 0.0110087\tvalid_1's multi_logloss: 0.340873\tvalid_1's loss is: : 0.591587\n",
      "[142]\ttraining's multi_logloss: 0.00531676\ttraining's loss is: : 0.0106072\tvalid_1's multi_logloss: 0.341771\tvalid_1's loss is: : 0.593489\n",
      "[143]\ttraining's multi_logloss: 0.00512512\ttraining's loss is: : 0.0102256\tvalid_1's multi_logloss: 0.34267\tvalid_1's loss is: : 0.595339\n",
      "Early stopping, best iteration is:\n",
      "[93]\ttraining's multi_logloss: 0.0339552\ttraining's loss is: : 0.0670518\tvalid_1's multi_logloss: 0.316638\tvalid_1's loss is: : 0.536914\n"
     ]
    }
   ],
   "source": [
    "train_features = [col for col in train_data_.columns if col!='label' and col!='file_id']\n",
    "train_label = 'label'\n",
    "print(len(train_features))\n",
    "train_X, test_X, train_Y, test_Y = train_test_split( train_data_[train_features],train_data_[train_label].values, test_size = 0.33) \n",
    "\n",
    "dtrain = lgb.Dataset(train_X,train_Y) \n",
    "dval   = lgb.Dataset(test_X,test_Y, reference = dtrain) \n",
    "\n",
    "params = {\n",
    "        'task':'train', \n",
    "        'num_leaves': 255,\n",
    "        'objective': 'multiclass',\n",
    "        'num_class':8,\n",
    "        #'min_data_in_leaf': 40,\n",
    "        'min_data_in_leaf': 10,\n",
    "        'learning_rate': 0.05,\n",
    "        'feature_fraction': 0.85,\n",
    "        'bagging_fraction': 0.9,\n",
    "        'bagging_freq': 5, \n",
    "        'max_bin':128,\n",
    "        'num_threads': 64,\n",
    "        'random_state':100\n",
    "    }  \n",
    "lgb_model_3_order = lgb.train(params, dtrain, num_boost_round=500,valid_sets=[dtrain,dval], early_stopping_rounds=50, feval=lgb_logloss)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "fea_imp = pd.DataFrame({'feature':train_features, 'imp':lgb_model_3_order.feature_importance()}).sort_values('imp')\n",
    "important_features = fea_imp.loc[fea_imp.imp >=1, 'feature'].values\n",
    "important_features = list(important_features)\n",
    "\n",
    "important_features.append('file_id')\n",
    "important_features.append('label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data_[important_features].to_csv('../feature_final/train_data_2gram.csv',index = None)\n",
    "\n",
    "train_ind = train_X.index\n",
    "test_ind = test_X.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(important_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:4: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min\n"
     ]
    }
   ],
   "source": [
    "api_opts = ['max','min']\n",
    "train_data_,groupby_features = groupby_pivot_features(train_data_[important_features], train.loc[train.api_2_count>=20], groupby_features, col1 = 'api_2', col2 = 'index', opts = api_opts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\ttraining's multi_logloss: 1.90856\ttraining's loss is: : 2.81654\tvalid_1's multi_logloss: 1.92087\tvalid_1's loss is: : 2.83077\n",
      "Training until validation scores don't improve for 50 rounds.\n",
      "[2]\ttraining's multi_logloss: 1.76571\ttraining's loss is: : 2.64745\tvalid_1's multi_logloss: 1.78921\tvalid_1's loss is: : 2.67497\n",
      "[3]\ttraining's multi_logloss: 1.64335\ttraining's loss is: : 2.49943\tvalid_1's multi_logloss: 1.67637\tvalid_1's loss is: : 2.53858\n",
      "[4]\ttraining's multi_logloss: 1.53597\ttraining's loss is: : 2.3668\tvalid_1's multi_logloss: 1.5782\tvalid_1's loss is: : 2.41745\n",
      "[5]\ttraining's multi_logloss: 1.44075\ttraining's loss is: : 2.24685\tvalid_1's multi_logloss: 1.49107\tvalid_1's loss is: : 2.30783\n",
      "[6]\ttraining's multi_logloss: 1.3547\ttraining's loss is: : 2.13643\tvalid_1's multi_logloss: 1.41293\tvalid_1's loss is: : 2.20775\n",
      "[7]\ttraining's multi_logloss: 1.27698\ttraining's loss is: : 2.03491\tvalid_1's multi_logloss: 1.34204\tvalid_1's loss is: : 2.11535\n",
      "[8]\ttraining's multi_logloss: 1.20605\ttraining's loss is: : 1.94062\tvalid_1's multi_logloss: 1.27811\tvalid_1's loss is: : 2.03057\n",
      "[9]\ttraining's multi_logloss: 1.14103\ttraining's loss is: : 1.85278\tvalid_1's multi_logloss: 1.2195\tvalid_1's loss is: : 1.95166\n",
      "[10]\ttraining's multi_logloss: 1.0812\ttraining's loss is: : 1.77064\tvalid_1's multi_logloss: 1.16593\tvalid_1's loss is: : 1.87834\n",
      "[11]\ttraining's multi_logloss: 1.02552\ttraining's loss is: : 1.69311\tvalid_1's multi_logloss: 1.11574\tvalid_1's loss is: : 1.80875\n",
      "[12]\ttraining's multi_logloss: 0.973664\ttraining's loss is: : 1.61985\tvalid_1's multi_logloss: 1.06962\tvalid_1's loss is: : 1.74385\n",
      "[13]\ttraining's multi_logloss: 0.925313\ttraining's loss is: : 1.55059\tvalid_1's multi_logloss: 1.02654\tvalid_1's loss is: : 1.68245\n",
      "[14]\ttraining's multi_logloss: 0.880372\ttraining's loss is: : 1.48533\tvalid_1's multi_logloss: 0.986352\tvalid_1's loss is: : 1.62442\n",
      "[15]\ttraining's multi_logloss: 0.838061\ttraining's loss is: : 1.42313\tvalid_1's multi_logloss: 0.948768\tvalid_1's loss is: : 1.56946\n",
      "[16]\ttraining's multi_logloss: 0.798185\ttraining's loss is: : 1.36384\tvalid_1's multi_logloss: 0.913706\tvalid_1's loss is: : 1.5176\n",
      "[17]\ttraining's multi_logloss: 0.760602\ttraining's loss is: : 1.30733\tvalid_1's multi_logloss: 0.88053\tvalid_1's loss is: : 1.46803\n",
      "[18]\ttraining's multi_logloss: 0.725431\ttraining's loss is: : 1.25382\tvalid_1's multi_logloss: 0.84948\tvalid_1's loss is: : 1.4211\n",
      "[19]\ttraining's multi_logloss: 0.692034\ttraining's loss is: : 1.2025\tvalid_1's multi_logloss: 0.820372\tvalid_1's loss is: : 1.37661\n",
      "[20]\ttraining's multi_logloss: 0.66084\ttraining's loss is: : 1.15401\tvalid_1's multi_logloss: 0.793067\tvalid_1's loss is: : 1.33447\n",
      "[21]\ttraining's multi_logloss: 0.631066\ttraining's loss is: : 1.10737\tvalid_1's multi_logloss: 0.767356\tvalid_1's loss is: : 1.29437\n",
      "[22]\ttraining's multi_logloss: 0.602775\ttraining's loss is: : 1.06266\tvalid_1's multi_logloss: 0.742825\tvalid_1's loss is: : 1.25585\n",
      "[23]\ttraining's multi_logloss: 0.57613\ttraining's loss is: : 1.02016\tvalid_1's multi_logloss: 0.719977\tvalid_1's loss is: : 1.21958\n",
      "[24]\ttraining's multi_logloss: 0.550826\ttraining's loss is: : 0.979467\tvalid_1's multi_logloss: 0.698204\tvalid_1's loss is: : 1.18486\n",
      "[25]\ttraining's multi_logloss: 0.526815\ttraining's loss is: : 0.940518\tvalid_1's multi_logloss: 0.677629\tvalid_1's loss is: : 1.15177\n",
      "[26]\ttraining's multi_logloss: 0.504013\ttraining's loss is: : 0.903269\tvalid_1's multi_logloss: 0.658216\tvalid_1's loss is: : 1.12033\n",
      "[27]\ttraining's multi_logloss: 0.482214\ttraining's loss is: : 0.867411\tvalid_1's multi_logloss: 0.639891\tvalid_1's loss is: : 1.09044\n",
      "[28]\ttraining's multi_logloss: 0.461599\ttraining's loss is: : 0.833237\tvalid_1's multi_logloss: 0.6227\tvalid_1's loss is: : 1.06213\n",
      "[29]\ttraining's multi_logloss: 0.442041\ttraining's loss is: : 0.800584\tvalid_1's multi_logloss: 0.606131\tvalid_1's loss is: : 1.03475\n",
      "[30]\ttraining's multi_logloss: 0.423349\ttraining's loss is: : 0.769185\tvalid_1's multi_logloss: 0.590357\tvalid_1's loss is: : 1.00865\n",
      "[31]\ttraining's multi_logloss: 0.405361\ttraining's loss is: : 0.738846\tvalid_1's multi_logloss: 0.575381\tvalid_1's loss is: : 0.98373\n",
      "[32]\ttraining's multi_logloss: 0.38829\ttraining's loss is: : 0.709874\tvalid_1's multi_logloss: 0.561571\tvalid_1's loss is: : 0.960484\n",
      "[33]\ttraining's multi_logloss: 0.372023\ttraining's loss is: : 0.682108\tvalid_1's multi_logloss: 0.548281\tvalid_1's loss is: : 0.938055\n",
      "[34]\ttraining's multi_logloss: 0.356577\ttraining's loss is: : 0.655576\tvalid_1's multi_logloss: 0.535722\tvalid_1's loss is: : 0.91675\n",
      "[35]\ttraining's multi_logloss: 0.341752\ttraining's loss is: : 0.630004\tvalid_1's multi_logloss: 0.523803\tvalid_1's loss is: : 0.896492\n",
      "[36]\ttraining's multi_logloss: 0.327575\ttraining's loss is: : 0.605448\tvalid_1's multi_logloss: 0.512377\tvalid_1's loss is: : 0.877007\n",
      "[37]\ttraining's multi_logloss: 0.314073\ttraining's loss is: : 0.58193\tvalid_1's multi_logloss: 0.501429\tvalid_1's loss is: : 0.858225\n",
      "[38]\ttraining's multi_logloss: 0.301198\ttraining's loss is: : 0.559409\tvalid_1's multi_logloss: 0.491126\tvalid_1's loss is: : 0.840488\n",
      "[39]\ttraining's multi_logloss: 0.288903\ttraining's loss is: : 0.537801\tvalid_1's multi_logloss: 0.481406\tvalid_1's loss is: : 0.823662\n",
      "[40]\ttraining's multi_logloss: 0.277206\ttraining's loss is: : 0.517148\tvalid_1's multi_logloss: 0.472108\tvalid_1's loss is: : 0.807555\n",
      "[41]\ttraining's multi_logloss: 0.265854\ttraining's loss is: : 0.497057\tvalid_1's multi_logloss: 0.463293\tvalid_1's loss is: : 0.792235\n",
      "[42]\ttraining's multi_logloss: 0.255063\ttraining's loss is: : 0.477885\tvalid_1's multi_logloss: 0.45489\tvalid_1's loss is: : 0.777557\n",
      "[43]\ttraining's multi_logloss: 0.244806\ttraining's loss is: : 0.459579\tvalid_1's multi_logloss: 0.447024\tvalid_1's loss is: : 0.763801\n",
      "[44]\ttraining's multi_logloss: 0.234904\ttraining's loss is: : 0.441845\tvalid_1's multi_logloss: 0.439296\tvalid_1's loss is: : 0.750265\n",
      "[45]\ttraining's multi_logloss: 0.225465\ttraining's loss is: : 0.424882\tvalid_1's multi_logloss: 0.431952\tvalid_1's loss is: : 0.737405\n",
      "[46]\ttraining's multi_logloss: 0.216386\ttraining's loss is: : 0.408519\tvalid_1's multi_logloss: 0.425105\tvalid_1's loss is: : 0.725385\n",
      "[47]\ttraining's multi_logloss: 0.207744\ttraining's loss is: : 0.392884\tvalid_1's multi_logloss: 0.418736\tvalid_1's loss is: : 0.714137\n",
      "[48]\ttraining's multi_logloss: 0.199455\ttraining's loss is: : 0.377834\tvalid_1's multi_logloss: 0.412657\tvalid_1's loss is: : 0.703386\n",
      "[49]\ttraining's multi_logloss: 0.191525\ttraining's loss is: : 0.363393\tvalid_1's multi_logloss: 0.406751\tvalid_1's loss is: : 0.69296\n",
      "[50]\ttraining's multi_logloss: 0.183917\ttraining's loss is: : 0.349498\tvalid_1's multi_logloss: 0.401374\tvalid_1's loss is: : 0.683354\n",
      "[51]\ttraining's multi_logloss: 0.17659\ttraining's loss is: : 0.336089\tvalid_1's multi_logloss: 0.396031\tvalid_1's loss is: : 0.673881\n",
      "[52]\ttraining's multi_logloss: 0.16958\ttraining's loss is: : 0.323231\tvalid_1's multi_logloss: 0.390834\tvalid_1's loss is: : 0.664689\n",
      "[53]\ttraining's multi_logloss: 0.162885\ttraining's loss is: : 0.310917\tvalid_1's multi_logloss: 0.385976\tvalid_1's loss is: : 0.656052\n",
      "[54]\ttraining's multi_logloss: 0.156498\ttraining's loss is: : 0.299128\tvalid_1's multi_logloss: 0.381328\tvalid_1's loss is: : 0.647799\n",
      "[55]\ttraining's multi_logloss: 0.150363\ttraining's loss is: : 0.287775\tvalid_1's multi_logloss: 0.37704\tvalid_1's loss is: : 0.640127\n",
      "[56]\ttraining's multi_logloss: 0.144448\ttraining's loss is: : 0.276819\tvalid_1's multi_logloss: 0.372944\tvalid_1's loss is: : 0.63283\n",
      "[57]\ttraining's multi_logloss: 0.138778\ttraining's loss is: : 0.266293\tvalid_1's multi_logloss: 0.368911\tvalid_1's loss is: : 0.625716\n",
      "[58]\ttraining's multi_logloss: 0.133322\ttraining's loss is: : 0.256147\tvalid_1's multi_logloss: 0.365084\tvalid_1's loss is: : 0.618963\n",
      "[59]\ttraining's multi_logloss: 0.128106\ttraining's loss is: : 0.246422\tvalid_1's multi_logloss: 0.361351\tvalid_1's loss is: : 0.612393\n",
      "[60]\ttraining's multi_logloss: 0.123115\ttraining's loss is: : 0.237091\tvalid_1's multi_logloss: 0.35796\tvalid_1's loss is: : 0.606445\n",
      "[61]\ttraining's multi_logloss: 0.118251\ttraining's loss is: : 0.227995\tvalid_1's multi_logloss: 0.354818\tvalid_1's loss is: : 0.600877\n",
      "[62]\ttraining's multi_logloss: 0.113634\ttraining's loss is: : 0.219343\tvalid_1's multi_logloss: 0.351774\tvalid_1's loss is: : 0.595464\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[63]\ttraining's multi_logloss: 0.109227\ttraining's loss is: : 0.21106\tvalid_1's multi_logloss: 0.348992\tvalid_1's loss is: : 0.590537\n",
      "[64]\ttraining's multi_logloss: 0.104992\ttraining's loss is: : 0.203088\tvalid_1's multi_logloss: 0.346622\tvalid_1's loss is: : 0.586319\n",
      "[65]\ttraining's multi_logloss: 0.100929\ttraining's loss is: : 0.195426\tvalid_1's multi_logloss: 0.344024\tvalid_1's loss is: : 0.581803\n",
      "[66]\ttraining's multi_logloss: 0.0969796\ttraining's loss is: : 0.18797\tvalid_1's multi_logloss: 0.341403\tvalid_1's loss is: : 0.577185\n",
      "[67]\ttraining's multi_logloss: 0.0931925\ttraining's loss is: : 0.180804\tvalid_1's multi_logloss: 0.338987\tvalid_1's loss is: : 0.572995\n",
      "[68]\ttraining's multi_logloss: 0.0896026\ttraining's loss is: : 0.173997\tvalid_1's multi_logloss: 0.336951\tvalid_1's loss is: : 0.569401\n",
      "[69]\ttraining's multi_logloss: 0.0861973\ttraining's loss is: : 0.167526\tvalid_1's multi_logloss: 0.334829\tvalid_1's loss is: : 0.56571\n",
      "[70]\ttraining's multi_logloss: 0.0829002\ttraining's loss is: : 0.161254\tvalid_1's multi_logloss: 0.33283\tvalid_1's loss is: : 0.562302\n",
      "[71]\ttraining's multi_logloss: 0.07969\ttraining's loss is: : 0.155142\tvalid_1's multi_logloss: 0.331024\tvalid_1's loss is: : 0.559239\n",
      "[72]\ttraining's multi_logloss: 0.0766467\ttraining's loss is: : 0.149335\tvalid_1's multi_logloss: 0.329337\tvalid_1's loss is: : 0.55632\n",
      "[73]\ttraining's multi_logloss: 0.0737028\ttraining's loss is: : 0.14371\tvalid_1's multi_logloss: 0.327751\tvalid_1's loss is: : 0.553626\n",
      "[74]\ttraining's multi_logloss: 0.0709025\ttraining's loss is: : 0.138348\tvalid_1's multi_logloss: 0.326291\tvalid_1's loss is: : 0.551145\n",
      "[75]\ttraining's multi_logloss: 0.0682038\ttraining's loss is: : 0.133179\tvalid_1's multi_logloss: 0.324739\tvalid_1's loss is: : 0.548556\n",
      "[76]\ttraining's multi_logloss: 0.0656016\ttraining's loss is: : 0.128188\tvalid_1's multi_logloss: 0.323531\tvalid_1's loss is: : 0.546578\n",
      "[77]\ttraining's multi_logloss: 0.0631202\ttraining's loss is: : 0.123422\tvalid_1's multi_logloss: 0.322209\tvalid_1's loss is: : 0.544378\n",
      "[78]\ttraining's multi_logloss: 0.0607134\ttraining's loss is: : 0.118797\tvalid_1's multi_logloss: 0.320998\tvalid_1's loss is: : 0.542421\n",
      "[79]\ttraining's multi_logloss: 0.058416\ttraining's loss is: : 0.114373\tvalid_1's multi_logloss: 0.319989\tvalid_1's loss is: : 0.540834\n",
      "[80]\ttraining's multi_logloss: 0.0562149\ttraining's loss is: : 0.110128\tvalid_1's multi_logloss: 0.318793\tvalid_1's loss is: : 0.538954\n",
      "[81]\ttraining's multi_logloss: 0.0540264\ttraining's loss is: : 0.105917\tvalid_1's multi_logloss: 0.317965\tvalid_1's loss is: : 0.537623\n",
      "[82]\ttraining's multi_logloss: 0.0519424\ttraining's loss is: : 0.101899\tvalid_1's multi_logloss: 0.317027\tvalid_1's loss is: : 0.536126\n",
      "[83]\ttraining's multi_logloss: 0.0499316\ttraining's loss is: : 0.098018\tvalid_1's multi_logloss: 0.316493\tvalid_1's loss is: : 0.535306\n",
      "[84]\ttraining's multi_logloss: 0.0480161\ttraining's loss is: : 0.0943168\tvalid_1's multi_logloss: 0.315846\tvalid_1's loss is: : 0.534344\n",
      "[85]\ttraining's multi_logloss: 0.0461856\ttraining's loss is: : 0.0907753\tvalid_1's multi_logloss: 0.315413\tvalid_1's loss is: : 0.533704\n",
      "[86]\ttraining's multi_logloss: 0.0444196\ttraining's loss is: : 0.0873581\tvalid_1's multi_logloss: 0.314913\tvalid_1's loss is: : 0.532988\n",
      "[87]\ttraining's multi_logloss: 0.0426963\ttraining's loss is: : 0.0840181\tvalid_1's multi_logloss: 0.314411\tvalid_1's loss is: : 0.532309\n",
      "[88]\ttraining's multi_logloss: 0.041071\ttraining's loss is: : 0.0808642\tvalid_1's multi_logloss: 0.314007\tvalid_1's loss is: : 0.531822\n",
      "[89]\ttraining's multi_logloss: 0.0395194\ttraining's loss is: : 0.077849\tvalid_1's multi_logloss: 0.313766\tvalid_1's loss is: : 0.531584\n",
      "[90]\ttraining's multi_logloss: 0.0380299\ttraining's loss is: : 0.0749524\tvalid_1's multi_logloss: 0.313581\tvalid_1's loss is: : 0.531378\n",
      "[91]\ttraining's multi_logloss: 0.036561\ttraining's loss is: : 0.0720928\tvalid_1's multi_logloss: 0.313303\tvalid_1's loss is: : 0.531137\n",
      "[92]\ttraining's multi_logloss: 0.0351847\ttraining's loss is: : 0.0694093\tvalid_1's multi_logloss: 0.313045\tvalid_1's loss is: : 0.53092\n",
      "[93]\ttraining's multi_logloss: 0.0338654\ttraining's loss is: : 0.0668345\tvalid_1's multi_logloss: 0.313055\tvalid_1's loss is: : 0.531068\n",
      "[94]\ttraining's multi_logloss: 0.0325848\ttraining's loss is: : 0.0643339\tvalid_1's multi_logloss: 0.312958\tvalid_1's loss is: : 0.531118\n",
      "[95]\ttraining's multi_logloss: 0.0313701\ttraining's loss is: : 0.0619593\tvalid_1's multi_logloss: 0.313066\tvalid_1's loss is: : 0.531509\n",
      "[96]\ttraining's multi_logloss: 0.0301774\ttraining's loss is: : 0.0596289\tvalid_1's multi_logloss: 0.313282\tvalid_1's loss is: : 0.532102\n",
      "[97]\ttraining's multi_logloss: 0.0290451\ttraining's loss is: : 0.0574143\tvalid_1's multi_logloss: 0.313334\tvalid_1's loss is: : 0.532427\n",
      "[98]\ttraining's multi_logloss: 0.0279471\ttraining's loss is: : 0.0552652\tvalid_1's multi_logloss: 0.313324\tvalid_1's loss is: : 0.532598\n",
      "[99]\ttraining's multi_logloss: 0.0268992\ttraining's loss is: : 0.0532118\tvalid_1's multi_logloss: 0.313394\tvalid_1's loss is: : 0.533006\n",
      "[100]\ttraining's multi_logloss: 0.0259088\ttraining's loss is: : 0.0512695\tvalid_1's multi_logloss: 0.313319\tvalid_1's loss is: : 0.53317\n",
      "[101]\ttraining's multi_logloss: 0.0249446\ttraining's loss is: : 0.0493781\tvalid_1's multi_logloss: 0.313482\tvalid_1's loss is: : 0.533695\n",
      "[102]\ttraining's multi_logloss: 0.0240324\ttraining's loss is: : 0.0475872\tvalid_1's multi_logloss: 0.313884\tvalid_1's loss is: : 0.53469\n",
      "[103]\ttraining's multi_logloss: 0.0231568\ttraining's loss is: : 0.0458667\tvalid_1's multi_logloss: 0.314036\tvalid_1's loss is: : 0.535219\n",
      "[104]\ttraining's multi_logloss: 0.0223152\ttraining's loss is: : 0.0442121\tvalid_1's multi_logloss: 0.314356\tvalid_1's loss is: : 0.53607\n",
      "[105]\ttraining's multi_logloss: 0.0215157\ttraining's loss is: : 0.0426391\tvalid_1's multi_logloss: 0.314524\tvalid_1's loss is: : 0.536611\n",
      "[106]\ttraining's multi_logloss: 0.0206912\ttraining's loss is: : 0.041019\tvalid_1's multi_logloss: 0.314821\tvalid_1's loss is: : 0.537429\n",
      "[107]\ttraining's multi_logloss: 0.0199123\ttraining's loss is: : 0.039487\tvalid_1's multi_logloss: 0.314992\tvalid_1's loss is: : 0.537994\n",
      "[108]\ttraining's multi_logloss: 0.0191758\ttraining's loss is: : 0.0380373\tvalid_1's multi_logloss: 0.315565\tvalid_1's loss is: : 0.539241\n",
      "[109]\ttraining's multi_logloss: 0.0184519\ttraining's loss is: : 0.0366114\tvalid_1's multi_logloss: 0.315922\tvalid_1's loss is: : 0.540145\n",
      "[110]\ttraining's multi_logloss: 0.0177683\ttraining's loss is: : 0.0352644\tvalid_1's multi_logloss: 0.316363\tvalid_1's loss is: : 0.541237\n",
      "[111]\ttraining's multi_logloss: 0.0170964\ttraining's loss is: : 0.0339402\tvalid_1's multi_logloss: 0.316859\tvalid_1's loss is: : 0.542405\n",
      "[112]\ttraining's multi_logloss: 0.0164412\ttraining's loss is: : 0.032648\tvalid_1's multi_logloss: 0.317175\tvalid_1's loss is: : 0.543214\n",
      "[113]\ttraining's multi_logloss: 0.0158187\ttraining's loss is: : 0.0314193\tvalid_1's multi_logloss: 0.317531\tvalid_1's loss is: : 0.544145\n",
      "[114]\ttraining's multi_logloss: 0.0152268\ttraining's loss is: : 0.0302503\tvalid_1's multi_logloss: 0.317817\tvalid_1's loss is: : 0.544971\n",
      "[115]\ttraining's multi_logloss: 0.0146469\ttraining's loss is: : 0.0291045\tvalid_1's multi_logloss: 0.318315\tvalid_1's loss is: : 0.546196\n",
      "[116]\ttraining's multi_logloss: 0.0141012\ttraining's loss is: : 0.0280261\tvalid_1's multi_logloss: 0.318916\tvalid_1's loss is: : 0.54742\n",
      "[117]\ttraining's multi_logloss: 0.0135816\ttraining's loss is: : 0.026999\tvalid_1's multi_logloss: 0.319392\tvalid_1's loss is: : 0.548526\n",
      "[118]\ttraining's multi_logloss: 0.0130827\ttraining's loss is: : 0.0260119\tvalid_1's multi_logloss: 0.320026\tvalid_1's loss is: : 0.5499\n",
      "[119]\ttraining's multi_logloss: 0.0126057\ttraining's loss is: : 0.025068\tvalid_1's multi_logloss: 0.320553\tvalid_1's loss is: : 0.551048\n",
      "[120]\ttraining's multi_logloss: 0.0121537\ttraining's loss is: : 0.024173\tvalid_1's multi_logloss: 0.321273\tvalid_1's loss is: : 0.55258\n",
      "[121]\ttraining's multi_logloss: 0.0117011\ttraining's loss is: : 0.0232771\tvalid_1's multi_logloss: 0.321973\tvalid_1's loss is: : 0.554042\n",
      "[122]\ttraining's multi_logloss: 0.0112617\ttraining's loss is: : 0.0224074\tvalid_1's multi_logloss: 0.322685\tvalid_1's loss is: : 0.555506\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[123]\ttraining's multi_logloss: 0.0108377\ttraining's loss is: : 0.0215677\tvalid_1's multi_logloss: 0.323428\tvalid_1's loss is: : 0.55709\n",
      "[124]\ttraining's multi_logloss: 0.0104344\ttraining's loss is: : 0.0207686\tvalid_1's multi_logloss: 0.324015\tvalid_1's loss is: : 0.558444\n",
      "[125]\ttraining's multi_logloss: 0.0100475\ttraining's loss is: : 0.0200016\tvalid_1's multi_logloss: 0.324785\tvalid_1's loss is: : 0.560056\n",
      "[126]\ttraining's multi_logloss: 0.00967195\ttraining's loss is: : 0.019257\tvalid_1's multi_logloss: 0.325642\tvalid_1's loss is: : 0.561782\n",
      "[127]\ttraining's multi_logloss: 0.00930444\ttraining's loss is: : 0.0185283\tvalid_1's multi_logloss: 0.326291\tvalid_1's loss is: : 0.563197\n",
      "[128]\ttraining's multi_logloss: 0.00895312\ttraining's loss is: : 0.0178314\tvalid_1's multi_logloss: 0.326888\tvalid_1's loss is: : 0.564504\n",
      "[129]\ttraining's multi_logloss: 0.0086164\ttraining's loss is: : 0.0171633\tvalid_1's multi_logloss: 0.327696\tvalid_1's loss is: : 0.566191\n",
      "[130]\ttraining's multi_logloss: 0.00829792\ttraining's loss is: : 0.0165312\tvalid_1's multi_logloss: 0.328397\tvalid_1's loss is: : 0.567624\n",
      "[131]\ttraining's multi_logloss: 0.0079909\ttraining's loss is: : 0.0159217\tvalid_1's multi_logloss: 0.329184\tvalid_1's loss is: : 0.569265\n",
      "[132]\ttraining's multi_logloss: 0.00769486\ttraining's loss is: : 0.0153339\tvalid_1's multi_logloss: 0.329996\tvalid_1's loss is: : 0.571017\n",
      "[133]\ttraining's multi_logloss: 0.00741302\ttraining's loss is: : 0.0147739\tvalid_1's multi_logloss: 0.331047\tvalid_1's loss is: : 0.573132\n",
      "[134]\ttraining's multi_logloss: 0.00714447\ttraining's loss is: : 0.0142402\tvalid_1's multi_logloss: 0.332074\tvalid_1's loss is: : 0.57521\n",
      "[135]\ttraining's multi_logloss: 0.006883\ttraining's loss is: : 0.0137205\tvalid_1's multi_logloss: 0.332896\tvalid_1's loss is: : 0.576965\n",
      "[136]\ttraining's multi_logloss: 0.00662988\ttraining's loss is: : 0.0132173\tvalid_1's multi_logloss: 0.333774\tvalid_1's loss is: : 0.578786\n",
      "[137]\ttraining's multi_logloss: 0.00638811\ttraining's loss is: : 0.0127367\tvalid_1's multi_logloss: 0.33461\tvalid_1's loss is: : 0.580465\n",
      "[138]\ttraining's multi_logloss: 0.00615925\ttraining's loss is: : 0.0122815\tvalid_1's multi_logloss: 0.335551\tvalid_1's loss is: : 0.582483\n",
      "[139]\ttraining's multi_logloss: 0.00594238\ttraining's loss is: : 0.01185\tvalid_1's multi_logloss: 0.336584\tvalid_1's loss is: : 0.584654\n",
      "[140]\ttraining's multi_logloss: 0.00573192\ttraining's loss is: : 0.0114312\tvalid_1's multi_logloss: 0.337496\tvalid_1's loss is: : 0.586552\n",
      "[141]\ttraining's multi_logloss: 0.0055185\ttraining's loss is: : 0.0110067\tvalid_1's multi_logloss: 0.338122\tvalid_1's loss is: : 0.587956\n",
      "[142]\ttraining's multi_logloss: 0.00531674\ttraining's loss is: : 0.0106053\tvalid_1's multi_logloss: 0.33908\tvalid_1's loss is: : 0.589865\n",
      "Early stopping, best iteration is:\n",
      "[92]\ttraining's multi_logloss: 0.0351847\ttraining's loss is: : 0.0694093\tvalid_1's multi_logloss: 0.313045\tvalid_1's loss is: : 0.53092\n"
     ]
    }
   ],
   "source": [
    "train_features = [col for col in train_data_.columns if col!='label' and col!='file_id' and 'std' not in col and 'quantile' not in col]\n",
    "train_label = 'label'\n",
    "\n",
    "train_ind = train_X.index\n",
    "test_ind = test_X.index\n",
    "\n",
    "dtrain = lgb.Dataset(train_data_.loc[train_ind,train_features],train_data_.loc[train_ind,train_label].values) \n",
    "dval   = lgb.Dataset(train_data_.loc[test_ind,train_features],train_data_.loc[test_ind,train_label].values, reference = dtrain) \n",
    "\n",
    "params = {\n",
    "        'task':'train', \n",
    "        'num_leaves': 255,\n",
    "        'objective': 'multiclass',\n",
    "        'num_class':8,\n",
    "        'min_data_in_leaf': 10,\n",
    "        #'min_data_in_leaf': 1,\n",
    "        'learning_rate': 0.05,\n",
    "        'feature_fraction': 0.85,\n",
    "        'bagging_fraction': 0.9,\n",
    "        'bagging_freq': 5, \n",
    "        'max_bin':128,\n",
    "        'num_threads': 64,\n",
    "        'random_state':100\n",
    "    }  \n",
    "lgb_model_3_order = lgb.train(params, dtrain, num_boost_round=500,valid_sets=[dtrain,dval], early_stopping_rounds=50, feval=lgb_logloss)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "fea_imp = pd.DataFrame({'feature':train_features, 'imp':lgb_model_3_order.feature_importance()}).sort_values('imp')\n",
    "important_features = fea_imp.loc[fea_imp.imp >=1, 'feature'].values\n",
    "important_features = list(important_features)\n",
    "\n",
    "important_features.append('file_id')\n",
    "important_features.append('label')\n",
    "\n",
    "train_data_[important_features].to_csv('./train_data_2gram.csv',index = None)\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 附录\n",
    "tf-idf的1Gram特征可以替换api的次数特征等，加入tf-idf有提升，提升较小"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "384px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}