SVM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Outline\n",
    "\n",
    "#### Data set versions:\n",
    "- Original dataset\n",
    "- Synthesized dataset using SMOTE\n",
    "\n",
    "#### Approaches:\n",
    "- SVM + Grid Search\n",
    "- SVM + Grid Search + Feature selection with F-score\n",
    "- Ensemble SVM\n",
    "  - Og data\n",
    "  - Synthesized data\n",
    "  - F-score filtered data\n",
    "\n",
    "#### Evaluation Metrics:\n",
    "- ROC\n",
    "- F-score (Precision Recall)\n",
    "- Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-25T02:58:44.019430Z",
     "start_time": "2018-04-25T02:58:40.654984Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.svm import OneClassSVM\n",
    "\n",
    "from sklearn.preprocessing import Imputer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "# from sklearn.metrics import precision_score\n",
    "# from sklearn.metrics import recall_score\n",
    "\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# DATA = 'dataset/loan_one_hot_encoded.csv'\n",
    "DATA = 'dataset/data.csv'\n",
    "SYN_DATA = 'dataset/syn_data.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-25T02:58:44.466908Z",
     "start_time": "2018-04-25T02:58:44.022994Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " OG dataset:  (219, 242) (219,) (array([0, 1]), array([210,   9]))\n",
      "SYN dataset:  (234, 242) (234,) (array([0, 1]), array([210,  24]))\n"
     ]
    }
   ],
   "source": [
    "original_df = pd.read_csv(DATA)\n",
    "synthesized_df = pd.read_csv(SYN_DATA)\n",
    "\n",
    "OG_TARGET = original_df['loan_created']\n",
    "OG_FEATURES = original_df.drop(['loan_created', 'application_id'], axis=1)\n",
    "\n",
    "SYN_TARGET = synthesized_df['loan_approved']\n",
    "SYN_FEATURES = synthesized_df.drop(['loan_approved'], axis=1)\n",
    "\n",
    "# scale features to 0 mean & unit var\n",
    "X = StandardScaler().fit_transform(OG_FEATURES)\n",
    "Y = OG_TARGET\n",
    "\n",
    "X_os = StandardScaler().fit_transform(SYN_FEATURES)\n",
    "Y_os = SYN_TARGET\n",
    "\n",
    "print(' OG dataset: ', X.shape, Y.shape, np.unique(Y, return_counts=True))\n",
    "print('SYN dataset: ', X_os.shape, Y_os.shape, np.unique(Y_os, return_counts=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-25T02:58:44.475120Z",
     "start_time": "2018-04-25T02:58:44.470293Z"
    }
   },
   "outputs": [],
   "source": [
    "# drop_cols = ['loan_created', 'application_id',\n",
    "# #              'firm_type_Proprietorship',\n",
    "#              'average_business_inflow'\n",
    "#             ]\n",
    "# df = pd.read_csv(DATA)\n",
    "# Y = df['loan_created']\n",
    "# og_X = df.drop(drop_cols, axis=1)\n",
    "\n",
    "# imp = Imputer()\n",
    "# imputed_X = imp.fit_transform(og_X)\n",
    "\n",
    "# # X = imputed_X\n",
    "# scl = StandardScaler()\n",
    "# X = scl.fit_transform(imputed_X)\n",
    "\n",
    "# X.shape, Y.shape, np.unique(Y, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-25T02:58:44.482753Z",
     "start_time": "2018-04-25T02:58:44.478917Z"
    }
   },
   "outputs": [],
   "source": [
    "# sm = SMOTE(random_state = 44, ratio = {0:221, 1:24})\n",
    "# X_os, Y_os = sm.fit_sample(X, Y)\n",
    "\n",
    "# X_os.shape, Y_os.shape, np.unique(Y_os, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-25T02:59:35.238945Z",
     "start_time": "2018-04-25T02:59:35.215467Z"
    }
   },
   "outputs": [],
   "source": [
    "class GridSearchCV:\n",
    "    def __init__(self, param_grid, clf, pos_label, cv=5):\n",
    "        self.param_list_ = list(ParameterGrid(param_grid))\n",
    "        self.cv = cv\n",
    "        self.pos_label = pos_label\n",
    "        self.clf = clf\n",
    "        \n",
    "        self.overfit_ = [[] for _ in range(len(self.param_list_))]\n",
    "        \n",
    "        # evaluation scores\n",
    "        self.accuracy_scores_ = ([[] for _ in range(len(self.param_list_))])\n",
    "        self.precision_ = [[] for _ in range(len(self.param_list_))]\n",
    "        self.recall_ = [[] for _ in range(len(self.param_list_))]\n",
    "        self.f1_scores_ = [[] for _ in range(len(self.param_list_))]\n",
    "        self.fpr_ = [[] for _ in range(len(self.param_list_))]\n",
    "        self.tpr_ = [[] for _ in range(len(self.param_list_))]\n",
    "        self.auc_scores_ = [[] for _ in range(len(self.param_list_))]        \n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        skf = StratifiedKFold(n_splits=self.cv)\n",
    "        for train_index, test_index in skf.split(X, y):\n",
    "            for idx, params in enumerate(self.param_list_):\n",
    "                X_train, X_test = X[train_index], X[test_index]\n",
    "                y_train, y_test = y[train_index], y[test_index]\n",
    "                self.clf.set_params(**params)\n",
    "                self.clf.fit(X_train, y_train) # probability=True, class_weight='balanced', \n",
    "                y_pred = self.clf.predict(X_test)\n",
    "                if getattr(self.clf, \"predict_proba\", None):\n",
    "                    y_probab = self.clf.predict_proba(X_test)\n",
    "                else:\n",
    "                    y_pred[y_pred == -1] = 0\n",
    "\n",
    "                if np.all(y_pred[0] == y_pred):\n",
    "                    # all values predicted are same;\n",
    "                    self.overfit_[idx].append(True)\n",
    "\n",
    "                    self.precision_[idx].append([])\n",
    "                    self.recall_[idx].append([])\n",
    "                    self.f1_scores_[idx].append(np.nan)\n",
    "\n",
    "                    if getattr(self.clf, \"predict_proba\", None):\n",
    "                        self.fpr_[idx].append([])\n",
    "                        self.tpr_[idx].append([])\n",
    "                        self.auc_scores_[idx].append(np.nan)\n",
    "                else:\n",
    "                    self.overfit_[idx].append(False)\n",
    "                    precision, recall, pr_thresholds = precision_recall_curve(y_test, y_pred, pos_label=self.pos_label)\n",
    "                    self.precision_[idx].append(precision)\n",
    "                    self.recall_[idx].append(recall)\n",
    "                    self.f1_scores_[idx].append(f1_score(y_test, y_pred))\n",
    "                    if getattr(self.clf, \"predict_proba\", None):\n",
    "                        fpr, tpr, roc_thresholds = roc_curve(y_test, y_probab[:,1], pos_label=self.pos_label)\n",
    "                        self.fpr_[idx].append(fpr)\n",
    "                        self.tpr_[idx].append(tpr)\n",
    "                        self.auc_scores_[idx].append(auc(fpr, tpr))\n",
    "\n",
    "\n",
    "                self.accuracy_scores_[idx].append(accuracy_score(y_test, y_pred))\n",
    "        \n",
    "        self.average_accuracy_param_ = np.array([np.average(_) for _ in self.accuracy_scores_])\n",
    "    \n",
    "        self.accuracy_scores_ = np.array(self.accuracy_scores_)\n",
    "        self.precision_ = np.array(self.precision_)\n",
    "        self.recall_ = np.array(self.recall_)\n",
    "        self.f1_scores_ = np.array(self.f1_scores_)\n",
    "        self.fpr_ = np.array(self.fpr_)\n",
    "        self.tpr_ = np.array(self.tpr_)\n",
    "        self.auc_scores_ = np.array(self.auc_scores_)\n",
    "            \n",
    "    def print_results(self, show_overfit=False):\n",
    "        for _idx in range(len(self.average_accuracy_param_)):\n",
    "            if not show_overfit and np.any(self.overfit_[_idx]):\n",
    "                continue\n",
    "            print('-'*40)\n",
    "            print('overfit: ', np.any(self.overfit_[_idx]), np.unique(self.overfit_[_idx], return_counts=True))\n",
    "            print('params:', self.param_list_[_idx])\n",
    "            print('avg acc: ', self.average_accuracy_param_[_idx])\n",
    "            print('f1 score:', np.average(self.f1_scores_[_idx]))\n",
    "            print('auc: ', np.average(self.auc_scores_[_idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-25T02:59:35.756746Z",
     "start_time": "2018-04-25T02:59:35.740873Z"
    }
   },
   "outputs": [],
   "source": [
    "class FScoreFeatureSelection:\n",
    "    ''' Feature Selection based on F-Score of features '''\n",
    "    \n",
    "    def __init__(self, clf, param_grid, pos_label):\n",
    "        self.clf = clf\n",
    "        self.param_grid = param_grid\n",
    "        self.pos_label = pos_label\n",
    "        \n",
    "    def _split_per_class(self, X, Y):\n",
    "        return (np.array([_.ravel() for _ in X[np.argwhere(Y == self.pos_label)]]), \n",
    "                np.array([_.ravel() for _ in X[np.argwhere(Y != self.pos_label)]]))\n",
    "    \n",
    "    def _new_x(self, low_th, high_th):\n",
    "        chosen_feature_indices = []\n",
    "        for idx, fc in enumerate(self.f_scores):\n",
    "            if fc >= low_th and fc <= high_th:\n",
    "                chosen_feature_indices.append(idx)\n",
    "        return self._X[:, chosen_feature_indices]\n",
    "#         tmp = self.f_scores[self.f_scores >= low_th]\n",
    "#         tmp = tmp[tmp <= high_th]\n",
    "#         chosen_feature_indices = np.searchsorted(self.f_scores, tmp)\n",
    "#         print(self.f_scores)\n",
    "#         print(tmp)\n",
    "#         print(chosen_feature_indices)\n",
    "#         return self._X[:, chosen_feature_indices]\n",
    "        \n",
    "    def fit(self, X, Y, thresholds):\n",
    "        self._thresholds = thresholds\n",
    "        self._pos_x, self._neg_x = self._split_per_class(X, Y)\n",
    "        self._X, self._Y = X, Y\n",
    "        self.f_scores = np.array([self._f(i) for i in range(X.shape[1])]) # f-scores per feature\n",
    "        gscv_objs = []\n",
    "        \n",
    "        for low_th, high_th in thresholds:\n",
    "            new_X = self._new_x(low_th, high_th) # generate the new feature matrix based on chosen features\n",
    "            \n",
    "            # do 5-fold cross-validation on new_x using self.clf\n",
    "            gscv = GridSearchCV(self.param_grid, self.clf, self.pos_label)\n",
    "            gscv.fit(new_X, self._Y)\n",
    "            gscv_objs.append(gscv)\n",
    "        self.gscv_objs = np.array(gscv_objs)\n",
    "    \n",
    "    def _f(self, i):\n",
    "        ''' Calculate F-score of the ith feature '''\n",
    "        X, Y = self._X, self._Y\n",
    "        x_ip_avg = np.average(self._pos_x[:, i]) # Average of the ith feature of positive dataset\n",
    "        x_in_avg = np.average(self._neg_x[:, i]) # Average of the ith feature of negative dataset\n",
    "        x_i_avg = np.average(X[:, i]) # Average of the ith feature of whole dataset\n",
    "        \n",
    "        _num = ((x_ip_avg - x_i_avg)**2 + (x_in_avg - x_i_avg)**2)\n",
    "        \n",
    "        _den = (((1/(self._pos_x.shape[0]-1)) * np.sum([(self._pos_x[k][i] - x_ip_avg)**2 for k in range(self._pos_x.shape[0])])) +\n",
    "                ((1/(self._neg_x.shape[0]-1)) * np.sum([(self._neg_x[k][i] - x_in_avg)**2 for k in range(self._neg_x.shape[0])])))\n",
    "        \n",
    "        return _num/_den\n",
    "    \n",
    "    def new_X(low_th, high_th):\n",
    "        return self._new_x(low_th, high_th)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-25T02:59:38.328703Z",
     "start_time": "2018-04-25T02:59:36.765839Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sahil/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:55: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "****************************************\n",
      "(0.004, 0.02)\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 0.1, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.5570824524312896\n",
      "f1 score: 0.15873015873015872\n",
      "auc:  0.6690476190476191\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 0.5, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.6758985200845665\n",
      "f1 score: 0.20478260869565218\n",
      "auc:  0.8071428571428572\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 0.5, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.42568710359408024\n",
      "f1 score: 0.10038588862118274\n",
      "auc:  0.3619047619047619\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 1, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.6758985200845665\n",
      "f1 score: 0.20478260869565218\n",
      "auc:  0.8095238095238095\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 1, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.3058139534883721\n",
      "f1 score: 0.0824964664964665\n",
      "auc:  0.3619047619047619\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 5, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.7031712473572939\n",
      "f1 score: 0.12298773690078038\n",
      "auc:  0.8095238095238095\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 5, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.5025369978858352\n",
      "f1 score: 0.09738095238095237\n",
      "auc:  0.43571428571428567\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 10, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.7031712473572939\n",
      "f1 score: 0.12298773690078038\n",
      "auc:  0.811904761904762\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 10, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.49788583509513745\n",
      "f1 score: 0.08090111642743222\n",
      "auc:  0.6857142857142857\n"
     ]
    }
   ],
   "source": [
    "param_grid = [\n",
    "  {'C': [.1, .5, 1, 5, 10], \n",
    "   'gamma': ['auto'], 'kernel': ['rbf', 'sigmoid'], \n",
    "   'probability': [True], 'class_weight': ['balanced']},\n",
    " ]\n",
    "thresholds = [(.004, .02)]#, (.2, .8), (.2, .9), (.2, .7), (.2, .6)]\n",
    "# gscv = GridSearchCV(param_grid, clf, pos_label=1, cv=5)\n",
    "fs = FScoreFeatureSelection(SVC(), param_grid, pos_label=1)\n",
    "fs.fit(X, Y, thresholds)\n",
    "\n",
    "for i, _g in enumerate(fs.gscv_objs):\n",
    "    print('**'*20)\n",
    "    print(fs._thresholds[i])\n",
    "    _g.print_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T05:35:11.097049Z",
     "start_time": "2018-04-21T05:35:09.903453Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sahil/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:55: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    }
   ],
   "source": [
    "param_grid = [\n",
    "  {'C': [.1, .5, 1, 5, 10], \n",
    "   'gamma': ['auto'], 'kernel': ['rbf', 'sigmoid'], \n",
    "   'probability': [True], 'class_weight': ['balanced']},\n",
    " ]\n",
    "thresholds = [(.004, .02)]#, (.2, .8), (.2, .9), (.2, .7), (.2, .6)]\n",
    "# gscv = GridSearchCV(param_grid, clf, pos_label=1, cv=5)\n",
    "fs = FScoreFeatureSelection(SVC(), param_grid, pos_label=1)\n",
    "fs.fit(X_os, Y_os, thresholds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T05:35:11.381432Z",
     "start_time": "2018-04-21T05:35:11.100891Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "239\n",
      "223\n",
      "127 96\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEKJJREFUeJzt3WuMpmV9x/Hvr6yIYC0LDARZ6WKypR4iaEdqa2NSVhSlcTcRGrS1G0uzL2qt1lpZtYm16QswprYvmjabok4TKyBKlqjRrlvsKQZdDiIL0oV1xZUtO3LwRKIu/vtibnRcZ/a5Z57nmcO1308yuU/XPc//mmf2t9fch+dOVSFJWv1+YbkLkCSNhoEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJasSapXyx0047rdavX7+ULylJq96tt976raqaGNRuSQN9/fr17N69eylfUpJWvSRf79POQy6S1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktSIJb1TdBzWb/vUnOv3X3XJElciScvLEbokNcJAl6RGGOiS1AgDXZIaYaBLUiN6BXqSP0uyJ8ldST6a5IQk5yS5JcneJNclOX7cxUqS5jcw0JOcBfwpMFlVzweOAy4HrgY+UFUbgEeBK8ZZqCTp6PoeclkDPC3JGuBE4CBwIXBDt30K2Dz68iRJfQ0M9Kr6JvB+4AFmgvzbwK3AY1V1uGt2ADhrXEVKkgbrc8hlLbAJOAd4JnAS8Ko5mtY8+29NsjvJ7unp6WFqlSQdRZ9DLi8HvlZV01X1I+ATwG8CJ3eHYADWAQ/OtXNVba+qyaqanJgY+NBqSdIi9Qn0B4CXJDkxSYCNwN3AzcClXZstwI7xlChJ6qPPMfRbmDn5eRvwlW6f7cCVwNuS3AecClwzxjolSQP0+rTFqnoP8J4jVu8DLhh5RZKkRfFOUUlqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSI/o8JPrcJHfM+vpOkrcmOSXJziR7u+napShYkjS3Po+gu7eqzq+q84FfAx4HbgS2AbuqagOwq1uWJC2ThR5y2QjcX1VfBzYBU936KWDzKAuTJC3MQgP9cuCj3fwZVXUQoJuePtcOSbYm2Z1k9/T09OIrlSQdVe9AT3I88BrgYwt5garaXlWTVTU5MTGx0PokST0tZIT+KuC2qnqoW34oyZkA3fTQqIuTJPW3kEB/HT893AJwE7Clm98C7BhVUZKkhesV6ElOBC4CPjFr9VXARUn2dtuuGn15kqS+1vRpVFWPA6cese5hZq56kSStAN4pKkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiP6PuDi5CQ3JPlqknuS/EaSU5LsTLK3m64dd7GSpPn1HaH/PfCZqvpV4DzgHmAbsKuqNgC7umVJ0jIZGOhJngG8DLgGoKp+WFWPAZuAqa7ZFLB5XEVKkgbrM0J/NjANfCjJ7Un+OclJwBlVdRCgm54+xjolSQP0CfQ1wIuAf6yqFwLfZwGHV5JsTbI7ye7p6elFlilJGqRPoB8ADlTVLd3yDcwE/ENJzgTopofm2rmqtlfVZFVNTkxMjKJmSdIcBgZ6Vf0f8I0k53arNgJ3AzcBW7p1W4AdY6lQktTLmp7t3gx8JMnxwD7gjcz8Z3B9kiuAB4DLxlOiJKmPXoFeVXcAk3Ns2jjaciRJi+WdopLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRvR6wEWS/cB3gSeAw1U1meQU4DpgPbAf+N2qenQ8ZUqSBlnICP23q+r8qnryyUXbgF1VtQHY1S1LkpbJMIdcNgFT3fwUsHn4ciRJi9U30Av4tyS3JtnarTujqg4CdNPTx1GgJKmfXsfQgZdW1YNJTgd2Jvlq3xfo/gPYCnD22WcvokRJUh+9RuhV9WA3PQTcCFwAPJTkTIBuemiefbdX1WRVTU5MTIymaknSzxkY6ElOSvKLT84DrwDuAm4CtnTNtgA7xlWkJGmwPodczgBuTPJk+3+tqs8k+RJwfZIrgAeAy8ZXpiRpkIGBXlX7gPPmWP8wsHEcRUmSFs47RSWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5Jjegd6EmOS3J7kk92y+ckuSXJ3iTXJTl+fGVKkgZZyAj9LcA9s5avBj5QVRuAR4ErRlmYJGlhegV6knXAJcA/d8sBLgRu6JpMAZvHUaAkqZ++I/S/A94B/LhbPhV4rKoOd8sHgLPm2jHJ1iS7k+yenp4eqlhJ0vwGBnqS3wEOVdWts1fP0bTm2r+qtlfVZFVNTkxMLLJMSdIga3q0eSnwmiSvBk4AnsHMiP3kJGu6Ufo64MHxlSlJGmTgCL2q3llV66pqPXA58O9V9XvAzcClXbMtwI6xVSlJGmiY69CvBN6W5D5mjqlfM5qSJEmL0eeQy09U1eeBz3fz+4ALRl+SJGkxvFNUkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRvR5pugJSb6Y5MtJ9iR5b7f+nCS3JNmb5Lokx4+/XEnSfPqM0H8AXFhV5wHnAxcneQlwNfCBqtoAPApcMb4yJUmD9HmmaFXV97rFp3RfBVwI3NCtnwI2j6VCSVIvvY6hJzkuyR3AIWAncD/wWFUd7pocAM4aT4mSpD56BXpVPVFV5wPrmHmO6HPmajbXvkm2JtmdZPf09PTiK5UkHdWCrnKpqseYeUj0S4CTkzz5kOl1wIPz7LO9qiaranJiYmKYWiVJR9HnKpeJJCd3808DXg7cA9wMXNo12wLsGFeRkqTB1gxuwpnAVJLjmPkP4Pqq+mSSu4Frk/wNcDtwzRjrlCQNMDDQq+pO4IVzrN/HzPF0SdIK4J2iktQIA12SGtHnGHpT1m/71Jzr9191yRJXIkmj5QhdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEX0eQfesJDcnuSfJniRv6dafkmRnkr3ddO34y5UkzafPCP0w8OdV9RxmHg79piTPBbYBu6pqA7CrW5YkLZOBgV5VB6vqtm7+u8w8IPosYBMw1TWbAjaPq0hJ0mALOoaeZD0zzxe9BTijqg7CTOgDp8+zz9Yku5Psnp6eHq5aSdK8egd6kqcDHwfeWlXf6btfVW2vqsmqmpyYmFhMjZKkHnoFepKnMBPmH6mqT3SrH0pyZrf9TODQeEqUJPXR5yqXANcA91TV387adBOwpZvfAuwYfXmSpL76PCT6pcAbgK8kuaNb9y7gKuD6JFcADwCXjadESVIfAwO9qv4byDybN462nJVn/bZPzbtt/1WXLGElknR03ikqSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1Ij+txYpAWa79p1r1uXNE6O0CWpEQa6JDXCQJekRhjoktSIVXNS9GgfkiVJcoQuSc0w0CWpEX2eWPTBJIeS3DVr3SlJdibZ203XjrdMSdIgfUboHwYuPmLdNmBXVW0AdnXLkqRlNDDQq+o/gUeOWL0JmOrmp4DNI65LkrRAiz2GfkZVHQTopqePriRJ0mKM/bLFJFuBrQBnn332uF9uVfKzXySNwmJH6A8lOROgmx6ar2FVba+qyaqanJiYWOTLSZIGWWyg3wRs6ea3ADtGU44kabH6XLb4UeALwLlJDiS5ArgKuCjJXuCiblmStIwGHkOvqtfNs2njiGuRJA3BO0UlqREGuiQ1wkCXpEYY6JLUCANdkhqxah5wodFbiXeorsSapNXCEbokNcJAl6RGGOiS1AgDXZIa4UlRjdVKO8m50uqRRskRuiQ1whH6KuQoU1qZlvvfpiN0SWqEgS5JjWj2kMt8f/os52uMu6ZR/bm3FH82LsX7MwoL/VkcrV8r7X1Y6HvgIb2Vb6gRepKLk9yb5L4k20ZVlCRp4RY9Qk9yHPAPzDyC7gDwpSQ3VdXdoyruWLfSRrGjHH2Oykr7GR3Nco24x/3eLPeJwHFabX0bZoR+AXBfVe2rqh8C1wKbRlOWJGmhhgn0s4BvzFo+0K2TJC2DVNXidkwuA15ZVX/ULb8BuKCq3nxEu63A1m7xXODeRdZ6GvCtRe67WtnnY4N9PjYM0+dfrqqJQY2GucrlAPCsWcvrgAePbFRV24HtQ7wOAEl2V9XksN9nNbHPxwb7fGxYij4Pc8jlS8CGJOckOR64HLhpNGVJkhZq0SP0qjqc5E+AzwLHAR+sqj0jq0yStCBD3VhUVZ8GPj2iWgYZ+rDNKmSfjw32+dgw9j4v+qSoJGll8bNcJKkRKyLQB32EQJKnJrmu235LkvWztr2zW39vklcuZd3DWGyfk1yU5NYkX+mmFy517Ys1zPvcbT87yfeSvH2pah7GkL/XL0jyhSR7uvf6hKWsfbGG+L1+SpKprq/3JHnnUte+WD36/LIktyU5nOTSI7ZtSbK3+9oydDFVtaxfzJxQvR94NnA88GXguUe0+WPgn7r5y4Hruvnndu2fCpzTfZ/jlrtPY+7zC4FndvPPB7653P0Zd59nbf848DHg7cvdnzG/x2uAO4HzuuVTj4Hf69cD13bzJwL7gfXL3acR9Xk98ALgX4BLZ60/BdjXTdd282uHqWcljND7fITAJmCqm78B2Jgk3fprq+oHVfU14L7u+610i+5zVd1eVU9e778HOCHJU5ek6uEM8z6TZDMzv/Cr5UqqYfr7CuDOqvoyQFU9XFVPLFHdwximzwWclGQN8DTgh8B3lqbsoQzsc1Xtr6o7gR8fse8rgZ1V9UhVPQrsBC4eppiVEOh9PkLgJ22q6jDwbWZGLav14weG6fNsrwVur6ofjKnOUVp0n5OcBFwJvHcJ6hyVYd7jXwEqyWe7P9XfsQT1jsIwfb4B+D5wEHgAeH9VPTLugkdgmAwaeX6thM9Dzxzrjrz0Zr42ffZdiYbp88zG5HnA1cyM5laDYfr8XuADVfW9bsC+GgzT3zXAbwEvBh4HdiW5tap2jbbEkRumzxcATwDPZObww38l+VxV7RttiSM3TAaNPL9Wwgi9z0cI/KRN9yfZLwGP9Nx3JRqmzyRZB9wI/EFV3T/2akdjmD7/OvC+JPuBtwLv6m5qW8mG/b3+j6r6VlU9zsy9Hi8ae8XDG6bPrwc+U1U/qqpDwP8Aq+GjAYbJoNHn1wo4qbCGmWOj5/DTkwrPO6LNm/jZEynXd/PP42dPiu5jdZw8GqbPJ3ftX7vc/ViqPh/R5q9YHSdFh3mP1wK3MXNycA3wOeCS5e7TmPt8JfAhZkatJwF3Ay9Y7j6Nos+z2n6Ynz8p+rXu/V7bzZ8yVD3L/QPpOvZq4H+ZOVv87m7dXwOv6eZPYObqhvuALwLPnrXvu7v97gVetdx9GXefgb9k5ljjHbO+Tl/u/oz7fZ71PVZFoA/bX+D3mTkBfBfwvuXuy7j7DDy9W7+nC/O/WO6+jLDPL2ZmNP594GFgz6x9/7D7WdwHvHHYWrxTVJIasRKOoUuSRsBAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEf8PVX3M2qHwb9MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "xs = fs.f_scores[~np.isnan(fs.f_scores)]\n",
    "print(len(xs))\n",
    "xs = xs[xs <= .1]\n",
    "print(len(xs))\n",
    "print(len(xs[xs >= .004]), len(xs[xs < .004]))\n",
    "plt.hist(xs, bins=50)\n",
    "plt.show()\n",
    "# fs.f_scores[~np.isnan(fs.f_scores)].shape, fs.f_scores.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T05:35:11.407525Z",
     "start_time": "2018-04-21T05:35:11.385558Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "****************************************\n",
      "(0.004, 0.02)\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 0.1, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.6069380203515264\n",
      "f1 score: 0.7166666666666666\n",
      "auc:  0.8514285714285714\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 0.5, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.6111933395004626\n",
      "f1 score: 0.8899999999999999\n",
      "auc:  0.8676190476190477\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 0.5, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.39814986123959295\n",
      "f1 score: 0.7238095238095238\n",
      "auc:  0.2928571428571428\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 1, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.6197039777983349\n",
      "f1 score: 0.9257142857142858\n",
      "auc:  0.8628571428571428\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 1, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.4401480111008326\n",
      "f1 score: 0.7483333333333333\n",
      "auc:  0.7280952380952381\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 5, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.6538390379278447\n",
      "f1 score: 0.9066666666666666\n",
      "auc:  0.8371428571428572\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 5, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.5513413506012951\n",
      "f1 score: 0.6554761904761904\n",
      "auc:  0.6383333333333334\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 10, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.6580943570767808\n",
      "f1 score: 0.9200000000000002\n",
      "auc:  0.8354761904761905\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'C': 10, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.5941720629047179\n",
      "f1 score: 0.6594444444444444\n",
      "auc:  0.6359523809523809\n"
     ]
    }
   ],
   "source": [
    "for i, _g in enumerate(fs.gscv_objs):\n",
    "    print('**'*20)\n",
    "    print(fs._thresholds[i])\n",
    "    _g.print_results()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## grid search on synthesized data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T02:01:12.601243Z",
     "start_time": "2018-04-21T02:01:07.548271Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([10]))\n",
      "params: {'C': 0.1, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.9489130434782609\n",
      "f1 score: 0.7166666666666666\n",
      "auc:  0.95\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([10]))\n",
      "params: {'C': 0.1, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.9356884057971016\n",
      "f1 score: 0.7166666666666667\n",
      "auc:  0.942063492063492\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([10]))\n",
      "params: {'C': 0.5, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.9789855072463769\n",
      "f1 score: 0.8899999999999999\n",
      "auc:  0.9841269841269842\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([10]))\n",
      "params: {'C': 0.5, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.9317028985507246\n",
      "f1 score: 0.7238095238095238\n",
      "auc:  0.9626984126984126\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([10]))\n",
      "params: {'C': 1, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.9831521739130435\n",
      "f1 score: 0.9257142857142858\n",
      "auc:  0.9825396825396826\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([10]))\n",
      "params: {'C': 1, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.927355072463768\n",
      "f1 score: 0.7483333333333333\n",
      "auc:  0.9595238095238094\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([10]))\n",
      "params: {'C': 5, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.9789855072463768\n",
      "f1 score: 0.9066666666666666\n",
      "auc:  0.9857142857142858\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([10]))\n",
      "params: {'C': 5, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8978260869565217\n",
      "f1 score: 0.6554761904761904\n",
      "auc:  0.9158730158730158\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([10]))\n",
      "params: {'C': 10, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
      "avg acc:  0.9831521739130435\n",
      "f1 score: 0.9200000000000002\n",
      "auc:  0.9841269841269842\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([10]))\n",
      "params: {'C': 10, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8980072463768115\n",
      "f1 score: 0.6594444444444444\n",
      "auc:  0.8825396825396826\n"
     ]
    }
   ],
   "source": [
    "param_grid = [\n",
    "  {'C': [.1, .5, 1, 5, 10], \n",
    "   'gamma': ['auto'], 'kernel': ['rbf', 'sigmoid'], \n",
    "   'probability': [True], 'class_weight': ['balanced']},\n",
    " ]\n",
    "clf = SVC()\n",
    "gscv = GridSearchCV(param_grid, clf, pos_label=1, cv=10)\n",
    "gscv.fit(X_os, Y_os)\n",
    "# vars(gscv)\n",
    "gscv.print_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## grid search on og data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T01:37:43.704735Z",
     "start_time": "2018-04-21T01:37:29.785801Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 5, 'class_weight': 'balanced', 'degree': 2, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8272222222222222\n",
      "f1 score: 0.024691358024691357\n",
      "auc:  0.4840982286634461\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 5, 'class_weight': 'balanced', 'degree': 3, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8272222222222222\n",
      "f1 score: 0.024691358024691357\n",
      "auc:  0.40881642512077293\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 5, 'class_weight': 'balanced', 'degree': 4, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8272222222222222\n",
      "f1 score: 0.024691358024691357\n",
      "auc:  0.4655797101449275\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 5, 'class_weight': 'balanced', 'degree': 5, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8272222222222222\n",
      "f1 score: 0.024691358024691357\n",
      "auc:  0.3641304347826087\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 10, 'class_weight': 'balanced', 'degree': 2, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8544444444444445\n",
      "f1 score: 0.05555555555555555\n",
      "auc:  0.34581320450885666\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 10, 'class_weight': 'balanced', 'degree': 3, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8544444444444445\n",
      "f1 score: 0.05555555555555555\n",
      "auc:  0.5475040257648953\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 10, 'class_weight': 'balanced', 'degree': 4, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8544444444444445\n",
      "f1 score: 0.05555555555555555\n",
      "auc:  0.3482286634460548\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 10, 'class_weight': 'balanced', 'degree': 5, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8544444444444445\n",
      "f1 score: 0.05555555555555555\n",
      "auc:  0.4037842190016103\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 20, 'class_weight': 'balanced', 'degree': 2, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8588888888888889\n",
      "f1 score: 0.0\n",
      "auc:  0.5269726247987117\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 20, 'class_weight': 'balanced', 'degree': 3, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8588888888888889\n",
      "f1 score: 0.0\n",
      "auc:  0.46095008051529796\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 20, 'class_weight': 'balanced', 'degree': 4, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8588888888888889\n",
      "f1 score: 0.0\n",
      "auc:  0.4786634460547504\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 20, 'class_weight': 'balanced', 'degree': 5, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8588888888888889\n",
      "f1 score: 0.0\n",
      "auc:  0.4134460547504026\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 40, 'class_weight': 'balanced', 'degree': 2, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8681481481481481\n",
      "f1 score: 0.0\n",
      "auc:  0.3987520128824477\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 40, 'class_weight': 'balanced', 'degree': 3, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8681481481481481\n",
      "f1 score: 0.0\n",
      "auc:  0.4591384863123994\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 40, 'class_weight': 'balanced', 'degree': 4, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8681481481481481\n",
      "f1 score: 0.0\n",
      "auc:  0.45873590982286633\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 40, 'class_weight': 'balanced', 'degree': 5, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8681481481481481\n",
      "f1 score: 0.0\n",
      "auc:  0.45873590982286633\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 80, 'class_weight': 'balanced', 'degree': 2, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8681481481481481\n",
      "f1 score: 0.0\n",
      "auc:  0.4446457326892109\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 80, 'class_weight': 'balanced', 'degree': 3, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8681481481481481\n",
      "f1 score: 0.0\n",
      "auc:  0.4688003220611916\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 80, 'class_weight': 'balanced', 'degree': 4, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8681481481481481\n",
      "f1 score: 0.0\n",
      "auc:  0.4446457326892109\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([9]))\n",
      "params: {'C': 80, 'class_weight': 'balanced', 'degree': 5, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
      "avg acc:  0.8681481481481481\n",
      "f1 score: 0.0\n",
      "auc:  0.5287842190016104\n"
     ]
    }
   ],
   "source": [
    "param_grid = [\n",
    "  {'C': [5, 10, 20, 40, 80], # .1, .5, 1, \n",
    "   'gamma': ['auto'], 'kernel': ['poly', 'sigmoid'], \n",
    "   'degree': [2, 3, 4, 5],\n",
    "   'probability': [True], 'class_weight': ['balanced']},\n",
    " ]\n",
    "clf = SVC()\n",
    "gscv = GridSearchCV(param_grid, clf, pos_label=1, cv=9)\n",
    "gscv.fit(X, Y)\n",
    "# vars(gscv)\n",
    "gscv.print_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## anomaly detection grid search on og data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T01:38:52.699264Z",
     "start_time": "2018-04-21T01:38:48.726125Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'degree': 2, 'gamma': 'auto', 'kernel': 'poly', 'nu': 0.1}\n",
      "avg acc:  0.8219873150105709\n",
      "f1 score: 0.07636363636363637\n",
      "auc:  nan\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'degree': 2, 'gamma': 'auto', 'kernel': 'poly', 'nu': 0.2}\n",
      "avg acc:  0.8587737843551798\n",
      "f1 score: 0.08444444444444446\n",
      "auc:  nan\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'degree': 2, 'gamma': 'auto', 'kernel': 'poly', 'nu': 0.30000000000000004}\n",
      "avg acc:  0.8770613107822411\n",
      "f1 score: 0.09444444444444444\n",
      "auc:  nan\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'degree': 2, 'gamma': 'auto', 'kernel': 'poly', 'nu': 0.4}\n",
      "avg acc:  0.8861522198731502\n",
      "f1 score: 0.10714285714285716\n",
      "auc:  nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sahil/anaconda3/lib/python3.6/site-packages/numpy/lib/function_base.py:1128: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "/Users/sahil/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:80: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    }
   ],
   "source": [
    "param_grid = [\n",
    "  {'nu': np.arange(.1, 1.0, 0.1), \n",
    "   'gamma': ['auto'], 'kernel': ['poly'],\n",
    "   'degree': [2, 3, 4, 5, 6, 7]},\n",
    " ]\n",
    "clf = OneClassSVM()\n",
    "gscv = GridSearchCV(param_grid, clf, pos_label=1, cv=5)\n",
    "gscv.fit(X, Y)\n",
    "# vars(gscv)\n",
    "gscv.print_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## anomaly detection grid search synthesized og data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T01:40:08.552151Z",
     "start_time": "2018-04-21T01:40:04.364074Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'degree': 2, 'gamma': 'auto', 'kernel': 'poly', 'nu': 0.1}\n",
      "avg acc:  0.8078630897317298\n",
      "f1 score: 0.36898944193061844\n",
      "auc:  nan\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'degree': 2, 'gamma': 'auto', 'kernel': 'poly', 'nu': 0.2}\n",
      "avg acc:  0.850786308973173\n",
      "f1 score: 0.4324675324675325\n",
      "auc:  nan\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'degree': 2, 'gamma': 'auto', 'kernel': 'poly', 'nu': 0.30000000000000004}\n",
      "avg acc:  0.8506012950971321\n",
      "f1 score: 0.3024708624708624\n",
      "auc:  nan\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'degree': 2, 'gamma': 'auto', 'kernel': 'poly', 'nu': 0.4}\n",
      "avg acc:  0.8506012950971323\n",
      "f1 score: 0.2630769230769231\n",
      "auc:  nan\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'degree': 3, 'gamma': 'auto', 'kernel': 'poly', 'nu': 0.1}\n",
      "avg acc:  0.9316373728029601\n",
      "f1 score: 0.5396825396825398\n",
      "auc:  nan\n",
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'degree': 3, 'gamma': 'auto', 'kernel': 'poly', 'nu': 0.2}\n",
      "avg acc:  0.9230342275670674\n",
      "f1 score: 0.5111111111111111\n",
      "auc:  nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sahil/anaconda3/lib/python3.6/site-packages/numpy/lib/function_base.py:1128: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "/Users/sahil/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:80: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    }
   ],
   "source": [
    "param_grid = [\n",
    "  {'nu': np.arange(.1, 1.0, 0.1), \n",
    "   'gamma': ['auto'], 'kernel': ['poly'],\n",
    "   'degree': [2, 3, 4, 5, 6, 7]},\n",
    " ]\n",
    "clf = OneClassSVM()\n",
    "gscv = GridSearchCV(param_grid, clf, pos_label=1, cv=5)\n",
    "gscv.fit(X_os, Y_os)\n",
    "# vars(gscv)\n",
    "gscv.print_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ensemble grid search on og data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T01:55:17.129868Z",
     "start_time": "2018-04-21T01:55:14.751240Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "overfit:  True (array([ True]), array([5]))\n",
      "params: {'max_samples': 1.0, 'n_estimators': 50}\n",
      "avg acc:  0.958985200845666\n",
      "f1 score: nan\n",
      "auc:  nan\n",
      "----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'param_list_': [{'max_samples': 1.0, 'n_estimators': 50}],\n",
       " 'cv': 5,\n",
       " 'pos_label': 1,\n",
       " '_clf': BaggingClassifier(base_estimator=SVC(C=10, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "   decision_function_shape='ovr', degree=3, gamma='auto', kernel='sigmoid',\n",
       "   max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "   tol=0.001, verbose=False),\n",
       "          bootstrap=True, bootstrap_features=False, max_features=1.0,\n",
       "          max_samples=1.0, n_estimators=50, n_jobs=1, oob_score=False,\n",
       "          random_state=None, verbose=0, warm_start=False),\n",
       " 'overfit_': [[True, True, True, True, True]],\n",
       " 'accuracy_scores_': array([[0.95454545, 0.95454545, 0.95454545, 0.95454545, 0.97674419]]),\n",
       " 'precision_': array([], shape=(1, 5, 0), dtype=float64),\n",
       " 'recall_': array([], shape=(1, 5, 0), dtype=float64),\n",
       " 'f1_scores_': array([[nan, nan, nan, nan, nan]]),\n",
       " 'fpr_': array([], shape=(1, 5, 0), dtype=float64),\n",
       " 'tpr_': array([], shape=(1, 5, 0), dtype=float64),\n",
       " 'auc_scores_': array([[nan, nan, nan, nan, nan]]),\n",
       " 'average_accuracy_param_': array([0.9589852])}"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'C': 10, 'class_weight': 'balanced', 'degree': 3, 'gamma': 'auto', 'kernel': 'sigmoid', 'probability': True}\n",
    "clf = BaggingClassifier(SVC(**params))\n",
    "clf.fit(X, Y)\n",
    "# clf.predict(X)\n",
    "# cross_val_score(clf, X, Y, cv=5)\n",
    "param_grid = [{'n_estimators': [50], 'max_samples': [1.0]}]\n",
    "gscv = GridSearchCV(param_grid, clf, pos_label=1, cv=5)\n",
    "gscv.fit(X, Y)\n",
    "gscv.print_results(show_overfit=True)\n",
    "print('-'*40)\n",
    "vars(gscv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ensemble grid search on synthesized data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T01:52:20.437388Z",
     "start_time": "2018-04-21T01:52:17.375721Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "overfit:  False (array([False]), array([5]))\n",
      "params: {'max_samples': 0.5, 'n_estimators': 10, 'n_jobs': -1}\n",
      "avg acc:  0.9617021276595745\n",
      "f1 score: 0.7563492063492064\n",
      "auc:  0.9828571428571429\n",
      "----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'param_list_': [{'max_samples': 0.5, 'n_estimators': 10, 'n_jobs': -1}],\n",
       " 'cv': 5,\n",
       " 'pos_label': 1,\n",
       " '_clf': BaggingClassifier(base_estimator=SVC(C=10, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "   decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
       "   max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "   tol=0.001, verbose=False),\n",
       "          bootstrap=True, bootstrap_features=False, max_features=1.0,\n",
       "          max_samples=0.5, n_estimators=10, n_jobs=-1, oob_score=False,\n",
       "          random_state=None, verbose=0, warm_start=False),\n",
       " 'overfit_': [[False, False, False, False, False]],\n",
       " 'accuracy_scores_': array([[0.93617021, 0.93617021, 0.9787234 , 0.95744681, 1.        ]]),\n",
       " 'precision_': array([[array([0.10638298, 1.        , 1.        ]),\n",
       "         array([0.10638298, 1.        , 1.        ]),\n",
       "         array([0.10638298, 1.        , 1.        ]),\n",
       "         array([0.10638298, 1.        , 1.        ]), array([1., 1.])]],\n",
       "       dtype=object),\n",
       " 'recall_': array([[array([1. , 0.4, 0. ]), array([1. , 0.4, 0. ]),\n",
       "         array([1. , 0.8, 0. ]), array([1. , 0.6, 0. ]), array([1., 0.])]],\n",
       "       dtype=object),\n",
       " 'f1_scores_': array([[0.57142857, 0.57142857, 0.88888889, 0.75      , 1.        ]]),\n",
       " 'fpr_': array([[array([0.        , 0.        , 0.02380952, 0.02380952, 0.33333333,\n",
       "        0.33333333, 1.        ]),\n",
       "         array([0.        , 0.        , 0.02380952, 0.02380952, 0.04761905,\n",
       "        0.04761905, 1.        ]),\n",
       "         array([0., 0., 1.]), array([0., 0., 1.]), array([0., 0., 1.])]],\n",
       "       dtype=object),\n",
       " 'tpr_': array([[array([0.2, 0.6, 0.6, 0.8, 0.8, 1. , 1. ]),\n",
       "         array([0.2, 0.6, 0.6, 0.8, 0.8, 1. , 1. ]),\n",
       "         array([0.2, 1. , 1. ]), array([0.2, 1. , 1. ]),\n",
       "         array([0.25, 1.  , 1.  ])]], dtype=object),\n",
       " 'auc_scores_': array([[0.92857143, 0.98571429, 1.        , 1.        , 1.        ]]),\n",
       " 'average_accuracy_param_': array([0.96170213])}"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'C': 10, 'class_weight': 'balanced', 'gamma': 'auto', 'kernel': 'rbf', 'probability': True}\n",
    "clf = BaggingClassifier(SVC(**params))\n",
    "clf.fit(X, Y)\n",
    "param_grid = [{'n_estimators': [10], 'n_jobs': [-1], 'max_samples': [0.5]}]\n",
    "gscv = GridSearchCV(param_grid, clf, pos_label=1, cv=5)\n",
    "gscv.fit(X_os, Y_os)\n",
    "gscv.print_results(show_overfit=True)\n",
    "print('-'*40)\n",
    "vars(gscv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}