Port_Opt_1st.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os.path\n",
    "try:\n",
    "    from urllib2 import urlopen\n",
    "except ImportError:\n",
    "    from urllib.request import urlopen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## This calculates the reward, Do not change this function\n",
    "def getReward(wt, wt_1, ri, l, k):\n",
    "    if (ri is None) or (wt is None):\n",
    "        port_returns.append(0)\n",
    "        sharpe_ratio.append(0)\n",
    "        port_volatility.append(0)\n",
    "        reward = 0\n",
    "    else:\n",
    "        ri.fillna(0, inplace=True)\n",
    "        returns = np.dot(wt, ri)\n",
    "        port_returns.append(returns)\n",
    "        downside_returns = [x for x in port_returns if x < 0]\n",
    "        volatility = np.std(downside_returns) #np.sqrt(np.dot(weights.T, np.dot(cov_annual, weights)))\n",
    "        sharpe = np.sqrt(12)*np.mean(port_returns) / volatility if volatility!=0 else 0\n",
    "        sharpe_ratio.append(sharpe)\n",
    "        port_volatility.append(volatility)\n",
    "        if (wt_1 is None):\n",
    "            phi = 0\n",
    "        else:\n",
    "            ix = wt.index | wt_1.index\n",
    "            tn = wt.reindex(ix) - wt_1.reindex(ix)\n",
    "            tn[tn.isnull()&tn.index.isin(wt.index)] = wt\n",
    "            tn[tn.isnull()&tn.index.isin(wt_1.index)] = -wt_1\n",
    "            phi = k*tn.abs().sum()\n",
    "        reward = returns - l*volatility - phi\n",
    "#         print('returns, volatility, phi, sharpe, reward')\n",
    "#         print(returns, volatility, phi, sharpe, reward)\n",
    "    return reward\n",
    "\n",
    "## Do not change this function, this verifies if all constraints are met\n",
    "def checkConstraints(wt, wt_1, wi, Dt, St, Qt, g, U, t, T, P, delta, chi, eta):\n",
    "    violated = False\n",
    "    tol = 0.009\n",
    "    if np.abs(wt.sum()-1)>tol:\n",
    "        print(wt.sum())\n",
    "        print(\"Fully Invested Constraint Violated: Sum of weights is not 1\")\n",
    "        violated = True\n",
    "    div_constraint = np.maximum(g, 1/float(Qt.sum()))\n",
    "    if (wt.abs()-div_constraint>tol).any():\n",
    "        print(wt[(wt.abs()-div_constraint>tol)])\n",
    "        print(\"Diversification Constraint Violated: All weights are not less than parameter %.2f\"%div_constraint )\n",
    "        violated = True\n",
    "    if wt_1 is None:\n",
    "        turnover = 0\n",
    "    else:\n",
    "        ix = wt.index | wt_1.index\n",
    "        tn = wt.reindex(ix) - wt_1.reindex(ix)\n",
    "        tn[tn.isnull()&tn.index.isin(wt.index)] = wt\n",
    "        tn[tn.isnull()&tn.index.isin(wt_1.index)] = -wt_1\n",
    "        turnover = (tn).abs().sum()/2\n",
    "    turnover_list.append(turnover)\n",
    "    if (np.sum(turnover_list[-12:])>U):\n",
    "        print(\"%0.2f Turnover Constraint Violated: Turnover Limit exceeded\"%np.sum(turnover_list[-12:]))\n",
    "        violated = True\n",
    "    if (wt<t).any():\n",
    "        print(\"Shortsell Constraint Violated: all weights are not greater than parameter t\")\n",
    "        violated = True\n",
    "    if wt[wt<0].sum()<T:\n",
    "        print(\"Max Shortsell Constraint Violated: sum of all weights are not greater than parameter T\")\n",
    "        violated = True\n",
    "    if wt[wt!=0].count() < np.minimum(P, len(wt)):\n",
    "        print(\"Min number of positions Constraint Violated: count of all weights <>0 %i are not greater than parameter P%i\"%(wt[wt!=0].count(), np.minimum(P, len(wt))))\n",
    "        violated = True\n",
    "    if np.abs((wt*Dt).sum()/ (wi*Dt).sum()  - 1) - delta > tol:\n",
    "        print(\"Duration Constraint Violated: wt*Dt/ wi*Dt %.2f is greater than parameter delta\"%((wt*Dt).sum()/ (wi*Dt).sum()))\n",
    "        violated = True\n",
    "    if np.abs((wt*St).sum()/ (wi*St).sum() -1) - chi > tol:\n",
    "        print(\"Spread Constraint Violated: wt*St/ wi*St %.2f is greater than parameter chi\"%((wt*St).sum()/ (wi*St).sum()))\n",
    "        violated = True\n",
    "    if (wt*(1-Qt)).abs().sum()>tol:\n",
    "        print(\"Qualification Constraint Violated: wt*(1-qt) is not zero %.2f\"%(wt*(1-Qt)).abs().sum())\n",
    "        violated = True\n",
    "#     if returns - Rlow/ volatility <= np.sqrt(1-eta):\n",
    "#         print(\"Max Risk probability Constraint Violated: returns - Rlow/ volatility <= np.sqrt(1-eta)\")\n",
    "            \n",
    "    return violated\n",
    "\n",
    "## Do not change this function, this verifies if final constraints are met\n",
    "def checkFinalConstraints(Rmin, volmax):\n",
    "    violated = False\n",
    "    if np.sum(port_returns)<Rmin:\n",
    "        print(\"Total Return Constraint Violated: Total Return is less than Index Return\")\n",
    "        violated = True\n",
    "    if port_volatility[-1]>volmax:\n",
    "        print(\"Volatility Constraint Violated: Vol is higher than Index Vol\")\n",
    "        violated = True\n",
    "    if sharpe_ratio[-1]<Rmin/volmax:\n",
    "        print(\"Sharpe Ratio Constraint Violated: SR is less than Index SR\")\n",
    "        violated = True\n",
    "        return violated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getSymbolsToTrade():\n",
    "    ################################################\n",
    "    ####   COPY FROM BELOW INTO TEMPLATE FILE   ####\n",
    "    ################################################\n",
    "    \n",
    "    return 'G1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3416422\n",
      "3411322\n",
      "Data Column Names:\n",
      "Index(['AssetGroup', 'Identifier', 'F2', 'F3', 'F4', 'F5', 'q', 'F7', 'F8',\n",
      "       'd', 'S', 'F11', 'F12', 'F13', 'F14', 'F15', 'F16', 'F17', 'wI', 'F19',\n",
      "       'TRR', 'F21', 'F22', 'F23', 'F24', 'F25', 'F26', 'F27', 'F28', 'F29',\n",
      "       'F30', 'F31', 'F32', 'F33'],\n",
      "      dtype='object')\n",
      "G1 1998-12-31 2018-11-30\n",
      "           AssetGroup Identifier        F2                  F3  \\\n",
      "TimeStamp                                                        \n",
      "1998-12-31         G1   86a1f1ee  bc068363       Capital Goods   \n",
      "1998-12-31         G1   de5b9bca  e5fb34b9  Telecommunications   \n",
      "1998-12-31         G1   2a0a4ee3  62835655      Basic Industry   \n",
      "1998-12-31         G1   a67fb965  81f811c5      Transportation   \n",
      "1998-12-31         G1   52f697cc  2becce58      Basic Industry   \n",
      "1998-12-31         G1   bf46c4bb  57847ea5  Telecommunications   \n",
      "1998-12-31         G1   e39dffda  9c69fc3a  Telecommunications   \n",
      "1998-12-31         G1   98eed4f4  2becce58      Basic Industry   \n",
      "1998-12-31         G1   53b20751  2becce58      Basic Industry   \n",
      "1998-12-31         G1   0ad1c30b  81f811c5      Transportation   \n",
      "\n",
      "                                                  F4  F5  q   F7    F8      d  \\\n",
      "TimeStamp                                                                       \n",
      "1998-12-31                 Diversified Capital Goods   1  1   B1  14.0  2.684   \n",
      "1998-12-31  Telecom - Wireline Integrated & Services   6  1  BB2  12.0  4.514   \n",
      "1998-12-31                                 Chemicals   1  1   B1  14.0  3.703   \n",
      "1998-12-31                                      Rail   2  0   B2  15.0  3.372   \n",
      "1998-12-31                        Building Materials   4  0  BB2  12.0  2.274   \n",
      "1998-12-31  Telecom - Wireline Integrated & Services   2  1   B3  16.0  6.721   \n",
      "1998-12-31  Telecom - Wireline Integrated & Services   2  0  BB2  12.0  3.021   \n",
      "1998-12-31                        Building Materials   4  1  BB2  12.0  4.663   \n",
      "1998-12-31                        Building Materials   4  0  BB2  12.0  1.326   \n",
      "1998-12-31                                      Rail   2  1   B2  15.0  4.254   \n",
      "\n",
      "           ...   F24  F25  F26  F27  F28  F29  F30  F31  F32  F33  \n",
      "TimeStamp  ...                                                     \n",
      "1998-12-31 ...     0    1  7.0  0.0    1  4.0  0.0    1    0 -100  \n",
      "1998-12-31 ...    -4   -4  NaN  NaN   -4  NaN  NaN   -4   -4   -4  \n",
      "1998-12-31 ...    -4   -4  NaN  NaN   -4  NaN  NaN   -4   -4   -4  \n",
      "1998-12-31 ...    -4   -4  NaN  NaN   -4  NaN  NaN   -4   -4   -4  \n",
      "1998-12-31 ...     1    1  7.0  0.0    1  4.0  0.0    1    1  -99  \n",
      "1998-12-31 ...    -4   -4  NaN  NaN   -4  NaN  NaN   -4   -4   -4  \n",
      "1998-12-31 ...     0    1  4.0  4.0    0  1.0  4.0    0    0 -100  \n",
      "1998-12-31 ...     1    1  7.0  0.0    1  4.0  0.0    1    1    1  \n",
      "1998-12-31 ...     1    1  7.0  0.0    1  4.0  0.0    1    1  -99  \n",
      "1998-12-31 ...    -4   -4  NaN  NaN   -4  NaN  NaN   -4   -4   -4  \n",
      "\n",
      "[10 rows x 34 columns]\n"
     ]
    }
   ],
   "source": [
    "index = getSymbolsToTrade()\n",
    "if not os.path.isfile('%s.csv'%index):\n",
    "    downloadFile(index)\n",
    "idx_data = pd.read_csv('%s.csv'%index, index_col='TimeStamp')\n",
    "idx_data.sort_index(axis=0, level=None, ascending=True, inplace=True)\n",
    "print(idx_data.size)\n",
    "idx_data=idx_data[~idx_data.duplicated()]\n",
    "print(idx_data.size)\n",
    "print(\"Data Column Names:\")\n",
    "print(idx_data.columns)\n",
    "print(index, idx_data.index[0], idx_data.index[-1])\n",
    "print(idx_data.head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AssetGroup object\n",
      "Identifier object\n",
      "F2 object\n",
      "F3 object\n",
      "F4 object\n",
      "F7 object\n",
      "F12 object\n",
      "F13 object\n"
     ]
    }
   ],
   "source": [
    "for c in idx_data.columns:\n",
    "    if idx_data[c].dtype==object:\n",
    "        print(c, idx_data[c].dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sachin/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "listf = ['F3', 'F4', 'F7', 'F12', 'F13']\n",
    "feature_dict ={}\n",
    "for feature in listf:    \n",
    "    fs = idx_data[feature].unique()\n",
    "    f_dict = {}\n",
    "    count = 0\n",
    "    for f in fs:\n",
    "        idx_data[feature][idx_data[feature]==f] = count\n",
    "        f_dict[f] = count\n",
    "        count = count+1\n",
    "    feature_dict[feature] = f_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'F12': {'1500-2500': 0, '2500-3500': 3, '3500+': 1, '500-1500': 2, '<500': 4},\n",
       " 'F13': {'DB1': 4, 'DB2': 2, 'DB3': 0, 'DB4': 3, 'DB5': 1},\n",
       " 'F3': {'Agency': 14,\n",
       "  'Automotive': 8,\n",
       "  'Basic Industry': 2,\n",
       "  'Capital Goods': 0,\n",
       "  'Consumer Goods': 6,\n",
       "  'Energy': 5,\n",
       "  'Government Guaranteed': 12,\n",
       "  'Healthcare': 17,\n",
       "  'Leisure': 16,\n",
       "  'Local-Authority': 7,\n",
       "  'Media': 4,\n",
       "  'Real Estate': 11,\n",
       "  'Retail': 10,\n",
       "  'Services': 13,\n",
       "  'Technology & Electronics': 15,\n",
       "  'Telecommunications': 1,\n",
       "  'Transportation': 3,\n",
       "  'Utility': 9},\n",
       " 'F4': {'Advertising': 39,\n",
       "  'Aerospace/Defense': 42,\n",
       "  'Agency': 26,\n",
       "  'Air Transportation': 48,\n",
       "  'Auto Loans': 46,\n",
       "  'Auto Parts & Equipment': 32,\n",
       "  'Automakers': 14,\n",
       "  'Beverage': 17,\n",
       "  'Building & Construction': 6,\n",
       "  'Building Materials': 4,\n",
       "  'Cable & Satellite TV': 5,\n",
       "  'Chemicals': 2,\n",
       "  'Department Stores': 44,\n",
       "  'Diversified Capital Goods': 0,\n",
       "  'Electric-Distr/Trans': 25,\n",
       "  'Electric-Generation': 21,\n",
       "  'Electric-Integrated': 15,\n",
       "  'Electronics': 31,\n",
       "  'Energy - Exploration & Production': 29,\n",
       "  'Environmental': 40,\n",
       "  'Food & Drug Retailers': 16,\n",
       "  'Food - Wholesale': 8,\n",
       "  'Forestry/Paper': 10,\n",
       "  'Gaming': 37,\n",
       "  'Gas Distribution': 12,\n",
       "  'Government Guaranteed': 19,\n",
       "  'Health Facilities': 47,\n",
       "  'Health Services': 55,\n",
       "  'Hotels': 38,\n",
       "  'Integrated Energy': 7,\n",
       "  'Local-Authority': 13,\n",
       "  'Machinery': 51,\n",
       "  'Media - Diversified': 43,\n",
       "  'Media Content': 22,\n",
       "  'Medical Products': 53,\n",
       "  'Metals/Mining Excluding Steel': 11,\n",
       "  'Non-Electric Utilities': 35,\n",
       "  'Oil Field Equipment & Services': 33,\n",
       "  'Oil Refining & Marketing': 28,\n",
       "  'Packaging': 36,\n",
       "  'Personal & Household Products': 34,\n",
       "  'Pharmaceuticals': 41,\n",
       "  'REITs': 50,\n",
       "  'Rail': 3,\n",
       "  'RealEstate Dev & Mgt': 18,\n",
       "  'Recreation & Travel': 45,\n",
       "  'Restaurants': 49,\n",
       "  'Software/Services': 52,\n",
       "  'Specialty Retail': 30,\n",
       "  'Steel Producers/Products': 23,\n",
       "  'Support-Services': 24,\n",
       "  'Tech Hardware & Equipment': 54,\n",
       "  'Telecom - Satellite': 27,\n",
       "  'Telecom - Wireless': 9,\n",
       "  'Telecom - Wireline Integrated & Services': 1,\n",
       "  'Transport Infrastructure/Services': 20},\n",
       " 'F7': {'A1': 17,\n",
       "  'A2': 16,\n",
       "  'A3': 11,\n",
       "  'AA1': 15,\n",
       "  'AA2': 20,\n",
       "  'AA3': 21,\n",
       "  'AAA': 13,\n",
       "  'B1': 0,\n",
       "  'B2': 2,\n",
       "  'B3': 3,\n",
       "  'BB1': 10,\n",
       "  'BB2': 1,\n",
       "  'BB3': 5,\n",
       "  'BBB1': 8,\n",
       "  'BBB2': 9,\n",
       "  'BBB3': 7,\n",
       "  'C': 14,\n",
       "  'CC': 6,\n",
       "  'CCC1': 12,\n",
       "  'CCC2': 4,\n",
       "  'CCC3': 18,\n",
       "  'D': 19}}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 2. break end period TRR into three buckets: over perform index, under perform index, +ve, underperform index -ve\n",
    "Create a target variable 'Y' and assign the following values:\n",
    "over perform index: 1\n",
    "under perform index but +ve return: 0\n",
    "underperform index and -ve return: -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates = idx_data.index.unique()\n",
    "idx_returns = pd.Series(0.0,index=dates)\n",
    "counter=0\n",
    "new_idx_data = pd.DataFrame(columns=idx_data.columns)\n",
    "while counter < len(dates):\n",
    "    date = dates[counter]\n",
    "    date_data = idx_data[idx_data.index == date]\n",
    "    \n",
    "    ## get all the identifiers for a date\n",
    "    \n",
    "    date_data.set_index( date_data['Identifier'], inplace=True)\n",
    "    date_data = date_data[~date_data.index.duplicated()]\n",
    "    \n",
    "    wi = pd.Series(date_data['wI']/100, index = date_data['Identifier'])\n",
    "    ri = pd.Series(date_data['TRR'], index = date_data['Identifier'])\n",
    "    idx_returns[date]=np.dot(wi, ri)\n",
    "    \n",
    "    date_data.set_index(date +'-'+date_data['Identifier'], inplace=True)\n",
    "    date_data.loc[date_data['TRR']<idx_returns[date], 'Y'] = 0\n",
    "    date_data.loc[date_data['TRR']<0, 'Y'] = -1\n",
    "    date_data.loc[date_data['TRR']>=idx_returns[date], 'Y'] = 1\n",
    "    new_idx_data = new_idx_data.append(date_data)\n",
    "    counter = counter+1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 3. Train a model which uses the given features to predict which bucket the performance will be in. Train a few models to check which one is the best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = new_idx_data.copy()\n",
    "del X['TRR']\n",
    "del X['AssetGroup']\n",
    "del X['Identifier']\n",
    "del X['Y']\n",
    "del X['F2']\n",
    "y = new_idx_data['Y'].copy()\n",
    "X['NAs'] = X.isnull().sum(axis=1)\n",
    "X = X.replace([np.inf, -np.inf], np.nan)\n",
    "X.fillna(-50, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 1.0    52157\n",
      "-1.0    24978\n",
      " 0.0    23090\n",
      "Name: Y, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(y.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random under-sampling:\n",
      "-1.0    24978\n",
      " 1.0    24978\n",
      " 0.0    23090\n",
      "Name: Y, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7ff38421aef0>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Class count\n",
    "count_class_0, count_class_1 , count_class_2 = y.value_counts()\n",
    "\n",
    "# Divide by class\n",
    "df_class_0 = X[y == 1]\n",
    "df_class_1 = X[y == 0]\n",
    "df_class_2 = X[y == -1]\n",
    "\n",
    "df_class_0_under = df_class_0.sample(count_class_1)\n",
    "df_test_under = pd.concat([df_class_0_under, df_class_1, df_class_2], axis=0)\n",
    "df_y_under = pd.concat([y[y.index.isin(df_class_0_under.index)], y[y == 0], y[y == -1]], axis=0)\n",
    "\n",
    "print('Random under-sampling:')\n",
    "print(df_y_under.value_counts())\n",
    "\n",
    "df_y_under.value_counts().plot(kind='bar', title='Count (target)')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix \n",
    "from sklearn.model_selection import train_test_split\n",
    "# dividing X, y into train and test data \n",
    "X_train, X_test, y_train, y_test = train_test_split(df_test_under, df_y_under, random_state = 10) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.620359215858066\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[5121, 1017,   63],\n",
       "       [1280, 4066,  495],\n",
       "       [1288, 2790, 2142]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " # training a DescisionTreeClassifier \n",
    "from sklearn.tree import DecisionTreeClassifier \n",
    "dtree_model = DecisionTreeClassifier(max_depth = 3).fit(X_train, y_train) \n",
    "dtree_predictions = dtree_model.predict(X_test) \n",
    "\n",
    "# accuracy on X_test \n",
    "accuracy = dtree_model.score(X_test, y_test) \n",
    "print (accuracy)\n",
    "\n",
    "# creating a confusion matrix \n",
    "cm = confusion_matrix(y_test, dtree_predictions)\n",
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sachin/anaconda3/lib/python3.6/site-packages/sklearn/ensemble/weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.\n",
      "  from numpy.core.umath_tests import inner1d\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6018508378052787\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[4216, 1049,  936],\n",
       "       [ 563, 2760, 2518],\n",
       "       [ 666, 1539, 4015]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rforest_model = RandomForestClassifier(n_estimators=100, max_depth=2,random_state=0).fit(X_train, y_train) \n",
    "rforest_predictions = rforest_model.predict(X_test) \n",
    "\n",
    "# accuracy on X_test \n",
    "accuracy = rforest_model.score(X_test, y_test) \n",
    "print (accuracy)\n",
    "\n",
    "# creating a confusion matrix \n",
    "cm = confusion_matrix(y_test, rforest_predictions)\n",
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.56422788e-01, 9.47014402e-02, 1.22461396e-01, 5.31429078e-03,\n",
       "       4.79124492e-02, 8.99744477e-03, 3.42467144e-02, 9.74603904e-02,\n",
       "       3.07970683e-01, 0.00000000e+00, 0.00000000e+00, 1.01399585e-03,\n",
       "       0.00000000e+00, 1.95385133e-03, 3.18663908e-03, 0.00000000e+00,\n",
       "       0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
       "       4.64050546e-04, 1.09485981e-03, 1.69098143e-04, 0.00000000e+00,\n",
       "       4.52689902e-03, 2.13775756e-02, 3.99912255e-02, 4.44127151e-02,\n",
       "       1.40473229e-03, 4.91676119e-03, 0.00000000e+00])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rforest_model.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # dividing X, y into train and test data \n",
    "# X_train, X_test, y_train, y_test = train_test_split(X, y, random_state = 0) \n",
    "  \n",
    "# # training a linear SVM classifier \n",
    "# from sklearn.svm import SVC \n",
    "# svm_model_linear = SVC(kernel = 'linear', C = 1).fit(X_train, y_train) \n",
    "# svm_predictions = svm_model_linear.predict(X_test) \n",
    "  \n",
    "# # model accuracy for X_test   \n",
    "# accuracy = svm_model_linear.score(X_test, y_test) \n",
    "  \n",
    "# # creating a confusion matrix \n",
    "# cm = confusion_matrix(y_test, svm_predictions) \n",
    "# cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4108531376629066\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[2629, 1549, 2082],\n",
       "       [1433, 2578, 1684],\n",
       "       [2176, 1835, 2296]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dividing X, y into train and test data \n",
    "X_train, X_test, y_train, y_test = train_test_split(df_test_under, df_y_under, random_state = 50) \n",
    "  \n",
    "# training a KNN classifier \n",
    "from sklearn.neighbors import KNeighborsClassifier \n",
    "knn = KNeighborsClassifier(n_neighbors = 7, weights='distance').fit(X_train, y_train) \n",
    "  \n",
    "# accuracy on X_test \n",
    "accuracy = knn.score(X_test, y_test) \n",
    "print (accuracy)\n",
    "  \n",
    "# creating a confusion matrix \n",
    "knn_predictions = knn.predict(X_test)  \n",
    "cm = confusion_matrix(y_test, knn_predictions) \n",
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.39491840981272586\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 630, 3927, 1702],\n",
       "       [  82, 5340,  315],\n",
       "       [ 240, 4784, 1242]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dividing X, y into train and test data \n",
    "X_train, X_test, y_train, y_test = train_test_split(df_test_under, df_y_under, random_state = 25) \n",
    "  \n",
    "# training a Naive Bayes classifier \n",
    "from sklearn.naive_bayes import GaussianNB \n",
    "gnb = GaussianNB().fit(X_train, y_train) \n",
    "gnb_predictions = gnb.predict(X_test) \n",
    "  \n",
    "# accuracy on X_test \n",
    "accuracy = gnb.score(X_test, y_test) \n",
    "print (accuracy)\n",
    "  \n",
    "# creating a confusion matrix \n",
    "cm = confusion_matrix(y_test, gnb_predictions) \n",
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.32783922900010953\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 814, 4861,  584],\n",
       "       [ 790, 4606,  341],\n",
       "       [ 830, 4869,  567]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Taking ensemble of all three\n",
    "\n",
    "y_avg = (gnb_predictions+knn_predictions+dtree_predictions)/3\n",
    "y_avg = [1 if x>0.5 else x for x in y_avg]\n",
    "y_avg = [-1 if x<-0.5 else x for x in y_avg]\n",
    "y_avg = [0 if (x>=-0.5)and(x<=0.5) else x for x in y_avg]\n",
    "from sklearn.metrics import accuracy_score\n",
    "print(accuracy_score(y_test, y_avg))\n",
    "cm = confusion_matrix(y_test, y_avg) \n",
    "cm\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 4. Define a new optimization objective function as wt*bucketValue - transaction costs (We want as much weight as possible to be assigned to highest bucket). Train an optimizer to maximize this reward while meeting constraints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#     ### In this sample case, I am using scipy.optimize \n",
    "from scipy.optimize import minimize\n",
    "# import cvxpy as cp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Here I define my objective function \n",
    "\n",
    "class CustomFeatures():\n",
    "    \n",
    "    def newFeature1(self, wt, wt_1, ri, k):\n",
    "    ################################################\n",
    "    ####   COPY FROM BELOW INTO TEMPLATE FILE   ####\n",
    "    ################################################\n",
    "#         ri = kwargs['ri']\n",
    "#         wt = kwargs['wt']\n",
    "#         wt_1 = kwargs['wt_1']\n",
    "#         l = kwargs['l']\n",
    "        ri.fillna(0, inplace=True)\n",
    "        returns = np.dot(wt, ri)\n",
    "#         volatility = np.std(port_returns) #np.sqrt(np.dot(weights.T, np.dot(cov_annual, weights)))\n",
    "        weights = pd.Series(wt, index = ri.index)\n",
    "    \n",
    "        if (wt_1 is None):\n",
    "            phi = 0\n",
    "        else:\n",
    "            ix = weights.index | wt_1.index\n",
    "            tn = weights.reindex(ix) - wt_1.reindex(ix)\n",
    "            tn[tn.isnull()&tn.index.isin(weights.index)] = weights\n",
    "            tn[tn.isnull()&tn.index.isin(wt_1.index)] = -wt_1\n",
    "            phi = k*tn.abs().sum()\n",
    "        reward = -(returns - phi)\n",
    "        return reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getWeights(identifiers, reward, wi, Dt, St, Qt, g, U, t, T, P, delta, chi, eta, df, trr, wt_1,**kwargs):\n",
    "    ################################################\n",
    "    ####   COPY FROM BELOW INTO TEMPLATE FILE   ####\n",
    "    ################################################\n",
    "    \n",
    "    ### I predict future returns as returns from t-1\n",
    "    ri = getPrediction(identifiers, wi, Dt, St, Qt, g, U, t, T, P, delta, chi, eta, df=df)\n",
    "    \n",
    "    div_constraint = np.maximum(g, 1/float(Qt.sum()))\n",
    "    \n",
    "    allowed = Qt[Qt==1].index   ###pre=process to assign weights only where Qt==1, this takes care of last constraint automatically\n",
    "    \n",
    "    bounds = ((t,div_constraint),) * len(allowed) #limits on weights\n",
    "    \n",
    "    cons = ({'type': 'eq', 'fun': lambda wt: 1.0 - np.sum(wt)},   ##sum of weights should be 1\n",
    "            {'type': 'ineq', 'args': (P, Qt,),'fun': lambda wt, P, Qt: np.count_nonzero(wt) - np.minimum(P, Qt.sum())}, ## minimum number of positions should be P\n",
    "            {'type': 'ineq', 'args': (delta, wi, Dt,),'fun': lambda wt, delta, wi, Dt: delta - np.abs((wt*Dt[Qt==1]).sum()/ (wi*Dt).sum() - 1)}, # duration should be within a % of index\n",
    "            {'type': 'ineq', 'args': (chi, wi, St,),'fun': lambda wt, chi, wi, St: chi - np.abs((wt*St[Qt==1]).sum()/ (wi*St).sum() - 1)}, # spread should be within a % of index\n",
    "#             {'type': 'eq', 'args': (Qt,),'fun': lambda wt, Qt: (wt*(1-Qt[Qt==1])).sum()} #only trade Q==1 assets\n",
    "           )\n",
    "    ## optimization function is defined in custom feature\n",
    "    CustomFeaturesCls = CustomFeatures()\n",
    "    weights = minimize(CustomFeaturesCls.newFeature1, wi[Qt==1], args=(wt_1, ri[Qt==1], k,), method='SLSQP', bounds=bounds,constraints=cons,\n",
    "                      options={'maxiter': 500, 'ftol': 1e-06, 'iprint': 1, 'disp': True})   \n",
    "    print(weights.success, weights.message)\n",
    "    \n",
    "    w = pd.Series(0, index = date_data['Identifier'])\n",
    "    w[qt==1] = pd.Series(weights.x , index = allowed) ## assets with q==1 are assigned weights, rest are zero\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getKwargs():   \n",
    "    return {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 3: Optional: Fill in the logic to return predictions for return on asset\n",
    "## This function takes in the same inputs as getWeights()\n",
    "\n",
    "### do not change the inputs to the function. If you want any extra inputs, specify them as **kwargs\n",
    "### you can lookup this tutorial on how to use **kwargs https://www.geeksforgeeks.org/args-kwargs-python/\n",
    "\n",
    "def getPrediction(identifiers, wi, Dt, St, Qt, g, U, t, T, P, delta, chi, eta, **kwargs):\n",
    "    ################################################\n",
    "    ####   COPY FROM BELOW INTO TEMPLATE FILE   ####\n",
    "    ################################################\n",
    "    \n",
    "    X = kwargs['df'].copy()\n",
    "    del X['AssetGroup']\n",
    "    del X['Identifier']\n",
    "    del X['F2']\n",
    "    X['NAs'] = X.isnull().sum(axis=1)\n",
    "    X = X.replace([np.inf, -np.inf], np.nan)\n",
    "    X.fillna(-50, inplace=True)\n",
    "#     y1 = dtree_model.predict(X)\n",
    "#     y2 = knn.predict(X)\n",
    "#     y3 = gnb.predict(X)\n",
    "    \n",
    "#     y_avg = (y1+y2+y3)/3\n",
    "#     y_avg = [1 if x>0.5 else x for x in y_avg]\n",
    "#     y_avg = [-1 if x<-0.5 else x for x in y_avg]\n",
    "#     y_avg = [0 if (x>=-0.5)and(x<=0.5) else x for x in y_avg]\n",
    "\n",
    "    y = rforest_model.predict(X)\n",
    "    \n",
    "    ri = pd.Series(y, index = date_data['Identifier'])\n",
    "    \n",
    "    return ri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# empty lists to store returns, volatility and weights of imaginary portfolios\n",
    "port_returns = []\n",
    "port_volatility = []\n",
    "sharpe_ratio = []\n",
    "asset_weights = []\n",
    "reward_list = []\n",
    "turnover_list = []\n",
    "idx_returns = []\n",
    "\n",
    "#port_returns ,port_volatility ,sharpe_ratio ,asset_weights ,reward_list ,turnover_list ,idx_returns \n",
    "\n",
    "#empty df to store previous period returns\n",
    "ri = None\n",
    "\n",
    "#empty dict to store values by date\n",
    "dict_metrics_by_date = {}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "################# \n",
      "DATE 1998-12-31\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "drop() got an unexpected keyword argument 'columns'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-41-1e1e84b5b250>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;31m## get new weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m     \u001b[0mwt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetWeights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcusips\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mqt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mU\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdelta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdate_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TRR'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mri\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwt_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mgetKwargs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: drop() got an unexpected keyword argument 'columns'"
     ]
    }
   ],
   "source": [
    "### Evaluator to getweights at everytime t and calcuate reward + check if constraints are met\n",
    "\n",
    "## specifying all the constants\n",
    "counter = 0\n",
    "l = 0.1\n",
    "k = 0.03\n",
    "g = 0.04\n",
    "U = 2.5\n",
    "t= 0\n",
    "T = 0\n",
    "P = 50\n",
    "delta = 0.5\n",
    "chi = 0.15\n",
    "eta = 0.95\n",
    "\n",
    "## initializing arrays\n",
    "dates = idx_data.index.unique()\n",
    "wi = None\n",
    "\n",
    "## looping over all dates\n",
    "while counter < len(dates):\n",
    "    date = dates[counter]\n",
    "    \n",
    "    print('################# \\nDATE %s'%date)\n",
    "    \n",
    "    ## load all the data for a date\n",
    "    date_data = idx_data[idx_data.index == date]\n",
    "    \n",
    "    ## get all the identifiers for a date\n",
    "    \n",
    "    date_data.set_index( date_data['Identifier'], inplace=True)\n",
    "    date_data = date_data[~date_data.index.duplicated()]\n",
    "    cusips = date_data['Identifier']\n",
    "    \n",
    "    ## old weights\n",
    "    wt_1 = asset_weights[-1] if len(asset_weights)>0 else None\n",
    "    wt_2 = asset_weights[-2] if len(asset_weights)>1 else None\n",
    "    \n",
    "    ##old index weights\n",
    "    wi_t_1 = None if wi is None else wi.copy()\n",
    "    \n",
    "    ## calculate reward at start of time t from weights allocated at time t-1\n",
    "    reward = getReward(wt_1, wt_2, ri, l, k)\n",
    "    \n",
    "    ## calculate index return at start of time t from weights allocated at time t-1\n",
    "    if ri is None or wi_t_1 is None:\n",
    "        idx_returns.append(0)\n",
    "    else:\n",
    "        idx_returns.append(np.dot(wi_t_1, ri))\n",
    "    \n",
    "    #load specific feature info for time t\n",
    "    wi = pd.Series(date_data['wI']/100, index = date_data['Identifier'])\n",
    "    Dt = pd.Series(date_data['d'], index = date_data['Identifier'])\n",
    "    St = pd.Series(date_data['S'], index = date_data['Identifier'])\n",
    "    qt = pd.Series(date_data['q'], index = date_data['Identifier'])\n",
    "    \n",
    "    ## get new weights\n",
    "    wt = getWeights(cusips, reward, wi, Dt, St, qt, g, U, t, T, P, delta, chi, eta, date_data.drop(columns=['TRR']), ri, wt_1, **getKwargs()) \n",
    "\n",
    "    \n",
    "    ## store relevant info in their lists\n",
    "    asset_weights.append(wt)\n",
    "    reward_list.append(reward)\n",
    "    \n",
    "    ## verify if all constraints are met\n",
    "    if checkConstraints(wt, wt_1, wi, Dt, St, qt, g, U, t, T, P, delta, chi, eta):\n",
    "        print(\"ERROR!!!! weights don't meet contraints, exiting\")\n",
    "#         break\n",
    "    \n",
    "    dict_metrics_by_date[date] = {'returns': port_returns[-1],\n",
    "                                  'volatility': port_volatility[-1] ,\n",
    "                                  'Sharpe Ratio': sharpe_ratio[-1],\n",
    "                                  'Index Returns': idx_returns[-1],\n",
    "                                  'Reward': reward_list[-1],\n",
    "                                  '12m turnover' : np.sum(turnover_list[-12:]),\n",
    "                                  'weights' :asset_weights[-1]}\n",
    "    \n",
    "    print(dict_metrics_by_date[date])\n",
    "    \n",
    "    counter += 1\n",
    "    \n",
    "    ## Store end of month returns to calculate reward in next period\n",
    "    del ri\n",
    "    ri = pd.Series(date_data['TRR'], index = date_data['Identifier'])\n",
    "\n",
    "## Calculate returns for last period    \n",
    "if ri is not None:\n",
    "    reward = getReward(wt, wt_1, ri, l, k)\n",
    "    reward_list.append(reward)\n",
    "    idx_returns.append(np.dot(wi, ri))\n",
    "## check if contraints on total return and risk are met        \n",
    "Rmin = np.sum(idx_returns)\n",
    "volmax = np.std(idx_returns)\n",
    "if checkFinalConstraints(Rmin, volmax):\n",
    "    print(\"ERROR!!!! weights don't meet return/risk limit contraints, exiting\")\n",
    "else:\n",
    "    print(\"Portfolio Metrics:\")\n",
    "    print(\"Total Return: %.2f\"%np.sum(port_returns))\n",
    "    print(\"Standard Deviation: %.2f\"%port_volatility[-1])\n",
    "    print(\"Sharpe Ratio: %.2f\"%sharpe_ratio[-1])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('run1','w') as data:\n",
    "    data.write(str(dict_metrics_by_date))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.2710803706444725, 0.8032437836394527, 1.2350242184662905)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(port_returns), np.std(port_returns), np.std([x for x in port_returns if x < 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.589812883126556, 2.2147595417241064, 2.5638555277373474)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(idx_returns), np.std(idx_returns), np.std([x for x in idx_returns if x < 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.1690721657712375, 0.9225253227613263)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sqrt(12)*np.mean(port_returns)/np.std(port_returns), np.sqrt(12)*np.mean(idx_returns)/np.std(idx_returns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7603494212832631, 0.7969137648215683)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sqrt(12)*np.mean(port_returns)/np.std([x for x in port_returns if x < 0]), np.sqrt(12)*np.mean(idx_returns)/np.std([x for x in idx_returns if x < 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.7685953052601675\n",
      "0.7736720808207398\n",
      "0.7752652584641955\n",
      "0.7679016904606271\n",
      "0.7673247857730527\n",
      "0.7625805380930513\n",
      "0.7616768815591277\n",
      "0.7532598105616266\n",
      "0.7532235913980986\n",
      "0.7551062498217509\n",
      "0.7741633428926226\n",
      "0.7862134726971751\n",
      "0.7812214827215072\n",
      "0.7960209010858181\n",
      "0.7999025874157809\n",
      "0.802684666500174\n",
      "0.8004423594099652\n",
      "0.817147458838857\n",
      "0.801758785249878\n",
      "0.8150078912760435\n",
      "0.822619593204505\n",
      "0.8264717101706762\n",
      "0.951934640137635\n",
      "0.8148597804851543\n",
      "0.7959227661265547\n",
      "0.7888476367758965\n",
      "0.7863910980443249\n",
      "0.7869636409128373\n",
      "0.7903986950795765\n",
      "0.8019448070622232\n",
      "0.8029626363594038\n",
      "0.8058414256980376\n",
      "0.8187981170400431\n",
      "0.7964732595985409\n",
      "0.8194685642825129\n",
      "0.8327179743102581\n",
      "0.8254660278361935\n",
      "0.8405338039145229\n",
      "0.8292202167597645\n",
      "0.839831824397471\n",
      "0.8396898895623903\n",
      "0.8538390536390533\n",
      "0.8271509844191867\n",
      "0.8407705323959702\n",
      "0.8483960542964697\n",
      "0.8495025164311225\n",
      "0.8402373587413654\n",
      "0.8509435276344715\n",
      "0.8519668575255362\n",
      "0.847618498538109\n",
      "0.8447571639552129\n",
      "0.8455425570807693\n",
      "0.8376722547484885\n",
      "0.8420252249967761\n",
      "0.8258298859432818\n",
      "0.8352044751716543\n",
      "0.8222933510892014\n",
      "0.7966972616371437\n",
      "0.7956321226674294\n",
      "0.745394244088976\n",
      "0.7661404705533752\n",
      "0.7527730401683974\n",
      "0.7473634978751087\n",
      "0.7580339991651676\n",
      "1.0258158860645576\n",
      "0.7639245758603865\n",
      "0.774483478996459\n",
      "0.8365008833879255\n",
      "0.8477194496157945\n",
      "0.9339776512842344\n",
      "0.8891447435419606\n",
      "0.9205212895038023\n",
      "0.8987028113103257\n",
      "0.9476382526198679\n",
      "0.965868977518246\n",
      "0.9015815317245923\n",
      "0.8978125895442839\n",
      "0.8976819142412833\n",
      "0.8913537210729945\n",
      "0.8563509844565096\n",
      "0.826669692111458\n",
      "0.8109587426434095\n",
      "0.8093552625957849\n",
      "0.8049941976925286\n",
      "1.0553556329764238\n",
      "0.8402162369410571\n",
      "0.8418507282457901\n",
      "0.8505153411821857\n",
      "0.8408384652030447\n",
      "0.8401997245140327\n",
      "0.8501400194885309\n",
      "0.8384641406259112\n",
      "0.8367468848199799\n",
      "0.8457566908894814\n",
      "0.8499793449095415\n",
      "0.8477663765739376\n",
      "0.8632335477381512\n",
      "0.975631515648694\n",
      "0.8697459439304993\n",
      "0.8971190310709205\n",
      "0.8952363481283262\n",
      "0.9190803246831489\n",
      "0.9094718662141232\n",
      "0.9082519949224781\n",
      "0.8985532639547554\n",
      "0.8949313739842815\n",
      "0.8979638331589707\n",
      "0.887914327150322\n",
      "0.866586493195737\n",
      "0.8576770114022518\n",
      "0.8514171510394852\n",
      "0.8532163367928378\n",
      "0.8535476375215562\n",
      "0.8427029721052925\n",
      "0.8636468475142537\n",
      "0.8634443166611636\n",
      "0.8514590909823765\n",
      "0.8622378705076066\n",
      "0.7926679231689095\n",
      "0.7887824515599915\n",
      "0.7783791626848533\n",
      "0.7461154599742839\n",
      "0.7678745401110944\n",
      "0.7607143625635855\n",
      "0.7469877051561327\n",
      "0.7660016960953684\n",
      "0.7685056240899801\n",
      "1.0308519021470928\n",
      "0.7660874632264116\n",
      "0.7480680493537946\n",
      "0.7600632419455025\n",
      "0.7626420359229764\n",
      "0.7686168847864866\n",
      "0.7771664353536953\n",
      "0.7818735365606003\n",
      "0.778160473843754\n",
      "0.7861784250075481\n",
      "0.7998349798592419\n",
      "0.7914105188474487\n",
      "0.7825386503895579\n",
      "0.7945718998957312\n",
      "0.7930724986606744\n",
      "0.7784354335675956\n",
      "0.7693111329739991\n",
      "0.7528986436279337\n",
      "0.7540341847935887\n",
      "0.7496091167862698\n",
      "0.7471160922650443\n",
      "0.7529554522757138\n",
      "0.7648557054419143\n",
      "0.7735786764208419\n",
      "0.7594305363953392\n",
      "0.7705237719608797\n",
      "0.7588530409765344\n",
      "0.7519406560808675\n",
      "1.0356322453463374\n",
      "0.7594927514533898\n",
      "0.737667007211283\n",
      "0.7642907899877318\n",
      "0.7459327608969948\n",
      "0.7384869106657929\n",
      "0.7254570903404036\n",
      "0.7108823272954947\n",
      "0.7185505339200536\n",
      "0.7206115117373627\n",
      "0.7165116234360307\n",
      "0.7303904981644684\n",
      "0.7367135527857744\n",
      "0.7279650536314709\n",
      "0.7199200932663121\n",
      "0.7337817448064354\n",
      "0.7465402880679215\n",
      "0.759937027681732\n",
      "0.7308784305567335\n",
      "0.7311714614655205\n",
      "0.741449246961347\n",
      "0.7704125777325177\n",
      "0.7659236430049294\n",
      "0.7617031381420944\n",
      "0.77088404135104\n",
      "0.7754169874320089\n",
      "0.7744529799966041\n",
      "0.7982161838023992\n",
      "0.8092647900328148\n",
      "0.8010025703079349\n",
      "0.8102075302152528\n",
      "0.8251898407538137\n",
      "0.8363782094238732\n",
      "0.8249952530975371\n",
      "0.8419367655441468\n",
      "0.8253613446053756\n",
      "0.8238763866043246\n",
      "0.8417881741997326\n",
      "0.8405633661479148\n",
      "0.844127145338812\n",
      "1.096315572341051\n",
      "0.8381490601571204\n",
      "0.8369466814042905\n",
      "0.8315435856228099\n",
      "0.8218020018844758\n",
      "0.8343853370156958\n",
      "0.8261122657419366\n",
      "0.8287121595761172\n",
      "0.8277596952465294\n",
      "1.000202043612851\n",
      "0.8556652220957582\n",
      "0.8501340941344118\n",
      "0.8521682918926684\n",
      "0.8612195616196876\n",
      "0.8613860600828981\n",
      "0.8721724159918116\n",
      "0.8741278392856018\n",
      "0.8694583299885551\n",
      "0.8642407875667237\n",
      "0.8524215163120086\n",
      "0.852045612152919\n",
      "0.8566518744351528\n",
      "0.8559366874911984\n",
      "0.8692977615169727\n",
      "0.856314908428953\n",
      "0.8561220965367449\n",
      "0.8578440722538283\n",
      "0.8822880022586996\n",
      "0.8713261814328902\n",
      "0.8707339789155569\n",
      "0.8761795389831074\n",
      "0.8728729165904698\n",
      "0.859086405222492\n",
      "0.8678758297230436\n",
      "0.8626040211989808\n",
      "0.8716107056528991\n",
      "0.8989706277305979\n",
      "0.9026008781096717\n",
      "0.8873187351638377\n",
      "1.0013143081429985\n",
      "0.8444302479131256\n",
      "0.8545613164787086\n",
      "0.8634165031823133\n",
      "0.8749075609943511\n",
      "0.844284059323938\n"
     ]
    }
   ],
   "source": [
    "counter=0\n",
    "while counter < len(dates):\n",
    "    date = dates[counter]\n",
    "    wt = dict_metrics_by_date[date]['weights']\n",
    "#     {'returns','volatility','Sharpe Ratio','Index Returns','Reward','12m turnover'}\n",
    "    if np.abs(wt.sum()-1)>tol:\n",
    "        print(wt.sum())\n",
    "    counter +=1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}