EM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from itertools import *\n",
    "# READ PREPROCESSED DATA FROM R FOR NN\n",
    "X_df = pd.read_csv(\"some_X.csv\", usecols=['NBR_EMPLOYEES', 'EMPL_GROWTH',\n",
    "       'COUNTRY_RISK_GROWTH_RATE', 'COUNTRY_RISK_DIVIDEND_YIELD',\n",
    "       'COUNTRY_RISK_PAYOUT_RATIO', #'VOLATILITY_30D', 'VOLATILITY_180D',\n",
    "       'PCT_CHG_1_YEAR', 'PCT_CHG_6_M', 'DIVIDEND_YIELD', 'MARKETCAP',\n",
    "       'TOTAL_ASSETS', 'TOTAL_LIABILITIES', 'CURRENT_ASSETS', 'EBIT',\n",
    "       'RETAINED_EARNINGS', 'SALES', 'SALES_GROWTH', 'INTEREST_EXPENSES',\n",
    "       'WHISTLE_BLOWER_POLICY', 'ETHICS_POLICY', 'BRIBERY_POLICY',\n",
    "               'PCT_WOMEN_EMPLOYEES', 'PCT_WOMEN_MGT'])\n",
    "\n",
    "Y_df = pd.read_csv(\"some_Y.csv\", usecols=[1])\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import shuffle\n",
    "def select_most_likely(mlp, loss_fn, x_row, y_row, random=0):\n",
    "    candidates = [] # TODO: For every -1 in row in the specific columns, try 0 and 1\n",
    "    \n",
    "    bad_values = {'WHISTLE_BLOWER_POLICY': 0,\n",
    "                            'ETHICS_POLICY': 0,\n",
    "                            'BRIBERY_POLICY': 0,\n",
    "                 'PCT_WOMEN_EMPLOYEES':-1,\n",
    "                  'PCT_WOMEN_MGT':-1}\n",
    "    \n",
    "    policies = {'WHISTLE_BLOWER_POLICY', 'ETHICS_POLICY', 'BRIBERY_POLICY'}\n",
    "\n",
    "    possible_values_dict = {'WHISTLE_BLOWER_POLICY':[[1,0], [0,1]],\n",
    "                            'ETHICS_POLICY':[[1,0], [0,1]],\n",
    "                            'BRIBERY_POLICY':[[1,0], [0,1]],\n",
    "                           'PCT_WOMEN_EMPLOYEES':[[-.5], [-.4], [-.3], [-.2], [-.1], [0], [0.1], [0.20], [0.30], [0.40]],\n",
    "                           'PCT_WOMEN_MGT':[[-.5], [-.4], [-.3], [-.2], [-.1], [0], [0.1], [0.20], [0.30], [0.40]]}\n",
    "    \n",
    "    # Make a dictionary for the candidate values that we will expand later\n",
    "    candidate_dict = {}\n",
    "    for k, v in x_row.iteritems():\n",
    "        candidate_dict[k] = []\n",
    "        if (k in bad_values): # We have to encode for the neural network\n",
    "            if (v == bad_values[k]):\n",
    "                for i in possible_values_dict[k]:\n",
    "                    # Directly encode the bad_value\n",
    "                    candidate_dict[k].append(i)\n",
    "            else:\n",
    "                # Distinquish between the policies that have to be encoded\n",
    "                if k in policies:\n",
    "                    candidate_dict[k] = [possible_values_dict[k][int(v) - 1]] # HACK: IT'S BECAUSE THEY HAVE VALUES 1 AND 2\n",
    "                else:\n",
    "                    # And the women percentages that can be left alone\n",
    "                    candidate_dict[k].append([v]) # Just add the value\n",
    "        else:\n",
    "            # Just add the value\n",
    "            candidate_dict[k].append([v])\n",
    "\n",
    "    best_candidate = None\n",
    "    lowest_loss = 100000000000\n",
    "\n",
    "    # In the dictionary we have the different candidates\n",
    "    if random==1:\n",
    "        p = list(product(*candidate_dict.values()))\n",
    "        shuffle(p)\n",
    "        candidate=p[0]\n",
    "        best_candidate = Variable(torch.FloatTensor(np.array(list(chain(*list(candidate)))))).cuda()\n",
    "    else:\n",
    "        for candidate in product(*candidate_dict.values()):\n",
    "            input = Variable(torch.FloatTensor(np.array(list(chain(*list(candidate)))))).cuda()\n",
    "            # Forward the candidate through the network   \n",
    "            y_pred = mlp(input)\n",
    "            loss = loss_fn(y_pred, y_row)\n",
    "            if loss < lowest_loss:\n",
    "                best_candidate = input\n",
    "    return best_candidate\n",
    "    \n",
    "def get_E_step(model, loss_fn, X_df, Y_df, random=0):\n",
    "    \"\"\" Return a torch CUDA variable of X\"\"\"\n",
    "    df_list = []        \n",
    "    for (k, j) in zip(X_df.iterrows(), Y_df.iterrows()):\n",
    "        (_, x) = k\n",
    "        (_, y) = j\n",
    "        df_list.append(select_most_likely(model, loss_fn, x, Variable(torch.FloatTensor(y.values).cuda()), random=random))\n",
    "    return torch.stack(df_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def EM_algorithm(X_df, Y_df, N_hidden=10, N_iter=30, N_iter_nn=10000):\n",
    "    # Whole data\n",
    "    X = Variable(torch.FloatTensor(X_df.values)).cuda()\n",
    "    Y = Variable(torch.FloatTensor(Y_df.values)).cuda() # Always stays the same\n",
    "    Y_train = Y[0:2500,:] # Always stays the same\n",
    "    Y_val = Y[2500:3000,:]\n",
    "    Y_test = Y[3000:,:] # Always stays the same\n",
    "\n",
    "    # Make an initial model\n",
    "    print(\"Making model\")\n",
    "    model = torch.nn.Sequential(\n",
    "        torch.nn.Linear(X.shape[1] + 3 - 2, N_hidden),\n",
    "        #orch.nn.Linear(X.shape[1] + 3, 1),\n",
    "\n",
    "        #torch.nn.ReLU(),\n",
    "        #torch.nn.Linear(N_hidden , N_hidden),\n",
    "        #torch.nn.ReLU(),\n",
    "\n",
    "        #torch.nn.Sigmoid(),\n",
    "        torch.nn.Tanh(), # best\n",
    "        torch.nn.Linear(N_hidden, Y.shape[1]),\n",
    "        torch.nn.Sigmoid() # If it doesn't work, look here\n",
    "    )\n",
    "\n",
    "    model = model.cuda()\n",
    "    loss_fn = torch.nn.MSELoss(size_average=False)\n",
    "    \n",
    "    loss = 10000000\n",
    "    loss_val = 10000000\n",
    "    previous_loss = 10000000\n",
    "    previous_loss_test = 10000000\n",
    "    # Init\n",
    "    for i in range(N_iter):\n",
    "        # Get the expected data\n",
    "        print(\"Doing iteration {} E step: finding most likely data\".format(i + 1))\n",
    "        #random = 1 if i==0 else 0\n",
    "        random = 0\n",
    "        # IMPUTE THE WHOLE DATASET BASED ON OUR MODEL (trained on the training)\n",
    "        X = get_E_step(model, loss_fn, X_df, Y_df, random=random)\n",
    "        \n",
    "        X_train = X[0:2500,:] # Do this every time\n",
    "        X_val = X[2500:3000,:] # \n",
    "        #X_test = X[3000:,:] # Do this every time\n",
    "                \n",
    "        # TODO: Check if the predicted data is the same still?\n",
    "        # Train the model for some iterations\n",
    "        print(\"Doing iteration {} M step: training neural network\".format(i + 1))\n",
    "        learning_rate = 1e-3\n",
    "        for t in range(N_iter_nn):\n",
    "            previous_loss = loss\n",
    "            previous_loss_val = loss_val\n",
    "\n",
    "            y_pred = model(X_train)\n",
    "            loss = loss_fn(y_pred, Y_train)\n",
    "            if (t % (N_iter_nn / 2) == 0): print(loss)\n",
    "            model.zero_grad()\n",
    "            loss.backward()\n",
    "            for param in model.parameters():\n",
    "                param.data -= learning_rate * param.grad.data\n",
    "\n",
    "            y_pred_val = model(X_val)\n",
    "            loss_val = loss_fn(y_pred_val, Y_val)\n",
    "\n",
    "            if (previous_loss_val < loss_val):\n",
    "                print(\"Validation loss {0} -> {1} increasing. Stopping\".format(previous_loss_val, loss_val))\n",
    "                break\n",
    "\n",
    "        if (-loss + previous_loss)<0.0001: \n",
    "            print(\"Loss not decreasing sufficiently: exit\")\n",
    "            break\n",
    "\n",
    "        \n",
    "        previous_loss = loss\n",
    "        previous_loss_val = loss_val\n",
    "    \n",
    "    #\n",
    "    return X, model, Y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def two_columns_to_one(bla):\n",
    "    if bla.equal(torch.FloatTensor([1,0]).cuda()):\n",
    "        return 1\n",
    "    elif bla.equal(torch.FloatTensor([0,1]).cuda()):\n",
    "        return 2\n",
    "    else:\n",
    "        print(\"ERROR\")\n",
    "        return 0\n",
    "\n",
    "\n",
    "def pytorch_vars_to_df(X_df, X_final):\n",
    "    new_df = X_df.copy()\n",
    "    # Map categorical variables back\n",
    "    cat = {'WHISTLE_BLOWER_POLICY','ETHICS_POLICY','BRIBERY_POLICY'}\n",
    "    new_df['WHISTLE_BLOWER_POLICY'] = pd.Series([two_columns_to_one(x) for x in X_final[:, 19:21].data]) - 1\n",
    "    new_df['ETHICS_POLICY'] = pd.Series([two_columns_to_one(x) for x in X_final[:, 21:23].data]) - 1\n",
    "    new_df['BRIBERY_POLICY'] = pd.Series([two_columns_to_one(x) for x in X_final[:, 23:25].data]) - 1\n",
    "    new_df['PCT_WOMEN_EMPLOYEES'] = pd.Series(X_final[:, 25].data)\n",
    "    new_df['PCT_WOMEN_MGT'] = pd.Series(X_final[:, 26].data)\n",
    "\n",
    "    return new_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Making model\n",
      "Doing iteration 1 E step: finding most likely data\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "size mismatch, m1: [1 x 25], m2: [23 x 40] at /home/dd/pytorch/aten/src/THC/generic/THCTensorMathBlas.cu:249",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-575087920927>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mX_final\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_logistic\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mY_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mEM_algorithm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_df\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_df\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mN_hidden\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m40\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mN_iter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mN_iter_nn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-5-f0911cb70a10>\u001b[0m in \u001b[0;36mEM_algorithm\u001b[0;34m(X_df, Y_df, N_hidden, N_iter, N_iter_nn)\u001b[0m\n\u001b[1;32m     37\u001b[0m         \u001b[0mrandom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0;31m# IMPUTE THE WHOLE DATASET BASED ON OUR MODEL (trained on the training)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_E_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_df\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_df\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m         \u001b[0mX_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m2500\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# Do this every time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-3-e4eb871bdb71>\u001b[0m in \u001b[0;36mget_E_step\u001b[0;34m(model, loss_fn, X_df, Y_df, random)\u001b[0m\n\u001b[1;32m     62\u001b[0m         \u001b[0;34m(\u001b[0m\u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0;34m(\u001b[0m\u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m         \u001b[0mdf_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselect_most_likely\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFloatTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-3-e4eb871bdb71>\u001b[0m in \u001b[0;36mselect_most_likely\u001b[0;34m(mlp, loss_fn, x_row, y_row, random)\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFloatTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcandidate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m             \u001b[0;31m# Forward the candidate through the network\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m             \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmlp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     53\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_row\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mlowest_loss\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    355\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.6/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     87\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    355\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.6/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m    970\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    971\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 972\u001b[0;31m     \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    973\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    974\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: size mismatch, m1: [1 x 25], m2: [23 x 40] at /home/dd/pytorch/aten/src/THC/generic/THCTensorMathBlas.cu:249"
     ]
    }
   ],
   "source": [
    "X_final, model_logistic , Y_test = EM_algorithm(X_df, Y_df, N_hidden=40, N_iter=5, N_iter_nn=30000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Making model\n",
      "Doing iteration 1 E step: finding most likely data\n",
      "Doing iteration 1 M step: training neural network\n",
      "Variable containing:\n",
      " 355.2236\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      "\n",
      "Validation loss Variable containing:\n",
      " 0.2224\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " -> Variable containing:\n",
      " 0.2224\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " increasing. Stopping\n",
      "Doing iteration 2 E step: finding most likely data\n",
      "Doing iteration 2 M step: training neural network\n",
      "Variable containing:\n",
      " 0.5597\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      "\n",
      "Validation loss Variable containing:\n",
      " 0.2224\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " -> Variable containing:\n",
      " 0.2224\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " increasing. Stopping\n",
      "Doing iteration 3 E step: finding most likely data\n",
      "Doing iteration 3 M step: training neural network\n",
      "Variable containing:\n",
      " 0.5534\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      "\n",
      "Validation loss Variable containing:\n",
      " 0.2224\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " -> Variable containing:\n",
      " 0.2224\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " increasing. Stopping\n",
      "Doing iteration 4 E step: finding most likely data\n",
      "Doing iteration 4 M step: training neural network\n",
      "Variable containing:\n",
      " 0.5473\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      "\n",
      "Validation loss Variable containing:\n",
      " 0.2224\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " -> Variable containing:\n",
      " 0.2225\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " increasing. Stopping\n",
      "Doing iteration 5 E step: finding most likely data\n",
      "Doing iteration 5 M step: training neural network\n",
      "Variable containing:\n",
      " 0.5414\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      "\n",
      "Validation loss Variable containing:\n",
      " 0.2225\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " -> Variable containing:\n",
      " 0.2225\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " increasing. Stopping\n"
     ]
    }
   ],
   "source": [
    "X_final, model_nn , Y_test = EM_algorithm(X_df, Y_df, N_hidden=10, N_iter=5, N_iter_nn=30000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = X_final[3000:,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      "1.00000e-03 *\n",
      "  1.0664\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " Variable containing:\n",
      "1.00000e-04 *\n",
      "  4.9186\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "loss_fn = torch.nn.MSELoss(size_average=True)\n",
    "\n",
    "y_pred_logistic = model_logistic(X_test)\n",
    "y_pred_nn = model_nn(X_test)\n",
    "\n",
    "loss_logistic = loss_fn(y_pred_logistic, Y_test)\n",
    "loss_nn = loss_fn(y_pred_nn, Y_test)\n",
    "\n",
    "print(loss_logistic, loss_nn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Making model\n",
      "Doing iteration 1 E step: finding most likely data\n",
      "Doing iteration 1 M step: training neural network\n",
      "Variable containing:\n",
      " 569.3817\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      "\n",
      "Validation loss Variable containing:\n",
      " 0.2753\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " -> Variable containing:\n",
      " 0.2753\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " increasing. Stopping\n",
      "Doing iteration 2 E step: finding most likely data\n",
      "Doing iteration 2 M step: training neural network\n",
      "Variable containing:\n",
      " 0.4726\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      "\n",
      "Validation loss Variable containing:\n",
      " 0.2753\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " -> Variable containing:\n",
      " 0.2754\n",
      "[torch.cuda.FloatTensor of size () (GPU 0)]\n",
      " increasing. Stopping\n",
      "Doing iteration 3 E step: finding most likely data\n"
     ]
    }
   ],
   "source": [
    "X_final, mlp = EM_algorithm(X_df, Y_df, N_hidden=40, N_iter=5, N_iter_nn=30000)\n",
    "#X_test = X_final[3000:,:] \n",
    "\n",
    "last_df = pytorch_vars_to_df(X_df, X_final)\n",
    "\n",
    "last_df[\"DEFAULT_PROB\"] = Y_df[\"DEFAULT_PROB\"]\n",
    "last_df.to_csv(\"EM_data_fuller2.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = (mlp(X_final).data).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.02813887],\n",
       "       [0.02176614],\n",
       "       [0.03820742],\n",
       "       ...,\n",
       "       [0.0071764 ],\n",
       "       [0.01487532],\n",
       "       [0.00735415]])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs.numpy()\n",
    "Y_df.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAecAAAFKCAYAAAAnj5dkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X1wU+edL/DvkWzJNpKNhCWTYFgcg0Lj1KlNYJs1hYba\nLZDkTnbSAYcUZ2d7yWUGF8KEuZNqCqa9wcN2UmaW9I/edLhpkiYdcVkNw0wzY5JAewmYGIJjEncT\nvyTrmCRYkpFlyy+yLOv+QaXYcHR0JFvSkfT9/BOOjx7p+fkQfnrehWAwGAQREREphirVFSAiIqLZ\nmJyJiIgUhsmZiIhIYZiciYiIFIbJmYiISGGYnImIiBQmJ9UVCHE6R5L6eQZDAdzusaR+ZqIwFmVi\nLMqTKXEAjEWpYonFZNJHvJe1LeecHHWqqzBvGIsyMRblyZQ4AMaiVPMVS9YmZyIiIqWSlZybm5ux\nbds21NfX49q1a6Kv+c1vfoMdO3bEVIaIiIjuFDU5t7W1oa+vDzabDYcPH8bhw4fveE1PTw8uX74c\nUxkiIiISFzU5t7a2ora2FgBQXl4Oj8cDr9c76zVHjhzBvn37YipDRERE4qImZ5fLBYPBEL42Go1w\nOp3ha7vdjrVr12LJkiWyyxAREVFkMS+lmnmI1dDQEOx2O1555RUMDAzIKhOJwVCQ9Bl7UtPY0w1j\nUSbGojyZEgfAWJRqPmKJmpzNZjNcLlf42uFwwGQyAQAuXbqEmzdv4qmnnsLk5CS++OILNDc3S5aJ\nJNlr3EwmfdLXVicKY1EmxqI8mRIHwFiUKpZY5rTOuaamBi0tLQCAzs5OmM1m6HQ6AMCmTZvw1ltv\n4cSJE/jtb3+LiooKWK1WyTJEREQkLWrLubq6GhUVFaivr4cgCGhqaoLdboder0ddXZ3sMkRERCSP\nEJQzIJwEye7SyNZuFKVjLMqUKbFkShwAY1GqpHVrExERUXIxORMRESkMkzMREZHCMDkTEREpDJMz\nERGRwjA5ExFR1vP5A3C4x+DzB1JdFQBxbN9JRESUKQLT07Cd7UF7lxM3h30wFmpRZTFh28YVUKtS\n135lciYioqxlO9uDd65cD18PDvvC19trLamqFru1iYgoO/n8AbR3iZ+Y2N7lSmkXN5MzERFlJY/X\nh5vDPtF77pEJeLzi95KByZmIiLJSkU4LY6FW9J5Bn4cinfi9ZGByJiKirKTNVaPKIn6ccZWlGNpc\ndZJr9A1OCCMioqy1beMKALfGmN0jEzDo81BlKQ7/PFWYnImIKGupVSpsr7XgiQ3l8Hh9KNJpU9pi\nDmFyJiKirKfNVcNsKEh1NcI45kxERKQwTM5EREQKw+RMRESkMEzORERECsPkTEREpDBMzkRERArD\n5ExERKQwTM5EREQKw+RMRESkMEzORERECiNr+87m5mZ0dHRAEARYrVZUVlaG7504cQInT56ESqXC\nqlWr0NTUhLa2NuzduxcrV64EAFgsFhw4cCAxERAREWWYqMm5ra0NfX19sNls6O3thdVqhc1mAwCM\nj4/jz3/+M9544w3k5uaioaEB7e3tAIC1a9fi2LFjia09ERFRBorard3a2ora2loAQHl5OTweD7xe\nLwAgPz8fr776KnJzczE+Pg6v1wuTSfxsTCIiIpInanJ2uVwwGAzha6PRCKfTOes1L7/8Murq6rBp\n0yYsXboUANDT04Ndu3bhySefxIULF+a52kRERJkr5iMjg8HgHT975pln0NDQgJ07d2L16tVYvnw5\nGhsbsXnzZvT396OhoQFnzpyBRqOJ+L4GQwFycpJ7hqbJpE/q5yUSY1EmxqI8mRIHwFiUaj5iiZqc\nzWYzXC5X+NrhcIS7roeGhtDd3Y01a9YgLy8P69evx9WrV7F69Wps2bIFALBs2TIUFxdjYGAg3KoW\n43aPzTWWmJhMejidI0n9zERhLMrEWJQnU+IAGItSxRKLVBKP2q1dU1ODlpYWAEBnZyfMZjN0Oh0A\nYGpqCs8//zxGR0cBAB999BHKyspw+vRpHD9+HADgdDoxODiIkpISWZUlIiLKdlFbztXV1aioqEB9\nfT0EQUBTUxPsdjv0ej3q6uqwe/duNDQ0ICcnB/feey9+8IMfYHR0FPv378e7774Lv9+PQ4cOSXZp\nExER0TeEoNggcgoku0sjW7tRlI6xKFOmxJIpcQCMRamS1q1NREREycXkTEREpDBMzkRERArD5ExE\nRKQwTM5EREQKw+RMRESkMEzORERECsPkTEREpDBMzkRERArD5ExERKQwTM5EREQKw+RMRESkMEzO\nRHPk8wfgcI/B5w+kuipElCGiHhlJROIC09Owne1Be5cTN4d9MBZqUWUxYdvGFVCr+L2XiOLH5EwU\nJ9vZHrxz5Xr4enDYF77eXmtJVbWIKAPw6z1RHHz+ANq7nKL32rtc7OImojlhciaKg8frw81hn+g9\n98gEPF7xe0REcjA5E8WhSKeFsVAres+gz0ORTvweEZEcTM5EcdDmqlFlMYneq7IUQ5urTnKNiCiT\ncEIYUZy2bVwB4NYYs3tkAgZ9HqosxeGfExHFi8mZKE5qlQrbay14YkM5PF4finRatpiJaF4wORPN\nkTZXDbOhINXVIKIMwjFnIiIihWFyJiIiUhgmZyIiIoWRNebc3NyMjo4OCIIAq9WKysrK8L0TJ07g\n5MmTUKlUWLVqFZqamiAIgmQZIiIiiixqcm5ra0NfXx9sNht6e3thtVphs9kAAOPj4/jzn/+MN954\nA7m5uWhoaEB7ezumpqYiliEiIiJpUbu1W1tbUVtbCwAoLy+Hx+OB1+sFAOTn5+PVV19Fbm4uxsfH\n4fV6YTKZJMsQERGRtKjJ2eVywWAwhK+NRiOcztkb/r/88suoq6vDpk2bsHTpUllliIiISFzM65yD\nweAdP3vmmWfQ0NCAnTt3YvXq1bLK3M5gKEBOTnI3cDCZ9En9vERiLMrEWJQnU+IAGItSzUcsUZOz\n2WyGy+UKXzscDphMt/YUHhoaQnd3N9asWYO8vDysX78eV69elSwTids9Fm8McTGZ9HA6R5L6mYnC\nWJSJsShPpsQBMBaliiUWqSQetVu7pqYGLS0tAIDOzk6YzWbodDoAwNTUFJ5//nmMjo4CAD766COU\nlZVJliEiIiJpUVvO1dXVqKioQH19PQRBQFNTE+x2O/R6Perq6rB79240NDQgJycH9957L37wgx9A\nEIQ7yhAREZE8QlDOgHASJLtLI1u7UZSOsShTpsSSKXEAjEWpktatTURERMnF5ExERKQwTM5EREQK\nw+RMRESkMEzORERECsPkTEREpDBMzkRERArD5ExERKQwTM5EREQKw+RMRESkMEzORERECsPkTERE\npDBMzkRERArD5ExERKQwTM5EREQKw+RMRESkMEzORERECsPkTEREpDBMzkRERArD5ExERKQwTM5E\nREQKw+RMRESkMEzORERECsPkTEREpDBMzkRERAqTI+dFzc3N6OjogCAIsFqtqKysDN+7dOkSjh49\nCpVKhbKyMhw+fBiXL1/G3r17sXLlSgCAxWLBgQMHEhMBERFRhomanNva2tDX1webzYbe3l5YrVbY\nbLbw/YMHD+K1117D4sWLsWfPHpw/fx55eXlYu3Ytjh07ltDKExERZaKo3dqtra2ora0FAJSXl8Pj\n8cDr9Ybv2+12LF68GABgNBrhdrsTVFUiIqLsEDU5u1wuGAyG8LXRaITT6Qxf63Q6AIDD4cCFCxew\nYcMGAEBPTw927dqFJ598EhcuXJjvehMREWUsWWPOMwWDwTt+Njg4iF27dqGpqQkGgwHLly9HY2Mj\nNm/ejP7+fjQ0NODMmTPQaDQR39dgKEBOjjrW6syJyaRP6uclEmNRJsaiPJkSB8BYlGo+YomanM1m\nM1wuV/ja4XDAZDKFr71eL3bu3Ilnn30W69atAwCUlJRgy5YtAIBly5ahuLgYAwMDWLp0acTPcbvH\n4g4iHiaTHk7nSFI/M1EYizIxFuXJlDgAxqJUscQilcSjdmvX1NSgpaUFANDZ2Qmz2RzuygaAI0eO\n4Omnn8b69evDPzt9+jSOHz8OAHA6nRgcHERJSYmsyhIREWW7qC3n6upqVFRUoL6+HoIgoKmpCXa7\nHXq9HuvWrcOpU6fQ19eHkydPAgAeffRRPPLII9i/fz/effdd+P1+HDp0SLJLm4iIiL4ha8x5//79\ns65XrVoV/vPHH38sWuZ3v/vdHKpFRESUvbhDGBEpgs8fgMM9Bp8/kOqqEKVczLO1iYjmU2B6Graz\nPWjvcuLmsA/GQi2qLCZs27gCahXbD5SdmJyJKKVsZ3vwzpXr4evBYV/4enutJVXVIkopfi0lopTx\n+QNo73KK3mvvcrGLm7IWkzMRpYzH68PNYZ/oPffIBDxe8XtEmY7JmYhSpkinhbFQK3rPoM9DkU78\nHlGmY3ImopTR5qpRZTGJ3quyFEObm9wtfYmUghPCiCiltm1cAeDWGLN7ZAIGfR6qLMXhnxNlIyZn\nIkoptUqF7bUWPLGhHB6vD0U6LVvMlPXYrU1ECSV3cxFtrhpmQwETMxHYciaiBOHmIkTxY3ImooTg\n5iJE8ePXVyKad9xchGhumJyJaN5xcxGiuWFyJqJ5x81FiOaGyZmI5h03FyGaG04II6KE4OYiRPFj\nciaihODmIkTxY3ImooQKbS5CRPJxzJmIiEhhmJyJiIgUhsmZiIhIYZiciYiIFIbJmYiISGGYnImI\niBRG1lKq5uZmdHR0QBAEWK1WVFZWhu9dunQJR48ehUqlQllZGQ4fPgyVSiVZhoiIiCKLmpzb2trQ\n19cHm82G3t5eWK1W2Gy28P2DBw/itddew+LFi7Fnzx6cP38e+fn5kmWIiIgosqjd2q2traitrQUA\nlJeXw+PxwOv1hu/b7XYsXrwYAGA0GuF2u6OWISIiosiitpxdLhcqKirC10ajEU6nEzqdDgDC/3U4\nHLhw4QL27t2Lo0ePSpYRYzAUICcnuVv7mUz6pH5eIjEWZWIsypMpcQCMRanmI5aYt+8MBoN3/Gxw\ncBC7du1CU1MTDAaDrDK3c7vHYq3KnJhMejidI0n9zERhLMrEWJQnU+IAGItSxRKLVBKP2q1tNpvh\ncrnC1w6HAybTN0fBeb1e7Ny5E88++yzWrVsnqwwRERFFFjU519TUoKWlBQDQ2dkJs9k8q3v6yJEj\nePrpp7F+/XrZZYiIiCiyqN3a1dXVqKioQH19PQRBQFNTE+x2O/R6PdatW4dTp06hr68PJ0+eBAA8\n+uij2LZt2x1liCh1fP4Aj20kSiOyxpz3798/63rVqlXhP3/88ceyyhBR8gWmp2E724P2LiduDvtg\nLNSiymLCto0roFZxDyIipeJ5zkQZzHa2B+9cuR6+Hhz2ha+311pSVS0iioJfnYkylM8fQHuXU/Re\ne5cLPn8gyTUiIrmYnIkylMfrw81hn+g998gEPF7xe0SUekzORBmqSKeFsVAres+gz0ORTvweEaUe\nkzNRhtLmqlFlEd9foMpSzFnbRArGCWEUxuU2mWfbxhUAbo0xu0cmYNDnocpSHP45ESkTkzNxuU0G\nU6tU2F5rwRMbyvnFiyiNMDkTl9tkAW2uGmZDQaqrQUQysVmU5bjchohIeZicsxyX2xARKQ+Tc5bj\nchsiIuVhcs5yXG5DRKQ8TM6EH3//Hiw166ASbl2rBGCpWYcff/+e1FaM4ubzB+Bwj3HOAFGa4mxt\nwsm/fIZ+hzd8PR0E+h1enPzLZ5ytnWa4LI4oM/D/1izH2dqZJbQsbnDYhyC+WRZnO9uT6qoRUQyY\nnLMcZ2snT6K7mvlFiyhzsFs7y4Vmaw+KJGjO1p4fyepqlvNFixuREKUHtpyzHGdrJ16yupq5LI4o\nczA5E7ZtXIHaB0uxqDAPKgFYVJiH2gdLeTjCPEhmVzO/aBFlDnZrEw9HSKBkdzXzFCqizMDkTGE8\nHGH+JXtMn1+0iDIDu7WJEihVXc2hL1pMzETpiS1nogRjVzMRxYrJmSjB2NVMRLFiciZKEo7pE5Fc\nspJzc3MzOjo6IAgCrFYrKisrw/d8Ph8OHjyI7u5u2O12AMD777+PvXv3YuXKlQAAi8WCAwcOJKD6\nREREmSdqcm5ra0NfXx9sNht6e3thtVphs9nC93/961/jW9/6Frq7u2eVW7t2LY4dOzb/NSaaJz5/\ngN3MRKRIUZNza2sramtrAQDl5eXweDzwer3Q6XQAgH379mFoaAinT59ObE2J5glPbiIipYuanF0u\nFyoqKsLXRqMRTqcznJx1Oh2GhobuKNfT04Ndu3bB4/GgsbERNTU1kp9jMBQgJye5rReTSZ/Uz0sk\nxiLf7099hHeuXA9fh7bTLMjXYOfj357Xz+JzUZ5MiQNgLEo1H7HEPCEsGAxGfc3y5cvR2NiIzZs3\no7+/Hw0NDThz5gw0Gk3EMm73WKxVmROTSQ+ncySpn5kojEU+nz+ACx1fit670PEVNq9dOm9d3Hwu\nypMpcQCMRaliiUUqiUftwzObzXC5XOFrh8MBk0l8U4WQkpISbNmyBYIgYNmyZSguLsbAwICsyhIl\nEo/IJKJ0EDU519TUoKWlBQDQ2dkJs9kc7tKO5PTp0zh+/DgAwOl0YnBwECUlJfNQXaK54clNRJQO\nonZrV1dXo6KiAvX19RAEAU1NTbDb7dDr9airq8OePXtw48YNfP7559ixYwe2bt2KjRs3Yv/+/Xj3\n3Xfh9/tx6NAhyS5tomQJbac5c8w5ZOZ2mpzJTUSpJATlDCInQbLHG7J1jEPpkhHLN7O1xbfTnK+Z\n3HwuypMpcQCMRanma8yZO4RR1pHaTvPNd7pEZ3IDwPZaS0rqS0TZh4s6ad75/AE43GPw+QOproqk\n209u8vkDaO9yir62vcul+HiIKHOw5UxxERuTjXdzj5nvlUpyZnJzb2wiSgYmZ4qJVAK2ne2JqUtY\n7L1qHliCxx5alpKdukIzuQdFEjRnchNRMrFbm2ISSsCDwz4E8U0CfvPtrpi7hMXe6/T5z2A725PY\nICIIzeQWM3MmNxFRojE5k2ySY7Ldrpg291Dq+O62jStQ+2ApFhXmQSUAiwrzUPtgaXgmNxFRMrBb\nm2STGpP1eCexUKeFW2SHLbEuYaWO70rN5CYiSha2nEk2qd21jIV5+I6lWPSeWJew0nfqun0mNxFR\nMjE5k2zRxmS3166U3SXM8V0iosjYrU0A5G9XGUq0YrtrxdolLPZeNQ/cjcceWja/wRERpRkm5ywX\n69pkOQk41CUcjdh7ld69MGO28SMiiheTc5aLdW1yiNwELMd8vhcRUSbgmHMWU+pyJiKibMfknMXk\nLGciIqLkY3LOYkpfzkRElK2YnLMYlzMRESkTJ4RlOamlUURElBpMzlkuk7erlLt2m4hIaZicCUBm\nLWeK91xpIiKlYHKmhElVyzXetdtERErB5Eyi5pJYU9lyjbZ2+4kN5eziJiLFY3JWsFS0POeaWH3+\nAP7Y8ikufHwj/LNktlyVehQlEVEsmJwVKFKCfPx7ZfCO+ROarOPtEg7V+eqnDtwcmRR9Tawt11i+\nnIRem6/NgbFQi0GRBM2120SULpicFShSgnzv2tfwTQYS1k08ly7h2+ssRm7LNZbWu9hrC/JyRZMz\n124TUbpgck6geLqlpRLkxOStva4T1U0cb5ewVJ1nkttyjaX1LvbawWEflpp1GJuY4tptIkpLspJz\nc3MzOjo6IAgCrFYrKisrw/d8Ph8OHjyI7u5u2O12WWUy3VzGbaUS5O1CrdlYRfrSENrOM9YuYbl1\nltNynZickt16l/pSMDYxhYP/8iDGfVNc50xEaSdqcm5ra0NfXx9sNht6e3thtVphs9nC93/961/j\nW9/6Frq7u2WXyXRzWcojlSBvF2rNlsqsV7QvDaHtPMW6p6USa7Q6L5rxOVFjGpbfeo/W0h/3TXHy\nFxGlpagDlq2traitrQUAlJeXw+PxwOv1hu/v27cvfF9umUw212MYpfa7vl2sE5xCXxoGh30I4psv\nDbazPeHXbNu4ArUPlmJRYR5UArCoMA+1D5ZKJlapOv/T/Yvxws7vYnutRdb4uKFQ/mEcPLiDiDJV\n1Jazy+VCRUVF+NpoNMLpdEKn0wEAdDodhoaGYiojxmAoQE5OcrseTSb9vL/n165R3ByJ3JpTa3Jh\nKl4g+R6NW6tQkK/BpY+/hmtoHFqNGuO+O5N6zQN3o/TuhQCixzIxOYVrvYOi9671DuJ/PJGPPM2t\nvw57n1yNickpuId9MBRqwz+Ppc6LivJQucKEZx6/HwX5mqjlZ8e1BKfPfyby82/ijee1qZCIv2Op\nkimxZEocAGNRqvmIJeYJYcFgMOYPkVPG7R6L+X3nwmTSw+kcmff3DfgDMOojj9sGJv2yPvfxmuXY\nvHYpPF4fdAUanDr/2R2HUzz20DI4nSOyYnG4x+B0j4vecw2No/e/Bu/oAs4BMOIZh9zf0uM1y/HD\nB0vxp7e78MkXbpy90o8PuxwRx9vFxr5NJj0ee2gZxsYnI8Y7UyyvTbZE/R1LhUyJJVPiABiLUsUS\ni1QSj5qczWYzXC5X+NrhcMBkku52jadMpoh33DbSe4US5lwPp5Az2Ws+Nj05df6zqBuQSI19A7Ed\nxpHJB3cQUfaKOghYU1ODlpYWAEBnZyfMZrNk93S8ZdKNzx+Awz0mOoYcz7itHKFkHU/ykRoXfmDl\nIvzHX3vxi99fws//9yX84veX8OY7XQhMT8f0GXLH2+WMfYfqLDfeufxuiIiUJmrLubq6GhUVFaiv\nr4cgCGhqaoLdboder0ddXR327NmDGzdu4PPPP8eOHTuwdetWPPbYY3eUyRRylkkptTUX6ezmYDAY\ndXa5nFa1nHXSRTqtZAKfmJyKOz4iokwha8x5//79s65XrVoV/vOxY8dklckUsSyTUtoxjGJfGgDg\nF7+/JPr69i4XHv9eGU6d/1zWmm2prvPcHBU0uaqoCdw97OPOOESU9Xi4bQzmukwqlULd8CNjk7MS\n82dfeiST5Ztvd8vqggaku859/mlYX34fLW1fSCx/0sIQ4R4RUTZhIyUG6Xji0e0HUqgEYDoI5Glu\nfS+bmJyGSgDEJtQb9Fp80ndT9H0j7bX9+PfuwXvXvsLE5J3j1ROTAZxr/wpLzTrR1vXohB+vv/Wf\neOyhZQk/WpKISMn4L2AMlLDphdRENDGhbvjQSVHTf0/CE5PT4QQ6HWGl28rShRFPmAp9Gbmdd2wS\nPpHEPNPouB8PVy9BnmZ2Yp+YnMbp85/hzbe7JMsTEWU6tpxjMJ/LpGIVbfmRGLkHUoSEWtAGvQYL\n8jXouj4U8bWFCzTI197510fO9qNDXh8e/s7d6Oh2hg/zmOmvH34FCAK2165kC5qIshL/5YtRopZJ\nRSO1/Ghickq0NR3LIRrArRZ0EMDoxBT6HV7JskPeSfzqD5fx5jtdGPN98/lyth9dqNPi5ogvYqt8\nOgicu/ql6Lg2EVE2YMs5RqlYJjXmm8J7174SvXe+4yt09A7C5R6/YyZ1LIdozOTzy1vf/M0501/B\nNzkd/vwff/8eAMB7174WbRmP+abw7//3Wnj8O5JoZ0gTEWUqtpzjNJ+bXkQbR/7T212iE6xulZ2G\n0z0uOpM6lkM05mJicnrW55/8y2fYXmvBi7trUHP/Yiwq1EIlIDzGPDEZQBDSiRmIPK5NRJTp2HJO\nITkbmvj8AXzyhTum953Z4gx1t1/91ImbI77bZmsLoi3buQp9foE2Bz999D74/AE43WP495PXYvo8\nnixFRNmKyTmF5Gxo4vH6Yu6Wnrms6/Zu+HxtDsZ9U+Gkd2NwFP/25tWILfN43L6sTJurhiZXHdP4\nN5D4SXZERErFbu05EuuSlrPcSe6GJkU6LRbqYjtyUazFqc1Vo0inDSdmba4a2lw1Lnx8I2JiVquE\nmD5X6vOllqEtKtTi4aq7kz7JjohIqdhyjpNYl/QDK4shAPiw2xV1q8toG5o43WPQ/D2hVq0sxrl2\n8QlhYm5vcUbqPn/8e/dE/IKgyRGwID8X7ggzqmP5fCDaMjQTttda4PMHoNbkIjDpZ4uZiLIak3Oc\nxLqkz37w5azXSO27LTWTWpOrxr+fvBZOpN9ZWYxS8wJcd4xGrVeeRo3Hv1cWta7vXLkO75g/4heE\nyakgJmUmZpVwawmW8e8HaURq8UY6eCP0c22uGqbiBRlzrisRUbyYnOMwMjaJK584ZL9ebEmQVEty\nYjIQnjg1OOzDux98CUtpIapWFOPzG8MY8kZOmpP+ALxjfhRocwFId5+3/ecANLkq2UunIgkGgf31\n38E9S4okW7xKPa2LiEhpmJxjEOoe/uATp2SCvF2kfbe3bVyBQGAa7d0ueLyTMOi1GPNNic5o7ro+\nHP6zJkeFySnxhFpYMHvnLqnu8+mg/DXNUoyFeVET80xKO62LiEhpOCEsBqHuYXeMa2/FJkiFEv21\n3kF4vJNYqNPCsmwhfDKWGkVKzAAwNDqJQ//nMl4/8ykC09OSE7FCNDnSE780udJ/TTirmohofjE5\nyxTrPtUz3bts4R0/e/Ptrlnbcbq9PlzqHIBWM/ck5/b6cO7ql/jVH64gRy1E3Yhkckp6N5ACjTri\njHGxMW4iIpobdmvL5Bwaj3mdbsjFj2/g0y/c4a0tbWd7bx3uIGI+NwXpd3jx5ttd2F5nwSd9blx3\nik8oi7aN5tCoP+I9sTFujicTEc0Nk3MUM89DjrLbpKTQDOlPvxhCv8Mr+do8jWreNgVp73Zhy3f/\nAd7xyAk22jaaUsk71GUvZ7czIiKSh8k5ituXIc3Vl07pxAxgXnfrGvJO4n+9egXDY5GTczRSybty\nxSJoc9V4/cynOHf1m6VkUsvIiIhIGps0EuYyzhxJtFZqIswlMUfzcNUSvN7yCf7a/qXo/fYup+RO\naUREdCcmZwmxnoecbRbqNHjnyhc41/5VxC8dg8M+vN5ya+Y4ERHJw+QsQc4ypHQX5/bZAG51mb93\n7UbU1138+Eb4GEsiIoqOyVlCss5DThdiiVxuN/3MwzyIiEgak3MU2zauQO2DpShaENvJUOkilFzz\nNOrwiVAYDl9cAAAQaklEQVSlpgWir11XuRiGOM9XDu2SRkRE0XG2tgyBwDSCwcweMw0GgzjwLw9i\nsXEBctTC35dFzT6g4uGqJTjfEb0bW4zYLmlERCROVnJubm5GR0cHBEGA1WpFZWVl+N7Fixdx9OhR\nqNVqrF+/Hrt378b777+PvXv3YuXKlQAAi8WCAwcOJCaCBApMT+OXf7gs6zSodOfzT+Pty9fx3x+9\nDwBED6jw+QMRT9JSCbcOwNBq1KIbqXCLTyIi+aIm57a2NvT19cFms6G3txdWqxU2my18/4UXXsDx\n48dRUlKCn/zkJ/jRj34EAFi7di2OHTuWuJonwZvvdGdFYg75pM8Nnz8QTqK3H1AhdZLWhqol+NGa\npdAV5OLU+c8jHgtJRETRRU3Ora2tqK2tBQCUl5fD4/HA6/VCp9Ohv78fRUVFuOuuuwAAGzZsQGtr\nKyyW9N90Ysznx4VrX6e6Gkk15PWFW8qRtuCUOpM5tBMYj4UkIpqbqMnZ5XKhoqIifG00GuF0OqHT\n6eB0OmE0Gmfd6+/vh8ViQU9PD3bt2gWPx4PGxkbU1NRIfo7BUICcnOT+I24y6SPeO/rmB5KnP2Wi\nRUV5+H8f3cCV/xyAc2gcpoX5+O79d+FfH6uAWv3N3MG9T67GxOQU3MM+GAq1yNOI/zUqjbMeUs8l\n3TAW5cmUOADGolTzEUvME8KCwehrZ5YvX47GxkZs3rwZ/f39aGhowJkzZ6DRRJ7x7HaPxVqVOTGZ\n9HA6R0Tvjfmm8Ner87dlZ7rQ5qrx1sX/Cl873OM4ff4zjI1Pim7BmQNgxDMO8d9ifKSeS7phLMqT\nKXEAjEWpYolFKolHXUplNpvhcrnC1w6HAyaTSfTewMAAzGYzSkpKsGXLFgiCgGXLlqG4uBgDAwOy\nKqsEf2z5NCXbbKZapMMxuAUnEVFyRU3ONTU1aGlpAQB0dnbCbDZDp9MBAEpLS+H1enH9+nVMTU3h\n3LlzqKmpwenTp3H8+HEAgNPpxODgIEpKShIYxvwZ8/lxtcuR6mok3UKdBkPeSdF7g8M+rlEmIkqi\nqN3a1dXVqKioQH19PQRBQFNTE+x2O/R6Perq6nDo0CE899xzAIAtW7agrKwMJpMJ+/fvx7vvvgu/\n349Dhw5JdmkryR/f7sLkVPY1m++/x4iLH90Q7TFQCUC+lkviiYiSRda/uPv37591vWrVqvCf16xZ\nM2tpFQDodDr87ne/m4fqJZfPH8DVT7Kv1QwA1StMEffJng4C474p6AvS4wsWEVG6y+rtO33+ABzu\nsfB4qtM9lpWtZgAo0mlg1IsnX6New929iIiSKCv7KgPT0/j9qY9woeNL3Bz2wVioRZXFhJpvL051\n1VIiT6PG3SYdqu81i24wMuabwn/8tXfWWmYiIkqcrEzOtrM9s5LQ4LAP71y5jkBgGnkRtp/MZNV/\nP3krtMHIe9e+nvU7mJicDv++xJZUERHR/Mq6ZpDPH0B7l1P03rXem/jH+9JjVvlcqFSAAECrUSFP\no8LFj2/gF7+/BNvZHjz+vTIUaMU3g+Gxj0REyZF1LWeP14ebIgc3ALeONfSOiS8nyiTT08BiYz5u\n3BwP/yzUezA2MQX3iPjvIHTs48z9tomIaP5lXcu5SKeFsVB8ctNCnRYfdrtE72WagRmJeaZP+tww\nRJgYxmMfiYiSI+uSc+hkJTErly5EIEsma0cK8+aID6PjU6L3eOwjEVFyZF1yBm5NfPpv37sHiwrz\noBKARYV5qH2wFA9X3ZXqqimC77YDP/I0atQ+WMpjH4mIkiTrxpwBQK1SYefj38bmtUvDxxrmqAVY\nX76U6qop0oK8HDyxoZzLqIiIkiSr/7XV5qphNhQgRy3gl69chnNoItVVSppFhVo8XHV3uPdgoU7i\nxLAR7q1NRJRMWdlyvt2bb3fhunM01dVICLUKCIgcS11lMWF7rQU+fwAerw/52hz86g+XMSgyk50T\nwYiIkiurW84AMDLux7n2r1JdjYQJJeY8jXrW+Hpo/DjUe6Av0EScKMeJYEREyZXVLefJqSk899J7\nqa5GUhRoc2DdsRqmhfkRE20oYbd3ueAemYBBn4cqSzEnghERJVlWJmefP4CvXaP45StXMCV2RmIG\nGvL6oMlR3ZGYQ93aRTottLlqbK+14IkN5bN+RkREyZVVyTkwPQ3b2R60dzlxc8SHYHbkZQB3jhvP\n+l3MOPxj28YV4a5uIiJKjaxKzrcfeJFNbh83jnT4B8DDLYiIUi1rJoRJHXiRibS5gugEMED6d8HD\nLYiIUi9rWs4er090mVCmmvQHsfa+Euz4kQUF2txZ96Id/sHDLYiIUitrWs752hyohFTXInmCAN7/\n2wBOnf/8jntSh39wTTMRUeplTXIe900hSyZmzyLWTS11+AfXNBMRpV7WdGsX6bRQCciYBC0A0OSq\n4POLbP81Q6Ruaq5pJiJSrqxJzpP+QMYk5kWFWuz9cSVMhgIEpoOwnevBex9+JXoMZKRuarVKxTXN\nREQKlTXd2p9/5Ul1FeZNlcWEUrMe2lw1CrQ5+J871uD7VXdHeK10N3VoTTMTMxGRcmRNy9k77k91\nFWRbYlqAQc8EJiZnjxUvmrFRyO2211mgVqvYTU1ElAGyJjlfTaM1zhO+AP5t10PwjE4CwSCKdFqM\n+6Yku57ZTU1ElDlkJefm5mZ0dHRAEARYrVZUVlaG7128eBFHjx6FWq3G+vXrsXv37qhlUmF0In1a\nzu6RCYz7plBq0oV/pi+IfN7yTNx6k4go/UUdc25ra0NfXx9sNhsOHz6Mw4cPz7r/wgsv4KWXXsKf\n/vQnXLhwAT09PVHLpMID94gvHUoFQQDWfXsxjHrxhMu1xkRE2S1qcm5tbUVtbS0AoLy8HB6PB16v\nFwDQ39+PoqIi3HXXXVCpVNiwYQNaW1sly6RKzQN3JeVz8nJVqLzHgH+8zwxtrviv9/vfuRv/+sh9\nqL7XLHqfa42JiLJb1G5tl8uFioqK8LXRaITT6YROp4PT6YTRaJx1r7+/H263O2KZSAyGAuTkJC4h\nmQCUmhfgumM0Ie+/xLQAP/+XNSgxLkCe5tavdWx8Ei+f+hjXepwY9EygeGE+vnv/XfjXxyqgVqvQ\nuLUKBfkaXPr4a7iGxu+4HwuTSZ+IsFKCsShTpsSSKXEAjEWp5iOWmCeEBeM4Z1FOGbd7LOb3jdUv\nGlbj0CttuDE4MS/vt6NuJfQ6De5dagiPCY94xjEy4zVP1a7EjzfcM2uS1s2b33xBeLxmOTavXRrx\nvhwmkx5O50j0F6YBxqJMmRJLpsQBMBaliiUWqSQeNTmbzWa4XK7wtcPhgMlkEr03MDAAs9mM3Nzc\niGVSSZOTg+ad/4Sbw2M4/McP4B6WP0lsSbEWP1pThusuLx6qKME/LC6SXTbaJC1O4iIiopmiJuea\nmhq89NJLqK+vR2dnJ8xmc7h7urS0FF6vF9evX8fixYtx7tw5vPjii3C73RHLKIGxsACvNW3BZ32D\n+PzrYagEYGhkEn0DHnw9OI5/WLwACKowOj6JB1aacF/ZIo4BExFR0kRNztXV1aioqEB9fT0EQUBT\nUxPsdjv0ej3q6upw6NAhPPfccwCALVu2oKysDGVlZXeUUSJ9gQaV5cXh63UQ32WLiIgomYRgPIPI\nCZDs8YZsHeNQOsaiTJkSS6bEATAWpZqvMees2VubiIgoXTA5ExERKQyTMxERkcIwORMRESkMkzMR\nEZHCMDkTEREpDJMzERGRwjA5ExERKQyTMxERkcIwORMRESkMkzMREZHCMDkTEREpDJMzERGRwjA5\nExERKQyTMxERkcIo5jxnIiIiuoUtZyIiIoVhciYiIlIYJmciIiKFYXImIiJSGCZnIiIihWFyJiIi\nUpicVFcgUZqbm9HR0QFBEGC1WlFZWRm+d/HiRRw9ehRqtRrr16/H7t27o5ZJpVhjef/997F3716s\nXLkSAGCxWHDgwIFUVT9MKg6fz4eDBw+iu7sbdrtdVplUijUWpT4TQDqWS5cu4ejRo1CpVCgrK8Ph\nw4ehUqnS8rmIxXL58uW0fC4nTpzAyZMnoVKpsGrVKjQ1NUEQhLR8LmKxtLW1KfK5yPn9/uY3v8GH\nH36I119/XXYZUcEM9P777wefeeaZYDAYDPb09AS3bt066/7mzZuDX331VTAQCASffPLJYHd3d9Qy\nqRJPLJcuXQr+7Gc/S0V1I4oWx69+9avgK6+8Evznf/5n2WVSJZ5YlPhMgsHosdTV1QW//vrrYDAY\nDP7sZz8L/uUvf0nb5yIWSzo+l7GxsWBDQ0NwcnIyGAwGgzt27Ah+8MEHaflcIsWixOci5/fb3d0d\n3LZtW/AnP/mJ7DKRZGS3dmtrK2prawEA5eXl8Hg88Hq9AID+/n4UFRXhrrvugkqlwoYNG9Da2ipZ\nJpXiiUWJov1+9+3bF74vt0yqxBOLUkWLxW63Y/HixQAAo9EIt9udts9FLBalkoolPz8fr776KnJz\nczE+Pg6v1wuTyZSWzyVSLEok5/d75MgR7Nu3L6YykWRkcna5XDAYDOFro9EIp9MJAHA6nTAajXfc\nkyqTSvHEAgA9PT3YtWsXnnzySVy4cCG5lRYR7fer0+liLpMq8cQCKO+ZAPJjcTgcuHDhAjZs2JD2\nz2VmLEB6PhcAePnll1FXV4dNmzZh6dKlaftcgDtjAZT3XKLFYbfbsXbtWixZskR2GSkZO+Y8UzCO\nHUrjKZMMcuq1fPlyNDY2YvPmzejv70dDQwPOnDkDjUaThBrKw2eivGcCiMcyODiIXbt2oampadY/\nNFJllEBOLOn8XJ555hk0NDRg586dWL16tawySiAnlnR4LjPjGBoagt1uxyuvvIKBgQFZZaLJyJaz\n2WyGy+UKXzscjnBXye33BgYGYDabJcukUjyxlJSUYMuWLRAEAcuWLUNxcbHkX5hkiOf3m47PJBIl\nPhMgeixerxc7d+7Es88+i3Xr1skqkyrxxJKOz2VoaAiXL18GAOTl5WH9+vW4evVqWj6XSLEo8blI\nxXHp0iXcvHkTTz31FBobG9HZ2Ynm5uY5PZOMTM41NTVoaWkBAHR2dsJsNoe7tEpLS+H1enH9+nVM\nTU3h3LlzqKmpkSyTSvHEcvr0aRw/fhzAra7vwcFBlJSUpCwGQDqO+SyTDPHUS4nPBIgey5EjR/D0\n009j/fr1ssukSjyxpONzmZqawvPPP4/R0VEAwEcffYSysrK0fC6RYlHic5GKY9OmTXjrrbdw4sQJ\n/Pa3v0VFRQWsVuucnknGnkr14osv4sqVKxAEAU1NTfjb3/4GvV6Puro6XL58GS+++CIA4Ic//CF+\n+tOfipZZtWpVKkMIizUWr9eL/fv3Y3h4GH6/H42NjeHxtVSSimPPnj24ceMGuru7cf/992Pr1q14\n7LHH0vKZiMXy8MMPK/KZAJFjWbduHdasWYOqqqrwax999FFs27Yt7Z5LpFgeeeSRtHsudXV1sNvt\neOONN5CTk4N7770Xv/zlLyEIQto9l0ixjI6OKvK5SMURcv36dfz85z8PL6WK95lkbHImIiJKVxnZ\nrU1ERJTOmJyJiIgUhsmZiIhIYZiciYiIFIbJmYiISGGYnImIiBSGyZmIiEhhmJyJiIgU5v8DG/p+\nLuhdNL4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0e49be5320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(outputs.numpy(), Y_df.values)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_df['res'] = np.abs(outputs.numpy() - Y_df.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.17347069184091402\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Int64Index([3132, 3068, 2937, 227, 331], dtype='int64')"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(last_df['res'].max())\n",
    "last_df.ix[last_df['res'].argmax()]\n",
    "\n",
    "\n",
    "last_df.nlargest(5, ['res']).index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_df.index[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}