Balse 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict, Counter\n",
    "\n",
    "god_rmse = defaultdict(list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from zero.als import MangakiALS\n",
    "from zero.svd import MangakiSVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "could not convert string to float: 'wontsee'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-16717eccce24>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/Users/jilljenn/code/mangaki/mangaki/data/balse/ratings.csv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mrating_values\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtitle_filename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'balse/works.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/code/mangaki/venv/lib/python3.6/site-packages/zero/dataset.py\u001b[0m in \u001b[0;36mload_csv\u001b[0;34m(self, filename, convert, title_filename)\u001b[0m\n\u001b[1;32m     56\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mload_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtitle_filename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m         \u001b[0mtriplets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'user'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'item'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rating'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m         \u001b[0;31m# noinspection PyTypeChecker\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0mvectorized_convert\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvectorize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0motypes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat64\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: could not convert string to float: 'wontsee'"
     ]
    }
   ],
   "source": [
    "from zero.dataset import Dataset\n",
    "from mangaki.utils.values import rating_values\n",
    "\n",
    "dataset = Dataset()\n",
    "dataset.load_csv('/Users/jilljenn/code/mangaki/mangaki/data/balse/ratings.csv', convert=lambda x: rating_values[x], title_filename='balse/works.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from mangaki.utils.balse import MangakiBALSE\n",
    "from mangaki.settings import DATA_DIR\n",
    "from scipy.sparse import load_npz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import ShuffleSplit\n",
    "\n",
    "NB_SPLIT = 5\n",
    "k_fold = ShuffleSplit(n_splits=NB_SPLIT)\n",
    "SETS = list(k_fold.split(dataset.anonymized.X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "GOD_I = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# START\n",
    "i_train, i_test = SETS[GOD_I]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(334390, 2)"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.anonymized.X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "NOTVAL = round(0.7 * len(i_train))\n",
    "\n",
    "X_train = dataset.anonymized.X[i_train]\n",
    "y_train = dataset.anonymized.y[i_train]\n",
    "X_subtrain = X_train[:NOTVAL]\n",
    "y_subtrain = y_train[:NOTVAL]\n",
    "X_val = X_train[NOTVAL:]\n",
    "y_val = y_train[NOTVAL:]\n",
    "X_test = dataset.anonymized.X[i_test]\n",
    "y_test = dataset.anonymized.y[i_test]\n",
    "nb_subtrain_rated = Counter(X_subtrain[:, 1])\n",
    "nb_train_rated = Counter(X_train[:, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_als(X_train, X_test, y_train, y_test):\n",
    "    als = MangakiALS(10, LAMBDA=0.1)\n",
    "    als.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "    #als.fit(X_train, y_train)\n",
    "    als.all_errors(X_train, X_test, y_train, y_test)\n",
    "    return als"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing M: (2079 × 9979)\n",
      "Chrono: fill and center matrix [0q, 1463ms]\n",
      "Shapes (2079, 10) (10, 9979)\n",
      "Chrono: factor matrix [0q, 12342ms]\n",
      "Train minmax -2.98166002478 4.52700465152\n",
      "Train error 0.980646617638\n",
      "Test minmax -2.80634401035 5.18515675231\n",
      "Test error 1.15149072211\n",
      "Computing M: (2079 × 9979)\n",
      "Chrono: fill and center matrix [0q, 988ms]\n",
      "Shapes (2079, 10) (10, 9979)\n",
      "Chrono: factor matrix [0q, 10569ms]\n",
      "Train minmax -2.93223724413 4.56488695928\n",
      "Train error 0.946329414823\n",
      "Test minmax -4.42400264331 4.49266330151\n",
      "Test error 1.18043112233\n"
     ]
    }
   ],
   "source": [
    "als = run_als(X_train, X_test, y_train, y_test)\n",
    "sub_als = run_als(X_subtrain, X_test, y_subtrain, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train minmax 0 0\n",
      "Train error 1.57335141405\n",
      "Test minmax 0 0\n",
      "Test error 1.5646916998\n"
     ]
    }
   ],
   "source": [
    "from mangaki.utils.zero import MangakiZero\n",
    "\n",
    "zero = MangakiZero()\n",
    "zero.all_errors(X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from mangaki.utils.lasso import MangakiLASSO\n",
    "\n",
    "def run_lasso(X_train, X_test, y_train, y_test):\n",
    "    lasso = MangakiLASSO(with_bias=True, alpha=0.01)\n",
    "    #lasso.load_tags()\n",
    "    lasso.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "    # lasso.fit(X_train, y_train)\n",
    "    lasso.all_errors(X_train, X_test, y_train, y_test)\n",
    "    return lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fitting user ID 0/2073…\n",
      "/Users/jin/Sites/mangaki/venv/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:484: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.\n",
      "  ConvergenceWarning)\n",
      "Fitting user ID 500/2073…\n",
      "Fitting user ID 1000/2073…\n",
      "Fitting user ID 1500/2073…\n",
      "Fitting user ID 2000/2073…\n",
      "Sparsity: 1.00000 ± 0.00000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train minmax -2.0 2.0\n",
      "Train error 0.763738670439\n",
      "Test minmax -2.0 2.0\n",
      "Test error 1.44732800804\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fitting user ID 0/2045…\n",
      "Fitting user ID 500/2045…\n",
      "Fitting user ID 1000/2045…\n",
      "Fitting user ID 1500/2045…\n",
      "Fitting user ID 2000/2045…\n",
      "Sparsity: 1.00000 ± 0.00000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train minmax -2.0 2.0\n",
      "Train error 0.694643658002\n",
      "Test minmax -2.0 2.0\n",
      "Test error 1.48473649764\n"
     ]
    }
   ],
   "source": [
    "lasso = run_lasso(X_train, X_test, y_train, y_test)\n",
    "sub_lasso = run_lasso(X_subtrain, X_test, y_subtrain, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "i_test_ordered = sorted(i_test, key=lambda i: nb_train_rated[dataset.anonymized.X[i][1]], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_tmp_test = dataset.anonymized.X[i_test_ordered]\n",
    "y_tmp_test = dataset.anonymized.y[i_test_ordered]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Attention\n",
    "# X_val = X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nb_r = list(map(lambda x: nb_subtrain_rated[x[1]], X_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_val_als = sub_als.predict(X_val)\n",
    "y_val_lasso = sub_lasso.predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_als = sub_als.predict(X_tmp_test)\n",
    "y_lasso = sub_lasso.predict(X_tmp_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "X = np.column_stack((nb_r, y_val_als, y_val_lasso))\n",
    "y = y_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90285, 3)"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90285,)"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_val_als.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90285,)"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "beta = tf.Variable(tf.random_normal([1]), name='beta')\n",
    "gamma = tf.Variable(tf.random_normal([1]), name='gamma')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "e1 = np.array([1, 0, 0])\n",
    "e2 = np.array([0, 1, 0])\n",
    "e3 = np.array([0, 0, 1])\n",
    "pred = tf.sigmoid(beta*(X.dot(e1) - gamma)) * X.dot(e2) + (1 - tf.sigmoid(beta*(X.dot(e1) - gamma))) * X.dot(e3)\n",
    "loss = tf.reduce_mean(tf.square(y - pred)) ** 0.5\n",
    "reg_loss = loss\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "starter_learning_rate = 0.9\n",
    "learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step,\n",
    "                                           20, 0.9965402628278678, staircase=True)\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "train_step = optimizer.minimize(reg_loss, var_list=[beta, gamma], global_step=global_step)\n",
    "\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess = tf.InteractiveSession()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 1.16952 beta [ 0.03688588] gamma [-0.69050276] 0.159098\n",
      "loss 1.16952 beta [ 0.03687955] gamma [-0.69555074] 0.145893\n",
      "loss 1.16952 beta [ 0.03687378] gamma [-0.70017523] 0.133785\n",
      "loss 1.16952 beta [ 0.03686853] gamma [-0.70441359] 0.122681\n",
      "loss 1.16952 beta [ 0.0368637] gamma [-0.70829797] 0.112499\n",
      "loss 1.16952 beta [ 0.03685927] gamma [-0.71185929] 0.103162\n",
      "loss 1.16952 beta [ 0.03685518] gamma [-0.71512151] 0.0946\n",
      "loss 1.16952 beta [ 0.03685147] gamma [-0.71811312] 0.0867485\n",
      "loss 1.16952 beta [ 0.03684811] gamma [-0.72085565] 0.0795487\n",
      "loss 1.16952 beta [ 0.03684492] gamma [-0.72336936] 0.0729465\n"
     ]
    }
   ],
   "source": [
    "for i in range(5000):\n",
    "    sess.run(train_step)\n",
    "    if i % 500 == 0:\n",
    "        print('loss', sess.run(reg_loss), 'beta', beta.eval(), 'gamma', gamma.eval(), learning_rate.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nb_rt = list(map(lambda x: nb_train_rated[x[1]], X_tmp_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Xt = np.column_stack((nb_rt, y_als, y_lasso))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.1695154,\n",
       " array([ 0.03684209], dtype=float32),\n",
       " array([-0.72566766], dtype=float32))"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss.eval(), beta.eval(), gamma.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_full_als = als.predict(X_tmp_test)\n",
    "y_full_lasso = lasso.predict(X_tmp_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "new_pred = tf.sigmoid(beta*(Xt.dot(e1) - gamma)) * y_full_als + (1 - tf.sigmoid(beta*(Xt.dot(e1) - gamma))) * y_full_lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.1804311223303001"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "als.compute_rmse(y_als, y_tmp_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test error 1.15149072211\n"
     ]
    }
   ],
   "source": [
    "WINDOW = 1000000\n",
    "\n",
    "rmse_als_full = als.compute_rmse(y_full_als, y_tmp_test)\n",
    "print('Test error', rmse_als_full)\n",
    "god_rmse['als'].append(rmse_als_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test error 1.14354738023\n"
     ]
    }
   ],
   "source": [
    "y_new_pred = new_pred.eval()\n",
    "rmse_balse_full = als.compute_rmse(y_new_pred, y_tmp_test)\n",
    "print('Test error', rmse_balse_full)\n",
    "god_rmse['balse'].append(rmse_balse_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test error 1.44732800804\n"
     ]
    }
   ],
   "source": [
    "rmse_lasso_full = als.compute_rmse(y_full_lasso, y_tmp_test)\n",
    "print('Test error', rmse_lasso_full)\n",
    "god_rmse['lasso'].append(rmse_lasso_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_rt[-1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.02990520051436945"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1000 / len(y_tmp_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.007386584527049254"
      ]
     },
     "execution_count": 219,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(len(nb_rt) - nb_rt.index(0)) / len(y_tmp_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for WINDOW, tag in [(1000, 'cold'), (len(nb_rt) - nb_rt.index(0), 'freeze')]:\n",
    "    god_rmse['als-%s' % tag].append(als.compute_rmse(y_full_als[-WINDOW:], y_tmp_test[-WINDOW:]))\n",
    "    y_new_pred = new_pred.eval()\n",
    "    god_rmse['balse-%s' % tag].append(als.compute_rmse(y_new_pred[-WINDOW:], y_tmp_test[-WINDOW:]))\n",
    "    god_rmse['lasso-%s' % tag].append(als.compute_rmse(y_full_lasso[-WINDOW:], y_tmp_test[-WINDOW:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "def avgstd(l):  # Displays mean and variance\n",
    "    n = len(l)\n",
    "    mean = float(sum(l)) / n\n",
    "    var = float(sum(i * i for i in l)) / n - mean * mean\n",
    "    return '%.5f ± %.5f' % (round(mean, 5), round(1.96 * math.sqrt(var / n), 3))  # '%.3f ± %.3f' % "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "als 1.15681 ± 0.00400\n",
      "balse 1.14954 ± 0.00400\n",
      "lasso 1.44444 ± 0.00200\n",
      "als-cold 1.29269 ± 0.02900\n",
      "balse-cold 1.22714 ± 0.03600\n",
      "lasso-cold 1.31331 ± 0.03600\n",
      "als-freeze 1.50047 ± 0.03500\n",
      "balse-freeze 1.34533 ± 0.04500\n",
      "lasso-freeze 1.37909 ± 0.05600\n"
     ]
    }
   ],
   "source": [
    "for key in god_rmse:\n",
    "    # print(key, god_rmse[key])\n",
    "    print(key, avgstd(god_rmse[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "240\n",
      "[1306 1483]\n",
      "Enfants d'Agartha (les) als [ 2.45263219] lasso [ 0.97617097] sub_als 1.343685313 sub_lasso 1.0957989608 balse 1.70751 truth 0.1 user bias 0.833885152613 nb ratings 0\n",
      "[1698 3047]\n",
      "Argento Soma Special als [ 1.09988402] lasso [ 0.66086675] sub_als 2.03313304362 sub_lasso 0.586069092746 balse 0.878325 truth 2.0 user bias 1.29502762431 nb ratings 0\n",
      "[2034 1180]\n",
      "Hacka Doll als [ 1.64694415] lasso [ 0.37655969] sub_als 0.672194576379 sub_lasso 0.315973101292 balse 1.00582 truth -2.0 user bias 1.20058548009 nb ratings 0\n"
     ]
    }
   ],
   "source": [
    "print(WINDOW)\n",
    "for i in range(1, 4):\n",
    "    encoded_user_id, encoded_work_id = X_tmp_test[-i]\n",
    "    if True:#encoded_user_id == 2015:\n",
    "        print(X_tmp_test[-i])\n",
    "        print(dataset.titles[encoded_work_id],\n",
    "              'als', als.predict(np.array([(encoded_user_id, encoded_work_id)])),\n",
    "              'lasso', lasso.predict(np.array([(encoded_user_id, encoded_work_id)])),\n",
    "              'sub_als', y_als[-i],\n",
    "              'sub_lasso', y_lasso[-i],\n",
    "              'balse', y_new_pred[-i],\n",
    "              'truth', y_tmp_test[-i],\n",
    "              'user bias', als.means[encoded_user_id],\n",
    "              'nb ratings', nb_train_rated[encoded_work_id])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'os' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-43-72488b1edefe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDATA_DIR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'balse/balse.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mtags\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m','\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mWORK_ID\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m665\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'os' is not defined"
     ]
    }
   ],
   "source": [
    "with open(os.path.join(DATA_DIR, 'balse/balse.csv')) as f:\n",
    "    tags = [line.split(',')[1] for line in f]\n",
    "\n",
    "WORK_ID = 665\n",
    "    \n",
    "for tag, weight in zip(tags, T[WORK_ID]):\n",
    "    if weight != 0:\n",
    "        print(tag.strip(), weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for tag, weight in zip(tags, lasso.reg[2015].coef_):\n",
    "    if weight != 0:\n",
    "        print(tag, weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.73747573])"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lasso.reg[2015].predict(np.array([T[WORK_ID]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lasso.reg[2015].coef_.dot(T[WORK_ID])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.73747572817848728"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lasso.reg[2015].intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for tag, weight in zip(tags, lasso.reg[2015].coef_ * T[WORK_ID]):\n",
    "    if weight:\n",
    "        print(tag.strip(), weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.34922168, -0.02603758, -0.31870448,  0.37174495, -0.04622742,\n",
       "        1.42612367, -0.84433803,  0.20316341,  0.80450289,  1.12107641])"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "als.VT.T[3748]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'dislike': 16,\n",
       "         'favorite': 23,\n",
       "         'like': 118,\n",
       "         'neutral': 9,\n",
       "         'willsee': 330,\n",
       "         'wontsee': 68})"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "Counter(User.objects.get(id=1).rating_set.values_list('choice', flat=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "77\n",
      "9163\n"
     ]
    }
   ],
   "source": [
    "for work_id, title in enumerate(dataset.titles):\n",
    "    if title == 'Millennium Actress':\n",
    "        print(work_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter, defaultdict\n",
    "\n",
    "nb = defaultdict(Counter)\n",
    "for (user_id, work_id), value in zip(dataset.anonymized.X, dataset.anonymized.y):\n",
    "    nb[user_id][value] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "981 Counter({-0.5: 126, 2.0: 92, -2.0: 51, 4.0: 46, 0.5: 42, 0.1: 28})\n",
      "121 Counter({-0.5: 75, 0.5: 64, 4.0: 62, 2.0: 51, 0.1: 22, -2.0: 5})\n",
      "1738 Counter({4.0: 21, -2.0: 8, 0.1: 7, 2.0: 7, 0.5: 4, -0.5: 1})\n",
      "1120 Counter({0.5: 742, 2.0: 316, -0.5: 208, -2.0: 137, 0.1: 126, 4.0: 39})\n",
      "326 Counter({2.0: 123, 4.0: 70, 0.1: 55, -2.0: 46, 0.5: 25, -0.5: 11})\n",
      "1689 Counter({0.1: 71, -2.0: 43, 2.0: 42, 4.0: 7})\n",
      "1979 Counter({2.0: 61, 0.5: 53, -0.5: 34, 4.0: 31, 0.1: 14})\n",
      "278 Counter({2.0: 103, -0.5: 85, 0.5: 31, 4.0: 26, 0.1: 24, -2.0: 10})\n",
      "407 Counter({0.1: 13, -2.0: 11, -0.5: 9, 2.0: 9, 0.5: 3, 4.0: 1})\n",
      "512 Counter({4.0: 7, 2.0: 6, -2.0: 2, -0.5: 2, 0.5: 1})\n",
      "453 Counter({-0.5: 518, 0.5: 437, 2.0: 278, 4.0: 77, 0.1: 17, -2.0: 11})\n",
      "2015 Counter({0.5: 333, 2.0: 121, -0.5: 72, 4.0: 25, -2.0: 16, 0.1: 9})\n",
      "1839 Counter({2.0: 197, -0.5: 73, 4.0: 62, 0.5: 57, 0.1: 46, -2.0: 16})\n",
      "242 Counter({2.0: 199, 4.0: 37, 0.1: 34, -2.0: 26, -0.5: 4, 0.5: 4})\n",
      "1678 Counter({2.0: 6, 0.5: 5, 4.0: 5, -2.0: 2})\n"
     ]
    }
   ],
   "source": [
    "for (user_id, work_id), value in zip(dataset.anonymized.X, dataset.anonymized.y):\n",
    "    if work_id == 77 and value == 4:\n",
    "        print(user_id, nb[user_id])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# JJ est 2015 !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os.path\n",
    "import numpy as np\n",
    "\n",
    "T = load_npz(os.path.join(DATA_DIR, 'balse/tag-matrix.npz')).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Try various lassos on myself\n",
    "USER_ID = 2015\n",
    "X = []\n",
    "y = []\n",
    "for (user_id, work_id), value in zip(dataset.anonymized.X, dataset.anonymized.y):\n",
    "    if user_id == USER_ID:\n",
    "        X.append(T[work_id])\n",
    "        y.append(value)\n",
    "X = np.array(X)\n",
    "y = np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "i_train, i_test = train_test_split(range(len(X)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = X[i_train]\n",
    "X_test = X[i_test]\n",
    "y_train = y[i_train]\n",
    "y_test = y[i_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(432, 503)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>493</th>\n",
       "      <th>494</th>\n",
       "      <th>495</th>\n",
       "      <th>496</th>\n",
       "      <th>497</th>\n",
       "      <th>498</th>\n",
       "      <th>499</th>\n",
       "      <th>500</th>\n",
       "      <th>501</th>\n",
       "      <th>502</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>576.000000</td>\n",
       "      <td>576.0</td>\n",
       "      <td>576.0</td>\n",
       "      <td>576.000000</td>\n",
       "      <td>576.000000</td>\n",
       "      <td>576.000000</td>\n",
       "      <td>576.000000</td>\n",
       "      <td>576.0</td>\n",
       "      <td>576.0</td>\n",
       "      <td>576.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>576.000000</td>\n",
       "      <td>576.000000</td>\n",
       "      <td>576.0</td>\n",
       "      <td>576.000000</td>\n",
       "      <td>576.0</td>\n",
       "      <td>576.000000</td>\n",
       "      <td>576.000000</td>\n",
       "      <td>576.000000</td>\n",
       "      <td>576.000000</td>\n",
       "      <td>576.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.000508</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000861</td>\n",
       "      <td>0.004601</td>\n",
       "      <td>0.009568</td>\n",
       "      <td>0.000916</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001197</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001010</td>\n",
       "      <td>0.037237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.078183</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001716</td>\n",
       "      <td>0.163196</td>\n",
       "      <td>0.000252</td>\n",
       "      <td>0.009215</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.008608</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.012030</td>\n",
       "      <td>0.028270</td>\n",
       "      <td>0.051571</td>\n",
       "      <td>0.016432</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.013173</td>\n",
       "      <td>...</td>\n",
       "      <td>0.014544</td>\n",
       "      <td>0.094976</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.122134</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.014617</td>\n",
       "      <td>0.262505</td>\n",
       "      <td>0.006056</td>\n",
       "      <td>0.046455</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.144202</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.226478</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.148561</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.187316</td>\n",
       "      <td>0.261796</td>\n",
       "      <td>0.672730</td>\n",
       "      <td>0.354625</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.201148</td>\n",
       "      <td>...</td>\n",
       "      <td>0.272276</td>\n",
       "      <td>0.613001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.589839</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.160001</td>\n",
       "      <td>0.977520</td>\n",
       "      <td>0.145337</td>\n",
       "      <td>0.402184</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 503 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              0      1      2           3           4           5    \\\n",
       "count  576.000000  576.0  576.0  576.000000  576.000000  576.000000   \n",
       "mean     0.000508    0.0    0.0    0.000861    0.004601    0.009568   \n",
       "std      0.008608    0.0    0.0    0.012030    0.028270    0.051571   \n",
       "min      0.000000    0.0    0.0    0.000000    0.000000    0.000000   \n",
       "25%      0.000000    0.0    0.0    0.000000    0.000000    0.000000   \n",
       "50%      0.000000    0.0    0.0    0.000000    0.000000    0.000000   \n",
       "75%      0.000000    0.0    0.0    0.000000    0.000000    0.000000   \n",
       "max      0.148561    0.0    0.0    0.187316    0.261796    0.672730   \n",
       "\n",
       "              6      7      8           9    ...           493         494  \\\n",
       "count  576.000000  576.0  576.0  576.000000  ...    576.000000  576.000000   \n",
       "mean     0.000916    0.0    0.0    0.001197  ...      0.001010    0.037237   \n",
       "std      0.016432    0.0    0.0    0.013173  ...      0.014544    0.094976   \n",
       "min      0.000000    0.0    0.0    0.000000  ...      0.000000    0.000000   \n",
       "25%      0.000000    0.0    0.0    0.000000  ...      0.000000    0.000000   \n",
       "50%      0.000000    0.0    0.0    0.000000  ...      0.000000    0.000000   \n",
       "75%      0.000000    0.0    0.0    0.000000  ...      0.000000    0.000000   \n",
       "max      0.354625    0.0    0.0    0.201148  ...      0.272276    0.613001   \n",
       "\n",
       "         495         496    497         498         499         500  \\\n",
       "count  576.0  576.000000  576.0  576.000000  576.000000  576.000000   \n",
       "mean     0.0    0.078183    0.0    0.001716    0.163196    0.000252   \n",
       "std      0.0    0.122134    0.0    0.014617    0.262505    0.006056   \n",
       "min      0.0    0.000000    0.0    0.000000    0.000000    0.000000   \n",
       "25%      0.0    0.000000    0.0    0.000000    0.000000    0.000000   \n",
       "50%      0.0    0.000000    0.0    0.000000    0.000000    0.000000   \n",
       "75%      0.0    0.144202    0.0    0.000000    0.226478    0.000000   \n",
       "max      0.0    0.589839    0.0    0.160001    0.977520    0.145337   \n",
       "\n",
       "              501    502  \n",
       "count  576.000000  576.0  \n",
       "mean     0.009215    0.0  \n",
       "std      0.046455    0.0  \n",
       "min      0.000000    0.0  \n",
       "25%      0.000000    0.0  \n",
       "50%      0.000000    0.0  \n",
       "75%      0.000000    0.0  \n",
       "max      0.402184    0.0  \n",
       "\n",
       "[8 rows x 503 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "pd.DataFrame(X).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "T = load_npz(os.path.join(DATA_DIR, 'balse/tag-matrix.npz')).tocsc()\n",
    "\n",
    "T = scale(T, with_mean=False).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'T_scaled' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-43-18a853b9eace>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mT_scaled\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdescribe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'T_scaled' is not defined"
     ]
    }
   ],
   "source": [
    "pd.DataFrame(T_scaled).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'T_scaled' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-44-df0736ce51e4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mT_scaled\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'T_scaled' is not defined"
     ]
    }
   ],
   "source": [
    "np.std(T_scaled[:, 0].data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'T_scaled' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-45-9ef787dcba69>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mT_scaled\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'T_scaled' is not defined"
     ]
    }
   ],
   "source": [
    "np.std(T_scaled[:, 1].data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'T_scaled' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-46-8aef8e108ffc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mT_scaled\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'T_scaled' is not defined"
     ]
    }
   ],
   "source": [
    "T_scaled.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jilljenn/code/mangaki/venv/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:484: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.5708497   0.50914856  0.86323306  1.08109931  0.50466447  0.45824748\n",
      "  1.06714965  0.78175205  0.69206829  0.60301172]\n",
      "[ 0.5  0.5  0.5  0.5  2.   0.5  2.   0.1  0.5  0.5]\n",
      "alpha 0.00675066364528\n",
      "1.14569988748\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression, Lasso, LassoLarsCV, LassoCV\n",
    "\n",
    "clf = LassoCV(cv=10, fit_intercept=False)\n",
    "# clf = Lasso(alpha=0.1, fit_intercept=False)\n",
    "clf.fit(X_train, y_train)\n",
    "y_pred = clf.predict(X_test)\n",
    "print(y_pred[:10])\n",
    "print(y_test[:10])\n",
    "print('alpha', clf.alpha_)\n",
    "print(als.compute_rmse(y_pred, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.81072104  0.76327843  0.48053424  0.60593274  0.68143621  0.84986515\n",
      "  0.85026601  0.84986515  0.65577524  0.8695655 ]\n",
      "[-0.5  0.5  0.5  0.5  0.5  2.   0.5  2.   0.5  2. ]\n",
      "0.849865148083\n",
      "1.0604782715\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression, Lasso, LassoLarsCV, LassoCV\n",
    "\n",
    "# clf = LassoCV(cv=10, fit_intercept=False)\n",
    "clf = Lasso(alpha=0.1, fit_intercept=True)\n",
    "clf.fit(X_train, y_train)\n",
    "y_pred = clf.predict(X_test)\n",
    "print(y_pred[:10])\n",
    "print(y_test[:10])\n",
    "# print('alpha', clf.alpha_)\n",
    "print(clf.intercept_)\n",
    "print(lasso.compute_rmse(y_pred, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(x < 0 for x in y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 petals 0.062048928418\n",
      "22 butterfly -0.00575388224009\n",
      "27 necktie -0.0441011232299\n",
      "29 kimono 0.0108843921645\n",
      "33 table 0.0609460759477\n",
      "48 instrument -0.00326064812932\n",
      "60 from behind -0.00583659841878\n",
      "69 traditional media 0.0147549217731\n",
      "73 magical girl 0.0312992390526\n",
      "76 younger -0.00337236166619\n",
      "92 child -0.010405530536\n",
      "102 3boys -0.0148483643512\n",
      "130 blonde hair -0.0562550649455\n",
      "146 dutch angle 0.00639426176897\n",
      "180 white hair -0.0311996357131\n",
      "228 mole -0.00851666561641\n",
      "229 blue hair -0.0126566933755\n",
      "244 zettai ryouiki -0.00325367435046\n",
      "282 backpack -0.0220788327809\n",
      "284 twin drills 0.0149344607109\n",
      "369 facial mark -0.0585771795163\n",
      "416 short twintails 0.00372983158992\n",
      "451 sky 0.0121431534467\n",
      "476 frog 0.039292267754\n"
     ]
    }
   ],
   "source": [
    "for i, (tag, weight) in enumerate(zip(tags, clf.coef_)):\n",
    "    if weight != 0:\n",
    "        print(i, tag.strip(), weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def search_tag(dataset, tag_id):\n",
    "    results = Counter()\n",
    "    for (user_id, work_id), choice in zip(dataset.anonymized.X, dataset.anonymized.y):\n",
    "        weight = T[work_id][tag_id]\n",
    "        if user_id == USER_ID and weight != 0:\n",
    "            results[dataset.titles[work_id], weight] = choice\n",
    "    return results.most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('Princesse Mononoké', 64.162086538829627), 4.0)]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_tag(dataset, 476)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9979, 503)"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       ..., \n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Django Shell-Plus",
   "language": "python",
   "name": "django_extensions"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}