sm_shared_svd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Obtain Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_df = pd.read_csv('../data/interim/final_ratings.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>listing_id</th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>reviewer_id</th>\n",
       "      <th>reviewer_name</th>\n",
       "      <th>comments</th>\n",
       "      <th>lang</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2539</td>\n",
       "      <td>55688172</td>\n",
       "      <td>2015-12-04</td>\n",
       "      <td>25160947</td>\n",
       "      <td>Peter</td>\n",
       "      <td>Great host</td>\n",
       "      <td>ENGLISH</td>\n",
       "      <td>0.6249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2539</td>\n",
       "      <td>97474898</td>\n",
       "      <td>2016-08-27</td>\n",
       "      <td>91513326</td>\n",
       "      <td>Liz</td>\n",
       "      <td>Nice room for the price. Great neighborhood. J...</td>\n",
       "      <td>ENGLISH</td>\n",
       "      <td>0.9228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2539</td>\n",
       "      <td>105340344</td>\n",
       "      <td>2016-10-01</td>\n",
       "      <td>90022459</td>\n",
       "      <td>Евгений</td>\n",
       "      <td>Very nice apt.  New remodeled.</td>\n",
       "      <td>ENGLISH</td>\n",
       "      <td>0.4754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2539</td>\n",
       "      <td>133131670</td>\n",
       "      <td>2017-02-20</td>\n",
       "      <td>116165195</td>\n",
       "      <td>George</td>\n",
       "      <td>Great place to stay for a while. John is a gre...</td>\n",
       "      <td>ENGLISH</td>\n",
       "      <td>0.9231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2539</td>\n",
       "      <td>157777930</td>\n",
       "      <td>2017-06-04</td>\n",
       "      <td>1806142</td>\n",
       "      <td>Isaac</td>\n",
       "      <td>I really enjoyed my time here in deep south Br...</td>\n",
       "      <td>ENGLISH</td>\n",
       "      <td>0.9881</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   listing_id         id        date  reviewer_id reviewer_name  \\\n",
       "0        2539   55688172  2015-12-04     25160947         Peter   \n",
       "1        2539   97474898  2016-08-27     91513326           Liz   \n",
       "2        2539  105340344  2016-10-01     90022459       Евгений   \n",
       "3        2539  133131670  2017-02-20    116165195        George   \n",
       "4        2539  157777930  2017-06-04      1806142         Isaac   \n",
       "\n",
       "                                            comments     lang  polarity  \n",
       "0                                         Great host  ENGLISH    0.6249  \n",
       "1  Nice room for the price. Great neighborhood. J...  ENGLISH    0.9228  \n",
       "2                     Very nice apt.  New remodeled.  ENGLISH    0.4754  \n",
       "3  Great place to stay for a while. John is a gre...  ENGLISH    0.9231  \n",
       "4  I really enjoyed my time here in deep south Br...  ENGLISH    0.9881  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1053172, 8)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    1.053172e+06\n",
       "mean     8.133919e-01\n",
       "std      2.722260e-01\n",
       "min     -1.000000e+00\n",
       "25%      7.906000e-01\n",
       "50%      9.215000e-01\n",
       "75%      9.666667e-01\n",
       "max      1.000000e+00\n",
       "Name: polarity, dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_df.polarity.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from surprise import SVD, Dataset, Reader, accuracy\n",
    "from surprise.model_selection import cross_validate, train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "reader = Reader(rating_scale=(-1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = Dataset.load_from_df(reviews_df[['listing_id', 'reviewer_id', 'polarity']], reader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method Dataset.read_ratings of <surprise.dataset.DatasetAutoFolds object at 0x10f616be0>>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.read_ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset, testset = train_test_split(data, test_size=.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "algo = SVD(n_factors=10, n_epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the algorithm on the trainset, and predict ratings for the testset\n",
    "algo.fit(trainset)\n",
    "predictions = algo.test(testset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 0.2661\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.2660984221479662"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Then compute RMSE\n",
    "accuracy.rmse(predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using SVDpp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from surprise import SVDpp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SVD Matrix Factorization\n",
    "algo = SVDpp(n_factors=10,n_epochs=10)\n",
    "algo.fit(trainset)\n",
    "predictions_svd = algo.test(testset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 0.2669\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.26688308324524357"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy.rmse(predictions_svd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Knn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "ge9eTEMuHgpk"
   },
   "outputs": [],
   "source": [
    "#lets configure some parameters for Collaborative Filtering Algo\n",
    "sim_options = {\n",
    "    'name': 'pearson', #similarity measure\n",
    "    'user_based': False #item based\n",
    "}\n",
    "#For user based ->True\n",
    "#For name ->pearson,cosine,msd,pearson_baseline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "9Nj19Vz2oHNb"
   },
   "outputs": [],
   "source": [
    "#this is baseline configuration for optimizing the error\n",
    "bsl_options = {'method': 'als',#another option is sgd\n",
    "               'n_epochs': 5, # number of iterations\n",
    "               'reg_u': 12, # user-regularisation parameter\n",
    "               'reg_i': 5   # item-regularisation parameter\n",
    "               }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GGG1vxslh0k_"
   },
   "source": [
    "We will try KNN with maximum neighbour as 3 and minimum as 1.We train our model on trainingset and generate prediction on test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "7McNpEyJIWiV",
    "outputId": "aa0616e1-db67-4ab3-abe4-ac83646045af"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing the pearson similarity matrix...\n",
      "Done computing similarity matrix.\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "knn = KNNBasic(sim_options=sim_options,k=3,min_k=1) #neighbours=3\n",
    "knn.fit(trainset)\n",
    "predictions_knn = knn.test(testset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predicting top 3 reccomendations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    " \n",
    "def get_top3_recommendations(predictions, topN = 3):\n",
    "     \n",
    "    top_recs = defaultdict(list)\n",
    "    for uid, iid, true_r, est, _ in predictions:\n",
    "        top_recs[uid].append((iid, est))\n",
    "     \n",
    "    for uid, user_ratings in top_recs.items():\n",
    "        user_ratings.sort(key = lambda x: x[1], reverse = True)\n",
    "        top_recs[uid] = user_ratings[:topN]\n",
    "     \n",
    "    return top_recs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "top3_recommendations = get_top3_recommendations(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6800277 [117180600, 223960747, 33160735]\n",
      "18402330 [66761764, 46562245, 184214926]\n",
      "10404638 [10234262, 1222411, 138698041]\n",
      "14805984 [98518310, 98410232, 122178507]\n",
      "16157731 [37187868, 89539814, 35971773]\n"
     ]
    }
   ],
   "source": [
    "# For each user we print 3 recommendation\n",
    "i=0;\n",
    "for uid, user_ratings in top3_recommendations.items():\n",
    "    print(uid, [iid for (iid, _) in user_ratings])\n",
    "    i=i+1;\n",
    "    if(i==5):\n",
    "        break;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}