Airbnb_Models_Collabarative_Filtering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TMlKEFYBAso8"
   },
   "source": [
    "## Collaborative Filtering Models for AirBnB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mtTdeixOAz_h"
   },
   "source": [
    "#### LIBRARIES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 126
    },
    "colab_type": "code",
    "id": "pYio5E51-E81",
    "outputId": "a5403558-06ac-4199-aa2b-77ba2cbc417b"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "from pyspark.ml.recommendation import ALS\n",
    "from pyspark.sql import Row\n",
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SQLContext\n",
    "from pyspark.sql import SparkSession\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "k6Rjba5fA62d"
   },
   "source": [
    "#### Reading a file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_55k4dS-42eR"
   },
   "outputs": [],
   "source": [
    "#I have used music track dataset.\n",
    "df=pd.read_csv('shared_final_prediction_file.csv', sep=',')# read csv into df\n",
    "df = df.drop('rowid', axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>listing_id</th>\n",
       "      <th>reviewer_id</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2539</td>\n",
       "      <td>25160947</td>\n",
       "      <td>0.6249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2539</td>\n",
       "      <td>91513326</td>\n",
       "      <td>0.9228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2539</td>\n",
       "      <td>90022459</td>\n",
       "      <td>0.4754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2539</td>\n",
       "      <td>116165195</td>\n",
       "      <td>0.9231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2539</td>\n",
       "      <td>1806142</td>\n",
       "      <td>0.9881</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   listing_id  reviewer_id  polarity\n",
       "0        2539     25160947    0.6249\n",
       "1        2539     91513326    0.9228\n",
       "2        2539     90022459    0.4754\n",
       "3        2539    116165195    0.9231\n",
       "4        2539      1806142    0.9881"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sc = SparkContext()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "sqlContext = SQLContext(sc)\n",
    "\n",
    "\n",
    "spark = SparkSession.builder \\\n",
    ".master('local') \\\n",
    ".appName('Data cleaning') \\\n",
    ".getOrCreate()\n",
    "\n",
    "df = spark.read.format('csv').option('header','true').option('mode','DROPMALFORMED').load('shared_final_prediction_file.csv')\n",
    "\n",
    "changedTypedf = df.withColumn(\"listing_id\", df[\"listing_id\"].cast(\"long\"))\n",
    "changedTypedf2 = changedTypedf.withColumn(\"reviewer_id\", changedTypedf[\"reviewer_id\"].cast(\"long\"))\n",
    "changedTypedf3 = changedTypedf2.withColumn(\"polarity\", changedTypedf2[\"polarity\"].cast(\"float\"))\n",
    "\n",
    "splitDF1, splitDF2 = changedTypedf3.randomSplit([0.7, 0.3])\n",
    "\n",
    "# Build the recommendation model using ALS on the training data\n",
    "# Note we set cold start strategy to 'drop' to ensure we don't get NaN evaluation metrics\n",
    "als = ALS(maxIter=10,  regParam=0.3, rank=10,nonnegative=False, userCol=\"reviewer_id\", itemCol=\"listing_id\", ratingCol=\"polarity\")\n",
    "#als = ALS(maxIter=10,  regParam=0.15, rank=5,  userCol=\"user_id\", itemCol=\"book_id\", ratingCol=\"rating\")\n",
    "model = als.fit(splitDF1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate the model by computing the RMSE on the test data\n",
    "predictions = model.transform(splitDF2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "changedPreddf = predictions.withColumn(\"polarity\", predictions[\"polarity\"].cast(\"double\"))\n",
    "changedPreddf2 = changedPreddf.withColumn(\"prediction\", changedPreddf[\"prediction\"].cast(\"double\"))\n",
    "changedPreddf2 = changedPreddf2.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "pandas_final_df = changedPreddf2.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rowid</th>\n",
       "      <th>listing_id</th>\n",
       "      <th>reviewer_id</th>\n",
       "      <th>polarity</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1485</td>\n",
       "      <td>5803</td>\n",
       "      <td>4213410</td>\n",
       "      <td>0.9690</td>\n",
       "      <td>0.036598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1421</td>\n",
       "      <td>5803</td>\n",
       "      <td>281328</td>\n",
       "      <td>0.9712</td>\n",
       "      <td>-0.068157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1435</td>\n",
       "      <td>5803</td>\n",
       "      <td>2727672</td>\n",
       "      <td>0.9805</td>\n",
       "      <td>0.101855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1480</td>\n",
       "      <td>5803</td>\n",
       "      <td>33059289</td>\n",
       "      <td>0.1010</td>\n",
       "      <td>-0.030405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1497</td>\n",
       "      <td>5803</td>\n",
       "      <td>56281844</td>\n",
       "      <td>0.9533</td>\n",
       "      <td>0.032358</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  rowid  listing_id  reviewer_id  polarity  prediction\n",
       "0  1485        5803      4213410    0.9690    0.036598\n",
       "1  1421        5803       281328    0.9712   -0.068157\n",
       "2  1435        5803      2727672    0.9805    0.101855\n",
       "3  1480        5803     33059289    0.1010   -0.030405\n",
       "4  1497        5803     56281844    0.9533    0.032358"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pandas_final_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "pandas_final_df = pandas_final_df.drop('rowid', axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>listing_id</th>\n",
       "      <th>reviewer_id</th>\n",
       "      <th>polarity</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5803</td>\n",
       "      <td>4213410</td>\n",
       "      <td>0.9690</td>\n",
       "      <td>0.036598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5803</td>\n",
       "      <td>281328</td>\n",
       "      <td>0.9712</td>\n",
       "      <td>-0.068157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5803</td>\n",
       "      <td>2727672</td>\n",
       "      <td>0.9805</td>\n",
       "      <td>0.101855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5803</td>\n",
       "      <td>33059289</td>\n",
       "      <td>0.1010</td>\n",
       "      <td>-0.030405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5803</td>\n",
       "      <td>56281844</td>\n",
       "      <td>0.9533</td>\n",
       "      <td>0.032358</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   listing_id  reviewer_id  polarity  prediction\n",
       "0        5803      4213410    0.9690    0.036598\n",
       "1        5803       281328    0.9712   -0.068157\n",
       "2        5803      2727672    0.9805    0.101855\n",
       "3        5803     33059289    0.1010   -0.030405\n",
       "4        5803     56281844    0.9533    0.032358"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pandas_final_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Root-mean-square error = 0.8535667149338011\n"
     ]
    }
   ],
   "source": [
    "evaluator = RegressionEvaluator(metricName=\"rmse\", labelCol=\"polarity\",\n",
    "                                predictionCol=\"prediction\")\n",
    "rmse = evaluator.evaluate(changedPreddf2)\n",
    "print(\"Root-mean-square error = \" + str(rmse))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oSdj6mmHmxCu"
   },
   "source": [
    "Now we will pull top three recommendation for each user;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fRhg2xYWH4I5"
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    " \n",
    "def get_top10_recommendations(predictions, topN = 10):\n",
    "     \n",
    "    top_recs = defaultdict(list)\n",
    "    print (top_recs)\n",
    "    for uid, iid, true_r, est, _ in predictions:\n",
    "        top_recs[uid].append((iid, est))\n",
    "     \n",
    "    for uid, user_ratings in top_recs.items():\n",
    "        user_ratings.sort(key = lambda x: x[1], reverse = True)\n",
    "        top_recs[uid] = user_ratings[:topN]\n",
    "     \n",
    "    return top_recs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "O9eyZyzYBORP"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'list'>, {})\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "too many values to unpack (expected 5)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-34-f0f60d948973>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtop10_recommendations\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_top10_recommendations\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpandas_final_df\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-33-c0347c846024>\u001b[0m in \u001b[0;36mget_top10_recommendations\u001b[0;34m(predictions, topN)\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mtop_recs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdefaultdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtop_recs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0muid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrue_r\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpredictions\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m         \u001b[0mtop_recs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: too many values to unpack (expected 5)"
     ]
    }
   ],
   "source": [
    "top10_recommendations = get_top10_recommendations(pandas_final_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>listing_id</th>\n",
       "      <th>reviewer_id</th>\n",
       "      <th>polarity</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5803</td>\n",
       "      <td>4213410</td>\n",
       "      <td>0.9690</td>\n",
       "      <td>0.036598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5803</td>\n",
       "      <td>281328</td>\n",
       "      <td>0.9712</td>\n",
       "      <td>-0.068157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5803</td>\n",
       "      <td>2727672</td>\n",
       "      <td>0.9805</td>\n",
       "      <td>0.101855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5803</td>\n",
       "      <td>33059289</td>\n",
       "      <td>0.1010</td>\n",
       "      <td>-0.030405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5803</td>\n",
       "      <td>56281844</td>\n",
       "      <td>0.9533</td>\n",
       "      <td>0.032358</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   listing_id  reviewer_id  polarity  prediction\n",
       "0        5803      4213410    0.9690    0.036598\n",
       "1        5803       281328    0.9712   -0.068157\n",
       "2        5803      2727672    0.9805    0.101855\n",
       "3        5803     33059289    0.1010   -0.030405\n",
       "4        5803     56281844    0.9533    0.032358"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pandas_final_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 108
    },
    "colab_type": "code",
    "id": "ihQWtfQacFNb",
    "outputId": "b919b6ba-c3e5-4e34-c8de-7138146d60b8"
   },
   "outputs": [],
   "source": [
    "# For each user we print 10 recommendation\n",
    "i=0;\n",
    "for uid, user_ratings in top3_recommendations.items():\n",
    "    print(uid, [iid for (iid, _) in user_ratings])\n",
    "    i=i+1;\n",
    "    if(i==10):\n",
    "        break;"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TyxQxxLpm_Sh"
   },
   "source": [
    "### Tips\n",
    "\n",
    "1.Surprise dataset function just takes three columns,user-item and ratings so be careful.\n",
    "\n",
    "2.Building Antitest set gives you all the unknown user-item ratings,you may not require all of them.\n",
    "\n",
    "3.Explore more and have fun!\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "256-DEMO.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}