Movie Recommendation System using PySpark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a9e7859c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark.ml.recommendation import ALS\n",
    "from pyspark.ml.evaluation import RegressionEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f0d791ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://DESKTOP-BJS1Q67:4044\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.5.0</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>movie_recommendation</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x2101d3f3fd0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark = SparkSession.builder.appName('movie_recommendation').getOrCreate()\n",
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "081bd1df",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- movieId: integer (nullable = true)\n",
      " |-- rating: double (nullable = true)\n",
      " |-- userId: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = spark.read.csv('movielens_ratings.csv',inferSchema=True,header=True)\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "41dc15a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+------+\n",
      "|movieId|rating|userId|\n",
      "+-------+------+------+\n",
      "|      2|   3.0|     0|\n",
      "|      3|   1.0|     0|\n",
      "|      5|   2.0|     0|\n",
      "|      9|   4.0|     0|\n",
      "|     11|   1.0|     0|\n",
      "+-------+------+------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9229e5cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1501"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cb7cf679",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+------------------+------------------+\n",
      "|summary|           movieId|            rating|            userId|\n",
      "+-------+------------------+------------------+------------------+\n",
      "|  count|              1501|              1501|              1501|\n",
      "|   mean| 49.40572951365756|1.7741505662891406|14.383744170552964|\n",
      "| stddev|28.937034065088994| 1.187276166124803| 8.591040424293272|\n",
      "|    min|                 0|               1.0|                 0|\n",
      "|    max|                99|               5.0|                29|\n",
      "+-------+------------------+------------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b342c7e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = df.randomSplit([0.7,0.3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7b32463c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1077, 424)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.count(), test.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ebb16039",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+------+\n",
      "|movieId|rating|userId|\n",
      "+-------+------+------+\n",
      "|      0|   1.0|     3|\n",
      "|      0|   1.0|     5|\n",
      "|      0|   1.0|     8|\n",
      "|      0|   1.0|    11|\n",
      "|      0|   1.0|    13|\n",
      "+-------+------+------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3ba26d5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+------+\n",
      "|movieId|rating|userId|\n",
      "+-------+------+------+\n",
      "|      0|   1.0|     6|\n",
      "|      0|   1.0|    15|\n",
      "|      0|   1.0|    19|\n",
      "|      0|   1.0|    26|\n",
      "|      1|   1.0|    28|\n",
      "+-------+------+------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "912fd013",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ALS_36bb09a9e678"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "als = ALS(userCol='userId',ratingCol='rating',itemCol='movieId')\n",
    "als"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3ff0d8a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ALSModel: uid=ALS_36bb09a9e678, rank=10"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "als_model = als.fit(train)\n",
    "als_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "acb4f52b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- movieId: integer (nullable = true)\n",
      " |-- rating: double (nullable = true)\n",
      " |-- userId: integer (nullable = true)\n",
      " |-- prediction: float (nullable = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = als_model.transform(test)\n",
    "predictions.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "dfcaae9f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+------+----------+\n",
      "|movieId|rating|userId|prediction|\n",
      "+-------+------+------+----------+\n",
      "|      1|   1.0|    28| 1.6128088|\n",
      "|      2|   4.0|    28| 2.4710596|\n",
      "|      3|   1.0|    28| 0.5800271|\n",
      "|      0|   1.0|    26| 1.6269283|\n",
      "|      4|   4.0|    26| 1.9392356|\n",
      "|      3|   1.0|    13| 1.0048172|\n",
      "|      0|   1.0|     6| 1.2605213|\n",
      "|      2|   1.0|     3| 1.4575107|\n",
      "|      0|   1.0|    19| 0.5118731|\n",
      "|      0|   1.0|    15| 1.0784113|\n",
      "|      4|   1.0|     9|  1.323154|\n",
      "|      3|   2.0|     8| 1.0388323|\n",
      "|      4|   2.0|     8| 1.9129245|\n",
      "|      4|   1.0|    23|0.78435105|\n",
      "|      2|   2.0|     7| 2.4162912|\n",
      "|      4|   1.0|     7| 1.8733845|\n",
      "|      2|   1.0|    25| 1.6303619|\n",
      "|      3|   1.0|    21| 1.2347553|\n",
      "|      3|   3.0|    14| 0.8342831|\n",
      "|      6|   1.0|     2|  1.234749|\n",
      "+-------+------+------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c050d6ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE of ALS: 1.0176051969384272\n"
     ]
    }
   ],
   "source": [
    "evaluator = RegressionEvaluator(predictionCol='prediction',labelCol='rating',metricName='rmse')\n",
    "rmse = evaluator.evaluate(predictions)\n",
    "print(\"RMSE of ALS:\", rmse)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "900973ed",
   "metadata": {},
   "source": [
    "## Making recommendations for a single user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3e034868",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+------+\n",
      "|movieId|userId|rating|\n",
      "+-------+------+------+\n",
      "|      2|    25|   1.0|\n",
      "|      7|    25|   1.0|\n",
      "|     12|    25|   3.0|\n",
      "|     17|    25|   1.0|\n",
      "|     41|    25|   1.0|\n",
      "|     43|    25|   1.0|\n",
      "|     47|    25|   4.0|\n",
      "|     71|    25|   4.0|\n",
      "|     74|    25|   1.0|\n",
      "|     76|    25|   1.0|\n",
      "|     85|    25|   1.0|\n",
      "|     92|    25|   1.0|\n",
      "+-------+------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "single_user = test.filter(test['userId'] == 25).select('movieId','userId','rating')\n",
    "single_user.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7352e8b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+------+----------+\n",
      "|movieId|userId|rating|prediction|\n",
      "+-------+------+------+----------+\n",
      "|     85|    25|   1.0| 1.3198066|\n",
      "|     76|    25|   1.0|  1.635262|\n",
      "|     12|    25|   3.0|  1.626037|\n",
      "|     47|    25|   4.0| 1.4469383|\n",
      "|     92|    25|   1.0| 2.4022336|\n",
      "|     41|    25|   1.0| 0.7240495|\n",
      "|     43|    25|   1.0| 1.4375576|\n",
      "|     17|    25|   1.0| 1.7582796|\n",
      "|      7|    25|   1.0| 0.9474437|\n",
      "|     71|    25|   4.0| 1.5368365|\n",
      "|      2|    25|   1.0| 1.6303619|\n",
      "|     74|    25|   1.0| 1.3975834|\n",
      "+-------+------+------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "recommendations = als_model.transform(single_user)\n",
    "recommendations.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "24307f29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+------+----------+\n",
      "|movieId|userId|rating|prediction|\n",
      "+-------+------+------+----------+\n",
      "|     92|    25|   1.0| 2.4022336|\n",
      "|     17|    25|   1.0| 1.7582796|\n",
      "|     76|    25|   1.0|  1.635262|\n",
      "|      2|    25|   1.0| 1.6303619|\n",
      "|     12|    25|   3.0|  1.626037|\n",
      "|     71|    25|   4.0| 1.5368365|\n",
      "|     47|    25|   4.0| 1.4469383|\n",
      "|     43|    25|   1.0| 1.4375576|\n",
      "|     74|    25|   1.0| 1.3975834|\n",
      "|     85|    25|   1.0| 1.3198066|\n",
      "|      7|    25|   1.0| 0.9474437|\n",
      "|     41|    25|   1.0| 0.7240495|\n",
      "+-------+------+------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "recommendations.orderBy('prediction',ascending=False).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d78c46a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE of movie rating prediction: 1.2364583181617885\n"
     ]
    }
   ],
   "source": [
    "print(\"RMSE of movie rating prediction:\", evaluator.evaluate(recommendations))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}