Code_Text_Matching.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Code_a1785760.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "78ckP2eE0yCg",
        "Bmdafn_x0yCq",
        "3_NjRMaeI8TQ",
        "sAsV-lU7-FPe",
        "ttzqHn-H0yC6"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YA3ViTrq0yCS"
      },
      "source": [
        "<h2 align=center> Building a Text Matching System for Question Matching </h2>\n",
        "\n",
        "<h4 align=center> APPLIED NATURAL LANGUAGE PROCESSING</h4>\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "melYlqyuVmNZ"
      },
      "source": [
        "### Note: After running the whole notebook,\n",
        "#### 1. For TFIDF Model, to get top matches (2 or 5), use code:\n",
        "- top5 = get_similarity_tfidf(\"sentence\",5) \n",
        "- print_top_k(top5)\n",
        "\n",
        "#### 2. For Average word Model, use code:\n",
        "- top5 = get_similarity_avg_word(\"sentence\" , 5 )\n",
        "- print_top_k(top5)\n",
        "\n",
        "#### 3. For Smooth Inverse Frequency  Model, use code:\n",
        "- top5 = get_similarity_SIF(\"sentence\" , 5 )\n",
        "- print_top_k(top5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "78ckP2eE0yCg"
      },
      "source": [
        "##  Importing Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wJh67gho0yCh"
      },
      "source": [
        "# imports\n",
        "import numpy as np \n",
        "import pandas as pd\n",
        "import sklearn\n",
        "import operator\n",
        "import sys\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import nltk\n",
        "from nltk import stem \n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.corpus import wordnet\n",
        "\n",
        "import random\n",
        "import re\n",
        "import math\n",
        "from math import sqrt\n",
        "from math import log\n",
        "from math import log10\n",
        "from statistics import mean\n",
        "from scipy.sparse import csr_matrix\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "\n",
        "import collections\n",
        "from collections import Counter\n",
        "import itertools\n",
        "from itertools import islice\n",
        "from itertools import chain\n",
        "import pickle\n",
        "\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RWKTLa3r-Ncy",
        "outputId": "43db85af-fc50-45e4-cd41-3830ea7d02b3"
      },
      "source": [
        "# nltk downloads\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('averaged_perceptron_tagger')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bmdafn_x0yCq"
      },
      "source": [
        "## Load and Explore the Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7bKKdAlv0yCq",
        "outputId": "baf63e0b-734d-43bb-a78d-ddb5947a4f87"
      },
      "source": [
        "# data loading\n",
        "df = pd.read_csv('data.tsv', sep='\\t',error_bad_lines=False)\n",
        "df.head(10) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "b'Skipping line 83032: expected 6 fields, saw 7\\n'\n",
            "b'Skipping line 154657: expected 6 fields, saw 7\\n'\n",
            "b'Skipping line 323916: expected 6 fields, saw 7\\n'\n",
            "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (1) have mixed types.Specify dtype option on import or set low_memory=False.\n",
            "  interactivity=interactivity, compiler=compiler, result=result)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>qid1</th>\n",
              "      <th>qid2</th>\n",
              "      <th>question1</th>\n",
              "      <th>question2</th>\n",
              "      <th>is_duplicate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>133273</td>\n",
              "      <td>213221</td>\n",
              "      <td>213222.0</td>\n",
              "      <td>How is the life of a math student? Could you d...</td>\n",
              "      <td>Which level of prepration is enough for the ex...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>402555</td>\n",
              "      <td>536040</td>\n",
              "      <td>536041.0</td>\n",
              "      <td>How do I control my horny emotions?</td>\n",
              "      <td>How do you control your horniness?</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>360472</td>\n",
              "      <td>364011</td>\n",
              "      <td>490273.0</td>\n",
              "      <td>What causes stool color to change to yellow?</td>\n",
              "      <td>What can cause stool to come out as little balls?</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>150662</td>\n",
              "      <td>155721</td>\n",
              "      <td>7256.0</td>\n",
              "      <td>What can one do after MBBS?</td>\n",
              "      <td>What do i do after my MBBS ?</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>183004</td>\n",
              "      <td>279958</td>\n",
              "      <td>279959.0</td>\n",
              "      <td>Where can I find a power outlet for my laptop ...</td>\n",
              "      <td>Would a second airport in Sydney, Australia be...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>119056</td>\n",
              "      <td>193387</td>\n",
              "      <td>193388.0</td>\n",
              "      <td>How not to feel guilty since I am Muslim and I...</td>\n",
              "      <td>I don't beleive I am bulimic, but I force thro...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>356863</td>\n",
              "      <td>422862</td>\n",
              "      <td>96457.0</td>\n",
              "      <td>How is air traffic controlled?</td>\n",
              "      <td>How do you become an air traffic controller?</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>106969</td>\n",
              "      <td>147570</td>\n",
              "      <td>787.0</td>\n",
              "      <td>What is the best self help book you have read?...</td>\n",
              "      <td>What are the top self help books I should read?</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>196763</td>\n",
              "      <td>297539</td>\n",
              "      <td>297540.0</td>\n",
              "      <td>Can I enter University of Melbourne if I could...</td>\n",
              "      <td>University of the Philippines: If I take a sec...</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>256389</td>\n",
              "      <td>37932</td>\n",
              "      <td>371478.0</td>\n",
              "      <td>Do you need a passport to go to Jamaica from t...</td>\n",
              "      <td>How can I move to Jamaica?</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       id  ... is_duplicate\n",
              "0  133273  ...          0.0\n",
              "1  402555  ...          1.0\n",
              "2  360472  ...          0.0\n",
              "3  150662  ...          1.0\n",
              "4  183004  ...          0.0\n",
              "5  119056  ...          0.0\n",
              "6  356863  ...          0.0\n",
              "7  106969  ...          1.0\n",
              "8  196763  ...          0.0\n",
              "9  256389  ...          0.0\n",
              "\n",
              "[10 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RZxPosyB0yCr",
        "outputId": "575dc7aa-19c8-40ca-dd96-c2703cef2815"
      },
      "source": [
        "df.info()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 363192 entries, 0 to 363191\n",
            "Data columns (total 6 columns):\n",
            " #   Column        Non-Null Count   Dtype  \n",
            "---  ------        --------------   -----  \n",
            " 0   id            363192 non-null  object \n",
            " 1   qid1          363192 non-null  object \n",
            " 2   qid2          363185 non-null  float64\n",
            " 3   question1     363181 non-null  object \n",
            " 4   question2     363180 non-null  object \n",
            " 5   is_duplicate  363180 non-null  float64\n",
            "dtypes: float64(2), object(4)\n",
            "memory usage: 16.6+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T1YgE1c89j0d"
      },
      "source": [
        "# change data types\n",
        "df[\"id\"] = pd.to_numeric(df[\"id\"],errors='coerce')\n",
        "df[\"qid1\"] = pd.to_numeric(df[\"qid1\"],errors='coerce') "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JsRCP5Tl9j0e",
        "outputId": "51132f92-ed59-464b-cb92-604d2cdc1357"
      },
      "source": [
        "df.info()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 363192 entries, 0 to 363191\n",
            "Data columns (total 6 columns):\n",
            " #   Column        Non-Null Count   Dtype  \n",
            "---  ------        --------------   -----  \n",
            " 0   id            363182 non-null  float64\n",
            " 1   qid1          363189 non-null  float64\n",
            " 2   qid2          363185 non-null  float64\n",
            " 3   question1     363181 non-null  object \n",
            " 4   question2     363180 non-null  object \n",
            " 5   is_duplicate  363180 non-null  float64\n",
            "dtypes: float64(4), object(2)\n",
            "memory usage: 16.6+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_pDw5Nu80yCr",
        "outputId": "1777961e-7e99-4e71-8daf-bc05199e993b"
      },
      "source": [
        "# check unique values\n",
        "df.nunique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "id              363182\n",
              "qid1            266111\n",
              "qid2            273126\n",
              "question1       265929\n",
              "question2       272961\n",
              "is_duplicate         2\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Co7BgU9Y0yCr",
        "outputId": "b09edcf0-4438-4f4c-931a-1b5e5776c57a"
      },
      "source": [
        "df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(363192, 6)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pXozyoFI9j0f",
        "outputId": "e9763da3-9b1d-4718-e9b0-c4225cd4f2c7"
      },
      "source": [
        "# Check for null values\n",
        "\n",
        "null_tr = df.isnull().sum()\n",
        "print(\"Null values in set: \")\n",
        "print(null_tr)\n",
        "print()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Null values in set: \n",
            "id              10\n",
            "qid1             3\n",
            "qid2             7\n",
            "question1       11\n",
            "question2       12\n",
            "is_duplicate    12\n",
            "dtype: int64\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X-TDcoal0yCs",
        "outputId": "8a721dd7-2b8f-45dd-a1cf-e2a599211805"
      },
      "source": [
        "# drop null values\n",
        "df = df.dropna()\n",
        "df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(363177, 6)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zBWwrQ489j0g",
        "outputId": "62b5f836-119c-4e08-bc38-2884d3c1325e"
      },
      "source": [
        "duplicateRowsQ1 = df[df.duplicated([ 'question1'])]\n",
        "print(duplicateRowsQ1.shape)\n",
        "\n",
        "duplicateRowsQ2 = df[df.duplicated([ 'question2'])]\n",
        "print(duplicateRowsQ2.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(97251, 6)\n",
            "(90218, 6)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GeKeWH9H9j0g",
        "outputId": "d5ed63b2-6a62-4208-e2ca-aa614e52eb4b"
      },
      "source": [
        "# drop rows with duplicated questions\n",
        "\n",
        "df = df.drop_duplicates(subset = ['question1'], keep = 'last').reset_index(drop = True)\n",
        "print(\"Shape of dataframe after dropping duplicated rows in question 1: \", df.shape)\n",
        "\n",
        "df = df.drop_duplicates(subset = ['question2'], keep = 'last').reset_index(drop = True)\n",
        "print(\"Shape after dropping duplicated rows in question 1 and 2 both : \", df.shape) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of dataframe after dropping duplicated rows in question 1:  (265926, 6)\n",
            "Shape after dropping duplicated rows in question 1 and 2 both :  (232203, 6)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FSKKEx0LCQrl",
        "outputId": "1956fb02-b06a-4aab-9a15-b9ae33a84ca4"
      },
      "source": [
        "# distribution of label\n",
        "df['is_duplicate'].value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0    169095\n",
              "1.0     63108\n",
              "Name: is_duplicate, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qfi_-XwS9j0h",
        "outputId": "9ce6bcb1-4a47-425d-9e95-ba31d3e0b6d9"
      },
      "source": [
        "# Check for null values\n",
        "null_tr = df.isnull().sum()\n",
        "print(\"Null values in set: \")\n",
        "print(null_tr)\n",
        "print()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Null values in set: \n",
            "id              0\n",
            "qid1            0\n",
            "qid2            0\n",
            "question1       0\n",
            "question2       0\n",
            "is_duplicate    0\n",
            "dtype: int64\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y_yYf3Ni0yCt",
        "outputId": "bca0ae0e-a414-4a23-b9a8-6260a72b8714"
      },
      "source": [
        "# plot distribution of label\n",
        "df['is_duplicate'].hist(edgecolor = 'black',bins=5,facecolor = 'teal')\n",
        "plt.suptitle(\"Histogram for Labels\",size = 14)\n",
        "plt.xlabel(\"Label\")\n",
        "plt.ylabel(\"Number of reviews\")\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEjCAYAAAD+PUxuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5hcVZnv8e8vCVE0QLiMLZMCEjU6E5lRocXMwUsLCgGVMAg2zCiByRCPIIMiysULjMgMHtKiKKAZySE4SoDIkYhhMhEoGOeQkHAxEBDShks6XIIkBBokJOSdP/ZqLJrq7kr1rqpU9+/zPPX03u9ee++1qi9v77VW7a2IwMzMLE8jGl0BMzMbepxczMwsd04uZmaWOycXMzPLnZOLmZnlzsnFzMxy5+Ri2yRJ4yWFpNZG12VbIGmGpEclbZF0TqPrU0rScZK6czhOUdIP8qiTNZ6Ti9WVpMslXV8m3pqSyfgUWg3sDtxd4XEflnRabhXdhkjaGbgYuAAYB8zM+fhlvydmgzGq0RUwKyciXgaeaHQ9ypE0OiJequMp9yL7Xb0+Ih6v9iANqLcNY75ysW1S724xSdtJukjSY5I2Slot6fy0rUj2B/iCtE+UHOcISfeU7PNVSSrZ3iJpvqQ/SnpE0vGS7i3tekrHPEnStZKeB/5F0khJl0l6KO27UtJXJI0o2e9ySddLOl3SE5I2SDpf0ghJ50ham+Kn9/M+HAfclVZXlV7dSfqspE5JL6WvJ/Ta9zX1rvJ7caqk5ZKel7RG0o8ljS1T7hOSHpT0oqSbJb2lzPY70vaHJJ0naXQ/5z0infePktZJukVSSzVtsPrzlYs1i38C/hY4GngYKADvSNuOAH4LzAYu7dlB0r7ANcC3gJ8C7wV+BDwLfD8Vm0PW/XYA8EeggyxR9XY2cBZwGhBk/5itAT4FPAXsB8wCngYuK9nvg0AX0Aa8J9Xj3WQJ4/3pvJdK+nVE3FHmvFcBjwP/kc6xGnhK0t8CPwC+CPwncDBwiaQnIuKX/dS7GluALwCryN6b76fXZ0rKvC6d63jgBeB7wLWS3hMRIeng1PZTgFuBPYEfpv1e050p6c3AXOBM4OfAGGBylfW3RogIv/yq2wu4HNgMdPd6vUD2x298Kjc+rbem9YuAGwH1cdyHgdN6xX4K3NQrdg7QlZbfkc4xuWT7HsDLwDklsQC+X0Hbzgd+3autq4GRJbFlwG8Hqnuv7a2l702K/Tcwu8x7+5sq6n05WZdbpd/DKcBGYERaPy6da/+SMnul9/Ejaf1W4Ou9jnN4+t4rrReBH6TlfdIx92r0z6xf1b3cLWaNcCvZf++lr78bYJ/LU7kHJV0s6WOlXVB9+EuyP8KlfgOMk7Qj8Bdk/5Uv69kYEauBx8oca1nvgKT/LWmZpKfSbKkvkv1HXuq+yMaPejwJ3NurzJPAmwZoS299tW3SQPXeWpIOkLRIUpek54BrgdHAm0uKbQFu71mJiEfI3see+uwLfFVSd88L+Bnwxl7H6fFb4NfAvZJ+Lulzkv5ssG2x+nFysUZ4ISI6S19kXUd9iog7ya5mziT7uZ0DLKogwfR5yK0s/3zpiqR24LtkSe9gssR3Cdkf3VKbypy3XCyv38Xe7Xq+bKkKSdoL+BVwP3AUWZL4h7S5d1v7e09HAP/Mq/+h+GtgIlm34qsPlCXkg9JrOTAdWCnpXdW2xerLycWaRkQ8FxHzIuJzwMfIxivelja/BIzstcv9wP69Yu8n6xZ7Dvgd2e/Avj0bJRWAP6+gOu8HlkTEDyLizpQg37q1bRqEvtp2X87naSVLIl+MiNsi4kHKvz8jyMaEAJC0Zyp3fwrdCfxF738q0mtzuRNH5raI+Gey8bLHgPb8mma15AF9awqSTiUb2L6b7D//vyMbmO+54nkY+ICkfwc2RsQfyAbnl6aZXz8j+wP1JbIBbiLiAUkLgR9K+hzwItlnSXrGf/rzIHCcpEOATrKJBh8C1ufR3gpcAFwj6Q6yAf0pwN+TTW6oxo6S3t0r9gywkixxfEHStWSD6l8os/9m4LuSTiGbGHEhsIKsawvgm8D1kh4Brk7l9wb2i4iv9D6YpMnAR4CFZN2G7yEbD8s7eVqN+MrFmsVzwJfJ+vXvJOtWOSQiXkjbv0H2x+f3pG6W1JV2FPBJsnGO89Or9FPgx5ElqCIwn2wSwFqyRNOfH5H9kfwZsJSsy66j6tZtpYj4BXAy2TjPfWSzsE6MV88U2xofIJvBVvqaGRHL07FPTef5R8rM7iIb4D8PuAJYQva35YiIbHQ+IhaSXW1+mOx7eDtwBvBoH/XZQHZldj1ZgusAzo2If6+yfVZnPbM0zAyQtBtZ98sxEfHzRtfHrFm5W8yGNUkHADsA95DN2DoP+APZ50rMrEpOLjbcbUf2Icu3kI21LAY+GBGDmmVlNty5W8zMzHLnAX0zM8udk4uZmeXOycXMzHLn5GJmZrlzcjEzs9w5uZiZWe6cXMzMLHdOLmZmljsnFzMzy52Ti5mZ5c7JxczMcufkYmZmuXNyMTOz3PmW+8luu+0W48ePr2rf559/nje+8Y35Vmgb5zYPD27z8DCYNt9xxx1/iIg/6x13cknGjx/PsmXLqtq3WCzS1taWb4W2cW7z8OA2Dw+DabOkR8rF3S1mZma5c3IxM7PcObmYmVnunFzMzCx3Ti5mZpY7JxczM8udk4uZmeXOycXMzHLn5GJmZrlzcsnB8uXLkTSsXsuXL2/0225m2zDf/iUHmzZtgnPOaXQ16mrTpk2NroKZbcN85WJmZrlzcjEzs9w5uZiZWe6cXMzMLHdOLmZmlruaJRdJsyWtlXRvr/jJkn4naYWk/1MSP1NSp6QHJB1cEp+SYp2SziiJT5C0JMWvkjQ6xV+X1jvT9vG1aqOZmZVXyyuXy4EppQFJHwamAu+KiHcCM1N8EnA08M60zyWSRkoaCVwMHAJMAo5JZQG+DVwYEW8D1gPTU3w6sD7FL0zlzMysjmqWXCLiVmBdr/DngPMjYmMqszbFpwJzI2JjRDwEdAL7pVdnRKyKiJeAucBUSQIOAOal/ecAh5cca05angccmMqbmVmd1HvM5e3AB1J31S2S3pvi44DVJeW6Uqyv+K7AMxGxuVf8VcdK2zek8mZmVif1/oT+KGAXYDLwXuBqSW+pcx1eIWkGMAOgpaWFYrFY1XEKhQIzt2zJsWbbvsKIEVW/X82qu7vbbR4G3OZ81Du5dAHXRkQAt0vaAuwGrAH2KClXSDH6iD8NjJU0Kl2dlJbvOVaXpFHATqn8a0TELGAWQGtra7S1tVXVqI6ODk7r7q5q32Y1c8wY2tvbG12NuioWi1T7M9Ks3ObhoRZtrne32C+ADwNIejswGvgDMB84Os30mgBMBG4HlgIT08yw0WSD/vNTcroZODIddxpwXVqen9ZJ229K5c3MrE5qduUi6UqgDdhNUhdwNjAbmJ2mJ78ETEt/+FdIuhq4D9gMnBQRL6fjfB5YCIwEZkfEinSK04G5kr4F3AVcluKXAT+R1Ek2oeDoWrXRzMzKq1lyiYhj+tj06T7KnwecVya+AFhQJr6KbDZZ7/iLwFFbVVkzM8uVP6FvZma5c3IxM7PcObmYmVnunFzMzCx3Ti5mZpY7JxczM8udk4uZmeXOycXMzHLn5GJmZrlzcjEzs9w5uZiZWe6cXMzMLHdOLmZmljsnFzMzy52Ti5mZ5c7JxczMclez5CJptqS16amTvbd9SVJI2i2tS9JFkjolLZe0T0nZaZJWpte0kvi+ku5J+1wkSSm+i6RFqfwiSTvXqo1mZlZeLa9cLgem9A5K2gM4CHi0JHwIMDG9ZgCXprK7kD0e+X1kT508uyRZXAqcULJfz7nOAG6MiInAjWndzMzqqGbJJSJuJXuGfW8XAl8BoiQ2FbgiMouBsZJ2Bw4GFkXEuohYDywCpqRtO0bE4ogI4Arg8JJjzUnLc0riZmZWJ6PqeTJJU4E1EfHb1IvVYxywumS9K8X6i3eViQO0RMTjafkJoKWf+swgu1KipaWFYrG4lS3KFAoFZm7ZUtW+zaowYkTV71ez6u7udpuHAbc5H3VLLpLeAJxF1iVWFxERkqKf7bOAWQCtra3R1tZW1Xk6Ojo4rbu7qn2b1cwxY2hvb290NeqqWCxS7c9Is3Kbh4datLmes8XeCkwAfivpYaAA3CnpzcAaYI+SsoUU6y9eKBMHeDJ1m5G+rs29JWZm1q+6JZeIuCci3hQR4yNiPFlX1j4R8QQwHzg2zRqbDGxIXVsLgYMk7ZwG8g8CFqZtz0qanGaJHQtcl041H+iZVTatJG5mZnVSy6nIVwK3Ae+Q1CVpej/FFwCrgE7g34ATASJiHXAusDS9vplipDI/Tvv8Hrghxc8HPippJfCRtG5mZnVUszGXiDhmgO3jS5YDOKmPcrOB2WXiy4C9y8SfBg7cyuqamVmO/Al9MzPLnZOLmZnlzsnFzMxy5+RiZma5c3IxM7PcObmYmVnunFzMzCx3Ti5mZpY7JxczM8udk4uZmeXOycXMzHLn5GJmZrkbMLlI2l/SG9PypyV9R9Jeta+amZk1q0quXC4FXpD0LuBLZLe3v6KmtTIzs6ZWSXLZnG6JPxX4QURcDOxQ22qZmVkzqyS5PCfpTODTwK8kjQC2G2gnSbMlrZV0b0nsAkm/k7Rc0v+TNLZk25mSOiU9IOngkviUFOuUdEZJfIKkJSl+laTRKf66tN6Zto+v5I0wM7P8VJJc2oGNwPT0SOICcEEF+10OTOkVWwTsHRF/DTwInAkgaRJwNPDOtM8lkkZKGglcDBwCTAKOSWUBvg1cGBFvA9YDPU+6nA6sT/ELUzkzM6ujSpLLx4BfRsR/AUTEoxEx4JhLRNwKrOsV+8+I2JxWF5MlKsi63OZGxMaIeIjs0cX7pVdnRKyKiJeAucBUSQIOAOal/ecAh5cca05angccmMqbmVmdVJJc9gR+JGmVpGsknZwG9wfrH/jTc+/HAatLtnWlWF/xXYFnShJVT/xVx0rbN6TyZmZWJ6MGKhARZwNI2h44Afgy8F1gZLUnlfRVYDPw02qPkQdJM4AZAC0tLRSLxaqOUygUmLllS4412/YVRoyo+v1qVt3d3W7zMOA252PA5CLpa8D+wBjgLuA04L+qPaGk44CPAwemWWgAa4A9SooVUow+4k8DYyWNSlcnpeV7jtUlaRSwUyr/GhExC5gF0NraGm1tbVW1qaOjg9O6u6vat1nNHDOG9vb2RlejrorFItX+jDQrt3l4qEWbK+kWO4KsW+nXwLXAdRHxeDUnkzQF+ApwWES8ULJpPnB0muk1AZgI3A4sBSammWGjyQb956ekdDNwZNp/GnBdybGmpeUjgZtKkpiZmdXBgMklIvYBPkL2x/6jwD2SfjPQfpKuBG4D3iGpS9J04Adkn5FZJOluST9M51gBXA3cB/wHcFJEvJyuSj4PLATuB65OZQFOB06V1EmW/C5L8cuAXVP8VOCV6ctmZlYflXSL7Q18APgQ0Eo2WD5gt1hEHFMmfFmZWE/584DzysQXAAvKxFeRzSbrHX8ROGqg+pmZWe0MmFyA88mSyUXA0ojYVNsqmZlZs6tkttjH00yxPZ1YzMysEpXcFfkTwN1kYyFIerek+bWumJmZNa9KZoudQza28QxARNwNTKhhnczMrMlVklw2RcSGXjFP7TUzsz5VMqC/QtLfASMlTQT+Cfj/ta2WmZk1s0quXE4mu1vxRuBK4FngC7WslJmZNbdKZou9AHw1vczMzAbUZ3KR9N2I+IKkX1JmjCUiDqtpzczMrGn1d+Xyk/R1Zj0qYmZmQ0efySUi7kiLuwK/ioiN9amSmZk1u0oG9D8BPCjpJ5I+nm5jb2Zm1qdK7op8PPA24BrgGOD3kn5c64qZmVnzqugqJCI2SbqBbGB/e7Ln1f9jLStmZmbNq5J7ix0i6XJgJfBJ4MfAm2tcLzMza2KVXLkcC1wFfNaD+mZmVolKxlyOAe4ie2AYkraXtMNA+0maLWmtpHtLYrtIWiRpZfq6c4pL0kWSOiUtl7RPyT7TUvmVkqaVxPeVdE/a5yJJ6u8cZmZWP5V0i50AzAN+lEIF4BcVHPtyYEqv2BnAjRExEbiRPz2C+BBgYnrNAC5N594FOBt4H9mdmc8uSRaXAieU7DdlgHOYmVmdVDIV+SRgf7J7ihERK4E3DbRTRNwKrOsVngrMSctzyCYG9MSviMxiYKyk3YGDgUURsS4i1gOLgClp244RsTgiArii17HKncPMzOqkkjGXjRHxUup1In3Opdpb7rdExONp+QmgJS2PA1aXlOtKsf7iXWXi/Z3jNSTNILtSoqWlhWKxuJXNyRQKBWZu2VLVvs2qMGJE1e9Xs+ru7nabhwG3OR+VJJdbJJ0FbC/po8CJwC8He+KICEk1fS7MQOeIiFnALIDW1tZoa2ur6jwdHR2c1t1d1b7NauaYMbS3tze6GnVVLBap9mekWbnNw0Mt2lxJt9jpwFPAPcBngQXA16o835OpS4v0dW2KrwH2KClXSLH+4oUy8f7OYWZmddJvcpE0Erg/Iv4tIo6KiCPTcrVXHPOBnhlf04DrSuLHplljk4ENqWtrIXCQpJ3TQP5BwMK07VlJk9MssWN7HavcOczMrE767RaLiJclPSBpz4h4dGsOLOlKoA3YTVIX2ayv84GrJU0HHgE+lYovAA4FOoEXgOPT+ddJOhdYmsp9MyJ6JgmcSDYjbXvghvSin3OYmVmdVDLmsjPZo45vB57vCQ70PJf0+ZhyDixTNshmpZU7zmxgdpn4MmDvMvGny53DzMzqp5Lk8vWa18LMzIaUSh5zfEs9KmJmZkNHJbPFzMzMtoqTi5mZ5a7P5CLpxvT12/WrjpmZDQX9jbnsLul/AYdJmguodGNE3FnTmpmZWdPqL7l8g2ymWAH4Tq9tARxQq0qZmVlz6zO5RMQ8YJ6kr0fEuXWsk5mZNblKpiKfK+kw4IMpVIyI62tbLTMza2aVPCzsX4FTgPvS6xRJ/1LripmZWfOq5BP6HwPeHRFbACTNIXvs8Vm1rJiZmTWvSj/nMrZkeadaVMTMzIaOSq5c/hW4S9LNZNORP4ifS29mZv2oZED/SklF4L0pdHpEPFHTWpmZWVOr5MqF9HCu+TWui5mZDRENubeYpC9KWiHpXklXSnq9pAmSlkjqlHSVpNGp7OvSemfaPr7kOGem+AOSDi6JT0mxTknuwjMzq7O6JxdJ44B/AlojYm9gJHA08G3gwoh4G7AemJ52mQ6sT/ELUzkkTUr7vROYAlwiaWR6NPPFwCHAJOCYVNbMzOqk3+SS/lj/rgbnHQVsL2kU8AbgcbLbycxL2+cAh6flqWmdtP1ASUrxuRGxMSIeIntE8n7p1RkRqyLiJWBuKmtmZnXSb3KJiJeBByTtmdcJI2INMBN4lCypbADuAJ6JiM2pWBcwLi2PA1anfTen8ruWxnvt01fczMzqpJIB/Z2BFZJuB57vCUbEYdWcUNLOZFcSE4BngGvIurXqTtIMYAZAS0sLxWKxquMUCgVmbtmSY822fYURI6p+v5pVd3e32zwMuM35qCS5fD3XM8JHgIci4ikASdcC+wNjJY1KVycFYE0qvwbYA+hK3Wg7AU+XxHuU7tNX/FUiYhYwC6C1tTXa2tqqalBHRwendXdXtW+zmjlmDO3t7Y2uRl0Vi0Wq/RlpVm7z8FCLNg84oB8RtwAPA9ul5aXAYJ7l8igwWdIb0tjJgWT3LLsZODKVmQZcl5bnp3XS9psiIlL86DSbbAIwEbg91W9imn02mmzQ39OozczqaMArF0knkHUd7QK8lWz84odkSWGrRcQSSfPIEtRmsvuUzQJ+BcyV9K0UuyztchnwE0mdwDqyZEFErJB0NVli2gyclMaIkPR5YCHZTLTZEbGimrqamVl1KukWO4lsBtYSgIhYKelNgzlpRJwNnN0rvCqdp3fZF4Gj+jjOecB5ZeILgAWDqaOZmVWvks+5bExTegFI4x5RuyqZmVmzqyS53CLpLLLPpXyUbHbXL2tbLTMza2aVJJczgKeAe4DPknU3fa2WlTIzs+ZWyV2Rt6QHhC0h6w57IM3WMjMzK6uS2WIfI5sd9nuy57lMkPTZiLih1pUzM7PmVMlssQ7gwxHRCSDprWTThp1czMysrErGXJ7rSSzJKuC5GtXHzMyGgD6vXCQdkRaXSVoAXE025nIU2afgzczMyuqvW+wTJctPAh9Ky08B29esRmZm1vT6TC4RcXw9K2JmZkNHJbPFJgAnA+NLy1d7y30zMxv6Kpkt9guym0f+EhheDy0xM7OqVJJcXoyIi2peEzMzGzIqSS7fk3Q28J/Axp5gRAzmmS5mZjaEVZJc/gr4DHAAf+oWi7RuZmb2GpUkl6OAt5Tedt/MzKw/lXxC/15gbJ4nlTRW0jxJv5N0v6S/kbSLpEWSVqavO6eyknSRpE5JyyXtU3Kcaan8SknTSuL7Sron7XNRepyymdmAli9fjqRh9Vq+fHnu72MlVy5jgd9JWsqrx1wGMxX5e8B/RMSR6Tn3bwDOAm6MiPMlnUF2q//TgUOAien1PuBS4H2SdiF7mmUrWTfdHZLmR8T6VOYEsjs5LwCm4HuhmVkFNm3aBOec0+hq1NWmTZtyP2YlyaX344gHRdJOwAeB4wBSd9tLkqYCbanYHKBIllymAlek2/wvTlc9u6eyiyJiXTruImCKpCKwY0QsTvErgMNxcjEzq5tKnudyS87nnEB2C5n/K+ldwB3AKUBLRDyeyjwBtKTlccDqkv27Uqy/eFeZ+GtImgHMAGhpaaFYLFbVoEKhwMwtw+sjQIURI6p+v5pVd3e32zwM+Pc5H5V8Qv85sm4ngNHAdsDzEbHjIM65D3ByRCyR9D2yLrBXRERIqvkDySJiFjALoLW1Ndra2qo6TkdHB6d1d+dYs23fzDFjaG9vb3Q16qpYLFLtz0izGo5t9u9zPgYc0I+IHSJix5RMtgc+CVwyiHN2AV0RsSStzyNLNk+m7i7S17Vp+xpgj5L9CynWX7xQJm5mZnVSyWyxV0TmF8DB1Z4wIp4AVkt6RwodCNwHzAd6ZnxNA65Ly/OBY9OsscnAhtR9thA4SNLOaWbZQcDCtO1ZSZPTLLFjS45lZmZ1UEm32BElqyPIZme9OMjzngz8NM0UWwUcn459taTpwCPAp1LZBcChQCfwQipLRKyTdC5/erbMN3sG94ETgcvJrrRuwIP5ZmZ1VclssdLnumwGHiabwVW1iLibLEn1dmCZsgGc1MdxZgOzy8SXAXsPpo5mZla9SmaL+bkuZma2Vfp7zPE3+tkvIuLcGtTHzMyGgP6uXJ4vE3sjMB3YFXByMTOzsvp7zHFHz7KkHcg+6Hg8MBfo6Gs/MzOzfsdc0v27TgX+nuyWLPuke3eZmZn1qb8xlwuAI8g+wf5XETG8PrJqZmZV6+9DlF8C/hz4GvCYpGfT6zlJz9anemZm1oz6G3PZqk/vm5mZ9XACMTOz3Dm5mJlZ7pxczMwsd04uZmaWOycXMzPLnZOLmZnlzsnFzMxy17DkImmkpLskXZ/WJ0haIqlT0lXpQWJIel1a70zbx5cc48wUf0DSwSXxKSnWKemMerfNzGy4a+SVyynA/SXr3wYujIi3AevJ7r5M+ro+xS9M5ZA0CTgaeCcwBbgkJayRwMXAIcAk4JhU1szM6qQhyUVSAfgY8OO0LuAAYF4qMgc4PC1PTeuk7Qem8lOBuRGxMSIeInsM8n7p1RkRqyLiJbK7OA/qyZlmZrZ1GnXl8l3gK8CWtL4r8ExEbE7rXcC4tDwOWA2Qtm9I5V+J99qnr7iZmdXJgI85zpukjwNrI+IOSW31Pn+vuswAZgC0tLRQLBarOk6hUGDmli0DFxxCCiNGVP1+Navu7m63eRjw73M+6p5cgP2BwyQdCrwe2BH4HjBW0qh0dVIA1qTya4A9gC5Jo4CdgKdL4j1K9+kr/ioRMYvskQK0trZGW1tbVQ3q6OjgtO7h9USCmWPG0N7e3uhq1FWxWKTan5FmNRzb7N/nfNS9WywizoyIQkSMJxuQvyki/h64GTgyFZsGXJeW56d10vabIiJS/Og0m2wCMBG4HVgKTEyzz0anc8yvQ9PMzCxpxJVLX04H5kr6FnAXcFmKXwb8RFInsI4sWRARKyRdDdwHbAZOioiXASR9HlgIjARmR8SKurbEzGyYa2hyiYgiUEzLq8hmevUu8yJwVB/7nwecVya+AFiQY1XNzGwr+BP6ZmaWOycXMzPLnZOLmZnlzsnFzMxy5+RiZma5c3IxM7PcObmYmVnunFzMzCx3Ti5mZpY7JxczM8udk4uZmeXOycXMzHLn5GJmZrlzcjEzs9w5uZiZWe6cXMzMLHd1Ty6S9pB0s6T7JK2QdEqK7yJpkaSV6evOKS5JF0nqlLRc0j4lx5qWyq+UNK0kvq+ke9I+F0lSvdtpZjacNeLKZTPwpYiYBEwGTpI0CTgDuDEiJgI3pnWAQ4CJ6TUDuBSyZAScDbyP7AmWZ/ckpFTmhJL9ptShXWZmltQ9uUTE4xFxZ1p+DrgfGAdMBeakYnOAw9PyVOCKyCwGxkraHTgYWBQR6yJiPbAImJK27RgRiyMigCtKjmVmZnUwqpEnlzQeeA+wBGiJiMfTpieAlrQ8DlhdsltXivUX7yoTL3f+GWRXQ7S0tFAsFqtqR6FQYOaWLVXt26wKI0ZU/X41q+7ubrd5GPDvcz4allwkjQF+DnwhIp4tHRaJiJAUta5DRMwCZgG0trZGW1tbVcfp6OjgtO7uHGu27Zs5Zgzt7e2NrkZdFYtFqv0ZaVbDsc3+fc5HQ2aLSdqOLLH8NCKuTeEnU5cW6evaFF8D7FGyeyHF+osXysTNzKxOGjFbTMBlwP0R8Z2STfOBnhlf04DrSuLHplljk4ENqftsIXCQpJ3TQP5BwMK07VlJk9O5ji05lpmZ1UEjusX2Bz4D3CPp7hQ7CzgfuFrSdOAR4FNp2wLgUKATeAE4HiAi1kk6F1iayn0zItal5ROBy4HtgRvSy8zM6qTuySUifgP09bmTA8uUD+CkPo41G5hdJr4M2HsQ1TQzs0HwJ84PCeIAAAXnSURBVPTNzCx3Ti5mZpY7JxczM8udk4uZmeXOycXMzHLn5GJmZrlzcjEzs9w5uZiZWe6cXMzMLHdOLmZmljsnFzMzy52Ti5mZ5c7JxczMcufkYmZmuXNyMTOz3A3Z5CJpiqQHJHVKOqPR9TEzG06GZHKRNBK4GDgEmAQcI2lSY2tlZjZ8DMnkAuwHdEbEqoh4CZgLTG1wnczMho2hmlzGAatL1rtSzMzM6kDZI+qHFklHAlMi4h/T+meA90XE53uVmwHMSKvvAB6o8pS7AX+oct9m5TYPD27z8DCYNu8VEX/WOzhqcPXZZq0B9ihZL6TYq0TELGDWYE8maVlEtA72OM3EbR4e3ObhoRZtHqrdYkuBiZImSBoNHA3Mb3CdzMyGjSF55RIRmyV9HlgIjARmR8SKBlfLzGzYGJLJBSAiFgAL6nS6QXetNSG3eXhwm4eH3Ns8JAf0zcyssYbqmIuZmTWQk8tWGOiWMpJeJ+mqtH2JpPH1r2W+KmjzqZLuk7Rc0o2S9mpEPfNU6a2DJH1SUkhq6plFlbRX0qfS93mFpJ/Vu455q+Dnek9JN0u6K/1sH9qIeuZJ0mxJayXd28d2SboovSfLJe0zqBNGhF8VvMgmBvweeAswGvgtMKlXmROBH6blo4GrGl3vOrT5w8Ab0vLnhkObU7kdgFuBxUBro+td4+/xROAuYOe0/qZG17sObZ4FfC4tTwIebnS9c2j3B4F9gHv72H4ocAMgYDKwZDDn85VL5Sq5pcxUYE5angccKEl1rGPeBmxzRNwcES+k1cVknylqZpXeOuhc4NvAi/WsXA1U0t4TgIsjYj1ARKytcx3zVkmbA9gxLe8EPFbH+tVERNwKrOunyFTgisgsBsZK2r3a8zm5VK6SW8q8UiYiNgMbgF3rUrva2Nrb6Ewn+8+nmQ3Y5tRdsEdE/KqeFauRSr7HbwfeLum/JS2WNKVutauNStp8DvBpSV1ks05Prk/VGirX22YN2anIVl+SPg20Ah9qdF1qSdII4DvAcQ2uSj2NIusaayO7Mr1V0l9FxDMNrVVtHQNcHhEdkv4G+ImkvSNiS6Mr1ix85VK5Sm4p80oZSaPILqefrkvtaqOi2+hI+gjwVeCwiNhYp7rVykBt3gHYGyhKepisb3p+Ew/qV/I97gLmR8SmiHgIeJAs2TSrSto8HbgaICJuA15Pdv+toayi3/dKOblUrpJbyswHpqXlI4GbIo2UNakB2yzpPcCPyBJLs/fFwwBtjogNEbFbRIyPiPFk40yHRcSyxlR30Cr5uf4F2VULknYj6yZbVc9K5qySNj8KHAgg6S/JkstTda1l/c0Hjk2zxiYDGyLi8WoP5m6xCkUft5SR9E1gWUTMBy4ju3zuJBs4O7pxNR68Ctt8ATAGuCbNXXg0Ig5rWKUHqcI2DxkVtnchcJCk+4CXgS9HRNNekVfY5i8B/ybpi2SD+8c1+T+KSLqS7J+E3dJY0tnAdgAR8UOysaVDgU7gBeD4QZ2vyd8vMzPbBrlbzMzMcufkYmZmuXNyMTOz3Dm5mJlZ7pxczMwsd04uZnUmqXsryp4j6bRaHd+sVpxczMwsd04uZtsASZ9IzwC6S9KvJbWUbH6XpNskrZR0Qsk+X5a0ND17458bUG2zPjm5mG0bfgNMjoj3kN0C/isl2/4aOAD4G+Abkv5c0kFk9/faD3g3sK+kD9a5zmZ98u1fzLYNBeCq9PyM0cBDJduui4g/An+UdDNZQnk/cBDZQ7wguwXPRLIHmJk1nJOL2bbh+8B3ImK+pDay54n06H2PpiB7WuC/RsSP6lM9s63jbjGzbcNO/On25tN6bZsq6fWSdiW78eBSspsu/oOkMQCSxkl6U70qazYQX7mY1d8b0l1pe3yH7ErlGknrgZuACSXblwM3kz1P5NyIeAx4LN0K/rZ0N+pu4NPAUHjsgQ0BviuymZnlzt1iZmaWOycXMzPLnZOLmZnlzsnFzMxy5+RiZma5c3IxM7PcObmYmVnunFzMzCx3/wNRWfrAOjRhvgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ttLfGD5Kp0Ri"
      },
      "source": [
        "- Test Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pKnqnBdG9j0i",
        "outputId": "4318f7cd-1d5b-4d2c-b621-52d2a3a423fc"
      },
      "source": [
        "df_allduplicate = df[df['is_duplicate'] == 1]\n",
        "df_test = df_allduplicate[0:100] \n",
        "df_test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>qid1</th>\n",
              "      <th>qid2</th>\n",
              "      <th>question1</th>\n",
              "      <th>question2</th>\n",
              "      <th>is_duplicate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>402555.0</td>\n",
              "      <td>536040.0</td>\n",
              "      <td>536041.0</td>\n",
              "      <td>How do I control my horny emotions?</td>\n",
              "      <td>How do you control your horniness?</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>11568.0</td>\n",
              "      <td>22332.0</td>\n",
              "      <td>22333.0</td>\n",
              "      <td>Which is the best book to study TENSOR for gen...</td>\n",
              "      <td>Which is the best book for tensor calculus?</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>33995.0</td>\n",
              "      <td>62359.0</td>\n",
              "      <td>62360.0</td>\n",
              "      <td>How does an IQ test work and what is determine...</td>\n",
              "      <td>How does IQ test works?</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>244506.0</td>\n",
              "      <td>357159.0</td>\n",
              "      <td>357160.0</td>\n",
              "      <td>Is it safe to use Xiaomi mobile phones?</td>\n",
              "      <td>Is it safe or unsafe to use Xiaomi Products?</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>375073.0</td>\n",
              "      <td>506056.0</td>\n",
              "      <td>506057.0</td>\n",
              "      <td>What are the best books on cosmology?</td>\n",
              "      <td>Which is the best book for cosmology?</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>379</th>\n",
              "      <td>17002.0</td>\n",
              "      <td>32363.0</td>\n",
              "      <td>32364.0</td>\n",
              "      <td>What is it like to work at Factual?</td>\n",
              "      <td>What is it like to work at Factual in 2016?</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>385</th>\n",
              "      <td>179873.0</td>\n",
              "      <td>275905.0</td>\n",
              "      <td>275906.0</td>\n",
              "      <td>How do I recover data from an external hard disk?</td>\n",
              "      <td>How can I recover data from an external hard d...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>390</th>\n",
              "      <td>236758.0</td>\n",
              "      <td>347752.0</td>\n",
              "      <td>347753.0</td>\n",
              "      <td>What is the perfect website that lists all for...</td>\n",
              "      <td>What is the perfect website that lists all typ...</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>392</th>\n",
              "      <td>72098.0</td>\n",
              "      <td>123946.0</td>\n",
              "      <td>123947.0</td>\n",
              "      <td>Can I install Mac OS on my Dell laptop? How?</td>\n",
              "      <td>How do I install Mac OS on a Dell laptop?</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>399</th>\n",
              "      <td>399184.0</td>\n",
              "      <td>360315.0</td>\n",
              "      <td>532420.0</td>\n",
              "      <td>How do I get the best deals for a cruise?</td>\n",
              "      <td>How can I get the absolute best deal on a cruise?</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>100 rows × 6 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "           id  ...  is_duplicate\n",
              "1    402555.0  ...           1.0\n",
              "6     11568.0  ...           1.0\n",
              "11    33995.0  ...           1.0\n",
              "12   244506.0  ...           1.0\n",
              "14   375073.0  ...           1.0\n",
              "..        ...  ...           ...\n",
              "379   17002.0  ...           1.0\n",
              "385  179873.0  ...           1.0\n",
              "390  236758.0  ...           1.0\n",
              "392   72098.0  ...           1.0\n",
              "399  399184.0  ...           1.0\n",
              "\n",
              "[100 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kiv6nZsPptK2",
        "outputId": "65e4094c-770f-42ad-f716-64c495dd70f3"
      },
      "source": [
        "ques1_list  = df_test['question1'].tolist()\n",
        "\n",
        "print(\"Length of question 1 list(first 100 with is_dup = 1): \", len(ques1_list))\n",
        "print(\"Unique question 1 Ids: \", df_test['qid1'].nunique()) # all unique ids"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Length of question 1 list(first 100 with is_dup = 1):  100\n",
            "Unique question 1 Ids:  100\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3_NjRMaeI8TQ"
      },
      "source": [
        "## 1. Data Prepocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yCDDSXxd0yCw"
      },
      "source": [
        "- Stop word Removal, Stemming/Lemmatization, Puntuation removal is done\n",
        "- Stemming is done for TF-IDF Method while for sentence embedding based methods, Lemmatization is used"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FSBaiZFKPJ6A"
      },
      "source": [
        "# function to remove puntuation\n",
        "def remove_puntuation(text):\n",
        "    text = [word for word in text if word.isalpha()] #  remove all puntuation and numbers\n",
        "    return text \n",
        "\n",
        "# function to remove stop word from text\n",
        "def remove_stopwords(text):\n",
        "    stop_words = list(stopwords.words('english')) # store nltk stopwords\n",
        "    text = [word for word in text if not word in stop_words] \n",
        "    return text \n",
        "\n",
        "# function to perform stemming\n",
        "def stemming(text):\n",
        "  stemer = stem.porter.PorterStemmer()\n",
        "  text = [stemer.stem(token) for token in text] \n",
        "  return text \n",
        "\n",
        "# function to perform lemmatization\n",
        "def lemmatization (text):\n",
        "    lemmatizer = stem.WordNetLemmatizer() \n",
        "    text = [lemmatizer.lemmatize(token) for token in text]\n",
        "    return text \n",
        "\n",
        "# function to perform text preprocessing (stemming)\n",
        "def clean_corpus(text):\n",
        "    \n",
        "    text = text.lower()                # lower case  \n",
        "    token = word_tokenize(text)         # tokenise\n",
        "  \n",
        "    token = remove_stopwords(token)    # remove stopwords\n",
        "    token = remove_puntuation(token)   # remove puntuation \n",
        "\n",
        "    token = stemming(token) \n",
        "\n",
        "    all_tokens = \" \".join(token)       # join all tokens\n",
        "\n",
        "    return all_tokens\n",
        "\n",
        "# function to perform text preprocessing (lemmatization and no stemming)\n",
        "def clean_corpus_2(text):\n",
        "    \n",
        "    text = text.lower()                # lower case  \n",
        "    token = word_tokenize(text)         # tokenise\n",
        "  \n",
        "    token = remove_stopwords(token)    # remove stopwords\n",
        "    token = remove_puntuation(token)   # remove puntuation \n",
        "    token = lemmatization(token)       # lemmatize\n",
        "\n",
        "    all_tokens = \" \".join(token)       # join all tokens\n",
        "\n",
        "    return all_tokens"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cAZthO6q0yCx"
      },
      "source": [
        "# preprocess training data\n",
        "df['processed_ques2'] = df.question2.apply(lambda x: clean_corpus(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_NSX6AgxQWIi",
        "outputId": "01ac08e5-9da6-4698-ae51-399e6cb5286e"
      },
      "source": [
        "#look at some preprocessed text\n",
        "print(df['processed_ques2'].values[6])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "best book tensor calculu\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nws4uVhb9j0k",
        "outputId": "3fd2823c-f45d-4228-97fd-8084077672ab"
      },
      "source": [
        "df[\"question2\"].values[6]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Which is the best book for tensor calculus?'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_8CJVButAmRT",
        "outputId": "4362898a-a84f-449c-db5d-e866b2d9ff48"
      },
      "source": [
        "df['processed_ques2'][0:15]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0                             level preprat enough exam\n",
              "1                                         control horni\n",
              "2     would second airport sydney australia need rai...\n",
              "3     beleiv bulim forc throw atleast day eat someth...\n",
              "4                             becom air traffic control\n",
              "5                              qualifi sap erp key user\n",
              "6                              best book tensor calculu\n",
              "7                                  contact donald knuth\n",
              "8                           die roll probabl number top\n",
              "9                                      best backend app\n",
              "10            type govern franc current benefit countri\n",
              "11                                         iq test work\n",
              "12                        safe unsaf use xiaomi product\n",
              "13                    creativ idea arrang fresher parti\n",
              "14                                   best book cosmolog\n",
              "Name: processed_ques2, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "debeMQHvvZnd"
      },
      "source": [
        "# function to get word count of a dataset\n",
        "def word_count(dataset):\n",
        "    \n",
        "    unique_words = [] \n",
        "    \n",
        "    for row in dataset:\n",
        "        for word in row.split(\" \"):\n",
        "            if len(word) > 2 and word not in unique_words:\n",
        "                unique_words.append(word) # Add each unique word of length > 2 to the list\n",
        "            \n",
        "    unique_words.sort()\n",
        "    vocab = {j:i for i,j in enumerate(unique_words)} # Enumerate the list, i.e., give consecutive numbers to each item, store in a dict\n",
        "    return vocab"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gzy4crX2vaSX",
        "outputId": "13f70666-4241-4728-8cb0-c7ea9071362a"
      },
      "source": [
        "#total number of unique words in question2 with length of word > 2\n",
        "unique_words_ques2 = word_count(df['processed_ques2'])\n",
        "print(\"Number of unique words in question2: \",len(unique_words_ques2))  "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of unique words in question2:  35937\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qWqrmDsEvAub"
      },
      "source": [
        "# function to remove words which have a count less than N\n",
        "def min_df(col , N):\n",
        "  v = col.str.split().tolist()              # split words into list\n",
        "  c = Counter(chain.from_iterable(v))       # compute word frequency\n",
        "  col = [' '.join([j for j in i if c[j] > N]) for i in v] #  join and reassign      \n",
        "  return col\n",
        "\n",
        "# function to remove words which have a count greater than N\n",
        "def max_df(col , N):\n",
        "  v = col.str.split().tolist()             # split words into list\n",
        "  c = Counter(chain.from_iterable(v))      # compute word frequency\n",
        "  col = [' '.join([j for j in i if c[j] < N]) for i in v] #  join and reassign      \n",
        "  return col"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0TLAXofrvYQJ",
        "outputId": "657818a8-a0c7-4a0c-bb5f-e856499af875"
      },
      "source": [
        "df[\"processed_ques2\"] = max_df(df[\"processed_ques2\"], 5000)  #remove words with count greater than 5k\n",
        "df[\"processed_ques2\"] = min_df(df[\"processed_ques2\"], 1) \n",
        "\n",
        "#total number of unique words in question2 after preprocessing\n",
        "unique_words_ques2 = word_count(df['processed_ques2'])\n",
        "print(\"Number of unique words in question2: \",len(unique_words_ques2))   #15479"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of unique words in question2:  19845\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bginKC9OvqS2",
        "outputId": "ab5eeff5-5144-4af8-afff-0738674bd5d7"
      },
      "source": [
        "df['processed_ques2'][0:20]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0                             level preprat enough exam\n",
              "1                                         control horni\n",
              "2     second airport sydney australia need rail link...\n",
              "3     bulim forc throw atleast day eat someth feel g...\n",
              "4                             becom air traffic control\n",
              "5                              qualifi sap erp key user\n",
              "6                                   book tensor calculu\n",
              "7                                  contact donald knuth\n",
              "8                           die roll probabl number top\n",
              "9                                           backend app\n",
              "10            type govern franc current benefit countri\n",
              "11                                              iq test\n",
              "12                            safe unsaf xiaomi product\n",
              "13                    creativ idea arrang fresher parti\n",
              "14                                        book cosmolog\n",
              "15                                 staten island racist\n",
              "16                             beauti hous around world\n",
              "17                                                medal\n",
              "18              recruit technolog execut san diego area\n",
              "19    got select infosi via campu placement septemb ...\n",
              "Name: processed_ques2, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7DbYhDA6vwpI",
        "outputId": "283efe27-37b8-46fb-bb4b-643939a43906"
      },
      "source": [
        "ques2_list = df['processed_ques2'].tolist()\n",
        "len(ques2_list)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "232203"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s3B3mUyQvyIV",
        "outputId": "2d429ac2-a98b-491e-875b-048470202f4f"
      },
      "source": [
        "# remove empty questions \n",
        "count = 0\n",
        "i = 0\n",
        "\n",
        "for line in ques2_list: \n",
        "  if line == \"\":\n",
        "    count = count+ 1\n",
        "    del ques2_list[i] # delete questions which are empty strings  \n",
        "    \n",
        "  i = i +1\n",
        "\n",
        "print(\"Total number of empty questions in question2: \",count)\n",
        "len(ques2_list)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Total number of empty questions in question2:  804\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "231399"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sAsV-lU7-FPe"
      },
      "source": [
        "## 2. TF IDF"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OuluuyEv9j0l",
        "outputId": "2633bfc9-a0f6-43bd-b1db-ba98afd932f4"
      },
      "source": [
        "vocab = word_count(df['processed_ques2'][:50000]) \n",
        "print(\"Vocabulary length: \",len(vocab))  #17899"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Vocabulary length:  14868\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_7BoubKV9j0m"
      },
      "source": [
        "# function to calculate Inverse document frequesny\n",
        "def get_doc_freq(data,word):\n",
        "    \n",
        "    freq = 0\n",
        "    for row in data:\n",
        "        if word in row:\n",
        "            freq = freq + 1\n",
        "    return freq\n",
        "\n",
        "# function to generate TF-IDF matrix\n",
        "def transform(data, vocab):\n",
        "    \n",
        "    rows = []              # stores which row number/question a word belongs to\n",
        "    each_word_freq = []   # stores the frequency of a word in the vocabulary \n",
        "    result = []           # stores TF IDF value of a word\n",
        "    \n",
        "    for indx, row in enumerate(data): # for each Qeuestion 2 in the dataset\n",
        "\n",
        "        sentence_word_freq = dict(Counter(row.split())) \n",
        "\n",
        "        for word, freq in sentence_word_freq.items(): \n",
        "            if len(word) < 2:                     # keep words with length > 2\n",
        "                continue \n",
        "\n",
        "            vocab_idx = vocab.get(word, -1)      # get the frequency of a word in the vocab,  if the word doesn't exits, return -1\n",
        "            \n",
        "            if vocab_idx != -1:\n",
        "\n",
        "                rows.append(indx)   \n",
        "                each_word_freq.append(vocab_idx)\n",
        "\n",
        "                doc_freq = get_doc_freq(data,word) # document frequency of a word\n",
        "           \n",
        "                TF = freq / len(row.split())        # term frequency\n",
        "                IDF = log10(len(data)/doc_freq)     # inverse document frequency\n",
        "                \n",
        "                val = TF * IDF \n",
        "                result.append(val)\n",
        "\n",
        "    x = len(data)\n",
        "    y = len(vocab)\n",
        "\n",
        "    # For TFIDF Matrix, row index = row number/question a word belongs to, column index = each word frequency           \n",
        "    return csr_matrix( (result, (rows, each_word_freq)), shape=(x,y) ) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eRSLmOPMw8md",
        "outputId": "5b5ecbc0-93be-4625-fcdd-2849374c793c"
      },
      "source": [
        "\n",
        "tfidf_cols = [] #store the word which represents each column in TFIDF matrix  \n",
        " \n",
        "for key, val in vocab.items():\n",
        "  tfidf_cols.append(key) \n",
        "\n",
        "print(len(tfidf_cols))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "14868\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v69xyr589j0m",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "45bb3c42-08f3-4149-ddc6-0c527f385e7f"
      },
      "source": [
        "tf_idf_vect = transform(ques2_list[0:50000] , vocab) # get TFIDF matrix\n",
        "\n",
        "print(\"TF-IDF shape: \" , str(tf_idf_vect.shape))\n",
        "print(tf_idf_vect)                              "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TF-IDF shape:  (50000, 14868)\n",
            "  (0, 4259)\t0.6715331949077117\n",
            "  (0, 4464)\t0.459807859534718\n",
            "  (0, 7543)\t0.6366704149882406\n",
            "  (0, 10295)\t1.0\n",
            "  (1, 2846)\t1.2529227029907786\n",
            "  (1, 6051)\t1.926935982160881\n",
            "  (2, 273)\t0.310790539730952\n",
            "  (2, 910)\t0.26234230429434885\n",
            "  (2, 3032)\t0.21284270644541214\n",
            "  (2, 7628)\t0.2552841968657781\n",
            "  (2, 8222)\t0.3744727494896694\n",
            "  (2, 8893)\t0.19385475209128067\n",
            "  (2, 10705)\t0.2866461091629783\n",
            "  (2, 11715)\t0.2661543506395395\n",
            "  (2, 12971)\t0.6017547848615011\n",
            "  (3, 861)\t0.3838044317833051\n",
            "  (3, 1791)\t0.3838044317833051\n",
            "  (3, 3286)\t0.15923057113172437\n",
            "  (3, 4026)\t0.13249304761267913\n",
            "  (3, 4695)\t0.17194863420610598\n",
            "  (3, 4937)\t0.21569401303702432\n",
            "  (3, 5589)\t0.3130634090211557\n",
            "  (3, 12310)\t0.29854515057865466\n",
            "  (3, 12314)\t0.2275518501751623\n",
            "  (3, 13154)\t0.18851432303403134\n",
            "  :\t:\n",
            "  (49994, 7713)\t0.49346887081363744\n",
            "  (49994, 10943)\t1.0\n",
            "  (49995, 3789)\t1.0120707242181481\n",
            "  (49995, 8792)\t0.7083129122027666\n",
            "  (49995, 13509)\t1.2329900014453394\n",
            "  (49996, 4834)\t0.4886600009635597\n",
            "  (49996, 4979)\t0.7192382236526805\n",
            "  (49996, 5985)\t0.3023607110052412\n",
            "  (49996, 7596)\t0.4940664035581309\n",
            "  (49996, 12416)\t0.28195577801122707\n",
            "  (49996, 13560)\t0.5378258674184866\n",
            "  (49997, 1096)\t0.9247425010840047\n",
            "  (49997, 2074)\t0.5176452685714268\n",
            "  (49997, 2170)\t0.6487915708450155\n",
            "  (49997, 2543)\t0.6540461585048922\n",
            "  (49998, 3350)\t1.073949583205452\n",
            "  (49998, 7543)\t0.4244469433254937\n",
            "  (49998, 13215)\t0.624121249149449\n",
            "  (49998, 14628)\t0.3331887130781288\n",
            "  (49999, 1092)\t0.41857488058325926\n",
            "  (49999, 2305)\t0.556207028053071\n",
            "  (49999, 10031)\t0.3395262197997929\n",
            "  (49999, 10510)\t0.4114201804929575\n",
            "  (49999, 12109)\t0.26467396034493307\n",
            "  (49999, 12366)\t0.4254736614429634\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KoGncxC_9j0n",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4b3263b2-6097-4874-ba29-32f3e3f74f44"
      },
      "source": [
        "tf_idf_array = tf_idf_vect.toarray()\n",
        "print(tf_idf_array )\n",
        "print(tf_idf_array.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0. 0. 0. ... 0. 0. 0.]\n",
            " [0. 0. 0. ... 0. 0. 0.]\n",
            " [0. 0. 0. ... 0. 0. 0.]\n",
            " ...\n",
            " [0. 0. 0. ... 0. 0. 0.]\n",
            " [0. 0. 0. ... 0. 0. 0.]\n",
            " [0. 0. 0. ... 0. 0. 0.]]\n",
            "(50000, 14868)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "duYDR3fQubvt"
      },
      "source": [
        "# function to generate inverted index\n",
        "def generate_inverted_index(data):\n",
        "\n",
        "    inv_idx_dict = {}\n",
        "\n",
        "    for index, rows in enumerate(data):\n",
        "        for word in rows.split():\n",
        "\n",
        "          if len(word) > 2:                        # discard words of length less than 2\n",
        "            if word not in inv_idx_dict.keys() : \n",
        "                inv_idx_dict[word] = [index]      # add word as key in dict\n",
        "\n",
        "            elif index not in inv_idx_dict[word] :\n",
        "                inv_idx_dict[word].append(index)  # add location of document(question2) as value \n",
        "\n",
        "    return inv_idx_dict"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vnyCYQfJulnD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c6e54b15-c2d3-41d0-98a5-d09777332042"
      },
      "source": [
        "inverted_index = generate_inverted_index(ques2_list[0:50000])\n",
        "len(inverted_index)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "14878"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wi6dp3yyESpj"
      },
      "source": [
        "# get the position of the column representing the word 'term' in TFIDF matrix\n",
        "def find_term_pos(term):\n",
        "  pos = -99\n",
        "  for i in [i for i,x in enumerate(tfidf_cols) if x == term]:\n",
        "    pos = i \n",
        "  return pos"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_VQbdj1GRw3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a6a0b73d-7018-42d7-ab3a-f1cf23051752"
      },
      "source": [
        "print(find_term_pos(\"safe\"))\n",
        "print(find_term_pos(\"iq\")) # there is no word name 'iq' in the TFIDF matrix representation"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "11447\n",
            "-99\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HLQcT0gcvYaN"
      },
      "source": [
        "# function to caculate topk similar questions , returns question number and similarity score\n",
        "def get_similarity_tfidf(ques1_sentence, k):\n",
        "\n",
        "  doc_sim = {}\n",
        "  ques1_sentence = clean_corpus(ques1_sentence) # preprocess input sentence\n",
        "  tokens = word_tokenize(ques1_sentence)\n",
        "\n",
        "  for term in tokens:\n",
        "    if term in inverted_index.keys() :\n",
        "\n",
        "      docs = inverted_index[term]     # get list of docs in which term is present   \n",
        "      term_pos = find_term_pos(term)  # find position of term in TFIDF matrix\n",
        "\n",
        "      if term_pos != -99:            #  if term has a TFIDF value\n",
        "        for doc in docs:\n",
        "          doc_score = tf_idf_array [doc][term_pos]   #get tfidf score\n",
        "        \n",
        "          if doc in doc_sim.keys():\n",
        "            doc_sim[doc] += doc_score\n",
        "          else:\n",
        "            doc_sim[doc] = doc_score\n",
        "\n",
        "  sorted_sim = sorted(doc_sim.items(), key = lambda kv:(kv[1], kv[0]),reverse = True ) # sort based on value\n",
        "  topk = sorted_sim[0:k]\n",
        "  \n",
        "  return topk\n",
        "\n",
        "# function to print topk sentences along with their scores\n",
        "def print_top_k(topk):\n",
        "\n",
        "  for item in topk :\n",
        "    val = item[0]\n",
        "    score = item[1]\n",
        "    sentence = df['question2'][val]\n",
        "    print(\" {}, Score: {}\".format( sentence, score))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t2unoOLo1ED4"
      },
      "source": [
        "-  For a single line of text :"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l3B5jwVW_v2g",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "90345ef5-c4b4-4e41-f51f-fcb096d64b11"
      },
      "source": [
        "df['question1'][16]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'What are some of the most beautiful houses in the world?'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jfGd0eQodA0h",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bdcfc97b-598f-4033-9377-1db989f3c349"
      },
      "source": [
        "ques_sim = get_similarity_tfidf(\"What are some of the most beautiful houses in the world?\",5) \n",
        "ques_sim"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(15238, 2.374687549038326),\n",
              " (11688, 2.374687549038326),\n",
              " (38786, 1.926648297613099),\n",
              " (16, 1.7651966495922062),\n",
              " (40786, 1.5620330164435)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oO3KXtMaAJ8r",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "67f8db9e-6bfd-4d01-cc15-458bd5e884f5"
      },
      "source": [
        "print_top_k(ques_sim)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " What is time complexity of heap and heap sort?, Score: 2.374687549038326\n",
            " If you think about someone are they thinking of you?, Score: 2.374687549038326\n",
            " How do you see the future of drones?, Score: 1.926648297613099\n",
            " Which are some of the most beautiful houses around the world?, Score: 1.7651966495922062\n",
            " WHAT are best practices FOR OUTBOUND MARKETING?, Score: 1.5620330164435\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xSWgwJ0U1Q5r"
      },
      "source": [
        "- For first 100 questions with is_dup = 1 :"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eob2tW7xdCqj"
      },
      "source": [
        "# function to perform all caculations and return accuracy\n",
        "def do_calcualtion(ques1_list, k):\n",
        "\n",
        "  all_similarty = [] # store top k sentence indices and scores\n",
        "\n",
        "  for sentence in ques1_list :\n",
        "    similarity = get_similarity_tfidf(sentence, k)\n",
        "    all_similarty.append(similarity )\n",
        "  \n",
        "  list_matching_id = all_similarity_index(all_similarty) # get ids of top k sentences\n",
        "  accuracy = get_accuracy(list_matching_id)\n",
        "  return accuracy\n",
        "\n",
        "# get index of top k matches\n",
        "def all_similarity_index(all_tfidf_similarty):\n",
        "  list_matching_id = []\n",
        "\n",
        "  for item in all_tfidf_similarty:  \n",
        "    matching_id = []\n",
        "    length = len(item)\n",
        "\n",
        "    for i in range(length):\n",
        "      matching_id.append(item[i][0])\n",
        "    list_matching_id.append( matching_id)\n",
        "  return list_matching_id\n",
        "\n",
        "# function to calculate accuracy\n",
        "def get_accuracy(list_matching_id):    \n",
        "  correct = 0\n",
        "  j = 0\n",
        "\n",
        "  for item in list_matching_id :\n",
        "    length = len(item)\n",
        "\n",
        "    for i in range(length):\n",
        "      if item[i] == df_test[\"question1\"].index[j]: #check if index matches\n",
        "        correct = correct + 1\n",
        "    j = j +1\n",
        "\n",
        "  accuracy = correct\n",
        "  return accuracy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JVWBFN0tkxjk"
      },
      "source": [
        "- Evaluation:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mnFL3fZD4IgP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9db9a448-54f8-4006-e092-d5cd7e862a5c"
      },
      "source": [
        "ques1_list  = df_test['question1'].tolist()\n",
        "\n",
        "top2  = do_calcualtion(ques1_list, 2) \n",
        "top5  = do_calcualtion(ques1_list, 5) \n",
        "\n",
        "print(\"Correctly predicted matches in top2: \", top2)\n",
        "print(\"Correctly predicted matches in top5: \", top5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Correctly predicted matches in top2:  17\n",
            "Correctly predicted matches in top5:  25\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ttzqHn-H0yC6"
      },
      "source": [
        "## 3. Average Word based Sentence Embedding Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "keIvGCY-9KAk"
      },
      "source": [
        "# preprocess training data \n",
        "df['processed_ques2_lemm'] = df.question2.apply(lambda x: clean_corpus_2(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c5r_ZqiCKZVY",
        "outputId": "83978fe9-784e-4847-8a94-090eefcfe066"
      },
      "source": [
        "ques2_all = df['processed_ques2_lemm'] \n",
        "ques2_all[0:25] # look at some preprocessed data"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0                          level prepration enough exam\n",
              "1                                     control horniness\n",
              "2     would second airport sydney australia needed r...\n",
              "3     beleive bulimic force throw atleast day eat so...\n",
              "4                         become air traffic controller\n",
              "5                            qualified sap erp key user\n",
              "6                             best book tensor calculus\n",
              "7                                  contact donald knuth\n",
              "8                     die rolled probability number top\n",
              "9                                      best backend app\n",
              "10    type government france currently benefited cou...\n",
              "11                                         iq test work\n",
              "12                       safe unsafe use xiaomi product\n",
              "13                creative idea arranging fresher party\n",
              "14                                  best book cosmology\n",
              "15                          people staten island racist\n",
              "16                         beautiful house around world\n",
              "17                                        indutal medal\n",
              "18    best recruiter technology executive san diego ...\n",
              "19    got selected infosys via campus placement sept...\n",
              "20    draw shear force bending moment diagram streng...\n",
              "21       good looking guy act like boyfriend thing mean\n",
              "22               artist interesting inspiring childhood\n",
              "23                             narendra modi win seat l\n",
              "24                            best website entrepreneur\n",
              "Name: processed_ques2_lemm, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5AlzHhT4-kNB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a0311e6b-bd44-4c43-b4ea-6bdba7a964d6"
      },
      "source": [
        "df['processed_ques2_lemm'].shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(232203,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wQHzLmeH92nZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "58ee70cb-24ed-4916-8264-71dec3d436c8"
      },
      "source": [
        "count = 0\n",
        "i = 0\n",
        "for line in ques2_all:\n",
        "  \n",
        "  if line == \"\":\n",
        "    count = count+ 1\n",
        "    del ques2_all[i]\n",
        "    \n",
        "  i = i +1\n",
        "\n",
        "print(\"Count: \",count)\n",
        "len(ques2_all)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Count:  288\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "231915"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gv9I99LoKdjt",
        "outputId": "98ddc160-4f19-4062-eba7-f89730b783de"
      },
      "source": [
        "# store vocabulary\n",
        "vocabulary = word_count(df['processed_ques2_lemm'] )\n",
        "print(\"Vocabulary length: \",len(vocabulary)) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Vocabulary length:  44799\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "khE0Xnm8EvLy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7a4bd488-4249-4564-ea49-faa217511c76"
      },
      "source": [
        "# code to download glove data\n",
        "# !wget http://nlp.stanford.edu/data/glove.6B.zip\n",
        "# !unzip -q glove.6B.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-05-17 12:06:18--  http://nlp.stanford.edu/data/glove.6B.zip\n",
            "Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:80... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://nlp.stanford.edu/data/glove.6B.zip [following]\n",
            "--2021-05-17 12:06:18--  https://nlp.stanford.edu/data/glove.6B.zip\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip [following]\n",
            "--2021-05-17 12:06:18--  http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip\n",
            "Resolving downloads.cs.stanford.edu (downloads.cs.stanford.edu)... 171.64.64.22\n",
            "Connecting to downloads.cs.stanford.edu (downloads.cs.stanford.edu)|171.64.64.22|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 862182613 (822M) [application/zip]\n",
            "Saving to: ‘glove.6B.zip’\n",
            "\n",
            "glove.6B.zip        100%[===================>] 822.24M  5.12MB/s    in 2m 43s  \n",
            "\n",
            "2021-05-17 12:09:01 (5.04 MB/s) - ‘glove.6B.zip’ saved [862182613/862182613]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NehD6SVyGDW3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "92bf3499-4acb-46ae-b0e1-9a7bf13012a0"
      },
      "source": [
        "# get word embeddings\n",
        "\n",
        "embedding_dim = 100\n",
        "\n",
        "embeddings_index = dict()\n",
        "glove_word_embedding = []\n",
        "glove_word_pos = {} \n",
        "\n",
        "f = open('glove.6B.100d.txt')\n",
        "\n",
        "for i,line in enumerate(f):\n",
        "    values = line.split()\n",
        "    word = values[0]\n",
        "    coefs = np.asarray(values[1:],dtype = 'float64') #store all excepy val[0]\n",
        "\n",
        "    glove_word_embedding.append(coefs)\n",
        "    glove_word_pos[word] = i\n",
        "\n",
        "    embeddings_index[word] = coefs #populate embeddings_index dict\n",
        "    \n",
        "f.close()\n",
        "\n",
        "glove_word_embedding = np.array(glove_word_embedding)\n",
        "\n",
        "print(f'Found {len(embeddings_index)} word vectors') "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 400000 word vectors\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rBVcLcd8LBEa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "61de9f98-ce72-479d-f524-7f35c83d5fbc"
      },
      "source": [
        "embeddings_index[\"book\"]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([-1.9744e-01,  4.4831e-01,  1.3689e-01, -1.5595e-01,  9.3600e-01,\n",
              "        7.2986e-01,  3.4099e-01, -3.3896e-01, -8.9569e-02, -4.7706e-01,\n",
              "        3.5112e-01, -4.2198e-01, -1.2221e-01, -6.3375e-02, -4.5820e-01,\n",
              "        7.8723e-01,  9.4045e-01,  8.1101e-02, -2.3224e-01,  4.0778e-01,\n",
              "        3.3258e-01, -4.4458e-01, -4.7117e-01,  1.4852e-01,  9.6308e-01,\n",
              "       -6.5267e-02, -5.3661e-02, -6.7474e-01, -4.2364e-01,  9.4392e-02,\n",
              "       -3.8668e-01,  1.8237e-01, -1.2846e-01, -2.1952e-01, -5.8993e-01,\n",
              "        7.3602e-01, -2.4009e-01,  3.2392e-01, -2.4663e-01, -4.0684e-01,\n",
              "       -5.2468e-01,  4.6174e-01, -1.4936e-01, -1.1999e-01, -1.3990e-01,\n",
              "       -4.4944e-01, -2.6565e-01, -7.0061e-01,  3.0188e-01, -1.1209e-01,\n",
              "        6.6323e-01,  3.9698e-01,  6.9158e-01,  8.3442e-01, -5.2717e-01,\n",
              "       -2.5314e+00,  1.3281e-01,  3.0253e-01,  1.1062e+00,  7.2221e-03,\n",
              "        2.6031e-01,  1.1584e+00, -7.9330e-02, -7.6659e-01,  1.2623e+00,\n",
              "       -6.2071e-01,  5.9821e-01,  7.3539e-01,  3.8573e-01, -4.0293e-01,\n",
              "       -3.1440e-02,  7.7863e-01,  3.1525e-01,  1.9003e-01, -6.5821e-01,\n",
              "        4.0548e-01,  5.3596e-03,  5.5274e-02, -1.2238e+00, -4.8912e-02,\n",
              "       -3.0511e-01,  4.4473e-01, -3.3826e-01, -2.2133e-01, -1.3214e+00,\n",
              "       -6.4761e-01, -4.4021e-01, -1.4910e+00, -2.2495e-02,  6.0346e-02,\n",
              "        1.4833e-01,  4.4162e-01,  7.9787e-01, -2.8076e-01, -2.9400e-02,\n",
              "       -1.5656e-01, -1.2650e-01, -5.6968e-01,  1.5374e-03,  6.6600e-01])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "53ggzsVcLVMw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e8fad876-31c7-442e-e8de-9496b7bd4bd8"
      },
      "source": [
        "embeddings_index[\"books\"].shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(100,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5UpCJfyKM2Ff"
      },
      "source": [
        "# function to calculate the average word based sentence embedding\n",
        "def get_mean_vector( sentence):\n",
        "\n",
        "    words_list = word_tokenize(sentence)\n",
        "\n",
        "    length = 0 \n",
        "    for word in words_list:\n",
        "      if word in embeddings_index:\n",
        "        length = length + 1         # calculate length of input sentence\n",
        "\n",
        "    sentence_emb = []\n",
        "    glove_index = []\n",
        "\n",
        "    for word in words_list:    \n",
        "      if word in embeddings_index: # check if word is present in glove embedding\n",
        "        glove_index.append (embeddings_index[word])\n",
        "    \n",
        "    score = []\n",
        "\n",
        "    if (length >= 1):             # if sentence has at least one word\n",
        "      glove_index = np.array(glove_index)\n",
        "\n",
        "      for i in range(0, 100):\n",
        "        sum = 0\n",
        "        for j in range(0, length ):\n",
        "          elem = glove_index[j][i] # get value from word embedding\n",
        "          sum = sum + elem\n",
        "\n",
        "        score.append(sum)\n",
        "\n",
        "      score = np.array(score)\n",
        "    \n",
        "      for elem in score:\n",
        "        elem = float(elem/length) # divide by length of sentence to calculate average\n",
        "        sentence_emb.append(elem)\n",
        "\n",
        "      sentence_emb = np.array(sentence_emb)\n",
        "\n",
        "    else:\n",
        "      sentence_emb = np.zeros(100)\n",
        "\n",
        "    return sentence_emb"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tebtI139yCsX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3ad2ca71-3b7c-4a85-edce-6ff36761ef34"
      },
      "source": [
        "get_mean_vector(\"level prepration enough exam\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([-4.95033333e-02,  2.87232000e-01,  2.88260667e-01,  1.08866667e-01,\n",
              "       -7.86246667e-01,  2.88300000e-02, -8.65536667e-02,  4.03348333e-01,\n",
              "       -7.74343333e-02,  5.09903333e-01,  3.84406667e-01, -1.50604000e-01,\n",
              "       -2.89943333e-02, -6.84843333e-02,  1.44096000e-01, -6.28460000e-02,\n",
              "        8.63333333e-03,  3.08110000e-01, -1.05685333e-01,  7.78300000e-03,\n",
              "       -1.86950000e-01, -1.95930000e-01, -2.30813333e-01, -4.68980000e-01,\n",
              "       -1.67722333e-01, -4.91675000e-01,  9.86133333e-02, -6.97816667e-01,\n",
              "       -6.53500000e-02, -5.65783333e-01, -2.17965333e-01,  1.89290000e-01,\n",
              "       -1.85226667e-01, -1.62690000e-02,  3.29442333e-01,  2.16933333e-01,\n",
              "       -6.58396667e-01,  1.27338667e-01, -3.94846333e-01,  1.44789333e-01,\n",
              "       -4.19693333e-01, -3.94495333e-01, -2.65376667e-01, -6.77313333e-01,\n",
              "        4.69593333e-02, -6.09566667e-02,  1.75477333e-01, -4.76270000e-01,\n",
              "       -2.60960333e-01, -6.39807000e-01,  9.34433333e-02, -5.93810000e-01,\n",
              "       -1.14260000e-01,  9.26783333e-01,  1.32131933e-01, -1.97155333e+00,\n",
              "        1.25298667e-01, -5.47450000e-01,  1.75133333e+00,  1.53660000e-01,\n",
              "       -8.37163333e-02,  5.49606667e-01, -3.39906667e-01,  1.07617667e-01,\n",
              "        2.45656667e-01,  5.44200000e-02,  3.61330000e-01, -5.75166667e-02,\n",
              "        5.50569667e-01, -2.69836333e-01,  9.56733333e-02, -1.35266667e-02,\n",
              "        5.93606667e-01, -5.66666667e-02, -1.55020000e-01,  4.12226667e-01,\n",
              "       -3.26002333e-01, -1.89253333e-01, -2.00450000e-01, -5.98050000e-01,\n",
              "        6.40213333e-01,  1.93900600e-01, -4.65536667e-01, -5.79962000e-01,\n",
              "       -1.33900000e+00,  3.37800000e-02,  3.90100000e-01,  1.78673333e-01,\n",
              "       -1.03333333e-03,  2.83233333e-02, -4.40606667e-01,  1.97460000e-01,\n",
              "       -3.75638667e-01,  5.74193333e-01, -1.82416667e-01,  4.36100333e-01,\n",
              "        1.64876667e-01, -3.35940000e-01,  2.67206667e-01,  2.51666667e-02])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MiMhGNY9OFxB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1fb12c63-4207-4bf5-d242-2bd1fde22e38"
      },
      "source": [
        "vec = []\n",
        "\n",
        "for ques in (ques2_all[0:200000]): \n",
        "  val = get_mean_vector(ques ) \n",
        "  vec.append(val)\n",
        "\n",
        "# store word average sentence embedding\n",
        "average_enbedding = np.array(vec) \n",
        "average_enbedding.shape "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(200000, 100)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PwT8D654jzj"
      },
      "source": [
        "#function to caculate cosine similarity between two vectors v1 and v2\n",
        "def cosine(v1, v2):\n",
        "\n",
        "    v1 = np.array(v1)\n",
        "    v2 = np.array(v2)\n",
        "    \n",
        "    denominator = (np.sqrt(np.sum(v1**2)) * np.sqrt(np.sum(v2**2)))\n",
        "\n",
        "    if  denominator != 0:\n",
        "      cos = float ( (np.dot(v1, v2)) / denominator )\n",
        "    else:\n",
        "      cos = -1.0 # lowest val of cos theata\n",
        "\n",
        "    return cos"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "frUP8sOj4j64"
      },
      "source": [
        "# return top k document indices and scores\n",
        "def get_similarity_avg_word(input, k) :\n",
        "\n",
        "  input = clean_corpus_2(input) # preprocess input text\n",
        "\n",
        "  val_input = get_mean_vector(input)\n",
        "\n",
        "  cosine_similarities = []\n",
        "  doc_simi = {}\n",
        "\n",
        "  i = 0\n",
        "  for x in average_enbedding:\n",
        "    sim_calculation = cosine(val_input,x)\n",
        "    cosine_similarities.append(sim_calculation)\n",
        "    doc_simi[i] = sim_calculation \n",
        "    i = i + 1\n",
        "  \n",
        "  sorted_sim = sorted(doc_simi.items(), key = lambda kv:(kv[1], kv[0]),reverse = True ) # sort by value\n",
        "  topk = sorted_sim[0:k]\n",
        "\n",
        "  return   topk"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZoIA_uMP6nWi"
      },
      "source": [
        "-  For a single line of text :"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YVEu-g-XbRKE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f3d0596e-52e6-4445-9add-54c7dd7868cf"
      },
      "source": [
        "print(df['question1'][24] )\n",
        "print(df['processed_ques2_lemm'][24])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "What are the best websites for entrepreneurs?\n",
            "best website entrepreneur\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ol42d_Fq4tuA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "930eb350-641b-44b8-d1e4-1e06a030f4f0"
      },
      "source": [
        "top5 = get_similarity_avg_word(df['question1'][24] , 5 )\n",
        "print_top_k(top5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(24, 1.0),\n",
              " (12595, 0.9196509186153524),\n",
              " (78416, 0.9172698824136616),\n",
              " (134826, 0.9100231715779249),\n",
              " (95849, 0.9009992677641179)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2l5t0iFL5PVp"
      },
      "source": [
        "- For first 100:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ulmI544t5Rtw"
      },
      "source": [
        "# function to do all calculations for average word model and return accuracy\n",
        "def do_calcualtion_avg (ques1_list, k):\n",
        "\n",
        "  all_similarty = []\n",
        "  for sentence in ques1_list :\n",
        "    similarity = get_similarity_avg_word(sentence, k)\n",
        "    all_similarty.append(similarity )\n",
        "\n",
        "  list_matching_id = all_similarity_index(all_similarty)\n",
        "  accuracy = get_accuracy(list_matching_id)\n",
        "  return accuracy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ElHKppt0DrkE"
      },
      "source": [
        "- Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZKasbBH45Rxx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f994155a-4041-4f9d-e2b1-f3c0a428955d"
      },
      "source": [
        "accuracy_top2  = do_calcualtion_avg(ques1_list, 2) # top 2\n",
        "accuracy_top5  = do_calcualtion_avg(ques1_list, 5) # top 5\n",
        " \n",
        "print(\"Correctly predicted matches in top2: \", accuracy_top2)\n",
        "print(\"Correctly predicted matches in top5: \", accuracy_top5) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Correctly predicted matches in top2:  67\n",
            "Correctly predicted matches in top5:  77\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m_EN0wxrCGZf"
      },
      "source": [
        "## 4. Smooth Inverse Frequency based Sentence Embedding Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_OVN-9eNxQmj"
      },
      "source": [
        "# function to calculate weight of all words present in vocabulary\n",
        "def compute_word_weight(vocab , a): \n",
        "  w_weight = {}\n",
        "  n = 0\n",
        "\n",
        "  for key,val in vocab.items():\n",
        "    w_weight[key] = float(val)\n",
        "    n += float(val)           \n",
        "                \n",
        "  for key, val in w_weight.items():\n",
        "\n",
        "      freq = float (val/ n)                  # estimated word frequency\n",
        "      if (a + freq) != 0:\n",
        "        w_weight[key] = a / (a + freq)  # weight of word\n",
        "      else:\n",
        "        w_weight[key] = 0.00001    \n",
        "\n",
        "  return w_weight\n",
        "\n",
        "# function to calculate weight of all words in pretrained word embedding\n",
        "def compute_weight(all_words, w_weight):\n",
        "    idx_weight = {}\n",
        "\n",
        "    for word, idx in all_words.items():\n",
        "        if word in w_weight:\n",
        "            idx_weight[idx] = w_weight[word]\n",
        "        else:\n",
        "           idx_weight[idx] = 0.25  # assign a weight of 0.25  for words not in vocab but in glove\n",
        "\n",
        "    return idx_weight\n",
        "\n",
        "# function to find index of  a word w \n",
        "def find_index(all_words, w):\n",
        "    w = w.lower()\n",
        "\n",
        "    if w in all_words:\n",
        "        return all_words[w]\n",
        "    else:\n",
        "        return len(all_words) - 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BCKyy22Y7KbC"
      },
      "source": [
        "# function to get weight\n",
        "def compute_index_weight(seq, m, idx_weight):\n",
        "  \n",
        "    weight = np.zeros(seq.shape).astype('float64')\n",
        "\n",
        "    for i in range(seq.shape[0]):\n",
        "        for j in range(seq.shape[1]):\n",
        "\n",
        "            if m[i,j] > 0 and seq[i,j] >= 0:\n",
        "                weight[i,j] = idx_weight[seq[i,j]]\n",
        "\n",
        "    weight = np.asarray(weight, dtype='float64')\n",
        "    return weight\n",
        "\n",
        "def compute_text_index(sentences, all_words):\n",
        "    seq1 = []\n",
        "\n",
        "    for l in sentences:\n",
        "      l = l.split()\n",
        "      X = []\n",
        "      for w in l:\n",
        "          X.append(find_index(all_words,w))        \n",
        "      seq1.append(X)\n",
        "\n",
        "    lengths = [len(s) for s in seq1]\n",
        "    n_samples = len(seq1)\n",
        "    maxlen = np.max(lengths)\n",
        "\n",
        "    x = np.zeros((n_samples, maxlen)).astype('int64')\n",
        "    matrix = np.zeros((n_samples, maxlen)).astype('float64')\n",
        "\n",
        "    for idx, s in enumerate(seq1):\n",
        "        x[idx, :lengths[idx]] = s\n",
        "        matrix[idx, :lengths[idx]] = 1\n",
        "\n",
        "    matrix = np.asarray(matrix, dtype='float64')\n",
        "\n",
        "    return x, matrix"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8rMusTEIyzgg"
      },
      "source": [
        "glove_word_emb = glove_word_embedding           # word and word embedding vector\n",
        "words = glove_word_pos                          # position of word in glove \n",
        "\n",
        "weight_word = compute_word_weight(vocabulary , 1e-3)  # weight of a word \n",
        "idx_weight = compute_weight(words, weight_word)      # weight for the ith word"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6zi2oO3Eq05w"
      },
      "source": [
        "# returns sentence embedding for a text\n",
        "def get_sentence_embedding( text, words,glove_word_emb,idx_weight):  \n",
        "\n",
        "  x, m  = compute_text_index(text, words) \n",
        "  weight =  compute_index_weight(x, m, idx_weight) # get word weights\n",
        "\n",
        "  embed = SIF_embedding(glove_word_emb, x, weight) ## calculate SIF embedding\n",
        "  return embed\n",
        "\n",
        "# function to calculate sentence embedding\n",
        "def SIF_embedding(glove_word_emb, x, w): \n",
        "    \n",
        "    n = x.shape[0]\n",
        "\n",
        "    emb = np.zeros((n, glove_word_emb.shape[1])) #create empty embedding\n",
        "\n",
        "    for i in range(n):\n",
        "        emb[i,:] = w[i,:].dot(glove_word_emb[x[i,:],:]) / np.count_nonzero(w[i,:]) # fill embedding\n",
        "\n",
        "    SVD = TruncatedSVD(n_components = 1 , n_iter = 5, random_state=0) \n",
        "    SVD.fit(emb)          # compute SVD\n",
        "    pc = SVD.components_  # array of shape (n_components, n_features)\n",
        "\n",
        "    embedd = emb - emb.dot(pc.transpose()) .dot(pc) # remove first component\n",
        "   \n",
        "    return embedd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MN5YfnDnI3uc",
        "outputId": "2666c7f1-25a1-4a05-af5c-05426e3161c6"
      },
      "source": [
        "# sentence embedding for first 50k question\n",
        "embedding = get_sentence_embedding( (ques2_all[0:50000]) , words, glove_word_emb, idx_weight) #\n",
        "print(embedding.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(50000, 100)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lWbm_dQDDv21"
      },
      "source": [
        "- Get matches for a single Question:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fm-4PbGZ929R"
      },
      "source": [
        "# returns top k similarities\n",
        "def get_similarity_SIF(input_text, k) : #embedding ,words, We, idx_weight\n",
        "  \n",
        "  input_text  = clean_corpus_2(input_text)\n",
        "  # print(input_text)\n",
        "\n",
        "  input = get_sentence_embedding([input_text], words, glove_word_emb, idx_weight)\n",
        "  input = np.array(input)\n",
        "  input_val = input[0]\n",
        "\n",
        "  cosine_similarities = []\n",
        "  doc_simi = {}\n",
        "\n",
        "  i = 0\n",
        "  for elem in embedding:     #(elem.shape) ->(100)\n",
        "    \n",
        "    sim_calculation = cosine(input_val,elem)\n",
        "    cosine_similarities.append(sim_calculation)\n",
        "    doc_simi[i] = sim_calculation \n",
        "    i = i + 1\n",
        "  \n",
        "  sorted_sim = sorted(doc_simi.items(), key = lambda kv:(kv[1], kv[0]),reverse = True )\n",
        "  topk = sorted_sim[0:k]\n",
        " \n",
        "  return   topk"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "HDKjHEHFIwXy",
        "outputId": "a6192eb9-0559-42bf-e7e8-6899e24fc49e"
      },
      "source": [
        "df['question1'][24]# qid1 = 360448"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'What are the best websites for entrepreneurs?'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 277
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_XmOy9YWBsJr",
        "outputId": "8263bd8e-1a38-435e-b125-6f9b49e8ebfb"
      },
      "source": [
        "top5 = get_similarity_SIF(df['question1'][24], 5)  \n",
        "print_top_k(top5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " What are the best websites for entrepreneur?, Score: 0.6114882541577606\n",
            " In the Bible, Eve plucks fruit from the Tree of Knowledge of good and evil, eats it, and then gives it to Adam to eat. Then, of course, they are cast out of Eden. What did Adam and/or Eve do that was so terribly bad?, Score: 0.5075702841056475\n",
            " How do you make notes from Hindu for IAS exam?, Score: 0.5023420505430883\n",
            " Is Steam compatible with Windows 10?, Score: 0.4845010548853627\n",
            " How can you compare and contrast qualitative and quantitative data?, Score: 0.4783229437364296\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "doHdomeDeqtI",
        "outputId": "ec05d2af-38a0-4f97-aa95-c6ceb087cfdc"
      },
      "source": [
        "top5 = get_similarity_SIF(df['question1'][12], 5)  \n",
        "print_top_k(top5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " What is the Guardian newspaper, and how trustworthy is it?, Score: 0.4951831784630833\n",
            " Which is a suitable inpatient drug and alcohol rehab center in Lee County KY?, Score: 0.4878813211130379\n",
            " What is the most peaceful city in the world, where there is no threat of terrorism?, Score: 0.48280480532091413\n",
            " Can I get college from neet phase-1 if I score 480?, Score: 0.4735006965835887\n",
            " What are some good ways to take a picture of myself?, Score: 0.47036962949995065\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nPGkDA7n09Nh"
      },
      "source": [
        "- Get matches for first 100 Question 1 with is_duplicate = 1:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eW54BX1jEvmd"
      },
      "source": [
        "# perform all calculations for smooth inverse freq model\n",
        "def do_calculation_SIF(ques1_list, k):\n",
        "\n",
        "  all_similarty = []\n",
        "\n",
        "  for sentence in ques1_list :\n",
        "    similarity = get_similarity_SIF(sentence, k)\n",
        "    all_similarty.append(similarity )\n",
        "\n",
        "  list_matching_id = all_similarity_index(all_similarty)\n",
        "  accuracy = get_accuracy(list_matching_id)\n",
        "  \n",
        "  return  accuracy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fbd-TDzMklTB"
      },
      "source": [
        "- Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EnkotkbiEvtQ",
        "outputId": "1aa37e80-bad2-45d5-bee3-c0db76c23153"
      },
      "source": [
        "np.seterr(divide='ignore', invalid='ignore')\n",
        "\n",
        "top2  = do_calculation_SIF(ques1_list, 2) # top 2\n",
        "top5  = do_calculation_SIF(ques1_list, 5) # top 5\n",
        " \n",
        "print(\"Correctly predicted matches in top2: {} out of 100 \".format(top2))\n",
        "print(\"Correctly predicted matches in top5: {} out of 100 \".format(top5))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Correctly predicted matches in top2: 34 out of 100 \n",
            "Correctly predicted matches in top5: 41 out of 100 \n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}