chunk_salience_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chunk salience classification\n",
    "We have trained a deep neural network to discriminate between useful and useless chunk setting a binary classification problem. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from rouge import Rouge\n",
    "\n",
    "from transcript_utils import get_transcription, semantic_segmentation, extract_features\n",
    "\n",
    "from tensorflow import keras\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Dataset preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1. Dataset reading\n",
    "A cleaned version of the golden set is used for the training and the test of the *chunck classifier*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>episode id</th>\n",
       "      <th>transcript</th>\n",
       "      <th>best_summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>spotify:episode:4KRC1TZ28FavN3J5zLHEtQ</td>\n",
       "      <td>What's up fellas? So I got a patron supported...</td>\n",
       "      <td>All right guys now as y'all guys might know so...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>spotify:episode:4tdDQcsBOUVWnA9XrpgTzS</td>\n",
       "      <td>If you are bored you are boring.  One of my ki...</td>\n",
       "      <td>It was the first and last time I ever said tha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spotify:episode:626YAxomH0HZ6nCW9NLlGY</td>\n",
       "      <td>Visit Larisa English club.com English everyday...</td>\n",
       "      <td>Prepositions of movement review two is the sec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>spotify:episode:6AUFl7KQWN6pzGFEIEKFQu</td>\n",
       "      <td>So so and salutations Summers and welcome to t...</td>\n",
       "      <td>It only seems fitting to walk you through a fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>spotify:episode:6IDbemwG5t6XMlctbqcna7</td>\n",
       "      <td>Hi everyone. This is Justin from a liquidy pla...</td>\n",
       "      <td>This week on Nothing But A Bob Thang, Nathan a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               episode id  \\\n",
       "0  spotify:episode:4KRC1TZ28FavN3J5zLHEtQ   \n",
       "1  spotify:episode:4tdDQcsBOUVWnA9XrpgTzS   \n",
       "2  spotify:episode:626YAxomH0HZ6nCW9NLlGY   \n",
       "3  spotify:episode:6AUFl7KQWN6pzGFEIEKFQu   \n",
       "4  spotify:episode:6IDbemwG5t6XMlctbqcna7   \n",
       "\n",
       "                                          transcript  \\\n",
       "0   What's up fellas? So I got a patron supported...   \n",
       "1  If you are bored you are boring.  One of my ki...   \n",
       "2  Visit Larisa English club.com English everyday...   \n",
       "3  So so and salutations Summers and welcome to t...   \n",
       "4  Hi everyone. This is Justin from a liquidy pla...   \n",
       "\n",
       "                                        best_summary  \n",
       "0  All right guys now as y'all guys might know so...  \n",
       "1  It was the first and last time I ever said tha...  \n",
       "2  Prepositions of movement review two is the sec...  \n",
       "3  It only seems fitting to walk you through a fe...  \n",
       "4  This week on Nothing But A Bob Thang, Nathan a...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# loading the dataset from the csv file\n",
    "dataset_path = os.path.join(os.path.abspath(\"\"), 'podcasts-no-audio-13GB')\n",
    "dataset = pd.read_csv(os.path.join(dataset_path, \"gold_set_cleaned.tsv\"), sep='\\t')\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 Ground truth targets creation\n",
    "In order to create the *ground truth targets* for the classifier,  we compare the chunk with the corresponding summary of the transcript it belongs to and, if the score obtained with a certain metric is below a threshold (strictly coupled with the metric), the chunk is not taken into account as a part of the transcript."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def isChunkUseful(chunk, summary, metric, threshold, verbose=False):\n",
    "    \"\"\"\n",
    "    Function to check if a chunk is useful or not\n",
    "\n",
    "    Parameters:\n",
    "        - chunk: part of the transcript\n",
    "        - summary: summary of a transcript\n",
    "        - metric: function of ariety 2 (chunk, summary) used to evaluate the summary\n",
    "        - threshold: value used to decide whether chunk is a good summary or not\n",
    "    Returns:\n",
    "        - True if the chunk is a good summary, False otherwise\n",
    "    \"\"\"\n",
    "    score = metric(chunk, summary)\n",
    "    if verbose: print(f\"\\tChunck: {chunk}\\n\\tSummary: {summary}\\n\\tScore: {score}\")\n",
    "\n",
    "    if score < threshold:\n",
    "        result = False\n",
    "    else:\n",
    "        result = True\n",
    "    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The chosen metric to compare the chuck with the description is ROUGE-L f1-score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rouge_score(candidate, reference, type='rouge-l', metric='f'):\n",
    "    \"\"\"\n",
    "    ROUGE score\n",
    "    Parameters:\n",
    "        reference: reference text\n",
    "        candidate: candidate text\n",
    "        type: type of ROUGE, it can be rouge-1, rouge-2, rouge-l (default)\n",
    "        metric: precision (p), recall (r) or f-score (f) (default)\n",
    "    \"\"\"\n",
    "    rouge = Rouge()\n",
    "    scores = rouge.get_scores(candidate, reference)\n",
    "    return scores[0][type][metric]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3 Extracting the input and target data for the chunck classifier\n",
    "We have all the tools to create the ground truth targets for the chunk selection classifier. \n",
    "The following code creates the dataset to train the chunk classifier:\n",
    "- the input features are the chunk encoding with a sentence transformer\n",
    "- the ground truth targets are created as aforementioned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting features and targets: 100%|██████████| 141/141 [1:04:23<00:00, 27.40s/it]\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.20\n",
    "metric = rouge_score\n",
    "verbose = False\n",
    "\n",
    "# creation of the dataset for chunk classification\n",
    "# creation of the targets\n",
    "\n",
    "features = []\n",
    "targets = []\n",
    "\n",
    "# initalize the model for the sentence transformer\n",
    "sentence_encoder = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "\n",
    "for i in tqdm(range(len(dataset)), desc=\"Extracting features and targets\"):\n",
    "    if verbose: print(f\"Episode: {i}\")\n",
    "    chunks = semantic_segmentation(dataset.transcript[i], sentence_encoder)\n",
    "    description = dataset.best_summary[i]\n",
    "\n",
    "    num_chunks = len(chunks)\n",
    "    if verbose: print(f\"Num chunks: {num_chunks}\")\n",
    "\n",
    "    for j in range(num_chunks):\n",
    "        if verbose: print(f\"\\tChunk {j}\")\n",
    "        features.append(extract_features(chunks[j], sentence_encoder))\n",
    "        if isChunkUseful(' '.join(chunks[j]), description, metric, threshold, verbose):\n",
    "            targets.append(1)\n",
    "        else:\n",
    "            targets.append(0)\n",
    "\n",
    "y = np.array(targets)\n",
    "y = y.reshape(y.shape[0], 1)\n",
    "X = np.array(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of useful chunks: 9.119270458363331%\n",
      "Percentage of unuseful chunks: 90.88072954163667%\n"
     ]
    }
   ],
   "source": [
    "# show the percentage of useful and unuseful chunks\n",
    "positive = y[y==1].shape[0]\n",
    "negative = y.shape[0] - positive\n",
    "print(f\"Percentage of useful chunks: {positive/(positive+negative)*100}%\")\n",
    "print(f\"Percentage of unuseful chunks: {negative/(positive+negative)*100}%\")\n",
    "\n",
    "# store chunk classification dataset\n",
    "chunck_classification_dataset = np.hstack((X, y))\n",
    "df_chunk = pd.DataFrame(chunck_classification_dataset)\n",
    "df_chunk.to_csv(os.path.join(dataset_path, \"chunk_classification_dataset.csv\"), header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Training the chunk classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load brass set\n",
    "chunck_classification_dataset = pd.read_csv(os.path.join(dataset_path, \"chunk_classification_dataset.csv\"), header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The dataset contains 384 features and 1 target\n",
    "y = chunck_classification_dataset.iloc[:,-1]\n",
    "X = chunck_classification_dataset.drop(chunck_classification_dataset.columns[[-1]], axis=1)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_positive = X_train[y_train>0]\n",
    "X_train_negative = X_train[y_train==0][:X_train_positive.shape[0]]\n",
    "y_train_positive = y_train[y_train>0]\n",
    "y_train_negative = y_train[y_train==0][:X_train_positive.shape[0]]\n",
    "\n",
    "X_train = np.vstack((X_train_positive,X_train_negative))\n",
    "y_train = np.hstack((y_train_positive, y_train_negative))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "54/54 [==============================] - 5s 23ms/step - loss: 2.9166 - precision: 0.5887 - recall: 0.9940 - val_loss: 1.7817 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 2/15\n",
      "54/54 [==============================] - 1s 16ms/step - loss: 1.0672 - precision: 0.5945 - recall: 0.9980 - val_loss: 1.0054 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 3/15\n",
      "54/54 [==============================] - 1s 16ms/step - loss: 0.7017 - precision: 0.6586 - recall: 0.8663 - val_loss: 0.9175 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 4/15\n",
      "54/54 [==============================] - 1s 14ms/step - loss: 0.6111 - precision: 0.7138 - recall: 0.8762 - val_loss: 1.0623 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 5/15\n",
      "54/54 [==============================] - 1s 16ms/step - loss: 0.5429 - precision: 0.7792 - recall: 0.8523 - val_loss: 0.6658 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 6/15\n",
      "54/54 [==============================] - 1s 15ms/step - loss: 0.4827 - precision: 0.8566 - recall: 0.8343 - val_loss: 0.9282 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 7/15\n",
      "54/54 [==============================] - 1s 16ms/step - loss: 0.3777 - precision: 0.8976 - recall: 0.8922 - val_loss: 1.2817 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 8/15\n",
      "54/54 [==============================] - 1s 15ms/step - loss: 0.2797 - precision: 0.9435 - recall: 0.9341 - val_loss: 1.0872 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 9/15\n",
      "54/54 [==============================] - 1s 15ms/step - loss: 0.1963 - precision: 0.9720 - recall: 0.9701 - val_loss: 1.1327 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 10/15\n",
      "54/54 [==============================] - 1s 16ms/step - loss: 0.1736 - precision: 0.9779 - recall: 0.9721 - val_loss: 2.5165 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 11/15\n",
      "54/54 [==============================] - 1s 12ms/step - loss: 0.1436 - precision: 0.9940 - recall: 0.9860 - val_loss: 1.5577 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 12/15\n",
      "54/54 [==============================] - 1s 16ms/step - loss: 0.1121 - precision: 1.0000 - recall: 0.9960 - val_loss: 1.4649 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 13/15\n",
      "54/54 [==============================] - 1s 16ms/step - loss: 0.1043 - precision: 1.0000 - recall: 0.9980 - val_loss: 1.9459 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 14/15\n",
      "54/54 [==============================] - 1s 16ms/step - loss: 0.0923 - precision: 1.0000 - recall: 1.0000 - val_loss: 1.8303 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "Epoch 15/15\n",
      "54/54 [==============================] - 1s 15ms/step - loss: 0.0855 - precision: 1.0000 - recall: 1.0000 - val_loss: 1.6726 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00\n",
      "INFO:tensorflow:Assets written to: modelChunkNN\\assets\n",
      "Accuracy: 0.5710650672482733\n",
      "Precision: [0.96262341 0.15453863]\n",
      "Recall: [0.54775281 0.7953668 ]\n"
     ]
    }
   ],
   "source": [
    "# Neural Network for chunk classification\n",
    "\n",
    "inputs = keras.Input(shape=(384))\n",
    "x = keras.layers.Dense(512, activation='relu')(inputs)\n",
    "x = keras.layers.Dense(256, activation='relu')(x)\n",
    "x = keras.layers.Dropout(0.4)(x)\n",
    "x = keras.layers.Dense(256, activation='relu', kernel_regularizer='l2')(x)\n",
    "x = keras.layers.Dropout(0.4)(x)\n",
    "x = keras.layers.Dense(128, activation='relu', kernel_regularizer='l2')(x)\n",
    "output = keras.layers.Dense(1, activation='sigmoid', kernel_regularizer='l2')(x)\n",
    "model = keras.Model(inputs, output)\n",
    "\n",
    "model.compile(\n",
    "    optimizer='Adam',\n",
    "    loss='binary_crossentropy',\n",
    "    metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()]\n",
    ")\n",
    "\n",
    "history = model.fit(\n",
    "    X_train,\n",
    "    y_train,\n",
    "    batch_size=16,\n",
    "    epochs=15,\n",
    "    validation_split=0.15,\n",
    "    validation_data=(X_test,y_test),\n",
    "    callbacks=[keras.callbacks.EarlyStopping(monitor='loss', patience=3)]\n",
    ")\n",
    "\n",
    "model.save(\"modelChunkNN\")\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "y_pred = [1 if y>0.5 else 0 for y in y_pred]\n",
    "accuracy = accuracy_score(y_test,y_pred)\n",
    "print(f\"Accuracy: {accuracy}\")\n",
    "print(f\"Precision: {precision_score(y_test, y_pred, average=None)}\")\n",
    "print(f\"Recall: {recall_score(y_test, y_pred, average=None)}\")"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "e800dd11dddfb1e3886769e91ed8bbe987a221798b85010fca298ae8afbc389e"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('nlp')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}