topic_classification_BERT.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "9SaUupOE8kl9"
      },
      "source": [
        "# Arash Hajian nezhad | BERT fine-tuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VXSVDC5N8vgG"
      },
      "source": [
        "#### Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ymC_gFoYF2LZ"
      },
      "outputs": [],
      "source": [
        "# %%capture\n",
        "# !pip install transformers==4.28.0"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Fy4PBvo82nl"
      },
      "source": [
        "#### Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "ziEqpTPUF6cR"
      },
      "outputs": [],
      "source": [
        "import ast\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import torch\n",
        "from torch.utils.data import Dataset\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import f1_score, roc_auc_score, accuracy_score\n",
        "\n",
        "from transformers import AutoModelForSequenceClassification, AutoTokenizer, logging, TrainingArguments, Trainer, EvalPrediction\n",
        "logging.set_verbosity_error()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1QGfdK-v8-IQ"
      },
      "source": [
        "#### Loading training dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "2lzqe85JGM0d",
        "outputId": "7ba44635-97b2-4f6b-f153-2394eb18d7b9"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-4f1a4f68-f3a3-4e4b-921e-5f2380d9255d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>body</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>hello and welcome to BBC News a woman who gave...</td>\n",
              "      <td>[1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>news now out of North Hollywood. A 14 yearold ...</td>\n",
              "      <td>[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>homelessness his city's greatest failure. That...</td>\n",
              "      <td>[1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Minneapolis police officer Kim Potter guilty o...</td>\n",
              "      <td>[1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Judy an update now to the wildfires that wiped...</td>\n",
              "      <td>[0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4f1a4f68-f3a3-4e4b-921e-5f2380d9255d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4f1a4f68-f3a3-4e4b-921e-5f2380d9255d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4f1a4f68-f3a3-4e4b-921e-5f2380d9255d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                body  \\\n",
              "0  hello and welcome to BBC News a woman who gave...   \n",
              "1  news now out of North Hollywood. A 14 yearold ...   \n",
              "2  homelessness his city's greatest failure. That...   \n",
              "3  Minneapolis police officer Kim Potter guilty o...   \n",
              "4  Judy an update now to the wildfires that wiped...   \n",
              "\n",
              "                                          labels  \n",
              "0  [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]  \n",
              "1  [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]  \n",
              "2  [1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]  \n",
              "3  [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]  \n",
              "4  [0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]  "
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df = pd.read_csv('data/processed/stories_processed.csv')\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vEpY9Ax99C4V"
      },
      "source": [
        "#### Dataset class definition"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "mqeYBG_MGOEw"
      },
      "outputs": [],
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "\n",
        "\n",
        "class StoriesDataset(Dataset):\n",
        "    def __init__(self, df):\n",
        "        super().__init__()\n",
        "\n",
        "        self.data = [tokenizer(text, padding='max_length', max_length=512, truncation=True, return_tensors='pt') for text in df['body']]\n",
        "        labels = [torch.tensor(ast.literal_eval(label)).float() for label in df['labels'].values]\n",
        "\n",
        "        for i, data_point in enumerate(self.data):\n",
        "            for key in data_point:\n",
        "                self.data[i][key] = self.data[i][key].squeeze(0)  # squeezing the extra first dimension in input_ids, mask_ids and attention tensors\n",
        "\n",
        "            self.data[i]['labels'] = labels[i]  # adding the labels\n",
        "\n",
        "        self.__size = len(self.data)\n",
        "    \n",
        "    def __len__(self):\n",
        "        return self.__size\n",
        "    \n",
        "    def __getitem__(self, idx):\n",
        "        return self.data[idx]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GVzYSGXD9F5V"
      },
      "source": [
        "#### Saving topics to labels mappings and vice versa"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "ZTmP8Y0IGcsf"
      },
      "outputs": [],
      "source": [
        "stories = pd.read_csv('data/processed/stories.csv')\n",
        "\n",
        "stories = stories[stories['body'] != ' '].reset_index().drop(['index'], axis=1)\n",
        "\n",
        "topics = []\n",
        "for i in stories['topic']:\n",
        "    current_topics = ast.literal_eval(i)\n",
        "    for topic in current_topics:\n",
        "        if topic not in topics:\n",
        "            topics.append(topic)\n",
        "\n",
        "\n",
        "topic_to_label = {topic: label for label, topic in enumerate(topics)}\n",
        "label_to_topic = {label: topic for label, topic in enumerate(topics)}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NCCgwzfU9QE4"
      },
      "source": [
        "#### Preparing model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "fFNwQyZJHOLs"
      },
      "outputs": [],
      "source": [
        "model = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased',\n",
        "                                                           problem_type='multi_label_classification',\n",
        "                                                           num_labels=15,\n",
        "                                                           id2label=topic_to_label,\n",
        "                                                           label2id=label_to_topic,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pQOusPUt9SYJ"
      },
      "source": [
        "#### Defining some training metric for Train API usage"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "j2nVXrnFHvKI"
      },
      "outputs": [],
      "source": [
        "def multi_label_metrics(predictions, labels, sigmoid_threshold=0.5):\n",
        "    \"\"\"\n",
        "    Function for computing metrics for usage in Trainer API.\n",
        "\n",
        "    Args:\n",
        "        predictions: model's output.\n",
        "        labels: true labels of the input data.\n",
        "        sigmoid_threshold: a `float` object representing the threshold which the label\n",
        "                           would be counted as `1` after applying `sigmoid`.\n",
        "    \n",
        "    Returns:\n",
        "        a dictionary containing some training metrics.\n",
        "    \"\"\"\n",
        "    # applying sigmoid on model's predictions\n",
        "    probabilities = torch.Tensor(predictions).sigmoid()\n",
        "\n",
        "    # applying threshold\n",
        "    y_pred = np.zeros(probabilities.shape)\n",
        "    y_pred[np.where(probabilities >= sigmoid_threshold)] = 1\n",
        "\n",
        "    # compute metrics\n",
        "    y_true = labels\n",
        "    f1_micro_average = f1_score(y_true=y_true, y_pred=y_pred, average='micro')\n",
        "    roc_auc = roc_auc_score(y_true, y_pred, average='micro')\n",
        "    accuracy = accuracy_score(y_true, y_pred)\n",
        "\n",
        "    # return the results as a dictionary\n",
        "    metrics = {'f1': f1_micro_average,\n",
        "               'roc_auc': roc_auc,\n",
        "               'accuracy': accuracy,\n",
        "    }\n",
        "\n",
        "    return metrics\n",
        "\n",
        "\n",
        "def compute_metrics(p: EvalPrediction):\n",
        "    \"\"\"\n",
        "    Function for fetching the training metrics.\n",
        "\n",
        "    Args:\n",
        "        p: an `EvalPrediction` object related to Trainer API.\n",
        "    \n",
        "    Returns:\n",
        "        some training metrics.\n",
        "    \"\"\"\n",
        "    predictions = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions\n",
        "    metrics = multi_label_metrics(\n",
        "        predictions=predictions, \n",
        "        labels=p.label_ids,\n",
        "    )\n",
        "\n",
        "    return metrics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6uxc7uRt9Xum"
      },
      "source": [
        "#### Splitting dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "y1RDdxgpktRX"
      },
      "outputs": [],
      "source": [
        "df_train, df_valid = train_test_split(df, train_size=0.85, shuffle=True)\n",
        "BATCH_SIZE = 16"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aKzqmtVf9cS0"
      },
      "source": [
        "#### Training arguments for Trainer API"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "BCP21XnnMZTp"
      },
      "outputs": [],
      "source": [
        "trainer_args = TrainingArguments(\n",
        "    f'bert-finetuned-stories-topic',\n",
        "    evaluation_strategy = 'epoch',\n",
        "    save_strategy = 'epoch',\n",
        "    learning_rate=2e-5,\n",
        "    per_device_train_batch_size=BATCH_SIZE,\n",
        "    per_device_eval_batch_size=BATCH_SIZE // 2,\n",
        "    num_train_epochs=5,\n",
        "    weight_decay=0.01,\n",
        "    load_best_model_at_end=True,\n",
        "    metric_for_best_model='f1',\n",
        "    push_to_hub=False,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Er2r6_9H9e6M"
      },
      "source": [
        "#### Preparing the final Trainer API"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "UJpjlXiwMubJ"
      },
      "outputs": [],
      "source": [
        "trainer = Trainer(\n",
        "    model,\n",
        "    trainer_args,\n",
        "    train_dataset=StoriesDataset(df_train),\n",
        "    eval_dataset=StoriesDataset(df_valid),\n",
        "    tokenizer=tokenizer,\n",
        "    compute_metrics=compute_metrics\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YWQtVhtd9hiq"
      },
      "source": [
        "#### Training the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kF0s_HzSmObH",
        "outputId": "3ca34dba-2b10-4380-881a-57a5301da8db"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'eval_loss': 0.19203034043312073, 'eval_f1': 0.5804962492787074, 'eval_roc_auc': 0.7140161528285432, 'eval_accuracy': 0.31953428201811124, 'eval_runtime': 24.3479, 'eval_samples_per_second': 31.748, 'eval_steps_per_second': 3.984, 'epoch': 1.0}\n",
            "{'loss': 0.2355, 'learning_rate': 1.27007299270073e-05, 'epoch': 1.82}\n",
            "{'eval_loss': 0.1526438593864441, 'eval_f1': 0.6862231534834992, 'eval_roc_auc': 0.7786105620691107, 'eval_accuracy': 0.4320827943078913, 'eval_runtime': 24.2918, 'eval_samples_per_second': 31.821, 'eval_steps_per_second': 3.993, 'epoch': 2.0}\n",
            "{'eval_loss': 0.13900378346443176, 'eval_f1': 0.7315136476426798, 'eval_roc_auc': 0.8129281547006817, 'eval_accuracy': 0.48124191461837, 'eval_runtime': 24.7301, 'eval_samples_per_second': 31.258, 'eval_steps_per_second': 3.922, 'epoch': 3.0}\n",
            "{'loss': 0.1361, 'learning_rate': 5.401459854014599e-06, 'epoch': 3.65}\n",
            "{'eval_loss': 0.13401706516742706, 'eval_f1': 0.7452322738386309, 'eval_roc_auc': 0.8235018169487525, 'eval_accuracy': 0.49547218628719275, 'eval_runtime': 24.5319, 'eval_samples_per_second': 31.51, 'eval_steps_per_second': 3.954, 'epoch': 4.0}\n",
            "{'eval_loss': 0.13227041065692902, 'eval_f1': 0.7476908118619349, 'eval_roc_auc': 0.8262900120687959, 'eval_accuracy': 0.49288486416558863, 'eval_runtime': 24.518, 'eval_samples_per_second': 31.528, 'eval_steps_per_second': 3.956, 'epoch': 5.0}\n",
            "{'train_runtime': 2128.8787, 'train_samples_per_second': 10.285, 'train_steps_per_second': 0.644, 'train_loss': 0.16619258797081718, 'epoch': 5.0}\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "TrainOutput(global_step=1370, training_loss=0.16619258797081718, metrics={'train_runtime': 2128.8787, 'train_samples_per_second': 10.285, 'train_steps_per_second': 0.644, 'train_loss': 0.16619258797081718, 'epoch': 5.0})"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "trainer.train()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y1yNvTkg9kG3"
      },
      "source": [
        "#### Functions for applying the predictions on the final dataframe\n",
        "The output of the model is logits of shape `batch_size x 15`. Here we are feeding the model only one text at a time, hence the batch size would be 1, which we need to eliminate this redundant shape. We do this using PyTorch's `squeeze` method, which does exactly that. We must also take into account that the models outputs must be between 0 and 1, so we apply sigmoid function to do just that. (this has already been applied at the train time using `BCEWithLogitsLoss`)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "W2nc96Glvy2K"
      },
      "outputs": [],
      "source": [
        "def get_label_ids(logits: torch.Tensor, sigmoid_threshold: float = 0.5) -> list:\n",
        "    \"\"\"\n",
        "    Function for getting transforming the model's output to its respective\n",
        "    topic ids.\n",
        "\n",
        "    Args:\n",
        "        logits: a `torch.Tensor` object which is the output of the model.\n",
        "        sigmoid_threshold: a `float` object representing the threshold which the label\n",
        "                           would be counted as `1` after applying `sigmoid`.\n",
        "    \n",
        "    Returns:\n",
        "        a list of topic ids based on the model's output and sigmoid threshold.\n",
        "    \"\"\"\n",
        "    probabilities = logits.squeeze().cpu().sigmoid()\n",
        "    predictions = np.zeros(probabilities.shape)\n",
        "    predictions[np.where(probabilities > sigmoid_threshold)] = 1\n",
        "\n",
        "    predicted_labels = str([str(label_to_topic[i]) for i, label in enumerate(predictions) if label == 1.0])\n",
        "    return predicted_labels\n",
        "\n",
        "\n",
        "def generate_topic(row: pd.DataFrame, sigmoid_threshold: float = 0.5) -> pd.DataFrame:\n",
        "    \"\"\"\n",
        "    Function for applying the prediction of topics of the text bodies.\n",
        "\n",
        "    Args:\n",
        "        row: a row of a pandas dataframe.\n",
        "        sigmoid_threshold: a `float` object representing the threshold which the label\n",
        "                           would be counted as `1` after applying `sigmoid`.\n",
        "    \n",
        "    Returns:\n",
        "        a row of a pandas dataframe containing a list of topics.\n",
        "    \"\"\"\n",
        "    text = row['body']\n",
        "    \n",
        "    encoding = tokenizer(text, padding='max_length', max_length=512, truncation=True, return_tensors='pt')\n",
        "    encoding = {key: value.to(trainer.model.device) for key, value in encoding.items()}\n",
        "\n",
        "    model_output = model(**encoding).logits\n",
        "\n",
        "    row['topic'] = get_label_ids(model_output, sigmoid_threshold=sigmoid_threshold)\n",
        "\n",
        "    return row"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vu8iSun19q9I"
      },
      "source": [
        "#### Loading the final dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "KDggIsatycZQ",
        "outputId": "aff1ab15-ca5e-49f8-ad93-220709593973"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-c793c845-4ce8-4711-83bb-4c1b7fcd559a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>first_words</th>\n",
              "      <th>last_words</th>\n",
              "      <th>source_video_id</th>\n",
              "      <th>body</th>\n",
              "      <th>start</th>\n",
              "      <th>end</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Well knew. This morning police need your help</td>\n",
              "      <td>gunpoint, beating him and stealing his cell ph...</td>\n",
              "      <td>18246</td>\n",
              "      <td>Well knew. This morning police need your help ...</td>\n",
              "      <td>464928</td>\n",
              "      <td>505910</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>a call. San Francisco firefighters rescued a man</td>\n",
              "      <td>all the way down to the ocean.</td>\n",
              "      <td>12387</td>\n",
              "      <td>a call. San Francisco firefighters rescued a m...</td>\n",
              "      <td>359020</td>\n",
              "      <td>385526</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Paul. Meanwhile, the state set a record in</td>\n",
              "      <td>night through conservation, some 4000 conserva...</td>\n",
              "      <td>16859</td>\n",
              "      <td>Paul. Meanwhile, the state set a record in ene...</td>\n",
              "      <td>60704</td>\n",
              "      <td>101238</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Emergency crews in Florida continue to search for</td>\n",
              "      <td>in Florida to more than 850,000 homes.</td>\n",
              "      <td>18246</td>\n",
              "      <td>Emergency crews in Florida continue to search ...</td>\n",
              "      <td>505290</td>\n",
              "      <td>534958</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>But even though the state never ordered rolling</td>\n",
              "      <td>feel since their power got cut out needlessly.</td>\n",
              "      <td>16859</td>\n",
              "      <td>But even though the state never ordered rollin...</td>\n",
              "      <td>100910</td>\n",
              "      <td>283606</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c793c845-4ce8-4711-83bb-4c1b7fcd559a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c793c845-4ce8-4711-83bb-4c1b7fcd559a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c793c845-4ce8-4711-83bb-4c1b7fcd559a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                         first_words  \\\n",
              "0      Well knew. This morning police need your help   \n",
              "1   a call. San Francisco firefighters rescued a man   \n",
              "2         Paul. Meanwhile, the state set a record in   \n",
              "3  Emergency crews in Florida continue to search for   \n",
              "4    But even though the state never ordered rolling   \n",
              "\n",
              "                                          last_words  source_video_id  \\\n",
              "0  gunpoint, beating him and stealing his cell ph...            18246   \n",
              "1                     all the way down to the ocean.            12387   \n",
              "2  night through conservation, some 4000 conserva...            16859   \n",
              "3             in Florida to more than 850,000 homes.            18246   \n",
              "4     feel since their power got cut out needlessly.            16859   \n",
              "\n",
              "                                                body   start     end  \n",
              "0  Well knew. This morning police need your help ...  464928  505910  \n",
              "1  a call. San Francisco firefighters rescued a m...  359020  385526  \n",
              "2  Paul. Meanwhile, the state set a record in ene...   60704  101238  \n",
              "3  Emergency crews in Florida continue to search ...  505290  534958  \n",
              "4  But even though the state never ordered rollin...  100910  283606  "
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "final = pd.read_csv('data/processed/to_fill_proccessed.csv')\n",
        "final.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cf8Px_4x9s7a"
      },
      "source": [
        "#### Applying the predictions and checking it out"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 970
        },
        "id": "RajYbzeVykOT",
        "outputId": "3186a755-d94f-4c14-859c-92f7eeef8eaa"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-1c21a1e2-a4c2-40f6-b3c2-205403b4f350\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>first_words</th>\n",
              "      <th>last_words</th>\n",
              "      <th>source_video_id</th>\n",
              "      <th>body</th>\n",
              "      <th>start</th>\n",
              "      <th>end</th>\n",
              "      <th>topic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Well knew. This morning police need your help</td>\n",
              "      <td>gunpoint, beating him and stealing his cell ph...</td>\n",
              "      <td>18246</td>\n",
              "      <td>Well knew. This morning police need your help ...</td>\n",
              "      <td>464928</td>\n",
              "      <td>505910</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>a call. San Francisco firefighters rescued a man</td>\n",
              "      <td>all the way down to the ocean.</td>\n",
              "      <td>12387</td>\n",
              "      <td>a call. San Francisco firefighters rescued a m...</td>\n",
              "      <td>359020</td>\n",
              "      <td>385526</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Paul. Meanwhile, the state set a record in</td>\n",
              "      <td>night through conservation, some 4000 conserva...</td>\n",
              "      <td>16859</td>\n",
              "      <td>Paul. Meanwhile, the state set a record in ene...</td>\n",
              "      <td>60704</td>\n",
              "      <td>101238</td>\n",
              "      <td>['83a09c6b-5f2f-421f-ae50-b38acca7e008']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Emergency crews in Florida continue to search for</td>\n",
              "      <td>in Florida to more than 850,000 homes.</td>\n",
              "      <td>18246</td>\n",
              "      <td>Emergency crews in Florida continue to search ...</td>\n",
              "      <td>505290</td>\n",
              "      <td>534958</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913', '9a06...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>But even though the state never ordered rolling</td>\n",
              "      <td>feel since their power got cut out needlessly.</td>\n",
              "      <td>16859</td>\n",
              "      <td>But even though the state never ordered rollin...</td>\n",
              "      <td>100910</td>\n",
              "      <td>283606</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>aid. And today, president Joe Biden and first</td>\n",
              "      <td>to view the destruction caused by Hurricane Ian.</td>\n",
              "      <td>18246</td>\n",
              "      <td>aid. And today, president Joe Biden and first ...</td>\n",
              "      <td>546494</td>\n",
              "      <td>593818</td>\n",
              "      <td>['83a09c6b-5f2f-421f-ae50-b38acca7e008']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>In the last month, there have been numerous</td>\n",
              "      <td>are necessary to crack down on those hackers.</td>\n",
              "      <td>18246</td>\n",
              "      <td>In the last month, there have been numerous da...</td>\n",
              "      <td>614910</td>\n",
              "      <td>700574</td>\n",
              "      <td>['83a09c6b-5f2f-421f-ae50-b38acca7e008', '9ff5...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>and the warriors are playing the Boston Celtics</td>\n",
              "      <td>that. We'll see if they get it tonight.</td>\n",
              "      <td>12387</td>\n",
              "      <td>and the warriors are playing the Boston Celtic...</td>\n",
              "      <td>419994</td>\n",
              "      <td>649502</td>\n",
              "      <td>['b49207eb-96eb-4b73-b534-adc0ef85022a']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>And San Leandro police searching for the person</td>\n",
              "      <td>footage to try to piece together more informat...</td>\n",
              "      <td>16859</td>\n",
              "      <td>And San Leandro police searching for the perso...</td>\n",
              "      <td>578612</td>\n",
              "      <td>619638</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>The updated Bivalent Coronavirus booster shot ...</td>\n",
              "      <td>on their vaccinations getting severe illness f...</td>\n",
              "      <td>16859</td>\n",
              "      <td>The updated Bivalent Coronavirus booster shot ...</td>\n",
              "      <td>619310</td>\n",
              "      <td>660654</td>\n",
              "      <td>['96326734-fd82-4350-b45c-513e7eb9147c']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>temperatures tonight, only back down to the un...</td>\n",
              "      <td>stuff. Stay hydrated, take breaks. Don't overd...</td>\n",
              "      <td>16859</td>\n",
              "      <td>temperatures tonight, only back down to the un...</td>\n",
              "      <td>1127008</td>\n",
              "      <td>1208198</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>And National Guard teams are also on the</td>\n",
              "      <td>north and South Carolina. Fasttracking federal...</td>\n",
              "      <td>18246</td>\n",
              "      <td>And National Guard teams are also on the groun...</td>\n",
              "      <td>534674</td>\n",
              "      <td>548314</td>\n",
              "      <td>['83a09c6b-5f2f-421f-ae50-b38acca7e008']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>A man died after falling from an escalator</td>\n",
              "      <td>name or what led to that fall.</td>\n",
              "      <td>18246</td>\n",
              "      <td>A man died after falling from an escalator aft...</td>\n",
              "      <td>593490</td>\n",
              "      <td>615274</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>All right. Well happening today. Friends and f...</td>\n",
              "      <td>Meanwhile, the suspected killer remains in the...</td>\n",
              "      <td>18246</td>\n",
              "      <td>All right. Well happening today. Friends and f...</td>\n",
              "      <td>218170</td>\n",
              "      <td>345758</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>detectives are searching for six people they say</td>\n",
              "      <td>with any information is asked to call police.</td>\n",
              "      <td>18246</td>\n",
              "      <td>detectives are searching for six people they s...</td>\n",
              "      <td>350830</td>\n",
              "      <td>459442</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>San Jose police say a security guard trying</td>\n",
              "      <td>not released yet by the Santa Clara County</td>\n",
              "      <td>12387</td>\n",
              "      <td>San Jose police say a security guard trying to...</td>\n",
              "      <td>98130</td>\n",
              "      <td>276282</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>San Jose police just announcing today an arrest</td>\n",
              "      <td>not reveal a motive for the deadly shooting.</td>\n",
              "      <td>12387</td>\n",
              "      <td>San Jose police just announcing today an arres...</td>\n",
              "      <td>283804</td>\n",
              "      <td>330978</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>shooting. Surveillance video shows how a car t...</td>\n",
              "      <td>this case, please give them a call.</td>\n",
              "      <td>12387</td>\n",
              "      <td>shooting. Surveillance video shows how a car t...</td>\n",
              "      <td>329682</td>\n",
              "      <td>360102</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1c21a1e2-a4c2-40f6-b3c2-205403b4f350')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1c21a1e2-a4c2-40f6-b3c2-205403b4f350 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1c21a1e2-a4c2-40f6-b3c2-205403b4f350');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                          first_words  \\\n",
              "0       Well knew. This morning police need your help   \n",
              "1    a call. San Francisco firefighters rescued a man   \n",
              "2          Paul. Meanwhile, the state set a record in   \n",
              "3   Emergency crews in Florida continue to search for   \n",
              "4     But even though the state never ordered rolling   \n",
              "5       aid. And today, president Joe Biden and first   \n",
              "6         In the last month, there have been numerous   \n",
              "7     and the warriors are playing the Boston Celtics   \n",
              "8     And San Leandro police searching for the person   \n",
              "9   The updated Bivalent Coronavirus booster shot ...   \n",
              "10  temperatures tonight, only back down to the un...   \n",
              "11           And National Guard teams are also on the   \n",
              "12         A man died after falling from an escalator   \n",
              "13  All right. Well happening today. Friends and f...   \n",
              "14   detectives are searching for six people they say   \n",
              "15        San Jose police say a security guard trying   \n",
              "16    San Jose police just announcing today an arrest   \n",
              "17  shooting. Surveillance video shows how a car t...   \n",
              "\n",
              "                                           last_words  source_video_id  \\\n",
              "0   gunpoint, beating him and stealing his cell ph...            18246   \n",
              "1                      all the way down to the ocean.            12387   \n",
              "2   night through conservation, some 4000 conserva...            16859   \n",
              "3              in Florida to more than 850,000 homes.            18246   \n",
              "4      feel since their power got cut out needlessly.            16859   \n",
              "5    to view the destruction caused by Hurricane Ian.            18246   \n",
              "6       are necessary to crack down on those hackers.            18246   \n",
              "7             that. We'll see if they get it tonight.            12387   \n",
              "8   footage to try to piece together more informat...            16859   \n",
              "9   on their vaccinations getting severe illness f...            16859   \n",
              "10  stuff. Stay hydrated, take breaks. Don't overd...            16859   \n",
              "11  north and South Carolina. Fasttracking federal...            18246   \n",
              "12                     name or what led to that fall.            18246   \n",
              "13  Meanwhile, the suspected killer remains in the...            18246   \n",
              "14      with any information is asked to call police.            18246   \n",
              "15         not released yet by the Santa Clara County            12387   \n",
              "16       not reveal a motive for the deadly shooting.            12387   \n",
              "17                this case, please give them a call.            12387   \n",
              "\n",
              "                                                 body    start      end  \\\n",
              "0   Well knew. This morning police need your help ...   464928   505910   \n",
              "1   a call. San Francisco firefighters rescued a m...   359020   385526   \n",
              "2   Paul. Meanwhile, the state set a record in ene...    60704   101238   \n",
              "3   Emergency crews in Florida continue to search ...   505290   534958   \n",
              "4   But even though the state never ordered rollin...   100910   283606   \n",
              "5   aid. And today, president Joe Biden and first ...   546494   593818   \n",
              "6   In the last month, there have been numerous da...   614910   700574   \n",
              "7   and the warriors are playing the Boston Celtic...   419994   649502   \n",
              "8   And San Leandro police searching for the perso...   578612   619638   \n",
              "9   The updated Bivalent Coronavirus booster shot ...   619310   660654   \n",
              "10  temperatures tonight, only back down to the un...  1127008  1208198   \n",
              "11  And National Guard teams are also on the groun...   534674   548314   \n",
              "12  A man died after falling from an escalator aft...   593490   615274   \n",
              "13  All right. Well happening today. Friends and f...   218170   345758   \n",
              "14  detectives are searching for six people they s...   350830   459442   \n",
              "15  San Jose police say a security guard trying to...    98130   276282   \n",
              "16  San Jose police just announcing today an arres...   283804   330978   \n",
              "17  shooting. Surveillance video shows how a car t...   329682   360102   \n",
              "\n",
              "                                                topic  \n",
              "0            ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "1            ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "2            ['83a09c6b-5f2f-421f-ae50-b38acca7e008']  \n",
              "3   ['9ff54ded-904b-4e0c-85ce-a3617f5cb913', '9a06...  \n",
              "4                                                  []  \n",
              "5            ['83a09c6b-5f2f-421f-ae50-b38acca7e008']  \n",
              "6   ['83a09c6b-5f2f-421f-ae50-b38acca7e008', '9ff5...  \n",
              "7            ['b49207eb-96eb-4b73-b534-adc0ef85022a']  \n",
              "8            ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "9            ['96326734-fd82-4350-b45c-513e7eb9147c']  \n",
              "10                                                 []  \n",
              "11           ['83a09c6b-5f2f-421f-ae50-b38acca7e008']  \n",
              "12           ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "13           ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "14           ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "15           ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "16           ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "17           ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  "
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "final = final.apply(generate_topic, axis=1)\n",
        "final"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XvPBMNfEBAmI"
      },
      "source": [
        "We observe that some of the rows have no topics predicted for them. We will try lowering the `sigmoid_threshold` this time to see if it gets any better."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 970
        },
        "id": "XFvfoHohBQ63",
        "outputId": "791a1cb3-f05d-4b86-8144-c1962a4f161f"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-217da875-1d22-46bc-a9de-096d2a61a09a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>first_words</th>\n",
              "      <th>last_words</th>\n",
              "      <th>source_video_id</th>\n",
              "      <th>body</th>\n",
              "      <th>start</th>\n",
              "      <th>end</th>\n",
              "      <th>topic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Well knew. This morning police need your help</td>\n",
              "      <td>gunpoint, beating him and stealing his cell ph...</td>\n",
              "      <td>18246</td>\n",
              "      <td>Well knew. This morning police need your help ...</td>\n",
              "      <td>464928</td>\n",
              "      <td>505910</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>a call. San Francisco firefighters rescued a man</td>\n",
              "      <td>all the way down to the ocean.</td>\n",
              "      <td>12387</td>\n",
              "      <td>a call. San Francisco firefighters rescued a m...</td>\n",
              "      <td>359020</td>\n",
              "      <td>385526</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Paul. Meanwhile, the state set a record in</td>\n",
              "      <td>night through conservation, some 4000 conserva...</td>\n",
              "      <td>16859</td>\n",
              "      <td>Paul. Meanwhile, the state set a record in ene...</td>\n",
              "      <td>60704</td>\n",
              "      <td>101238</td>\n",
              "      <td>['83a09c6b-5f2f-421f-ae50-b38acca7e008', '3982...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Emergency crews in Florida continue to search for</td>\n",
              "      <td>in Florida to more than 850,000 homes.</td>\n",
              "      <td>18246</td>\n",
              "      <td>Emergency crews in Florida continue to search ...</td>\n",
              "      <td>505290</td>\n",
              "      <td>534958</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913', '9a06...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>But even though the state never ordered rolling</td>\n",
              "      <td>feel since their power got cut out needlessly.</td>\n",
              "      <td>16859</td>\n",
              "      <td>But even though the state never ordered rollin...</td>\n",
              "      <td>100910</td>\n",
              "      <td>283606</td>\n",
              "      <td>['39822b5f-e37e-43e8-b997-7142fe55c3ea']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>aid. And today, president Joe Biden and first</td>\n",
              "      <td>to view the destruction caused by Hurricane Ian.</td>\n",
              "      <td>18246</td>\n",
              "      <td>aid. And today, president Joe Biden and first ...</td>\n",
              "      <td>546494</td>\n",
              "      <td>593818</td>\n",
              "      <td>['83a09c6b-5f2f-421f-ae50-b38acca7e008', '9a06...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>In the last month, there have been numerous</td>\n",
              "      <td>are necessary to crack down on those hackers.</td>\n",
              "      <td>18246</td>\n",
              "      <td>In the last month, there have been numerous da...</td>\n",
              "      <td>614910</td>\n",
              "      <td>700574</td>\n",
              "      <td>['83a09c6b-5f2f-421f-ae50-b38acca7e008', '9ff5...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>and the warriors are playing the Boston Celtics</td>\n",
              "      <td>that. We'll see if they get it tonight.</td>\n",
              "      <td>12387</td>\n",
              "      <td>and the warriors are playing the Boston Celtic...</td>\n",
              "      <td>419994</td>\n",
              "      <td>649502</td>\n",
              "      <td>['b49207eb-96eb-4b73-b534-adc0ef85022a']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>And San Leandro police searching for the person</td>\n",
              "      <td>footage to try to piece together more informat...</td>\n",
              "      <td>16859</td>\n",
              "      <td>And San Leandro police searching for the perso...</td>\n",
              "      <td>578612</td>\n",
              "      <td>619638</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>The updated Bivalent Coronavirus booster shot ...</td>\n",
              "      <td>on their vaccinations getting severe illness f...</td>\n",
              "      <td>16859</td>\n",
              "      <td>The updated Bivalent Coronavirus booster shot ...</td>\n",
              "      <td>619310</td>\n",
              "      <td>660654</td>\n",
              "      <td>['96326734-fd82-4350-b45c-513e7eb9147c']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>temperatures tonight, only back down to the un...</td>\n",
              "      <td>stuff. Stay hydrated, take breaks. Don't overd...</td>\n",
              "      <td>16859</td>\n",
              "      <td>temperatures tonight, only back down to the un...</td>\n",
              "      <td>1127008</td>\n",
              "      <td>1208198</td>\n",
              "      <td>['0d817400-3f5d-41e0-929c-c31fdbe75d31']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>And National Guard teams are also on the</td>\n",
              "      <td>north and South Carolina. Fasttracking federal...</td>\n",
              "      <td>18246</td>\n",
              "      <td>And National Guard teams are also on the groun...</td>\n",
              "      <td>534674</td>\n",
              "      <td>548314</td>\n",
              "      <td>['83a09c6b-5f2f-421f-ae50-b38acca7e008']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>A man died after falling from an escalator</td>\n",
              "      <td>name or what led to that fall.</td>\n",
              "      <td>18246</td>\n",
              "      <td>A man died after falling from an escalator aft...</td>\n",
              "      <td>593490</td>\n",
              "      <td>615274</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913', 'b492...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>All right. Well happening today. Friends and f...</td>\n",
              "      <td>Meanwhile, the suspected killer remains in the...</td>\n",
              "      <td>18246</td>\n",
              "      <td>All right. Well happening today. Friends and f...</td>\n",
              "      <td>218170</td>\n",
              "      <td>345758</td>\n",
              "      <td>['83a09c6b-5f2f-421f-ae50-b38acca7e008', '9ff5...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>detectives are searching for six people they say</td>\n",
              "      <td>with any information is asked to call police.</td>\n",
              "      <td>18246</td>\n",
              "      <td>detectives are searching for six people they s...</td>\n",
              "      <td>350830</td>\n",
              "      <td>459442</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>San Jose police say a security guard trying</td>\n",
              "      <td>not released yet by the Santa Clara County</td>\n",
              "      <td>12387</td>\n",
              "      <td>San Jose police say a security guard trying to...</td>\n",
              "      <td>98130</td>\n",
              "      <td>276282</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>San Jose police just announcing today an arrest</td>\n",
              "      <td>not reveal a motive for the deadly shooting.</td>\n",
              "      <td>12387</td>\n",
              "      <td>San Jose police just announcing today an arres...</td>\n",
              "      <td>283804</td>\n",
              "      <td>330978</td>\n",
              "      <td>['83a09c6b-5f2f-421f-ae50-b38acca7e008', '9ff5...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>shooting. Surveillance video shows how a car t...</td>\n",
              "      <td>this case, please give them a call.</td>\n",
              "      <td>12387</td>\n",
              "      <td>shooting. Surveillance video shows how a car t...</td>\n",
              "      <td>329682</td>\n",
              "      <td>360102</td>\n",
              "      <td>['9ff54ded-904b-4e0c-85ce-a3617f5cb913']</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-217da875-1d22-46bc-a9de-096d2a61a09a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-217da875-1d22-46bc-a9de-096d2a61a09a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-217da875-1d22-46bc-a9de-096d2a61a09a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                          first_words  \\\n",
              "0       Well knew. This morning police need your help   \n",
              "1    a call. San Francisco firefighters rescued a man   \n",
              "2          Paul. Meanwhile, the state set a record in   \n",
              "3   Emergency crews in Florida continue to search for   \n",
              "4     But even though the state never ordered rolling   \n",
              "5       aid. And today, president Joe Biden and first   \n",
              "6         In the last month, there have been numerous   \n",
              "7     and the warriors are playing the Boston Celtics   \n",
              "8     And San Leandro police searching for the person   \n",
              "9   The updated Bivalent Coronavirus booster shot ...   \n",
              "10  temperatures tonight, only back down to the un...   \n",
              "11           And National Guard teams are also on the   \n",
              "12         A man died after falling from an escalator   \n",
              "13  All right. Well happening today. Friends and f...   \n",
              "14   detectives are searching for six people they say   \n",
              "15        San Jose police say a security guard trying   \n",
              "16    San Jose police just announcing today an arrest   \n",
              "17  shooting. Surveillance video shows how a car t...   \n",
              "\n",
              "                                           last_words  source_video_id  \\\n",
              "0   gunpoint, beating him and stealing his cell ph...            18246   \n",
              "1                      all the way down to the ocean.            12387   \n",
              "2   night through conservation, some 4000 conserva...            16859   \n",
              "3              in Florida to more than 850,000 homes.            18246   \n",
              "4      feel since their power got cut out needlessly.            16859   \n",
              "5    to view the destruction caused by Hurricane Ian.            18246   \n",
              "6       are necessary to crack down on those hackers.            18246   \n",
              "7             that. We'll see if they get it tonight.            12387   \n",
              "8   footage to try to piece together more informat...            16859   \n",
              "9   on their vaccinations getting severe illness f...            16859   \n",
              "10  stuff. Stay hydrated, take breaks. Don't overd...            16859   \n",
              "11  north and South Carolina. Fasttracking federal...            18246   \n",
              "12                     name or what led to that fall.            18246   \n",
              "13  Meanwhile, the suspected killer remains in the...            18246   \n",
              "14      with any information is asked to call police.            18246   \n",
              "15         not released yet by the Santa Clara County            12387   \n",
              "16       not reveal a motive for the deadly shooting.            12387   \n",
              "17                this case, please give them a call.            12387   \n",
              "\n",
              "                                                 body    start      end  \\\n",
              "0   Well knew. This morning police need your help ...   464928   505910   \n",
              "1   a call. San Francisco firefighters rescued a m...   359020   385526   \n",
              "2   Paul. Meanwhile, the state set a record in ene...    60704   101238   \n",
              "3   Emergency crews in Florida continue to search ...   505290   534958   \n",
              "4   But even though the state never ordered rollin...   100910   283606   \n",
              "5   aid. And today, president Joe Biden and first ...   546494   593818   \n",
              "6   In the last month, there have been numerous da...   614910   700574   \n",
              "7   and the warriors are playing the Boston Celtic...   419994   649502   \n",
              "8   And San Leandro police searching for the perso...   578612   619638   \n",
              "9   The updated Bivalent Coronavirus booster shot ...   619310   660654   \n",
              "10  temperatures tonight, only back down to the un...  1127008  1208198   \n",
              "11  And National Guard teams are also on the groun...   534674   548314   \n",
              "12  A man died after falling from an escalator aft...   593490   615274   \n",
              "13  All right. Well happening today. Friends and f...   218170   345758   \n",
              "14  detectives are searching for six people they s...   350830   459442   \n",
              "15  San Jose police say a security guard trying to...    98130   276282   \n",
              "16  San Jose police just announcing today an arres...   283804   330978   \n",
              "17  shooting. Surveillance video shows how a car t...   329682   360102   \n",
              "\n",
              "                                                topic  \n",
              "0            ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "1            ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "2   ['83a09c6b-5f2f-421f-ae50-b38acca7e008', '3982...  \n",
              "3   ['9ff54ded-904b-4e0c-85ce-a3617f5cb913', '9a06...  \n",
              "4            ['39822b5f-e37e-43e8-b997-7142fe55c3ea']  \n",
              "5   ['83a09c6b-5f2f-421f-ae50-b38acca7e008', '9a06...  \n",
              "6   ['83a09c6b-5f2f-421f-ae50-b38acca7e008', '9ff5...  \n",
              "7            ['b49207eb-96eb-4b73-b534-adc0ef85022a']  \n",
              "8            ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "9            ['96326734-fd82-4350-b45c-513e7eb9147c']  \n",
              "10           ['0d817400-3f5d-41e0-929c-c31fdbe75d31']  \n",
              "11           ['83a09c6b-5f2f-421f-ae50-b38acca7e008']  \n",
              "12  ['9ff54ded-904b-4e0c-85ce-a3617f5cb913', 'b492...  \n",
              "13  ['83a09c6b-5f2f-421f-ae50-b38acca7e008', '9ff5...  \n",
              "14           ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "15           ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  \n",
              "16  ['83a09c6b-5f2f-421f-ae50-b38acca7e008', '9ff5...  \n",
              "17           ['9ff54ded-904b-4e0c-85ce-a3617f5cb913']  "
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "final = final.apply(generate_topic, sigmoid_threshold=0.3, axis=1)\n",
        "final"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9CzbuXjCBXWI"
      },
      "source": [
        "Much better now! This looks very good."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1sQ6Vmi-9vM7"
      },
      "source": [
        "#### Saving the dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "crzFDVG5yrlT"
      },
      "outputs": [],
      "source": [
        "final.to_csv('to_fill_finalized_BERT.csv', index=False)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.10"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}