bibliometric_topic_modeling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "E64QVb3q3pDX"
   },
   "source": [
    "Install and import library modules. Download English stopwords and WordNet. Set logging to INFO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-hl1Pyqi2WeK",
    "outputId": "505681b2-eb84-40cb-dc3d-af42e4fe5fbb",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gensim in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (3.8.3)\n",
      "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from gensim) (4.0.1)\n",
      "Requirement already satisfied: six>=1.5.0 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from gensim) (1.14.0)\n",
      "Requirement already satisfied: scipy>=0.18.1 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from gensim) (1.5.2)\n",
      "Requirement already satisfied: numpy>=1.11.3 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from gensim) (1.19.2)\n",
      "\u001b[33mWARNING: You are using pip version 20.3.3; however, version 21.2.3 is available.\n",
      "You should consider upgrading via the '/usr/local/Cellar/jupyterlab/2.1.2/libexec/bin/python3.8 -m pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: pyLDAvis in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (2.1.2)\n",
      "Requirement already satisfied: joblib>=0.8.4 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pyLDAvis) (0.16.0)\n",
      "Requirement already satisfied: funcy in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pyLDAvis) (1.15)\n",
      "Requirement already satisfied: future in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pyLDAvis) (0.18.2)\n",
      "Requirement already satisfied: numpy>=1.9.2 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pyLDAvis) (1.19.2)\n",
      "Requirement already satisfied: pytest in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pyLDAvis) (6.2.1)\n",
      "Requirement already satisfied: scipy>=0.18.0 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pyLDAvis) (1.5.2)\n",
      "Requirement already satisfied: wheel>=0.23.0 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pyLDAvis) (0.34.2)\n",
      "Requirement already satisfied: jinja2>=2.7.2 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pyLDAvis) (2.11.2)\n",
      "Requirement already satisfied: numexpr in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pyLDAvis) (2.7.2)\n",
      "Requirement already satisfied: pandas>=0.17.0 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pyLDAvis) (1.1.2)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from jinja2>=2.7.2->pyLDAvis) (1.1.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pandas>=0.17.0->pyLDAvis) (2020.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pandas>=0.17.0->pyLDAvis) (2.8.1)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas>=0.17.0->pyLDAvis) (1.14.0)\n",
      "Requirement already satisfied: pluggy<1.0.0a1,>=0.12 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pytest->pyLDAvis) (0.13.1)\n",
      "Requirement already satisfied: toml in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pytest->pyLDAvis) (0.10.2)\n",
      "Requirement already satisfied: py>=1.8.2 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pytest->pyLDAvis) (1.10.0)\n",
      "Requirement already satisfied: attrs>=19.2.0 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pytest->pyLDAvis) (19.3.0)\n",
      "Requirement already satisfied: packaging in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pytest->pyLDAvis) (20.7)\n",
      "Requirement already satisfied: iniconfig in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pytest->pyLDAvis) (1.1.1)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from packaging->pytest->pyLDAvis) (2.4.7)\n",
      "\u001b[33mWARNING: You are using pip version 20.3.3; however, version 21.2.3 is available.\n",
      "You should consider upgrading via the '/usr/local/Cellar/jupyterlab/2.1.2/libexec/bin/python3.8 -m pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: pandas in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (1.1.2)\n",
      "Requirement already satisfied: pytz>=2017.2 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pandas) (2020.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pandas) (2.8.1)\n",
      "Requirement already satisfied: numpy>=1.15.4 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from pandas) (1.19.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas) (1.14.0)\n",
      "\u001b[33mWARNING: You are using pip version 20.3.3; however, version 21.2.3 is available.\n",
      "You should consider upgrading via the '/usr/local/Cellar/jupyterlab/2.1.2/libexec/bin/python3.8 -m pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: numpy in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (1.19.2)\n",
      "\u001b[33mWARNING: You are using pip version 20.3.3; however, version 21.2.3 is available.\n",
      "You should consider upgrading via the '/usr/local/Cellar/jupyterlab/2.1.2/libexec/bin/python3.8 -m pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: matplotlib in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (3.3.2)\n",
      "Requirement already satisfied: numpy>=1.15 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from matplotlib) (1.19.2)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from matplotlib) (2.8.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from matplotlib) (7.2.0)\n",
      "Requirement already satisfied: certifi>=2020.06.20 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from matplotlib) (2020.6.20)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from matplotlib) (2.4.7)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from matplotlib) (1.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from matplotlib) (0.10.0)\n",
      "Requirement already satisfied: six in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from cycler>=0.10->matplotlib) (1.14.0)\n",
      "\u001b[33mWARNING: You are using pip version 20.3.3; however, version 21.2.3 is available.\n",
      "You should consider upgrading via the '/usr/local/Cellar/jupyterlab/2.1.2/libexec/bin/python3.8 -m pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: nltk in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (3.5)\n",
      "Requirement already satisfied: click in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from nltk) (7.1.2)\n",
      "Requirement already satisfied: tqdm in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from nltk) (4.54.0)\n",
      "Requirement already satisfied: joblib in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from nltk) (0.16.0)\n",
      "Requirement already satisfied: regex in /usr/local/Cellar/jupyterlab/2.1.2/libexec/lib/python3.8/site-packages (from nltk) (2020.11.13)\n",
      "\u001b[33mWARNING: You are using pip version 20.3.3; however, version 21.2.3 is available.\n",
      "You should consider upgrading via the '/usr/local/Cellar/jupyterlab/2.1.2/libexec/bin/python3.8 -m pip install --upgrade pip' command.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /Users/joost/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /Users/joost/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "!{sys.executable} -m pip install gensim\n",
    "!{sys.executable} -m pip install pyLDAvis\n",
    "!{sys.executable} -m pip install pandas\n",
    "!{sys.executable} -m pip install numpy\n",
    "!{sys.executable} -m pip install matplotlib\n",
    "!{sys.executable} -m pip install nltk\n",
    "\n",
    "import re, gensim, pyLDAvis, logging, datetime, multiprocessing, os, pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from math import floor, ceil\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import download\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "download('stopwords')\n",
    "download('wordnet')\n",
    "\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read the data as pandas dataframe, remove items with missing abstracts, and fill other missing data with empty strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kTYs7ND53iEL",
    "outputId": "6840fdf0-09ac-4a57-ace1-c33c7f964926"
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/data_merged_and_filtered.csv', usecols=['scopusId', 'title', 'keywords', 'abstract', 'modularity_class'], index_col='scopusId')\n",
    "df = df[df['abstract'].notna()]\n",
    "df.fillna(\"\", inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Combine the text fiels into one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "dhV7-XfC6LSC",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-09 20:31:31,337 : INFO : NumExpr defaulting to 8 threads.\n"
     ]
    }
   ],
   "source": [
    "df[\"data\"] = df[\"abstract\"] + \" \" + df[\"keywords\"] + \" \" + df[\"title\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove copyright notices from the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "tOkh2Nlc4Mbf"
   },
   "outputs": [],
   "source": [
    "copyright_re = \"(©|&#x00A9;|Copyright)(.*?(?=B\\\\.V\\\\.|Ltd|\\\\.))(B\\\\.V\\\\.|Ltd|\\\\.)\"\n",
    "rights = \"All rights reserved.\"\n",
    "df.replace(copyright_re, '', regex=True, inplace=True)\n",
    "df.replace(rights, '', regex=False, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rZbCoBCTBP5G"
   },
   "source": [
    "Tokenize, convert to lowercase, remove punctuation, remove stop-words, lemmatize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "  result = [word for word in gensim.utils.simple_preprocess(text, deacc=True) if word not in stop_words]\n",
    "  wordnet_lemmatizer = WordNetLemmatizer()\n",
    "  result = [wordnet_lemmatizer.lemmatize(word) for word in result]\n",
    "  return result\n",
    "\n",
    "stop_words = stopwords.words('english')\n",
    "df[\"data_preprocessed\"] = df[\"data\"].map(preprocess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate the Dictionairy and perform relative pruning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-09 20:34:31,443 : INFO : adding document #0 to Dictionary(0 unique tokens: [])\n",
      "2021-08-09 20:34:33,489 : INFO : adding document #10000 to Dictionary(27074 unique tokens: ['ability', 'acceptable', 'additionally', 'analysis', 'anti']...)\n",
      "2021-08-09 20:34:35,623 : INFO : adding document #20000 to Dictionary(38089 unique tokens: ['ability', 'acceptable', 'additionally', 'analysis', 'anti']...)\n",
      "2021-08-09 20:34:37,456 : INFO : adding document #30000 to Dictionary(46717 unique tokens: ['ability', 'acceptable', 'additionally', 'analysis', 'anti']...)\n",
      "2021-08-09 20:34:39,543 : INFO : adding document #40000 to Dictionary(53696 unique tokens: ['ability', 'acceptable', 'additionally', 'analysis', 'anti']...)\n",
      "2021-08-09 20:34:41,572 : INFO : adding document #50000 to Dictionary(59648 unique tokens: ['ability', 'acceptable', 'additionally', 'analysis', 'anti']...)\n",
      "2021-08-09 20:34:43,434 : INFO : adding document #60000 to Dictionary(64997 unique tokens: ['ability', 'acceptable', 'additionally', 'analysis', 'anti']...)\n",
      "2021-08-09 20:34:45,986 : INFO : adding document #70000 to Dictionary(70041 unique tokens: ['ability', 'acceptable', 'additionally', 'analysis', 'anti']...)\n",
      "2021-08-09 20:34:48,214 : INFO : built Dictionary(75148 unique tokens: ['ability', 'acceptable', 'additionally', 'analysis', 'anti']...) from 79433 documents (total 9349837 corpus positions)\n",
      "2021-08-09 20:34:48,733 : INFO : discarding 72727 tokens: [('competent', 111), ('conceal', 295), ('conceals', 59), ('curtail', 30), ('db', 204), ('ee', 34), ('embed', 297), ('excellent', 301), ('histogram', 277), ('hyper', 79)]...\n",
      "2021-08-09 20:34:48,735 : INFO : keeping 2421 tokens which were in no less than 397 and no more than 78638 (=99.0%) documents\n",
      "2021-08-09 20:34:48,790 : INFO : resulting dictionary: Dictionary(2421 unique tokens: ['ability', 'acceptable', 'additionally', 'analysis', 'anti']...)\n"
     ]
    }
   ],
   "source": [
    "dictionary = gensim.corpora.Dictionary(df[\"data_preprocessed\"])\n",
    "dictionary.filter_extremes(no_below=(floor(0.005 * len(df))), no_above=0.99)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make the bag of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = [dictionary.doc2bow(doc) for doc in df[\"data_preprocessed\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate all models for evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "juzFGaeamQSN",
    "outputId": "03cb29ac-d423-4e56-a1c2-ed53f08f38e4",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20:34:59 Model rs24k25 already in directory. Skipped\n",
      "20:34:59 Model rs24k30 already in directory. Skipped\n",
      "20:34:59 Model rs24k36 already in directory. Skipped\n",
      "20:34:59 Model rs24k40 already in directory. Skipped\n",
      "20:34:59 Model rs24k42 already in directory. Skipped\n",
      "20:34:59 Model rs24k49 already in directory. Skipped\n",
      "20:34:59 Model rs27k25 already in directory. Skipped\n",
      "20:34:59 Model rs27k30 already in directory. Skipped\n",
      "20:34:59 Model rs27k36 already in directory. Skipped\n",
      "20:34:59 Model rs27k40 already in directory. Skipped\n",
      "20:34:59 Model rs27k42 already in directory. Skipped\n",
      "20:34:59 Model rs27k49 already in directory. Skipped\n",
      "20:34:59 Model rs1024k25 already in directory. Skipped\n",
      "20:34:59 Model rs1024k30 already in directory. Skipped\n",
      "20:34:59 Model rs1024k36 already in directory. Skipped\n",
      "20:34:59 Model rs1024k40 already in directory. Skipped\n",
      "20:34:59 Model rs1024k42 already in directory. Skipped\n",
      "20:34:59 Model rs1024k49 already in directory. Skipped\n",
      "20:34:59 Model rs1027k25 already in directory. Skipped\n",
      "20:34:59 Model rs1027k30 already in directory. Skipped\n",
      "20:34:59 Model rs1027k36 already in directory. Skipped\n",
      "20:34:59 Model rs1027k40 already in directory. Skipped\n",
      "20:34:59 Model rs1027k42 already in directory. Skipped\n",
      "20:34:59 Model rs1027k49 already in directory. Skipped\n"
     ]
    }
   ],
   "source": [
    "k = [25, 30, 36, 40, 42, 49] #[1, 4, 9, 16, 25, 36, 49, 64, 81, 100, 121, 144, 169, 196, 225, 256, 289, 324, 361] later added [30, 40, 42]\n",
    "random_states = [24, 27, 1024, 1027] #[24, 1024] later added [27, 1027]\n",
    "\n",
    "texts = df[\"data_preprocessed\"]\n",
    "chunksize = ceil(len(texts))\n",
    "passes = 20\n",
    "iterations = 100\n",
    "eval_every = None\n",
    "dir_prior = 'auto'\n",
    "\n",
    "for random_state in random_states:\n",
    "    for num_topics in k:\n",
    "        \n",
    "        name = \"rs{}k{}\".format(random_state, num_topics)\n",
    "        \n",
    "        \n",
    "        if 'model_' + name in os.listdir('models'):\n",
    "            print(datetime.datetime.now().strftime('%H:%M:%S') + ' Model ' + name + ' already in directory. Skipped')\n",
    "        \n",
    "        else:\n",
    "            print(datetime.datetime.now().strftime('%H:%M:%S') + \" Building model \" + name)\n",
    "            lda_model = gensim.models.LdaModel(corpus, num_topics=num_topics, id2word=dictionary, alpha=dir_prior, eta=dir_prior, random_state=random_state, chunksize=chunksize, passes=passes, iterations=iterations)\n",
    "\n",
    "            lda_model.save(os.path.join('models', 'model_' + name))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate all models present in the current directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "models_to_evaluate = []\n",
    "\n",
    "for entry in os.scandir('models'):\n",
    "    if entry.path.endswith('.state'):\n",
    "        models_to_evaluate.append(entry.path[:-6])\n",
    "        \n",
    "models_to_evaluate.sort()        \n",
    "        \n",
    "for path in models_to_evaluate:\n",
    "    print(datetime.datetime.now().strftime('%H:%M:%S') + ' Starting evaluation of ' + path)\n",
    "    model = gensim.models.LdaModel.load(path)\n",
    "    coherence_model_lda = gensim.models.coherencemodel.CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')\n",
    "    coherence = coherence_model_lda.get_coherence()\n",
    "        \n",
    "    with open(os.path.join('models', 'scores.txt'), \"a\") as f:\n",
    "        f.write(\"\\\"\" + str(os.path.split(path)[1]) + \"\\\"\" ' : ' + str(coherence) + ',\\n')   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the preprocessed data, corpus and dictionary for later exploration  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-08-09 20:35:18,346 : INFO : saving Dictionary object under models/dictionary, separately None\n",
      "2021-08-09 20:35:18,350 : INFO : saved models/dictionary\n"
     ]
    }
   ],
   "source": [
    "df.to_csv('data/data_for_interpretation.csv')\n",
    "\n",
    "with open('models/corpus.pkl', 'wb') as f:\n",
    "    pickle.dump(corpus, f)\n",
    "    \n",
    "dictionary.save('models/dictionary')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyOp+BISxaCQJk/X2OSJaTWP",
   "collapsed_sections": [],
   "include_colab_link": true,
   "mount_file_id": "1D4ZE74ez76G3KnQhaa4vLdv8vQLh3Yz8",
   "name": "Untitled0.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}