real_or_not.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### I really  like potential of modern NLP tools and frameworks like Flair, Texthero, Distillbert\n",
    "... with couple of lines you could do quite a lot with quite impresive results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from flair.data import Sentence, Dictionary\n",
    "from flair.datasets import ClassificationCorpus\n",
    "from flair.embeddings import WordEmbeddings, DocumentRNNEmbeddings, FlairEmbeddings, TransformerDocumentEmbeddings\n",
    "from flair.models import TextClassifier\n",
    "from flair.trainers import ModelTrainer\n",
    "from torch.optim.adam import Adam\n",
    "\n",
    "from pathlib import Path\n",
    "from texthero import preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 7613 rows and 5 columns in train\n",
      "There are 3263 rows and 4 columns in train\n"
     ]
    }
   ],
   "source": [
    "df= pd.read_csv('train.csv')\n",
    "test=pd.read_csv('test.csv')\n",
    "submission = pd.read_csv(\"sample_submission.csv\")\n",
    "print('There are {} rows and {} columns in train'.format(df.shape[0],df.shape[1]))\n",
    "print('There are {} rows and {} columns in train'.format(test.shape[0],test.shape[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Preprocessing column 'text' with predefined tools from TextHero package \n",
    "#### ( steps are selfexplanatory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "custom_pipeline = [preprocessing.fillna,\n",
    "                   preprocessing.remove_stopwords,\n",
    "                   preprocessing.lowercase,\n",
    "                   preprocessing.remove_whitespace,\n",
    "                   preprocessing.remove_digits,\n",
    "                   preprocessing.remove_html_tags,\n",
    "                   preprocessing.remove_punctuation,\n",
    "                   \n",
    "                   ] \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['text_cleaner'] = preprocessing.clean(df[\"text\"], custom_pipeline)\n",
    "test['text_cleaner'] = preprocessing.clean(test[\"text\"], custom_pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "      <th>text_cleaner</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6929</th>\n",
       "      <td>9938</td>\n",
       "      <td>trouble</td>\n",
       "      <td>on twitter</td>\n",
       "      <td>why is it trouble@niallhariss / @simply_vain l...</td>\n",
       "      <td>0</td>\n",
       "      <td>trouble niallhariss    simply vain live http c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4540</th>\n",
       "      <td>6454</td>\n",
       "      <td>injured</td>\n",
       "      <td>Ikorodu</td>\n",
       "      <td>Photos: 17 people killed and over 25 injured i...</td>\n",
       "      <td>1</td>\n",
       "      <td>photos    people killed   injured deadly saudi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7205</th>\n",
       "      <td>10321</td>\n",
       "      <td>weapon</td>\n",
       "      <td>statesboro/vidalia</td>\n",
       "      <td>@ThatRussianMan you're too busy finishing thos...</td>\n",
       "      <td>0</td>\n",
       "      <td>thatrussianman   busy finishing weapon designs</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id  keyword            location  \\\n",
       "6929   9938  trouble         on twitter    \n",
       "4540   6454  injured             Ikorodu   \n",
       "7205  10321   weapon  statesboro/vidalia   \n",
       "\n",
       "                                                   text  target  \\\n",
       "6929  why is it trouble@niallhariss / @simply_vain l...       0   \n",
       "4540  Photos: 17 people killed and over 25 injured i...       1   \n",
       "7205  @ThatRussianMan you're too busy finishing thos...       0   \n",
       "\n",
       "                                           text_cleaner  \n",
       "6929  trouble niallhariss    simply vain live http c...  \n",
       "4540  photos    people killed   injured deadly saudi...  \n",
       "7205     thatrussianman   busy finishing weapon designs  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>text_cleaner</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1016</th>\n",
       "      <td>3343</td>\n",
       "      <td>demolished</td>\n",
       "      <td>Bolton, England</td>\n",
       "      <td>@OpTic_DKarma dude they demolished you!</td>\n",
       "      <td>optic dkarma dude demolished</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1777</th>\n",
       "      <td>6008</td>\n",
       "      <td>hazardous</td>\n",
       "      <td>NaN</td>\n",
       "      <td>#diablo #dsp Olap #world pres: http://t.co/LFE...</td>\n",
       "      <td>diablo  dsp olap  world pres  http co lfetnrx...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2533</th>\n",
       "      <td>8460</td>\n",
       "      <td>screamed</td>\n",
       "      <td>Holland,MI</td>\n",
       "      <td>I just screamed when I saw Jordan come out the...</td>\n",
       "      <td>i screamed i saw jordan come door behind donni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id     keyword         location  \\\n",
       "1016  3343  demolished  Bolton, England   \n",
       "1777  6008   hazardous              NaN   \n",
       "2533  8460    screamed       Holland,MI   \n",
       "\n",
       "                                                   text  \\\n",
       "1016            @OpTic_DKarma dude they demolished you!   \n",
       "1777  #diablo #dsp Olap #world pres: http://t.co/LFE...   \n",
       "2533  I just screamed when I saw Jordan come out the...   \n",
       "\n",
       "                                           text_cleaner  \n",
       "1016                     optic dkarma dude demolished    \n",
       "1777   diablo  dsp olap  world pres  http co lfetnrx...  \n",
       "2533  i screamed i saw jordan come door behind donni...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.sample(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing input for Distilbert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__label__0\ttrauma team needs come american e shop \n",
      "__label__1\t ptsd chat yes  i feel root shame   found rubble trauma   ptsdchat\n",
      "__label__1\thiroshima  they told paint story  eighty nine year old man recalls terror trauma   http co spe7u8t40k\n",
      "__label__1\tphoto  lavenderpoetrycafe  the forgotten history sexual trauma hysteria affliction seen primarily  http co u2es0uk1u3\n",
      "__label__1\ttrauma injuries involving kids sport usually cycling related   cbc ca http co 0dqjeretxu\n",
      "__label__1\tbutt trauma extraordinaire\n",
      "__label__0\tauthor interview michele rosenthal author your life after trauma \n",
      "__label__0\ta1  i started writing i   talk trauma therapy way i could communicate  gravitychat\n",
      "__label__0\t ashghebranious civil rights continued 60s  and trans generational trauma  anything listen americans \n",
      "__label__0\t thetimepast  saalon i childhood trauma resolved   actual trauma  fricken babies \n"
     ]
    }
   ],
   "source": [
    "!head tweet_train/tweet_text/dev.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " #### This is the format of file which Flair classifier understand\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p tweet_train/tweet_text "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating three files aout of training data\n",
    "def save_like_fasttext(text_feat, target_feat, dir_path):\n",
    "\n",
    "    df['label'] = '__label__' + df['target'].astype(str)\n",
    "\n",
    "    df[ ['label', text_feat] ].iloc[0:int(len(df)*0.8)].to_csv(dir_path + '/train.txt', sep='\\t', index=False, header=False)\n",
    "    df[ ['label', text_feat] ].iloc[int(len(df)*0.8):int(len(df)*0.9)].to_csv(dir_path + '/test.txt', sep='\\t', index=False, header=False)\n",
    "    df[ ['label', text_feat] ].iloc[int(len(df)*0.9):].to_csv(dir_path + '/dev.txt', sep='\\t', index=False, header=False);\n",
    "    \n",
    "    \n",
    "    \n",
    "save_like_fasttext(\"text_cleaner\", \"target\", \"tweet_train/tweet_text\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data_folder = Path('./tweet_train/tweet_text').resolve()\n",
    "\n",
    "#creating corpus  for Classifier\n",
    "corpus = ClassificationCorpus(\n",
    "    data_folder,\n",
    "    test_file='test.txt',\n",
    "    dev_file='dev.txt',\n",
    "    train_file='train.txt')\n",
    "\n",
    "\n",
    "# print(corpus.obtain_statistics())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DistillBert in action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-11-09 17:58:58,694 Computing label dictionary. Progress:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6851/6851 [00:02<00:00, 2656.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-11-09 17:59:01,439 [b'1', b'0']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "label_dict = corpus.make_label_dictionary()\n",
    "document_embeddings = TransformerDocumentEmbeddings('distilbert-base-uncased', fine_tune=True)\n",
    "classifier = TextClassifier(document_embeddings, label_dictionary=label_dict, multi_label=False)\n",
    "trainer = ModelTrainer(classifier, corpus, optimizer=Adam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.train('models/tweets',\n",
    "              learning_rate=3e-5, # use very small learning rate\n",
    "              mini_batch_size=16,\n",
    "              mini_batch_chunk_size=4, # optionally set this if transformer is too much for your machine\n",
    "              max_epochs=5, # terminate after 5 epochs\n",
    "              )"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "After 5 epochs of training results are like below( they ar ealso saved in training.log file \n",
    "\n",
    "...\n",
    "2020-11-09 20:41:14,996 ----------------------------------------------------------------------------------------------------\n",
    "2020-11-09 20:41:14,999 EPOCH 5 done: loss 0.1242 - lr 0.0000300\n",
    "2020-11-09 20:41:51,895 DEV : loss 1.2005043029785156 - score 0.8176\n",
    "2020-11-09 20:41:52,219 BAD EPOCHS (no improvement): 3\n",
    "2020-11-09 20:41:52,683 ----------------------------------------------------------------------------------------------------\n",
    "2020-11-09 20:41:52,685 Testing using best model ...\n",
    "2020-11-09 20:41:52,687 loading file models/tweets/best-model.pt\n",
    "2020-11-09 20:42:31,402 \t0.8068\n",
    "2020-11-09 20:42:31,404 \n",
    "Results:\n",
    "- F-score (micro) 0.8068\n",
    "- F-score (macro) 0.8062\n",
    "- Accuracy 0.8068\n",
    "\n",
    "By class:\n",
    "              precision    recall  f1-score   support\n",
    "\n",
    "           1     0.7851    0.8051    0.7950       354\n",
    "           0     0.8266    0.8084    0.8174       407\n",
    "\n",
    "   micro avg     0.8068    0.8068    0.8068       761\n",
    "   macro avg     0.8059    0.8067    0.8062       761\n",
    "weighted avg     0.8073    0.8068    0.8070       761\n",
    " samples avg     0.8068    0.8068    0.8068       761\n",
    "\n",
    "2020-11-09 20:42:31,405 ----------------------------------------------------------------------------------------------------"
   ]
  },
  {
   "source": [
    "# what should be done is to predict values for the test part based on the trained model best-model.pt"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": null,
   "outputs": []
  },
  {
   "source": [
    "#### This semi automatic approach it gives  similar score on kaggle ie __0.817__ on Leaderboard.\n",
    "#### Of course it is just a beginning as no info at all was used from other columns and other actions to improve the score."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}