Data Cleaning and Preprocessing Obama.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import warnings\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer,WordNetLemmatizer\n",
    "from wordsegment import segment\n",
    "from html import unescape\n",
    "import itertools\n",
    "import random\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xl_file = pd.ExcelFile(\"training-Obama-Romney-tweets.xlsx\", encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfs = {sheet_name: xl_file.parse('Obama',encoding='utf-8') \n",
    "          for sheet_name in xl_file.sheet_names}\n",
    "obamaData=dfs.get('Obama')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clearColumns(dataDF):\n",
    "    dataDF=dataDF.drop(dataDF.columns[[0, 1, 2, 5]], axis=1)\n",
    "    dataDF.columns = ['tweet', 'label']\n",
    "    dataDF = dataDF.ix[1:]\n",
    "    print(dataDF)\n",
    "    print(dataDF['label'].unique())\n",
    "    dataDF=dataDF[dataDF['label'].isin([0,1,-1,u'-1',u'1',u'0'])]\n",
    "    return dataDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                  tweet label\n",
      "1     Kirkpatrick, who wore a baseball cap embroider...     0\n",
      "2     Question: If <e>Romney</e> and <e>Obama</e> ha...     2\n",
      "3     #<e>obama</e> debates that Cracker Ass Cracker...     1\n",
      "4     RT @davewiner Slate: Blame <e>Obama</e> for fo...     2\n",
      "5     @Hollivan @hereistheanswer  Youre missing the ...     0\n",
      "6     <e>Mitt Romney</e> made all of his money himse...     2\n",
      "7     I was raised as a Democrat  left the party yea...    -1\n",
      "8     The <e>Obama camp</e> can't afford to lower ex...     0\n",
      "9     Tonight's debate has that \"Game 7\" feel! This ...     2\n",
      "10    <e>Obama</e> pot <a>policy</a> disappointing -...    -1\n",
      "11    Not all of Hollywood has his back! RT @RedAler...    -1\n",
      "12    @hblodget i'd be grateful for scoop from you g...     2\n",
      "13    <e>Obama</e> must, to a degree, hit <e>MR</e> ...     2\n",
      "14    <e>Obama</e>'s Expedient Speak fair in order t...     0\n",
      "15    I had a dream that i was smoking with <e>Obama...     0\n",
      "16    The Washington Times: The President’s Populari...    -1\n",
      "17    <e>Obama</e>'s priorities: $30K for \"gaydar\" r...    -1\n",
      "18    United states of Islam ? Does <e>Obama</e> rea...    -1\n",
      "19    @FightForJobs :they created this debt now plam...    -1\n",
      "20    @mittromney <e>Obama</e> has not kept ONE of h...    -1\n",
      "21    <e>Obama</e> vs <e>Romney</e> pt.2 tonight. Ob...     2\n",
      "22    This sub is here talking shit about <e>Obama</e>.     0\n",
      "23                                 <e>obama</e> ist tot     0\n",
      "24    @rnixonjr when you go against your own ideas t...     0\n",
      "25    I just don't see how <e>Obama</e> can recover ...     2\n",
      "26             Still my Idol Mr President <e>Obama</e>.     1\n",
      "27    @ArturasR Pretty creepy! No wonder <e>Obama</e...     0\n",
      "28    Just saw a truck with a NASA sticker and an <e...     0\n",
      "29    debate #2 tonight! I hope <e>Obama</e> brings ...     0\n",
      "30    @thedailybeast, you said so much good things a...     1\n",
      "...                                                 ...   ...\n",
      "7169   Best statement by a focus group member:  <e>O...    -1\n",
      "7170   @RichardLui <e>president</e> <e>Obama</e> is ...     2\n",
      "7171   <e>Obama</e> should start challenging leaders...    -1\n",
      "7172   I feel so blessed. I have been inside the <a>...     2\n",
      "7173                                             #NAME?     2\n",
      "7174   FACT CHECK: A Look At Some Of the Claims Made...     2\n",
      "7175   #emptybinder Obama's Business Experience     ...     0\n",
      "7176   FACT CHECK: A Look At Some Of the Claims Made...     2\n",
      "7177   #theview When Candy said <e>Obama</e> wants t...     0\n",
      "7178   Good morning my fellow Americans!  How are y'...     2\n",
      "7179   #<e>Romney</e> won the <a>#debate</a> he held...     2\n",
      "7180   With all eyes on <e>Obama</e> or <e>Romney</e...     2\n",
      "7181   7) But   BUT‰Û_ It is a mistake to assume tha...     0\n",
      "7182   Ya I did @BarackObama RT @CarlJohnson70: @Cha...     2\n",
      "7183   MT RT @ErikWemple Guy from town hall debate s...    -1\n",
      "7184   Mitt-Gangnam Style was funnier  but just for ...     0\n",
      "7185   Obama  <e>Romney</e> clash over <a>energy iss...     2\n",
      "7186   Yo look at some of the shit <e>Obama</e> sign...    -1\n",
      "7187   There is  and always has been a glass ceiling...    -1\n",
      "7188   @wolfbIitzercnn  What choice did <e>Obama</e>...     2\n",
      "7189   ‰ÛÏ@9inyochick: When they for Obama  who kan ...     1\n",
      "7190   When Jacobs blames <e>Obama</e> for her compu...     0\n",
      "7191   except for <a>women</a> who work in the WH (t...     0\n",
      "7192   Vote for whomever you choose. #Obama vs #<e>R...     2\n",
      "7193   20 Days to Election & Selection.  Elect Lewis...     1\n",
      "7194   The Reason <e>Ann Romney</e> And <e>Michelle ...     0\n",
      "7195   <e>Obama</e> Kenakan Cincin Syahadat Sejak SM...     0\n",
      "7196   Bitches be like \"Obama<3\" bitches just want <...     0\n",
      "7197   <e>president</e> Barack <e>Obama</e> and Repu...     2\n",
      "7198   #ThatsSoRude you trying to get into Obama's f...     2\n",
      "\n",
      "[7198 rows x 2 columns]\n",
      "[0 2 1 -1 'irrelevant' 'irrevelant' nan '0' '2' '-1' '1']\n"
     ]
    }
   ],
   "source": [
    "obamaDataDF=clearColumns(obamaData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# stopwords=['i','me','my','myself','we','our','ours','ourselves','you','your','yours','yourself','yourselves','he','him','his',\n",
    "#            'himself','she','her','hers','herself','it','its','itself','they','them','their','theirs','themselves','what',\n",
    "#            'which','who','whom','this','that','these','those','am','is','are','was','were','be','been','being','have','has',\n",
    "#            'had','having','do','does','did','doing','a','an','the','and','but','if','or','because','as','until','while','of',\n",
    "#            'at','by','for','with','about','against','between','into','through','during','before','after','above','below','to',\n",
    "#            'from','up','down','in','out','on','off','over','under','again','further','then','once','here','there','when',\n",
    "#            'where','why','how','all','any','both','each','few','more','most','other','some','such','only',\n",
    "#            'own','same','so','than','too','very','can','will','just','should','now','im']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cleanURLS(tweet):\n",
    "    tweet=re.sub('http[s]?://(?:[a-z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-f][0-9a-f]))+', '', tweet)\n",
    "    return tweet\n",
    "    \n",
    "def getText_fromHTML(tweet):\n",
    "    tweet = BeautifulSoup(tweet)\n",
    "    tweet = tweet.get_text()\n",
    "    return tweet\n",
    "\n",
    "def removeAppostophes(tweet):\n",
    "    appostophes = {\"s\": \"is\", \"re\": \"are\", \"em\": \"them\", \"ll\": \"will\", \"t\": \"it\", \"m\": \"am\", \"ve\": \"have\", \"d\": \"did\"}\n",
    "    words = []\n",
    "    [words.append(terms) for word in tweet.split() for terms in [appostophes[term] if term in appostophes else term for\n",
    "                                                                 term in word.split(\"'\")]]\n",
    "    tweet = \" \".join(words)\n",
    "    return tweet\n",
    "\n",
    "def splitAttachedwords(tweet):\n",
    "    tweet = \" \".join(segment(tweet))\n",
    "    return tweet\n",
    "\n",
    "def lemmatisation(tweet):\n",
    "    return \" \".join(WordNetLemmatizer().lemmatize(word) for word in tweet.split())\n",
    "\n",
    "def stemming(tweet):\n",
    "    return \" \".join(PorterStemmer().stem(word) for word in tweet.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cleanTweets(dataDF):\n",
    "    rowsToBeRemoved=[]\n",
    "    TAG_RE = re.compile(r'<[^>]+>')\n",
    "    for index, row in dataDF.iterrows():\n",
    "        tweet=row['tweet']\n",
    "        if isinstance(row['label'], str) == True:\n",
    "            row['label']=int(row['label'])\n",
    "        output=\"\"      \n",
    "        if isinstance(tweet, str) == False:\n",
    "            rowsToBeRemoved.append(index)\n",
    "            continue\n",
    "        tweet=tweet.lower()\n",
    "        \n",
    "        # remove the urls from tweet\n",
    "        tweet = cleanURLS(tweet)\n",
    "\n",
    "        # remove HTML tags\n",
    "        tweet = getText_fromHTML(tweet)\n",
    "        \n",
    "        # remove appostophes\n",
    "        tweet = removeAppostophes(tweet)\n",
    "        \n",
    "        # split attached words ex: goodboy = good boy\n",
    "        tweet = splitAttachedwords(tweet)\n",
    "        \n",
    "        for word in tweet.split(\" \"):\n",
    "            # removing usernames\n",
    "            if word.strip().startswith('@'):\n",
    "                word=\"\"\n",
    "            # removing hashtags\n",
    "            if word.strip().startswith('#'):\n",
    "                word=word[1:]\n",
    "            # strip punctuation\n",
    "            word=re.sub(r'[^\\w\\s]','',word)\n",
    "            if word != \"\" and word.isalpha():\n",
    "                output+=\" \"+word\n",
    "        if len(output)<1:\n",
    "            rowsToBeRemoved.append(index)\n",
    "        output = lemmatisation(output)\n",
    "        output = stemming(output)   \n",
    "        row['tweet']=output.strip()\n",
    "    return dataDF.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "obamaCleanedData=cleanTweets(obamaDataDF.copy(deep=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Shuffling the cleaned data\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from numpy.random import random_integers as ri\n",
    "\n",
    "final_data = obamaCleanedData.copy(deep=True)\n",
    "for i in range(0,500):\n",
    "    rand_int = ri(0,500)\n",
    "    X_train,X_test,y_train,y_test = train_test_split(final_data['tweet'],final_data['label'],test_size=0.5,random_state=rand_int)\n",
    "    train_dataDF = pd.concat([X_train,y_train],axis=1)\n",
    "    test_dataDF = pd.concat([X_test,y_test],axis=1)\n",
    "    final_data = pd.concat([train_dataDF,test_dataDF],axis=0)\n",
    "final_data.to_csv('obamaCleanedData.csv',sep=',',encoding='utf-8',header=['tweet','label'],index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del obamaData,obamaDataDF,obamaCleanedData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}