ScrapeTweets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Tweet Scraper**\n",
    "\n",
    "This notebook uses tweepy and a previously made twitter account to scrape every tweet from given twitter accounts.  Edit the handles.txt file to change the twitter accounts set to download\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import oauth2 as oauth\n",
    "import urllib.request\n",
    "from pprint import pprint\n",
    "import csv\n",
    "import re\n",
    "import os\n",
    "import tweepy\n",
    "from tweepy import OAuthHandler\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "keys_path = './twitterkeys.txt'\n",
    "keys=[]\n",
    "\n",
    "#load the twitter api keys\n",
    "with open(keys_path) as f:\n",
    "    keys=json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "accountToUse = 'Donald Trump'\n",
    "keysToUse = keys[accountToUse]\n",
    "auth = OAuthHandler(keysToUse['cons_key'], keysToUse['cons_secret'])\n",
    "auth.set_access_token(keysToUse['access_token'], keysToUse['access_token_secret'])\n",
    "\n",
    "api = tweepy.API(auth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Ted Cruz': 'tedcruz', 'Hillary Clinton': 'HillaryClinton', 'Barack Obama': 'BarackObama', 'Bernie Sanders': 'BernieSanders', 'President Obama': 'potus44', 'Donald Trump': 'realDonaldTrump'}\n"
     ]
    }
   ],
   "source": [
    "#load the list of twitter handles to scrape\n",
    "handles=[]\n",
    "\n",
    "with open('./Twitter/twitterhandles.json') as f:\n",
    "    handles= json.load(f)\n",
    "print(handles)\n",
    "\n",
    "root = './Twitter/tweets/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#accepts a twitter handle and a name for the candidate, downloads all the tweets for that candidate and stores them in \n",
    "def scrapeTweetsToFile(handle, name):\n",
    "    \n",
    "    print(handle)\n",
    "    alltweets = []\n",
    "\n",
    "    newtweets=api.user_timeline(screen_name = handle, count=200)\n",
    "    alltweets.extend(newtweets)\n",
    "    oldest=alltweets[-1].id -1\n",
    "\n",
    "    while len(newtweets) > 0:\n",
    "        print('getting tweets before %s' %(oldest))\n",
    "        newtweets = api.user_timeline(screen_name=handle, count=200, max_id=oldest)    \n",
    "        alltweets.extend(newtweets)    \n",
    "        oldest=alltweets[-1].id - 1\n",
    "        print('...%s tweets downloaded so far' % len(alltweets))\n",
    "\n",
    "    outtweets = [[re.sub(r'[^\\x00-\\x7f]',r' ',tweet.text.replace('&amp;', '&').strip(\"'\").replace('\"','').replace('\\n', ' '))] for tweet in alltweets]\n",
    "\n",
    "    with open(os.path.join(root, '%s.txt' % name.replace(' ','-')) , 'w+', encoding='utf8') as f:\n",
    "\n",
    "        previous=''\n",
    "        \n",
    "        #These skip flags ensure that a continued tweet is not split in two\n",
    "        skipnext=False\n",
    "        skipcurrent=False\n",
    "        \n",
    "        #loop through all the tweets\n",
    "        for t in outtweets:\n",
    "            #move the skip next flag to the skip current flag\n",
    "            skipcurrent=skipnext\n",
    "            skipnext=False\n",
    "            \n",
    "            #if there is a previous tweet and the current tweet starts with '...'\n",
    "            if previous!='':\n",
    "                #if the current tweet has .. in the last 6 chars (Trump is messy with his ellipsies)\n",
    "                if '..' in t[0][-6:]:\n",
    "                    #set previous to the current tweet with the previous tweet appended to the end, removing all ellipsis-like patterns\n",
    "                    previous=t[0].strip('...').replace('...',' ')+' '+previous.strip('...').replace('...',' ')\n",
    "                    #set the flag to skip the next entry\n",
    "                    skipnext=True\n",
    "            #if there is a previous tweet that is not a retweet or a reply, and there are no links in the tweet\n",
    "            if previous != '' and previous[:2] != 'RT' and not '@' in previous[:2] and 'http' not in previous and not skipcurrent: \n",
    "                #write the previous tweet to its file\n",
    "                f.write(previous+'\\n') \n",
    "            \n",
    "            #set the previous tweet\n",
    "            previous=t[0]\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tedcruz\n",
      "getting tweets before 846782074677837823\n",
      "...400 tweets downloaded so far\n",
      "getting tweets before 831141818624536576\n",
      "...600 tweets downloaded so far\n",
      "getting tweets before 819232954513813504\n",
      "...800 tweets downloaded so far\n",
      "getting tweets before 806560962421620735\n",
      "...999 tweets downloaded so far\n",
      "getting tweets before 796126100615217152\n",
      "...1195 tweets downloaded so far\n",
      "getting tweets before 789272928256790527\n",
      "...1395 tweets downloaded so far\n",
      "getting tweets before 781490806817366016\n",
      "...1595 tweets downloaded so far\n",
      "getting tweets before 772509287012704255\n",
      "...1795 tweets downloaded so far\n",
      "getting tweets before 759008258065510399\n",
      "...1995 tweets downloaded so far\n",
      "getting tweets before 748131629235765247\n",
      "...2195 tweets downloaded so far\n",
      "getting tweets before 734173054801039359\n",
      "...2393 tweets downloaded so far\n",
      "getting tweets before 726026976679448575\n",
      "...2592 tweets downloaded so far\n",
      "getting tweets before 723883307906174975\n",
      "...2791 tweets downloaded so far\n",
      "getting tweets before 721741682216083455\n",
      "...2989 tweets downloaded so far\n",
      "getting tweets before 719675652828303359\n",
      "...3187 tweets downloaded so far\n",
      "getting tweets before 716729038773964799\n",
      "...3224 tweets downloaded so far\n",
      "getting tweets before 716304071011885056\n",
      "...3224 tweets downloaded so far\n",
      "HillaryClinton\n",
      "getting tweets before 795440369039187967\n",
      "...400 tweets downloaded so far\n",
      "getting tweets before 793513555035291647\n",
      "...600 tweets downloaded so far\n",
      "getting tweets before 790730465175011327\n",
      "...800 tweets downloaded so far\n",
      "getting tweets before 788909459586314239\n",
      "...1000 tweets downloaded so far\n",
      "getting tweets before 785894612686757887\n",
      "...1200 tweets downloaded so far\n",
      "getting tweets before 783657327396265984\n",
      "...1400 tweets downloaded so far\n",
      "getting tweets before 781890393185058815\n",
      "...1600 tweets downloaded so far\n",
      "getting tweets before 780081425714335746\n",
      "...1800 tweets downloaded so far\n",
      "getting tweets before 776782482352009215\n",
      "...2000 tweets downloaded so far\n",
      "getting tweets before 774021824053116927\n",
      "...2200 tweets downloaded so far\n",
      "getting tweets before 770291230966251519\n",
      "...2400 tweets downloaded so far\n",
      "getting tweets before 765288353419890687\n",
      "...2600 tweets downloaded so far\n",
      "getting tweets before 761602482774675456\n",
      "...2800 tweets downloaded so far\n",
      "getting tweets before 758805145190707199\n",
      "...3000 tweets downloaded so far\n",
      "getting tweets before 757973312706318335\n",
      "...3200 tweets downloaded so far\n",
      "getting tweets before 756314843376918527\n",
      "...3239 tweets downloaded so far\n",
      "getting tweets before 755950742310154240\n",
      "...3239 tweets downloaded so far\n",
      "BarackObama\n",
      "getting tweets before 773549731901349887\n",
      "...400 tweets downloaded so far\n",
      "getting tweets before 747152944047362048\n",
      "...600 tweets downloaded so far\n",
      "getting tweets before 723638166243016703\n",
      "...800 tweets downloaded so far\n",
      "getting tweets before 702936121659920383\n",
      "...1000 tweets downloaded so far\n",
      "getting tweets before 686966611064926207\n",
      "...1200 tweets downloaded so far\n",
      "getting tweets before 666354177400897539\n",
      "...1400 tweets downloaded so far\n",
      "getting tweets before 646093630315192319\n",
      "...1600 tweets downloaded so far\n",
      "getting tweets before 627914442227888127\n",
      "...1800 tweets downloaded so far\n",
      "getting tweets before 614505702296453119\n",
      "...2000 tweets downloaded so far\n",
      "getting tweets before 596717288566358015\n",
      "...2200 tweets downloaded so far\n",
      "getting tweets before 578308711833214975\n",
      "...2399 tweets downloaded so far\n",
      "getting tweets before 557738590818795519\n",
      "...2599 tweets downloaded so far\n",
      "getting tweets before 544580050012491775\n",
      "...2798 tweets downloaded so far\n",
      "getting tweets before 532909143573528575\n",
      "...2998 tweets downloaded so far\n",
      "getting tweets before 515870800424222719\n",
      "...3198 tweets downloaded so far\n",
      "getting tweets before 497412858599075841\n",
      "...3231 tweets downloaded so far\n",
      "getting tweets before 494959325102620672\n",
      "...3231 tweets downloaded so far\n",
      "BernieSanders\n",
      "getting tweets before 831288596778905599\n",
      "...400 tweets downloaded so far\n",
      "getting tweets before 814195718810238976\n"
     ]
    },
    {
     "ename": "TweepError",
     "evalue": "[{'code': 131, 'message': 'Internal error'}]",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTweepError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-e73c09dcaa48>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#loop through the twitter handles and scrape the tweets of each one into a file\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mhandles\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m     \u001b[0mscrapeTweetsToFile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandles\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-5-aa2adec5a815>\u001b[0m in \u001b[0;36mscrapeTweetsToFile\u001b[1;34m(handle, name)\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[1;32mwhile\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnewtweets\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'getting tweets before %s'\u001b[0m \u001b[1;33m%\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moldest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m         \u001b[0mnewtweets\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mapi\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muser_timeline\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscreen_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcount\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m200\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_id\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moldest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     14\u001b[0m         \u001b[0malltweets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnewtweets\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m         \u001b[0moldest\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0malltweets\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mid\u001b[0m \u001b[1;33m-\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\nick\\Anaconda3\\lib\\site-packages\\tweepy\\binder.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    243\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 245\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    246\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    247\u001b[0m     \u001b[1;31m# Set pagination mode\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\nick\\Anaconda3\\lib\\site-packages\\tweepy\\binder.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    227\u001b[0m                     \u001b[1;32mraise\u001b[0m \u001b[0mRateLimitError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merror_msg\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresp\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    228\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 229\u001b[1;33m                     \u001b[1;32mraise\u001b[0m \u001b[0mTweepError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merror_msg\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mapi_code\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mapi_error_code\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    230\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    231\u001b[0m             \u001b[1;31m# Parse the response payload\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTweepError\u001b[0m: [{'code': 131, 'message': 'Internal error'}]"
     ]
    }
   ],
   "source": [
    "#loop through the twitter handles and scrape the tweets of each one into a file\n",
    "for handle in handles:\n",
    "    scrapeTweetsToFile(handles[handle], handle)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}