json_play.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:b964cad082ebbb997965bbfac31f0578338ed85764dc4ecc7f3ebfb53affb0ca"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import json"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "jfile = open('json_1.txt')\n",
      "jdata = jfile.read()\n",
      "jfile.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "myjson = json.JSONDecoder()\n",
      "obj = myjson.decode(jdata)\n",
      "obj.keys()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "[u'search_metadata', u'statuses']"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for tweet in obj['statuses']:\n",
      "    print tweet['text']\n",
      "    print tweet['favorite_count']\n",
      "    print tweet['retweet_count']\n",
      "    print tweet['user']['followers_count']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "RT @nicholalexander: amazing watching the apps roll into the repo -https://t.co/oea693GBqN - amazing talent here. #hacktheuniverse\n",
        "0\n",
        "4\n",
        "1982\n",
        "On my way back to #hacktheuniverse at @AMNH! Excited to see what's been going on\n",
        "0\n",
        "0\n",
        "158\n",
        "Great to see all the #hacktheuniverse @AMNH guys bright and early this morning. So much fun and the music was great http://t.co/HclPp4L1zU\n",
        "0\n",
        "0\n",
        "146\n",
        "RT @kosamari: You wake up to this view :) Inside of the American Museum of Natural History #hacktheuniverse http://t.co/TCSiytvg5m\n",
        "0\n",
        "3\n",
        "146\n",
        "@shubhabala are you at #hacktheuniverse today? We're trying to come by later!\n",
        "0\n",
        "0\n",
        "1434\n",
        "@MacDiva So sad to miss this weekend, please share the results! Would love it if anyone wanted to come present on #HackTheUniverse @ITP_NYU\n",
        "0\n",
        "0\n",
        "9022\n",
        "LOOK I MADE A PLAYLIST PLEASE USE THIS http://t.co/KueNrl81Vd #hacktheuniverse #nomorearenarockplz\n",
        "0\n",
        "0\n",
        "115\n",
        "It's @Nrefzen!! \u201cAMNH #HacktheUniverse participants working through the night to build apps using Digital Universe http://t.co/Vhlsb39gOP\u201d\n",
        "0\n",
        "0\n",
        "2484\n",
        "RT @kosamari: You wake up to this view :) Inside of the American Museum of Natural History #hacktheuniverse http://t.co/TCSiytvg5m\n",
        "0\n",
        "3\n",
        "867\n",
        "22 projects created in 24 hours at #HackTheUniverse! INTERGALACTIC!!! @amnh\n",
        "2\n",
        "0\n",
        "2484\n",
        "#hacktheuniverse has become a HELLVERSE w this awful music. Who's down to make an intergalactic alliance for silence?!\n",
        "0\n",
        "0\n",
        "115\n",
        "Good morning, Hayden Planetarium \ud83c\udf1e\ud83c\udf1e #hacktheuniverse #amnh @ Hayden Planetarium @ The American\u2026 http://t.co/JpgRsSaF6R\n",
        "0\n",
        "0\n",
        "206\n",
        "#hacktheuniverse any front end / design folks want to help translate some wireframes into a prettier / interactive UI for Partiview?\n",
        "1\n",
        "0\n",
        "1386\n",
        "Breakfast while we hack away in the #amnh #hacktheuniverse #hackathon @ American Museum of Natural\u2026 http://t.co/INPaK9BdtK\n",
        "1\n",
        "0\n",
        "1431\n",
        "breakfast & hacking with team cosmos #cosmosis #hacktheuniverse @amnh @ Hayden Planetarium @ The\u2026 http://t.co/zzRoAmyoal\n",
        "1\n",
        "0\n",
        "241\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Function for putting together tweet text\n",
      "def tweet_text( decoded_obj ):\n",
      "    result = \"\"\n",
      "    for tweet in decoded_obj['statuses']:\n",
      "        result += tweet['text'] + \" \"\n",
      "    return result"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "all_tweets = tweet_text( obj )\n",
      "print all_tweets"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "RT @nicholalexander: amazing watching the apps roll into the repo -https://t.co/oea693GBqN - amazing talent here. #hacktheuniverse On my way back to #hacktheuniverse at @AMNH! Excited to see what's been going on Great to see all the #hacktheuniverse @AMNH guys bright and early this morning. So much fun and the music was great http://t.co/HclPp4L1zU RT @kosamari: You wake up to this view :) Inside of the American Museum of Natural History #hacktheuniverse http://t.co/TCSiytvg5m @shubhabala are you at #hacktheuniverse today? We're trying to come by later! @MacDiva So sad to miss this weekend, please share the results! Would love it if anyone wanted to come present on #HackTheUniverse @ITP_NYU LOOK I MADE A PLAYLIST PLEASE USE THIS http://t.co/KueNrl81Vd #hacktheuniverse #nomorearenarockplz It's @Nrefzen!! \u201cAMNH #HacktheUniverse participants working through the night to build apps using Digital Universe http://t.co/Vhlsb39gOP\u201d RT @kosamari: You wake up to this view :) Inside of the American Museum of Natural History #hacktheuniverse http://t.co/TCSiytvg5m 22 projects created in 24 hours at #HackTheUniverse! INTERGALACTIC!!! @amnh #hacktheuniverse has become a HELLVERSE w this awful music. Who's down to make an intergalactic alliance for silence?! Good morning, Hayden Planetarium \ud83c\udf1e\ud83c\udf1e #hacktheuniverse #amnh @ Hayden Planetarium @ The American\u2026 http://t.co/JpgRsSaF6R #hacktheuniverse any front end / design folks want to help translate some wireframes into a prettier / interactive UI for Partiview? Breakfast while we hack away in the #amnh #hacktheuniverse #hackathon @ American Museum of Natural\u2026 http://t.co/INPaK9BdtK breakfast & hacking with team cosmos #cosmosis #hacktheuniverse @amnh @ Hayden Planetarium @ The\u2026 http://t.co/zzRoAmyoal \n"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Try going through text file to capture json objects"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ff = open(\"jsondata.txt\")\n",
      "jdata_text_all = ff.read()\n",
      "ff.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import re"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "re_statuses = re.compile('{\\s*?\"statuses\":[\\s\\S]*?\\n}\\n')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "retest = re_statuses.search(jdata_text_all)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len(retest.group())\n",
      "retest.span()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 23,
       "text": [
        "(121, 89441)"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "otest = myjson.decode(retest.group())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print tweet_text(otest)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "RT @nicholalexander: amazing watching the apps roll into the repo -https://t.co/oea693GBqN - amazing talent here. #hacktheuniverse On my way back to #hacktheuniverse at @AMNH! Excited to see what's been going on Great to see all the #hacktheuniverse @AMNH guys bright and early this morning. So much fun and the music was great http://t.co/HclPp4L1zU RT @kosamari: You wake up to this view :) Inside of the American Museum of Natural History #hacktheuniverse http://t.co/TCSiytvg5m @shubhabala are you at #hacktheuniverse today? We're trying to come by later! @MacDiva So sad to miss this weekend, please share the results! Would love it if anyone wanted to come present on #HackTheUniverse @ITP_NYU LOOK I MADE A PLAYLIST PLEASE USE THIS http://t.co/KueNrl81Vd #hacktheuniverse #nomorearenarockplz It's @Nrefzen!! \u201cAMNH #HacktheUniverse participants working through the night to build apps using Digital Universe http://t.co/Vhlsb39gOP\u201d RT @kosamari: You wake up to this view :) Inside of the American Museum of Natural History #hacktheuniverse http://t.co/TCSiytvg5m 22 projects created in 24 hours at #HackTheUniverse! INTERGALACTIC!!! @amnh #hacktheuniverse has become a HELLVERSE w this awful music. Who's down to make an intergalactic alliance for silence?! Good morning, Hayden Planetarium \ud83c\udf1e\ud83c\udf1e #hacktheuniverse #amnh @ Hayden Planetarium @ The American\u2026 http://t.co/JpgRsSaF6R #hacktheuniverse any front end / design folks want to help translate some wireframes into a prettier / interactive UI for Partiview? Breakfast while we hack away in the #amnh #hacktheuniverse #hackathon @ American Museum of Natural\u2026 http://t.co/INPaK9BdtK breakfast & hacking with team cosmos #cosmosis #hacktheuniverse @amnh @ Hayden Planetarium @ The\u2026 http://t.co/zzRoAmyoal \n"
       ]
      }
     ],
     "prompt_number": 25
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "** IT'S ALIVE!! **"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ALL_TWEETS = \"\"\n",
      "\n",
      "keep_going = True\n",
      "leftover_string = jdata_text_all\n",
      "count = 0\n",
      "while( keep_going ):\n",
      "    json_chunk = re_statuses.search(leftover_string)\n",
      "    try:\n",
      "        jobj = myjson.decode(json_chunk.group())\n",
      "        print tweet_text(jobj)\n",
      "        ALL_TWEETS += tweet_text(jobj)\n",
      "        end_line = json_chunk.span()[1]\n",
      "        print \"END LINE = \", end_line\n",
      "        leftover_string = leftover_string[end_line+1:]\n",
      "    except:\n",
      "        keep_going = False"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "RT @nicholalexander: amazing watching the apps roll into the repo -https://t.co/oea693GBqN - amazing talent here. #hacktheuniverse On my way back to #hacktheuniverse at @AMNH! Excited to see what's been going on Great to see all the #hacktheuniverse @AMNH guys bright and early this morning. So much fun and the music was great http://t.co/HclPp4L1zU RT @kosamari: You wake up to this view :) Inside of the American Museum of Natural History #hacktheuniverse http://t.co/TCSiytvg5m @shubhabala are you at #hacktheuniverse today? We're trying to come by later! @MacDiva So sad to miss this weekend, please share the results! Would love it if anyone wanted to come present on #HackTheUniverse @ITP_NYU LOOK I MADE A PLAYLIST PLEASE USE THIS http://t.co/KueNrl81Vd #hacktheuniverse #nomorearenarockplz It's @Nrefzen!! \u201cAMNH #HacktheUniverse participants working through the night to build apps using Digital Universe http://t.co/Vhlsb39gOP\u201d RT @kosamari: You wake up to this view :) Inside of the American Museum of Natural History #hacktheuniverse http://t.co/TCSiytvg5m 22 projects created in 24 hours at #HackTheUniverse! INTERGALACTIC!!! @amnh #hacktheuniverse has become a HELLVERSE w this awful music. Who's down to make an intergalactic alliance for silence?! Good morning, Hayden Planetarium \ud83c\udf1e\ud83c\udf1e #hacktheuniverse #amnh @ Hayden Planetarium @ The American\u2026 http://t.co/JpgRsSaF6R #hacktheuniverse any front end / design folks want to help translate some wireframes into a prettier / interactive UI for Partiview? Breakfast while we hack away in the #amnh #hacktheuniverse #hackathon @ American Museum of Natural\u2026 http://t.co/INPaK9BdtK breakfast & hacking with team cosmos #cosmosis #hacktheuniverse @amnh @ Hayden Planetarium @ The\u2026 http://t.co/zzRoAmyoal \n",
        "END LINE =  89441\n",
        "Good morning, Hayden Planetarium \ud83c\udf1e\ud83c\udf1e #hacktheuniverse #amnh @ Hayden Planetarium @ The American\u2026 http://t.co/JpgRsSaF6R breakfast & hacking with team cosmos #cosmosis #hacktheuniverse @amnh @ Hayden Planetarium @ The\u2026 http://t.co/zzRoAmyoal RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 RT @joespens: #HackTheUniverse in the Hayden Planetarium at @AMNH! http://t.co/7sDWjJASaR I just liked \"Clip From \"Dark Universe\" | Hayden Planetarium, American Museum of Natural History\" on Vimeo: http://t.co/YX3pQYOaoQ RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 RT @joespens: #HackTheUniverse in the Hayden Planetarium at @AMNH! http://t.co/7sDWjJASaR sleeping over at the hayden planetarium in the natural history museum #nightatthemuseum ... http://t.co/TZmiQEvHtu http://t.co/XJTDYV3D7T RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 Spending the night in the Hall of the Universe. No, for real - Hayden Planetarium, y'all. Hope Neil\u2026 http://t.co/jyzzXnqt6K RT @palebluejen: Thrilled for tonight\u2019s slumber party hackathon at the Hayden Planetarium! #HackTheUniverse http://t.co/vMj1LJbUFX RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 \n",
        "END LINE =  135876\n",
        "LOOK I MADE A PLAYLIST PLEASE USE THIS http://t.co/KueNrl81Vd #hacktheuniverse #nomorearenarockplz RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 Hack the Universe - http://t.co/VH8tBThEFc Hack the Universe - http://t.co/QJW7GXrvcN @pruss thought you might find this interesting via @AMNH: #HacktheUniverse http://t.co/ovZdNGEKxn\" RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 \n",
        "END LINE =  118858\n"
       ]
      }
     ],
     "prompt_number": 45
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print jdata_text_all[89442:][135877:][118859:]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 46
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print ALL_TWEETS"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "RT @nicholalexander: amazing watching the apps roll into the repo -https://t.co/oea693GBqN - amazing talent here. #hacktheuniverse On my way back to #hacktheuniverse at @AMNH! Excited to see what's been going on Great to see all the #hacktheuniverse @AMNH guys bright and early this morning. So much fun and the music was great http://t.co/HclPp4L1zU RT @kosamari: You wake up to this view :) Inside of the American Museum of Natural History #hacktheuniverse http://t.co/TCSiytvg5m @shubhabala are you at #hacktheuniverse today? We're trying to come by later! @MacDiva So sad to miss this weekend, please share the results! Would love it if anyone wanted to come present on #HackTheUniverse @ITP_NYU LOOK I MADE A PLAYLIST PLEASE USE THIS http://t.co/KueNrl81Vd #hacktheuniverse #nomorearenarockplz It's @Nrefzen!! \u201cAMNH #HacktheUniverse participants working through the night to build apps using Digital Universe http://t.co/Vhlsb39gOP\u201d RT @kosamari: You wake up to this view :) Inside of the American Museum of Natural History #hacktheuniverse http://t.co/TCSiytvg5m 22 projects created in 24 hours at #HackTheUniverse! INTERGALACTIC!!! @amnh #hacktheuniverse has become a HELLVERSE w this awful music. Who's down to make an intergalactic alliance for silence?! Good morning, Hayden Planetarium \ud83c\udf1e\ud83c\udf1e #hacktheuniverse #amnh @ Hayden Planetarium @ The American\u2026 http://t.co/JpgRsSaF6R #hacktheuniverse any front end / design folks want to help translate some wireframes into a prettier / interactive UI for Partiview? Breakfast while we hack away in the #amnh #hacktheuniverse #hackathon @ American Museum of Natural\u2026 http://t.co/INPaK9BdtK breakfast & hacking with team cosmos #cosmosis #hacktheuniverse @amnh @ Hayden Planetarium @ The\u2026 http://t.co/zzRoAmyoal Good morning, Hayden Planetarium \ud83c\udf1e\ud83c\udf1e #hacktheuniverse #amnh @ Hayden Planetarium @ The American\u2026 http://t.co/JpgRsSaF6R breakfast & hacking with team cosmos #cosmosis #hacktheuniverse @amnh @ Hayden Planetarium @ The\u2026 http://t.co/zzRoAmyoal RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 RT @joespens: #HackTheUniverse in the Hayden Planetarium at @AMNH! http://t.co/7sDWjJASaR I just liked \"Clip From \"Dark Universe\" | Hayden Planetarium, American Museum of Natural History\" on Vimeo: http://t.co/YX3pQYOaoQ RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 RT @joespens: #HackTheUniverse in the Hayden Planetarium at @AMNH! http://t.co/7sDWjJASaR sleeping over at the hayden planetarium in the natural history museum #nightatthemuseum ... http://t.co/TZmiQEvHtu http://t.co/XJTDYV3D7T RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 Spending the night in the Hall of the Universe. No, for real - Hayden Planetarium, y'all. Hope Neil\u2026 http://t.co/jyzzXnqt6K RT @palebluejen: Thrilled for tonight\u2019s slumber party hackathon at the Hayden Planetarium! #HackTheUniverse http://t.co/vMj1LJbUFX RT @AMNH: #HacktheUniverse participants will build applications using the Museum's Digital Universe data http://t.co/gtt386NI4A http://t.co\u2026 LOOK I MADE A PLAYLIST PLEASE USE THIS http://t.co/KueNrl81Vd #hacktheuniverse #nomorearenarockplz RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 Hack the Universe - http://t.co/VH8tBThEFc Hack the Universe - http://t.co/QJW7GXrvcN @pruss thought you might find this interesting via @AMNH: #HacktheUniverse http://t.co/ovZdNGEKxn\" RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 RT @AMNH: For the next 24hrs, over 100 people will be participating in the Museum's first-ever hackathon! #HacktheUniverse http://t.co/wN1F\u2026 \n"
       ]
      }
     ],
     "prompt_number": 47
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      " ## Making a word cloud"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.feature_extraction.text import CountVectorizer"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## http://peekaboo-vision.blogspot.com/2012/11/a-wordcloud-in-python.html\n",
      "cv = CountVectorizer(min_df = 0, decode_error=\"ignore\", stop_words=\"english\")\n",
      "counts = cv.fit_transform([ALL_TWEETS])\n",
      "words  = np.array(cv.get_feature_names())\n",
      "counts = counts / max(counts.data)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 87
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "words"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 88,
       "text": [
        "array([u'100', u'22', u'24', u'24hrs', u'7sdwjjasar', u'alliance',\n",
        "       u'amazing', u'american', u'amnh', u'applications', u'apps', u'away',\n",
        "       u'awful', u'breakfast', u'bright', u'build', u'clip', u'come',\n",
        "       u'cosmos', u'cosmosis', u'created', u'dark', u'data', u'design',\n",
        "       u'digital', u'early', u'end', u'excited', u'folks', u'fun',\n",
        "       u'going', u'good', u'great', u'gtt386ni4a', u'guys', u'hack',\n",
        "       u'hackathon', u'hacking', u'hacktheuniverse', u'hall', u'hayden',\n",
        "       u'hclpp4l1zu', u'hellverse', u'help', u'history', u'hope', u'hours',\n",
        "       u'http', u'https', u'inpak9bdtk', u'inside', u'interactive',\n",
        "       u'interesting', u'intergalactic', u'itp_nyu', u'joespens',\n",
        "       u'jpgrssaf6r', u'just', u'jyzzxnqt6k', u'kosamari', u'kuenrl81vd',\n",
        "       u'later', u'liked', u'look', u'love', u'macdiva', u'make', u'miss',\n",
        "       u'morning', u'museum', u'music', u'natural', u'neil',\n",
        "       u'nicholalexander', u'night', u'nightatthemuseum',\n",
        "       u'nomorearenarockplz', u'nrefzen', u'oea693gbqn', u'ovzdngekxn',\n",
        "       u'palebluejen', u'participants', u'participating', u'partiview',\n",
        "       u'party', u'people', u'planetarium', u'playlist', u'present',\n",
        "       u'prettier', u'projects', u'pruss', u'qjw7gxrvcn', u'real', u'repo',\n",
        "       u'results', u'roll', u'rt', u'sad', u'share', u'shubhabala',\n",
        "       u'silence', u'sleeping', u'slumber', u'spending', u'talent',\n",
        "       u'tcsiytvg5m', u'team', u'thought', u'thrilled', u'today',\n",
        "       u'tonight', u'translate', u'trying', u'tzmiqevhtu', u'ui',\n",
        "       u'universe', u'use', u'using', u'vh8tbthefc', u'vhlsb39gop',\n",
        "       u'view', u'vimeo', u'vmj1ljbufx', u'wake', u'want', u'wanted',\n",
        "       u'watching', u'way', u'weekend', u'wireframes', u'wn1f', u'working',\n",
        "       u'xjtdyv3d7t', u'yx3pqyoaoq', u'zzroamyoal'], \n",
        "      dtype='<U18')"
       ]
      }
     ],
     "prompt_number": 88
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "counts.data"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 89,
       "text": [
        "array([ 0.52173913,  0.02173913,  0.04347826,  0.02173913,  0.04347826,\n",
        "        0.02173913,  0.02173913,  0.02173913,  0.02173913,  0.02173913,\n",
        "        0.86956522,  0.02173913,  0.65217391,  0.02173913,  0.02173913,\n",
        "        0.04347826,  0.02173913,  0.02173913,  0.02173913,  0.06521739,\n",
        "        0.02173913,  0.04347826,  1.        ,  0.02173913,  0.04347826,\n",
        "        0.04347826,  0.04347826,  0.04347826,  0.13043478,  0.5       ,\n",
        "        0.10869565,  0.08695652,  0.04347826,  0.02173913,  0.02173913,\n",
        "        0.02173913,  0.04347826,  0.02173913,  0.02173913,  0.02173913,\n",
        "        0.02173913,  0.02173913,  0.02173913,  0.02173913,  0.02173913,\n",
        "        0.02173913,  0.02173913,  0.02173913,  0.04347826,  0.04347826,\n",
        "        0.04347826,  0.04347826,  0.04347826,  0.02173913,  0.17391304,\n",
        "        0.02173913,  0.04347826,  0.17391304,  0.17391304,  0.17391304,\n",
        "        0.26086957,  0.02173913,  0.02173913,  0.02173913,  0.02173913,\n",
        "        0.02173913,  0.02173913,  0.04347826,  0.02173913,  0.02173913,\n",
        "        0.02173913,  0.02173913,  0.02173913,  0.04347826,  0.26086957,\n",
        "        0.26086957,  0.04347826,  0.02173913,  0.02173913,  0.02173913,\n",
        "        0.02173913,  0.02173913,  0.02173913,  0.02173913,  0.02173913,\n",
        "        0.02173913,  0.02173913,  0.02173913,  0.06521739,  0.06521739,\n",
        "        0.02173913,  0.2826087 ,  0.02173913,  0.04347826,  0.04347826,\n",
        "        0.04347826,  0.04347826,  0.04347826,  0.15217391,  0.15217391,\n",
        "        0.15217391,  0.04347826,  0.04347826,  0.02173913,  0.02173913,\n",
        "        0.02173913,  0.02173913,  0.02173913,  0.02173913,  0.02173913,\n",
        "        0.02173913,  0.02173913,  0.02173913,  0.02173913,  0.02173913,\n",
        "        0.02173913,  0.02173913,  0.02173913,  0.02173913,  0.02173913,\n",
        "        0.02173913,  0.02173913,  0.02173913,  0.02173913,  0.02173913,\n",
        "        0.23913043,  0.23913043,  0.23913043,  0.23913043,  0.23913043,\n",
        "        0.02173913,  0.02173913,  0.02173913,  0.02173913,  0.02173913,\n",
        "        0.02173913])"
       ]
      }
     ],
     "prompt_number": 89
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "isort = np.argsort(counts.data)[::-1]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 98
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for i in isort[0:10]:\n",
      "    print words[i], ':', counts.data[i]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "data : 1.0\n",
        "apps : 0.869565217391\n",
        "awful : 0.652173913043\n",
        "100 : 0.521739130435\n",
        "fun : 0.5\n",
        "pruss : 0.282608695652\n",
        "nightatthemuseum : 0.260869565217\n",
        "night : 0.260869565217\n",
        "kuenrl81vd : 0.260869565217\n",
        "weekend : 0.239130434783\n"
       ]
      }
     ],
     "prompt_number": 99
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}