real_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Web Scraping IRL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This file is an example of using scraping to accomplish a task.\n",
    "# Synopsis: I play the game Android Netrunner and would like to make a data set/database\n",
    "# of tournment winning decks with the name and the card list.  The names will be specially crafted\n",
    "# so that it identifies the set (it's important if you play the game), the tournament name, and the\n",
    "# faction (corp or runner).  All this information is important for my purposes, if you don't\n",
    "# understand what was just said don't worry about it!  The desired endstate is to have the card sets\n",
    "# then saved as a document with that name.  I'll be able to use that information for later analysis\n",
    "# and use in online collections so that I can read through each card.\n",
    "# The target site will be https://stimhack.com/tournament-decklists/ which contains tables\n",
    "# with hyperlinks to the winning decks for a given entry. This notebook will be a little messy as I'm intending\n",
    "# to leave in the bits where I figure this out (I.e. adjust my targeting on the site to scrape it, finding tables\n",
    "# following links, etc.).\n",
    "# Plan:\n",
    "# 1) Check robots.txt - ensure we're not breaking rules!\n",
    "# 2) Parse the target document so that we can programatically links and names\n",
    "# 3) Put the deck names and links in a data frame\n",
    "# 4) Follow each link to get the web doc in which the cards are listed\n",
    "# 5) Parse the page with card lists\n",
    "# 6) (finally) store the card lists with proper names for later use\n",
    "from requests import get\n",
    "from requests.exceptions import RequestException\n",
    "from contextlib import closing\n",
    "from bs4 import BeautifulSoup as bs\n",
    "from urllib.parse import urlparse\n",
    "import re\n",
    "\n",
    "\n",
    "# First, setup some get, response check, and \"log\" functions <- all borrowed from the mathematicians file.\n",
    "# These will allow us to re-use code better.\n",
    "# Simple Get:  Pass it a URL, if the URL is valid and the response looks like HTML, we'll send the content back\n",
    "#              to the caller.\n",
    "def simple_get(url):\n",
    "    # Attempts to get the content at 'url' by making an HTTP GET request.\n",
    "    # If the content-type of the response is some kind of HTML/XML, return the\n",
    "    # text content, otherwise return None\n",
    "    try:\n",
    "        # Contextlib's closing allows us to open, parse, and pass the contents of the url\n",
    "        # without having to explicitly close the connection\n",
    "        with closing(get(url, stream=True)) as resp:\n",
    "            if is_good_response(resp):\n",
    "                return resp.content\n",
    "            else:\n",
    "                return None\n",
    "    # Handle Request exception\n",
    "    except RequestException as e:\n",
    "        log_error('Error during requests to {} : {}'.format(url, str(e)))\n",
    "        return None\n",
    "    \n",
    "    \n",
    "def is_good_response(resp):\n",
    "    # Returns true if the response seems to be HTML, false otherwise\n",
    "    content_type = resp.headers['Content-Type'].lower()\n",
    "    print('[*] Get results content type: {}'.format(content_type))\n",
    "    return (resp.status_code == 200 and content_type is not None and\n",
    "            content_type.find('html') > -1)\n",
    "\n",
    "\n",
    "def log_error(e):\n",
    "    # It is always a good idea to log errors.  This function just prints them,\n",
    "    # but you can make it do anything\n",
    "    print(e)\n",
    "    \n",
    "    \n",
    "def find_robots(url):\n",
    "    # Check the domain for a robots.txt file. We must check this to ensure that\n",
    "    # we are not violating any rules of the site - please play nice when scraping!\n",
    "    # Get just the domain from the target\n",
    "    parser = urlparse(url)\n",
    "    rbts = str(parser.scheme + '://' + parser.netloc + '/robots.txt')\n",
    "    \n",
    "    try:\n",
    "        # Contextlib's closing allows us to open, parse, and pass the contents of the url\n",
    "        # without having to explicitly close the connection\n",
    "        with closing(get(rbts, stream=True)) as resp:\n",
    "            return (resp.content).decode('UTF-8')\n",
    "    except RequestException as e:\n",
    "        log_error('Error during requests to {} : {}'.format(rbts, str(e)))\n",
    "        return None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing 1 - Investigating the Target Structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*] Get results content type: text/html; charset=utf-8\n",
      "21\n",
      "All That Remains\n",
      "Up and Over\n",
      "First Contact\n"
     ]
    }
   ],
   "source": [
    "# The first thing one should do before conducting scraping is check to see what's allowed - please scrape responsibly.\n",
    "# I've checked this site and there is only one place not allowed.\n",
    "# test = find_robots('https://stimhack.com/tournament-decklists/')\n",
    "# print(test)\n",
    "\n",
    "# Instantiate a \"target\" variable to contain the value of the site we're scraping\n",
    "target = 'https://stimhack.com/tournament-decklists/'\n",
    "\n",
    "# Grab the page's HTML\n",
    "page = simple_get(target)\n",
    "\n",
    "# Parse the page's HTML\n",
    "html = bs(page, 'html.parser')\n",
    "\n",
    "# Display the HTML - using .prettify() method to make the output more \"human readable\"\n",
    "# print(html.prettify())\n",
    "# Find all <table> tags with class \"tablepres\" <- tried using dataTable (seemed more appropriate) but it\n",
    "# wouldn't grab all the tables.  After some tinkering, came up with the below\n",
    "tables = html.find_all('table', {'class': 'tablepress'})\n",
    "# Check the number of items in our list of tables - comes out to be 21 which is the number of tables there\n",
    "# are on the page (sigh) I counted them at the time of writing so trust me?\n",
    "print(len(tables))\n",
    "\n",
    "# After doing a little more digging I figured out how to get better delineators for my tables in the \"older\" section of\n",
    "# the document.  If this seems jarringly out of place it's because it should.  I need a unique name for each of my tables\n",
    "# in order to better sort the data later on\n",
    "older_expansions = html.find_all('h2', {'class': 'tablepress-table-name'})\n",
    "\n",
    "# Showing a little of the information we grabbed\n",
    "for name in older_expansions[:3]:\n",
    "    print(name.string)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing 2 - Parsing the tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*] Displaying the type of object each element in the list is:\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "<class 'bs4.element.Tag'>\n",
      "\n",
      "\n",
      "[*] The unique identifier for each of our tables:\n",
      "decklists\n",
      "tablepress-20\n",
      "tablepress-19\n",
      "\n",
      "\n",
      "[*] The headers for the tables:\n",
      "Table decklists columns:\n",
      "     DataPack\n",
      "     Location\n",
      "     Type\n",
      "     Players\n",
      "     Winner\n",
      "     Runner\n",
      "Table tablepress-20 columns:\n",
      "     No\n",
      "     Tournament\n",
      "     Players\n",
      "     Champion\n",
      "     Runner\n",
      "     Corp\n",
      "Table tablepress-19 columns:\n",
      "     No\n",
      "     Tournament\n",
      "     Players\n",
      "     Champion\n",
      "     Runner\n",
      "     Corp\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# At this point we've:\n",
    "#    Retrieved the document from the website containing the information target\n",
    "#    Parsed the document to pull out the tables of information\n",
    "# Now, we need to pull the information out and store it in a useful manner.\n",
    "# The first of those to things we'll do is pull the info out of a table:\n",
    "# Found that each instance of an item in the list can still be treated like a bs4\n",
    "# object...\n",
    "print('[*] Displaying the type of object each element in the list is:')\n",
    "for table in tables[:3]:\n",
    "    print(type(table))\n",
    "print('\\n')\n",
    "\n",
    "# So we continue to parse through each one finding a unique name for late use\n",
    "print('[*] The unique identifier for each of our tables:')\n",
    "for table in tables[:3]:\n",
    "    print(table['id'])\n",
    "print('\\n')\n",
    "\n",
    "# Now that we can name each table something, we'll find the headers next to create\n",
    "# the structure for our table\n",
    "print('[*] The headers for the tables:')\n",
    "for table in tables[:3]:\n",
    "    print('Table {} columns:'.format(table['id']))\n",
    "    for col in table.thead.find_all('th'):\n",
    "        print('     {}'.format(col.string))\n",
    "print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: blood_money_games_of_berkeley_berkeley_ca\n",
      "    Link: https://stimhack.com/gnk-games-of-berkeley-berkeley-ca-14-players/\n",
      "Name: 23_seconds_gamespace_kashiwagi\n",
      "    Link: https://stimhack.com/gnk-gamespace-kashiwagi-6-players/\n",
      "Name: 23_seconds_ropecon_helsinki_finland\n",
      "    Link: https://stimhack.com/regional-ropecon-helsinki-finland-23-players/\n"
     ]
    }
   ],
   "source": [
    "# Data Frame Structure:\n",
    "#    Name: expansion_tournament_faction\n",
    "#    Link: <hyperlink>\n",
    "#    Cards: [maybe list?]\n",
    "# The site has divided its records into (essentially) two different sets:\n",
    "# \"modern\" and \"older\" <- I made those up, just don't think about it\n",
    "# The table titled decklists I'm calling \"modern\" and everything else I'm\n",
    "# calling older.  We'll need to parse them a bit differently so let's put them\n",
    "# into two different variables so we can approach them appropriately and,\n",
    "# ultimately, store them in the same dataframe!\n",
    "# Had to switch this up because the tbody was not a part of the table for\n",
    "# some reason...\n",
    "modern = html.find('tbody', {'class': 'list'})\n",
    "\n",
    "# Had to use a bit o' the ol' re to get the job done, the expression below\n",
    "# says \"find all the tables (html.find_all('table'...)) where the id is\n",
    "# equal to 'tablepress' followed by anything else ({'id': re.compile('tablepress.*')})\"\n",
    "older = html.find_all('table', {'id': re.compile('tablepress.*')})\n",
    "\n",
    "# Now, we pull out the beginnings of our information - deck name\n",
    "# we need expansion_tournament_faction, however, we can't get faction\n",
    "# from this table, that's tied to the card list. So, we'll grab the first to pieces\n",
    "# and store that with the link as a dictionary\n",
    "names_links = {}\n",
    "for row in modern.find_all('tr'):\n",
    "    # Get the cell that has the data pack name and modify it\n",
    "    set_name = row.find('td', {'class': 'data-pack'}).string.replace(' ', '_').lower()\n",
    "    dual_purpose = row.find('td', {'class': 'location'})\n",
    "    tourny = dual_purpose.string.replace(' ', '_').replace(',', '').lower()\n",
    "    name = set_name + '_' + tourny\n",
    "    link = dual_purpose.a['href']\n",
    "    names_links[name] = link\n",
    "\n",
    "# Now the \"modern\" table has been parsed\n",
    "for key, value in list(names_links.items())[:3]:\n",
    "    print('Name: {}\\n    Link: {}'.format(key, value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Set honor_and_profit error 'NoneType' object has no attribute 'replace'\n",
      "Set double_time error 'NoneType' object has no attribute 'replace'\n",
      "Set double_time error 'NoneType' object has no attribute 'replace'\n",
      "Set double_time error 'NoneType' object has no attribute 'replace'\n",
      "Set fear_and_loathing error 'NoneType' object has no attribute 'replace'\n",
      "Set fear_and_loathing error 'NoneType' object has no attribute 'replace'\n",
      "Set fear_and_loathing error 'NoneType' object has no attribute 'replace'\n",
      "Set fear_and_loathing error 'NoneType' object has no attribute 'replace'\n",
      "Set fear_and_loathing error 'NoneType' object has no attribute 'replace'\n",
      "Set fear_and_loathing error 'NoneType' object has no attribute 'replace'\n",
      "Set fear_and_loathing error 'NoneType' object has no attribute 'replace'\n",
      "Set fear_and_loathing error 'NoneType' object has no attribute 'replace'\n",
      "Set true_colors error 'NoneType' object has no attribute 'replace'\n",
      "Set true_colors error 'NoneType' object has no attribute 'replace'\n",
      "Set second_thoughts error 'NoneType' object is not subscriptable\n",
      "Set opening_moves error 'NoneType' object has no attribute 'replace'\n",
      "Set what_lies_ahead error 'NoneType' object is not subscriptable\n"
     ]
    }
   ],
   "source": [
    "# Now we need to parse the \"older\" tables - remember, this isn't just one but a list of tables\n",
    "# Itterate over the tables\n",
    "counter = -1\n",
    "for table in older:\n",
    "    # Increment the counter to change the value from the old set list variable\n",
    "    counter += 1\n",
    "    for row in table.tbody.find_all('tr'):\n",
    "        try:\n",
    "            set_name = older_expansions[counter].string.replace(' ', '_').lower()\n",
    "            dual_purpose = row.find('td', {'class': 'column-2'})\n",
    "            tourny = dual_purpose.string.replace(',', '').replace(' ', '_').lower()\n",
    "            link = dual_purpose.a['href']\n",
    "            name = set_name + '_' + tourny\n",
    "            names_links[name] = link\n",
    "        except Exception as e:\n",
    "            print('Set {} error {}'.format(set_name, e))\n",
    "            continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*] Get results content type: text/html; charset=utf-8\n"
     ]
    }
   ],
   "source": [
    "# So now, names_links has a bunch of links to follow to get the covetted card sets.\n",
    "# While I wasn't able to parse out everything, for one reason or another, there's still\n",
    "# a LOT to work with\n",
    "#print('Number of links to follow: {}\\n'.format(len(names_links.keys())))\n",
    "\n",
    "# Need some links to test the next bit - parsing out the deck list\n",
    "test_1 = names_links['blood_money_games_of_berkeley_berkeley_ca']\n",
    "test_2 = names_links['23_seconds_gamespace_kashiwagi']\n",
    "test_3 = names_links['23_seconds_ropecon_helsinki_finland']\n",
    "\n",
    "card_page = simple_get(test_1)\n",
    "card_html = bs(card_page, 'html.parser')\n",
    "set_1 = card_html.find('div', {'class': 'wc-shortcodes-column-first'})\n",
    "# print(set_1.text)\n",
    "# print(type(set_1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vex'ahlia <- Something relavent for a single line\n",
      "(45 cards) <- wasn't in tag but single line relavent\n",
      "Kate \"Mac\" McCaffrey: Digital Tinker <- Something relavent for a single line\n",
      "Event (12) <- wasn't in tag but single line relavent\n",
      "3 <- place_holder\n",
      "3 x Career Fair < - Number and Card\n",
      " <- What got here?!\n",
      "3 <- place_holder\n",
      "3 x Diesel < - Number and Card\n",
      "1 <- place_holder\n",
      "1 x \"Freedom Through Equality\" < - Number and Card\n",
      "2 <- place_holder\n",
      "2 x Indexing < - Number and Card\n",
      "3 <- place_holder\n",
      "3 x Sure Gamble < - Number and Card\n",
      "Hardware (7) <- wasn't in tag but single line relavent\n",
      "2 <- place_holder\n",
      "2 x Clone Chip < - Number and Card\n",
      "2 <- place_holder\n",
      "2 x Mirror < - Number and Card\n",
      "1 <- place_holder\n",
      "1 x Plascrete Carapace < - Number and Card\n",
      "2 <- place_holder\n",
      "2 x R&D Interface < - Number and Card\n",
      "Resource (17) <- wasn't in tag but single line relavent\n",
      "2 <- place_holder\n",
      "2 x Beth Kilrain-Chang < - Number and Card\n",
      "3 <- place_holder\n",
      "3 x Daily Casts < - Number and Card\n",
      "2 <- place_holder\n",
      "2 x Film Critic < - Number and Card\n",
      "3 <- place_holder\n",
      "3 x Ghost Runner < - Number and Card\n",
      "2 <- place_holder\n",
      "2 x New Angeles City Hall < - Number and Card\n",
      "3 <- place_holder\n",
      "3 x Temujin Contract < - Number and Card\n",
      " <- What got here?!\n",
      "2 <- place_holder\n",
      "2 x The Turning Wheel < - Number and Card\n",
      " <- What got here?!\n",
      "Icebreaker (3) <- wasn't in tag but single line relavent\n",
      "1 <- place_holder\n",
      "1 x Corroder < - Number and Card\n",
      " <- What got here?!\n",
      "1 <- place_holder\n",
      "1 x Dagger < - Number and Card\n",
      "1 <- place_holder\n",
      "1 x Refractor < - Number and Card\n",
      "Program (6) <- wasn't in tag but single line relavent\n",
      "3 <- place_holder\n",
      "3 x Cloak < - Number and Card\n",
      "3 <- place_holder\n",
      "3 x Self-modifying Code < - Number and Card\n",
      "Built with <- wasn't in tag but single line relavent\n",
      "http://meteor.stimhack.com/ <- Something relavent for a single line\n"
     ]
    }
   ],
   "source": [
    "for linebreak in set_1.find_all('br'):\n",
    "    linebreak.extract()\n",
    "\n",
    "place_holder = ''\n",
    "for child in set_1.children:\n",
    "    try:\n",
    "        # The deck name and the card names are in a tags which have a text attribute.\n",
    "        # If the child does not it will drop down to the exception logic below.\n",
    "        try_var = child.text.strip()\n",
    "        # If that child object is a card name, there should have been a number before it.\n",
    "        # If there was a number the exception logic below would have stored that value in the place_holder variable.\n",
    "        if place_holder != '':\n",
    "            print('{} x {} < - Number and Card'.format(place_holder, try_var))\n",
    "            # reset the place_holder var to be empty\n",
    "            place_holder = ''\n",
    "            # Start the loop over\n",
    "            continue\n",
    "        # If placeholder has nothing in it, it's not a card bet we need to see if it should be printed on its own line\n",
    "        try:\n",
    "            # If the child wasn't a card it was instead a single line of text relavent for one line\n",
    "            # I.e. (<card_subtype> <count>), or the name of the deck\n",
    "            # if re.match(\"^[A-Za-z0-9_-]*$\", my_little_string)\n",
    "            # re.search('[a-zA-Z]', try_var)\n",
    "            if any(c.isalpha() for c in try_var):\n",
    "                print('{} <- Something relavent for a single line'.format(try_var))\n",
    "                continue\n",
    "        except TypeError:\n",
    "            # If the child contained no letters, it was something I didn't care to have\n",
    "            # I.e. the amount of influence (something specific to the game) a given card consumes\n",
    "            print('{} <- don\\'t need this'.format(try_var))\n",
    "    except Exception as e:\n",
    "        # print('Error occurred at {}: {}'.format(child, e))\n",
    "        # Get rid of spaces around the text - because that's what this is, just text, meaning not really a child element\n",
    "        # Looking at the code reveals that the number of cards there is in a specific deck isn't between any tags.\n",
    "        exc_var = str(child).strip()\n",
    "        try:\n",
    "            # Check the contents of the variable to see if it's a number\n",
    "            int(exc_var)\n",
    "            # If so, we want to save that number to print it on a line with the card to designate how many should\n",
    "            # be in the deck\n",
    "            place_holder = exc_var\n",
    "            print('{} <- place_holder'.format(exc_var))\n",
    "            continue\n",
    "        except Exception as e:\n",
    "            pass\n",
    "        if re.search('[a-zA-Z]', child):\n",
    "            print('{} <- wasn\\'t in tag but single line relavent'.format(exc_var))\n",
    "        else:\n",
    "            print('{} <- What got here?!'.format(exc_var))\n",
    "            continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3'"
      ]
     },
     "execution_count": 273,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}