basic_plus_beautifulsoup4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target: http://dataquestio.github.io/web-scraping-pages/simple.html\n"
     ]
    }
   ],
   "source": [
    "# This code is based off a tutorial found at:\n",
    "# https://www.dataquest.io/blog/web-scraping-tutorial-python/\n",
    "# This bit of code builds upon basic_request_usage notebook to\n",
    "# demonstrate parsing of the HTML.\n",
    "import requests\n",
    "from bs4 import BeautifulSoup as bs\n",
    "\n",
    "# Designating a 'target', not neccessary to do as variable\n",
    "target = \"http://dataquestio.github.io/web-scraping-pages/simple.html\"\n",
    "print('Target: {}'.format(target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Response object: <Response [200]>\n",
      "\n",
      "Status code: 200\n",
      "\n",
      "HTML code:\n",
      "b'<!DOCTYPE html>\\n<html>\\n    <head>\\n        <title>A simple example page</title>\\n    </head>\\n    <body>\\n        <p>Here is some simple content for this page.</p>\\n    </body>\\n</html>'\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Request a webpage using the requests.get() method\n",
    "page = requests.get(target)\n",
    "\n",
    "# Display the response from that page\n",
    "# The object obtained is a response object, the below will print the response\n",
    "print('Response object: {}\\n'.format(page))\n",
    "\n",
    "# The response object also has a status code, indicating that the page\n",
    "# downloaded successfully (status code 200)\n",
    "print('Status code: {}\\n'.format(page.status_code))\n",
    "\n",
    "# Display the HTML content of the page using content property\n",
    "print('HTML code:\\n{}\\n'.format(page.content))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restructuring HTML printout:\n",
      "\n",
      "<!DOCTYPE html>\n",
      "<html>\n",
      " <head>\n",
      "  <title>\n",
      "   A simple example page\n",
      "  </title>\n",
      " </head>\n",
      " <body>\n",
      "  <p>\n",
      "   Here is some simple content for this page.\n",
      "  </p>\n",
      " </body>\n",
      "</html>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Pass the page downloaded to BeautifulSoup - specifically, the HTML\n",
    "soup = bs(page.content, 'html.parser')\n",
    "\n",
    "# BeautifulSoup will help format the contents to be more \"human readable\"\n",
    "print('Restructuring HTML printout:\\n')\n",
    "print('{}\\n'.format(soup.prettify()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "List child elements of the HTML document:\n",
      "\n",
      "['html', '\\n', <html>\n",
      "<head>\n",
      "<title>A simple example page</title>\n",
      "</head>\n",
      "<body>\n",
      "<p>Here is some simple content for this page.</p>\n",
      "</body>\n",
      "</html>]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Moving through the elements in the HTML\n",
    "# Select all elements at the top level of the page using the children property\n",
    "# of soup.  Children returns a list generator so list function is called.\n",
    "print('List child elements of the HTML document:\\n')\n",
    "\n",
    "print('{}\\n'.format(list(soup.children)))\n",
    "# The above shows that there are two tags at the top level of the page -- the\n",
    "# initial <!DOCTYPE html> tag and the <html> tag.  There is also a newline\n",
    "# character in the list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Show the \"type\" of each child element:\n",
      "\n",
      "[<class 'bs4.element.Doctype'>, <class 'bs4.element.NavigableString'>, <class 'bs4.element.Tag'>]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Display what each element in the list is\n",
    "print('Show the \"type\" of each child element:\\n')\n",
    "\n",
    "print('{}\\n'.format([type(item) for item in list(soup.children)]))\n",
    "# Each item is a BeautifulSoup object.  The first is a Doctype object, which\n",
    "# contains the information about the type of document.  The second is a\n",
    "# NavigableString, which represents text found in the HTML document.  The last\n",
    "# is a Tag object, which contains other nested tags.  This is the most\n",
    "# important object type and the one dealt with most often."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selecting \"html\" child from the HTML document.\n",
      "\n",
      "<html>\n",
      "<head>\n",
      "<title>A simple example page</title>\n",
      "</head>\n",
      "<body>\n",
      "<p>Here is some simple content for this page.</p>\n",
      "</body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "# Select the html tag and its children by taking the third item in the list.\n",
    "print('Selecting \"html\" child from the HTML document.\\n')\n",
    "html = list(soup.children)[2]\n",
    "\n",
    "print(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selecting \"body\" tag.\n",
      "\n",
      "<body>\n",
      "<p>Here is some simple content for this page.</p>\n",
      "</body>\n"
     ]
    }
   ],
   "source": [
    "# The 'body' child is selected in order to get the 'p' tag's text.\n",
    "print('Selecting \"body\" tag.\\n')\n",
    "body = list(html.children)[3]\n",
    "\n",
    "print(body)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying contents of \"body\" tag:\n",
      "\n",
      "['\\n', <p>Here is some simple content for this page.</p>, '\\n']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Now, the contents of the 'p' tag can be grabbed by finding the children of\n",
    "# the body tag.\n",
    "print('Displaying contents of \"body\" tag:\\n')\n",
    "print('{}\\n'.format(list(body.children)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selecting \"p\" tag.\n",
      "\n",
      "<p>Here is some simple content for this page.</p>\n"
     ]
    }
   ],
   "source": [
    "# The 'p' tag can now be isolated from the list.\n",
    "print('Selecting \"p\" tag.\\n')\n",
    "p = list(body.children)[1]\n",
    "\n",
    "print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying contents of the \"p\" tag:\n",
      "\n",
      "Here is some simple content for this page.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# The 'get_text' method can be used to extract all of the text inside the tag.\n",
    "print('Displaying contents of the \"p\" tag:\\n')\n",
    "print('{}\\n'.format(p.get_text()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}