Web_Crawlers/Multi_Thread_Article_Scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing Dependencies\n",
    "import requests\n",
    "from urllib.parse import urljoin, urlunsplit, urlparse\n",
    "import bs4\n",
    "from bs4 import BeautifulSoup\n",
    "from bs4.element import Comment\n",
    "\n",
    "from collections import Counter\n",
    "from string import punctuation\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import newspaper\n",
    "from newspaper import Article\n",
    "from newspaper import fulltext\n",
    "import requests\n",
    "\n",
    "\n",
    "#Tokenization Of Sentences\n",
    "import nltk\n",
    "# nltk.download('punkt')\n",
    "\n",
    "#Readability Scores\n",
    "import textstat\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Importing Libraries For Multi-threading\n",
    "from queue import Queue, Empty\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "from urllib.parse import urljoin, urlparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_dict = {\n",
    "'HTML_Content': [],\n",
    "'Full_Text': [],\n",
    "'URL': [],\n",
    "\n",
    "\n",
    "#### Article Information \n",
    "'Authors': [],\n",
    "'Publish_Date': [],\n",
    "'Article_Text': [],\n",
    "'Article_Text_Length': [],\n",
    "'Has_Top_Image': [],\n",
    "'Number_of_Movies': [],\n",
    "'Article_Is_Media_News': [],\n",
    "'Website_Has_Favicon': [],\n",
    "'Number_Of_Images':[],\n",
    "'Is_Valid_Body': [],\n",
    "\n",
    "#### NLP Features\n",
    "'Setences_Text': [],\n",
    "'Number_Of_Sentences': [],\n",
    "'Lexicon_Count': [],\n",
    "\n",
    "#### Readability Scores\n",
    "'Flesch_Reading_Ease_formula': [],\n",
    "'Flesch_Kincaid_Grade_Level': [],\n",
    "'FOG_Scale': [],\n",
    "'SMOG_Index': [],\n",
    "'ARI_Index': [],\n",
    "\n",
    "\n",
    "#### Meta_Data\n",
    "'Title_Text': [],\n",
    "'Title_Tag_Length' : [],\n",
    "'Meta_Description' : [],\n",
    "'Meta_Description_Length': [],\n",
    "\n",
    "\n",
    "#### Extract_page_features\n",
    "'Body_Content_Links': [],\n",
    "'Number_Of_Links': [],\n",
    "'Links_To_Text_Ratio': [],\n",
    "\n",
    "\n",
    "#### Technical Page Metrics\n",
    "'Page_Size_In_Bytes': [],\n",
    "'Plain_Text_Size': [],\n",
    "'Plain_Text_Rate': [],\n",
    "'Encoding': [],\n",
    "'SSL': []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping URL: https://gatheringdreams.com/affiliate-marketing-for-dummies/\n",
      "Scraping URL: https://www.entrepreneur.com/article/319017\n",
      "Scraping URL: https://maybethisway.com/blogging-tips/intro-affiliate-marketing/\n"
     ]
    }
   ],
   "source": [
    "## Importing Libraries For Multi-threading\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from queue import Queue, Empty\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "from urllib.parse import urljoin, urlparse\n",
    "\n",
    "class MultiThreadScraper:\n",
    " \n",
    "\n",
    "    def __init__(self, links, data_dict):\n",
    "        self.pool = ThreadPoolExecutor(max_workers=20)\n",
    "        self.scraped_pages = set([])\n",
    "        self.to_crawl = Queue()\n",
    "        self.start_url = 'https://gatheringdreams.com/affiliate-marketing-for-dummies/'\n",
    "        self.to_crawl.put(self.start_url)\n",
    "        self.links = ['https://www.entrepreneur.com/article/319017',\n",
    "                     'https://maybethisway.com/blogging-tips/intro-affiliate-marketing/']\n",
    "        \n",
    "        self.data_dict = data_dict\n",
    "    \n",
    "    \n",
    "    def technical_page_metrics(self, req):\n",
    "        #Page_Size_In_Bytes\n",
    "        page_size_in_bytes = len(req.content)\n",
    "\n",
    "        text = fulltext(req.text)\n",
    "\n",
    "        #Plain_text_size\n",
    "        plain_text_size = len(text)\n",
    "\n",
    "        #plain_text_rate --> plaintext rate value (plain_text_size / page_size)\n",
    "        plain_text_rate = (plain_text_size / page_size_in_bytes) * 100\n",
    "\n",
    "        #Encoding \n",
    "        encoding = req.encoding\n",
    "\n",
    "        #Detecting SSL Encryption\n",
    "        if 's' in req.url:\n",
    "            SSL = True\n",
    "        else:\n",
    "            SSL = False\n",
    "            \n",
    "        self.data_dict['Page_Size_In_Bytes'].append(page_size_in_bytes)\n",
    "        self.data_dict['Plain_Text_Size'].append(plain_text_size)\n",
    "        self.data_dict['Plain_Text_Rate'].append(plain_text_rate)\n",
    "        self.data_dict['Encoding'].append(encoding)\n",
    "        \n",
    "        if SSL == True:\n",
    "            self.data_dict['SSL'].append(1)\n",
    "        else:\n",
    "            self.data_dict['SSL'].append(0)\n",
    "            \n",
    "        \n",
    "    def get_article_links(self, article_text, soup):\n",
    "        soup = BeautifulSoup(article_text)\n",
    "        body_links = soup.find_all('a')\n",
    "        number_of_links = len(body_links)\n",
    "        #Links To Text Ratio\n",
    "        Links_To_Text_Ratio = len(body_links) / len(article_text)\n",
    "        return body_links, number_of_links, Links_To_Text_Ratio\n",
    "        \n",
    " \n",
    "    def parse_links(self, html):\n",
    "        for url in self.links:\n",
    "            if url not in self.scraped_pages:\n",
    "                self.to_crawl.put(url)\n",
    "        self.links.pop(0)\n",
    "        \n",
    " \n",
    "    def scrape_info(self, html, status_code):\n",
    "        article = self.article\n",
    "        article.download()\n",
    "        article.parse()\n",
    "        \n",
    "        #Scraping Article Metrics\n",
    "        author = article.authors\n",
    "        publish_date = article.publish_date\n",
    "        text = fulltext(html)\n",
    "        article_text = article.text\n",
    "        article_top_image = len(article.top_image)\n",
    "        article_movies = len(article.movies)\n",
    "        article_is_media_news = article.is_media_news()\n",
    "        favicon = article.meta_favicon\n",
    "        has_top_image = article.has_top_image()\n",
    "        number_of_images = len(article.images)\n",
    "        is_valid_body = article.is_valid_body()\n",
    "        \n",
    "        #Article NLP\n",
    "        article.nlp()\n",
    "        main_keywords = article.keywords\n",
    "        lexicon_count = textstat.lexicon_count(article_text, removepunct=True)\n",
    "        \n",
    "        #Extracting Sentences\n",
    "        sentences = nltk.sent_tokenize(article_text)\n",
    "        number_of_sentences = len(sentences)\n",
    "        \n",
    "        \n",
    "        #Extract Article Readability Scores\n",
    "        Flesch_Reading_Ease_formula = textstat.flesch_reading_ease(article_text)\n",
    "        Flesch_Kincaid_Grade_Level = textstat.flesch_kincaid_grade(article_text)\n",
    "        FOG_Scale = textstat.gunning_fog(article_text)\n",
    "        SMOG_Index = textstat.smog_index(article_text)\n",
    "        ARI_Index = textstat.automated_readability_index(article_text)\n",
    "        \n",
    "        \n",
    "        #BeautifulSoup Extraction\n",
    "        soup = BeautifulSoup(html)\n",
    "        #Get Title Tag Of Page\n",
    "        title_text = soup.title.getText()\n",
    "        #Title Tag Length In Characters\n",
    "        title_tag_length = len(title_text)\n",
    "        # First get the meta description tag\n",
    "        description = soup.find('meta', attrs={'name':'og:description'}) or soup.find('meta', attrs={'property':'description'}) or soup.find('meta', attrs={'name':'description'})\n",
    "        # If description meta tag was found, then get the content attribute and save it to db entry\n",
    "        description = description.get('content')\n",
    "        \n",
    "        #Extract Text Metrics From Article Text\n",
    "        body_links , number_of_links,  Links_To_Text_Ratio = self.get_article_links(article_text, soup)\n",
    "        \n",
    "        \n",
    "        #### Dictionary Inserts #####\n",
    "        self.data_dict['HTML_Content'].append(html)\n",
    "        self.data_dict['Full_Text'].append(text)\n",
    "        self.data_dict['Authors'].append(author)\n",
    "        self.data_dict['Publish_Date'].append(publish_date)\n",
    "        self.data_dict['Article_Text'].append(article_text) \n",
    "        self.data_dict['Article_Text_Length'].append(len(article_text))\n",
    "        \n",
    "        if has_top_image == True:\n",
    "            self.data_dict['Has_Top_Image'] .append(1)\n",
    "        else:\n",
    "            self.data_dict['Has_Top_Image'].append(np.nan)\n",
    "\n",
    "        if article_movies != 0:\n",
    "            self.data_dict['Number_of_Movies'].append(article_movies)\n",
    "        else:\n",
    "            self.data_dict['Number_of_Movies'].append(0)\n",
    "\n",
    "        if article_is_media_news == True:\n",
    "            self.data_dict['Article_Is_Media_News'].append(1)\n",
    "        else:\n",
    "            self.data_dict['Article_Is_Media_News'].append(0)\n",
    "\n",
    "        if favicon == True:\n",
    "            self.data_dict['Website_Has_Favicon'] .append(1)\n",
    "        else:\n",
    "            self.data_dict['Website_Has_Favicon'] .append(0)\n",
    "            \n",
    "        if number_of_images != 0:\n",
    "            self.data_dict['Number_Of_Images'].append(number_of_images)\n",
    "        else:\n",
    "            self.data_dict['Number_Of_Images'].append(0)\n",
    "\n",
    "        if is_valid_body == True:\n",
    "            self.data_dict['Is_Valid_Body'].append(1)\n",
    "        else: \n",
    "            self.data_dict['Is_Valid_Body'].append(0)\n",
    "        \n",
    "        ### NLP Features\n",
    "        self.data_dict['Setences_Text'].append(sentences)\n",
    "        self.data_dict['Number_Of_Sentences'].append(number_of_sentences)\n",
    "        self.data_dict['Lexicon_Count'].append(lexicon_count)\n",
    "        \n",
    "        ### Readability Scores   \n",
    "        self.data_dict['Flesch_Reading_Ease_formula'].append(Flesch_Reading_Ease_formula)\n",
    "        self.data_dict['Flesch_Kincaid_Grade_Level'].append(Flesch_Kincaid_Grade_Level)\n",
    "        self.data_dict['FOG_Scale'].append(FOG_Scale)\n",
    "        self.data_dict['SMOG_Index'].append(SMOG_Index)\n",
    "        self.data_dict['ARI_Index'].append(ARI_Index)\n",
    "        \n",
    "        self.data_dict['Title_Text'].append(title_text)\n",
    "        self.data_dict['Title_Tag_Length'].append(title_tag_length)\n",
    "        \n",
    "        if description:\n",
    "            self.data_dict['Meta_Description'].append(description)\n",
    "            self.data_dict['Meta_Description_Length'].append(len(description))\n",
    "        else:\n",
    "            self.data_dict['Meta_Description'].append(np.nan)\n",
    "            self.data_dict['Meta_Description_Length'].append(0)\n",
    "            \n",
    "        ### Additional Page Features\n",
    "        if len(body_links) != 0:\n",
    "            master_dict['Body_Content_Links'].append(body_links)\n",
    "        else:\n",
    "            master_dict['Body_Content_Links'].append(0)\n",
    "            \n",
    "        if number_of_links != 0:\n",
    "            master_dict['Number_Of_Links'].append(number_of_links)\n",
    "        else:\n",
    "            master_dict['Number_Of_Links'].append(0)\n",
    "            \n",
    "        if Links_To_Text_Ratio != 0:\n",
    "            master_dict['Links_To_Text_Ratio'].append(Links_To_Text_Ratio)\n",
    "        else:\n",
    "            master_dict['Links_To_Text_Ratio'].append(0)\n",
    " \n",
    "   \n",
    "    def post_scrape_callback(self, res):\n",
    "        result = res.result()\n",
    "        self.data_dict['URL'].append(result.url)\n",
    "        status = result.status_code\n",
    "        if result and result.status_code == 200:\n",
    "            self.parse_links(result.text)\n",
    "            self.scrape_info(result.text, status)\n",
    "            self.technical_page_metrics(result)\n",
    " \n",
    "   \n",
    "    def scrape_page(self, url):\n",
    "        try:\n",
    "            res = requests.get(url, timeout=(3, 30))\n",
    "            self.article = Article(url)\n",
    "            return res\n",
    "        except requests.RequestException:\n",
    "            return\n",
    " \n",
    "    \n",
    "    def run_scraper(self):\n",
    "        while True:\n",
    "            try:\n",
    "                target_url = self.to_crawl.get(timeout=6)\n",
    "                if target_url not in self.scraped_pages:\n",
    "                    print(\"Scraping URL: {}\".format(target_url))\n",
    "                    self.scraped_pages.add(target_url)\n",
    "                    job = self.pool.submit(self.scrape_page, target_url)\n",
    "                    job.add_done_callback(self.post_scrape_callback)\n",
    "            except Empty:\n",
    "                return\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "                continue\n",
    "               \n",
    "            \n",
    "if __name__ == '__main__':\n",
    "    s = MultiThreadScraper(\"https://edmundmartin.com\", master_dict)\n",
    "    s.run_scraper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://gatheringdreams.com/affiliate-marketing-for-dummies/',\n",
       " 'https://www.entrepreneur.com/article/319017',\n",
       " 'https://maybethisway.com/blogging-tips/intro-affiliate-marketing/']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.data_dict['URL'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Learn three simple strategies to help you stack on more revenue without the headaches of fulfillment and customer support.',\n",
       " 'A step-by-step affiliate marketing for dummies guide: everything you need to know to make your first sale if you are a beginner in affiliate marketing!']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.data_dict['Meta_Description']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HTML_Content 2\n",
      "Full_Text 2\n",
      "URL 2\n",
      "Authors 2\n",
      "Publish_Date 2\n",
      "Article_Text 2\n",
      "Article_Text_Length 2\n",
      "Has_Top_Image 2\n",
      "Number_of_Movies 2\n",
      "Article_Is_Media_News 2\n",
      "Website_Has_Favicon 2\n",
      "Number_Of_Images 2\n",
      "Is_Valid_Body 2\n",
      "Setences_Text 2\n",
      "Number_Of_Sentences 2\n",
      "Lexicon_Count 2\n",
      "Flesch_Reading_Ease_formula 2\n",
      "Flesch_Kincaid_Grade_Level 2\n",
      "FOG_Scale 2\n",
      "SMOG_Index 2\n",
      "ARI_Index 2\n",
      "Title_Text 2\n",
      "Title_Tag_Length 2\n",
      "Meta_Description 2\n",
      "Meta_Description_Length 2\n",
      "Body_Content_Links 2\n",
      "Number_Of_Links 2\n",
      "Links_To_Text_Ratio 2\n",
      "Page_Size_In_Bytes 2\n",
      "Plain_Text_Size 2\n",
      "Plain_Text_Rate 2\n",
      "Encoding 2\n",
      "SSL 2\n"
     ]
    }
   ],
   "source": [
    "for key, value in s.data_dict.items():\n",
    "    print(key, len(value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = requests.get('https://maybethisway.com/blogging-tips/intro-affiliate-marketing/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "s.data_dict['URL'] = s.data_dict['URL'][1:]x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HTML_Content                                    object\n",
       "Full_Text                                       object\n",
       "URL                                             object\n",
       "Authors                                         object\n",
       "Publish_Date                   datetime64[ns, tzutc()]\n",
       "Article_Text                                    object\n",
       "Article_Text_Length                              int64\n",
       "Has_Top_Image                                    int64\n",
       "Number_of_Movies                                 int64\n",
       "Article_Is_Media_News                            int64\n",
       "Website_Has_Favicon                              int64\n",
       "Number_Of_Images                                 int64\n",
       "Is_Valid_Body                                    int64\n",
       "Setences_Text                                   object\n",
       "Number_Of_Sentences                              int64\n",
       "Lexicon_Count                                    int64\n",
       "Flesch_Reading_Ease_formula                    float64\n",
       "Flesch_Kincaid_Grade_Level                     float64\n",
       "FOG_Scale                                      float64\n",
       "SMOG_Index                                     float64\n",
       "ARI_Index                                      float64\n",
       "Title_Text                                      object\n",
       "Title_Tag_Length                                 int64\n",
       "Meta_Description                                object\n",
       "Meta_Description_Length                          int64\n",
       "Body_Content_Links                               int64\n",
       "Number_Of_Links                                  int64\n",
       "Links_To_Text_Ratio                              int64\n",
       "Page_Size_In_Bytes                               int64\n",
       "Plain_Text_Size                                  int64\n",
       "Plain_Text_Rate                                float64\n",
       "Encoding                                        object\n",
       "SSL                                              int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(s.data_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#How to debug function callbacks....\n",
    "x, y = s.scrape_page('https://edmundmartin.com')\n",
    "s.post_scrape_callback(x, y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}