reel_scrap.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import dependencies\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.wait import WebDriverWait\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.common.exceptions import NoSuchElementException, TimeoutException\n",
    "import time\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import config\n",
    "import json\n",
    "import os\n",
    "from urllib.parse import urlparse\n",
    "from selenium.webdriver.chrome.service import Service"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#specify the path to chromedriver.exe (download and save on your computer)\n",
    "serv = Service(executable_path=r'C:\\Users\\kirti\\chromedriver-win64\\chromedriver-win64\\chromedriver.exe')\n",
    "driver = webdriver.Chrome(service=serv)\n",
    "\n",
    "#open the webpage\n",
    "driver.get(\"https://www.instagram.com/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#target username\n",
    "username = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, \"input[name='username']\")))\n",
    "password = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, \"input[name='password']\")))\n",
    "\n",
    "#enter username and password\n",
    "username.clear()\n",
    "username.send_keys('your_username')\n",
    "password.clear()\n",
    "password.send_keys('your_password')\n",
    "\n",
    "#target the login button and click it\n",
    "button = WebDriverWait(driver, 2).until(EC.element_to_be_clickable((By.CSS_SELECTOR, \"button[type='submit']\"))).click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# not_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[2]/div/div/div[2]/div/div/div[1]/div[1]/div[2]/section/main/div/div/div/div/div')))\n",
    "\n",
    "# not_button.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# not_button2 = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[3]/div[1]/div/div[2]/div/div/div/div/div[2]/div/div/div[3]/button[2]')))\n",
    "# not_button2.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Wait up to 10 seconds for the search button to be clickable on the web page\n",
    "reel_button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'svg[aria-label=\"Explore\"]')))\n",
    "\n",
    "# Click the search button once it becomes clickable\n",
    "reel_button.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "NoSuchWindowException",
     "evalue": "Message: no such window: target window already closed\nfrom unknown error: web view not found\n  (Session info: chrome=123.0.6312.123)\nStacktrace:\n\tGetHandleVerifier [0x00007FF66E527032+63090]\n\t(No symbol) [0x00007FF66E492C82]\n\t(No symbol) [0x00007FF66E32EC65]\n\t(No symbol) [0x00007FF66E30CA7C]\n\t(No symbol) [0x00007FF66E39D687]\n\t(No symbol) [0x00007FF66E3B2AC1]\n\t(No symbol) [0x00007FF66E396D83]\n\t(No symbol) [0x00007FF66E3683A8]\n\t(No symbol) [0x00007FF66E369441]\n\tGetHandleVerifier [0x00007FF66E9225AD+4238317]\n\tGetHandleVerifier [0x00007FF66E95F70D+4488525]\n\tGetHandleVerifier [0x00007FF66E9579EF+4456495]\n\tGetHandleVerifier [0x00007FF66E600576+953270]\n\t(No symbol) [0x00007FF66E49E54F]\n\t(No symbol) [0x00007FF66E499224]\n\t(No symbol) [0x00007FF66E49935B]\n\t(No symbol) [0x00007FF66E489B94]\n\tBaseThreadInitThunk [0x00007FFFA28F257D+29]\n\tRtlUserThreadStart [0x00007FFFA36CAA48+40]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNoSuchWindowException\u001b[0m                     Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[7], line 15\u001b[0m\n\u001b[0;32m     12\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m10\u001b[39m)\n\u001b[0;32m     14\u001b[0m \u001b[38;5;66;03m# Parse the HTML\u001b[39;00m\n\u001b[1;32m---> 15\u001b[0m html \u001b[38;5;241m=\u001b[39m \u001b[43mdriver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpage_source\u001b[49m\n\u001b[0;32m     17\u001b[0m \u001b[38;5;66;03m# Create a BeautifulSoup object from the scraped HTML\u001b[39;00m\n\u001b[0;32m     18\u001b[0m soups\u001b[38;5;241m.\u001b[39mappend(BeautifulSoup(html, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhtml.parser\u001b[39m\u001b[38;5;124m'\u001b[39m))\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:448\u001b[0m, in \u001b[0;36mWebDriver.page_source\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    439\u001b[0m \u001b[38;5;129m@property\u001b[39m\n\u001b[0;32m    440\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpage_source\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mstr\u001b[39m:\n\u001b[0;32m    441\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Gets the source of the current page.\u001b[39;00m\n\u001b[0;32m    442\u001b[0m \n\u001b[0;32m    443\u001b[0m \u001b[38;5;124;03m    :Usage:\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    446\u001b[0m \u001b[38;5;124;03m            driver.page_source\u001b[39;00m\n\u001b[0;32m    447\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 448\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mCommand\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mGET_PAGE_SOURCE\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:347\u001b[0m, in \u001b[0;36mWebDriver.execute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    345\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcommand_executor\u001b[38;5;241m.\u001b[39mexecute(driver_command, params)\n\u001b[0;32m    346\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response:\n\u001b[1;32m--> 347\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merror_handler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    348\u001b[0m     response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_unwrap_value(response\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m))\n\u001b[0;32m    349\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py:229\u001b[0m, in \u001b[0;36mErrorHandler.check_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    227\u001b[0m         alert_text \u001b[38;5;241m=\u001b[39m value[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124malert\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    228\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m exception_class(message, screen, stacktrace, alert_text)  \u001b[38;5;66;03m# type: ignore[call-arg]  # mypy is not smart enough here\u001b[39;00m\n\u001b[1;32m--> 229\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exception_class(message, screen, stacktrace)\n",
      "\u001b[1;31mNoSuchWindowException\u001b[0m: Message: no such window: target window already closed\nfrom unknown error: web view not found\n  (Session info: chrome=123.0.6312.123)\nStacktrace:\n\tGetHandleVerifier [0x00007FF66E527032+63090]\n\t(No symbol) [0x00007FF66E492C82]\n\t(No symbol) [0x00007FF66E32EC65]\n\t(No symbol) [0x00007FF66E30CA7C]\n\t(No symbol) [0x00007FF66E39D687]\n\t(No symbol) [0x00007FF66E3B2AC1]\n\t(No symbol) [0x00007FF66E396D83]\n\t(No symbol) [0x00007FF66E3683A8]\n\t(No symbol) [0x00007FF66E369441]\n\tGetHandleVerifier [0x00007FF66E9225AD+4238317]\n\tGetHandleVerifier [0x00007FF66E95F70D+4488525]\n\tGetHandleVerifier [0x00007FF66E9579EF+4456495]\n\tGetHandleVerifier [0x00007FF66E600576+953270]\n\t(No symbol) [0x00007FF66E49E54F]\n\t(No symbol) [0x00007FF66E499224]\n\t(No symbol) [0x00007FF66E49935B]\n\t(No symbol) [0x00007FF66E489B94]\n\tBaseThreadInitThunk [0x00007FFFA28F257D+29]\n\tRtlUserThreadStart [0x00007FFFA36CAA48+40]\n"
     ]
    }
   ],
   "source": [
    "# Get the initial page height\n",
    "initial_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "\n",
    "# Create a list to store htmls\n",
    "soups = []\n",
    "count = 0\n",
    "while True:\n",
    "    # Scroll down to the bottom of the page\n",
    "    driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "\n",
    "    # Wait for a moment to allow new content to load (adjust as needed)\n",
    "    time.sleep(10)\n",
    "    \n",
    "    # Parse the HTML\n",
    "    html = driver.page_source\n",
    "    \n",
    "    # Create a BeautifulSoup object from the scraped HTML\n",
    "    soups.append(BeautifulSoup(html, 'html.parser'))\n",
    "\n",
    "    # Get the current page height\n",
    "    current_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "\n",
    "    if count == 5:\n",
    "        break  # Exit the loop when you can't scroll further\n",
    "\n",
    "    initial_height = current_height  # Update the initial height for the next iteration\n",
    "    count+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "before: 280, after: 120\n"
     ]
    }
   ],
   "source": [
    "# List to store the post image URLs\n",
    "post_urls = []\n",
    "\n",
    "for soup in soups:\n",
    "    # Find all anchor elements with href attributes\n",
    "    anchors = soup.find_all('a', href=True)\n",
    "    \n",
    "    # Filter URLs that start with \"/p/\" or \"/reel/\"\n",
    "    post_urls.extend([anchor['href'] for anchor in anchors if anchor['href'].startswith((\"/p/\", \"/reel/\"))])\n",
    "\n",
    "# Convert the list to a set to remove duplicates\n",
    "unique_post_urls = list(set(post_urls))\n",
    "\n",
    "print(f\"before: {len(post_urls)}, after: {len(unique_post_urls)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error processing URL /p/C5X3zuIPUnW/?next=%2F: Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"/html/body/div[2]/div/div/div[2]/div/div/div[1]/div[1]/div[2]/section/main/div/div[1]/div/div[2]/div/div[2]/div/div[1]/div/div[2]/div/span/div/span\"}\n",
      "  (Session info: chrome=123.0.6312.122); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#no-such-element-exception\n",
      "Stacktrace:\n",
      "\tGetHandleVerifier [0x00007FF733927032+63090]\n",
      "\t(No symbol) [0x00007FF733892C82]\n",
      "\t(No symbol) [0x00007FF73372EC65]\n",
      "\t(No symbol) [0x00007FF73377499D]\n",
      "\t(No symbol) [0x00007FF733774ADC]\n",
      "\t(No symbol) [0x00007FF7337B5B37]\n",
      "\t(No symbol) [0x00007FF73379701F]\n",
      "\t(No symbol) [0x00007FF7337B3412]\n",
      "\t(No symbol) [0x00007FF733796D83]\n",
      "\t(No symbol) [0x00007FF7337683A8]\n",
      "\t(No symbol) [0x00007FF733769441]\n",
      "\tGetHandleVerifier [0x00007FF733D225AD+4238317]\n",
      "\tGetHandleVerifier [0x00007FF733D5F70D+4488525]\n",
      "\tGetHandleVerifier [0x00007FF733D579EF+4456495]\n",
      "\tGetHandleVerifier [0x00007FF733A00576+953270]\n",
      "\t(No symbol) [0x00007FF73389E54F]\n",
      "\t(No symbol) [0x00007FF733899224]\n",
      "\t(No symbol) [0x00007FF73389935B]\n",
      "\t(No symbol) [0x00007FF733889B94]\n",
      "\tBaseThreadInitThunk [0x00007FFD11C0257D+29]\n",
      "\tRtlUserThreadStart [0x00007FFD1220AA48+40]\n",
      "\n",
      "Error processing URL /p/C4haSFlSfFq/?next=%2F: Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"/html/body/div[2]/div/div/div[2]/div/div/div[1]/div[1]/div[2]/section/main/div/div[1]/div/div[2]/div/div[2]/div/div[1]/div/div[2]/div/span/div/span\"}\n",
      "  (Session info: chrome=123.0.6312.122); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#no-such-element-exception\n",
      "Stacktrace:\n",
      "\tGetHandleVerifier [0x00007FF733927032+63090]\n",
      "\t(No symbol) [0x00007FF733892C82]\n",
      "\t(No symbol) [0x00007FF73372EC65]\n",
      "\t(No symbol) [0x00007FF73377499D]\n",
      "\t(No symbol) [0x00007FF733774ADC]\n",
      "\t(No symbol) [0x00007FF7337B5B37]\n",
      "\t(No symbol) [0x00007FF73379701F]\n",
      "\t(No symbol) [0x00007FF7337B3412]\n",
      "\t(No symbol) [0x00007FF733796D83]\n",
      "\t(No symbol) [0x00007FF7337683A8]\n",
      "\t(No symbol) [0x00007FF733769441]\n",
      "\tGetHandleVerifier [0x00007FF733D225AD+4238317]\n",
      "\tGetHandleVerifier [0x00007FF733D5F70D+4488525]\n",
      "\tGetHandleVerifier [0x00007FF733D579EF+4456495]\n",
      "\tGetHandleVerifier [0x00007FF733A00576+953270]\n",
      "\t(No symbol) [0x00007FF73389E54F]\n",
      "\t(No symbol) [0x00007FF733899224]\n",
      "\t(No symbol) [0x00007FF73389935B]\n",
      "\t(No symbol) [0x00007FF733889B94]\n",
      "\tBaseThreadInitThunk [0x00007FFD11C0257D+29]\n",
      "\tRtlUserThreadStart [0x00007FFD1220AA48+40]\n",
      "\n",
      "Error processing URL /p/C5lRtqKyKoB/?next=%2F: Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"/html/body/div[2]/div/div/div[2]/div/div/div[1]/div[1]/div[2]/section/main/div/div[1]/div/div[2]/div/div[2]/div/div[1]/div/div[2]/div/span/div/span\"}\n",
      "  (Session info: chrome=123.0.6312.122); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#no-such-element-exception\n",
      "Stacktrace:\n",
      "\tGetHandleVerifier [0x00007FF733927032+63090]\n",
      "\t(No symbol) [0x00007FF733892C82]\n",
      "\t(No symbol) [0x00007FF73372EC65]\n",
      "\t(No symbol) [0x00007FF73377499D]\n",
      "\t(No symbol) [0x00007FF733774ADC]\n",
      "\t(No symbol) [0x00007FF7337B5B37]\n",
      "\t(No symbol) [0x00007FF73379701F]\n",
      "\t(No symbol) [0x00007FF7337B3412]\n",
      "\t(No symbol) [0x00007FF733796D83]\n",
      "\t(No symbol) [0x00007FF7337683A8]\n",
      "\t(No symbol) [0x00007FF733769441]\n",
      "\tGetHandleVerifier [0x00007FF733D225AD+4238317]\n",
      "\tGetHandleVerifier [0x00007FF733D5F70D+4488525]\n",
      "\tGetHandleVerifier [0x00007FF733D579EF+4456495]\n",
      "\tGetHandleVerifier [0x00007FF733A00576+953270]\n",
      "\t(No symbol) [0x00007FF73389E54F]\n",
      "\t(No symbol) [0x00007FF733899224]\n",
      "\t(No symbol) [0x00007FF73389935B]\n",
      "\t(No symbol) [0x00007FF733889B94]\n",
      "\tBaseThreadInitThunk [0x00007FFD11C0257D+29]\n",
      "\tRtlUserThreadStart [0x00007FFD1220AA48+40]\n",
      "\n",
      "Error processing URL /p/C5fwoHFRcMF/?next=%2F: Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"/html/body/div[2]/div/div/div[2]/div/div/div[1]/div[1]/div[2]/section/main/div/div[1]/div/div[2]/div/div[2]/div/div[1]/div/div[2]/div/span/div/span\"}\n",
      "  (Session info: chrome=123.0.6312.122); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#no-such-element-exception\n",
      "Stacktrace:\n",
      "\tGetHandleVerifier [0x00007FF733927032+63090]\n",
      "\t(No symbol) [0x00007FF733892C82]\n",
      "\t(No symbol) [0x00007FF73372EC65]\n",
      "\t(No symbol) [0x00007FF73377499D]\n",
      "\t(No symbol) [0x00007FF733774ADC]\n",
      "\t(No symbol) [0x00007FF7337B5B37]\n",
      "\t(No symbol) [0x00007FF73379701F]\n",
      "\t(No symbol) [0x00007FF7337B3412]\n",
      "\t(No symbol) [0x00007FF733796D83]\n",
      "\t(No symbol) [0x00007FF7337683A8]\n",
      "\t(No symbol) [0x00007FF733769441]\n",
      "\tGetHandleVerifier [0x00007FF733D225AD+4238317]\n",
      "\tGetHandleVerifier [0x00007FF733D5F70D+4488525]\n",
      "\tGetHandleVerifier [0x00007FF733D579EF+4456495]\n",
      "\tGetHandleVerifier [0x00007FF733A00576+953270]\n",
      "\t(No symbol) [0x00007FF73389E54F]\n",
      "\t(No symbol) [0x00007FF733899224]\n",
      "\t(No symbol) [0x00007FF73389935B]\n",
      "\t(No symbol) [0x00007FF733889B94]\n",
      "\tBaseThreadInitThunk [0x00007FFD11C0257D+29]\n",
      "\tRtlUserThreadStart [0x00007FFD1220AA48+40]\n",
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[22], line 18\u001b[0m\n\u001b[0;32m     15\u001b[0m modified_url \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps://www.instagram.com/\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m url\n\u001b[0;32m     17\u001b[0m \u001b[38;5;66;03m# Navigate to the post URL\u001b[39;00m\n\u001b[1;32m---> 18\u001b[0m \u001b[43mdriver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodified_url\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     20\u001b[0m \u001b[38;5;66;03m# Wait for the page to load (you might need to adjust the wait time)\u001b[39;00m\n\u001b[0;32m     21\u001b[0m driver\u001b[38;5;241m.\u001b[39mimplicitly_wait(\u001b[38;5;241m15\u001b[39m)\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:356\u001b[0m, in \u001b[0;36mWebDriver.get\u001b[1;34m(self, url)\u001b[0m\n\u001b[0;32m    354\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget\u001b[39m(\u001b[38;5;28mself\u001b[39m, url: \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    355\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Loads a web page in the current browser session.\"\"\"\u001b[39;00m\n\u001b[1;32m--> 356\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mCommand\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mGET\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43murl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:345\u001b[0m, in \u001b[0;36mWebDriver.execute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    342\u001b[0m     \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msessionId\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m params:\n\u001b[0;32m    343\u001b[0m         params[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msessionId\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msession_id\n\u001b[1;32m--> 345\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcommand_executor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdriver_command\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    346\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response:\n\u001b[0;32m    347\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39merror_handler\u001b[38;5;241m.\u001b[39mcheck_response(response)\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\remote_connection.py:302\u001b[0m, in \u001b[0;36mRemoteConnection.execute\u001b[1;34m(self, command, params)\u001b[0m\n\u001b[0;32m    300\u001b[0m trimmed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_trim_large_entries(params)\n\u001b[0;32m    301\u001b[0m LOGGER\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, command_info[\u001b[38;5;241m0\u001b[39m], url, \u001b[38;5;28mstr\u001b[39m(trimmed))\n\u001b[1;32m--> 302\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcommand_info\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\selenium\\webdriver\\remote\\remote_connection.py:322\u001b[0m, in \u001b[0;36mRemoteConnection._request\u001b[1;34m(self, method, url, body)\u001b[0m\n\u001b[0;32m    319\u001b[0m     body \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m    321\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkeep_alive:\n\u001b[1;32m--> 322\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_conn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    323\u001b[0m     statuscode \u001b[38;5;241m=\u001b[39m response\u001b[38;5;241m.\u001b[39mstatus\n\u001b[0;32m    324\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\request.py:78\u001b[0m, in \u001b[0;36mRequestMethods.request\u001b[1;34m(self, method, url, fields, headers, **urlopen_kw)\u001b[0m\n\u001b[0;32m     74\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequest_encode_url(\n\u001b[0;32m     75\u001b[0m         method, url, fields\u001b[38;5;241m=\u001b[39mfields, headers\u001b[38;5;241m=\u001b[39mheaders, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39murlopen_kw\n\u001b[0;32m     76\u001b[0m     )\n\u001b[0;32m     77\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m---> 78\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest_encode_body\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m     79\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43murlopen_kw\u001b[49m\n\u001b[0;32m     80\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\request.py:170\u001b[0m, in \u001b[0;36mRequestMethods.request_encode_body\u001b[1;34m(self, method, url, fields, headers, encode_multipart, multipart_boundary, **urlopen_kw)\u001b[0m\n\u001b[0;32m    167\u001b[0m extra_kw[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mheaders\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mupdate(headers)\n\u001b[0;32m    168\u001b[0m extra_kw\u001b[38;5;241m.\u001b[39mupdate(urlopen_kw)\n\u001b[1;32m--> 170\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mextra_kw\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\poolmanager.py:376\u001b[0m, in \u001b[0;36mPoolManager.urlopen\u001b[1;34m(self, method, url, redirect, **kw)\u001b[0m\n\u001b[0;32m    374\u001b[0m     response \u001b[38;5;241m=\u001b[39m conn\u001b[38;5;241m.\u001b[39murlopen(method, url, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw)\n\u001b[0;32m    375\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 376\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mu\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest_uri\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    378\u001b[0m redirect_location \u001b[38;5;241m=\u001b[39m redirect \u001b[38;5;129;01mand\u001b[39;00m response\u001b[38;5;241m.\u001b[39mget_redirect_location()\n\u001b[0;32m    379\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m redirect_location:\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connectionpool.py:714\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[0;32m    711\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_prepare_proxy(conn)\n\u001b[0;32m    713\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[1;32m--> 714\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    715\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    716\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    717\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    718\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    719\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    720\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    721\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    722\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    724\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[0;32m    725\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[0;32m    726\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[0;32m    727\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n\u001b[0;32m    728\u001b[0m response_conn \u001b[38;5;241m=\u001b[39m conn \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m release_conn \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connectionpool.py:466\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[1;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[0;32m    461\u001b[0m             httplib_response \u001b[38;5;241m=\u001b[39m conn\u001b[38;5;241m.\u001b[39mgetresponse()\n\u001b[0;32m    462\u001b[0m         \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m    463\u001b[0m             \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[0;32m    464\u001b[0m             \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[0;32m    465\u001b[0m             \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[1;32m--> 466\u001b[0m             \u001b[43msix\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraise_from\u001b[49m\u001b[43m(\u001b[49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[0;32m    467\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m    468\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_raise_timeout(err\u001b[38;5;241m=\u001b[39me, url\u001b[38;5;241m=\u001b[39murl, timeout_value\u001b[38;5;241m=\u001b[39mread_timeout)\n",
      "File \u001b[1;32m<string>:3\u001b[0m, in \u001b[0;36mraise_from\u001b[1;34m(value, from_value)\u001b[0m\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\urllib3\\connectionpool.py:461\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[1;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[0;32m    458\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[0;32m    459\u001b[0m     \u001b[38;5;66;03m# Python 3\u001b[39;00m\n\u001b[0;32m    460\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 461\u001b[0m         httplib_response \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    462\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m    463\u001b[0m         \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[0;32m    464\u001b[0m         \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[0;32m    465\u001b[0m         \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[0;32m    466\u001b[0m         six\u001b[38;5;241m.\u001b[39mraise_from(e, \u001b[38;5;28;01mNone\u001b[39;00m)\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\http\\client.py:1374\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1372\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1373\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m-> 1374\u001b[0m         \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbegin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1375\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n\u001b[0;32m   1376\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\http\\client.py:318\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    316\u001b[0m \u001b[38;5;66;03m# read until we get a non-100 response\u001b[39;00m\n\u001b[0;32m    317\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m--> 318\u001b[0m     version, status, reason \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    319\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m!=\u001b[39m CONTINUE:\n\u001b[0;32m    320\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\http\\client.py:279\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    278\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_read_status\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m--> 279\u001b[0m     line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfp\u001b[38;5;241m.\u001b[39mreadline(_MAXLINE \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m), \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124miso-8859-1\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    280\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) \u001b[38;5;241m>\u001b[39m _MAXLINE:\n\u001b[0;32m    281\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m LineTooLong(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstatus line\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[1;32mc:\\Users\\kirti\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\socket.py:706\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[1;34m(self, b)\u001b[0m\n\u001b[0;32m    704\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m    705\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 706\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    707\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[0;32m    708\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# json_list = []\n",
    "# url_list = []\n",
    "hashtag_list = []\n",
    "# count2 = 0\n",
    "# Define the query parameters to add\n",
    "# query_parameters = \"__a=1&__d=dis\"\n",
    "\n",
    "# go through all urls\n",
    "for url in unique_post_urls:\n",
    "    try:\n",
    "        # Get the current URL of the page\n",
    "        current_url = driver.current_url\n",
    "\n",
    "        # Append the query parameters to the current URL\n",
    "        modified_url = \"https://www.instagram.com/\" + url\n",
    "\n",
    "        # Navigate to the post URL\n",
    "        driver.get(modified_url)\n",
    "\n",
    "        # Wait for the page to load (you might need to adjust the wait time)\n",
    "        driver.implicitly_wait(15)\n",
    "\n",
    "        caption_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div/div[2]/div/div/div[1]/div[1]/div[2]/section/main/div/div[1]/div/div[2]/div/div[2]/div/div[1]/div/div[2]/div/span/div/span')\n",
    "        caption_text = caption_element.text\n",
    "        # Extract hashtags from elements\n",
    "        # hashtags = [element.text for element in hashtag_elements]\n",
    "        words = caption_text.split()\n",
    "\n",
    "        # Extract words starting with #\n",
    "        hashtags = [word for word in words if word.startswith('#')]\n",
    "        hashtag_list.append((modified_url, hashtags))\n",
    "        # if count2 == 20:\n",
    "        #     break\n",
    "        # count2+=1\n",
    "\n",
    "    except (NoSuchElementException, TimeoutException, json.JSONDecodeError) as e:\n",
    "        print(f\"Error processing URL {url}: {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "76"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# len(hashtag_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "# Open a CSV file in write mode\n",
    "with open('instagram_data.csv', 'w', newline='', encoding='utf-8') as csvfile:\n",
    "    # Create a CSV writer object\n",
    "    csv_writer = csv.writer(csvfile)\n",
    "    \n",
    "    # Write header row\n",
    "    csv_writer.writerow(['URL', 'Hashtags'])\n",
    "    \n",
    "    # Write data rows\n",
    "    for post_url, hashtags in hashtag_list:\n",
    "        csv_writer.writerow([post_url, ', '.join(hashtags)])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}