Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

stack overflow result fixes #93

Open
wants to merge 21 commits into
base: master
Choose a base branch
from
Open

stack overflow result fixes #93

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
680cd4d
Update rebound.py
derhnyel Nov 30, 2021
d9e4a40
Update requirements.txt
derhnyel Nov 30, 2021
bbcf0a9
Delete rebound.py
derhnyel Nov 30, 2021
ae6e54b
Merge branch 'master' of https://github.com/derhnyel/rebound
derhnyel Nov 30, 2021
b830535
update
derhnyel Dec 1, 2021
c2dc793
update
derhnyel Dec 1, 2021
8a64a73
Update
derhnyel Dec 1, 2021
4377886
Update test.py
derhnyel Dec 1, 2021
2da9d79
update
derhnyel Dec 1, 2021
cdb48da
Update import.py
derhnyel Dec 1, 2021
3f6307d
Update import.py
derhnyel Dec 1, 2021
2ed5322
Update import.py
derhnyel Dec 1, 2021
b6f08b2
Update import.py
derhnyel Dec 1, 2021
4d5f5e7
Update import.py
derhnyel Dec 1, 2021
4bf8583
Update import.py
derhnyel Dec 1, 2021
a756e27
fixes
derhnyel Dec 1, 2021
2127bca
Merge branch 'master' of https://github.com/derhnyel/rebound
derhnyel Dec 1, 2021
a5c9730
Update requirements.txt
derhnyel Dec 1, 2021
a666724
update
derhnyel Dec 1, 2021
7b7bdd6
Update rebound.py
derhnyel Feb 16, 2022
84c9c2e
Merge branch 'shobrook:master' into master
derhnyel Feb 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,5 @@ BeautifulSoup4
requests
urllib3
urwid
search_engine_parser
fake_useragent
189 changes: 106 additions & 83 deletions rebound/rebound.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,10 @@
from subprocess import PIPE, Popen
from threading import Thread
import webbrowser
import time
from urwid.widget import (BOX, FLOW, FIXED)
import random
from search_engine_parser.core.engines.google import Search as GoogleSearch
from fake_useragent import UserAgent


SO_URL = "https://stackoverflow.com"

Expand All @@ -41,39 +42,39 @@
SCROLLBAR_LEFT = "left"
SCROLLBAR_RIGHT = "right"

USER_AGENTS = [
"Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Firefox/59",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36',
'Mozilla/5.0 (Windows NT 5.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
'Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1)',
'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko',
'Mozilla/5.0 (Windows NT 6.2; WOW64; Trident/7.0; rv:11.0) like Gecko',
'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)',
'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64; Trident/7.0; rv:11.0) like Gecko',
'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)',
]
# USER_AGENTS = [
# "Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
# "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
# "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
# "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
# "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
# "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
# "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Firefox/59",
# "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
# "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
# "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
# 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
# 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36',
# 'Mozilla/5.0 (Windows NT 5.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36',
# 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36',
# 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36',
# 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
# 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
# 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
# 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
# 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
# 'Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1)',
# 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',
# 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
# 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko',
# 'Mozilla/5.0 (Windows NT 6.2; WOW64; Trident/7.0; rv:11.0) like Gecko',
# 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
# 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)',
# 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko',
# 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)',
# 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; Trident/7.0; rv:11.0) like Gecko',
# 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)',
# ]


##################
Expand Down Expand Up @@ -240,36 +241,36 @@ def stylize_code(soup):
return urwid.Text(stylized_text)


def get_search_results(soup):
"""Returns a list of dictionaries containing each search result."""
search_results = []
# def get_search_results(soup):
# """Returns a list of dictionaries containing each search result."""
# search_results = []

for result in soup.find_all("div", class_="question-summary search-result"):
title_container = result.find_all("div", class_="result-link")[0].find_all("a")[0]
# for result in soup.find_all("div", class_="question-summary search-result"):
# title_container = result.find_all("div", class_="result-link")[0].find_all("a")[0]

if result.find_all("div", class_="status answered") != []: # Has answers
answer_count = int(result.find_all("div", class_="status answered")[0].find_all("strong")[0].text)
elif result.find_all("div", class_="status answered-accepted") != []: # Has an accepted answer (closed)
answer_count = int(result.find_all("div", class_="status answered-accepted")[0].find_all("strong")[0].text)
else: # No answers
answer_count = 0
# if result.find_all("div", class_="status answered") != []: # Has answers
# answer_count = int(result.find_all("div", class_="status answered")[0].find_all("strong")[0].text)
# elif result.find_all("div", class_="status answered-accepted") != []: # Has an accepted answer (closed)
# answer_count = int(result.find_all("div", class_="status answered-accepted")[0].find_all("strong")[0].text)
# else: # No answers
# answer_count = 0

search_results.append({
"Title": title_container["title"],
#"Body": result.find_all("div", class_="excerpt")[0].text,
#"Votes": int(result.find_all("span", class_="vote-count-post ")[0].find_all("strong")[0].text),
"Answers": answer_count,
"URL": SO_URL + title_container["href"]
})
# search_results.append({
# "Title": title_container["title"],
# #"Body": result.find_all("div", class_="excerpt")[0].text,
# #"Votes": int(result.find_all("span", class_="vote-count-post ")[0].find_all("strong")[0].text),
# "Answers": answer_count,
# "URL": SO_URL + title_container["href"]
# })

return search_results
# return search_results


def souper(url):
"""Turns a given URL into a BeautifulSoup object."""

UAgent = UserAgent()
try:
html = requests.get(url, headers={"User-Agent": random.choice(USER_AGENTS)})
html = requests.get(url, headers={"User-Agent": UAgent.random})
except requests.exceptions.RequestException:
sys.stdout.write("\n%s%s%s" % (RED, "Rebound was unable to fetch Stack Overflow results. "
"Please check that you are connected to the internet.\n", END))
Expand All @@ -284,16 +285,30 @@ def souper(url):
## Main ##


def search_stackoverflow(query):
"""Wrapper function for get_search_results."""
soup = souper(SO_URL + "/search?pagesize=50&q=%s" % query.replace(' ', '+'))
# def search_stackoverflow(query):
# """Wrapper function for get_search_results."""
# soup = souper(SO_URL + "/search?pagesize=50&q=%s" % query.replace(' ', '+'))

# TODO: Randomize the user agent
# # TODO: Randomize the user agent

if soup == None:
return (None, True)
else:
return (get_search_results(soup), False)
# if soup == None:
# return (None, True)
# else:
# return (get_search_results(soup), False)

def search_google(query):
try:
query = query+" :"+SO_URL[8:]
google_search = GoogleSearch()
SearchArgs=(query,1)
google_search.clear_cache()
SearchDict=google_search.search(*SearchArgs)
except Exception as e:
sys.stdout.write("\n%s%s%s" % (RED,"Rebound was unable to fetch results. "
+str(e)+"\n Try again Later.", END))
sys.exit(1)

return [result for result in SearchDict]


def get_question_and_answers(url):
Expand Down Expand Up @@ -776,14 +791,16 @@ def _get_selected_link(self):

for result in self.search_results:
if title == self._stylize_title(result): # Found selected title's search_result dict
return result["URL"]
return result["link"]
#return result["URL"]


def _stylize_title(self, search_result):
if search_result["Answers"] == 1:
return "%s (1 Answer)" % search_result["Title"]
else:
return "%s (%s Answers)" % (search_result["Title"], search_result["Answers"])
return search_result["title"]
# if search_result["Answers"] == 1:
# return "%s (1 Answer)" % search_result["title"]
# else:
# return "%s (%s Answers)" % (search_result["title"], search_result["Answers"])


def _stylize_question(self, title, desc, stats):
Expand Down Expand Up @@ -830,20 +847,24 @@ def print_help():


def main():

if len(sys.argv) == 1 or sys.argv[1].lower() == "-h" or sys.argv[1].lower() == "--help":
print_help()
elif sys.argv[1].lower() == "-q" or sys.argv[1].lower() == "--query":
query = ' '.join(sys.argv[2:])
search_results, captcha = search_stackoverflow(query)

#search_results, captcha = search_stackoverflow(query)
search_results = search_google(query)

if search_results != []:
if captcha:
print("\n%s%s%s" % (RED, "Sorry, Stack Overflow blocked our request. Try again in a minute.\n", END))
return
else:
App(search_results) # Opens interface

# if captcha:
# print("\n%s%s%s" % (RED, "Sorry, Stack Overflow blocked our request. Try again in a minute.\n", END))
# return
# else:
App(search_results) # Opens interface
else:
print("\n%s%s%s" % (RED, "No Stack Overflow results found.\n", END))
print("\n%s%s%s" % (RED, "No Google results found.\n", END))
else:
language = get_language(sys.argv[1].lower()) # Gets the language name
if language == '': # Unknown language
Expand All @@ -860,17 +881,19 @@ def main():
error_msg = get_error_message(error, language) # Prepares error message for search
if error_msg != None:
language = 'java' if language == 'javac' else language # Fix language compiler command
query = "%s %s" % (language, error_msg)
search_results, captcha = search_stackoverflow(query)
query = "%s %s %s" % (language, error_msg)
#search_results, captcha = search_stackoverflow(query)
search_results = search_google(query)

if search_results != []:
if captcha:
print("\n%s%s%s" % (RED, "Sorry, Stack Overflow blocked our request. Try again in a minute.\n", END))
return
elif confirm("\nDisplay Stack Overflow results?"):
# if captcha:
# print("\n%s%s%s" % (RED, "Sorry, Stack Overflow blocked our request. Try again in a minute.\n", END))
# return
#elif confirm("\nDisplay Stack Overflow results?"):
if confirm("\nDisplay Stack Overflow results?"):
App(search_results) # Opens interface
else:
print("\n%s%s%s" % (RED, "No Stack Overflow results found.\n", END))
print("\n%s%s%s" % (RED, "No Google results found.\n", END))
else:
print("\n%s%s%s" % (CYAN, "No error detected :)\n", END))

Expand Down