-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathcount_file_md5.py
257 lines (219 loc) · 7.74 KB
/
count_file_md5.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 18/4/15 上午2:07
# @Author : 重剑无锋
# @Email : [email protected]
import random, os, mysql.connector
import urllib2, re, requests
import time, hashlib, urllib
import sys
mysql_config = {
'host': '127.0.0.1',
# 'host': '192.168.1.201',
'port': 3306,
'db_name': 'finger',
'username': 'root',
'password': '123456'
}
def url_protocol(url):
domain = re.findall(r'.*(?=://)', url)
if domain:
return domain[0]
else:
return url
def get_domain(target):
try:
url = target
if url[0:4] == 'http':
proto, rest = urllib.splittype(url)
host, rest = urllib.splithost(rest)
if host[0:3] == 'www':
host = host[4:]
elif url[0:3] == 'www':
host = url[4:]
else:
host = url
if ':' in host:
host = host.split(':')[0]
if '/' in host:
host = host.split('/')[0]
return host
except:
return target
def get_main_domain(domain):
double_exts = ['.com.cn', '.edu.cn', '.gov.cn', '.org.cn', '.net.cn']
main_domain = domain
for ext in double_exts:
if ext in domain:
if len(domain.split('.')) > 3:
# print "yuanshi",domain
domain_split = domain.split('.')
domain_new = "%s.%s.%s" % (domain_split[-3], domain_split[-2],
domain_split[-1])
# print "exact",domain
main_domain = domain_new
else:
main_domain = domain
break
else:
if len(domain.split('.')) > 2:
domain_split = domain.split('.')
domain_new = "%s.%s" % (domain_split[-2], domain_split[-1])
main_domain = domain_new
else:
main_domain = domain
return main_domain
def requests_headers():
'''
Random UA for every requests && Use cookie to scan
'''
user_agent = [
'Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.8.1) Gecko/20061010 Firefox/2.0',
'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.6 Safari/532.0',
'Mozilla/5.0 (Windows; U; Windows NT 5.1 ; x64; en-US; rv:1.9.1b2pre) Gecko/20081026 Firefox/3.1b2pre',
'Opera/10.60 (Windows NT 5.1; U; zh-cn) Presto/2.6.30 Version/10.60',
'Opera/8.01 (J2ME/MIDP; Opera Mini/2.0.4062; en; U; ssr)',
'Mozilla/5.0 (Windows; U; Windows NT 5.1; ; rv:1.9.0.14) Gecko/2009082707 Firefox/3.0.14',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
'Mozilla/5.0 (Windows; U; Windows NT 6.0; fr; rv:1.9.2.4) Gecko/20100523 Firefox/3.6.4 ( .NET CLR 3.5.30729)',
'Mozilla/5.0 (Windows; U; Windows NT 6.0; fr-FR) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16',
'Mozilla/5.0 (Windows; U; Windows NT 6.0; fr-FR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5'
]
UA = random.choice(user_agent)
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'User-Agent':
UA,
'Upgrade-Insecure-Requests':
'1',
'Connection':
'keep-alive',
'Cache-Control':
'max-age=0',
'Accept-Encoding':
'gzip, deflate, sdch',
'Accept-Language':
'zh-CN,zh;q=0.8',
"Referer":
"http://www.baidu.com/link?url=www.so.com&url=www.soso.com&&url=www.sogou.com"
}
return headers
excludeext = [
'.png', '.ico', '.gif', '.svg', '.jpeg', 'js', 'css', 'xml', 'txt'
]
def getPageLinks(url):
try:
headers = requests_headers()
content = requests.get(url, timeout=5, headers=headers,
verify=False).text.encode('utf-8')
links = []
tags = ['a', 'A', 'link', 'script', 'area', 'iframe', 'form'] # img
tos = ['href', 'src', 'action']
if url[-1:] == '/':
url = url[:-1]
try:
for tag in tags:
for to in tos:
link1 = re.findall(r'<%s.*?%s="(.*?)"' % (tag, to),
str(content))
link2 = re.findall(r'<%s.*?%s=\'(.*?)\'' % (tag, to),
str(content))
for i in link1:
links.append(i)
for i in link2:
if i not in links:
links.append(i)
except Exception, e:
print e
print '[!] Get link error'
pass
return links
except:
return []
def getMD5(c):
m = hashlib.md5()
m.update(c)
psw = m.hexdigest()
return psw
def request_url(url):
try:
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:59.0) Gecko/20100101 Firefox/59.0'
}
r = requests.get(
url=url,
headers=requests_headers(),
timeout=5,
verify=False,
)
r.encoding = 'utf-8'
if r.status_code == 200:
return r.content
else:
return ''
except Exception, e:
# print e
return ''
def update_mysql(conn, url, filename, md5):
try:
cur = conn.cursor()
sql = "select * from file_md5 where md5 = '" + str(md5) + "'"
print sql
cur.execute(sql)
exist_md5 = cur.fetchone()
if exist_md5:
url = exist_md5[2] + ',' + url
hint = exist_md5[3] + 1
sql = "update file_md5 set url = '%s',hint='%s' where md5 = '%s'" % (
url, hint, md5)
else:
sql = "INSERT INTO file_md5 (url,filename,md5,hint) VALUES ('%s','%s','%s','%s')" % (
url, filename, md5, 1)
print sql
cur.execute(sql)
conn.commit()
except Exception, e:
print e
pass
conn = mysql.connector.connect(user=mysql_config['username'],
password=mysql_config['password'],
host=mysql_config['host'],
database=mysql_config['db_name'],
charset='utf8')
# url = 'http://www.sddlr.gov.cn/'
for url in open('url.txt', 'r').readlines():
url = url.strip()
print url
urlprotocol = url_protocol(url)
domain_url = get_domain(url)
pageLinks = getPageLinks(url)
print pageLinks
true_url = []
for suburl in pageLinks:
for ext in excludeext:
if ext in suburl[-4:]:
# print ext,' ',suburl
if re.findall(r'/', suburl):
if re.findall(r':', suburl):
true_url.append(suburl)
else:
true_url.append(urlprotocol + '://' + domain_url +
'/' + suburl)
else:
true_url.append(urlprotocol + '://' + domain_url + '/' +
suburl)
url_tmp1 = set(list(true_url))
print "原长度: ", len(url_tmp1)
url_tmp2 = []
for x in url_tmp1:
url_tmp2.append(x)
print "去重后: ", len(url_tmp2)
for url_tmp in url_tmp2:
url_content = request_url(url_tmp)
filename = os.path.basename(url_tmp)
md5 = getMD5(url_content)
print url_tmp, ' ', filename, md5
update_mysql(conn, url_tmp, filename, md5)