import lxml.html
import random
import time
import requests
class CsdnSpider():
USER_AGENTS = [
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
'Opera/8.0 (Windows NT 5.1; U; en)',
'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',
'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36'
]
url_list = [
"https://blog.csdn.net/qq_26397239/article/details/97482475",
"https://blog.csdn.net/qq_26397239/article/details/99403826",
"https://blog.csdn.net/qq_26397239/article/details/99403826",
"https://blog.csdn.net/qq_26397239/article/details/93773968",
"https://blog.csdn.net/qq_26397239/article/details/96353890",
"https://blog.csdn.net/qq_26397239/article/details/97482251",
"https://blog.csdn.net/qq_26397239/article/details/88895506",
"https://blog.csdn.net/qq_26397239/article/details/92403575",
"https://blog.csdn.net/qq_26397239/article/details/91348230",
"https://blog.csdn.net/qq_26397239/article/details/88895097",
"https://blog.csdn.net/qq_26397239/article/details/98724355",
]
def __init__(self):
self.page = 0
self.proxy = []
def get_proxy(self):
self.page += 1
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"}
request = requests.get("https://www.kuaidaili.com/free/inha/" + str(self.page), headers=headers)
html = request.content
etree = lxml.html.etree
content = etree.HTML(html)
# print html
ip = content.xpath('//td[@data-title="IP"]/text()')
port = content.xpath('//td[@data-title="PORT"]/text()')
# 将對應的ip和port進行拼接
for i in range(len(ip)):
for p in range(len(port)):
if i == p:
if ip[i] + ':' + port[p] not in self.proxy:
self.proxy.append(ip[i] + ':' + port[p])
# print self.proxy
if self.proxy:
print("this use" + str(self.page) + "page IP")
self.spider()
def spider(self):
num = 0 # 用于通路計數
err_num = 0 # 用于異常錯誤計數
while True:
# 從清單中随機選擇UA和代理
user_agent = random.choice(self.USER_AGENTS)
proxy = random.choice(self.proxy)
referer = random.choice(self.url_list) # 随機選擇通路url位址
headers = {
"Host": "blog.csdn.net",
"Connection": "keep-alive",
"Cache-Control": "max-age=0",
"Upgrade-Insecure-Requests": "1",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Referer": "https://blog.csdn.net/qq_24347541",
"User-Agent": user_agent,
"Accept-Language": "zh-CN,zh;q=0.9",
"Cookie": "smidV2=2018061109563875e1d84afd822e152ccb71c406cf4e44009bb058182434650; uuid_tt_dd=10_37292961410-1553592218451-585832; UN=RUSH00000; Hm_ct_6bcd52f51e9b3dce32bec4a3997715ac=6525*1*10_37292961410-1553592218451-585832!5744*1*RUSH00000; acw_tc=2760824015580561068492252e3b68add4b06393ed45927064f907a6c5015c; __yadk_uid=szPOcQVFaUokWOOM2tzYYKteKJO5RAK6; acw_sc__v3=5ce219647dc08d09158df14c726ba51f46249bfd; acw_sc__v2=5ce21963f50279c4296b98184746d6a85a5f877d; dc_session_id=10_1558321509369.910879; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1558054651,1558321511; SESSION=25351c44-6b87-48dc-b04b-a336aab6db77; UserName=RUSH00000; UserInfo=338ce29d15c840c98dbe43f5da028079; UserToken=338ce29d15c840c98dbe43f5da028079; UserNick=RUSH00000; AU=AAC; BT=1558321532133; dc_tos=prs8ld; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1558321538"
}
try:
# 建構一個Handler處理器對象，參數是一個字典類型，包括代理類型和代理伺服器IP+PROT
request = requests.get(referer, headers=headers, proxies={"http": proxy})
html = request.content
etree = lxml.html.etree
content = etree.HTML(html)
# 使用xpath比對閱讀量
read_num = content.xpath('//span[@class="read-count"]/text()')
# 将清單轉為字元串
new_read_num = ''.join(read_num)
# 通過xpath比對的頁面為blog.csdn.net/qq_41782425/article/details/84934224是以比對其他頁面傳回的為空
if len(new_read_num) != 0:
print(new_read_num)
num += 1
print('The' + str(num) + 'few visits')
print(request.url + "proxy ip: " + str(proxy))
# print request.headers
time.sleep(6)
# 當通路數量達到100時，退出循環，并調用get_proxy方法擷取第二頁的代理
if num > 100:
break
except Exception as result:
err_num += 1
print("error message(%d):%s" % (err_num, result))
# 當錯誤資訊大于等于30時，初始化代理頁面page，重新從第一頁開始擷取代理ip，并退出循環
if err_num >= 30:
self.__init__()
break
# 當退出循環時，看就會執行get_proxy擷取代理的方法
print("Re acquiring agent proxy IP")
self.get_proxy()
if __name__ == "__main__":
CsdnSpider().get_proxy()
![TestLink導出用例轉換工具(XML2Excel)[圖]](data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=)