#coded by 伊瑪目的門徒
import re
import requests
import time
from bs4 import BeautifulSoup
urllist=[]
titlelist=[]
start = time.clock() # 計時-開始
from concurrent.futures import ThreadPoolExecutor
header={'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.XXXX.XXX Safari/537.36'}
def do(i):
try:
cd=[]
html=requests.get('http://futures.hexun.com/domestic/index-'+str(i)+'.html',headers=header)
html.encoding='gbk'
Soup = BeautifulSoup(html.text, "lxml")
#ab=Soup.select('li a[target="_blank"]')
ab=Soup.select('div.temp01 ul li a[target="_blank"]')
for x in range(len(ab)):
if (x % 2) == 1:
cd.append (ab[x])
print ('-----------------')
pattern = re.compile(r'<a href="(.*?)" target="_blank" rel="external nofollow" target="_blank">',re.S) # 查找數字
result1 = pattern.findall(str(cd))
pattern2 = re.compile(r'target="_blank">(.*?)</a>',re.S)
result2 = pattern2.findall(str(cd))
print (result1)
urllist.extend(result1)
print (result2)
titlelist.extend(result2)
list1.remove(i)
except:
pass
# 多線程
def multithreading():
sum=0
while len(list1)>0:
with ThreadPoolExecutor(max_workers=10) as executor:
for result in executor.map(do, list1):
sum+=1
return sum
list1=list(range(1,393,1))
sum=multithreading()
print ('還剩下{}頁'.format(list1))
end = time.clock() # 計時-結束
print (("爬取完成 用時:"))
print ((end - start))
print ('總爬取 %d 頁 '%(sum))
while None in titlelist:
titlelist.remove(None)
while None in urllist:
urllist.remove(None)
print (titlelist)
print (urllist)
'''
#可作為TXT輸出
with open("test.txt","w") as f:
for thing in urllist:
f.write(thing)
f.write('\r\n')
'''
示範視訊:
https://www.bilibili.com/video/av80003976/
python多線程爬蟲和訊網 标題和正文URL效果展示
後續可看
https://blog.csdn.net/qq_37195257/article/details/103643238