爬蟲練習--使用正規表達式爬取B站排行榜

1.首先打開B站網頁

從中可以擷取的資訊是網站的URL是“https://www.bilibili.com/v/popular/rank/all”

2.打開開發者工具，在Network頁籤下檢視Cookie

3.檢視網站源代碼，編寫正規表達式

觀察源代碼，我們可以寫出正規表達式

<li data-id.*?num.*?>(.*?)</div>.*?img.*?href="(.*?)" target="_blank" rel="external nofollow" target="_blank" rel="external nofollow" target="_blank" rel="external nofollow" .*?</a>.*?title.*?>(.*?)</a>.*?b-icon play.*?</i>(.*?)</span>.*?b-icon view.*?</i>(.*?)</span>.*?b-icon author.*?</i>(.*?)</span>.*?pts.*?<div>(.*?)</div>.*?</li> 

4.抓取頁面

#抓取網頁
def get_one_page(url):
	headers = {
		'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
		'Cookie' : "_uuid=50C67A24-10FD-241D-42BC-63060BB3AD4640961infoc; buvid3=B2CBDB74-9521-4C16-9283-36AC6E99CD3234758infoc; PVID=1; bsource=search_baidu; fingerprint=86c13fa55e20a8f09e66c6b108be9688; buvid_fp=B2CBDB74-9521-4C16-9283-36AC6E99CD3234758infoc; buvid_fp_plain=3BBDF6C0-3345-4F0F-934E-8F76451AC03B13425infoc"
		}
	response = requests.get(url,headers = headers)
	if response.status_code == 200:
		return response.text
	return None

def main():
	url = 'https://www.bilibili.com/v/popular/rank/all'
	html = get_one_page(url)

main()
           

5.利用正規表達式提取網頁資訊 

def main():
	url = 'https://www.bilibili.com/v/popular/rank/all'
	html = get_one_page(url)
	
	#檢視網站源代碼，分析源代碼格式，編寫正規表達式
	pattern = '<li data-id.*?num.*?>(.*?)</div>.*?img.*?href="(.*?)" target="_blank" rel="external nofollow"  target="_blank" rel="external nofollow"  target="_blank" rel="external nofollow" .*?</a>.*?title.*?>(.*?)</a>.*?b-icon play.*?</i>(.*?)</span>.*?b-icon view.*?</i>(.*?)</span>.*?b-icon author.*?</i>(.*?)</span>.*?pts.*?<div>(.*?)</div>.*?</li>'
	items = re.findall(pattern, html,re.S)

    #将提取的資訊寫入字典
	for item in items:
		
		d = {}
		d['index'] = item[0]
		d['image'] = item[1]
		d['title'] = item[2].strip()
		d['play'] = item[3].strip()
		d['view'] = item[4].strip()
		d['author'] = item[5].strip()
		d['point'] = item[6].strip()
           

 6.将整理好的字典寫入檔案

def write_to_file(content):
    #寫入“result1.txt”檔案
	with open('result1.txt','a',encoding = 'utf-8') as f:
		print(type(json.dumps(content)))
		f.write(json.dumps(content, ensure_ascii = False) + '\n')
           

7.結果

 工作台：

result1檔案：

大功告成！

前20是我爬貓眼TOP的資料（doge）

8.完整代碼

import requests
import re
import json

def get_one_page(url):
	headers = {
		'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
		'Cookie' : "_uuid=50C67A24-10FD-241D-42BC-63060BB3AD4640961infoc; buvid3=B2CBDB74-9521-4C16-9283-36AC6E99CD3234758infoc; PVID=1; bsource=search_baidu; fingerprint=86c13fa55e20a8f09e66c6b108be9688; buvid_fp=B2CBDB74-9521-4C16-9283-36AC6E99CD3234758infoc; buvid_fp_plain=3BBDF6C0-3345-4F0F-934E-8F76451AC03B13425infoc"
		}
	response = requests.get(url,headers = headers)
	if response.status_code == 200:
		return response.text
	return None

def write_to_file(content):
	with open('result1.txt','a',encoding = 'utf-8') as f:
		print(type(json.dumps(content)))
		f.write(json.dumps(content, ensure_ascii = False) + '\n')

def main():
	url = 'https://www.bilibili.com/v/popular/rank/all'
	html = get_one_page(url)
	
	pattern = '<li data-id.*?num.*?>(.*?)</div>.*?img.*?href="(.*?)" target="_blank" rel="external nofollow"  target="_blank" rel="external nofollow"  target="_blank" rel="external nofollow" .*?</a>.*?title.*?>(.*?)</a>.*?b-icon play.*?</i>(.*?)</span>.*?b-icon view.*?</i>(.*?)</span>.*?b-icon author.*?</i>(.*?)</span>.*?pts.*?<div>(.*?)</div>.*?</li>'
	items = re.findall(pattern, html,re.S)
	for item in items:
		
		d = {}
		d['index'] = item[0]
		d['image'] = item[1]
		d['title'] = item[2].strip()
		d['play'] = item[3].strip()
		d['view'] = item[4].strip()
		d['author'] = item[5].strip()
		d['point'] = item[6].strip()
		print(d)
		write_to_file(d)

main()
           

本文是學習了崔大神的《爬蟲》的爬貓眼執行個體的練習，如果有錯誤的地方的還請大神們指出！對我是莫大的幫助！