爬蟲之爬取豆瓣圖書的評論

from urllib import request
from bs4 import BeautifulSoup as bs

#爬取豆瓣最受關注圖書榜
resp = request.urlopen('https://book.douban.com/chart?subcat=I')
html_data = resp.read().decode('utf-8')

#轉化為BeautifulSoup對象
soup = bs(html_data,'html.parser')

#搜尋最受關注的圖書清單
topchart_book =soup.find_all('ul',class_='chart-dashed-list')

#搜尋清單中所有圖書
topchart_book_list = topchart_book[0].find_all('li',class_='media clearfix')

#建立數組用于存放後續的資料
topchart_list = []

#周遊圖書館清單，從中過濾出我們所需的資訊
for item in topchart_book_list:
    #建立字典用于存放我們的圖書資訊，之後可用class來存儲
    topchart_dict = {}

    #搜尋到具體資訊的位置
    book_item = item.find('a',class_='fleft')

    #得到圖書ID
    topchart_dict['id'] = book_item['href'].split('/')[4]   

    #得到圖書名稱
    topchart_dict['name'] = book_item.getText().replace('\t','').replace('\n','').replace(' ','')  #圖書名字
    
    #将圖書資訊加入到數組中
    topchart_list.append(topchart_dict)
# print(topchart_list)

#拼接出圖書對應的詳情頁
requrl = 'https://book.douban.com/subject/'+topchart_list[0]['id']+'/comments/hot'+'?'+'p-1'

#爬取熱門第一頁中的評論資訊
resp = request.urlopen(requrl)
html_data = resp.read().decode('utf-8')
soup = bs(html_data,'html.parser')

#搜尋到評論所在div
comment_div_lits = soup.find_all('div',class_='comment')

#建立數組用于存放評論資訊
eachCommentList = []

for item in comment_div_lits:
    if item.find_all('p')[0].string is not None:
        eachCommentList.append(item.find_all('p')[0].string)
print(eachCommentList)      

轉載于:https://www.cnblogs.com/lsm-boke/p/9940313.html