豆瓣電影資料爬蟲

import requests
from bs4 import BeautifulSoup
import json
import re
import codecs
import time


#定義 http的頭
headers = {
    'Cookie':'xxxxxxxx',
    'Host':'movie.douban.com',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0',
}

#擷取頁面函數
def get_html(url):
    try:
        r = requests.get(url,headers = headers) #通路頁面
    except Exception as e :
        print("失敗伺服器傳回：%s" % e)
        return
    return(r.text)#傳回頁面文本

#擷取影片清單函數
def get_movies(html):
    result=json.loads(html)['data'] #資料json化
    return(result) #傳回影片清單

#擷取影片詳情頁函數
def get_movie_info(movies):
    for movie in movies: #周遊影片清單
        html = get_html(movie['url'])   #通過每個影片的url通路頁面
        soup = BeautifulSoup(html,'lxml') #BeautifulSoup解析頁面
        movie_info = soup.find('div',attrs={'id':'info'}) #通過特定标簽，過濾出影片資訊
        movie_info_text = movie_info.get_text() #擷取影片資訊文本
        #過濾導演、主演、語言、上映時間、類型等資料
        directed =    "".join([str(x) for x in re.findall('導演: (.*)',movie_info_text)]).replace(" / ", ";")
        star =        "".join([str(x) for x in re.findall('主演: (.*)',movie_info_text)]).replace(" / ", ";")
        language =    "".join([str(x) for x in re.findall('語言: (.*)',movie_info_text)]).replace(" / ", ";")
        releaseDate = "".join([str(x) for x in re.findall('上映日期: (.*)',movie_info_text)]).replace(" / ", ";")
        genre =       "".join([str(x) for x in re.findall('類型: (.*)',movie_info_text)]).replace(" / ", ";")
        #拼湊資料
        movie = "%s|%s|%s|%s|%s|%s|%s|%s|%s\n" % (movie['title'],movie['rate'],movie['url'],movie['cover'],genre,directed,star,language,releaseDate)
        print(movie)
        #儲存到檔案中
        file_object.write(movie)
        time.sleep(1)

#主函數由于豆瓣反爬機制，目前通過指定start和end值，來爬取電影資料
def movies_down(start,end):
    for page in range(start,end,20):#通路的url每次傳回20條電影資料頁面
        url = 'https://movie.douban.com/j/new_search_subjects?sort=T&range=0,10&tags=%E7%94%B5%E5%BD%B1&start='
        url = url+str(page) #拼湊url
        html = get_html(url) #調用頁面函數
        movies = get_movies(html) #調用影片清單函數
        #print(movies)
        get_movie_info(movies) #調用詳情頁函數

#定義檔案句柄
file_object = codecs.open('douban_movies.txt', 'a' ,"utf-8")
#調用主函數，需要指定爬取的個數範圍
movies_down(700,1300)
#關閉檔案句柄
file_object.close()