小白爬蟲第一彈之抓取妹子圖【更新版】

最近對爬蟲感興趣，參考了  http://cuiqingcai.com/3179.html   這篇文章，琢磨的小試身手，但是按照原文發現圖檔下載下傳下來全是打不開的圖檔，這不是我想要的啊，嘗試直接把圖檔的連結在浏覽器中打開，發現已經看過的沒有問題，新打開的就不行了，一直懷疑是在cookie方面做了文章，在request裡也确實發現了有兩個cookie的身影，但就是怎麼找也找不到這兩個cookie怎麼來的，一度想放棄，爬個别的站算了，對妹子們的不舍還是又堅持研究了半天，在浏覽器裡的抓包工具裡一直檢視各種參數，做各種嘗試，發現了一點門道，原來請求圖檔的時候需要增加一個參數，具體看代碼吧，運作後中途連接配接被切斷了，遂又增加了ua随機替換的方法，不廢話了，代碼如下：

import requests
from bs4 import BeautifulSoup
import os
import urllib
import random
 
 
class mzitu():
 
    def all_url(self, url):
        html = self.request(url)
        all_a = BeautifulSoup(html.text, 'lxml').find('div', class_='all').find_all('a')
        for a in all_a:
            title = a.get_text()
            if title == '早期圖檔':
                continue
            print(u'開始儲存：', title)
            path = str(title).replace("?", '_')
            if not self.mkdir(path): ##跳過已存在的檔案夾
                print(u'已經跳過：', title)
                continue
            href = a['href']
            self.html(href)
    def html(self, href):
        html = self.request(href)
        max_span = BeautifulSoup(html.text, 'lxml').find('div', class_='pagenavi').find_all('span')[-2].get_text()
        for page in range(1, int(max_span) + 1):
            page_url = href + '/' + str(page)
            self.img(page_url)
 
    def img(self, page_url):
        img_html = self.request(page_url)
        img_url = BeautifulSoup(img_html.text, 'lxml').find('div', class_='main-image').find('img')['src']
        self.save(img_url,page_url)
 
    def save(self, img_url, page_url):
        name = img_url[-9:-4]
        try:
            img = self.requestpic(img_url,page_url)
            f = open(name + '.jpg', 'ab')
            f.write(img.content)
            f.close()
        except FileNotFoundError: ##捕獲異常，繼續往下走
            print(u'圖檔不存在已跳過：', img_url)
            return False
 
    def mkdir(self, path): ##這個函數建立檔案夾
        path = path.strip()
        isExists = os.path.exists(os.path.join("D:\mzitu", path))
        if not isExists:
            print(u'建了一個名字叫做', path, u'的檔案夾！')
            os.makedirs(os.path.join("D:\mzitu", path))
            os.chdir(os.path.join("D:\mzitu", path)) ##切換到目錄
            return True
        else:
            print(u'名字叫做', path, u'的檔案夾已經存在了！')
            return False
 
    def requestpic(self, url, Referer): ##這個函數擷取網頁的response 然後傳回
        user_agent_list = [ \
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1" \
            "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", \
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", \
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", \
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", \
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", \
            "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", \
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3", \
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", \
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
        ]
        ua = random.choice(user_agent_list)
        headers = {'User-Agent': ua,"Referer":Referer} ##較之前版本擷取圖檔關鍵參數在這裡
        content = requests.get(url, headers=headers)
        return content
 
    def request(self, url): ##這個函數擷取網頁的response 然後傳回
        headers = {'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"}
        content = requests.get(url, headers=headers)
        return content
 
Mzitu = mzitu() ##執行個體化
Mzitu.all_url('http://www.mzitu.com/all') ##給函數all_url傳入參數  你可以當作啟動爬蟲（就是入口）
print(u'恭喜您下載下傳完成啦！')      

感謝前人們的無私分享， 感謝 http://cuiqingcai.com/ ， 感謝mzitu的站長提供給我們這麼多的優質妹子。

轉載于:https://www.cnblogs.com/wangbg/p/7282543.html