python爬蟲入門學習10-scrapy-站長之家圖檔爬取

class ImgSpider(scrapy.Spider):
    name = 'img'
    start_urls = ['https://sc.chinaz.com/tupian/']
	
	# 爬蟲處理方法
    def parse(self, response):
        div_list = response.xpath('//*[@id="container"]/div')
        for div in div_list:
            # 圖檔懶加載
            src = 'https:' + div.xpath('./div/a/img/@src2').extract_first()
            # print(src)
            item = ImgproItem()
            item['src'] = src
            yield item

# 圖檔對象item
class ImgproItem(scrapy.Item):
    src = scrapy.Field()

# 圖檔管道類重寫
class imgsPipeline(ImagesPipeline):
    # 該方法負責 根據圖檔url 發起下載下傳的請求
    def get_media_requests(self, item, info):
        yield scrapy.Request(item['src'])

    # 該方法負責 處理下載下傳的圖檔名稱
    def file_path(self, request, response=None, info=None, *, item=None):
        imgName = request.url.split('/')[-1]
        return imgName

    # 該方法負責 執行完重寫方法後 傳回item對象
    def item_completed(self, results, item, info):
        return item


# 配置相關
IMAGES_STORE = './imgs'
# 使用自己重寫的管道類
ITEM_PIPELINES = {
   'imgPro.pipelines.imgsPipeline': 300,
}
# 不遵循robots協定
ROBOTSTXT_OBEY = False
# 日志級别
LOG_LEVEL = 'ERROR'
# 請求頭
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'