使用scrapy中crawlspider爬取csdn文章

生成crawlspider指令：

scrapy genspider -t crawl csdn "csdn.cn"
           

在csdn_spider.py

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class CsdnSpdierSpider(CrawlSpider):
name = 'csdn_spdier'
allowed_domains = ['blog.csdn.net']
start_urls = ['http://blog.csdn.net/hbblzjy']

rules = (
    #提取專家的部落格位址
    Rule(LinkExtractor(allow=r'blog.csdn.net/\w+$'), follow=True),
    #提取部落格詳情頁的位址
    Rule(LinkExtractor(allow=r'/\w+/article/details/\d+$'),callback="parse_item"),
    #提取專家翻頁位址
    Rule(LinkExtractor(allow=r'channelid=\d+&page=\d+$'), follow=True),
    #提取部落格清單頁翻頁
    Rule(LinkExtractor(allow=r'/\w+/article/list/\d+$'), follow=True),
)

def parse_item(self, response):
    item = {}
    item["title"] = response.xpath("//h1/text()").extract_first()
    item['update_time'] = response.xpath("//span[@class='time']/text()").extract_first()
    item["tag"] = response.xpath("//ul[@class='article_tags clearfix csdn-tracking-statistics']//text()").extract()
    # item["content"] = response.xpath("//div[@id='article_content']")
    # print(item)
    yield item
           

在pipelines.py

import re

class CsdnPipeline(object):
def process_item(self, item, spider):
    item["tag"] = [re.sub(r"\s+|/","",i,re.S) for i in item["tag"]]
    item["tag"] =[i for i in item["tag"] if len(i)>0 and i!='标簽：']
    print(item)
           

在settings.py

BOT_NAME = 'csdn'

SPIDER_MODULES = ['csdn.spiders']

NEWSPIDER_MODULE = 'csdn.spiders'

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'

ROBOTSTXT_OBEY = False