Python+scrapy+mysql实现爬取磁力链接
作为老司机中的一员,所以试试爬取磁力链接,看看效果咋样。
直接上代码:
class torrentSpider(scrapy.Spider):
name = "torrent"
allowed_domains = ["www....org"]
start_urls = ['http:(网址不公布了)/so/abp-first-asc-1?f=h']
def parse(self, response):
#下面元组这里填一些简单的关键词信息,如什么前缀之类的
destination = ('')
通过css选择器选取对应的内容
post_nodes = response.css('.list a')
#从磁链列表解析每一个结果,然后通过parse_detail()解析名字和对应磁链
for post_node in post_nodes:
post_url = post_node.css('::attr(href)').extract_first('')
url = parse.urljoin(response.url, post_url)
yield Request(url=parse.urljoin(response.url, post_url),
callback=self.parse_detail)
for i in range(,len(destination)):
#可以自己选择一个合适的页数
for num in range(, ?):
#根据元组和页数拼成url
url = 'http://.../so/'+destination[i]+'-first-asc-' + str(num) + '?f=h'
#交给yield处理
yield Request(url, callback=self.parse)
def parse_detail(self, response):
my_torrent = MyTorrentItem()
item_loader = MyTorrentItemLoader(item=MyTorrentItem(), response=response)
#//*[@id="container"]/div[1]/dl/p[6]/a
item_loader.add_css('name', '.nobg small::text')
item_loader.add_css('torrent', '.magta::text')
my_torrent = item_loader.load_item()
yield my_torrent
下面是对应的MyTorrentItem:
class MyTorrentItemLoader(ItemLoader):
# 自定义ItemLoader
default_output_processor = TakeFirst()
class MyTorrentItem(scrapy.Item):
name = scrapy.Field()
torrent = scrapy.Field()
def get_insert_sql(self):
insert_sql = """
insert into my_torrent(name, torrent)
VALUES (%s, %s)
"""
params = (self["name"], self["torrent"])
return insert_sql, params
下面是pipelines.py的与数据库交互的代码
class MysqlTwistedPipeline(object):
def __init__(self, dbpool):
self.dbpool = dbpool
@classmethod
def from_settings(cls, settings):
dbparms = dict(
host=settings["MYSQL_HOST"],
db=settings["MYSQL_DBNAME"],
user=settings["MYSQL_USER"],
passwd=settings["MYSQL_PASSWORD"],
charset='utf8',
cursorclass=MySQLdb.cursors.DictCursor,
use_unicode=True,
)
dbpool = adbapi.ConnectionPool("MySQLdb", **dbparms)
return cls(dbpool)
def process_item(self, item, spider):
# 使用twisted将mysql插入变成异步执行
query = self.dbpool.runInteraction(self.do_insert, item)
query.addErrback(self.handle_error, item, spider) # 处理异常
def handle_error(self, failure, item, spider):
# 处理异步插入的异常
print(failure)
def do_insert(self, cursor, item):
# 执行具体的插入
# 根据不同的item 构建不同的sql语句并插入到mysql中
insert_sql, params = item.get_insert_sql()
# print(insert_sql, params)
cursor.execute(insert_sql, params)
这里采用异步插入,
到现在爬了半个钟,大概爬了50000+条,由于调的是打印输出日志,导致效率低了一些,,
仅供参考学习,欢迎大家指导。