Python+scrapy+mysql實作爬取磁力連結
作為老司機中的一員,是以試試爬取磁力連結,看看效果咋樣。
直接上代碼:
class torrentSpider(scrapy.Spider):
name = "torrent"
allowed_domains = ["www....org"]
start_urls = ['http:(網址不公布了)/so/abp-first-asc-1?f=h']
def parse(self, response):
#下面元組這裡填一些簡單的關鍵詞資訊,如什麼字首之類的
destination = ('')
通過css選擇器選取對應的内容
post_nodes = response.css('.list a')
#從磁鍊清單解析每一個結果,然後通過parse_detail()解析名字和對應磁鍊
for post_node in post_nodes:
post_url = post_node.css('::attr(href)').extract_first('')
url = parse.urljoin(response.url, post_url)
yield Request(url=parse.urljoin(response.url, post_url),
callback=self.parse_detail)
for i in range(,len(destination)):
#可以自己選擇一個合适的頁數
for num in range(, ?):
#根據元組和頁數拼成url
url = 'http://.../so/'+destination[i]+'-first-asc-' + str(num) + '?f=h'
#交給yield處理
yield Request(url, callback=self.parse)
def parse_detail(self, response):
my_torrent = MyTorrentItem()
item_loader = MyTorrentItemLoader(item=MyTorrentItem(), response=response)
#//*[@id="container"]/div[1]/dl/p[6]/a
item_loader.add_css('name', '.nobg small::text')
item_loader.add_css('torrent', '.magta::text')
my_torrent = item_loader.load_item()
yield my_torrent
下面是對應的MyTorrentItem:
class MyTorrentItemLoader(ItemLoader):
# 自定義ItemLoader
default_output_processor = TakeFirst()
class MyTorrentItem(scrapy.Item):
name = scrapy.Field()
torrent = scrapy.Field()
def get_insert_sql(self):
insert_sql = """
insert into my_torrent(name, torrent)
VALUES (%s, %s)
"""
params = (self["name"], self["torrent"])
return insert_sql, params
下面是pipelines.py的與資料庫互動的代碼
class MysqlTwistedPipeline(object):
def __init__(self, dbpool):
self.dbpool = dbpool
@classmethod
def from_settings(cls, settings):
dbparms = dict(
host=settings["MYSQL_HOST"],
db=settings["MYSQL_DBNAME"],
user=settings["MYSQL_USER"],
passwd=settings["MYSQL_PASSWORD"],
charset='utf8',
cursorclass=MySQLdb.cursors.DictCursor,
use_unicode=True,
)
dbpool = adbapi.ConnectionPool("MySQLdb", **dbparms)
return cls(dbpool)
def process_item(self, item, spider):
# 使用twisted将mysql插入變成異步執行
query = self.dbpool.runInteraction(self.do_insert, item)
query.addErrback(self.handle_error, item, spider) # 處理異常
def handle_error(self, failure, item, spider):
# 處理異步插入的異常
print(failure)
def do_insert(self, cursor, item):
# 執行具體的插入
# 根據不同的item 建構不同的sql語句并插入到mysql中
insert_sql, params = item.get_insert_sql()
# print(insert_sql, params)
cursor.execute(insert_sql, params)
這裡采用異步插入,
到現在爬了半個鐘,大概爬了50000+條,由于調的是列印輸出日志,導緻效率低了一些,,
僅供參考學習,歡迎大家指導。