一、安裝splash
#docker安裝
#拉取鏡像
docker pull scrapinghub/splash
#運作容器
docker run -p 8050:8050 scrapinghub/splash 通路你自己伺服器的ip，http://10.0.0.11:8050
二、安裝scrapy-splash建立項目
pip install scrapy-splash 建立scrapy項目
scrapy startproject JDspider 配置setting
ROBOTSTXT_OBEY = False
SPIDER_MIDDLEWARES = {
'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
}
DOWNLOADER_MIDDLEWARES = {
'scrapy_splash.SplashCookiesMiddleware': 723,
'scrapy_splash.SplashMiddleware': 725,
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810
}
SPLASH_URL = 'http://10.0.0.11:8050' #你自己的伺服器位址
DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'
HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage' 建立spider檔案
import scrapy
from scrapy_splash import SplashRequest
import logging
search_script = '''
function main(splash, args)
splash.images_enabled = false
splash:set_user_agent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36')
assert(splash:go(args.url))
splash:wait(0.5)
local input = splash:select("#keyword")
input:send_text('{}')
splash:wait(0.5)
local form = splash:select('.input_submit')
form:click()
splash:wait(2)
splash:runjs("document.getElementsByClassName('bottom-search')[0].scrollIntoView(true)")
splash:wait(6)
return splash:html()
end
'''
class JsSpider(scrapy.Spider):
name = "jd"
allowed_domains = ["www.jd.com"]
start_urls = [
"https://search.jd.com/"
]
def start_requests(self):
splash_args = {
'wait': 2,
'lua_source': search_script.format("小米10")
}
for url in self.start_urls:
yield SplashRequest(url, self.parse_result, endpoint='execute',
args=splash_args)
def parse_result(self, response):
if response.status == 200:
ul_list = response.xpath('//*[@id="J_goodsList"]/ul/li')
print(ul_list)
print(len(ul_list))
for i in range(1, len(ul_list) + 1):
logging.info(u'----------使用splash爬取京東網異步加載内容-----------')
xm10_price = response.xpath(
'//*[@id="J_goodsList"]/ul/li[{}]/div/div[3]/strong/i/text()'.format(i)).extract_first()
logging.info(u"find：%s" % xm10_price)
logging.info(u'---------------success----------------') 建立項目啟動檔案
from scrapy.cmdline import execute
execute(['scrapy', 'crawl', 'jd']) 三、運作項目輸出結果
2020-02-16 22:15:26 [scrapy.utils.log] INFO: Scrapy 1.8.0 started (bot: JDspider)
2020-02-16 22:15:26 [scrapy.utils.log] INFO: Versions: lxml 4.3.2.0, libxml2 2.9.5, cssselect 1.0.3, parsel 1.5.2, w3lib 1.20.0, Twisted 19.2.0, Python 3.6.7 |Anaconda, Inc.| (default, Oct 28 2018, 19:44:12) [MSC v.1915 64 bit (AMD64)], pyOpenSSL 18.0.0 (OpenSSL 1.1.0i 14 Aug 2018), cryptography 2.3.1, Platform Windows-10-10.0.17763-SP0
2020-02-16 22:15:26 [scrapy.crawler] INFO: Overridden settings: {'BOT_NAME': 'JDspider', 'DUPEFILTER_CLASS': 'scrapy_splash.SplashAwareDupeFilter', 'HTTPCACHE_STORAGE': 'scrapy_splash.SplashAwareFSCacheStorage', 'NEWSPIDER_MODULE': 'JDspider.spiders', 'SPIDER_MODULES': ['JDspider.spiders']}
2020-02-16 22:15:26 [scrapy.extensions.telnet] INFO: Telnet Password: e23c115bddb3c3fa
2020-02-16 22:15:26 [scrapy.middleware] INFO: Enabled extensions:
['scrapy.extensions.corestats.CoreStats',
'scrapy.extensions.telnet.TelnetConsole',
'scrapy.extensions.logstats.LogStats']
2020-02-16 22:15:26 [scrapy.middleware] INFO: Enabled downloader middlewares:
['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
'scrapy.downloadermiddlewares.retry.RetryMiddleware',
'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
'scrapy_splash.SplashCookiesMiddleware',
'scrapy_splash.SplashMiddleware',
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
'scrapy.downloadermiddlewares.stats.DownloaderStats']
2020-02-16 22:15:26 [scrapy.middleware] INFO: Enabled spider middlewares:
['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
'scrapy_splash.SplashDeduplicateArgsMiddleware',
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
'scrapy.spidermiddlewares.referer.RefererMiddleware',
'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
'scrapy.spidermiddlewares.depth.DepthMiddleware']
2020-02-16 22:15:26 [scrapy.middleware] INFO: Enabled item pipelines:
[]
2020-02-16 22:15:26 [scrapy.core.engine] INFO: Spider opened
2020-02-16 22:15:26 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2020-02-16 22:15:26 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023
2020-02-16 22:15:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://search.jd.com/ via http://10.0.0.11:8050/execute> (referer: None)
[<Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item gl-item-presell" d...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item gl-item-presell" d...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="1000053...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6549112...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="1000035...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6545310...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="1000054...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6550526...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6545274...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="5873228...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item gl-item-presell" d...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6554474...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6549278...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6212269...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6111394...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6549862...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6551420...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6112452...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6121304...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6556373...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6551411...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6103996...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6554278...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6079772...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6555427...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6557779...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6363949...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6199133...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6553114...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="4723083...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6363895...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6128068...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6088586...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6364131...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6549037...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6550853...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6549124...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6546202...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6557937...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6557971...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6362546...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6557774...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6557828...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6557978...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6549308...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6558412...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6558070...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6551411...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6558723...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6558417...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6558373...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6561199...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6551616...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6560815...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6559425...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6557857...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6559487...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6557723...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item gl-item-presell" d...'>, <Selector xpath='//*[@id="J_goodsList"]/ul/li' data='<li class="gl-item" data-sku="6557979...'>]
60
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4299.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：5499.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：2799.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4489.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：599.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4489.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：3199.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4399.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4399.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：2699.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4999.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4199.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4199.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：2599.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：2599.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4178.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4799.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：2799.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：3299.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4199.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4499.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：3099.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4899.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：5199.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4199.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4199.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：2679.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：2999.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4999.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：2999.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：3059.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：2859.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：799.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：589.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4199.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4399.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：5499.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：5499.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：4199.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:36 [root] INFO: find：5499.00
2020-02-16 22:15:36 [root] INFO: ---------------success----------------
2020-02-16 22:15:36 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：4799.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：4199.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：5499.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：5499.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：5499.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：5299.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：4699.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：5499.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：5488.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：4199.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：5499.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：4799.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：4099.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：4499.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：4399.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：4199.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：5399.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：4799.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：5499.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [root] INFO: ----------使用splash爬取京東網異步加載内容-----------
2020-02-16 22:15:37 [root] INFO: find：4499.00
2020-02-16 22:15:37 [root] INFO: ---------------success----------------
2020-02-16 22:15:37 [scrapy.core.engine] INFO: Closing spider (finished)
2020-02-16 22:15:37 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
{'downloader/request_bytes': 1123,
'downloader/request_count': 1,
'downloader/request_method_count/POST': 1,
'downloader/response_bytes': 458413,
'downloader/response_count': 1,
'downloader/response_status_count/200': 1,
'elapsed_time_seconds': 10.35831,
'finish_reason': 'finished',
'finish_time': datetime.datetime(2020, 2, 16, 14, 15, 37, 94205),
'log_count/DEBUG': 1,
'log_count/INFO': 190,
'response_received_count': 1,
'scheduler/dequeued': 2,
'scheduler/dequeued/memory': 2,
'scheduler/enqueued': 2,
'scheduler/enqueued/memory': 2,
'splash/execute/request_count': 1,
'splash/execute/response_count/200': 1,
'start_time': datetime.datetime(2020, 2, 16, 14, 15, 26, 735895)}
2020-02-16 22:15:37 [scrapy.core.engine] INFO: Spider closed (finished)