爬蟲小練習：Chrome無彈窗爬取淘寶美食資訊

# coding:utf-8

from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from lxml import etree

import re


options = webdriver.ChromeOptions()
options.add_argument('--headless')
driver = webdriver.Chrome(options=options)

def search():
	try:
		print('打開網頁中...')
		driver.get('https://www.taobao.com/')
		driver.maximize_window()

		search = WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH,'//*[@id="q"]')))
		submit = WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.XPATH,'//*[@id="J_TSearchForm"]/div[1]/button')))
		search.send_keys('美食')
		submit.click()
		print('搜尋中...')
		# 等待頁面關鍵元素完全加載，以頁面底部的總頁數為參照
		total_page = WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH,'//*[@id="mainsrp-pager"]/div/div/div/div[1]')))
		print('搜尋完成！')
		get_product()
		# print(total_page.text)
		return total_page.text
	except:
		print('搜尋失敗！再來一次...')
		search()

def next_page(number):
	try:
		number_input = WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH,'//*[@id="mainsrp-pager"]/div/div/div/div[2]/input')))
		submit = WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.XPATH,'//*[@id="mainsrp-pager"]/div/div/div/div[2]/span[3]')))
		number_input.clear()
		number_input.send_keys(number)
		submit.click()
		print('搜尋第%s頁中...'%(number))
		WebDriverWait(driver,20).until(EC.presence_of_element_located((By.XPATH,'//*[@id="mainsrp-itemlist"]/div/div/div[1]/div')))
		print('搜尋第%s頁完成...'%(number))
		get_product()
	except:
		next_page(number)

def get_product():
	print('加載商品頁面中...')
	WebDriverWait(driver,20).until(EC.presence_of_element_located((By.XPATH,'//*[@id="mainsrp-itemlist"]/div/div/div[1]/div')))
	print('目前頁商品加載完成...')
	selector = etree.HTML(driver.page_source)
	# print(html)
	shop_names = selector.xpath('//*[@id="mainsrp-itemlist"]//div[1]/a/span[2]/text()')
	product_names = selector.xpath('//*[@id="mainsrp-itemlist"]//div/div[1]/a/img/@alt')
	product_prices = selector.xpath('//*[@id="mainsrp-itemlist"]//div[2]/div[1]/div[1]/strong/text()')
	image_urls = selector.xpath('//*[@id="mainsrp-itemlist"]//div/div[1]/a/img/@src')

	for item in zip(shop_names,product_names,product_prices,image_urls):
		# print(item)
		# print(type(item[0])) <class 'lxml.etree._ElementUnicodeResult'>
		# print(type(item[1])) <class 'lxml.etree._ElementUnicodeResult'>
		# print(type(item[2])) <class 'lxml.etree._ElementUnicodeResult'>
		# print(type(item[3])) <class 'lxml.etree._ElementUnicodeResult'>
		product = {
			'shop_names': item[0],
			'product_names': item[1],
			'product_prices': '¥' + item[2],
			'image_urls': item[3]
		}
		print(product)


def main():
	count_text = search()
	count = int(re.compile('(\d+)').search(count_text).group(1))
	for i in range(2,count+1):
		next_page(i)


if __name__ == '__main__':
	main()