爬取全部的校園新聞

作業來源：https://edu.cnblogs.com/campus/gzcc/GZCC-16SE1/homework/3002

0.從新聞url擷取點選次數，并整理成函數

• newsUrl
• newsId(re.search())
• clickUrl(str.format())
• requests.get(clickUrl)
• re.search()/.split()
• str.lstrip(),str.rstrip()
• int
• 整理成函數

• # 點選次數
  def clickCount(url):
      # clickNum=re.findall('/(\d*).html',url)[0]
      clickNum = list(re.search('/(\d*).html', url).groups(1))[0]
      clickurl = 'http://oa.gzcc.cn/api.php?op=count&id={}&modelid=80'.format(clickNum)
      count = html(clickurl).text.split()[0].split('html')[-1]
      ss = "()';"
      for i in ss:
          count = count.replace(i, '')
      co = int(count)
      return co      

• 擷取新聞釋出時間及類型轉換也整理成函數

• # 時間
  def time(url):
      time1 = html(url).select('div .show-info')[0].text.split()[0].split(':')[1]
      time2 = html(url).select('div .show-info')[0].text.split()[1]
      Time = time1 + ' ' + time2
      now1 = datetime.strptime(Time, '%Y-%m-%d %H:%M:%S')
      newstime = datetime.strftime(now1, '%Y{y}-%m{m}-%d{d} %H{H}:%M{M}:%S{S}').format(y='年', m='月', d='日', H='時', M='分',S='秒')
      return newstime      

1.從新聞url擷取新聞詳情： 字典,anews

# 新聞字典
def newsdist(url):
    newsdis = {}
    newsdis['newstitle'] = title(url)
    newsdis['newstime'] = time(url)
    newsdis['newsfrom'] = froms(url)
    newsdis['newswrite'] = write(url)
    newsdis['newsclick'] = clickCount(url)
    # newsdis['newscont']=cont(url)
    return newsdis      

2.從清單頁的url擷取新聞url：清單append(字典) alist

# 每一頁新聞連結清單
def alist(url):
    nets = html(url).select('div .list-container li')
    k = 0
    newslist = []
    for i in nets:
        # 每一條新聞url
        net = i.find('a').get('href')
        # 每一條新聞描述
        description = html(url).select('div .news-list-description')[k].text
        k = k + 1
        newsdis = newsdist(net)
        newsdis['newurl'] = net
        newsdis['newdscription'] = description
        newslist.append(newsdis)
    return newslist      

3.生成所頁清單頁的url并擷取全部新聞 ：清單extend(清單) allnews

for i in range(43, 45):
    if i == 1:
        url = 'http://news.gzcc.cn/html/xiaoyuanxinwen/'
    else:
        url = 'http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html'.format(i)
    allnews = alist(url)
    # 清單疊加
    allnews.extend(alist(url))      

*每個同學爬學号尾數開始的10個清單頁

4.設定合理的爬取間隔

import time

import random

time.sleep(random.random()*3)

5.用pandas做簡單的資料處理并儲存

儲存到csv或excel檔案 

newsdf.to_csv(r'F:\duym\爬蟲\gzccnews.csv')

newsdf = pd.DataFrame(allnews)
print(newsdf)
newsdf.to_csv(r'F:\xinwen.csv')      

全部代碼：

import requests
import bs4
from bs4 import BeautifulSoup as bs
from datetime import datetime
import re
import pandas as pd


def html(url):
    response = requests.get(url=url)
    response.encoding = 'utf-8'
    soup = bs(response.text, 'html.parser')
    return soup
# 點選次數
def clickCount(url):
    # clickNum=re.findall('/(\d*).html',url)[0]
    clickNum = list(re.search('/(\d*).html', url).groups(1))[0]
    clickurl = 'http://oa.gzcc.cn/api.php?op=count&id={}&modelid=80'.format(clickNum)
    count = html(clickurl).text.split()[0].split('html')[-1]
    ss = "()';"
    for i in ss:
        count = count.replace(i, '')
    co = int(count)
    return co
# 标題
def title(url):
    title = html(url).select('div .show-title')[0].text
    return title
# 時間
def time(url):
    time1 = html(url).select('div .show-info')[0].text.split()[0].split(':')[1]
    time2 = html(url).select('div .show-info')[0].text.split()[1]
    Time = time1 + ' ' + time2
    now1 = datetime.strptime(Time, '%Y-%m-%d %H:%M:%S')
    newstime = datetime.strftime(now1, '%Y{y}-%m{m}-%d{d} %H{H}:%M{M}:%S{S}').format(y='年', m='月', d='日', H='時', M='分',S='秒')
    return newstime
# 釋出機關
def froms(url):
    comFrom = html(url).select('div .show-info')[0].text.split()[4].split('：')[1]
    return comFrom
# 作者
def write(url):
    write = html(url).select('div .show-info')[0].text.split()[2].split('：')[1]
    return write
# 内容
def cont(url):
    cont = html(url).select('div .show-content')[0].text
    ss = "\n\u3000\r\xa0"
    for i in ss:
        cont = cont.replace(i, '')
    return cont
# 新聞字典
def newsdist(url):
    newsdis = {}
    newsdis['newstitle'] = title(url)
    newsdis['newstime'] = time(url)
    newsdis['newsfrom'] = froms(url)
    newsdis['newswrite'] = write(url)
    newsdis['newsclick'] = clickCount(url)
    # newsdis['newscont']=cont(url)
    return newsdis
# 每一頁新聞連結清單
def alist(url):
    nets = html(url).select('div .list-container li')
    k = 0
    newslist = []
    for i in nets:
        # 每一條新聞url
        net = i.find('a').get('href')
        # 每一條新聞描述
        description = html(url).select('div .news-list-description')[k].text
        k = k + 1
        newsdis = newsdist(net)
        newsdis['newurl'] = net
        newsdis['newdscription'] = description
        newslist.append(newsdis)
    return newslist


for i in range(43, 45):
    if i == 1:
        url = 'http://news.gzcc.cn/html/xiaoyuanxinwen/'
    else:
        url = 'http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html'.format(i)
    allnews = alist(url)
    # 清單疊加
    allnews.extend(alist(url))
newsdf = pd.DataFrame(allnews)
print(newsdf)
newsdf.to_csv(r'F:\xinwen.csv')      

運作結果：