爬取全部的校園新聞

　　作業要求來自于：https://edu.cnblogs.com/campus/gzcc/GZCC-16SE2/homework/2941

1.從新聞url擷取新聞詳情： 字典,anews

#擷取新聞頁面資訊
def getNewsInfo(urlList):
    newsDict={};
    newInfo = getHtml(urlList);
    soup = BeautifulSoup(newInfo, "html.parser");
    newsTitle=soup.select(".show-title")[0].text;
    detail=soup.select(".show-info")[0].text;
    dt=changeType(detail);
    newsClick=getClick(urlList);
    newsDict["标題"] = newsTitle;
    newsDict["時間"] = dt;
    newsDict["點選數"] = newsClick;
    return newsDict;      

2.從清單頁的url擷取新聞url：清單append(字典) alist

#擷取新聞清單資訊
def getNewsList(mainUrl):
    newlistInfo = getHtml(mainUrl);
    newsList = [];
    soup=BeautifulSoup(newlistInfo,"html.parser");
    li=soup.select("li");
    for news in li:
        if len(news.select(".news-list-title"))>0:
            #newsTitle=news.select(".news-list-title")[0].text;
            href=news.select("a")[0]["href"];
            description = news.select(".news-list-description")[0].text;
            source = news.select("span")[1].text;
            newsDict = getNewsInfo(href);#擷取新聞頁面的url。
            newsDict["描述"] = description;
            newsDict["連結"] = href;
            newsDict["來源"] = source;
            newsList.append(newsDict);
    print(newsList);
    return newsList;      

3.生成所頁清單頁的url并擷取全部新聞 ：清單extend(清單) allnews

*每個同學爬學号尾數開始的10個清單頁

#主函數
def main():
    infoList=[];
    newsDict = {};
    for htmlNum in range(100,110):
        mainUrl = "http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html".format(htmlNum);  #新聞清單
        info=getNewsList(mainUrl);
        infoList.extend(info);  #将爬取的資訊放置到清單中
        time.sleep(random.random() * 5);  #爬取網頁時間間隔
    setCSV(infoList);      

4.設定合理的爬取間隔

import time

import random

time.sleep(random.random()*3)

5.用pandas做簡單的資料處理并儲存

儲存到csv或excel檔案 

newsdf.to_csv(r'F:\duym\爬蟲\gzccnews.csv')

儲存到資料庫

import sqlite3

with sqlite3.connect('gzccnewsdb.sqlite') as db:

    newsdf.to_sql('gzccnewsdb',db)

#用pandas做簡單的資料處理并儲存
def setCSV(infoList):
    newsdf = pd.DataFrame(infoList); #建立二維表格型資料結構。
    newsdf.to_csv("news.csv",encoding="utf-8");
    print(newsdf);
    with sqlite3.connect('gzccnewsdb.sqlite') as db: #儲存到資料庫
        newsdf.to_sql('gzccnewsdb', db)
      

完整代碼：

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import re
import pandas as pd
import sqlite3
import time
import random


#爬取頁面
def getHtml(url):
    r=requests.get(url);
    r.raise_for_status();
    r.encoding = r.status_code;
    return r.text;

#擷取新聞頁面資訊
def getNewsInfo(urlList):
    newsDict={};
    newInfo = getHtml(urlList);
    soup = BeautifulSoup(newInfo, "html.parser");
    newsTitle=soup.select(".show-title")[0].text;
    detail=soup.select(".show-info")[0].text;
    dt=changeType(detail);
    newsClick=getClick(urlList);
    newsDict["标題"] = newsTitle;
    newsDict["時間"] = dt;
    newsDict["點選數"] = newsClick;
    return newsDict;


#擷取新聞清單資訊
def getNewsList(mainUrl):
    newlistInfo = getHtml(mainUrl);
    newsList = [];
    soup=BeautifulSoup(newlistInfo,"html.parser");
    li=soup.select("li");
    for news in li:
        if len(news.select(".news-list-title"))>0:
            #newsTitle=news.select(".news-list-title")[0].text;
            href=news.select("a")[0]["href"];
            description = news.select(".news-list-description")[0].text;
            source = news.select("span")[1].text;
            newsDict = getNewsInfo(href);#擷取新聞頁面的url。
            newsDict["描述"] = description;
            newsDict["連結"] = href;
            newsDict["來源"] = source;
            newsList.append(newsDict);
    print(newsList);
    return newsList;

#擷取點選數
def getClick(urlList):
    id = re.findall('(\d{1,5})', urlList)[-1];
    clickUrl = 'http://oa.gzcc.cn/api.php?op=count&id={}&modelid=80'.format(id);
    resClick = requests.get(clickUrl);
    newsClick = int(resClick.text.split('.html')[-1].lstrip("('").rstrip("');"));
    return newsClick;


#擷取datetime類型
def changeType(detail):
    newsDate = detail.split()[0].split(':')[1];
    newsTime = detail.split()[1];
    dt = newsDate + ' ' + newsTime;
    #dt = datetime.strptime(newsDT, '%Y-%m-%d %H:%M:%S');
    return dt;



#用pandas做簡單的資料處理并儲存
def setCSV(infoList):
    newsdf = pd.DataFrame(infoList); #建立二維表格型資料結構。
    newsdf.to_csv("news.csv",encoding="utf-8");
    print(newsdf);
    with sqlite3.connect('gzccnewsdb.sqlite') as db: #儲存到資料庫
        newsdf.to_sql('gzccnewsdb', db)


#主函數
def main():
    infoList=[];
    newsDict = {};
    for htmlNum in range(100,110):
        mainUrl = "http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html".format(htmlNum);  #新聞清單
        info=getNewsList(mainUrl);
        infoList.extend(info);  #将爬取的資訊放置到清單中
        time.sleep(random.random() * 5);  #爬取網頁時間間隔
    setCSV(infoList);


main();