C#網絡爬蟲--多線程處理強化版

上次做了一個幫公司妹子做了爬蟲，不是很精緻，這次公司項目裡要用到，于是有做了一番修改，功能添加了網址圖檔采集，下載下傳，線程處理界面網址圖檔下載下傳等。

說說思路：首相擷取初始網址的所有内容 在初始網址采集圖檔 去初始網址采集連結 把采集到的連結放入隊列 繼續采集圖檔，然後繼續采集連結，無限循環

還是上圖檔大家看一下，在上代碼！

處理網頁内容抓取跟網頁網址爬取都做了改進，下面還是大家來看看代碼，有不足之處，還請之處!

網頁内容抓取HtmlCodeRequest,

網頁網址爬取GetHttpLinks，用正則去篩選html中的Links

圖檔抓取GetHtmlImageUrlList，用正則去篩選html中的Img

都寫進了一個封裝類裡面 HttpHelper

/// <summary>   
        /// 取得HTML中所有圖檔的 URL。   
        /// </summary>   
        /// <param name="sHtmlText">HTML代碼</param>   
        /// <returns>圖檔的URL清單</returns> 
public static string HtmlCodeRequest(string Url)
        {
            if (string.IsNullOrEmpty(Url))
            {
                return "";
            }
            try
            {
                //建立一個請求
                HttpWebRequest httprequst = (HttpWebRequest)WebRequest.Create(Url);
                //不建立持久性連結
                httprequst.KeepAlive = true;
                //設定請求的方法
                httprequst.Method = "GET";
                //設定标頭值
                httprequst.UserAgent = "User-Agent:Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705";
                httprequst.Accept = "*/*";
                httprequst.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
                httprequst.ServicePoint.Expect100Continue = false;
                httprequst.Timeout = 5000;
                httprequst.AllowAutoRedirect = true;//是否允許302
                ServicePointManager.DefaultConnectionLimit = 30;
                //擷取響應
                HttpWebResponse webRes = (HttpWebResponse)httprequst.GetResponse();
                //擷取響應的文本流
                string content = string.Empty;
                using (System.IO.Stream stream = webRes.GetResponseStream())
                {
                    using (System.IO.StreamReader reader = new StreamReader(stream, System.Text.Encoding.GetEncoding("utf-8")))
                    {
                        content = reader.ReadToEnd();
                    }
                }
                //取消請求
                httprequst.Abort();
                //傳回資料内容
                return content;
            }
            catch (Exception)
            {

                return "";
            }
        }
/// <summary>
        /// 提取頁面連結
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
public static List<string> GetHtmlImageUrlList(string url)
        {
            string html = HttpHelper.HtmlCodeRequest(url);
            if (string.IsNullOrEmpty(html))
            {
                return new List<string>();
            }
            // 定義正規表達式用來比對 img 标簽   
            Regex regImg = new Regex(@"<img\b[^<>]*?\bsrc[\s\t\r\n]*=[\s\t\r\n]*[""']?[\s\t\r\n]*(?<imgUrl>[^\s\t\r\n""'<>]*)[^<>]*?/?[\s\t\r\n]*>", RegexOptions.IgnoreCase);

            // 搜尋比對的字元串   
            MatchCollection matches = regImg.Matches(html);
            List<string> sUrlList = new List<string>();

            // 取得比對項清單   
            foreach (Match match in matches)
                sUrlList.Add(match.Groups["imgUrl"].Value);
            return sUrlList;
        }


        /// <summary>
        /// 提取頁面連結
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
        public static List<string> GetHttpLinks(string url)
        {
            //擷取網址内容
            string html = HttpHelper.HtmlCodeRequest(url);
            if (string.IsNullOrEmpty(html))
            {
                return new List<string>();
            }
            //比對http連結
            const string pattern2 = @"http(s)?://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?";
            Regex r2 = new Regex(pattern2, RegexOptions.IgnoreCase);
            //獲得比對結果
            MatchCollection m2 = r2.Matches(html);
            List<string> links = new List<string>();
            foreach (Match url2 in m2)
            {
                if (StringHelper.CheckUrlIsLegal(url2.ToString()) || !StringHelper.IsPureUrl(url2.ToString()) || links.Contains(url2.ToString()))
                    continue;
                links.Add(url2.ToString());
            }
            //比對href裡面的連結
            const string pattern = @"(?i)<a\s[^>]*?href=(['""]?)(?!javascript|__doPostBack)(?<url>[^'""\s*#<>]+)[^>]*>"; ;
            Regex r = new Regex(pattern, RegexOptions.IgnoreCase);
            //獲得比對結果
            MatchCollection m = r.Matches(html);
            foreach (Match url1 in m)
            {
                string href1 = url1.Groups["url"].Value;
                if (!href1.Contains("http"))
                {
                    href1 = Global.WebUrl + href1;
                }
                if (!StringHelper.IsPureUrl(href1) || links.Contains(href1)) continue;
                links.Add(href1);
            }
            return links;
        }          

這邊下載下傳圖檔有個任務條數限制，限制是200條。如果超過的話線程等待5秒，這裡下載下傳圖檔是異步調用的委托

public string DownLoadimg(string url)
        {
            if (!string.IsNullOrEmpty(url))
            {
                try
                {
                    if (!url.Contains("http"))
                    {
                        url = Global.WebUrl + url;
                    }
                    HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
                    request.Timeout = 2000;
                    request.UserAgent = "User-Agent:Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705";
                    //是否允許302
                    request.AllowAutoRedirect = true;
                    WebResponse response = request.GetResponse();
                    Stream reader = response.GetResponseStream();
                    //檔案名
                    string aFirstName = Guid.NewGuid().ToString();
                    //擴充名
                    string aLastName = url.Substring(url.LastIndexOf(".") + 1, (url.Length - url.LastIndexOf(".") - 1));
                    FileStream writer = new FileStream(Global.FloderUrl + aFirstName + "." + aLastName, FileMode.OpenOrCreate, FileAccess.Write);
                    byte[] buff = new byte[512];
                    //實際讀取的位元組數
                    int c = 0;
                    while ((c = reader.Read(buff, 0, buff.Length)) > 0)
                    {
                        writer.Write(buff, 0, c);
                    }
                    writer.Close();
                    writer.Dispose();
                    reader.Close();
                    reader.Dispose();
                    response.Close();
                    return (aFirstName + "." + aLastName);
                }
                catch (Exception)
                {
                    return "錯誤：位址" + url;
                }
            }
            return "錯誤：位址為空";
        }      

話不多說，更多的需要大家自己去改進咯！歡迎讀者來與樓主進行交流。如果本文對您有參考價值，歡迎幫部落客點下文章下方的推薦，謝謝

有興趣可加入企鵝群一起進步：495104593

下面源碼送上：嘿嘿要分的哦！

http://download.csdn.net/detail/nightmareyan/9627215