通過HtmlAgilityPack插件和xpath解析html完成爬蟲抓取資料

爬蟲抓取資料的思路是，根據url位址去擷取html，然後解析html，取出需要的資料

首先需要引入HtmlAgilityPack的dll（下載下傳HtmlAgilityPack.dll）

主要是使用HtmlDocument類來加載擷取到的html代碼，轉換為HtmlDocument對象操作

HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();

doc.LoadHtml(html);

擷取到HtmlDocument對象以後，根據xpath過濾出對應的節點

//找到class=zg_itemImmersion的div節點

string xpathDiv = "//div[@class='zg_itemImmersion']";

HtmlNodeCollection allDivs = doc.DocumentNode.SelectNodes(xpathDiv);

xpath文法可以自行網上查找，簡單實用，很好了解

完整代碼如下：

public static void GetData(string url, ref DataTable dt)
        {
            try
            {
                //WebClient擷取Amazon的html會傳回校驗頁面的html
                //WebClient wc = new WebClient();
                //string html = wc.DownloadString(url);

                //HtmlWeb方式擷取html，擷取多次以後，後續會加載不到html
                //HtmlWeb web = new HtmlWeb();
                //HtmlAgilityPack.HtmlDocument doc = web.Load(url);
                //找到排行的每個商品節點

                //通過HttpWebRequest方式擷取html
                string html = WebRequestPost(url);
                HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
                doc.LoadHtml(html);
                string xpathDiv = "//div[@class='zg_itemImmersion']";//找到class=zg_itemImmersion的div節點
                HtmlNodeCollection allDivs = doc.DocumentNode.SelectNodes(xpathDiv);
                for (int i = 0; i < allDivs.Count; i++)
                {
                    if (i > 2) break;

                    //需要把allDivs裡面的node重新轉換為HtmlNode才能取到對應節點的資訊，否則取到的一直都是第一個
                    HtmlNode node = HtmlNode.CreateNode(allDivs[i].InnerHtml);
                    DataRow dr = dt.NewRow();

                    //過濾商品排名
                    string xpath = "//span[@class='zg_rankNumber']";//找到class=zg_rankNumber的span節點
                    string indexText = node.SelectSingleNode(xpath).InnerText.Replace(".", "").Replace("\n", "").TrimStart().TrimEnd();
                    int rank = int.Parse(indexText);
                    dr["排名"] = rank;

                    //過濾商品名稱
                    xpath = "//div[@class='p13n-sc-truncate p13n-sc-truncated-hyphen p13n-sc-line-clamp-2']";//找到對應class的div節點
                    string name = node.SelectSingleNode(xpath).InnerText.Replace("\n", "").TrimStart().TrimEnd();
                    dr["商品名稱"] = name;

                    //過濾商品價格
                    xpath = "//span[@class='p13n-sc-price']";//找到class=p13n-sc-price的span節點
                    string price = node.SelectSingleNode(xpath).InnerText.Replace("\n", "");
                    dr["售價"] = price;

                    //過濾商品明細連接配接，position()從1開始
                    xpath = "//a[@class='a-link-normal' and position()=1]";//找到class=a-link-normal的并且位置是第一個的a節點
                    string href = node.SelectSingleNode(xpath).Attributes["href"].Value;
                    href = "https://www.amazon.com" + href;
                    string htmlDetail = WebRequestPost(href);
                    HtmlAgilityPack.HtmlDocument docDetail = new HtmlAgilityPack.HtmlDocument();
                    docDetail.LoadHtml(htmlDetail);
                    xpath = "//div[@id='detailBulletsWrapper_feature_div']";//找到id=detailBulletsWrapper_feature_div的div節點
                    HtmlNode nodeDetail = docDetail.DocumentNode.SelectSingleNode(xpath);
                    if (nodeDetail != null)
                    {
                        //過濾商品首次上架日期節點
                        //xpath = "//li[position()=5]//span[position()=2]";//不能直接取固定位置的li，因為有些商品的li數量不一緻有些5個，有些6個
                        //找到包含有Date first available at Amazon.com文本内容的span節點的第一個span兄弟節點
                        xpath = "//span[contains(text(), 'Date first available at Amazon.com')]/following-sibling::span[1]";
                        string dateFrist = nodeDetail.SelectSingleNode(xpath).InnerText;
                        dr["首次上架日期"] = dateFrist;

                        //過濾商品分類排名資訊
                        xpath = "//li[@id='SalesRank']/b/following::text()[1]";//找到id=SalesRank的li節點裡面b節點相鄰的第一個文本節點
                        string categoryRank = nodeDetail.SelectSingleNode(xpath).InnerText.Replace("(", "");//擷取主分類排名
                        xpath = "//li[@id='SalesRank']/ul[@class='zg_hrsr']";//找到id=SalesRank的li節點裡面class=zg_hrsr的ul節點
                        string detailRank = nodeDetail.SelectSingleNode(xpath).InnerText.Replace("&nbsp;", " ").Replace("&gt;", ">");//擷取具體分類排名
                        dr["排名資訊"] = categoryRank + detailRank;
                    }
                    dt.Rows.Add(dr);
                }
            }
            catch (Exception ex)
            {
                MessageBox.Show("爬蟲抓取失敗，失敗資訊：" + ex.Message);
            }
        }      

View Code

最後将html裡面的内容解析出來以後，添加到DataTable，然後再導入到Excel

轉載于:https://www.cnblogs.com/zfylzl/p/6949954.html