using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Net;
using System.IO;
using System.Web;
using System.Drawing;
using ICSharpCode.SharpZipLib.GZip;
using System.Collections;
using System.Diagnostics;
using System.Text.RegularExpressions;
using System.Windows.Forms;
namespace Squeen.Common
{
public partial class HttpUtility
{
#region private member
internal const string FAIL_MSG = "fail";
const int MAX_TRY_NUMBER = 1;
private ArrayList alCookie = new ArrayList();
private ArrayList alPostUrl = new ArrayList();
private string _useWebProxy = null;
private CookieContainer _pContainer;
private WebProxy _currentProxy;
#endregion
/// <summary>
/// Cookie瀹瑰?? /// </summary>
public CookieContainer PageCookieContainer
{
set { _pContainer = value; }
get
{
if (_pContainer == null) _pContainer = new CookieContainer();
return _pContainer;
}
}
/// <summary>
/// 浠ｇ?????″?? /// </summary>
public WebProxy CurrentProxy
{
set { _currentProxy = value; }
get { return _currentProxy; }
}
private string _responseUrl = string.Empty;
public string ResponseUrl
{
get { return _responseUrl; }
set { _responseUrl = value; }
}
/// <summary>
/// Get Method
/// </summary>
/// <param name="url"></param>
/// <param name="encoding"></param>
/// <param name="checkPoint"></param>
/// <param name="refer"></param>
/// <returns></returns>
public string Get(string url, string VeriPoint, string SuccessPoint, int? MinSize, System.Text.Encoding encoding, string refer)
{
return Get(url,VeriPoint,SuccessPoint ,MinSize, encoding, refer, null);
}
public string Get(string url, string VeriPoint, string SuccessPoint, int? MinSize, System.Text.Encoding encoding, string refer, LogMonitor logMonitor)
{
return TryRequest(url,VeriPoint,SuccessPoint ,MinSize, null, false, refer, this.PageCookieContainer, null, encoding, true, 2, logMonitor);
}
public string Get(string url, string VeriPoint, string SuccessPoint, int? MinSize, System.Text.Encoding encoding, string refer, CookieContainer container, LogMonitor logMonitor)
{
return TryRequest(url,VeriPoint,SuccessPoint ,MinSize, null, false, refer, container, null, encoding, true, 2, logMonitor);
}
public string Get(string url)
{
return TryRequest(url,"","",null, null, false,"", null, null, Encoding.UTF8, false, 1,null);
}
/// <summary>
/// ??浜? /// </summary>
/// <param name="postData"></param>
/// <param name="url"></param>
/// <param name="referurl"></param>
/// <param name="encoding"></param>
/// <param name="method"></param>
/// <param name="sbLog"></param>
/// <param name="isWorkMode"></param>
/// <returns></returns>
public string Post(string postData, string url, string VeriPoint, string SuccessPoint, int? MinSize, string referurl, System.Text.Encoding encoding, int method, bool redirect, LogMonitor logMonitor)
{
return Post(postData, url,VeriPoint,SuccessPoint ,MinSize, referurl, encoding, this.PageCookieContainer,null, method, redirect, logMonitor);
}
public string Post(string url, string VeriPoint, string SuccessPoint, int? MinSize, string postData, System.Text.Encoding encoding, LogMonitor logMonitor)
{
return TryRequest(url,VeriPoint,SuccessPoint ,MinSize, encoding.GetBytes(postData), true, "", null, null, encoding, false, 1, logMonitor);
}
/// <summary>
/// ??浜ら〉?? /// </summary>
/// <param name="url">椤甸?㈠?板??</param>
/// <param name="postData">浼????版??濡?锛?id=222&type=1</param>
/// <param name="container">cookie锛??婚??淇℃??</param>
/// <param name="logMonitor">??娴?淇℃??</param>
/// <returns></returns>
public string Post(string url, string VeriPoint, string SuccessPoint, int? MinSize, string postData, CookieContainer container, LogMonitor logMonitor)
{
this.PageCookieContainer = container;
return Post(postData, url,VeriPoint,SuccessPoint ,MinSize, url, Encoding.UTF8, 0, false, logMonitor);
}
/// <summary>
/// POST method
/// </summary>
/// <param name="pPostData"></param>
/// <param name="Url"></param>
/// <param name="pEncoding"></param>
/// <param name="pContainer"></param>
/// <param name="pPorxy"></param>
/// <returns></returns>
public string Post(string pPostData, string Url, string VeriPoint, string SuccessPoint, int? MinSize, string referUrl, System.Text.Encoding encoding, CookieContainer container, List<WebProxy> lstPxy, int method, bool redirect, LogMonitor logMonitor)
{
if (string.IsNullOrEmpty(pPostData)) return FAIL_MSG;
if (pPostData.StartsWith("&")) pPostData = pPostData.Substring(1);
if (method == 0)
{
byte[] byte1 = encoding.GetBytes(pPostData);
return TryRequest(Url, VeriPoint,SuccessPoint ,MinSize, byte1, true, referUrl, container, lstPxy, encoding, redirect, 0, logMonitor);
}
return TryRequest(GetUrlofGet(Url, pPostData),VeriPoint,SuccessPoint ,MinSize, new byte[0], false, referUrl, container, lstPxy, encoding, redirect, 0,logMonitor);
}
public static object @Watch = new object();
public void GetFileByWebClient(string url, string fileName, LogMonitor logMonitor) {
string strPath = Path.GetFullPath(fileName);
string strFileName = Path.GetFileName(fileName);
lock (@Watch)
{
Stopwatch watch = new Stopwatch();
try
{
watch.Start();
logMonitor.BeginRequestTime = DateTime.Now;
GetImg(url, strPath, strFileName, false, url);
watch.Stop();
logMonitor.Spend = watch.ElapsedMilliseconds;
logMonitor.EndResponseTime = DateTime.Now;
}
catch (System.Exception ex)
{
watch.Stop();
logMonitor.Spend = watch.ElapsedMilliseconds;
logMonitor.EndResponseTime = DateTime.Now;
logMonitor.Description = ex.Message;
}
}
}
private string GetUrlofGet(string url, string postData)
{
if (url.IndexOf("?") > 0)
{
postData = "&" + postData;
return url + postData;
}
return url + "?" + postData;
}
private string TryRequest(string url, string VeriPoint, string SuccessPoint, int? MinSize, byte[] postData, bool isPostMethod, string referUrl, CookieContainer container, List<WebProxy> pProxy, System.Text.Encoding encoding, bool redirect, int maxtimes, LogMonitor logMonitor)
{
WebProxy proxy = null;
if (pProxy != null && pProxy.Count > 0) proxy = pProxy[0];
if (maxtimes <= 0)
{
maxtimes = MAX_TRY_NUMBER;
}
for (int i = 0; i < maxtimes; i++)
{
try
{
return SendRequest(url,VeriPoint,SuccessPoint ,MinSize, postData, isPostMethod, referUrl, container, proxy, encoding, redirect,logMonitor);
}
catch (System.Exception ex)
{
}
}
return FAIL_MSG;
}
private string GetDns(string strUrl)
{
Regex reg = new Regex(@"^http(s)?://[^/]+", RegexOptions.IgnoreCase);
string strDNS = reg.Match(strUrl).Value;
return strDNS;
}
private string SendRequest(string url, string VeriPoint,string SuccessPoint,int? MinSize, byte[] postData, bool isPostMethod, string referUrl, CookieContainer container, WebProxy pProxy, System.Text.Encoding encoding, bool redirect, LogMonitor logMonitor)
{
HttpWebRequest request = GetRequest(url, postData, isPostMethod, referUrl, container, pProxy, encoding, redirect);
if (logMonitor == null) logMonitor = new LogMonitor();
string result = "";
//???版??
lock (@Watch)
{
logMonitor.BeginRequestTime = DateTime.Now;
Stopwatch watch = new Stopwatch();
watch.Start();
try
{
if (isPostMethod)
{
System.IO.Stream PostStream = request.GetRequestStream();
PostStream.Write(postData, 0, postData.Length);
PostStream.Close();
}
//????搴?
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream streamReceive;
string gzip = response.ContentEncoding;
if (string.IsNullOrEmpty(gzip) || gzip.ToLower() != "gzip")
{
streamReceive = response.GetResponseStream();
}
else
{
streamReceive = new System.IO.Compression.GZipStream(response.GetResponseStream(), System.IO.Compression.CompressionMode.Decompress);
}
StreamReader streamReader = new StreamReader(streamReceive, encoding);
result = streamReader.ReadToEnd();
if (response.ResponseUri != null && !string.IsNullOrEmpty(response.ResponseUri.OriginalString))
{
this.ResponseUrl = response.ResponseUri.OriginalString;
}
if (GetDns(ResponseUrl) != GetDns(url))
{
logMonitor.HttpCode = 601;
logMonitor.Description = "璁块??缃?椤靛????璺宠浆锛?";
}
if (ResponseUrl != url)
{
if (ResponseUrl.IndexOf("error.aspx") > -1)
{
logMonitor.HttpCode = 604;
logMonitor.Description = "璁块??椤甸?㈠?洪??锛?";
}
}
if (!string.IsNullOrEmpty(VeriPoint))
{
string[] strErrors = VeriPoint.Split(',');
foreach (string error in strErrors)
{
if (!string.IsNullOrEmpty(error) && result.IndexOf(error) > -1)
{
logMonitor.HttpCode = 607;
logMonitor.Description = "璁块??椤甸?㈠??瀹瑰?洪??锛?";
break;
}
}
}
if (!string.IsNullOrEmpty(SuccessPoint))
{
string[] strSuccesses = SuccessPoint.Split(',');
foreach (string success in strSuccesses)
{
if (!string.IsNullOrEmpty(success) && result.IndexOf(success) < 0)
{
logMonitor.HttpCode = 608;
logMonitor.Description = "璁块??椤甸?㈡???芥?剧ず姝ｇ‘??瀹瑰?洪??锛?";
break;
}
}
}
if ((MinSize ?? 0) != 0 && result.Length < MinSize*1024)
{
logMonitor.HttpCode = 609;
logMonitor.Description = "璁块??椤甸?㈠??瀹瑰?浜?" + MinSize + "K?洪??锛?";
}
response.Close();
streamReader.Close();
}
catch (WebException ex)
{
HttpStatusCode status = ((HttpWebResponse)ex.Response).StatusCode;
//?峰??httpCode
logMonitor.HttpCode = (int)status;
logMonitor.Description = "httpCode=" + ((int)status).ToString();
result = ex.Message;
}
catch (Exception e)
{
logMonitor.HttpCode = 600;
result = e.Message;
logMonitor.Description = e.Message;
}
finally
{
//logMonitor.EndResponseTime = DateTime.Now;
watch.Stop();
logMonitor.Spend = watch.ElapsedMilliseconds;
logMonitor.EndResponseTime = DateTime.Now;
watch.Reset();
}
}
return result;
}
private HttpWebRequest GetRequest(string url, byte[] postData, bool isPostMethod, string referUrl, CookieContainer container, WebProxy wp, System.Text.Encoding pEncoding, bool redirect)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 18000;//30000
request.Headers.Set("Pragma", "no-cache");
request.AllowAutoRedirect = true;
request.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
request.Headers.Add("UA-CPU", "x86");
request.Headers.Add("Accept-Charset", "iso-8859-1,*,utf-8,gb2312,gbk");
request.KeepAlive = true;
request.ContentType = "application/x-www-form-urlencoded";
request.Headers.Add("Accept-Encoding", "gzip, deflate");
request.Proxy = null;
//request.ContentType = "text/html; charset=gb2312";
if (isPostMethod) request.ContentLength = postData.Length;
//request.ContentLength = 1284;
request.Method = isPostMethod ? "POST" : "GET";
request.Referer = string.IsNullOrEmpty(referUrl) ? url : referUrl;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
request.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-silverlight, */*";
request.CookieContainer = container;
request.ServicePoint.ConnectionLimit = 512;
return request;
}
/// <summary>
/// 浠?浜???缃??峰???剧??
/// </summary>
/// <param name="url">such as http://www2.baidu.com/vcode.jsp?sessionid=YtUxeffaePagm</param>
/// <param name="pPorxy"></param>
/// <returns></returns>
public string GetImg(string url, string path, string fileName, bool rndName, string refer)
{
Random rd = new Random();
string fname = rndName ? rd.Next(10000, 99999).ToString() + ".bmp" : fileName;
string bmpPath = path + fname;
for (int i = 0; i < MAX_TRY_NUMBER; i++)
{
if (i > MAX_TRY_NUMBER) return "";
try
{
return GetBmpImg(url, path, fname, null, refer, this.PageCookieContainer);
}
catch (System.Exception ex)
{
Logger.Log(string.Format("\r\n{0}\r\n", url), "code");
}
}
return "";
}
/// <summary>
/// 浠?浜???缃?涓?杞藉?剧??
/// </summary>
/// <param name="url">?剧??url?板??</param>
/// <param name="fileName">???拌矾寰?</param>
public void GegImg(string url,string fileName)
{
Uri uri = new Uri(url);
WebRequest request =FileWebRequest.Create(uri);
WebResponse response = request.GetResponse();
Stream reader = response.GetResponseStream();
FileStream writer = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.Write);
byte[] buff = new byte[512];
int c = 0; //瀹???璇诲????瀛????? while ((c = reader.Read(buff, 0, buff.Length)) > 0)
{
writer.Write(buff, 0, c);
}
writer.Close();
writer.Dispose();
reader.Close();
reader.Dispose();
response.Close();
}
private HttpWebRequest GetImageRequest(string url, byte[] postData, bool isPostMethod, string referUrl, CookieContainer container, WebProxy wp, System.Text.Encoding pEncoding, bool redirect)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 12000; //30000
request.Headers.Set("Pragma", "no-cache");
request.AllowAutoRedirect = !redirect;
request.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
request.Headers.Add("UA-CPU", "x86");
request.Headers.Add("Accept-Charset", "iso-8859-1,*,utf-8,gb2312,gbk");
request.KeepAlive = true;
request.Proxy = null;
if (isPostMethod) request.ContentLength = postData.Length;
request.Method = isPostMethod ? "POST" : "GET";
request.Referer = string.IsNullOrEmpty(referUrl) ? url : referUrl;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
request.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-silverlight, */*";
request.CookieContainer = container;
request.ServicePoint.ConnectionLimit = 512;
return request;
}
private void MakeCookieContainer(HttpWebResponse response)
{
CookieCollection col = response.Cookies;
if (col != null && col.Count > 0)
{
if (this.PageCookieContainer == null) this.PageCookieContainer = new CookieContainer();
for (int i = 0; i < col.Count; i++)
{
this.PageCookieContainer.Add(col[i]);
}
}
}
public void SetCookie(string cookie, string url)
{
CookieManager ckManager = new CookieManager();
ckManager.SetCookie(cookie, url, ref _pContainer);
}
public static Encoding GetEncoding(string encoding)
{
if (encoding.ToLower() == "utf-8") return Encoding.UTF8;
else if (encoding.ToLower() == "gbk") return Encoding.GetEncoding(936);
else if (encoding.ToLower() == "gb2312") return System.Text.Encoding.GetEncoding("GB2312");
return Encoding.Default;
}
/// <summary>
/// GZip Compress
/// </summary>
/// <param name="Source">gzip stream</param>
/// <returns></returns>
public byte[] GZipCompress(Stream Source)
{
Source.Seek(0, SeekOrigin.Begin);
MemoryStream objMem = new MemoryStream();
GZipOutputStream objGzip = new GZipOutputStream(objMem);
const int BUFFER_SIZE = 1024 * 10;
byte[] arrBuffer = new byte[BUFFER_SIZE];
int nGetedCount = 0;
do
{
nGetedCount = Source.Read(arrBuffer, 0, BUFFER_SIZE);
objGzip.Write(arrBuffer, 0, nGetedCount);
} while (nGetedCount > 0);
objGzip.Finish();
byte[] arrResult = objMem.ToArray();
objGzip.Close(); //??缂╁??????锛?杈??烘?灏变?琚??抽??
objGzip = null;
objMem.Close();
objMem = null;
return arrResult;
}
#region private method
/// <summary>
/// 浠?URL?峰???剧??
/// </summary>
/// <param name="url">such as http://www2.baidu.com/vcode.jsp?sessionid=uxeJfax0eg</param>
/// <param name="path">physical path</param>
/// <param name="fileName">assign a file name</param>
/// <param name="wp">web proxy</param>
/// <param name="refer">header refer</param>
/// <param name="pContainer">a instance of cookie container </param>
/// <returns>bmp physical path</returns>
private string GetBmpImg(string url, string path, string fileName, WebProxy wp, string refer, CookieContainer pContainer)
{
HttpWebRequest request = GetImageRequest(url, null, false, refer, pContainer, wp, Encoding.Default, true);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream streamReceive;
string gzip = response.ContentEncoding;
if (string.IsNullOrEmpty(gzip) || gzip.ToLower() != "gzip")
{
streamReceive = response.GetResponseStream();
}
else
{
streamReceive = new System.IO.Compression.GZipStream(response.GetResponseStream(), System.IO.Compression.CompressionMode.Decompress);
}
string bmpPath = path + "\\" + fileName;
byte[] bBuffer = new byte[1024];
int nGetedCount = 0;
do
{
nGetedCount = streamReceive.Read(bBuffer, 0, bBuffer.Length);
} while (nGetedCount > 0);
response.Close();
return bmpPath;
}
public void ClientDownImage(string url, string fileName)
{
using (WebClient client = new WebClient())
{
Stream myStream = client.OpenRead(url);
myStream.ReadTimeout = 500;
//FileStream fs = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.Write);
byte[] bBuffer = new byte[1024];
int nGetedCount = 0;
do
{
nGetedCount = myStream.Read(bBuffer, 0, bBuffer .Length);
//fs.Write(bBuffer, 0, nGetedCount);
} while (nGetedCount > 0);
//fs.Dispose();
//fs.Close();
client.Dispose();
}
}
#endregion
}
}
![【Netty】使用 Netty 开发 HTTP 服务器[图]](data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=)