封装HttpClient 4.3.4 ,方便日后抓取数据，数据提交（待Cookie访问、302跳转访问）：
package util;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CookieStore;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
/**
* 网页抓取，httpclient-4.3.4 调用
* 类名称：HttpUtil.java
* 创建时间：2014-12-1
* 修改备注：
* @version 1.0
*/
public class HttpUtil {
private static final Log log = LogFactory.getLog(HttpUtil.class);
public static String get(String url){
//初始化一个httpClient
/* 带Cookie方式初始化 httpclient 作为全局变量，不要关闭
CloseableHttpClient httpclient;
CookieStore cookieStore;
String cookies ="user=155;pwd=2545454weafadfwef";
if (cookieStore==null) {
cookieStore = new BasicCookieStore();
String[] ckArr = cookies.split(";");
for (String ck : ckArr) {
String[] tempArr = ck.split("=");
if (tempArr.length==2) {
BasicClientCookie cookie = new BasicClientCookie(tempArr[0].trim(), tempArr[1].trim());
cookie.setVersion(1);
cookie.setDomain("pub.alimama.com");
cookie.setPath("/");
cookieStore.addCookie(cookie);
}
}
}
httpclient = HttpClients.custom().setDefaultCookieStore(cookieStore).build();
*/
//普通方式构造 httpclient
CloseableHttpClient httpclient = HttpClients.createDefault();
return get(url,httpclient);
}
/**
* get方式获取网页内容
* @param url 网址
* @param httpclient httpclient 实例
* @param referer 构造来源地址
* @return
*/
public static String get(String url,CloseableHttpClient httpclient,String referer){
String html ="";
try {
HttpGet httpGet = new HttpGet(url);
Integer timeOut = 30000; //30 秒 超时
RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(timeOut) .setConnectTimeout(timeOut).setSocketTimeout(timeOut).build();
httpGet.setConfig(requestConfig);
if (referer!=null) {
httpGet.setHeader("referer", referer);
}
CloseableHttpResponse response = null;
try {
response = httpclient.execute(httpGet);
HttpEntity entity = response.getEntity();
html = EntityUtils.toString(entity);
} catch (ClientProtocolException e) {
log.error("http协议异常："+e.getMessage());
} catch (IOException e) {
log.error("http读取异常："+e.getMessage());
}catch (Exception e) {
log.error("httpException："+e.getMessage());
}finally {
try {
response.close();
} catch (IOException e) {
log.error("http输出流关闭错误："+e.getMessage());
}
}
} finally {
}
return html;
}
/**
* get方式获取网页内容
* @param url 网址
* @param httpclient httpclient 实例
* @return
*/
public static String get(String url,CloseableHttpClient httpclient){
return get(url, httpclient, null);
}
/**
* post方式提交数据，获取网页内容
* @param url 网址
* @param parmas Map<String, String>方式存放数据名称和值
* @param httpclient httpclient 实例
* @param referer 构造来源地址
* @return
*/
public static String post(String url, Map<String, String> parmas,CloseableHttpClient httpclient,String referer){
String html ="";
try {
HttpPost httpPost = new HttpPost(url);
Integer timeOut = 30000; //30 秒 超时
RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(timeOut) .setConnectTimeout(timeOut).setSocketTimeout(timeOut).build();
httpPost.setConfig(requestConfig);
if (referer!=null) {
httpPost.setHeader("referer", referer);
}
CloseableHttpResponse response = null;
try {
//参数设置
List <NameValuePair> nvps = new ArrayList <NameValuePair>();
for (String key : parmas.keySet()) {
nvps.add(new BasicNameValuePair(key, parmas.get(key)));
}
httpPost.setEntity(new UrlEncodedFormEntity(nvps));
response = httpclient.execute(httpPost);
HttpEntity entity = response.getEntity();
html = EntityUtils.toString(entity);
} catch (ClientProtocolException e) {
log.error("http协议异常："+e.getMessage());
} catch (IOException e) {
log.error("http读取异常："+e.getMessage());
}catch (Exception e) {
log.error("httpException："+e.getMessage());
}finally {
try {
response.close();
} catch (IOException e) {
log.error("http输出流关闭错误："+e.getMessage());
}
}
} finally {
}
return html;
}
/**
* post方式提交数据，获取网页内容
* @param url 网址
* @param parmas Map<String, String>方式存放数据名称和值
* @param httpclient httpclient 实例
* @return
*/
public static String post(String url, Map<String, String> parmas,CloseableHttpClient httpclient){
return post(url, parmas, httpclient, null);
}
/**
* 简单方式，post方式提交数据，获取网页内容
* @param url 网址
* @param parmas Map<String, String>方式存放数据名称和值
* @return
*/
public static String post(String url, Map<String, String> parmas){
CloseableHttpClient httpclient = HttpClients.createDefault();
return post(url, parmas,httpclient);
}
/**
* 302连续跳转，获取真实连接
* @param url
* @param cookieStore
* @return
*/
public static String get302(String url,CookieStore cookieStore) {
CloseableHttpClient httpclient = HttpClients.custom().setDefaultCookieStore(cookieStore).build();
HttpGet httpget = new HttpGet(url);
Integer timeOut = 20000; //30 秒 超时
RequestConfig requestConfig = RequestConfig.custom().setRedirectsEnabled(false).setCircularRedirectsAllowed(false).setConnectionRequestTimeout(timeOut) .setConnectTimeout(timeOut).setSocketTimeout(timeOut).build();
httpget.setConfig(requestConfig);
CloseableHttpResponse response = null;
try {
response = httpclient.execute(httpget);
int status = response.getStatusLine().getStatusCode();
if (status==302) {
url = response.getFirstHeader("Location").getValue();
return get302(url,cookieStore);
}else {
HttpEntity entity = response.getEntity();
return EntityUtils.toString(entity);
}
} catch (Exception e) {
System.err.println("获取失败");
} finally {
try {
response.close();
httpclient.close();
} catch (IOException e) {}
}
return null;
}
} 转载于:https://www.cnblogs.com/hi007/p/httpclient.html