package com.uniubi.management.util; import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.UnsupportedEncodingException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import javax.net.ssl.SSLException;
import javax.net.ssl.SSLHandshakeException; import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.NoHttpResponseException;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils; import com.alibaba.fastjson.JSON;
import com.uniubi.management.model.ResultWrapper; public class HttpClientUtil { static class HttpResult{
public String text = ""; public HttpResult(String text) {
super();
this.text = text;
}
} private static final int THREAD_COUNT = 10;
private static CloseableHttpClient httpClient;
private static ExecutorService executors;
private static PoolingHttpClientConnectionManager cm;
static{
executors = Executors.newFixedThreadPool(THREAD_COUNT); ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory();
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", plainsf)
.register("https", sslsf)
.build();
cm = new PoolingHttpClientConnectionManager(registry);
// 将最大连接数增加到200
cm.setMaxTotal(200);
// 将每个路由基础的连接增加到20
cm.setDefaultMaxPerRoute(20);
// 将目标主机的最大连接数增加到50
HttpHost localhost = new HttpHost(WebService.WS_ADDRESS);
cm.setMaxPerRoute(new HttpRoute(localhost), 50); //请求重试处理
HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception,int executionCount, HttpContext context) {
if (executionCount >= 5) {// 如果已经重试了5次，就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接，那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return false;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// ssl握手异常
return false;
} HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的，就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
}; httpClient = HttpClients.custom()
.setConnectionManager(cm)
.setRetryHandler(httpRequestRetryHandler)
.build();
} public static void release() {
if (executors != null) {
executors.shutdown();
executors = null;
}
if (cm != null) {
cm.shutdown();
cm = null;
}
} private static void config(HttpRequestBase httpRequestBase) {
httpRequestBase.setHeader("User-Agent", "Mozilla/5.0");
httpRequestBase.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
httpRequestBase.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");//"en-US,en;q=0.5");
httpRequestBase.setHeader("Accept-Charset", "ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7"); // 配置请求的超时设置
RequestConfig requestConfig = RequestConfig.custom()
.setConnectionRequestTimeout(3000)
.setConnectTimeout(3000)
.setSocketTimeout(3000)
.build();
httpRequestBase.setConfig(requestConfig);
} private static String executeMethod(HttpRequestBase httpget) {//, CountDownLatch countDownLatch
CloseableHttpResponse response = null;
String result = "";
try {
response = httpClient.execute(httpget,HttpClientContext.create());
HttpEntity entity = response.getEntity();
result = EntityUtils.toString(entity, "utf-8");
EntityUtils.consume(entity);
} catch (IOException e) {
e.printStackTrace();
} finally {
// countDownLatch.countDown();
try {
if(response != null)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return result;
} public static String doGet(String url){
return doGet(url,null);
}
public static String doGet(String url,String param) {
HttpResult result = new HttpResult("");
try {
CountDownLatch countDownLatch = new CountDownLatch(1);
url = url + "?apiKey="+WebService.WS_APIKEY;
if(!StringUtil.isEmpty(param)){
url = url + "&" + param;
}
HttpGet request = new HttpGet(url);
config(request);
//启动线程抓取
executors.execute(new GetRunnable(result,request,countDownLatch));
countDownLatch.await();
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
// System.out.println("线程" + Thread.currentThread().getName() + "," + System.currentTimeMillis() + ", 所有线程已完成，开始进入下一步！");
}
return result.text;
} public static String doPost(String url) {
return doPost(url,null);
}
public static String doPost(String url,Map<String,String> param) {
// HttpResult result = new HttpResult("");
String responeText = "";
try {
// CountDownLatch countDownLatch = new CountDownLatch(1);
HttpPost request = new HttpPost(url);
config(request);
List<NameValuePair> nvps = new ArrayList<NameValuePair>();
//增加apikey
nvps.add(new BasicNameValuePair("apiKey", WebService.WS_APIKEY)); if(!param.isEmpty()){
Iterator<Entry<String, String>> iter = param.entrySet().iterator();
if(iter.hasNext()){
Entry<String, String> next = iter.next();
nvps.add(new BasicNameValuePair(next.getKey(), next.getValue()));
}
}
request.setEntity(new UrlEncodedFormEntity(nvps));
//启动线程抓取
// executors.execute(new GetRunnable(result,request,countDownLatch));
responeText = executeMethod(request);
// countDownLatch.await();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} finally {
// System.out.println("线程" + Thread.currentThread().getName() + "," + System.currentTimeMillis() + ", 所有线程已完成，开始进入下一步！");
}
// return result.text;
return responeText;
} private static class GetRunnable implements Runnable {
private CountDownLatch countDownLatch;
private final HttpRequestBase request;
private HttpResult result; public GetRunnable(HttpResult result, HttpRequestBase request, CountDownLatch countDownLatch){
this.result = result;
this.request = request; this.countDownLatch = countDownLatch;
}
@Override
public void run() {
CloseableHttpResponse response = null;
try {
response = httpClient.execute(request,HttpClientContext.create());
if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK){
HttpEntity entity = response.getEntity();
EntityUtils.consume(entity);
result.text = EntityUtils.toString(entity, "utf-8");
}
else{
result.text = JSON.toJSONString(ResultWrapper.wrapFailure());
}
} catch (IOException e) {
e.printStackTrace();
} finally {
countDownLatch.countDown(); try {
if(response != null)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
} public static void main(String[] args) {
// String url = WebService.USER_GET_INFO;
// String result = doGet(url,"id=2");
// System.out.println(result); Random r = new Random();
for (int i = 0; i < 10; i++) {
long l1 = System.currentTimeMillis();
String resultText = doGet("http://www.baidu.com/s?wd=" + r.nextInt(5000));
System.out.println("查询耗时" + (System.currentTimeMillis() - l1));
}
}
}
package com.uniubi.management.util;
import java.io.IOException;import java.io.InterruptedIOException;import java.io.UnsupportedEncodingException;import java.net.UnknownHostException;import java.util.ArrayList;import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.Map.Entry;import java.util.Random;import java.util.concurrent.CountDownLatch;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;
import javax.net.ssl.SSLException;import javax.net.ssl.SSLHandshakeException;
import org.apache.http.HttpEntity;import org.apache.http.HttpEntityEnclosingRequest;import org.apache.http.HttpHost;import org.apache.http.HttpRequest;import org.apache.http.HttpStatus;import org.apache.http.NameValuePair;import org.apache.http.NoHttpResponseException;import org.apache.http.client.HttpRequestRetryHandler;import org.apache.http.client.config.RequestConfig;import org.apache.http.client.entity.UrlEncodedFormEntity;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.client.methods.HttpPost;import org.apache.http.client.methods.HttpRequestBase;import org.apache.http.client.protocol.HttpClientContext;import org.apache.http.config.Registry;import org.apache.http.config.RegistryBuilder;import org.apache.http.conn.ConnectTimeoutException;import org.apache.http.conn.routing.HttpRoute;import org.apache.http.conn.socket.ConnectionSocketFactory;import org.apache.http.conn.socket.LayeredConnectionSocketFactory;import org.apache.http.conn.socket.PlainConnectionSocketFactory;import org.apache.http.conn.ssl.SSLConnectionSocketFactory;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;import org.apache.http.message.BasicNameValuePair;import org.apache.http.protocol.HttpContext;import org.apache.http.util.EntityUtils;
import com.alibaba.fastjson.JSON;import com.uniubi.management.model.ResultWrapper;
public class HttpClientUtil {static class HttpResult{public String text = "";
public HttpResult(String text) {super();this.text = text;}}
private static final int THREAD_COUNT = 10;private static CloseableHttpClient httpClient;private static ExecutorService executors;private static PoolingHttpClientConnectionManager cm; static{ executors = Executors.newFixedThreadPool(THREAD_COUNT); ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory(); LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory(); Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create() .register("http", plainsf) .register("https", sslsf) .build(); cm = new PoolingHttpClientConnectionManager(registry); // 将最大连接数增加到200 cm.setMaxTotal(200); // 将每个路由基础的连接增加到20 cm.setDefaultMaxPerRoute(20); // 将目标主机的最大连接数增加到50 HttpHost localhost = new HttpHost(WebService.WS_ADDRESS); cm.setMaxPerRoute(new HttpRoute(localhost), 50); //请求重试处理 HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() { public boolean retryRequest(IOException exception,int executionCount, HttpContext context) { if (executionCount >= 5) {// 如果已经重试了5次，就放弃 return false; } if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接，那么就重试 return true; } if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常 return false; } if (exception instanceof InterruptedIOException) {// 超时 return false; } if (exception instanceof UnknownHostException) {// 目标服务器不可达 return false; } if (exception instanceof ConnectTimeoutException) {// 连接被拒绝 return false; } if (exception instanceof SSLException) {// ssl握手异常 return false; } HttpClientContext clientContext = HttpClientContext.adapt(context); HttpRequest request = clientContext.getRequest(); // 如果请求是幂等的，就再次尝试 if (!(request instanceof HttpEntityEnclosingRequest)) { return true; } return false; } }; httpClient = HttpClients.custom() .setConnectionManager(cm) .setRetryHandler(httpRequestRetryHandler) .build(); } public static void release() { if (executors != null) { executors.shutdown(); executors = null; } if (cm != null) { cm.shutdown(); cm = null; } } private static void config(HttpRequestBase httpRequestBase) { httpRequestBase.setHeader("User-Agent", "Mozilla/5.0"); httpRequestBase.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); httpRequestBase.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");//"en-US,en;q=0.5"); httpRequestBase.setHeader("Accept-Charset", "ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7"); // 配置请求的超时设置 RequestConfig requestConfig = RequestConfig.custom() .setConnectionRequestTimeout(3000) .setConnectTimeout(3000) .setSocketTimeout(3000) .build(); httpRequestBase.setConfig(requestConfig); } private static String executeMethod(HttpRequestBase httpget) {//, CountDownLatch countDownLatch CloseableHttpResponse response = null; String result = ""; try { response = httpClient.execute(httpget,HttpClientContext.create()); HttpEntity entity = response.getEntity(); result = EntityUtils.toString(entity, "utf-8"); EntityUtils.consume(entity); } catch (IOException e) { e.printStackTrace(); } finally {// countDownLatch.countDown(); try { if(response != null) response.close(); } catch (IOException e) { e.printStackTrace(); } } return result;} public static String doGet(String url){ return doGet(url,null); } public static String doGet(String url,String param) { HttpResult result = new HttpResult("");try { CountDownLatch countDownLatch = new CountDownLatch(1); url = url + "?apiKey="+WebService.WS_APIKEY; if(!StringUtil.isEmpty(param)){ url = url + "&" + param; } HttpGet request = new HttpGet(url); config(request); //启动线程抓取 executors.execute(new GetRunnable(result,request,countDownLatch)); countDownLatch.await(); } catch (InterruptedException e) { e.printStackTrace(); } finally {// System.out.println("线程" + Thread.currentThread().getName() + "," + System.currentTimeMillis() + ", 所有线程已完成，开始进入下一步！"); }return result.text;} public static String doPost(String url) { return doPost(url,null); } public static String doPost(String url,Map<String,String> param) {// HttpResult result = new HttpResult(""); String responeText = ""; try {// CountDownLatch countDownLatch = new CountDownLatch(1); HttpPost request = new HttpPost(url); config(request); List<NameValuePair> nvps = new ArrayList<NameValuePair>(); //增加apikey nvps.add(new BasicNameValuePair("apiKey", WebService.WS_APIKEY)); if(!param.isEmpty()){ Iterator<Entry<String, String>> iter = param.entrySet().iterator(); if(iter.hasNext()){ Entry<String, String> next = iter.next(); nvps.add(new BasicNameValuePair(next.getKey(), next.getValue())); } } request.setEntity(new UrlEncodedFormEntity(nvps)); //启动线程抓取// executors.execute(new GetRunnable(result,request,countDownLatch)); responeText = executeMethod(request);// countDownLatch.await(); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } finally {// System.out.println("线程" + Thread.currentThread().getName() + "," + System.currentTimeMillis() + ", 所有线程已完成，开始进入下一步！"); }// return result.text; return responeText; }
private static class GetRunnable implements Runnable { private CountDownLatch countDownLatch; private final HttpRequestBase request;private HttpResult result; public GetRunnable(HttpResult result, HttpRequestBase request, CountDownLatch countDownLatch){ this.result = result; this.request = request; this.countDownLatch = countDownLatch;}@Override public void run() { CloseableHttpResponse response = null; try { response = httpClient.execute(request,HttpClientContext.create()); if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK){ HttpEntity entity = response.getEntity(); EntityUtils.consume(entity); result.text = EntityUtils.toString(entity, "utf-8"); } else{ result.text = JSON.toJSONString(ResultWrapper.wrapFailure()); } } catch (IOException e) { e.printStackTrace(); } finally { countDownLatch.countDown(); try { if(response != null) response.close(); } catch (IOException e) { e.printStackTrace(); } } } } public static void main(String[] args) {// String url = WebService.USER_GET_INFO;// String result = doGet(url,"id=2");// System.out.println(result);Random r = new Random();for (int i = 0; i < 10; i++) {long l1 = System.currentTimeMillis();String resultText = doGet("http://www.baidu.com/s?wd=" + r.nextInt(5000));System.out.println("查询耗时" + (System.currentTimeMillis() - l1));} }}