package com.yami.trading.huobi.hobi.http; import com.alibaba.fastjson.JSONObject; import org.apache.commons.lang3.StringUtils; import org.apache.http.*; import org.apache.http.client.CookieStore; import org.apache.http.client.HttpClient; import org.apache.http.client.HttpResponseException; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.client.utils.URLEncodedUtils; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.HttpConnectionFactory; import org.apache.http.conn.ManagedHttpClientConnection; import org.apache.http.conn.routing.HttpRoute; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.cookie.Cookie; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.BasicCookieStore; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.DefaultHttpResponseParserFactory; import org.apache.http.impl.conn.ManagedHttpClientConnectionFactory; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.impl.conn.SystemDefaultDnsResolver; import org.apache.http.impl.io.DefaultHttpRequestWriterFactory; import org.apache.http.io.HttpMessageWriterFactory; import org.apache.http.message.BasicNameValuePair; import org.apache.http.protocol.BasicHttpContext; import org.apache.http.protocol.HttpContext; import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.SSLContext; import java.io.IOException; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; /** *

* //请求信息类型MIME每种响应类型的输出(普通文本、html 和 XML,json)。允许的响应类型应当匹配资源类中生成的 MIME 类型 * //资源类生成的 MIME 类型应当匹配一种可接受的 MIME 类型。如果生成的 MIME 类型和可接受的 MIME 类型不 匹配,那么将 //生成 * com.sun.jersey.api.client.UniformInterfaceException。例如,将可接受的 MIME 类型设置为 * text/xml,而将 //生成的 MIME 类型设置为 application/xml。将生成 UniformInterfaceException。 * //代理: new HttpHost("10.0.0.172", 80, "http"); *

*/ public class HttpHelper { private static Logger logger = LoggerFactory.getLogger(HttpHelper.class); private static Map> cookiesMap = Collections .synchronizedMap(new HashMap>()); static private HttpClient httpclient; private static Map> globalParam = new HashMap>(5); private static Map headers = new HashMap(); static { RequestConfig config = RequestConfig.copy(RequestConfig.DEFAULT).setConnectionRequestTimeout(30000) .setSocketTimeout(40000).build(); SSLContext sslcontext = SSLContexts.createSystemDefault(); HttpMessageWriterFactory requestWriterFactory = new DefaultHttpRequestWriterFactory(); Registry socketFactoryRegistry = RegistryBuilder.create() .register("http", PlainConnectionSocketFactory.INSTANCE) .register("https", new SSLConnectionSocketFactory(sslcontext, NoopHostnameVerifier.INSTANCE)).build(); HttpConnectionFactory connFactory = new ManagedHttpClientConnectionFactory( requestWriterFactory, new DefaultHttpResponseParserFactory()); PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry, connFactory, new SystemDefaultDnsResolver()); connManager.setMaxTotal(100); connManager.setDefaultMaxPerRoute(100); httpclient = HttpClientBuilder.create().setConnectionManager(connManager).setDefaultRequestConfig(config) .build(); } public static HttpClient getHttpclient(){ return httpclient; } /** * 通过HTTP协议访问站点,并且将返回的数据转换成json对象 注意 对方返回的数据最外层对象必须是单个对象 * * @param url * @param param * @param method * @return * @throws IOException * @throws IOException */ public static String getJSONFromHttp(String url, Map param, HttpMethodType method) throws RuntimeException { String rs; switch (method) { case GET: { rs = sendGetHttp(url, param); break; } case POST: { rs = sendPostHttp(url, param, false); break; } default: { throw new IllegalArgumentException("HTTP访问方式设置有误"); } } // logger.debug("return is:" + rs); return rs == null || "".equals(rs) ? null : rs; } public static String getJSONFromHttpNew(String url, Map param, HttpMethodType method) throws RuntimeException { String rs; switch (method) { case GET: { rs = sendGetHttpNew(url, param); break; } default: { throw new IllegalArgumentException("HTTP访问方式设置有误"); } } // logger.debug("return is:" + rs); return rs == null || "".equals(rs) ? null : rs; } public static String sendHttp(String url, Map param, HttpMethodType method) throws RuntimeException { switch (method) { case GET: { return sendGetHttp(url, param); } case POST: { return sendPostHttp(url, param, false); } default: throw new IllegalArgumentException("参数中的HTTP访问方式有误,只支持GET、POST、FILE"); } } private static final Pattern paramPat = Pattern.compile("([^&]+)=([^&]+)"); /** * 发送一个HTTP协议的GET请求 * * @param url * @param param * @return * @throws IOException */ public static String sendGetHttp(String url, Map param) throws RuntimeException { StringBuilder parmStr = new StringBuilder(); if (null != param && !param.isEmpty()) { List parm = new ArrayList(param.size()); for (Map.Entry paramEntity : param.entrySet()) { Object value = paramEntity.getValue(); if (null != value && !StringUtils.isBlank(value.toString())) { parm.add(new BasicNameValuePair(paramEntity.getKey(), value.toString())); } } parmStr.append(URLEncodedUtils.format(parm, "UTF-8")); } return sendGetHttp(url, parmStr.toString()); } public static String sendGetHttpNew(String url, Map param) { StringBuilder stringBuilder = new StringBuilder(); for (Map.Entry entry : param.entrySet()) { if (!("").equals(stringBuilder.toString())) { stringBuilder.append("&"); } stringBuilder.append(entry.getKey()); stringBuilder.append("="); stringBuilder.append(entry.getValue()); } return sendGetHttp(url, stringBuilder.toString()); } /** * 发送一个HTTP协议的GET请求 * * @param url * @param param * @return * @throws IOException */ public static String sendGetHttp(String url, String param) throws RuntimeException { HttpContext localContext = new BasicHttpContext(); setCookie(localContext, url); if (!StringUtils.isBlank(param)) url = url + ((url.indexOf("?") > 0) ? "&" + param : "?" + param); url = appendGlobalParam(url, param); // logger.debug("远程URL:{}", url); // 创建HttpGet对象 HttpGet request = new HttpGet(url); setHeader(request); String result; try { HttpResponse response = httpclient.execute(request, localContext); result = responseProc(response); } catch (Exception e) { logger.error("url采集失败:{}", url, e.getMessage()); throw new RuntimeException(e); } finally { request.reset(); } return result; } /** * 发送一个HTTP协议的GET请求 * * @param url * @param param * @return * @throws IOException */ public static String sendGetHttp(String url, String param, String cookie) throws RuntimeException { HttpContext localContext = new BasicHttpContext(); setCookie(localContext, url); if (!StringUtils.isBlank(param)) url = url + ((url.indexOf("?") > 0) ? "&" + param : "?" + param); url = appendGlobalParam(url, param); // logger.debug("远程URL:{}", url); // 创建HttpGet对象 HttpGet request = new HttpGet(url); setHeader(request); request.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36"); request.setHeader("Cookie", "cookiesu=151734335486035; device_id=db4f11ccdb5ab5620448eba443476969; HMACCOUNT=71DD59B7799087D2; xq_a_token=cccb558956c11f5aaf8b9a30bcf1f214117e8d67; xqat=cccb558956c11f5aaf8b9a30bcf1f214117e8d67; xq_r_token=4e5ca4e31de699c97d46d00138f4460861fd1778; xq_id_token=eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJ1aWQiOi0xLCJpc3MiOiJ1YyIsImV4cCI6MTczNjk4ODkwMCwiY3RtIjoxNzM0Njg4NjYxNTYwLCJjaWQiOiJkOWQwbjRBWnVwIn0.QP9PIOO3Ta_PXMsOyQHjaVMvVq2k4B5MrnXu-X4Wk4s5p-VEETfjhKyjKot7IiyYvOeuA0GBx98DO8DlvRzf9W5fopsrbMlGz3mzVAJMfTroDEmfv_aPQkPbKadU-dq0iKjgZUjbsYBB3HEXc66mEQvw8BEGvCnHDjx8fGjKqVKn2E0JemcuX72HwC_txjXyPUuZ4cqDEEZP9mFowthCsRLBJK6fQ6oODG6ryr3_LVpl7iAjvKM4mwfVp_AL7F0aMD4GY4UuFpqPF4ZaV71Frx_jTvrNQe76zQnaWg8n4wSRnPfNnC_iT7wS2xexPBf-FCADS6_94bd1z0NxsJOKDA; u=151734335486035; Hm_lvt_1db88642e346389874251b5a1eded6e3=1734688671; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1734688686; ssxmod_itna=YqUxgDc70QG=DtqDQDXKDHmyxPeqOxGKA5YmhiefpPND/SWxnqD=GFDK40o3M0v3TKGi44eUY9fwho0mDeKToUbj0cnGWDCPGnDB94t=ADYY1Dt4DTD34DYDir5DLDmeD+ATKDd6vsNskvxiOD7eDXxGCDQIGx4DaDGak1Wme6gYH0oxDBohvbnxtrDi36O5HY3A+Diy49cr56xG1DQ5DshG6F4DCyILelOvL0tfdvOAhDCKDjcY1zEBP0g5zm=fPhB43WenqWCW5dn2qKe+ete+P=iDYhlr5ClUBYerYEyxyz0=Di6iF1YD; ssxmod_itna2=YqUxgDc70QG=DtqDQDXKDHmyxPeqOxGKA5YmhiefpPG98szDBuBx7PoM7gP+xMtb4Ok0DeRKn2G=D6ng544RijAObkZmcxrupwnzwgbF8VoKkYO=8Rpl5LgFQTOuty4p9C51gLqmQTqW4rDcDx9Ep==Q94ljH=7cB+9e4xr3TYzapY8=W5qgIp1jOwhluAA=aavrKHxzELM2Oij1EOvROGUt08D8H1m3TtQaa5iGmpey9DRBnbQPRIhn4DwrDx5rMqdjHtHvZzkX8BYD08DYIe4D"); String result; try { HttpResponse response = httpclient.execute(request, localContext); result = responseProc(response); } catch (Exception e) { logger.error("url采集失败:{}", url, e.getMessage()); throw new RuntimeException(e); } finally { request.reset(); } return result; } private static String appendGlobalParam(String url, Object param) { for (Map.Entry> stringMapEntry : globalParam.entrySet()) { if (url.startsWith(stringMapEntry.getKey())) { for (Map.Entry paramEntry : stringMapEntry.getValue().entrySet()) { logger.debug("HTTP处理过程发送了参数:" + paramEntry.getKey() + "|" + paramEntry.getValue()); if (param instanceof List) ((List) param).add(new BasicNameValuePair(paramEntry.getKey(), paramEntry.getValue())); else url += "&" + paramEntry.getKey() + "=" + paramEntry.getValue(); } } } return url; } /** * 发送一个HTTP协议的POST请求 * * @param url * @param param * @return * @throws IOException */ public static String sendPostHttp(String url, Map param, boolean postTxtBody) throws RuntimeException { HttpContext localContext = new BasicHttpContext(); setCookie(localContext, url); // logger.debug("远程URL:{}", url); HttpPost request = new HttpPost(url); List parm = new ArrayList(); if (null != param && !param.isEmpty()) for (Map.Entry paramEntity : param.entrySet()) { Object value = paramEntity.getValue(); if (null != value && !StringUtils.isBlank(value.toString())) { logger.debug("HTTP处理过程发送了参数:" + paramEntity.getKey() + "|" + value); parm.add(new BasicNameValuePair(paramEntity.getKey(), value.toString())); } } appendGlobalParam(url, parm); HttpResponse response; try { request.setEntity(generyEntity(parm, "UTF-8", postTxtBody)); setHeader(request); response = httpclient.execute(request, localContext); } catch (Exception e) { logger.error("url采集失败:{}", url, e.getMessage()); throw new RuntimeException(e); } String result; try { result = responseProc(response); } catch (IOException e) { logger.error("url采集失败:{}", url, e.getMessage()); throw new RuntimeException(e); } finally { request.reset(); } // logger.debug("return is:" + result); return result; } static Pattern urlPrePat = Pattern.compile("https?://([^/]*)?/?"); private static String getUrlPerfix(String url) { Matcher mat = urlPrePat.matcher(url); if (mat.find()) return mat.group(1); return ""; } private static void setCookie(HttpContext localContext, String url) { String urlPrefix = getUrlPerfix(url); CookieStore cookieStore = new BasicCookieStore(); List cookieList = cookiesMap.get(urlPrefix); if (cookieList != null && cookieList.size() > 0) { for (Cookie cookie : cookiesMap.get(urlPrefix)) { cookieStore.addCookie(cookie); } localContext.setAttribute(HttpClientContext.COOKIE_STORE, cookieStore); } } private static void parseCookie(HttpClientContext context, String url) { List cookies = context.getCookieStore().getCookies(); String urlPrefix = getUrlPerfix(url); List oldCookies = cookiesMap.get(urlPrefix); if (oldCookies != null) { for (Cookie cookie : cookies) { for (Cookie oldCookie : oldCookies) { if (cookie.getName().equals(oldCookie.getName())) { oldCookies.remove(oldCookie); oldCookies.add(cookie); } } } } else cookiesMap.put(urlPrefix, cookies); } public static String responseProc(HttpResponse response) throws IOException { switch (response.getStatusLine().getStatusCode()) { case 200: { HttpEntity entity = response.getEntity(); return EntityUtils.toString(entity, "UTF-8"); } case 302: { return sendGetHttp(response.getFirstHeader("location").getValue(), ""); } case 303: case 304: { Header[] headers = response.getAllHeaders(); for (Header header : headers) { logger.debug(header.getName() + " : " + header.getValue()); } } default: throw new HttpResponseException(response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); } } public static void setHeader(HttpRequestBase request) { for (Map.Entry headerEntry : headers.entrySet()) { request.setHeader(headerEntry.getKey(), headerEntry.getValue()); } } public static HttpEntity generyEntity(List parm, String encode, boolean postTxtBody) throws Exception { if (postTxtBody && parm.size() > 0) { JSONObject paramJson = new JSONObject(); for (NameValuePair nameValuePair : parm) { paramJson.put(nameValuePair.getName(), nameValuePair.getValue()); } return (new StringEntity(paramJson.toString(), encode)); } else return (new UrlEncodedFormEntity(parm, encode)); } public static String getCookie(String url) throws RuntimeException { HttpContext localContext = new BasicHttpContext(); setCookie(localContext, url); // logger.debug("远程URL:{}", url); // 创建HttpGet对象 HttpGet request = new HttpGet(url); setHeader(request); request.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36"); String result; try { HttpResponse response = httpclient.execute(request, localContext); Header[] setCookieHeaders = response.getHeaders("Set-Cookie"); if (setCookieHeaders.length > 0) { return joinCookies(setCookieHeaders); } return null; } catch (Exception e) { logger.error("url采集失败:{}", url, e.getMessage()); throw new RuntimeException(e); } finally { request.reset(); } } private static String joinCookies(Header[] setCookieHeaders) { return Arrays.stream(setCookieHeaders) .map(header -> header.getValue().split(";")[0]) .collect(Collectors.joining("; ")); } }