package com.yami.trading.huobi.hobi.internal; import cn.hutool.core.util.StrUtil; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.yami.trading.bean.cms.Infomation; import com.yami.trading.bean.data.domain.*; import com.yami.trading.common.util.RedisUtil; import com.yami.trading.huobi.data.DataCache; import com.yami.trading.huobi.data.internal.DepthTimeObject; import com.yami.trading.huobi.data.internal.KlineService; import com.yami.trading.huobi.data.internal.TradeTimeObject; import com.yami.trading.huobi.hobi.http.HttpHelper; import com.yami.trading.huobi.hobi.http.HttpMethodType; import com.yami.trading.service.cms.InfomationService; import com.yami.trading.service.item.ItemService; import org.apache.commons.lang3.StringUtils; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; import org.apache.http.message.BasicNameValuePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; /** * 完成需求k线图采集 */ @Component public class XueQiuDataServiceImpl { // https://stock.xueqiu.com/v5/stock/chart/kline.json?symbol=TSLA&begin=1682695800000&period=120m&type=before&count=-500&indicator=kline"; public final static String klineUrl = "https://stock.xueqiu.com/v5/stock/chart/kline.json?symbol={}&begin={}&period={}&type=before&count=-500&indicator=kline"; @Autowired private InfomationService infomationService; /** * live */ public final static String live = "https://stock.xueqiu.com/v5/stock/quote.json"; public final static String markets = "https://stock.xueqiu.com/v5/stock/quote.json"; public final static String pankou = "https://stock.xueqiu.com/v5/stock/realtime/pankou.json"; public final static String tradeList = "https://stock.xueqiu.com/v5/stock/history/trade.json"; private static Logger logger = LoggerFactory.getLogger(XueQiuDataServiceImpl.class); @Autowired private KlineService klineService; @Autowired private ItemService itemService; public static List pankous(String symbols) { Map param = new HashMap<>(); param.put("symbol", symbols); String result = HttpHelper.getJSONFromHttpNew(pankou, param, HttpMethodType.GET); JSONObject resultJson = JSON.parseObject(result); String code = resultJson.getString("error_code"); if ("0".equalsIgnoreCase(code)) { JSONObject jsonObject = JSON.parseObject(resultJson.getString("data")); List depths = new ArrayList<>(); Depth depth = new Depth(); depth.setSymbol(jsonObject.getString("symbol")); depth.setTs(jsonObject.getLong("timestamp")); List asks = new ArrayList<>(); List bids = new ArrayList<>(); DepthEntry asks1 = new DepthEntry(); asks1.setAmount(jsonObject.getDouble("sc1") / 100); asks1.setPrice(jsonObject.getDouble("sp1")); DepthEntry asks2 = new DepthEntry(); asks2.setAmount(jsonObject.getDouble("sc2") / 100); asks2.setPrice(jsonObject.getDouble("sp2")); DepthEntry asks3 = new DepthEntry(); asks3.setAmount(jsonObject.getDouble("sc3") / 100); asks3.setPrice(jsonObject.getDouble("sp3")); DepthEntry asks4 = new DepthEntry(); asks4.setAmount(jsonObject.getDouble("sc4") / 100); asks4.setPrice(jsonObject.getDouble("sp4")); DepthEntry asks5 = new DepthEntry(); asks5.setAmount(jsonObject.getDouble("sc5") / 100); asks5.setPrice(jsonObject.getDouble("sp5")); DepthEntry bids1 = new DepthEntry(); bids1.setAmount(jsonObject.getDouble("bc1") / 100); bids1.setPrice(jsonObject.getDouble("bp1")); DepthEntry bids2 = new DepthEntry(); bids2.setAmount(jsonObject.getDouble("bc2") / 100); bids2.setPrice(jsonObject.getDouble("bp2")); DepthEntry bids3 = new DepthEntry(); bids3.setAmount(jsonObject.getDouble("bc3") / 100); bids3.setPrice(jsonObject.getDouble("bp3")); DepthEntry bids4 = new DepthEntry(); bids4.setAmount(jsonObject.getDouble("bc4") / 100); bids4.setPrice(jsonObject.getDouble("bp4")); DepthEntry bids5 = new DepthEntry(); bids5.setAmount(jsonObject.getDouble("bc5") / 100); bids5.setPrice(jsonObject.getDouble("bp5")); asks.add(asks1); asks.add(asks2); asks.add(asks3); asks.add(asks4); asks.add(asks5); depth.setAsks(asks); bids.add(bids1); bids.add(bids2); bids.add(bids3); bids.add(bids4); bids.add(bids5); depth.setBids(bids); depths.add(depth); return depths; } return Lists.newArrayList(); } public static List getMarkets(String symbols) { try{ String cookie = HttpHelper.getCookie("https://xueqiu.com/"); List stockMarkets = new ArrayList<>(); List strings = Arrays.asList(symbols.split(",")); for (String symbol: strings) { String result = HttpHelper.sendGetHttp(markets, "symbol=" + symbol, cookie); JSONObject resultJson = JSON.parseObject(result); String error_code = resultJson.getString("error_code"); if("0".equalsIgnoreCase(error_code)){ JSONObject data = resultJson.getJSONObject("data"); JSONObject market = data.getJSONObject("market"); market.put("symbol",symbol); StockMarket stockMarket = market.toJavaObject(StockMarket.class); stockMarkets.add(stockMarket); } } return stockMarkets; }catch (Exception e){ logger.error("XueQiuData{}",e); return Lists.newArrayList(); } } public static void main(String[] args) { XueQiuDataServiceImpl service = new XueQiuDataServiceImpl(); // List sz300750 = service.buildOneYearPeriod("AAPL"); // System.out.println(sz300750.size()); List markets1 = service.getMarkets("AAPL,OIL"); System.out.println(markets1); } public void getInformation() { String cookie = HttpHelper.getCookie("https://xueqiu.com/"); String url = "https://xueqiu.com/statuses/livenews/list.json?since_id=-1&max_id=-1&count=15"; String json = HttpHelper.sendGetHttp(url, null, cookie); JSONArray items = JSONObject.parseObject(json).getJSONArray("items"); List infomations = new ArrayList<>(); items.forEach(d -> { Infomation infom = new Infomation(); JSONObject data = (JSONObject) d; String dataId = data.getString("id"); infom.setDataId(dataId); String description = data.getString("text"); infom.setDescription(description); String createAt = data.getString("created_at"); infom.setCreatedAt(createAt); infom.setType("1"); String originUrl = data.getString("target"); infom.setOriginUrl(originUrl); String source = getSource(description); infom.setSource(source); infom.setLang("zh-CN"); String key = "infomation" + originUrl; if (RedisUtil.get(key) == null) { infomations.add(infom); // url存一周 RedisUtil.set(key, 1, 60 * 60 * 24 * 7); } }); if(infomations.size()>1){ infomationService.saveBatch(infomations); } } public static String getSource(String text) { String pattern = "(([^()]*))$"; // 匹配最后一个括号内的文本 Pattern regex = Pattern.compile(pattern); Matcher matcher = regex.matcher(text); if (matcher.find()) { String endText = matcher.group(1); return endText; } else { return ""; } } /** * 可以多个 * * @param symbols */ public static void tradeList(String symbols, boolean isUSStock) { try{ List strings = Arrays.asList(symbols.split(",")); String cookie = HttpHelper.getCookie("https://xueqiu.com/"); for (String symbol: strings) { Map param = new HashMap<>(); param.put("symbol", symbol); StringBuilder parmStr = new StringBuilder(); if (null != param && !param.isEmpty()) { List parm = new ArrayList(param.size()); for (Map.Entry paramEntity : param.entrySet()) { Object value = paramEntity.getValue(); if (null != value && !StringUtils.isBlank(value.toString())) { parm.add(new BasicNameValuePair(paramEntity.getKey(), value.toString())); } } parmStr.append(URLEncodedUtils.format(parm, "UTF-8")); } String result = HttpHelper.sendGetHttp(tradeList, parmStr.toString(), cookie); JSONObject resultJson = JSON.parseObject(result); String code = resultJson.getString("error_code"); if ("0".equals(code)) { JSONObject data = resultJson.getJSONObject("data"); JSONArray items = data.getJSONArray("items"); List tradeDetails = items.toJavaList(TradeDetails.class); if (tradeDetails.size() >= 1) { String symbol_flag = tradeDetails.get(0).getSymbol(); DataCache.putStockTradeList(symbol_flag, tradeDetails); tradeListToTrade(symbol_flag, tradeDetails); if (isUSStock) { setTradeListToDepth(symbol_flag, tradeDetails); } } } } }catch (Exception e){} } public static void setTradeListToDepth(String symbol, List tradeDetails) { Depth depth = new Depth(); depth.setTs(tradeDetails.get(0).getTimestamp() / 1000); depth.setSymbol(symbol); List asks = tradeDetails.stream().filter(t -> t.getSide() == -1).map(t -> { DepthEntry depthEntry = new DepthEntry(); depthEntry.setAmount((double) t.getTrade_volume()); depthEntry.setPrice(t.getCurrent()); return depthEntry; }).collect(Collectors.toList()); List bids = tradeDetails.stream().filter(t -> t.getSide() == 1).map(t -> { DepthEntry depthEntry = new DepthEntry(); depthEntry.setAmount((double) t.getTrade_volume()); depthEntry.setPrice(t.getCurrent()); return depthEntry; }).collect(Collectors.toList()); depth.setAsks(asks); depth.setBids(bids); DepthTimeObject timeObject = new DepthTimeObject(); timeObject.setLastTime(new Date()); timeObject.setDepth(depth); DataCache.getDepth().put(depth.getSymbol(), timeObject); } public static void tradeListToTrade(String symbol, List tradeDetails) { TradeTimeObject timeObject = DataCache.getTrade().get(symbol); if (timeObject == null) { timeObject = new TradeTimeObject(); } timeObject.setLastTime(new Date()); List data = tradeDetails.stream().map(a -> { TradeEntry tradeEntry = new TradeEntry(); tradeEntry.setDirection(a.getSide() == 1 ? "sell" : "buy"); tradeEntry.setAmount((double) a.getTrade_volume()); tradeEntry.setPrice(a.getCurrent()); tradeEntry.setTs(a.getTimestamp() / 1000); return tradeEntry; }).collect(Collectors.toList()); timeObject.put(symbol, data); DataCache.getTrade().put(symbol, timeObject); } /** * 获取原始的K线图数据 * * @param symbol * @param cookie * @return */ public String getRawTimeseriesByMinute(String symbol, String cookie) { long nowTs = System.currentTimeMillis(); long begin = nowTs; String url = StrUtil.format(klineUrl, symbol, begin, "1m"); return HttpHelper.sendGetHttp(url, null, cookie); } public List realtimeSingle(String symbols) { List list = new ArrayList(); try { List strings = Arrays.asList(symbols.split(",")); String cookie = HttpHelper.getCookie("https://xueqiu.com/"); for (String symbol: strings) { String result = HttpHelper.sendGetHttp(live, "symbol=" + symbol, cookie); JSONObject resultJson = JSON.parseObject(result); String code = resultJson.getString("error_code"); if ("0".equals(code)) { JSONObject jsonObject = resultJson.getJSONObject("data").getJSONObject("quote"); Realtime realtime = new Realtime(); String currency; currency = symbol; int decimal = itemService.getDecimal(currency); realtime.setSymbol(currency); realtime.setName(currency); Long timestamp = jsonObject.getLong("timestamp"); if (timestamp.toString().length() > 13) { timestamp = timestamp / 1000; } realtime.setTs(timestamp); realtime.setOpen(jsonObject.getBigDecimal("open").setScale(decimal, RoundingMode.HALF_UP)); realtime.setClose(jsonObject.getBigDecimal("current").setScale(decimal, RoundingMode.HALF_UP)); realtime.setHigh(jsonObject.getBigDecimal("high").setScale(decimal, RoundingMode.HALF_UP)); realtime.setLow(jsonObject.getBigDecimal("low").setScale(decimal, RoundingMode.HALF_UP)); realtime.setMarketCapital(jsonObject.getLong("market_capital")); realtime.setFloatMarketCapital(jsonObject.getLong("float_market_capital")); realtime.setPeForecast(jsonObject.getBigDecimal("pe_forecast")); realtime.setVolumeRatio(jsonObject.getBigDecimal("volume_ratio")); realtime.setTurnoverRate(jsonObject.getBigDecimal("turnover_rate")); BigDecimal amount = jsonObject.getBigDecimal("amount"); if (amount == null) { amount = BigDecimal.ZERO; } realtime.setAmount(amount.setScale(decimal, RoundingMode.HALF_UP)); BigDecimal volume = jsonObject.getBigDecimal("volume"); if (volume == null) { volume = BigDecimal.ZERO; } realtime.setVolume(volume.setScale(decimal, RoundingMode.HALF_UP)); // realtime.setAsk(realtimeJson.getBigDecimal("ask").setScale(decimal, RoundingMode.HALF_UP)); // realtime.setBid(realtimeJson.getBigDecimal("pb").setScale(decimal, RoundingMode.HALF_UP)); list.add(realtime); } } } catch (Exception e) { logger.error("error", e); } return list; } public List realtime(String symbols) { List realtimeList = realtimeSingle(symbols); return realtimeList; } /** * 1day 历史数据 : 周期 1年 * 1week,1mon 历史数据 : 周期 5年 * 请求 350次 */ public Map> getDailyWeekMonthHistory(String symbol) { Map> map = new HashMap<>(); map.put(Kline.PERIOD_1WEEK, buildOneWeekPeriod(symbol)); map.put(Kline.PERIOD_1MON, buildOneMonthPeriod(symbol)); map.put(Kline.PERIOD_1DAY, buildOneDayPeriod(symbol)); map.put(Kline.PERIOD_5DAY, buildFiveDayPeriod(symbol)); map.put(Kline.PERIOD_QUARTER, buildOneQuarterPeriod(symbol)); map.put(Kline.PERIOD_YEAR, buildOneYearPeriod(symbol)); return map; } public List buildOneDayPeriod(String currency) { return getTimeseriesByPeriod(currency, "day", Kline.PERIOD_1DAY, 365); } public List buildOneWeekPeriod(String currency) { return getTimeseriesByPeriod(currency, "week", Kline.PERIOD_1WEEK, 365 * 5); } public List buildOneMonthPeriod(String currency) { return getTimeseriesByPeriod(currency, "month", Kline.PERIOD_1MON, 365 * 5); } public List buildOneQuarterPeriod(String currency) { return getTimeseriesByPeriod(currency, "quarter", Kline.PERIOD_QUARTER, 365 * 100); } public List buildOneYearPeriod(String currency) { return getTimeseriesByPeriod(currency, "year", Kline.PERIOD_YEAR, 365 * 100); } /** * 雪球支持120分钟的,讲120分钟的, * Hourly * 4hourly 3月 */ public List buildFiveDayPeriod(String currency) { List result = Lists.newArrayList(); List timeseriesByOneDay = getTimeseriesByPeriod(currency, "day", Kline.PERIOD_1DAY, 1000); Collections.sort(timeseriesByOneDay, Kline::compareTo); int lastIndex = timeseriesByOneDay.size() - 1; for (int i = lastIndex; i >= 5; i = i - 5) { // 1天K线最新的5条数据 List klineOneTop5 = new ArrayList<>(); klineOneTop5.add(timeseriesByOneDay.get(i)); klineOneTop5.add(timeseriesByOneDay.get(i - 1)); klineOneTop5.add(timeseriesByOneDay.get(i - 2)); klineOneTop5.add(timeseriesByOneDay.get(i - 3)); klineOneTop5.add(timeseriesByOneDay.get(i - 4)); Double high = null; Double low = null; for (Kline kline : klineOneTop5) { if (high == null || high <= kline.getHigh().doubleValue()) { high = kline.getHigh().doubleValue(); } if (low == null || low >= kline.getLow().doubleValue()) { low = kline.getLow().doubleValue(); } } // 保存K线到数据库 Kline kline = new Kline(); kline.setSymbol(currency); kline.setTs(klineOneTop5.get(4).getTs()); kline.setOpen(klineOneTop5.get(4).getOpen()); kline.setHigh(new BigDecimal(high)); kline.setLow(new BigDecimal(low)); kline.setClose(klineOneTop5.get(0).getClose()); kline.setPeriod(Kline.PERIOD_5DAY); // 格式化小数点位 klineService.formatPoint(kline); BigDecimal sumAmount = klineOneTop5.stream() .map(Kline::getAmount) .filter(amount -> amount != null) .reduce(BigDecimal.ZERO, BigDecimal::add); BigDecimal sumVolume = klineOneTop5.stream() .map(Kline::getVolume) .filter(volume -> volume != null) .reduce(BigDecimal.ZERO, BigDecimal::add); kline.setAmount(sumAmount); kline.setVolume(sumVolume); klineService.repairKline(kline); result.add(kline); } Collections.sort(result); // 不需要首尾连 // int len = result.size(); // for (int i = 1; i < len; i++) { // result.get(i).setOpen(result.get(i - 1).getClose()); // } // 默认是升序。最后一个开始,每次step为2 取数据 return result; } /** * 雪球支持120分钟的,讲120分钟的, * Hourly * 4hourly 3月 */ public List getTimeseriesForFourHourly(String currency) { List result = Lists.newArrayList(); List timeseriesByMinute = getTimeseriesByMinute(currency, 120, 90); Collections.sort(timeseriesByMinute, Kline::compareTo); int lastIndex = timeseriesByMinute.size() - 1; for (int i = lastIndex; i >= 1; i = i - 2) { Kline first = timeseriesByMinute.get(i); Kline secnd = timeseriesByMinute.get(i - 1); Kline kline = new Kline(); kline.setPeriod(Kline.PERIOD_4HOUR); kline.setSymbol(currency); Long timestamp = first.getTs(); if (timestamp.toString().length() > 13) { timestamp = timestamp / 1000; } kline.setTs(timestamp); kline.setOpen(secnd.getOpen()); kline.setClose(first.getClose()); kline.setHigh(first.getHigh().compareTo(secnd.getHigh()) > 0 ? first.getHigh() : secnd.getHigh()); kline.setLow(first.getLow().compareTo(secnd.getLow()) < 0 ? first.getLow() : secnd.getLow()); kline.setAmount(first.getAmount().add(secnd.getAmount())); kline.setVolume(first.getVolume().add(secnd.getVolume())); klineService.repairKline(kline); result.add(kline); } Collections.sort(result); // int len = result.size(); // for (int i = 1; i < len; i++) { // result.get(i).setOpen(result.get(i - 1).getClose()); // } // 默认是升序。最后一个开始,每次step为2 取数据 return result; } /** * Hourly * 1hourly 2个小时 */ public List getTimeseriesForTwoHourly(String currency) { return getTimeseriesByMinute(currency, 120, 300); } /** * Hourly * 1hourly 1月 */ public List getTimeseriesForOneHourly(String currency) { return getTimeseriesByMinute(currency, 60, 300); } /** * Minute * 30minute 10天 * 15minute 5天 * 5minute 2天 * 1minute 1天 */ public List getTimeseriesOneMinute(String currency) { return getTimeseriesByMinute(currency, 1, 15); } /** * Minute * 30minute 10天 * 15minute 5天 * 5minute 2天 * 1minute 1天 */ public List getTimeseriesFiveMinute(String currency) { return getTimeseriesByMinute(currency, 5, 15); } /** * Minute * 30minute 10天 * 15minute 5天 * 5minute 2天 * 1minute 1天 */ public List getTimeseriesFifteenMinute(String currency) { return getTimeseriesByMinute(currency, 15, 15); } /** * Minute * 30minute 15 * 15minute 15 * 5minute 15 * 1minute 15 */ public List getTimeseriesThirtyMinute(String currency) { return getTimeseriesByMinute(currency, 30, 10); } public List getTimeseriesByPeriod(String currency, String periodXieQiu, String sysPeriod, long limitDays) { List resList = new ArrayList<>(); long nowTs = System.currentTimeMillis(); long startTs = System.currentTimeMillis() - limitDays * 24 * 60 * 60 * 1000; long begin = nowTs; String cookie = HttpHelper.getCookie("https://xueqiu.com/"); Set tsSet = new HashSet<>(); while (begin > startTs) { String url = StrUtil.format(klineUrl, currency, begin, periodXieQiu); String json = HttpHelper.sendGetHttp(url, null, cookie); JSONObject resultJson = JSON.parseObject(json); JSONArray dataArray = resultJson.getJSONObject("data").getJSONArray("item"); List lists = dataArray.toJavaList(List.class); long minTS = begin; for (List result : lists) { Kline kline = new Kline(); kline.setSymbol(currency); kline.setPeriod(sysPeriod); // 毫秒 long ts = Long.parseLong(result.get(0).toString()); if (Long.toString(ts).length() > 13) { ts = ts / 1000; } if (tsSet.contains(ts)) { continue; } else { tsSet.add(ts); } kline.setTs(ts); kline.setOpen(new BigDecimal(result.get(2).toString())); kline.setClose(new BigDecimal(result.get(5).toString())); kline.setHigh(new BigDecimal(result.get(3).toString())); kline.setLow(new BigDecimal(result.get(4).toString())); kline.setVolume(new BigDecimal(result.get(1).toString())); kline.setAmount(new BigDecimal(result.get(9).toString())); if (klineService != null) { klineService.repairKline(kline); } resList.add(kline); if (ts < minTS) { minTS = ts; } } if (minTS == begin) { break; } begin = minTS; if (begin < startTs) { break; } } Collections.sort(resList); int len = resList.size(); for (int i = 1; i < len; i++) { resList.get(i).setOpen(resList.get(i - 1).getClose()); } return resList; } public List getTimeseriesByMinute(String currency, int minute, long limitDays) { return getTimeseriesByPeriod(currency, minute + "m", minute + "min", limitDays); } }