package link.at17.mid.tushare.data.crawler.tushare;

import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.TreeSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.Function;
import java.util.stream.Stream;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;

import com.alibaba.fastjson2.JSONObject;

import link.at17.mid.tushare.data.crawler.QueryWay;
import link.at17.mid.tushare.data.crawler.RetryAndDelay;
import link.at17.mid.tushare.data.models.StockInfo;
import link.at17.mid.tushare.data.models.interfaces.ITsStockInfo;
import link.at17.mid.tushare.data.models.interfaces.ITsTradeDate;
import link.at17.mid.tushare.data.service.StockCalendarService;
import link.at17.mid.tushare.data.service.StockInfoService;
import link.at17.mid.tushare.enums.StockMarket;
import link.at17.mid.tushare.enums.StockSpan;
import lombok.extern.slf4j.Slf4j;

/**
 * Tushare 爬虫
 * @author Barry
 *
 */
@Component
@Slf4j
public class TushareCrawler {
    

    @Autowired
    private StockInfoService stockInfoService;
    @Autowired
    private StockCalendarService stockCalendarService;
    
    private static final int MAX_THREADS = 5;
    /**
     * 根据日期滚动查询日内所有股票相关信息<br>
     * 该查询的起始日是指交易日历内的最新日，故在调用该方法之前，请确保交易日历已最新，且所有股票已有的数据都是最新日数据<br>
     * 
     * 该接口使用 trade_date 进行轮询
     * @param baseRequest
     * @param ITsTradeDate 实现了 ITsTradeDate 接口的实体
     * @param processDataFunc 获取 List 以后的处理方法
     * @param QueryWay 查询方法
     * @param start 开始日期，留空则默认根据 ITsTradeDate、stockCalendarDao 或个股 listDate 查询
     * @param useTradeDate 是否使用 trade_date 字段来获取数据，否则使用 start_date 和 end_date 来获取（如分钟数据）
     * @return
     * @see TushareCrawler#rollingQueryByStock
     */
    public List<Future<TushareCrawlerResult>> rollingQueryByDate(TushareRequestBody baseRequest, ITsTradeDate iTradeDate,
            Function<List<JSONObject>, Boolean> processDataFunc, QueryWay queryWay, LocalDateTime start){
        ExecutorService es = Executors.newFixedThreadPool(MAX_THREADS);
        RetryAndDelay retryAndDelay = new RetryAndDelay().setThreadNum(MAX_THREADS);
        List<Future<TushareCrawlerResult>> executeResult = new ArrayList<>();
        
        // 生成指纹
        baseRequest.generateFingerPrint();
        
        if (queryWay != QueryWay.ByDateCrossCheck) {
            // 查询最新日到当日
            
            boolean all = queryWay.equals(QueryWay.ByDateAll) || iTradeDate == null;
            
            if (start == null) {
                if (!all && queryWay != QueryWay.ByDateUpdate) {
                    queryWay = QueryWay.ByDateCrossCheck;
                    return rollingQueryByDate(baseRequest, iTradeDate, processDataFunc, queryWay, start);
                }
                start = stockCalendarService.getGreatestLocalDateTime();
            }
            
            if (queryWay == QueryWay.ByDateUpdate && iTradeDate != null) {
                LocalDateTime iTradeDateLatest = iTradeDate.getLatestTradeDate(null);
                if (iTradeDateLatest != null && 
                        iTradeDateLatest.toLocalDate().compareTo(start.toLocalDate()) > 0) {
                    // 数据库内最新数据存在，且大于开市日期，说明有一定数据，从数据库内最新日期开始更新
                    start = iTradeDateLatest;
                }
            }
            start = start.toLocalDate().atStartOfDay();
            LocalDateTime end = LocalDateTime.now().toLocalDate().atStartOfDay();
            
            while(!start.isAfter(end)) {
                if (all || stockCalendarService.isOpen(start, (StockMarket)null)) {
                    TushareRequestBody rq = baseRequest.clone()
                            .addDateParam("trade_date", start);
                    executeResult.add(es.submit(new TushareResponseCallable(rq, processDataFunc)
                            .setRetryAndDelay(retryAndDelay)));
                }
                start = start.plusDays(1);
            }
        }
        else {
            if (iTradeDate == null) {
                throw new RuntimeException("For queryWay = QueryWay.ByDateCrossCheck, iTradeDate must not be null!");
            }
            // 交叉检查
            final LocalDateTime currentDate = LocalDateTime.now();
            final List<LocalDateTime> szStockCalendars = stockCalendarService.getAllOpenDatesBetween(StockMarket.SZ, null, currentDate);
            final List<LocalDateTime> shStockCalendars = stockCalendarService.getAllOpenDatesBetween(StockMarket.SH, null, currentDate);
            final List<LocalDateTime> allStockCalendars = stockCalendarService.getAllOpenDatesBetween((StockMarket)null, null, currentDate);
            final List<StockInfo> stockInfos = stockInfoService.list();
            TreeSet<LocalDateTime> needUpdates = new TreeSet<>();
            stockInfos.forEach(stockInfo -> {
                List<LocalDateTime> stockTradeDates = iTradeDate.getAllTradeDates(stockInfo);
                StockMarket exchange = stockInfo.getExchange();
                List<LocalDateTime> crossToList = exchange == StockMarket.SZ ? szStockCalendars :
                    exchange == StockMarket.SH ? shStockCalendars : allStockCalendars;
                Stream<LocalDateTime> crossTo;
                LocalDateTime listDate = stockInfo.getListDate(), delistDate = stockInfo.getDelistDate();
                if (delistDate != null) {
                    crossTo = crossToList.stream().filter(date -> (date.isAfter(listDate) || date.equals(listDate)) && date.isBefore(delistDate));
                }
                else {
                    crossTo = crossToList.stream().filter(date -> date.isAfter(listDate) || date.equals(listDate));
                }
                
                crossTo.filter(date -> !stockTradeDates.contains(date)).forEach(needUpdates::add);
            });
            needUpdates.descendingSet().forEach(date -> {
                TushareRequestBody rq = baseRequest.clone()
                        .addDateParam("trade_date", date);
                executeResult.add(es.submit(new TushareResponseCallable(rq, processDataFunc)
                        .setRetryAndDelay(retryAndDelay)));
            });
        }
        es.shutdown();
        while (!es.isTerminated()) {
            try { 
                Thread.sleep(100);
            } catch (InterruptedException e) {}
        }
        return executeResult;
    }

    
    /**
     * 根据股票滚动查询，用于获取从上市以来/已有最新记录日期到结束日期的股票的信息轮询<br>
     * 除最初初始化时以外，不建议采用该方法进行轮询，请采用 rollingQueryByDate 方法进行轮询<br>
     * 
     * 存储在数据库内相关联的最新日期的字段名必须为 trade_date<br>
     * 必须是采用 ts_code, start_date 和 end_date 进行查询的接口才可以用
     * @param baseRequest 基本请求，给定接口名和 fields 就行了
     * @param stockList 
     * @param iTradeDate 对应的根据 ts_code 获取数据库内最新一条记录的接口实现，可以检查 list_date 和 delist_date，当为空时，只根据 ts_code 轮询
     * @param afterRespFunc 获取结果后处理的方法，返回是否成功
     * @param singleMax 单次最大请求条数
     * @return 每条请求的执行结果
     * @see TushareCrawler#rollingQueryByDate
     */
    public List<Future<TushareCrawlerResult>> rollingQueryByStock(
            TushareRequestBody baseRequest, 
            List<? extends ITsStockInfo> stockList,
            ITsTradeDate iTradeDate,
            Function<List<JSONObject>, Boolean> afterRespFunc,
            long singleMax,
            QueryWay queryWay,
            StockSpan stockSpan) {

        ExecutorService es = Executors.newFixedThreadPool(MAX_THREADS);
        RetryAndDelay retryAndDelay = new RetryAndDelay().setThreadNum(MAX_THREADS);
        List<Future<TushareCrawlerResult>> executeResult = new ArrayList<>();
        
        // 生成指纹
        baseRequest.generateFingerPrint();
        
        if (stockSpan == null) {
            stockSpan = StockSpan.Daily;
        }

        if (queryWay != QueryWay.ByStockCrossCheck) {
            Assert.isTrue(singleMax > 0, "单次最大请求条数 singleMax 必须大于 0");;
            
            boolean needCheckDate = iTradeDate != null;
            
            for (ITsStockInfo stockInfo : stockList) {
                String stockCode = stockInfo.getTsCode();
                if (needCheckDate) {
                    LocalDateTime start = iTradeDate.getLatestTradeDate(stockInfo);
                    LocalDateTime end = stockInfo.getDelistDate();
                    // 是否退市
                    if (null == end) {
                        // 未退市或没有 delist_date 字段
                        end = LocalDateTime.now().toLocalDate().atStartOfDay();
                    }
                    if (null == start) {
                        // 最新日K为空，则从头开始查询该股所有日K
                        start = stockInfo.getListDate();
                    }
                    else {
                        // 最新日K存在，则查询从最新日开始至今的所有日K并更新
                        start = start.toLocalDate().atStartOfDay();
                        if (start.compareTo(end) == 0) {
                            // 已经是最新了
                            continue;
                        }
                        start = start.plusDays(1);
                        LocalDateTime today = LocalDateTime.now();
                        long todayPassHours = ChronoUnit.HOURS.between(today.toLocalDate().atStartOfDay(), today);
                        if (start.compareTo(end) == 0 && todayPassHours < 15) {
                            // 请求当日数据但未到当日收盘时间
                            continue;
                        }
                    }
                    // 各宽限一天，保证数据补缺、分钟数据
                    start = start.minusDays(1);
                    end = end.plusDays(1);
                    TushareRequestBody rq = baseRequest.clone()
                        .addDateParam("start_date", start)
                        .addParam("ts_code", stockCode);
                    
                    // TODO: 如果是 BJ？
                    StockMarket exchange =  stockCode.endsWith("SZ") ? StockMarket.SZ : StockMarket.SH   ; //stockCode.endsWith("SZ") ? "SZSE" : stockCode.endsWith("SH") ? "SSE" : null;
                    
                    // 计算 start 和 end 之间一共有多少个交易日
                    long dataBetween = stockCalendarService.countOpenDaysBetween(exchange, start, end);
                    long dataPerDay = 1;
                    if (stockSpan.compareTo(StockSpan.Daily) == -1 && stockSpan.compareTo(StockSpan.Minute) >= 0) {
                        // 分钟数，重新算下 dataBetween
                        dataPerDay = 240L / stockSpan.getMin() + 1;
                    }
                    dataBetween *= dataPerDay;
                    long daySpan = singleMax / dataPerDay;
                    LocalDateTime tmpEndDate = stockCalendarService.getOpenDateOffset(exchange, start, daySpan);
                    while (dataBetween > singleMax) {
                        rq.addDateParam("end_date",tmpEndDate);
                        executeResult.add(es.submit(new TushareResponseCallable(rq.clone(), afterRespFunc)
                                .setRetryAndDelay(retryAndDelay)));
                        
                        start = tmpEndDate;
                        tmpEndDate = stockCalendarService.getOpenDateOffset(exchange, start, daySpan);
                        rq.addDateParam("start_date", start);
                        dataBetween -= daySpan * dataPerDay;
                    }
                    rq.addDateParam("end_date", end);
                    executeResult.add(es.submit(new TushareResponseCallable(rq.clone(), afterRespFunc)
                            .setRetryAndDelay(retryAndDelay)));
                }
                else {
                    TushareRequestBody rq = baseRequest.clone()
                        .addParam("ts_code", stockCode);
                    executeResult.add(es.submit(new TushareResponseCallable(rq.clone(), afterRespFunc)
                            .setRetryAndDelay(retryAndDelay)));
                }
            }
        }
        else {
         // 交叉检查
            final List<StockInfo> stockInfos = stockInfoService.list();
            for(StockInfo stockInfo : stockInfos) {
                StockMarket exchange = stockInfo.getExchange();
                List<LocalDateTime> missingDates = iTradeDate.getAllMissingDates(stockInfo);
                if (missingDates.size() > 0) {
                    Collections.sort(missingDates);
                    int i = 0, maxI = missingDates.size() - 1;
                    LocalDateTime start = missingDates.get(0), end = missingDates.get(maxI);
                    end = end.plusDays(1); // end 宽限 1 天，对应分钟数据的情况
                    TushareRequestBody rq = baseRequest.clone()
                        .addDateParam("start_date", start)
                        .addParam("ts_code", stockInfo.getTsCode());
                    
                    // 需要注意的是：交叉检查后的 start 和 end 和非交叉检查的不一样
                    // 不应成为一个范围。比如如果只缺失了 start = 2021-01-09 和 end
                    // = 2022-12-06 两天，成为范围以后将会变成更新 start 和 end 之间
                    // 的所有数据，这样将会大大增加不必要的更新请求。
                    
                    long dataBetween = stockCalendarService.countOpenDaysBetween(exchange, start, end);
                    long dataPerDay = 1;
                    if (stockSpan.compareTo(StockSpan.Daily) == -1 && stockSpan.compareTo(StockSpan.Minute) >= 0) {
                        // 分钟数，重新算下 dataBetween
                        dataPerDay = 240L / stockSpan.getMin() + 1;
                    }
                    dataBetween *= dataPerDay;
                    long daySpan = singleMax / dataPerDay;
                    
                    LocalDateTime tmpEndDate = stockCalendarService.getOpenDateOffset(exchange, start, daySpan);
                    while (dataBetween > singleMax) {
                        rq.addDateParam("end_date", tmpEndDate);
                        executeResult.add(es.submit(new TushareResponseCallable(rq.clone(), afterRespFunc)
                                .setRetryAndDelay(retryAndDelay)));
                        
                        start = tmpEndDate;
                        while (i < maxI && start.isBefore(missingDates.get(i + 1))) {
                            start = missingDates.get(++i);
                        }
                        tmpEndDate = stockCalendarService.getOpenDateOffset(exchange, start, daySpan);
                        rq.addDateParam("start_date", start);
                        dataBetween -= daySpan * dataPerDay;
                    }
                    rq.addDateParam("end_date", end);
                    executeResult.add(es.submit(new TushareResponseCallable(rq.clone(), afterRespFunc)
                            .setRetryAndDelay(retryAndDelay)));
                }
            };
        }
        es.shutdown();
        while (!es.isTerminated()) {
            try { 
                Thread.sleep(100);
            } catch (InterruptedException e) {}
        }
        return executeResult;
    }
}
