from jqdata import *
import datetime
import pandas as pd
import numpy as np
from six import StringIO
import warnings
import time
import pickle
from jqfactor import standardlize
from jqfactor import winsorize_med
from jqfactor import neutralize
import matplotlib.pyplot as plt
import scipy.stats as st
warnings.filterwarnings("ignore")
plt.rcParams['axes.unicode_minus']=False

#获取指定周期的日期列表 'W、M、Q'
def get_period_date(peroid,start_date, end_date):
    #设定转换周期period_type  转换为周是'W',月'M',季度线'Q',五分钟'5min',12天'12D'
    stock_data = get_price('000001.XSHE',start_date,end_date,'daily',fields=['close'])
    #记录每个周期中最后一个交易日
    stock_data['date']=stock_data.index
    #进行转换，周线的每个变量都等于那一周中最后一个交易日的变量值
    period_stock_data=stock_data.resample(peroid,how='last')
    date=period_stock_data.index
    pydate_array = date.to_pydatetime()
    date_only_array = np.vectorize(lambda s: s.strftime('%Y-%m-%d'))(pydate_array )
    date_only_series = pd.Series(date_only_array)
    start_date = datetime.datetime.strptime(start_date, "%Y-%m-%d")
    start_date=start_date-datetime.timedelta(days=1)
    start_date = start_date.strftime("%Y-%m-%d")
    date_list=date_only_series.values.tolist()
    date_list.insert(0,start_date)
    return date_list
get_period_date('M','2018-01-01', '2019-01-01')

#去除上市距beginDate不足3个月的股票
def delect_stop(stocks,beginDate,n=30*3):
    stockList = []
    beginDate = datetime.datetime.strptime(beginDate, "%Y-%m-%d")
    for stock in stocks:
        start_date = get_security_info(stock).start_date
        if start_date < (beginDate-datetime.timedelta(days = n)).date():
            stockList.append(stock)
    return stockList

#获取股票池
def get_stock_ZZ800(begin_date):
    begin_date = str(begin_date)
    stockList = get_index_stocks('000906.XSHG')
    #剔除ST股
    st_data = get_extras('is_st', stockList, count = 1, end_date=begin_date)
    stockList = [stock for stock in stockList if not st_data[stock][0]]
    #剔除停牌、新股及退市股票
    stockList = delect_stop(stockList, begin_date)
    return stockList
get_stock_ZZ800('2016-08-31')

start = time.clock()
begin_date = '2013-01-01'
end_date = '2018-01-01'
dateList = get_period_date('M',begin_date, end_date)
factor = {}
for date in dateList[:-1]:
    stockList = get_stock_ZZ800(date)
    df_data= get_fundamentals(query(valuation.code, valuation.market_cap, indicator.inc_operation_profit_year_on_year).filter(valuation.code.in_(stockList)), date = date)         
    df_data = df_data.set_index(['code'])
    df_data['market_cap'] = -1 * df_data['market_cap'] 
    # 去极值
    df_data = winsorize_med(df_data, scale=1, inclusive=True, inf2nan=True, axis=0)
    # 数据标准化
    df_data = standardlize(df_data, inf2nan=True, axis=0)
    
    df_data['total_55'] = 0.5 * df_data['market_cap'] + 0.5 * df_data['inc_operation_profit_year_on_year']
    df_data['total_64'] = 0.6 * df_data['market_cap'] + 0.4 * df_data['inc_operation_profit_year_on_year']
    # 获取横截面收益率
    df_close = get_price(stockList, date, dateList[dateList.index(date)+1], 'daily', ['close'])
    if df_close.empty:
        continue
    df_pchg = df_close['close'].iloc[-1,:]/df_close['close'].iloc[0,:]-1
    # 上证指数收益率
    index_close = get_price('000906.XSHG', date, dateList[dateList.index(date)+1], 'daily', ['close'])
    index_pchg = index_close['close'].iloc[-1]/index_close['close'].iloc[0]-1
    df_data['excess_pchg'] = df_pchg - index_pchg
    factor[date] = df_data  
end = time.clock()
print "time: ", end-start

# 净值计算
def NetValue(temPchg):
    # 费率
    cost = 0.0015
    netValue = []
    netValue.append(1)
    for i in range(len(temPchg)):
        netValue.append(netValue[i]*(1+temPchg[i] - cost))
    netValue.pop(0)
    return netValue

# 净值曲线绘制
def plotNetValue(dates, netValue):
    # 通过figsize参数可以指定绘图对象的宽度和高度，单位为英寸；
    fig = plt.figure(figsize=(20,8))
    ax = fig.add_subplot(111)
    xticks = range(len(netValue[netValue.keys()[0]]))
    for key in netValue.keys():
        ax.plot(xticks, netValue[key], label=key)
        for i in range(len(xticks)):  
            xticks[i] = xticks[i] + 0.1 
    # 设置图例样式
    ax.legend(loc = 2, fontsize = 10)
    # 设置y标签样式
    ax.set_ylabel('NetValue',fontsize=20)
    ax.set_title("Strategy's NetValue Performances", fontsize=20)
    plt.show() 

# 月度收益柱状图绘制
def plotBar(dates, pchg, label):
    # 通过figsize参数可以指定绘图对象的宽度和高度，单位为英寸；
    fig = plt.figure(figsize=(20,8))
    ax = fig.add_subplot(111)
    xticks = range(len(pchg))
    ax.bar(xticks, pchg, label = label) 
    plt.xticks([x+0.3 for x in xticks], xticks)
    ticks = [int(x) for x in np.linspace(0, len(dates)-1, 11)]
    plt.xticks(ticks, [dates[i] for i in ticks])
    # 设置图例样式
    ax.legend(loc='best',fontsize=15)
    # 设置y标签样式
    ax.set_ylabel('Algorithm_return', fontsize=15)
    # 设置图片标题样式
    ax.set_title("月度收益柱状图", fontsize=15)
    plt.show()

def MaxDrawdown(return_list):
    '''最大回撤率'''
    i = np.argmax((np.maximum.accumulate(return_list) - return_list) / np.maximum.accumulate(return_list))  # 结束位置
    if i == 0:
        return 0
    j = np.argmax(return_list[:i])  # 开始位置
    return (return_list[j] - return_list[i]) / (return_list[j])

# 收益风险统计
def cal_indictor(netValue):
    result = {}
    total_return = netValue[-1] / netValue[0] - 1
    ann_return = pow((1+total_return), 240/float(len(netValue) * 20))-1
    pchg = []
    #计算收益率
    for i in range(1, len(netValue)):
        pchg.append(netValue[i]/netValue[i-1] - 1)
    temp = 0
    for i in pchg:
        temp += pow(i-mean(pchg), 2)
    annualVolatility = sqrt(240/float((len(pchg) * 20-1))*temp)
    sharpe_ratio = (ann_return - 0.04)/annualVolatility
    result['TotalReturn'] = total_return
    result['AnnReturn'] = ann_return
    result['AnnVol'] = annualVolatility
    result['SR'] = sharpe_ratio
    result['MaxDown'] = MaxDrawdown(netValue)
    return result

pchg_55 = []
pchg_64 = []
tradingDays = sort(list(factor.keys()))
for date in tradingDays:
    temp = factor[date]
    temp_55 = temp.sort(['total_55'], ascending = False)
    temp_64 = temp.sort(['total_64'], ascending = False)
    pchg_55.append(np.mean(temp_55['excess_pchg'].iloc[:100]))
    pchg_64.append(np.mean(temp_64['excess_pchg'].iloc[:100]))
    
# 绘制净值曲线
netValue = {}
netValue['等权组合'] = NetValue(pchg_55)
netValue['60/40 组合'] = NetValue(pchg_64)
plotNetValue(tradingDays, netValue)
plotBar(tradingDays, pchg_55, label = '等权组合')
plotBar(tradingDays, pchg_64, label = '60/40 组合')
MonthWinRate1 = len([i for i in pchg_55 if i >0])/float(len(pchg_55))
MonthWinRate2 = len([i for i in pchg_64 if i >0])/float(len(pchg_64))
# 收益风险指标统计
result55 = cal_indictor(netValue['等权组合'])
result64 = cal_indictor(netValue['60/40 组合'])
result = pd.DataFrame()
result['等权组合'] = pd.Series(result55)
result['60/40 组合'] =pd.Series(result64)
result_output = result.T
result_output = result_output[['TotalReturn', 'AnnReturn', 'AnnVol', 'SR', 'MaxDown']]
result_output['MonthWinRate'] = [MonthWinRate1, MonthWinRate2]
result_output

IC_marketcap = []
IC_iop = []
for date in tradingDays:
    temp = factor[date]
    IC_marketcap.append(st.pearsonr(temp['market_cap'], temp['excess_pchg'])[0])
    IC_iop.append(st.pearsonr(temp['inc_operation_profit_year_on_year'], temp['excess_pchg'])[0])
IC = pd.DataFrame(index = ['市值', '营业利润同比增长率'])
IC['均值'] = [np.nanmean(IC_marketcap), np.nanmean(IC_iop)]
IC['标准差'] = [np.nanstd(IC_marketcap), np.nanstd(IC_iop)]
IC['IR'] = [np.nanmean(IC_marketcap) / np.nanstd(IC_iop), np.nanmean(IC_iop) / np.nanstd(IC_iop)]
IC

# 计算市值因子的权重
ratio_marketcap = np.nanmean(IC_marketcap) / (abs(np.nanmean(IC_marketcap)) + np.nanmean(IC_iop))
print '市值因子权重: ', ratio_marketcap
print '营业利润同比增长率因子权重: ', 1-ratio_marketcap
pchg_IC_avg = []
for date in tradingDays:
    temp = factor[date]
    temp['IC_avg'] = ratio_marketcap * temp['market_cap'] + (1 - ratio_marketcap) * temp['inc_operation_profit_year_on_year']
    temp = temp.sort(['IC_avg'], ascending = False)
    pchg_IC_avg.append(np.mean(temp['excess_pchg'].iloc[:100]))
netValue['IC 加权组合'] = NetValue(pchg_IC_avg)
plotNetValue(tradingDays, netValue)
plotBar(tradingDays, pchg_IC_avg, label = 'IC 加权组合')
MonthWinRate3 = len([i for i in pchg_IC_avg if i >0])/float(len(pchg_IC_avg))
# 收益风险指标统计
result_IC_avg = cal_indictor(netValue['IC 加权组合'])
result['IC 加权组合'] = pd.Series(result_IC_avg)
result_output = result.T
result_output = result_output[['TotalReturn', 'AnnReturn', 'AnnVol', 'SR', 'MaxDown']]
result_output['MonthWinRate'] = [MonthWinRate1, MonthWinRate2, MonthWinRate3]
result_output

start = time.clock()
begin_date = '2013-01-01'
end_date = '2018-01-01'
dateList = get_period_date('M',begin_date, end_date)
factor = {}
for date in dateList[:-1]:
    stockList = get_stock_ZZ800(date)
    df_data = get_fundamentals(query(valuation.code, valuation.pb_ratio, valuation.pcf_ratio, valuation.pe_ratio, indicator.inc_operation_profit_year_on_year, balance.total_liability, balance.total_assets).filter(valuation.code.in_(stockList)), date = date)         
    df_data = df_data.set_index(['code'])
    df_turnover = get_fundamentals_continuously(query(valuation.code, valuation.turnover_ratio).filter(valuation.code.in_(stockList)), end_date = date, count = 10)
    df_data['pb_ratio'] = df_data['pb_ratio']
    df_data['turnover_ratio'] = -1 * np.mean(df_turnover['turnover_ratio'])
    df_data['pe_ratio'] = -1 * df_data['pe_ratio']
    df_data['pcf_ratio'] = -1 * df_data['pcf_ratio']
    df_close = get_price(stockList, count = 21, end_date=date, frequency='daily', fields=['close'])['close']
    df_data['pchg'] = -1 * (df_close.iloc[-1] / df_close.iloc[0] - 1)
    df_data['AssetsLiab'] = df_data['total_assets'] / df_data['total_liability']
    # 去极值
    df_data = winsorize_med(df_data, scale=1, inclusive=True, inf2nan=True, axis=0)
    # 数据标准化
    df_data = standardlize(df_data, inf2nan=True, axis=0)
    # 获取横截面收益率
    df_close = get_price(stockList, date, dateList[dateList.index(date)+1], 'daily', ['close'])
    if df_close.empty:
        continue
    df_pchg = df_close['close'].iloc[-1,:]/df_close['close'].iloc[0,:]-1
    # 上证指数收益率
    index_close = get_price('000906.XSHG', date, dateList[dateList.index(date)+1], 'daily', ['close'])
    index_pchg = index_close['close'].iloc[-1]/index_close['close'].iloc[0]-1
    df_data['excess_pchg'] = df_pchg - index_pchg
    factor[date] = df_data  
end = time.clock()
print "time: ", end-start

pchg_mean = []
tradingDays = sort(list(factor.keys()))
for date in tradingDays:
    temp = factor[date]
    temp['total_mean'] = 1/float(7) * temp['pb_ratio'] + 1/float(7) * temp['pcf_ratio'] + 1/float(7) * temp['turnover_ratio'] + 1/float(7) * temp['pchg'] + \
                           1/float(7) * temp['pe_ratio'] + 1/float(7) * temp['AssetsLiab'] + 1/float(7) * temp['inc_operation_profit_year_on_year']
    temp_mean = temp.sort(['total_mean'], ascending = False)
    pchg_mean.append(np.mean(temp_mean['excess_pchg'].iloc[:100]))

IC_pb_ratio = []
IC_turnover = []
IC_pe_ratio = []
IC_pcf_ratio = []
IC_pchg = []
IC_AssetsLiab = []
IC_iop = []

for date in tradingDays:
    temp = factor[date]
    IC_pb_ratio.append(st.pearsonr(temp['pb_ratio'], temp['excess_pchg'])[0])
    IC_turnover.append(st.pearsonr(temp['turnover_ratio'], temp['excess_pchg'])[0])
    IC_pe_ratio.append(st.pearsonr(temp['pe_ratio'], temp['excess_pchg'])[0])
    IC_pcf_ratio.append(st.pearsonr(temp['pcf_ratio'], temp['excess_pchg'])[0])
    IC_pchg.append(st.pearsonr(temp['pchg'], temp['excess_pchg'])[0])
    IC_AssetsLiab.append(st.pearsonr(temp['AssetsLiab'], temp['excess_pchg'])[0])
    IC_iop.append(st.pearsonr(temp['inc_operation_profit_year_on_year'], temp['excess_pchg'])[0])
    
# 计算市值因子的权重
totalWeight =  np.nanmean(IC_pb_ratio) + np.nanmean(IC_turnover) + np.nanmean(IC_pe_ratio) + np.nanmean(IC_pchg) +\
                np.nanmean(IC_AssetsLiab) + np.nanmean(IC_iop) + + np.nanmean(IC_pcf_ratio)
ratio_pb_ratio = np.nanmean(IC_pb_ratio) / totalWeight
ratio_turnover = np.nanmean(IC_turnover) / totalWeight
ratio_pe_ratio = np.nanmean(IC_pe_ratio) / totalWeight
ratio_pcf_ratio = np.nanmean(IC_pcf_ratio) / totalWeight
ratio_pchg = np.nanmean(IC_pchg) / totalWeight
ratio_AssetsLiab = np.nanmean(IC_AssetsLiab) / totalWeight
ratio_iop = np.nanmean(IC_iop) / totalWeight

pchg_IC_avg = []
tradingDays = sort(list(factor.keys()))
for date in tradingDays:
    temp = factor[date]
    temp['total_IC_avg'] = ratio_pb_ratio * temp['pb_ratio'] + ratio_turnover * temp['turnover_ratio'] + ratio_pchg * temp['pchg'] + \
                           ratio_pe_ratio * temp['pe_ratio'] + ratio_AssetsLiab * temp['AssetsLiab'] + ratio_iop * temp['inc_operation_profit_year_on_year'] + ratio_pcf_ratio * temp['pcf_ratio']
    temp_IC_avg = temp.sort(['total_IC_avg'], ascending = False)
    pchg_IC_avg.append(np.mean(temp_IC_avg['excess_pchg'].iloc[:100]))

tempData = pd.DataFrame()
for date in tradingDays[:-1]:
    temp = factor[date][['pb_ratio', 'turnover_ratio', 'pe_ratio', 'inc_operation_profit_year_on_year', 'pchg', 'AssetsLiab', 'pcf_ratio']]
    if tempData.empty:
        tempData = temp
    else:
        tempData = tempData.append(temp)
a = temp.cov()
weight = np.dot(np.linalg.inv(np.array(a)), np.array([np.nanmean(IC_pb_ratio), np.nanmean(IC_turnover), np.nanmean(IC_pe_ratio), np.nanmean(IC_iop), np.nanmean(IC_pchg), np.nanmean(IC_AssetsLiab), np.nanmean(IC_pcf_ratio)]))
pchg_IC_max = []
tradingDays = sort(list(factor.keys()))
for date in tradingDays:
    temp = factor[date]
    temp['total_IC_max'] = weight[0] * temp['pb_ratio'] + weight[1] * temp['turnover_ratio'] + weight[4] * temp['pchg'] + \
                           weight[2] * temp['pe_ratio'] + weight[5] * temp['AssetsLiab'] + weight[3] * temp['inc_operation_profit_year_on_year'] + weight[6] * temp['pcf_ratio']
    temp_IC_max = temp.sort(['total_IC_max'], ascending = False)
    pchg_IC_max.append(np.mean(temp_IC_max['excess_pchg'].iloc[:100]))

# 绘制净值曲线
netValue = {}
netValue['等权组合'] = NetValue(pchg_mean)
netValue['IC 加权组合'] = NetValue(pchg_IC_avg)
netValue['最优 IC 加权组合'] = NetValue(pchg_IC_max)
plotNetValue(tradingDays, netValue)
MonthWinRate1 = len([i for i in pchg_mean if i >0])/float(len(pchg_mean))
MonthWinRate2 = len([i for i in pchg_IC_avg if i >0])/float(len(pchg_IC_avg))
MonthWinRate3 = len([i for i in pchg_IC_max if i >0])/float(len(pchg_IC_max))
# 收益风险指标统计
result_temp = cal_indictor(netValue['等权组合'])
result = pd.DataFrame()
result['等权组合'] = pd.Series(result_temp)
result_temp = cal_indictor(netValue['IC 加权组合'])
result['IC 加权组合'] = pd.Series(result_temp)
result_temp = cal_indictor(netValue['最优 IC 加权组合'])
result['最优 IC 加权组合'] = pd.Series(result_temp)

result_output = result.T
result_output = result_output[['TotalReturn', 'AnnReturn', 'AnnVol', 'SR', 'MaxDown']]
result_output['MonthWinRate'] = [MonthWinRate1, MonthWinRate2, MonthWinRate3]
result_output