import pandas as pd
import numpy as np
from jqlib.alpha191 import *
import jqdata
import datetime
import time
import statsmodels.api as sm
import scipy.stats as st
import math
from statsmodels import regression
import matplotlib.pyplot as plt

#计算并保存alpha002因子值
#参数：开始时间，结束时间，股票池，不同alpha因子的API
#输出：index为日期，column是股票名称，values是因子值的dataframe
#将alpha因子值保存为CSV格式文件，使用时直接调用
df_sh50 = get_price('000300.XSHG',start_date='2017-5-10',end_date='2018-5-10')
index_date = df_sh50.index
#对于不同时期的成分股不一致的情况，经过检查，对对columns进行校验
columns_stock = get_index_stocks('000300.XSHG')
alpha_002_factor = pd.DataFrame(index = index_date,columns = columns_stock)
alpha_002_factor_neu = pd.DataFrame(index = index_date,columns = columns_stock)
for i in index_date:
    print i
    code= list(get_index_stocks('000300.XSHG'))
    temp_a = alpha_002(code,i)
    #temp_a_neu = get_factor_ne(temp_a,i)['factor_ne']
    temp_a = temp_a.dropna()
    temp_a_neu = get_factor_neutra(temp_a,i)
    #print temp_a_neu
    alpha_002_factor.loc[i,:] = temp_a
    #中性化后的因子值
    alpha_002_factor_neu.loc[i,:] = temp_a_neu
alpha_002_factor.to_csv('alpha_002.csv')    
alpha_002_factor_neu.to_csv('alpha_002_neutra.csv')

#计算并保存alpha003因子值
#参数：开始时间，结束时间，股票池，不同alpha因子的API
#输出：index为日期，column是股票名称，values是因子值的dataframe
#将alpha因子值保存为CSV格式文件，使用时直接调用
df_sh50 = get_price('000300.XSHG',start_date='2017-5-10',end_date='2018-5-10')
index_date = df_sh50.index
#对于不同时期的成分股不一致的情况，经过检查，对对columns进行校验
columns_stock = get_index_stocks('000300.XSHG')
alpha_003_factor = pd.DataFrame(index = index_date,columns = columns_stock)
alpha_003_factor_neu = pd.DataFrame(index = index_date,columns = columns_stock)
for i in index_date:
    print i
    code= list(get_index_stocks('000300.XSHG'))
    temp_a = alpha_003(code,i)
    #temp_a_neu = get_factor_ne(temp_a,i)['factor_ne']
    temp_a = temp_a.dropna()
    temp_a_neu = get_factor_neutra(temp_a,i)
    #print temp_a_neu
    alpha_003_factor.loc[i,:] = temp_a
    #中性化后的因子值
    alpha_003_factor_neu.loc[i,:] = temp_a_neu
alpha_003_factor.to_csv('alpha_003.csv')    
alpha_003_factor_neu.to_csv('alpha_003_neutra.csv')

#中性化（回归拟合后取残差值为因子的最新值）
#传入：mkt_cap：以股票为index，市值为value的Series,
#factor：以股票code为index，因子值为value的Series,
#输出：中性化后的因子值series
def neutralization(factor,mkt_cap = False, industry = True,date=None):
    y = factor
    if type(mkt_cap) == pd.Series:
        LnMktCap = mkt_cap.apply(lambda x:math.log(x))
        if industry: #行业、市值
            dummy_industry = get_industry_exposure(factor.index,date)
            x = pd.concat([LnMktCap,dummy_industry.T],axis = 1)
        else: #仅市值
            x = LnMktCap
    elif industry: #仅行业
        dummy_industry = get_industry_exposure(factor.index)
        x = dummy_industry.T
    result = sm.OLS(y.astype(float),x.astype(float)).fit()
    return result.resid

#为股票池添加行业标记,return df格式 ,为中性化函数的子函数   
def get_industry_exposure(stock_list,date):
    df = pd.DataFrame(index=jqdata.get_industries(name='sw_l1').index, columns=stock_list)
    for stock in stock_list:
        try:
            df[stock][get_industry_code_from_security(stock,date)] = 1
        except:
            continue
    return df.fillna(0)#将NaN赋为0


#查询个股所在行业函数代码（申万一级） ,为中性化函数的子函数    
def get_industry_code_from_security(security,date=None):
    industry_index=jqdata.get_industries(name='sw_l1').index
    for i in range(0,len(industry_index)):
        try:
            index = get_industry_stocks(industry_index[i],date=date).index(security)
            return industry_index[i]
        except:
            continue
    return u'未找到'    

def get_factor_neutra(alpha_factor,date):
    h=get_fundamentals(query(valuation.code,valuation.market_cap)\
        .filter(valuation.code.in_(alpha_factor.index)),date=date)
    stocks_mktcap_se=pd.Series(list(h.market_cap),index=list(h.code))
    stocks_neutra_se=neutralization(alpha_factor,stocks_mktcap_se,date)
    return stocks_neutra_se 

#按行业进行标准化处理的方法
#参数：
#输入：股票名称为index的因子序列，检查日期
#输出： df,包括市值，包括因子值
def get_factor_ne(factor_se,date):
    industries = jqdata.get_industries(name='sw_l1').index
    df_factor = get_fundamentals(query(valuation.code,valuation.market_cap).filter(valuation.code.in_(factor_se.index)),date)
    df_factor.index = df_factor.code
    del df_factor['code']
    df_factor.columns = ['market_cap']
    df_factor['factor'] = factor_se
    industries_series = pd.Series()
    for aa in industries:
        alist = []
        stock_list = list(get_industry_stocks(aa, date=date))
        for bb in stock_list:
            if bb in list(df_factor.index):
                alist.append(bb)
        aseries = pd.Series(aa, index = alist)
#         print aseries
        industries_series = pd.concat([industries_series, aseries])

    duplicated = industries_series.index.duplicated()
    index_list = []
    for x in range(len(industries_series.index)):
        if duplicated[x] ==True:
            index_list.append(list(industries_series.index)[x])
    industries_series.drop(index_list, inplace = True)
    df_factor['industries_detail'] = industries_series

    df_factor.dropna(how = 'any', inplace = True) # 1173/1018 2016/12/31
    df_factor['factor_mean'] = NaN
    df_factor['factor_std'] = NaN

    for x in range(len(industries)):
        # 生成均值数据
        df_factor['factor_mean'][df_factor['industries_detail'] == industries[x]] \
            = df_factor['factor'][df_factor['industries_detail'] == industries[x]].mean()
        # 生成方差数据
        df_factor['factor_std'][df_factor['industries_detail'] == industries[x]] \
            = df_factor['factor'][df_factor['industries_detail'] == industries[x]].std()
        # 标准化
        df_factor['factor_ne'] = (df_factor['factor'] - df_factor['factor_mean'])/df_factor['factor_std']
        
    #df_factor.dropna(how = 'any', inplace = True)
    return df_factor    

#进行收益统计构造
#参数：
#输入：开始时间，结束时间，股票池，计算天数
pool = get_index_stocks('000300.XSHG')
start_d='2016-5-9'
end_d='2018-5-10'
days = 1
def get_pool_pct(pool,start_date,end_date,days=1):
    df_sh50 = get_price(pool,start_date=start_d,end_date=end_d,fields=['close'])['close']
    pct_close = (df_sh50[days:].values-df_sh50[:-days].values)/df_sh50[:-days].values
    df_temp = df_sh50#.ix[1:,:]
    df_temp.iloc[days:,:]=pct_close
    df_temp.iloc[0:days,:] = np.nan
    return df_temp
factor_return = get_pool_pct(pool=pool,start_date=start_d,end_date=end_d,days=days)

#计算因子IC值，P值
#输入：因子序列和收益序列
#返回IC和IR
#注意事项：输入的是df格式内容，函数会剔除空值
def get_rank_ic(factor, factor_return,days = 1):
    ic_df = pd.DataFrame(index=factor.index, columns=['IC','pValue'])
    # 计算相关系数
    date_index = factor.index
    for i in range(len(date_index)-days):
        tmp_factor = factor.ix[date_index[i]]
        tmp_ret = factor_return.ix[date_index[i+days]]
        cor = pd.DataFrame(tmp_factor)
        ret = pd.DataFrame(tmp_ret)
        cor.columns = ['corr']
        ret.columns = ['ret']
        cor['ret'] = ret['ret']
        #去除空值
        cor = cor[~pd.isnull(cor['corr'])][~pd.isnull(cor['ret'])]

        ic, p_value = st.spearmanr(cor['corr'], cor['ret'])   # 计算秩相关系数RankIC
        ic_df['IC'][i] = ic
        ic_df['pValue'][i] = p_value
    return ic_df

#因子数据获取比较耗时，这里直接读取的已经存在本地的CSV文件
alpha_001_factor = pd.read_csv('alpha_001.csv')
alpha_001_factor_neu = pd.read_csv('alpha_001_neu.csv')

alpha_001_factor.index = alpha_001_factor.iloc[:,0] 
alpha_001_factor_neu.index = alpha_001_factor_neu.iloc[:,0]

#用于计算不同周期的信息系数
factor_ic = {}
factor_neu_ic = {}
for i in range(5):
    ic_alpha_day=get_rank_ic(alpha_001_factor,factor_return,days=i+1)
    ic_alpha_neu_day=get_rank_ic(alpha_001_factor_neu,factor_return,days=i+1)
    factor_ic[str(i+1)+'day']=ic_alpha_day
    factor_neu_ic[str(i+1)+'day']=ic_alpha_neu_day
factor_ic

alpha_report = pd.DataFrame(index=['1DAY', '2DAY', '3DAY','4DAY','5DAY'], 
                      columns=[['alpha_001因子','alpha_001因子','alpha_001因子', '行业和市值中性化后因子', '行业和市值中性化后因子','行业和市值中性化后因子'], 
                               ['IC', 'IC_IR', 'pvalue', 'IC', 'IC_IR', 'pvalue']])
for i in range(1,6):
    l = [factor_ic[str(i)+'day']['IC'].mean(),factor_ic[str(i)+'day']['IC'].mean()/factor_ic[str(i)+'day']['IC'].std(),factor_ic[str(i)+'day']['pValue'].mean()]
    l_neu = [factor_neu_ic[str(i)+'day']['IC'].mean(),factor_neu_ic[str(i)+'day']['IC'].mean()/factor_neu_ic[str(i)+'day']['IC'].std(),factor_neu_ic[str(i)+'day']['pValue'].mean()]
    alpha_report.loc[str(i)+'DAY',:] = l+l_neu
alpha_report

alpha_report['alpha_001因子']['IC'].plot(kind='bar')

alpha_report['行业和市值中性化后因子']['IC'].plot(kind='bar')

factor_ic['1day']['IC'].hist(),factor_neu_ic['1day']['IC'].hist()

factor_ic['1day']['IC'].plot(),factor_neu_ic['1day']['IC'].plot()

factor_ic['2day']['IC'].hist(),factor_neu_ic['2day']['IC'].hist()

factor_ic['3day']['IC'].hist(),factor_neu_ic['3day']['IC'].hist()

factor_ic['4day']['IC'].hist(),factor_neu_ic['4day']['IC'].hist()

factor_ic['5day']['IC'].hist(),factor_neu_ic['5day']['IC'].hist()

#按因子值进行收益分组统计
#定义一个大函数
import pandas as pd
import numpy as np
#import matplotlib.pyplot as plt
#plt.style.use('ggplot')

def get_fact_ret(fact,pool_index='000300.XSHG',fdate='2017-01-01',e_date='2017-02-01'):
    
    def get_df(pool,fdate):
        q = query(
        valuation.code,
        valuation.circulating_market_cap,
        valuation.pe_ratio,
        valuation.pe_ratio/indicator.inc_net_profit_year_on_year,
        valuation.pb_ratio
        ).filter(
        valuation.code.in_(pool))
        df = get_fundamentals(q, fdate)
        df.columns=['code','circulating_market_cap','pe_ratio','peg','pb_ratio']
        #print(df)
        return df
    
    def ret_se(sl,start_date=fdate,end_date=e_date):
        #得到股票的历史价格数据
        df = get_price(sl,start_date=start_date,end_date=end_date,fields=['close']).close
        df = df.dropna(axis=1)
        #print(df.head(3))
        #相当于昨天的百分比变化
        pct = df.pct_change()+1
        pct.ix[0,:] = 1
        #pct = pct.dropna(axis=0)
        #累积相乘得到总收益的变化情况
        #df1 = pct.cumprod()
        #等权重平均收益结果
        se1 = pct.cumsum(axis=1).ix[:,-1]/pct.shape[1]
        return se1
    '''
    #获取因子值，进行排序
    pool = get_index_stocks(pool_index)
    df = get_df(pool,fdate=fdate)
    df.index = df.code
    '''
    df_d1 = fact.loc[fdate:fdate,:].T.sort_values(fdate,ascending=0)[1:]
    #print(df_d1.head(3))
    
    #step2 分组
    part = list(df_d1.index)
    a = len(df_d1)//5
    part1 = list(df_d1.index)[:a]
    part2 = list(df_d1.index)[a:2*a]
    part3 = list(df_d1.index)[2*a:-2*a]
    part4 = list(df_d1.index)[-2*a:-a]
    part5 = list(df_d1.index)[-a:]
    
    #step3 计算收益
    se  = ret_se(part)
    se1 = ret_se(part1)
    se2 = ret_se(part2)
    se3 = ret_se(part3)
    se4 = ret_se(part4)
    se5 = ret_se(part5)
    
    return se,se1,se2,se3,se4,se5

#数据组拼接
def fact_tu(fact,l_all):
    l1 = l_all[:-1]
    l2 = l_all[1:]
    date_list = list(zip(l1,l2))

    num = 1
    for i,j in date_list:
        se,se1,se2,se3,se4,se5 = get_fact_ret(fact,fdate=i,e_date=j)
        if num == 1:
            #se_list = [ for i in se_list]
            df_all_1 = pd.concat([se,se1,se2,se3,se4,se5],axis=1)
            df_all_1.columns = ['benchmark','part1','part2','part3','part4','part5']
            #df_all_1.ix[0,:] = 1
            num = 0
            #print(df_all_1)
        else:
            df_all_2 = pd.concat([se,se1,se2,se3,se4,se5],axis=1)
            df_all_2.columns = ['benchmark','part1','part2','part3','part4','part5']
            #df_all_2.ix[0,:] = 1
            #将各个月份的收益拼接为大表
            df_all_1 = pd.concat([df_all_1,df_all_2])
    #print(df_all_1)
    
    return df_all_1.cumprod()
#获取时间段内的月初首个交易日日期
#获取每个月的第一个交易日日期
'''
def date_list(start,end):
    df_date = pd.read_csv('df_date.csv')
    df_1=df_date[(df_date['dt_list']>=start) & (df_date['dt_list']<=end) ]
    df_2 = df_1.drop_duplicates(['year','month'])
    l_3 = [i for i in df_2.dt_list.values]
    return l_3
'''
#每天进行调仓
def date_list(start,end):
    l1 = get_price('000001.XSHG',start_date=start,end_date=end)
    l2 = [str(i)[:10] for i in l1.index]
    return l2

#输入起止日期
l_date=date_list('2017-05-10','2018-05-10')
#输入目标因子进行分组测试
df = fact_tu(alpha_001_factor,l_date)
#对分组收益进行制图
df.plot(figsize=(15,10))

#输入起止日期
l_date=date_list('2017-05-10','2018-05-10')
#输入目标因子进行分组测试
df = fact_tu(alpha_001_factor_neu,l_date)
#对分组收益进行制图
df.plot(figsize=(15,10))