import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from jqdata import jy
pd.set_option('precision', 3)
import datetime
import jqdata
from jqdata import *
import matplotlib.pyplot as plt
plt.style.use('ggplot')

#统计各指数当天涨跌
#输入日期
#返回df
class report():
    #1.指数涨跌
    def get_index_pct(self,date):
        #指数涨跌幅统计
        #总数据
        zhishu_list = ['000001.XSHG','000300.XSHG','000016.XSHG','000905.XSHG','000852.XSHG','399006.XSHE','399678.XSHE','399005.XSHE'] 
        pl = get_price(zhishu_list,end_date=date,count=6)#.ix[:,0,:]

        #涨跌幅数据
        pl_pct = (pl.iloc[:,-1,:]/pl.iloc[:,-2,:]-1)*100
        pl_pct_5 = (pl.iloc[:,-1,:]/pl.iloc[:,-6,:]-1)*100

        #5日money均量
        df_money = pl.loc['money',:,:]
        def f(x):
            return x.mean()
        se_money = df_money.apply(f)

        #5日volume均量
        df_volume = pl.loc['volume',:,:]
        def f(x):
            return x.mean()
        se_volume = df_volume.apply(f)

        #最新一天指数值数据
        zv = pl.iloc[:,-1,:]
        zv['name'] = ['上证','沪深300','上证50','中证500','中证1000','创业板','次新股','中小板']
        df_fin = pd.concat([zv['close'],pl_pct['close'],pl_pct_5['close'],pl_pct['volume'],zv['volume']/(10**8),se_volume/(10**8),pl_pct['money'],zv['money']/(10**8),se_money/(10**8)],axis=1)
        df_fin.index = zv['name'].values
        df_fin.columns = ['指数值','涨跌幅%','5日涨跌幅%','成交量%','成交量（亿）','5日成交均量(亿)','成交额%','成交额（亿）','5日成交均额（亿）']

        return df_fin
    
    #2.行业涨跌幅统计
    def get_hy_pct(self,y=2018,m=7,d=6,days = 5):

        #行业涨跌幅
        def get_SW_index(SW_index = 801010,start_date = '2017-01-31',end_date = '2018-01-31'):
            index_list = ['PrevClosePrice','OpenPrice','HighPrice','LowPrice','ClosePrice','TurnoverVolume','TurnoverValue','TurnoverDeals','ChangePCT','UpdateTime']
            jydf = jy.run_query(query(jy.SecuMain).filter(jy.SecuMain.SecuCode==str(SW_index)))
            link=jydf[jydf.SecuCode==str(SW_index)]
            rows=jydf[jydf.SecuCode==str(SW_index)].index.tolist()
            result=link['InnerCode'][rows]

            df = jy.run_query(query(jy.QT_SYWGIndexQuote).filter(jy.QT_SYWGIndexQuote.InnerCode==str(result[0]),\
                                                           jy.QT_SYWGIndexQuote.TradingDay>=start_date,\
                                                                 jy.QT_SYWGIndexQuote.TradingDay<=end_date
                                                                ))
            df.index = df['TradingDay']
            df = df[index_list]
            return df

        def ShiftTradingDay(date,shift=5):
            # 获取所有的交易日，返回一个包含所有交易日的 list,元素值为 datetime.date 类型.
            tradingday = get_all_trade_days()
            # 得到date之后shift天那一天在列表中的行标号 返回一个数
            shiftday_index = list(tradingday).index(date)+shift
            # 根据行号返回该日日期 为datetime.date类型
            return tradingday[shiftday_index]  

        date = datetime.date(y,m,d)
        date_s = ShiftTradingDay(date,-days)

        #指数涨跌幅统计
        #总数据
        sw_hy = jqdata.get_industries(name='sw_l1')
        sw_hy_dict  = {}
        for i in sw_hy.index:
            value = get_SW_index(i,start_date=date_s,end_date=date)
            sw_hy_dict[i] = value
        pl_hy = pd.Panel(sw_hy_dict)
        pl = pl_hy.transpose(2,1,0)
        pl = pl.loc[['PrevClosePrice','OpenPrice','HighPrice','LowPrice','ClosePrice','TurnoverVolume','TurnoverValue'],:,:]
        #涨跌幅数据
        pl_pct = (pl.iloc[:,-1,:]/pl.iloc[:,-2,:]-1)*100
        pl_pct_5 = (pl.iloc[:,-1,:]/pl.iloc[:,-6,:]-1)*100

        #5日money均量
        df_money = pl.loc['TurnoverValue',:,:]
        def f(x):
            return x.mean()
        se_money = df_money.apply(f)

        #5日volume均量
        df_volume = pl.loc['TurnoverVolume',:,:]
        def f(x):
            return x.mean()
        se_volume = df_volume.apply(f)

        #最新一天指数值数据
        zv = pl.iloc[:,-1,:]
        zv['name'] = sw_hy['name']
        df_fin = pd.concat([zv['ClosePrice'],pl_pct['ClosePrice'],pl_pct_5['ClosePrice'],pl_pct['TurnoverVolume'],zv['TurnoverVolume']/(10**8),se_volume/(10**8),pl_pct['TurnoverValue'],zv['TurnoverValue']/(10**8),se_money/(10**8)],axis=1)
        df_fin.columns = ['指数值','涨跌幅%','5日涨跌幅%','成交量%','成交量（亿）','5日成交均量(亿)','成交额%','成交额（亿）','5日成交均额（亿）']
        df_fin['name'] = sw_hy['name']
        df_fin['code'] = df_fin.index
        df_fin.index = df_fin['name'].values
        del df_fin['name'] 

        return df_fin.sort_values(by = '涨跌幅%',ascending=0) 

    #指数贡献点数
    def get_index_contribution(self,date='2018-6-30',index='000001.XSHG'):
        def get_pct(date,pool):
            #指数涨跌幅统计
            #总数据
            #zhishu_list = ['000001.XSHG','000300.XSHG','000016.XSHG','000905.XSHG','000852.XSHG','399006.XSHE','399678.XSHE','399005.XSHE'] 
            zhishu_list = pool#[:8]
            pl = get_price(zhishu_list,end_date=date,count=6)#.ix[:,0,:]

            #涨跌幅数据
            pl_pct = (pl.iloc[:,-1,:]/pl.iloc[:,-2,:]-1)*100
            pl_pct_5 = (pl.iloc[:,-1,:]/pl.iloc[:,-6,:]-1)*100

            #5日money均量
            df_money = pl.loc['money',:,:]
            def f(x):
                return x.mean()
            se_money = df_money.apply(f)

            #5日volume均量
            df_volume = pl.loc['volume',:,:]
            def f(x):
                return x.mean()
            se_volume = df_volume.apply(f)

            #最新一天指数值数据
            zv = pl.iloc[:,-1,:]
            #zv['name'] = ['上证','沪深300','上证50','中证500','中证1000','创业板','次新股','中小板']
            zv['name'] = zhishu_list
            df_fin = pd.concat([zv['close'],pl_pct['close'],pl_pct_5['close'],pl_pct['volume'],zv['volume']/(10**8),se_volume/(10**8),pl_pct['money'],zv['money']/(10**8),se_money/(10**8)],axis=1)
            df_fin.index = zv['name'].values
            df_fin.columns = ['当前价格','涨跌幅%','5日涨跌幅%','成交量%','成交量（亿）','5日成交均量(亿)','成交额%','成交额（亿）','5日成交均额（亿）']

            return df_fin

        #获取大盘成分股权重
        def get_weight():
            q = query(jy.LC_IndexComponentsWeight).filter(jy.LC_IndexComponentsWeight.UpdateTime>'2018-6-1',\
                                                         jy.LC_IndexComponentsWeight.IndexCode == '1')
            df = jy.run_query(q)
            df_weight = df[['InnerCode','Weight']]
            #获取证券主表
            df_code_main = jy.run_query(query(jy.SecuMain).filter(jy.SecuMain.InnerCode.in_(list(df_weight['InnerCode'].values))))
            df_code = df_code_main[['InnerCode','SecuCode']]
            #print(df_code)
            #进行拼接，得到股票的权重数据
            df_weight_tab = pd.merge(df_code,df_weight,on='InnerCode')
            #转换成需要的代码格式即可
            df_weight_tab['code'] = [normalize_code(i) for i in df_weight_tab['SecuCode'].values]
            df_weight_tab.index = df_weight_tab.code
            weight_se = df_weight_tab['Weight']/100.0
            return weight_se

        pool = get_index_stocks(index)
        df_weight_main = get_pct(date,pool)
        weight_se = get_weight()

        #5天前大盘的值
        point = get_price('000001.XSHG',end_date=date,count=5)['close'][0]
        #5天贡献点数计算
        df_weight_main['weight_se'] = weight_se
        df_weight_main['point'] = df_weight_main['weight_se']*df_weight_main['5日涨跌幅%']*point/100.0
        df_index_weight = df_weight_main.sort_values(by='point',ascending=0)
        df_index_weight = df_index_weight[['5日涨跌幅%','point']]
        f_pool = df_index_weight[:10]
        b_pool = df_index_weight.dropna()[-10:]
        return f_pool,b_pool

a = report()
index_pct_df = a.get_index_pct('2018-7-6')
hy_pct_df = a.get_hy_pct(2018,7,6)
index_crb_df = a.get_index_contribution('2018-7-6')

index_pct_df = index_pct_df.sort_values(by='涨跌幅%',ascending=0)
index_pct_df

def get_name(code,Type='stock'):
    all_data = get_all_securities(types=Type)
    i = list(all_data.index).index(code)
    return all_data['display_name'][i]
name_list0 = [get_name(i) for i in index_crb_df[0].index]
name_list1 = [get_name(i) for i in index_crb_df[1].index]
index_crb_df[0]['股票名称'] = name_list0
index_crb_df[1]['股票名称'] = name_list1
index_crb_df[0]

index_crb_df[1]

#图表展示
#指数单日和5日涨跌幅
index_pct_df = index_pct_df.sort_values(by='5日涨跌幅%',ascending=0)
index_pct_df[['涨跌幅%','5日涨跌幅%']].plot(kind='bar')

#图表展示
#指数单日和5日涨跌幅
index_pct_df = index_pct_df.sort_values(by='涨跌幅%',ascending=0)
index_pct_df[['涨跌幅%','5日涨跌幅%']].plot(kind='bar')

index_pct_df = index_pct_df.sort_values(by='成交量%',ascending=0)
index_pct_df[['涨跌幅%','成交量%','成交额%']].plot(kind='bar')

hy_pct_df = hy_pct_df.sort_values(by = '涨跌幅%',ascending=0)
hy_pct_df[['涨跌幅%','5日涨跌幅%']].plot(kind='bar',figsize=(12,8))

hy_pct_df = hy_pct_df.sort_values(by = '5日涨跌幅%',ascending=0)
hy_pct_df[['涨跌幅%','5日涨跌幅%']].plot(kind='bar',figsize=(12,8))

'''
因子效果统计
'''
#指定时间段的收益拼接(多空组合)
def get_fact_ret(fact,pool_index='000300.XSHG',fdate='2017-01-01',e_date='2017-02-01'):
    
    #获取因子值（主要是财务因子值）
    def get_df(pool,fdate):
        q = query(
        valuation.code,
        valuation.circulating_market_cap,
        valuation.pe_ratio,
        valuation.ps_ratio, # PS 市销率
        indicator.eps,
        valuation.pe_ratio/indicator.inc_net_profit_year_on_year,
        valuation.pb_ratio,
        indicator.roa, # ROA
        indicator.roe, # ROE
        indicator.inc_revenue_year_on_year
        ).filter(
            
        valuation.code.in_(pool))
        df = get_fundamentals(q, fdate)
        df.columns=['code','circulating_market_cap','pe_ratio','ps_ratio','eps','peg','pb_ratio','roa','roe','inc_revenue_year_on_year']
        df = df[df['pe_ratio']>0]
        df = df[df['ps_ratio']>0]
        df = df[df['eps']>0]
        df = df[df['peg']>0]
        df = df[df['pb_ratio']>0]
        df = df[df['roa']>0]
        df = df[df['roe']>0]
        df = df[df['inc_revenue_year_on_year']>0]
        #调整排序
        df['eps'] = 1/df['eps']
        df['roa'] = 1/df['roa']
        df['roe'] = 1/df['roe']
        df['inc_revenue_year_on_year'] = 1/df['inc_revenue_year_on_year']
        #print(df)
        return df
    
    #获取股票池的组合收益
    def ret_se_1(sl,start_date=fdate,end_date=e_date):
        #得到股票的历史价格数据
        df = get_price(sl,start_date=start_date,end_date=end_date,fields=['close']).close
        df = df.dropna(axis=1)
        #print(df.head(3))
        #相当于昨天的百分比变化
        pct = df.pct_change()+1
        pct.iloc[0,:] = 1
        #pct = pct.dropna(axis=0)
        #累积相乘得到总收益的变化情况
        #df1 = pct.cumprod()
        #等权重平均收益结果
        se1 = pct.cumsum(axis=1).iloc[:,-1]/pct.shape[1]
        return se1
    
    #获取股票池的组合收益(市值加权的方式)
    def ret_se(sl,start_date=fdate,end_date=e_date):
        #得到股票的历史价格数据
        df = get_price(sl,start_date=start_date,end_date=end_date,fields=['close']).close
        df = df.dropna(axis=1)

        #获取列表中的股票流通市值对数值
        df_mkt = get_fundamentals(query(valuation.code,valuation.circulating_market_cap).filter(valuation.code.in_(df.columns)))
        df_mkt.index = df_mkt['code'].values
        fact_se =pd.Series(df_mkt['circulating_market_cap'].values,index = df_mkt['code'].values)
        fact_se = np.log(fact_se)
        #print(fact_se)

        #相当于昨天的百分比变化
        pct = df.pct_change()+1
        pct.iloc[0,:] = 1
        #df1 = pct.cumprod()
        #等权重平均收益结果
        #按权重的方式计算
        se1 = (pct*fact_se).cumsum(axis=1).iloc[:,-1]/sum(fact_se)
        return se1
    
    #获取因子值，进行排序
    pool = get_index_stocks(pool_index)
    df = get_df(pool,fdate=fdate)
    df.index = df['code'].values
    #将因子值进行排序，从小到大
    #print(df)
    df_d1 = df.sort_values(by=fact,axis=0)
    #print(df_d1.head(3))
    
    #step2 分组
    part = list(df_d1.index)
    a = len(df_d1)//5
    part1 = list(df_d1.index)[:a]
    part2 = list(df_d1.index)[a:2*a]
    part3 = list(df_d1.index)[2*a:-2*a]
    part4 = list(df_d1.index)[-2*a:-a]
    part5 = list(df_d1.index)[-a:]
    
    #step3 计算收益
    se  = ret_se(part)
    se1 = ret_se(part1)
    se2 = ret_se(part2)
    se3 = ret_se(part3)
    se4 = ret_se(part4)
    se5 = ret_se(part5)
    
    return se,se1,se2,se3,se4,se5

#数据组拼接
def fact_return_df(fact,l_all):
    l1 = l_all[:-1]
    l2 = l_all[1:]
    date_list = list(zip(l1,l2))

    num = 1
    for i,j in date_list:
        se,se1,se2,se3,se4,se5 = get_fact_ret(fact,fdate=i,e_date=j)
        if num == 1:
            #se_list = [ for i in se_list]
            df_all_1 = pd.concat([se,se1,se2,se3,se4,se5],axis=1)
            df_all_1.columns = ['benchmark','part1','part2','part3','part4','part5']
            #df_all_1.ix[0,:] = 1
            num = 0
            #print(df_all_1)
        else:
            df_all_2 = pd.concat([se,se1,se2,se3,se4,se5],axis=1)
            df_all_2.columns = ['benchmark','part1','part2','part3','part4','part5']
            #df_all_2.ix[0,:] = 1
            #将各个月份的收益拼接为大表
            df_all_1 = pd.concat([df_all_1,df_all_2])
    #print(df_all_1)
    df_all_1 = df_all_1.drop_duplicates()
    return df_all_1.cumprod()
#获取时间段内的月初首个交易日日期
#获取每个月的第一个交易日日期
'''
def date_list(start,end):
    df_date = pd.read_csv('df_date.csv')
    df_1=df_date[(df_date['dt_list']>=start) & (df_date['dt_list']<=end) ]
    df_2 = df_1.drop_duplicates(['year','month'])
    l_3 = [i for i in df_2.dt_list.values]
    return l_3
'''
#每天进行组合调整
def date_list(start,end):
    l = get_price('000001.XSHG',start_date=start,end_date=end)
    return list(l.index)
'''
多空组合收益部分内容（折线图）
'''
#输入起止日期
l_date=date_list('2017-7-6','2018-7-6')
#输入目标因子进行分组测试
fact_list = ['circulating_market_cap','pe_ratio','ps_ratio','eps','peg','pb_ratio','roa','roe','inc_revenue_year_on_year']
fact_se = []

for fact in fact_list:
    df = fact_return_df(fact,l_date)
    print('准备计算 %s 因子累计收益'%fact)
    #做多第一组，做空第五组（从小到大排序）
    se_long_part1 = df['part1']
    se_short_part5 = 1+(1-df['part5'])
    se = se_short_part5*0.5+se_long_part1*0.5
    #se = df['part5']-df['part1']
    #print(se)
    fact_se.append(se)
fact_df = pd.concat(fact_se,axis=1)
name_list = ['市值','市盈率','市销率','1/每股收益','peg','市净率','1/roa','1/roe','1/营收增长']
fact_df.columns = name_list
fact_df

fact_df.plot(figsize=(15,10))

#计算多空因子组合收益
end_date = '2018-7-6'
date_week = date_list('2018-6-29',end_date)
date_mouth =  date_list('2018-6-6',end_date)
date_halfyear =  date_list('2018-1-6',end_date)
date_year =  date_list('2017-7-6',end_date)
date_3year =  date_list('2015-7-6',end_date)
#输入目标因子进行分组测试
fact_list = ['circulating_market_cap','pe_ratio','ps_ratio','eps','peg','pb_ratio','roa','roe','inc_revenue_year_on_year']
fact_fin = []
for i in [date_week,date_mouth,date_halfyear,date_year,date_3year]:
    fact_se = []
    for fact in fact_list:
        df = fact_return_df(fact,i)
        #print(df)
        se_long_part1 = df['part1']
        se_short_part5 = 1+(1-df['part5'])
        se = se_short_part5*0.5+se_long_part1*0.5
        #se = df['part5']-df['part1']
        fact_se.append(se)
    fact_df = pd.concat(fact_se,axis=1)
    name_list = ['市值','市盈率','市销率','1/每股收益','peg','市净率','1/roa','1/roe','1/营收增长']
    fact_df.columns = name_list
    print('当前时间周期因子计算完毕')
    fact_fin.append(fact_df)
    
df_tab = pd.DataFrame(index =['市值','市盈率','市销率','1/每股收益','peg','市净率','1/roa','1/roe','1/营收增长'],columns=['近一周','近一个月','近半年','近一年','近三年'])
    
df_tab['近一周'] = 100*(fact_fin[0].iloc[-1,:]-1)
df_tab['近一个月'] = 100*(fact_fin[1].iloc[-1,:]-1)
df_tab['近半年'] = 100*(fact_fin[2].iloc[-1,:]-1)
df_tab['近一年'] = 100*(fact_fin[3].iloc[-1,:]-1)
df_tab['近三年'] = 100*(fact_fin[4].iloc[-1,:]-1)
df_tab

df_tab.sort_values(by='近半年',ascending=0)

df_tab = df_tab.sort_values(by='近一周',ascending=0)
df_tab['近一周'].plot(kind='bar')

df_tab = df_tab.sort_values(by='近一个月',ascending=0)
df_tab['近一个月'].plot(kind='bar')

df_tab = df_tab.sort_values(by='近半年',ascending=0)
df_tab['近半年'].plot(kind='bar')

df_tab = df_tab.sort_values(by='近一年',ascending=0)
df_tab['近一年'].plot(kind='bar')

df_tab = df_tab.sort_values(by='近三年',ascending=0)
df_tab['近三年'].plot(kind='bar')