#工具函数
import time
from datetime import datetime, timedelta
import jqdata
import numpy as np
import pandas as pd
import math
from statsmodels import regression
import statsmodels.api as sm
import matplotlib.pyplot as plt
from jqfactor import get_factor_values
import datetime
from jqlib.technical_analysis import *
from scipy import stats

#设置画图样式
plt.style.use('ggplot')

#输入起止日期，返回所有自然日日期
def get_date_list(begin_date, end_date):
    dates = []
    dt = datetime.strptime(begin_date,"%Y-%m-%d")
    date = begin_date[:]
    while date <= end_date:
        dates.append(date)
        dt += timedelta(days=1)
        date = dt.strftime("%Y-%m-%d")
    return dates

#去极值函数
#mad中位数去极值法
def filter_extreme_MAD(series,n): #MAD: 中位数去极值 
    median = series.quantile(0.5)
    new_median = ((series - median).abs()).quantile(0.50)
    max_range = median + n*new_median
    min_range = median - n*new_median
    return np.clip(series,min_range,max_range)

#进行标准化处理
def winsorize(factor, std=3, have_negative = True):
    '''
    去极值函数 
    factor:以股票code为index，因子值为value的Series
    std为几倍的标准差，have_negative 为布尔值，是否包括负值
    输出Series
    '''
    r=factor.dropna().copy()
    if have_negative == False:
        r = r[r>=0]
    else:
        pass
    #取极值
    edge_up = r.mean()+std*r.std()
    edge_low = r.mean()-std*r.std()
    r[r>edge_up] = edge_up
    r[r<edge_low] = edge_low
    return r

#标准化函数：
def standardize(s,ty=2):
    '''
    s为Series数据
    ty为标准化类型:1 MinMax,2 Standard,3 maxabs 
    '''
    data=s.dropna().copy()
    if int(ty)==1:
        re = (data - data.min())/(data.max() - data.min())
    elif ty==2:
        re = (data - data.mean())/data.std()
    elif ty==3:
        re = data/10**np.ceil(np.log10(data.abs().max()))
    return re
    

#中性化函数
#传入：mkt_cap：以股票为index，市值为value的Series,
#factor：以股票code为index，因子值为value的Series,
#输出：中性化后的因子值series
def neutralization(factor,mkt_cap = False, industry = True):
    y = factor
    if type(mkt_cap) == pd.Series:
        LnMktCap = mkt_cap.apply(lambda x:math.log(x))
        if industry: #行业、市值
            dummy_industry = get_industry_exposure(factor.index)
            x = pd.concat([LnMktCap,dummy_industry.T],axis = 1)
        else: #仅市值
            x = LnMktCap
    elif industry: #仅行业
        dummy_industry = get_industry_exposure(factor.index)
        x = dummy_industry.T
    result = sm.OLS(y.astype(float),x.astype(float)).fit()
    return result.resid

#为股票池添加行业标记,return df格式 ,为中性化函数的子函数   
def get_industry_exposure(stock_list):
    df = pd.DataFrame(index=jqdata.get_industries(name='sw_l1').index, columns=stock_list)
    for stock in stock_list:
        try:
            df[stock][get_industry_code_from_security(stock)] = 1
        except:
            continue
    return df.fillna(0)#将NaN赋为0


#查询个股所在行业函数代码（申万一级） ,为中性化函数的子函数    
def get_industry_code_from_security(security,date=None):
    industry_index=jqdata.get_industries(name='sw_l1').index
    for i in range(0,len(industry_index)):
        try:
            index = get_industry_stocks(industry_index[i],date=date).index(security)
            return industry_index[i]
        except:
            continue
    return u'未找到'    

def get_win_stand_neutra(stocks):
    h=get_fundamentals(query(valuation.pb_ratio,valuation.code,valuation.market_cap)\
        .filter(valuation.code.in_(stocks)))
    stocks_pb_se=pd.Series(list(h.pb_ratio),index=list(h.code))
    stocks_pb_win_standse=standardize(winsorize(stocks_pb_se))
    stocks_mktcap_se=pd.Series(list(h.market_cap),index=list(h.code))
    stocks_neutra_se=neutralization(stocks_pb_win_standse,stocks_mktcap_se)
    return stocks_neutra_se 

#获取日期列表
def get_tradeday_list(start,end,frequency=None,count=None):
    if count != None:
        df = get_price('000001.XSHG',end_date=end,count=count)
    else:
        df = get_price('000001.XSHG',start_date=start,end_date=end)
    if frequency == None or frequency =='day':
        return df.index
    else:
        df['year-month'] = [str(i)[0:7] for i in df.index]
        if frequency == 'month':
            return df.drop_duplicates('year-month').index
        elif frequency == 'quarter':
            df['month'] = [str(i)[5:7] for i in df.index]
            df = df[(df['month']=='01') | (df['month']=='04') | (df['month']=='07') | (df['month']=='10') ]
            return df.drop_duplicates('year-month').index
        elif frequency =='halfyear':
            df['month'] = [str(i)[5:7] for i in df.index]
            df = df[(df['month']=='01') | (df['month']=='06')]
            return df.drop_duplicates('year-month').index 
        
def ret_se(start_date='2018-6-1',end_date='2018-7-1',stock_pool=None,weight=0):
    pool = stock_pool
    if len(pool) != 0:
        #得到股票的历史价格数据
        df = get_price(list(pool),start_date=start_date,end_date=end_date,fields=['close']).close
        df = df.dropna(axis=1)
        #获取列表中的股票流通市值对数值
        df_mkt = get_fundamentals(query(valuation.code,valuation.circulating_market_cap).filter(valuation.code.in_(df.columns)))
        df_mkt.index = df_mkt['code'].values
        fact_se =pd.Series(df_mkt['circulating_market_cap'].values,index = df_mkt['code'].values)
        fact_se = np.log(fact_se)
    else:
        df = get_price('000001.XSHG',start_date=start_date,end_date=end_date,fields=['close'])
        df['v'] = [1]*len(df)
        del df['close']
    #相当于昨天的百分比变化
    pct = df.pct_change()+1
    pct.iloc[0,:] = 1
    if weight == 0:
        #等权重平均收益结果
        se = pct.cumsum(axis=1).iloc[:,-1]/pct.shape[1]
        return se
    else:
        #按权重的方式计算
        se = (pct*fact_se).cumsum(axis=1).iloc[:,-1]/sum(fact_se)
        return se
    
#获取所有分组pct
def get_all_pct(pool_dict,trade_list,groups=5):
    num = 1
    for s,e in zip(trade_list[:-1],trade_list[1:]):
        stock_list = pool_dict[s]
        stock_num = len(stock_list)//groups
        if num == 0:
            pct_se_list = []
            for i in range(groups):
                pct_se_list.append(ret_se(start_date=s,end_date=e,stock_pool=stock_list[i*stock_num:(i+1)*stock_num]))
            pct_df1 = pd.concat(pct_se_list,axis=1)
            pct_df = pd.concat([pct_df,pct_df1],axis=0)
        else:
            pct_se_list = []
            for i in range(groups):
                pct_se_list.append(ret_se(start_date=s,end_date=e,stock_pool=stock_list[i*stock_num:(i+1)*stock_num]))
            pct_df = pd.concat(pct_se_list,axis=1)    
            num = 0
    return pct_df

def tradedays_before(date,count):#获取指定交易日往前推count天交易日
    date = get_price('000001.XSHG',end_date=date,count=count+1).index[0]
    return date

#获取时间为date的全部因子数据
def get_factor_data(stock,date):
    data=pd.DataFrame(index=stock)
    q = query(valuation,balance,cash_flow,income,indicator).filter(valuation.code.in_(stock))
    df = get_fundamentals(q, date)
    df['market_cap']=df['market_cap']*100000000
    factor_data=get_factor_values(stock,['roe_ttm','roa_ttm','total_asset_turnover_rate',\
                               'net_operate_cash_flow_ttm','net_profit_ttm','net_profit_ratio',\
                              'cash_to_current_liability','current_ratio',\
                             'gross_income_ratio','non_recurring_gain_loss',\
                            'operating_revenue_ttm','net_profit_growth_rate'],end_date=date,count=1)
    factor=pd.DataFrame(index=stock)
    for i in factor_data.keys():
        factor[i]=factor_data[i].iloc[0,:]
    df.index = df['code']
    data['code'] = df['code']
    del df['code'],df['id']
    #合并得大表
    df=pd.concat([df,factor],axis=1)
    #总市值取对数
    data['size_lg']=np.log(df['market_cap'])
    #净利润(TTM)/总市值
    data['EP']=df['net_profit_ttm']/df['market_cap']
    #净资产/总市值
    data['BP']=1/df['pb_ratio']
    #营业收入(TTM)/总市值
    data['SP']=1/df['ps_ratio']
    #净现金流(TTM)/总市值
    data['NCFP']=1/df['pcf_ratio']
    #经营性现金流(TTM)/总市值
    data['OCFP']=df['net_operate_cash_flow_ttm']/df['market_cap']
    #净利润同比增长率
    data['net_g'] = df['net_profit_growth_rate']
    #净利润(TTM)同比增长率/PE_TTM
    data['G/PE']=df['net_profit_growth_rate']/df['pe_ratio']
    #ROE_ttm
    data['roe_ttm']=df['roe_ttm']
    #ROE_YTD
    data['roe_q']=df['roe']
    #ROA_ttm
    data['roa_ttm']=df['roa_ttm']
    #ROA_YTD
    data['roa_q']=df['roa']
    #净利率
    data['netprofitratio_ttm'] = df['net_profit_ratio']
    #毛利率TTM
    data['grossprofitmargin_ttm']=df['gross_income_ratio']
    #毛利率YTD
    data['grossprofitmargin_q']=df['gross_profit_margin']

    #扣除非经常性损益后净利润率YTD
    data['profitmargin_q']=df['adjusted_profit']/df['operating_revenue']
    #资产周转率TTM
    data['assetturnover_ttm']=df['total_asset_turnover_rate']
    #总资产周转率YTD 营业收入/总资产
    data['assetturnover_q']=df['operating_revenue']/df['total_assets']
    #经营性现金流/净利润TTM
    data['operationcashflowratio_ttm']=df['net_operate_cash_flow_ttm']/df['net_profit_ttm']
    #经营性现金流/净利润YTD
    data['operationcashflowratio_q']=df['net_operate_cash_flow']/df['net_profit']
    #净资产
    df['net_assets']=df['total_assets']-df['total_liability']
    #总资产/净资产
    data['financial_leverage']=df['total_assets']/df['net_assets']
    #非流动负债/净资产
    data['debtequityratio']=df['total_non_current_liability']/df['net_assets']
    #现金比率=(货币资金+有价证券)÷流动负债
    data['cashratio']=df['cash_to_current_liability']
    #流动比率=流动资产/流动负债*100%
    data['currentratio']=df['current_ratio']
    #总市值取对数
    data['ln_capital']=np.log(df['market_cap'])
    #TTM所需时间
    his_date = [pd.to_datetime(date) - datetime.timedelta(90*i) for i in range(0, 4)]
    tmp = pd.DataFrame()
    tmp['code']=list(stock)
    for i in his_date:
        tmp_adjusted_dividend = get_fundamentals(query(indicator.code, indicator.adjusted_profit, \
                                                     cash_flow.dividend_interest_payment).
                                               filter(indicator.code.in_(stock)), date = i)
        tmp=pd.merge(tmp,tmp_adjusted_dividend,how='outer',on='code')

        tmp=tmp.rename(columns={'adjusted_profit':'adjusted_profit'+str(i.month), \
                                'dividend_interest_payment':'dividend_interest_payment'+str(i.month)})
    tmp=tmp.set_index('code')
    tmp_columns=tmp.columns.values.tolist()
    tmp_adjusted=sum(tmp[[i for i in tmp_columns if 'adjusted_profit'in i ]],1)
    tmp_dividend=sum(tmp[[i for i in tmp_columns if 'dividend_interest_payment'in i ]],1)
    #扣除非经常性损益后净利润(TTM)/总市值
    data['EPcut']=tmp_adjusted/df['market_cap']
    #近12个月现金红利(按除息日计)/总市值
    data['DP']=tmp_dividend/df['market_cap']
    #扣除非经常性损益后净利润率TTM
    data['profitmargin_ttm']=tmp_adjusted/df['operating_revenue_ttm']
    #营业收入(YTD)同比增长率
    #_x现在 _y前一年
    his_date = pd.to_datetime(date) - datetime.timedelta(365)
    name=['operating_revenue','net_profit','net_operate_cash_flow','roe']
    temp_data=df[name]
    his_temp_data = get_fundamentals(query(valuation.code, income.operating_revenue,income.net_profit,\
                                            cash_flow.net_operate_cash_flow,indicator.roe).
                                      filter(valuation.code.in_(stock)), date = his_date)
    his_temp_data=his_temp_data.set_index('code')
    #重命名 his_temp_data last_year
    for i in name:
        his_temp_data=his_temp_data.rename(columns={i:i+'last_year'})

    temp_data =pd.concat([temp_data,his_temp_data],axis=1)
    #营业收入(YTD)同比增长率
    data['sales_g_q']=temp_data['operating_revenue']/temp_data['operating_revenuelast_year']-1
    #净利润(YTD)同比增长率
    data['profit_g_q']=temp_data['net_profit']/temp_data['net_profitlast_year']-1
    #经营性现金流(YTD)同比增长率
    data['ocf_g_q']=temp_data['net_operate_cash_flow']/temp_data['net_operate_cash_flowlast_year']-1
    #ROE(YTD)同比增长率
    data['roe_g_q']=temp_data['roe']/temp_data['roelast_year']-1
    
    
    #计算beta部分
    #辅助线性回归的函数
    def linreg(X,Y,columns=3):
        X=sm.add_constant(array(X))
        Y=array(Y)
        if len(Y)>1:
            results = regression.linear_model.OLS(Y, X).fit()
            return results.params
        else:
            return [float("nan")]*(columns+1)
    #个股60个月收益与上证综指回归的截距项与BETA
    stock_close=get_price(list(stock), count = 12*20+1, end_date=date, frequency='daily', fields=['close'])['close']
    SZ_close=get_price('000001.XSHG', count = 12*20+1, end_date=date, frequency='daily', fields=['close'])['close']
    stock_pchg=stock_close.pct_change().iloc[1:]
    SZ_pchg=SZ_close.pct_change().iloc[1:]
    beta=[]
    stockalpha=[]
    for i in stock:
        temp_beta, temp_stockalpha = stats.linregress(SZ_pchg, stock_pchg[i])[:2]
        beta.append(temp_beta)
        stockalpha.append(temp_stockalpha)
    #此处alpha beta为list
    #data['alpha']=stockalpha
    data['beta']=beta
    
    #反转
    data['reverse_1m']=stock_close.iloc[-21]/stock_close.iloc[-1]-1
    data['reverse_3m']=stock_close.iloc[-63]/stock_close.iloc[-1]-1
    
    #波动率（一个月、三个月标准差）
    data['std_1m']=stock_close[-20:].std()
    data['std_3m']=stock_close[-60:].std()
    
    #换手率
    #tradedays_1m = get_tradeday_list(start=date,end=date,frequency='day',count=21)#最近一个月交易日
    tradedays_3m = get_tradeday_list(start=date,end=date,frequency='day',count=63)#最近一个月交易日
    data_turnover_ratio=pd.DataFrame()
    data_turnover_ratio['code']=list(stock)
    for i in tradedays_3m:
        q = query(valuation.code,valuation.turnover_ratio).filter(valuation.code.in_(stock))
        temp = get_fundamentals(q, i)
        data_turnover_ratio=pd.merge(data_turnover_ratio, temp,how='left',on='code')
        data_turnover_ratio=data_turnover_ratio.rename(columns={'turnover_ratio':i})
    data['turn_3m']= (data_turnover_ratio.set_index('code').T).mean()
    data['turn_1m']= (data_turnover_ratio.set_index('code').T)[-21:].mean()    
    
    #技术指标部分
    date_1 = tradedays_before(date,1)
    data['PSY']=pd.Series(PSY(stock, date_1, timeperiod=20))
    data['RSI']=pd.Series(RSI(stock, date_1, N1=20))
    data['BIAS']=pd.Series(BIAS(stock,date_1, N1=20)[0])
    dif,dea,macd=MACD(stock, date_1, SHORT = 10, LONG = 30, MID = 15)
    #data['DIF']=pd.Series(dif)
    #data['DEA']=pd.Series(dea)
    data['MACD']=pd.Series(macd)
    
    return data

factor_test_list = ['turn_1m','turn_3m','reverse_3m','size_lg','roa_q','roe_g_q','roe_q','assetturnover_q','RSI',\
                   'BIAS','currentratio','reverse_1m']

#设置因子检查日期
start_date = '2018-1-1'
end_date = '2019-1-1'
#设置股票池
index = '000906.XSHG'
trade_list = get_tradeday_list(start=start_date,end=end_date,frequency='month')

#因子列表
factor_list = ['code','EP','BP','SP','net_g','roe_g_q','sales_g_q','roe_q','roa_q','grossprofitmargin_ttm','netprofitratio_ttm',\
              'assetturnover_q','currentratio','size_lg','reverse_1m','reverse_3m','std_1m','std_3m','turn_1m','turn_3m',\
              'PSY','RSI','BIAS','MACD']
factor_name = ['股票代码','净利润/总市值','净资产/总市值','营业收入/总市值','净利润同比增长率','ROE同比增长率','营业收入同比增长率',\
               'ROE','ROA','毛利率ttm','净利率ttm','总资产周转率q','流动比率',\
               '市值','反转因子（1个月）','反转因子（3个月）','波动率（1个月）','波动率（3个月）','换手率（1个月）',\
               '换手率（3个月）','PSY','RSI','BIAS','MACD']

df_dict = {}
pool = {}

#获取多期所有涉及到的股票
for d in trade_list[:]:
    pool = set(pool) | set(get_index_stocks(index,date=d))
pool = list(pool)
print(len(pool))
#进行多期因子数据获取
for date in trade_list[:]:
    temp_df = get_factor_data(pool,date)
    temp_df = temp_df[factor_list]
    df_dict[date] = temp_df#pd.DataFrame(temp_df,index=df.index,columns=df.columns)
pl = pd.Panel(df_dict)
pl = pl.transpose(2,0,1)
pl

random_matrix = np.matrix([[random.random() for i in range(len(pl.major_axis))] for j in range(len(pl.minor_axis))])
pl['random'] = pd.DataFrame(random_matrix.T,index=pl.major_axis,columns=pl.minor_axis)
pl

#保留原始数据，方便后面修改使用
pl_pro = pl.copy()

#去空值
print('去除空值前个数：%s'%len(pl.minor_axis))
pl_pro = (pl_pro.loc[:,:,:]).dropna(axis=2)
pl_pro

#进行因子截面去极值、标准化处理
start = time.time()
print('计算%s截面去极值、标准化因子中......'%len(pl_pro.minor_axis))

for factor in factor_list[1:]:
    factor_df = pl_pro.loc[factor,:,:].copy() #获取因子df
    for i in factor_df.index:
        factor_df.loc[i,:] = filter_extreme_MAD(factor_df.loc[i,:],3)#去极值 
        factor_df.loc[i,:] = standardize(factor_df.loc[i,:],ty=2)    #标准化
    pl_pro[factor] = pd.DataFrame(factor_df,index=pl_pro.major_axis, columns=pl_pro.minor_axis)
    print('%s因子处理完毕'%factor)
end = time.time()
print('因子标准化处理完毕,统计股票个数：%s,耗时：%s 秒'%(len(pl_pro.minor_axis),end-start))
pl_pro

#获得行业哑变量矩阵
start = time.time()
print('行业中性化处理中......')
from jqdata import *
sw=get_industries(name='sw_l1').index
industry_df=pd.DataFrame(0,columns=pl_pro.minor_axis,index=range(len(sw)))

for i in range(len(sw)):
    temp=list(set(pl_pro.minor_axis).intersection(set(get_industry_stocks(sw[i])))) #?
    industry_df.loc[i,temp]=1
for factor in factor_list[1:]:
    data = pl_pro.loc[factor,:,:].copy()
    data = data.applymap(lambda x:float(x))

    #去除市值、行业因素，得到新的因子值 
    data_df=pd.DataFrame()
    for i in range(len(data.index)):
        '''
        未做市值中性化
        m= get_fundamentals(query(valuation.circulating_market_cap,valuation.code).filter(valuation.code.in_(data.columns)), date=data.index[i])
        m.index=np.array(m['code'])
        m=m.iloc[:,0]
        m=(m-mean(m))/std(m)
        '''
        x=data.iloc[i,:]
        conc=pd.concat([x,industry_df.T],axis=1)#.fillna(mean(m)) 这里只做了行业中性化，全部为[x,m,industry_df.T]
        est=sm.OLS(conc.iloc[:,:1],conc.iloc[:,1:]).fit()
        y_fitted = est.fittedvalues
        data_df[i]=est.resid
    data_df=data_df.T
    data_df.index=data.index
    pl_pro[factor] = data_df
    print('%s行业中性化处理完毕'%factor)
end = time.time()
print('行业中性化处理完毕，耗时：%s 秒'%(end-start))

trade_list1 = get_tradeday_list(start='2017-12-1',end='2019-1-10',frequency='month')
trade_list1

pubdate_df = pl_pro.loc['code',:,:].copy() 
pubdate_df.columns

#用于获取股票的统计期涨跌幅
price_df = get_price(list(pl_pro.minor_axis),start_date=trade_list1[0],end_date=trade_list1[-1],fields=['close'])['close'].fillna(method='ffill')
price_df = price_df.loc[trade_list1,:].shift(-1) #获取指定日期列表前个各加一个月,并往前推一个周期
pct_df = price_df/price_df.shift(1)-1 #统计下期收益记录再当前日期上
pct_df

#加入时间序列价格
start = time.time()
print('计算%s只股票各周期收益......'%len(pl_pro.minor_axis))
#加入价格序列
#price_df = get_price(list(pl_pro.minor_axis),start_date=trade_list1[0],end_date=trade_list1[-1],fields=['close'])['close']
pubdate_df = pl_pro.loc['code',:,:].copy() #获取因子df
for colm in pubdate_df.columns:
    pubdate_df[colm] = pct_df.loc[trade_list,colm].values
pl_pro['pct'] = pd.DataFrame(pubdate_df,index=pl_pro.major_axis, columns=pl_pro.minor_axis)
#计算收益变化值
#pl_pro['pct'] = (pl_pro.loc['close',:,:]/pl_pro.loc['close',:,:].shift(1)).fillna(1)
end = time.time()
print('数据准备完毕，耗时：%s 秒'%str(end-start))
pl_pro

pl_pro.major_axis

pl_pro.major_axis[-1]
end_label = pl_pro.major_axis[-1]
start_label = pl_pro.major_axis[0]

#将因子值和y值匹配
start = time.time()
print('计算RLM回归中......')
end_label = pl_pro.major_axis[-1]
start_label = pl_pro.major_axis[0]
t_dict = {}
f_dict = {}
IC_spearman_dict = {}
IC_pearson_dict = {}
for factor in factor_list[1:]:
    x_df = pl_pro.loc[factor,:end_label,:]  #设置不同的输入因子值  factor_mad_std
    x_df = x_df.applymap(lambda x:float(x))
    y_df = pl_pro.loc['pct',start_label:,:]
    t_list = []
    f_list = []
    IC_spearman_list = []
    IC_pearson_list = []  
    
    for i in range(len(y_df.index)):
        rlm_model = sm.RLM(y_df.iloc[i,:], x_df.iloc[i,:], M=sm.robust.norms.HuberT()).fit()
        f_list.append(float(rlm_model.params))
        t_list.append(float(rlm_model.tvalues))
        
    for i in range(len(x_df.index)):
        pearson_corr = x_df.iloc[i,:].corr(y_df.iloc[i,:],method='pearson')
        spearman_corr = x_df.iloc[i,:].corr(y_df.iloc[i,:],method='spearman')
        IC_pearson_list.append(pearson_corr)
        IC_spearman_list.append(spearman_corr)
    
    t_dict[factor] = t_list
    f_dict[factor] = f_list
    IC_spearman_dict[factor] = IC_spearman_list
    IC_pearson_dict[factor] = IC_pearson_list
    
    print('%s因子计算完毕'%factor)
end = time.time()
print('回归计算完毕,耗时%s'%(end-start))

factor_name = ['股票代码','净利润/总市值','净资产/总市值','营业收入/总市值','净利润同比增长率','ROE同比增长率','营业收入同比增长率',\
               'ROE','ROA','毛利率ttm','净利率ttm','总资产周转率q','流动比率',\
               '市值','反转因子（1个月）','反转因子（3个月）','波动率（1个月）','波动率（3个月）','换手率（1个月）',\
               '换手率（3个月）','PSY','RSI','BIAS','MACD']

index_list = ['f均值','fi>0','abs（T）均值','abs（T）>2','IC均值','IR值','abs（IC）>0.02','IC>0']
summary_df = pd.DataFrame(index=index_list)

for i in range(1,len(factor_list)):
    factor = factor_list[i]
    IC_spearman_list = IC_spearman_dict[factor]
    IC_pearson_list = IC_pearson_dict[factor]
    f_list = f_dict[factor]
    t_list = t_dict[factor]
    f_mean = np.mean(f_list)
    f_ratio = sum(np.where(np.array(f_list)>0,1,0))*1.0/len(f_list)
    #print('因子收益序列fi大于0概率：%s'%round(f_ratio,4))
    t_abs_mean = np.mean([abs(i) for i in t_list])
    #print('t值绝对值的均值：%s'%round(t_abs_mean,4))
    t_abs_dayu2 = sum(np.where(((np.array(t_list)>2) | (np.array(t_list)<-2)),1,0))*1.0/len(t_list)
    #print('t值绝对值大于等于2的概率：%s'%round(t_abs_dayu2,4))

    ic_mean = np.mean(IC_spearman_list)
    ic_std = np.std(IC_spearman_list)
    ic_dayu0 = sum(np.where(np.array(IC_spearman_list)>0,1,0))*1.0/len(IC_spearman_list)
    ic_abs_dayu = sum(np.where(((np.array(IC_spearman_list)>0.02) | (np.array(IC_spearman_list)<-0.02)),1,0))*1.0/len(IC_spearman_list)
    ir = ic_mean/ic_std
    #print('IC均值：%s,标准差：%s,IR值：%s'%(round(ic_mean,4),round(ic_std,4),round(ir,4)))
    #print('IC值大于0概率：%s'%round(ic_dayu0,4))
    #print('IC值绝对值大于0.02的均值：%s'%round(ic_abs_dayu,4))

    index_values = [round(f_mean,4),round(f_ratio,4),round(t_abs_mean,4),round(t_abs_dayu2,4),round(ic_mean,4),round(ir,4),\
                    round(ic_abs_dayu,4),round(ic_dayu0,4)]
    
    summary_df[factor_name[i]] = index_values
summary_df

def showColor(val):
    color = 'red' if val > 2 else 'black'
    return 'color:%s'%color
#summary_df.style.applymap(showColor)#全表格
#summary_df.style.applymap(showColor,subset=pd.IndexSlice[3:4,:])#指定表格位置?
summary_df.T.sort_values('IC均值',ascending=0)

factor_list

#获取各因子IC序列值
#获取各因子收益率序列
factor_ic = pd.DataFrame(index=pl_pro.major_axis[:])
factor_fi = pd.DataFrame(index=pl_pro.major_axis[:])
for i in range(1,len(factor_list)):
    factor = factor_list[i]
    factor_ic[factor_name[i]]=IC_spearman_dict[factor]
    factor_fi[factor_name[i]]=f_dict[factor]

#因子2018年月均收益
factor_fi.mean().sort_values().plot(kind='bar',color='blue',figsize=(12,8))#展示所有因子的IC均值情况

#因子2018年月均IC 
factor_ic.mean().sort_values().plot(kind='bar',figsize=(12,8))#展示所有因子的IC均值情况

factor = 'sizi_lg'
factor_fi['市值'].plot(kind='bar')#因子收益序列图

factor_ic['市值'].plot(kind='bar')#IC序列图

#设置用于分组检查的因子值
factor = 'roa_q'
pool_dict = {}
group = 5

x_df = pl_pro.loc[factor,:end_label,:]  #设置不同的输入因子值  factor_mad_std
x_df = x_df.applymap(lambda x:float(x))

for i in range(len(x_df.index)):
    temp_se = x_df.iloc[0,:].sort_values()#从小到大排序
    pool = temp_se[temp_se>0].index #去掉小于0的值
    #pool = temp_se.index #不做负值处理
    num = int(len(pool)/group)
    #print('第%s期每组%s只股票'%(i,num))
    pool_dict[x_df.index[i]] = pool
group_pct = get_all_pct(pool_dict,trade_list,groups=group)
group_pct.columns = ['group'+str(i) for i in range(len(group_pct.columns))]
group_pct.cumprod().plot(figsize=(12,8))

#factor_test_list = ['BP','reverse_1m','size_lg','SP','turn_1m','turn_3m','std_3m','net_g']\
factor_test_list = ['turn_1m','turn_3m','reverse_3m','size_lg','roa_q','roe_g_q','roe_q','assetturnover_q','RSI',\
                   'BIAS','currentratio','reverse_1m']
group_bt_df = pd.DataFrame()
for factor in factor_test_list:
    pl_pro = pl
    pool_dict = {}
    x_df = pl_pro.loc[factor,:end_label,:]  #设置不同的输入因子值  factor_mad_std
    x_df = x_df.applymap(lambda x:float(x))

    for i in range(len(x_df.index)):
        temp_se = x_df.iloc[0,:].sort_values()#从小到大排序
        pool = temp_se[temp_se>0].index #去掉小于0的值
        #pool = temp_se.index #不做负值处理
        num = int(len(pool)/group)
        #print('第%s期每组%s只股票'%(i,num))
        pool_dict[x_df.index[i]] = pool
    group_pct = get_all_pct(pool_dict,trade_list,groups=group)
    group_pct.columns = ['group'+str(i) for i in range(len(group_pct.columns))]
    
    group_bt_df[factor_name_df.loc[factor][0]] = group_pct.cumprod().iloc[-1,:]
group_bt_df.T

factor_name_df.loc['EP'][0]

factor_name_df = pd.DataFrame(factor_name,index=factor_list)
factor_name_df