【筆記】因子測試框架¶

# 【基礎框架函數】輪動框架 v9 
# coding: utf-8
# 制作人：Howard董先森， wechat:dhl19988882，e-mail: 16hldong.stu.edu.cn
# copy並分享的時候記得給小弟一下credit就好~

import warnings
warnings.filterwarnings("ignore")

'''
導入函數庫
'''
import pandas as pd
import numpy as np
import time
import datetime
import statsmodels.api as sm
import pickle
import warnings
from itertools import combinations
import itertools
from jqfactor import get_factor_values
from jqdata import *
from jqfactor import neutralize
from jqfactor import standardlize
from jqfactor import winsorize
from jqfactor import neutralize
import calendar
from scipy import stats
import statsmodels.api as sm
from statsmodels import regression
import csv
from six import StringIO
from jqlib.technical_analysis import *
#導入pca
from sklearn.decomposition import PCA
from sklearn import svm
from sklearn.model_selection import train_test_split
from sklearn.grid_search import GridSearchCV
from sklearn import metrics

import matplotlib.dates as mdates
import matplotlib.pyplot as plt

import seaborn as sns

import matplotlib.pyplot as plt
from jqfactor import neutralize

'''
普通工具
'''
# 多重列表分解函數
def splitlist(b):
    alist = []
    a = 0

    for sublist in b:
        try: #用try來判斷是列表中的元素是不是可迭代的，可以迭代的繼續迭代
            for i in sublist:
                alist.append (i)
        except TypeError: #不能迭代的就是直接取出放入alist
            alist.append(sublist)
    for i in alist:
        if type(i) == type([]):#判斷是否還有列表
            a =+ 1
            break
    if a==1:
        return printlist(alist) #還有列表，進行遞歸
    if a==0:

        return alist

#PCA降維
def pca_analysis(data,n_components='mle'):
    index = data.index
    model = PCA(n_components=n_components)
    model.fit(data)
    data_pca = model.transform(data)
    df = pd.DataFrame(data_pca,index=index)
    return df


'''
數據準備
'''
# 指數池子清洗函數
def get_stock_code(index, date):
    # 指數池子提取
    stockList = get_index_stocks(index , date=date)
    #剔除ST股
    st_data=get_extras('is_st',stockList, count = 1,end_date = start_date)
    stockList = [stock for stock in stockList if not st_data[stock][0]]
    #剔除停牌、新股及退市股票
    stockList=delect_stop(stockList,date,date)
    
    return stockList 

# 因子排列組合函數
def factor_combins(data):
    '''
    輸入：
        格式為Dataframe
         data
    輸出:
        格式為list
        所有因子的排列組合
    '''
    # 排列組合
    factor_combins =[]

    for i in range(len(data.columns)+1):
        combins = [c for c in combinations((data.columns),i)]

        factor_combins.append(combins)

    # 剔除排列組合中空的情況
    factor_combins.pop(0)

    # 多層列表展開平鋪成一層
    factor_list = list(itertools.chain(*factor_combins))
    
    return factor_list

#獲取時間為date的全部因子數據
def get_factor_data(stock_list,date):

    data=pd.DataFrame(index=stock_list)

    # 第一波因子提取
    q = query(valuation,balance,cash_flow,income,indicator).filter(valuation.code.in_(stock_list))
    df = get_fundamentals(q, date)
    df['market_cap']=df['market_cap']*100000000
    df.index = df['code']
    del df['code'],df['id']

    # 第二波因子提取
    factor_data=get_factor_values(stock_list,['roe_ttm','roa_ttm','total_asset_turnover_rate',\
                               'net_operate_cash_flow_ttm','net_profit_ttm',\
                              'cash_to_current_liability','current_ratio',\
                             'gross_income_ratio','non_recurring_gain_loss',\
                            'operating_revenue_ttm','net_profit_growth_rate'],end_date=date,count=1)
    # 第二波因子轉dataframe格式
    factor=pd.DataFrame(index=stock_list)
    for i in factor_data.keys():
        factor[i]=factor_data[i].iloc[0,:]


    #合並得大表
    df=pd.concat([df,factor],axis=1)

    # 有些因子沒有的，用別的近似代替
    df.fillna(0, inplace = True)

    # profitability 因子
    # proft/net revenue
    data['gross_profit_margin'] = df['gross_profit_margin']
    data['net_profit_margin'] = df['net_profit_margin']
    data['operating_profit_margin'] = df['operating_profit']/(df['total_operating_revenue']-df['total_operating_cost'])
    data['pretax_margin'] = (df['net_profit'] + df['income_tax_expense'])/(df['total_operating_revenue']-df['total_operating_cost'])
    # profit/capital
    data['ROA'] = df['roa']
    data['EBIT_OA'] = df['operating_profit']/df['total_assets']
    data['ROTC'] = df['operating_profit']/(df['total_owner_equities'] + df['longterm_loan'] + df['shortterm_loan']+df['borrowing_from_centralbank']+df['bonds_payable']+df['longterm_account_payable'])
    data['ROE'] = df['roe']
    # Others
    data['inc_total_revenue_annual'] = df['inc_total_revenue_annual']


    # Activity 因子
    # inventory
    data['inventory_turnover'] = df['operating_cost']/df['inventories']
    data['inventory_processing_period'] = 365/data['inventory_turnover']
    # account receivables
    data['receivables_turnover'] =df['total_operating_revenue']/df['account_receivable']
    data['receivables_collection_period'] = 365/data['receivables_turnover']

    # activity cycle
    data['operating_cycle'] = data['receivables_collection_period']+data['inventory_processing_period']
    
    # Other turnovers
    data['total_asset_turnover'] = df['operating_revenue']/df['total_assets']
    data['fixed_assets_turnover'] = df['operating_revenue']/df['fixed_assets']
    data['working_capital_turnover'] = df['operating_revenue']/(df['total_current_assets']-df['total_current_liability'])


    # Liquidity 因子
    data['current_ratio'] =df['current_ratio']
    data['quick_ratio'] =(df['total_current_assets']-df['inventories'])/df['total_current_liability']
    data['cash_ratio'] = (df['cash_equivalents']+df['proxy_secu_proceeds'])/df['total_current_liability']
    # leverage 
    data['debt_to_equity'] = (df['longterm_loan'] + df['shortterm_loan']+df['borrowing_from_centralbank']+df['bonds_payable']+df['longterm_account_payable'])/df['total_owner_equities']
    data['debt_to_capital'] =  (df['longterm_loan'] + df['shortterm_loan']+df['borrowing_from_centralbank']+df['bonds_payable']+df['longterm_account_payable'])/(df['longterm_loan'] + df['shortterm_loan']+df['borrowing_from_centralbank']+df['bonds_payable']+df['longterm_account_payable']+df['total_owner_equities'])                               
    data['debt_to_assets'] =  (df['longterm_loan'] + df['shortterm_loan']+df['borrowing_from_centralbank']+df['bonds_payable']+df['longterm_account_payable'])/df['total_assets']
    data['financial_leverage'] = df['total_assets'] / df['total_owner_equities']


    # Valuation 因子
    # Price
    data['EP']=df['net_profit_ttm']/df['market_cap']
    data['PB']=df['pb_ratio']
    data['PS']=df['ps_ratio']
    data['P_CF']=df['market_cap']/df['net_operate_cash_flow']



    # 技術面 因子
    # 下面技術因子來自於西交大元老師量化小組的SVM研究，report，https://www.joinquant.com/view/community/detail/15371
    stock = stock_list
    #個股60個月收益與上證綜指回歸的截距項與BETA
    stock_close=get_price(stock, count = 60*20+1, end_date=date, frequency='daily', fields=['close'])['close']
    SZ_close=get_price('000001.XSHG', count = 60*20+1, end_date=date, frequency='daily', fields=['close'])['close']
    stock_pchg=stock_close.pct_change().iloc[1:]
    SZ_pchg=SZ_close.pct_change().iloc[1:]
    beta=[]
    stockalpha=[]
    for i in stock:
        temp_beta, temp_stockalpha = stats.linregress(SZ_pchg, stock_pchg[i])[:2]
        beta.append(temp_beta)
        stockalpha.append(temp_stockalpha)
    #此處alpha beta為list
    data['alpha']=stockalpha
    data['beta']=beta

    #動量
    data['return_1m']=stock_close.iloc[-1]/stock_close.iloc[-20]-1
    data['return_3m']=stock_close.iloc[-1]/stock_close.iloc[-60]-1
    data['return_6m']=stock_close.iloc[-1]/stock_close.iloc[-120]-1
    data['return_12m']=stock_close.iloc[-1]/stock_close.iloc[-240]-1

    #取換手率數據
    data_turnover_ratio=pd.DataFrame()
    data_turnover_ratio['code']=stock
    trade_days=list(get_trade_days(end_date=date, count=240*2))
    for i in trade_days:
        q = query(valuation.code,valuation.turnover_ratio).filter(valuation.code.in_(stock))
        temp = get_fundamentals(q, i)
        data_turnover_ratio=pd.merge(data_turnover_ratio, temp,how='left',on='code')
        data_turnover_ratio=data_turnover_ratio.rename(columns={'turnover_ratio':i})
    data_turnover_ratio=data_turnover_ratio.set_index('code').T   

    #個股個股最近N個月內用每日換手率乘以每日收益率求算術平均值
    data['wgt_return_1m']=mean(stock_pchg.iloc[-20:]*data_turnover_ratio.iloc[-20:])
    data['wgt_return_3m']=mean(stock_pchg.iloc[-60:]*data_turnover_ratio.iloc[-60:])
    data['wgt_return_6m']=mean(stock_pchg.iloc[-120:]*data_turnover_ratio.iloc[-120:])
    data['wgt_return_12m']=mean(stock_pchg.iloc[-240:]*data_turnover_ratio.iloc[-240:])
    #個股個股最近N個月內用每日換手率乘以函數exp(-x_i/N/4)再乘以每日收益率求算術平均值
    temp_data=pd.DataFrame(index=data_turnover_ratio[-240:].index,columns=stock)
    temp=[]
    for i in range(240):
        if i/20<1:
            temp.append(exp(-i/1/4))
        elif i/20<3:
            temp.append(exp(-i/3/4))
        elif i/20<6:
            temp.append(exp(-i/6/4))
        elif i/20<12:
            temp.append(exp(-i/12/4))  
    temp.reverse()
    for i in stock:
        temp_data[i]=temp
    data['exp_wgt_return_1m']=mean(stock_pchg.iloc[-20:]*temp_data.iloc[-20:]*data_turnover_ratio.iloc[-20:])
    data['exp_wgt_return_3m']=mean(stock_pchg.iloc[-60:]*temp_data.iloc[-60:]*data_turnover_ratio.iloc[-60:])
    data['exp_wgt_return_6m']=mean(stock_pchg.iloc[-120:]*temp_data.iloc[-120:]*data_turnover_ratio.iloc[-120:])
    data['exp_wgt_return_12m']=mean(stock_pchg.iloc[-240:]*temp_data.iloc[-240:]*data_turnover_ratio.iloc[-240:])

    #波動率
    data['std_1m']=stock_pchg.iloc[-20:].std()
    data['std_3m']=stock_pchg.iloc[-60:].std()
    data['std_6m']=stock_pchg.iloc[-120:].std()
    data['std_12m']=stock_pchg.iloc[-240:].std()

    #股價
    data['ln_price']=np.log(stock_close.iloc[-1])

    #換手率
    data['turn_1m']=mean(data_turnover_ratio.iloc[-20:])
    data['turn_3m']=mean(data_turnover_ratio.iloc[-60:])
    data['turn_6m']=mean(data_turnover_ratio.iloc[-120:])
    data['turn_12m']=mean(data_turnover_ratio.iloc[-240:])

    data['bias_turn_1m']=mean(data_turnover_ratio.iloc[-20:])/mean(data_turnover_ratio)-1
    data['bias_turn_3m']=mean(data_turnover_ratio.iloc[-60:])/mean(data_turnover_ratio)-1
    data['bias_turn_6m']=mean(data_turnover_ratio.iloc[-120:])/mean(data_turnover_ratio)-1
    data['bias_turn_12m']=mean(data_turnover_ratio.iloc[-240:])/mean(data_turnover_ratio)-1

    #技術指標
    data['PSY']=pd.Series(PSY(stock, date, timeperiod=20))
    data['RSI']=pd.Series(RSI(stock, date, N1=20))
    data['BIAS']=pd.Series(BIAS(stock,date, N1=20)[0])
    dif,dea,macd=MACD(stock, date, SHORT = 10, LONG = 30, MID = 15)
    data['DIF']=pd.Series(dif)
    data['DEA']=pd.Series(dea)
    data['MACD']=pd.Series(macd)

    # 去inf
    a = np.array(data)
    where_are_inf = np.isinf(a)
    a[where_are_inf] = nan
    factor_data =pd.DataFrame(a, index=data.index, columns = data.columns)

    return factor_data

# 行業代碼獲取函數
def get_industry(date,industry_old_code,industry_new_code):
    '''
    輸入：
        date: str, 時間點
        industry_old_code: list, 舊的行業代碼
        industry_new_code: list, 新的行業代碼
        
    輸出:
        
        list, 行業代碼
    ''' 
    #獲取行業因子數據
    if datetime.datetime.strptime(date,"%Y-%m-%d").date()<datetime.date(2014,2,21):
        industry_code=industry_old_code
    else:
        industry_code=industry_new_code
    len(industry_code)
    
    return industry_code

#取股票對應行業
def get_industry_name(i_Constituent_Stocks, value):
    return [k for k, v in i_Constituent_Stocks.items() if value in v]

#缺失值處理
def replace_nan_indu(factor_data, stockList, industry_code, date): 
    '''
    輸入：
        factor_data: dataframe, 有缺失值的因子值
        stockList： list, 指數池子所有標的
        industry_code: list, 行業代碼
        date: str, 時間點
        
    輸出:
        格式為list
        所有因子的排列組合
    ''' 
    # 創建一個字典，用於儲存每個行業對應的成分股（行業標的和指數標的取交集）
    i_Constituent_Stocks={}

    # 創建一個行業因子數據表格，待填因子數據（暫定為均值）
    ind_factor_data = pd.DataFrame(index=industry_code,columns=factor_data.columns)


    # 遍曆所有行業
    for i in industry_code:

        # 獲取單個行業所有的標的
        stk_code = get_industry_stocks(i, date) 

        # 儲存每個行業對應的所有成分股到剛剛創建的字典中（行業標的和指數標的取交集）
        i_Constituent_Stocks[i] = list(set(stk_code).intersection(set(stockList)))

        # 對於某個行業，根據提取出來的指數池子，把所有交集內的因子值取mean，賦值到行業因子值的表格上
        ind_factor_data.loc[i]=mean(factor_data.loc[i_Constituent_Stocks[i],:])

    # 對於行業值缺失的（行業標的和指數標的取交集），用所有行業的均值代替
    for factor in ind_factor_data.columns:

        # 提取行業缺失值所在的行業，並放到list去
        null_industry=list(ind_factor_data.loc[pd.isnull(ind_factor_data[factor]),factor].keys())

        # 遍曆行業缺失值所在的行業
        for i in null_industry:

            # 所有行業的均值代替,行業缺失的數值
            ind_factor_data.loc[i,factor]=mean(ind_factor_data[factor])

        # 提取含缺失值的標的
        null_stock=list(factor_data.loc[pd.isnull(factor_data[factor]),factor].keys())

        # 用行業值（暫定均值）去填補，指數因子缺失值
        for i in null_stock:

            industry = get_industry_name(i_Constituent_Stocks, i)

            if industry:

                factor_data.loc[i, factor] = ind_factor_data.loc[industry[0], factor]

            else:

                factor_data.loc[i, factor] = mean(factor_data[factor])
    
    return factor_data

#去除上市距beginDate不足1年且退市在endDate之後的股票
def delect_stop(stocks,beginDate,endDate,n=365):
    stockList=[]
    beginDate = datetime.datetime.strptime(beginDate, "%Y-%m-%d")
    endDate = datetime.datetime.strptime(endDate, "%Y-%m-%d")
    for stock in stocks:
        start_date=get_security_info(stock).start_date
        end_date=get_security_info(stock).end_date
        if start_date<(beginDate-datetime.timedelta(days=n)).date() and end_date>endDate.date():
            stockList.append(stock)
    return stockList

# 獲取預處理後的因子
def prepare_data(date,index):
    
    '''
    輸入：
        date: str, 時間點
        index: 指數
        
    輸出: 
        
        data: 預備處理好的了dataframe 
    ''' 
    # 指數清洗（剔除ST、退市、新上市不足三個月的標的）
    stockList = get_stock_code(index, date)
    
    # 獲取數據
    factor_data = get_factor_data(stockList, date)
    
    # 數據預處理
    
    # 缺失值處理
    data = replace_nan_indu(factor_data, stockList, industry_code, date)
    # 啟動內置函數去極值
    data = winsorize(data, qrange=[0.05,0.93], inclusive=True, inf2nan=True, axis=0)
    # 啟動內置函數標準化
    data = standardlize(data, inf2nan=True, axis=0)
    # 啟動內置函數中性化，行業為'sw_l1'
    data = neutralize(data, how=['sw_l1','market_cap'], date=date , axis=0)

    
    return data

'''
時間準備
'''
# 時間管理函數
def time_handle(start_date, end_date, period, period_pre,index):
    '''
    輸入：
        start_date: 開始時間，str
        end_date: 截至時間，str
        period: 周期間隔，str(轉換為周是'W',月'M',季度線'Q',五分鍾'5min',12天'12D')
        period_pre: 小周期間隔，str(為了提取第一個T0數據，作為輔助用途)


    輸出:
        格式為dataframe
        columns:
        T0: T0時間點的集合
        T1: T1時間點的集合
        T2: T2時間點的集合

    '''
    # 設定轉換周期 period_type  轉換為周是'W',月'M',季度線'Q',五分鍾'5min',12天'12D'
    time = get_price(index ,start_date,end_date,'daily',fields=['close'])
    # 記錄每個周期中最後一個交易日
    time['date']=time.index
    # 進行轉換，周線的每個變量都等於那一周中最後一個交易日的變量值
    period_handle=time.resample(period,how='last')
    period_handle['t1'] = period_handle['date']
    # 創造超前數據T2
    period_handle['t2']=period_handle['t1'].shift(-1)
    # 重新設定index，設為模型運行次數
    period_handle.index = range(len(period_handle))
    # 構建T0
    period_handle['t0'] = period_handle['t1'].shift(1)

    # 重新獲取小一個周期的數據
    time1 = get_price(index ,start_date, end_date,'daily',fields=['close'])
    # 記錄每個周期中最後一個交易日
    time1['date']=time1.index
    # 進行轉換，周線的每個變量都等於那一周中最後一個交易日的變量值
    ph =time1.resample(period_pre,how='last').dropna()
    # 通過更小的周期來提取的最後一個交易日的值，來提取大表格的第一個T0
    period_handle['t0'][0] = ph['date'][0]
    # 記錄模型運行次數
    period_handle['Replication#'] = period_handle.index
    period_handle['Replication'] = period_handle['Replication#'].shift(-1)
    del period_handle['Replication#']
    del period_handle['date']
    # 重新排列列表順序
    period_handle['T0'] = period_handle['t0']
    period_handle['T1'] = period_handle['t1']
    period_handle['T2'] = period_handle['t2']
    del period_handle['close']
    del period_handle['t0']
    del period_handle['t1']
    del period_handle['t2']

    period_handle.to_csv('period_handle.csv')

    return period_handle

# 時間格式修改（從timestamp改為str）
def time_fix(period_handle):

    # 導入時間周期管理函數
    p = time_handle(start_date, end_date, period, period_pre,index)

    # dropna 並且list化
    datelist_T0 = list(p.dropna()['T0'])
    datelist_T1 = list(p.dropna()['T1'])
    datelist_T2 = list(p.dropna()['T2'])

    # timestamp 轉str
    datelist_T0 = list(np.vectorize(lambda s: s.strftime('%Y-%m-%d'))(datelist_T0))
    datelist_T1 = list(np.vectorize(lambda s: s.strftime("%Y-%m-%d"))(datelist_T1))
    datelist_T2 = list(np.vectorize(lambda s: s.strftime("%Y-%m-%d"))(datelist_T2))

    # list字典化，並轉為dateframe
    t = {'T0': datelist_T0, 'T1':datelist_T1, 'T2':datelist_T2}
    test_timing = pd.DataFrame(t)

    return test_timing 



'''
分類函數
'''

# 獲取指數在所有周期內的池子，然後取不重複set的並集,變成新的池子
def get_all_index_stock(index, period_handle):
    '''
    輸入： 
        stock_list: 所有周期池子的出現過的股票, list
        period_handle: 周期管理函數出來的時間集合, dataframe

    輸出:
            stock_list: list, 所有周期內，於指數出現過的標的池子
    '''
    # 時間周期管理
    p = time_handle(start_date, end_date, period, period_pre,index)
    
    # 大池子提取
    stock_list_pre = []

    for date in p['T0']:

        # 獲取每個T0的指數池子
        stk_list = get_index_stocks(index , date=date)

        # 放到stock_list里面
        stock_list_pre.append(stk_list)

    # 分解多重列表，去重複，返回list
    s = splitlist(stock_list_pre)
    stock_list = list(set(s))   
    
    return stock_list

# 所有周期，指數池子里面所有股票的周期收益率
def get_index_big_pool_return(stock_list, period_handle):
    '''
    輸入： 
        stock_list: 所有周期池子的出現過的股票, list
        period_handle: 周期管理函數出來的時間集合, dataframe

    輸出:
        格式為dataframe
        columns:

        T0: T0時間點的集合
        code1: 標的1在T0到T1的收益率
        code2: 標的2在T0到T1的收益率
        以此類推
    '''
    # 獲取指數池子所有的索引
    stock_code = stock_list
    p = time_handle(start_date, end_date, period, period_pre,index)

    # 獲取時間表格，剔除T1、T2和Replication列
    stk_return = p.copy()

    # 收益率計算
    for j in stock_code:
        j_str = str(j)
        p0 = [float(get_price(j,i,i)['close'])for i in p.iloc[:,1]]
        p1 = [float(get_price(j,i,i)['close'])for i in p.iloc[:,2]]
        a = [(m-n)/n for m,n in zip (p1,p0)]
        stk_return[str(j)] = a

    # 刪除T1列，保留T0列 ,並賦值給index     
    del stk_return['T1']
    stk_return.index = stk_return['T0']
    del stk_return['T0']
    del stk_return['T2']
    del stk_return['Replication']
    
    # 缺失值用0填補
    stk_return.fillna(0)
    
    # 保存
    stk_return.to_csv('stock_return.csv')
    
    return stk_return

# 訓練：分類函數
def train_classification(stk_return, index_return):
    '''
    輸入： 
        skt_retun: 所有標的在T0-T1時間集當中的周期收益率, dataframe
        index_return: 指數在T0-T1時間集合中的周期收益率, dataframe

    輸出:
        格式為dataframe
        columns:
        T0: T0時間點的集合

        index:
        code1: 在T0-T1這個周期里面是否跑贏大盤，跑贏為'1',否則為'0'
        code2: 是否跑贏大盤，分類數據

        以此類推
    '''
    # 複制skt_return的dataframe為最終的輸出格式
    stock_cl = stk_return.copy()

    # 遍曆比較，賦值
    for j in range(len(stock_cl.index)):

        for i in range(len(stock_cl.columns)):

            if stock_cl.iloc[j,i] > index_return.iloc[j,0]: 
                stock_cl.iloc[j,i] = 1
            else:
                stock_cl.iloc[j,i] = 0
    # 轉置
    stock_cl = stock_cl.T

    # 保存
    stock_cl.to_csv('stock_classification_data.csv')    

    return stock_cl





# 獲取T0 到 T1時間段，指數周期收益率
def get_index_return_train(index,period_handle):
    '''
    輸入：
        index: 指數, 元素str
        period_handle: 周期管理函數出來的時間集合, dataframe

    輸出:
        格式為dataframe
        columns:
        T0: T0時間點的集合
        index_return: 指數在T0到T1的收益率
    '''

    # 獲取時間表格，剔除T1、T2和Replication列
    p = time_handle(start_date, end_date, period, period_pre,index)
    index_return = p.copy()
    del index_return['Replication']
    del index_return['T2']

    # 收益率計算 
    p0 = [float(get_price(index,i,i)['close']) for i in p.iloc[:,1]]
    p1 = [float(get_price(index,i,i)['close']) for i in p.iloc[:,2]]
    a = [(m-n)/n for m,n in zip(p1,p0) ]
    index_return['index_return'] = a
    del index_return['T1']
    index_return.index = index_return['T0']
    del index_return['T0']

    # 保存現成的表格
    index_return.to_csv('index_return.csv')

    return index_return

# 獲取T1 到 T2 時間段， 指數周期的收益率
def get_index_return_test(index,period_handle):
    '''
    輸入：
        index: 指數, 元素str
        period_handle: 周期管理函數出來的時間集合, dataframe

    輸出:
        格式為dataframe
        columns:
        T2: T2時間點的集合
        index_return: 指數在T0到T1的收益率
    '''

    # 獲取時間表格，剔除T1、T2和Replication列
    p = time_handle(start_date, end_date, period, period_pre,index).dropna()
    index_return = p.copy()
    del index_return['Replication']
    del index_return['T0']

    # 收益率計算 
    p0 = [float(get_price(index,i,i)['close']) for i in p.iloc[:,2]]
    p1 = [float(get_price(index,i,i)['close']) for i in p.iloc[:,3]]
    a = [(m-n)/n for m,n in zip(p1,p0) ]
    index_return['index_return'] = a
    del index_return['T1']
    index_return.index = index_return['T2']
    del index_return['T2']

    # 保存現成的表格
    index_return.to_csv('index_return.csv')

    return index_return

# 獲取T0 到 T1時間段，指數池子里面所有股票的周期收益率
def get_stock_return_train(data, period_handle):
    '''
    輸入： 
        data: 因子提取的大表格, dataframe
        period_handle: 周期管理函數出來的時間集合, dataframe

    輸出:
        格式為dataframe
        columns:

        T0: T0時間點的集合
        code1: 標的1在T0到T1的收益率
        code2: 標的2在T0到T1的收益率
        以此類推
    '''
    # 獲取指數池子所有的索引
    stock_code = list(data.index)
    p = time_handle(start_date, end_date, period, period_pre,index)

    # 獲取時間表格，剔除T1、T2和Replication列
    stk_return = p.copy()

    # 收益率計算
    for j in stock_code:
        j_str = str(j)
        p0 = [float(get_price(j,i,i)['close'])for i in p.iloc[:,1]]
        p1 = [float(get_price(j,i,i)['close'])for i in p.iloc[:,2]]
        a = [(m-n)/n for m,n in zip (p1,p0)]
        stk_return[str(j)] = a

    # 刪除T1列，保留T0列 ,並賦值給index     
    del stk_return['T1']
    stk_return.index = stk_return['T0']
    del stk_return['T0']
    del stk_return['T2']
    del stk_return['Replication']

    # 保存文件
    stk_return.to_csv('stk_return_train.csv')

    return stk_return

# 獲取 T1 到 T2時間段，預測池子里面所有股票的周其收益率
def get_stock_return_test(pool_list, period_handle):
    '''
    輸入： 
        pool_list: 根據預測出來的標的池子, list
        period_handle: 周期管理函數出來的時間集合, dataframe

    輸出:
        格式為dataframe
        columns: T2時間點集合
        index: 股票對應的收益率

    '''
    p = time_handle(start_date, end_date, period, period_pre,index)

    # 獲取時間表格，剔除T1、T2和Replication列
    stk_return = p.copy()


    # 收益率計算
    for j in pool_list:
        j_str = str(j)
        p0 = [float(get_price(j,i,i)['close'])for i in p.iloc[:,2]]
        p1 = [float(get_price(j,i,i)['close'])for i in p.iloc[:,3]]
        a = [(m-n)/n for m,n in zip (p1,p0)]
        stk_return[str(j)] = a

    # 刪除T1列，保留T0列 ,並賦值給index     
    del stk_return['T1']
    stk_return.index = stk_return['T0']
    del stk_return['T0']
    del stk_return['T2']
    del stk_return['Replication']

    # 保存文件
    stk_return.to_csv('stk_return_train.csv')

    return stk_return



'''
訓練和回測準備
'''
# 根據預測並提取預測池子函數
def model_pool(stock_cl, num, clf, clf1, test_timing):
    '''
    輸入： 
        stock_cl: 用以訓練的分類數據
        num = 第幾次
        clf： 模型
        test_timing: str化的時間周期

    輸出:
        pool_list: 根據模型出來的股票池子

    '''
    # 獲取T0時間點的因子數據
    data = prepare_data(test_timing['T0'][num], index)

    # 獲取T1時間點的因子數據
    data_test = prepare_data(test_timing['T1'][num],index)

    # 如果 T0 和 T1 兩個時間點出來的因子數據，因為某標的剔除導致index不匹配
    if len(data.index) != len(data_test.index):


        # 取交集作為分類值，訓練還有預測值
        data_adj = data.loc[data.index & data_test.index,:] 

        data_test_adj = data_test.loc[data.index & data_test.index,:]

        stock_cl_adj = stock_cl.loc[data.index & data_test.index,:]

        # 訓練
        X_train = np.array(data_adj)
        Y_train = np.array(stock_cl_adj.T)[num]

        # 算法訓練
        clf.fit(X_train, Y_train)

        # 預測
        X_predict = np.array(data_test_adj) 


        # 根據T0到T1的訓練模型，基於T1的因子數據，做T1-T2的分類結果預測
        Y_predict = clf.predict(X_predict)   

        # 如果用第一種算法預測結果都為0，用第二種算法fit一次
        if all (Y_predict == 0):

            # 算法訓練第二次
            clf1.fit(X_train, Y_train)

            # 預測第二遍
            X_predict = np.array(data_test) 

            Y_predict = clf1.predict(X_predict)



    # 如果匹配則，按照原計劃進行
    else: 


        X_train = np.array(data)

        stock_cl_adj = stock_cl.loc[data.index,:]

        Y_train = np.array(stock_cl_adj.T)[num]

        # 算法訓練
        clf.fit(X_train, Y_train)
        
        
        # 預測
        data_test_adj = data_test.copy()
        
        X_predict = np.array(data_test_adj) 


        # 根據T0到T1的訓練模型，基於T1的因子數據，做T1-T2的分類結果預測
        Y_predict = clf.predict(X_predict)

        # 如果用第一種算法預測結果都為0，用第二種算法fit一次
        if all (Y_predict == 0):

            # 算法訓練第二次
            clf1.fit(X_train, Y_train)

            # 預測第二遍
            X_predict = np.array(data_test_adj) 

            Y_predict = clf1.predict(X_predict)



    # 構建在T1預測T2的分類表格
    s = {'predict_clf': list(Y_predict)}
    s = pd.DataFrame(s)
    s.index = data_test_adj.index

    # 獲取預測池子的平均收益率
    model_predict = []

    pool_list = []

    # 提取預測的分類值為1的標的
    for a in range(len(s)):    
            if s.iloc[a,0] == 1:

                # 捕捉分類為1的index，添加到信的list中
                pool_list.append(str(s.index[a]))

    return pool_list

# 根據模型篩選的股票池子出來的平均收益率
def get_model_return(pool_list, test_timing,num):
    '''
    輸入： 
        pool_list: 篩選出來的股票池子
        test_timing: 周期函數


    輸出:
        model_mean_return: 預測出來標的的平均收益率

    '''

    return_test_list = []
    for j in pool_list:
            j_str = str(j)

            datelist_T1 = test_timing['T1']
            datelist_T2 = test_timing['T2']

            p1 = [float(get_price(j, datelist_T1[num],datelist_T1[num])['close'])]
            p2 = [float(get_price(j, datelist_T2[num],datelist_T2[num])['close'])]
            a = [(m-n)/n for m,n in zip (p2,p1)]
            # 捕捉分類為1的index，添加到信的list中
            return_test_list.append(a[0])

    stk_test = {'pool_list':pool_list, 'return_test_list':return_test_list}
    stk_test = pd.DataFrame(stk_test)
    stk_test.index = stk_test['pool_list']
    del stk_test['pool_list']

    # 收益率獲取
    model_mean_return = stk_test.mean()

    return model_mean_return

# 將預測出來的股票池子轉化為dataframe格式
def predict_pool(pool_list):
    # 字典化為dataframe
    predict_pool ={'predict_pool':pool_list, 'predict_score':1}
    predict_pool = pd.DataFrame(predict_pool)
    predict_pool.index = predict_pool['predict_pool']
    del predict_pool['predict_pool']

    return predict_pool        

# 實際分類池子函數
def actural_pool(act_cl, test_timing):

    '''
    輸入： 
        act_cl: 分類池子
        test_timing: str化的時間周期

    輸出:
        格式為dataframe
        實際分類池子

    '''
    # 提取真實的分類數據
    act_c = act_cl.copy()
    del act_c[(act_c.columns)[0]]
    act_c.columns = test_timing['T2']
    actural_cl = act_c[act_c.columns[num]]

    # 字典化為dataframe
    actural_pool = {'act_pool':actural_cl.index, 'act_score':(actural_cl)}
    actural_pool = pd.DataFrame(actural_pool)

    del actural_pool['act_pool']

    return actural_pool

# 大模型測試函數
# 自動化生成對比收益率函數
def test_return(start_date, end_date, period, period_pre, index, clf, clf1):
    '''
    輸入： 
        index: 指數
        clf： 模型


    輸出:
        格式為dataframe
        index: T2時間節點
        index_return: 大盤在T1-T2的收益
        model_return: 模型在T1-T2的收益
        pool_number: 模型在T1-T2預測池子數量
    '''
    # 獲取時間周期
    p = time_handle(start_date, end_date, period, period_pre,index)
    test_timing = time_fix(p)

    # 輔助
    data = prepare_data(start_date,index)

    # 獲取 T0 到 T1 時間段，指數池子里面所有股票的周期收益率
    stk_return = get_stock_return_train(data,p)

    # 獲取 T0 到 T1 時間段，指數周期收益率
    index_return = get_index_return_train(index,p)

    # 獲取用來訓練的分類數據
    stock_cl = train_classification(stk_return, index_return)

    # 回測分類，實際的分類情況獲取
    act_cl = actural_classification(p, stock_cl)

    # 剔除最後一列（因為p的最後一行的T2是nan值）
    del stock_cl[stock_cl.columns[-1]]

    # 獲取T1 到 T2 時間段， 指數周期的收益率
    index_return_test =  get_index_return_test(index,p)


    # 模型收益率提取
    predict_return = []

    # 每一次預測池子標的數量提取
    pool_num = []

    for num in range(len(test_timing['T0'])):

        print('正在進行第'+str(num+1)+'次訓練、預測')

        # 獲取預測的pool
        pool = model_pool(stock_cl,num,clf, clf1, test_timing)

        # 根據預測的pool獲取周期均收益率
        model_mean_return = get_model_return(pool, test_timing, num)

        # 把均收益率放到predict_return這個list中
        predict_return.append(model_mean_return)

        # 記錄預測池子數量
        pool_num.append(len(pool))

    # 對比預測池子周期均收益率和大盤周期均收益率
    compare = index_return_test.copy()

    cp_return = pd.DataFrame(predict_return)
    cp_return.index = compare.index
    cp_return['index_return'] = compare['index_return']
    cp_return['model_return'] = cp_return['return_test_list']
    del cp_return['return_test_list']
    cp_return['premium'] = cp_return['model_return'] - cp_return['index_return']

    # 記錄預測池子數量
    cp_return['pool_number'] = pool_num

    print('所有周期數據訓練、預測已經完成')


    return cp_return 

# 結果可視化
def visualization(df):
    '''
    輸入： 
        df: 自動化生成對比收益率表格


    輸出:
        可視化結果

    '''
    del df['pool_number'] 
    
    df.columns = ['指數大盤收益率','模型收益率','模型超額收益率']
    df.plot(figsize=(15,8),title= "模型收益率（折線圖）")
    df.plot.bar(figsize=(15,8),title= "模型收益率（柱狀圖）")
        
    # 跑贏or輸結果
    # 記錄對比大盤結果的次數
    count1 = []
    for j in range(len(df)):
        if df.iloc[j,2] < 0:
            count1.append(j)
    
    count2 = []
    for j in range(len(df)):
        if df.iloc[j,2] > 0:
            count2.append(j) 
    
    return df.plot(figsize=(15,8),title= "模型收益率（折線圖）"), df.plot.bar(figsize=(15,8),title= "模型收益率（柱狀圖）"), '總回測次數：', len(df), '跑輸的次數：', len(count1), '跑贏的次數：', len(count2)

# 【因子測試函數】測試框架 v4 
# 制作人：Howard董先森， wechat:dhl19988882，e-mail: 16hldong.stu.edu.cn
# copy並分享的時候記得給小弟一下credit就好~

'''
導入庫
'''
from sklearn import linear_model
regr = linear_model.LinearRegression()

'''
篩選輔助工具函數以及數據準備函數
'''
# 提取超額收益率函數
def premium_dataframe(index_return, stock_return):
    
    # 構建待填補的dataframe，index和inde_return的index相同 
    premium_df = pd.DataFrame(index = index_return.index)

    # 遍曆stk_return的所有標的
    for i in stk_return[stk_return.columns]:
        
        # 列運算，把結果賦值給premium_df
        premium_df[i] = stk_return[i] - index_return['index_return']

    return premium_df

# 已知標的池子，獲取T0-T1周期收益率
def get_group_return(group, test_timing):
    '''
    輸入： 
        group: 標的池子的list
        test_timing: 周期管理函數出來的時間集合, dataframe

    輸出:
        格式為dataframe
        columns:

        T0: T0時間點的集合
        code1: 標的1在T0到T1的收益率
        code2: 標的2在T0到T1的收益率
        以此類推
    '''
    # 獲取時間表格，剔除T1、T2和Replication列
    stk_group_return = test_timing.copy()

    # 收益率計算
    for j in group:
        j_str = str(j)
        p0 = [float(get_price(j,i,i)['close'])for i in test_timing.iloc[:,1]]
        p1 = [float(get_price(j,i,i)['close'])for i in test_timing.iloc[:,2]]
        a = [(m-n)/n for m,n in zip (p1,p0)]
        stk_group_return[str(j)] = a

    # 表格整理
    del stk_group_return['T0']
    del stk_group_return['T1']
    del stk_group_return['T2']
    
    return stk_group_return

# 已知標的池子，獲取T0-T1周期收益率
def get_single_period_group_return(group, test_timing):
    '''
    輸入： 
        group: 標的池子的list
        test_timing: 周期管理函數出來的時間集合, dataframe

    輸出:
        格式為dataframe
        columns: 單個標的的T0-T1周期收益率
        index: 各種標的代碼
    '''
   # 創建字典，代填入單個標的單個周期的收益率
    stk_group_return = {}
    # 創建list，代填入單個標的單個周期的收益率
    stk_return_list = []

    # 收益率計算
    for j in group:
        time_t0 = test_timing.iloc[num,1]
        time_t1 = test_timing.iloc[num,0]
        j_str = str(j)
        p0 = float(get_price(j,time_t0,time_t0)['close']) # 獲得T0時間點的價格
        p1 = float(get_price(j,time_t1,time_t1)['close']) # 獲得T1時間點的價格
        ret = (p1-p0)/p0 # 獲得單個標的在T0-T1時間點的收益率

        stk_return_list.append(ret)

    # 構建表格
    stk_group_return = pd.DataFrame(stk_return_list,index = group)
    stk_group_return[str(test_timing.iloc[num,0])] = stk_group_return[stk_group_return.columns[0]]
    del stk_group_return[stk_group_return.columns[0]]

    return stk_group_return
    

# 提取因子大表格數據函數
def get_big_data(test_timing):
    '''
    輸入：
        test_timing: dataframe格式， 時間周期管理函數
        
    輸出:
        
        big_data_t0: 字典，存放T0所有時間點的因子數據
        big_data_t1: 字典，存放T1所有時間點的因子數據
        
    '''
    # 提取因子大表格數據，放在字典里面
    big_data_t0 = {}
    big_data_t1 = {}


    for i in range(len(test_timing['T0'])):   
        print('提取第'+str(i+1)+'個周期的T0因子數據') 
        t0 = test_timing['T0'][i]
        t1 = test_timing['T1'][i]
        big_data_t0[t0] = prepare_data(t0, index)
        print('提取第'+str(i+1)+'個周期的T1因子數據')  
        big_data_t1[t1] = prepare_data(t1, index)
        i = i+1    


    print('提取成功！周期數為：'+ str(i))
    return big_data_t0,big_data_t1

'''
Normal IC框架
'''
# Normal IC 情況下，因子收益率（回歸系數）、因子收益率的T值、IC值 (原始數據表格獲取)
def get_normal_ic_original_factor_testing_data(stk_return, test_timing, big_data):
    
    '''
    
    輸入：
        stk_return: dataframe格式， 所有標的在所有周期的收益率
        
        test_timing: dataframe格式， 時間周期管理函數
        
        big_data: 兩個dict:
                            big_data_t0: 字典，存放T0所有時間點的因子數據
                            big_data_t1: 字典，存放T1所有時間點的因子數據        
    輸出:
        
        normal_ic_df: dataframe格式， Normal_ic表格
        
        t_df： dataframe格式，因子數值（Barra因子暴露）對標的周期收益率RLM回歸後的T值
        
        param_df: dataframe格式， 因子收益率（回歸系數）表格
           
    '''
    # 因子收益率（回歸系數）、因子收益率序列的T值、IC值 三個表格獲取

    # 周期因子IC提取

    # 收益率數據準備
    stk_return1 = stk_return.T

    # 構建字典，待存放因子IC值
    factor_for_ic = {}

    # 構建字典，待存放因子t_value
    factor_for_t = {}

    # 構建字典，待存放因子parameter
    factor_for_parameter = {}


    # 遍曆所有的T0時間點
    for num in range(len(test_timing['T0'])):

        print('正在進行第'+str(num+1)+'周期因子測試原始數據獲取……')

        # 獲取單周期的因子值

        big_data_t0 = big_data[0]
        big_Data_t1 = big_data[1]
        
        t0 = test_timing['T0'][num]
        t1 = test_timing['T1'][num]
        
        data = big_data_t0[t0]
        data1 = big_data_t1[t1]


        # 調整stk_return的index
        stk_return2 = stk_return1.loc[data.index,:]
        del stk_return2[stk_return2.columns[-1]]
        stk_return2.columns = test_timing['T0']

        # 對於T0有因子值，T1沒有的情況，提取T0和T1兩個index的交集
        data_adj = data.loc[data.index & data1.index,:]

        stk_return2_adj = stk_return2.loc[data_adj.index & stk_return2.index,:]

        # list1，待存放ic
        list1 = []

        # list2, 待存放t—value
        list2 = []

        # list3, 待存放parameter
        list3 = []

        # 求相關系數和T值,對於第i個因子
        for i in range(len(data_adj.columns)):

            IC_X = list(data_adj[data_adj.columns[i]])

            IC_Y = list((stk_return2_adj[stk_return2_adj.columns[num]]))

            # 將單個周期所有因子ic值放到cor中
            cor = (list(corrcoef(IC_X, IC_Y)[0])[-1])

            # 回歸
            rlm_model = sm.RLM(IC_Y , IC_X , M=sm.robust.norms.HuberT()).fit()

            # 存放相關系數cor    
            list1.append(cor)

            # 存放t_value
            list2.append(float(rlm_model.tvalues))

            # 存放parameter
            list3.append(float(rlm_model.params))

        # 相關系數，放到字典里
        factor_for_ic[num] = list1

        # t_value， 放到字典里
        factor_for_t[num] = list2

        # parameter, 放到字典里
        factor_for_parameter[num] =list3

    # 整合
    # Normal ICdf整合
    normal_ic_df = pd.DataFrame(factor_for_ic)
    normal_ic_df.index = data.columns
    normal_ic_df.index.name = 'factor'
    normal_ic_df.columns = test_timing['T0']

    # t值整合
    t_df = pd.DataFrame(factor_for_t)
    t_df.index = data.columns
    t_df.index.name ='factor'
    t_df.columns = test_timing['T0']

    # Barra體系下的因子收益率（回歸系數）整合
    param_df = pd.DataFrame(factor_for_parameter)
    param_df.index = data.columns
    param_df.index.name = 'factor'
    param_df.columns = test_timing['T0']

    print('所有周期因子測試在Normal_IC框架下原始數據獲取完畢！')

    return normal_ic_df, t_df, param_df

# 根據 Normal_IC原始數據，提取各種打分指標
def get_pre_scoring_data_for_normal_ic(normal_ic_og):
    '''
    輸入：
        normal_ic_og: dataframe，原始數據大表格
    
    輸出:
        factor_mean_return: dataframe, 因子年化收益率均值
        ic_indicator_df： dataframe, normal_ic各項指標
        
               
    '''
    # 初始數據準備
   
    t_df = normal_ic_og[1]
    normal_ic_df = normal_ic_og[0]
    param_df = normal_ic_og[2]
    
    # 提取因子收益率年化均值
    factor_mean_list =[]
    factor_return = param_df.T
    # 遍曆每個因子
    for i in factor_return.columns:

        # 計算單個因子所有周期的均值
        mean_return = (factor_return[i].mean())

        # 轉為年化收益率
        annualized_mean_return = (1+mean_return)**12 - 1

        # 計算單個因子所有周期年化均值，放到list中去
        factor_mean_list.append((annualized_mean_return))

    # 轉為dataframe格式
    factor_mean_return = pd.DataFrame(factor_return.columns)
    factor_mean_return['abs_annualized_factor_mean_return'] = factor_mean_list
    factor_mean_return.index = factor_mean_return[factor_mean_return.columns[0]]
    del factor_mean_return[factor_mean_return.columns[0]]
    
    # IC各種指標提取 
    ic = normal_ic_df.T
    # 創建字典，待提取每一個因子的所有ic值序列
    features_ic = {}

    # 創建字典，待填入每一個因子的IC各種指標
    ic_indicator ={}

    # 創建dataframe
    indicator = pd.DataFrame(columns = ic.columns)

    # 提取每個factor的ic值，轉為列表
    for i in ic.columns:

        # 提取每一個因子的所有ic值序列，放到字典中
        features_ic[i] = list(ic[i])

        # 提取空值，放到字典ic_indicator中
        ic_indicator[i] = list(indicator[i])


    # 提取每一個因子的所有indicator
    for i in ic.columns:

        # 計算IC值的均值
        ic_indicator[i].append(mean(features_ic[i]))    

        # 計算IC值的標準差
        ic_indicator[i].append(std(features_ic[i]))

        # IC大於0的比例
        ic_indicator[i].append(sum(pd.Series(features_ic[i])>0)/len(features_ic[i]))

        # IC絕對值大於0.02的比例
        abs_list = []
        for d in ic_indicator[i]:
            abs_list.append(abs(d))

        ic_indicator[i].append(sum(pd.Series(abs_list)>0.02)/len(features_ic[i]))

        # IC絕對值
        ic_indicator[i].append(mean(abs_list)) 

        # IR值
        ic_indicator[i].append(mean(features_ic[i])/std(features_ic[i]))

        # IR絕對值
        ic_indicator[i].append(abs(mean(features_ic[i])/std(features_ic[i])))


    # 整合成表格    
    ic_indicator_df = pd.DataFrame(ic_indicator, index = ['Normal_IC_mean','Normal_IC_std','Normal_IC>0_%','Normal_IC>0.02_%','abs_Normal_IC','Normal_IR','abs_IR']).T

    
    # T值各種指標提取 
    t = t_df.T
    # 創建字典，待提取每一個因子的所有t值序列
    features_t = {}

    # 創建字典，待填入每一個因子的t各種指標
    t_indicator ={}

    # 創建dataframe
    indicator = pd.DataFrame(columns = t.columns)

    # 提取每個factor的ic值，轉為列表
    for i in t.columns:

        # 提取每一個因子的所有ic值序列，放到字典中
        features_t[i] = list(t[i])

        # 提取空值，放到字典ic_indicator中
        t_indicator[i] = list(indicator[i])


    # 提取每一個因子的所有indicator
    for i in t.columns:

        # T值均值
        t_indicator[i].append(mean(features_t[i]))

        # 計算T絕對值的均值
        abs_list = []
        for d in features_t[i]:
            abs_list.append(abs(d))

        t_indicator[i].append(mean(abs_list))    

        # T絕對值大於2的比例 
        t_indicator[i].append(sum(pd.Series(abs_list)> 2 )/len(features_t[i]))


    # 整合成表格    
    t_indicator_df = pd.DataFrame(t_indicator, index = ['T_mean','abs(t_value)_mean','abs(t_value)_>_2_%']).T

    return factor_mean_return, t_indicator_df, ic_indicator_df

# 參照光大的因子初步篩選打分表格
def get_scoring_data_for_normal_ic(pre_scoring_data):
    '''
    輸入：
        pre_scoring_data: dataframe，原始數據大表格處理過後的三個各種打分指標表格
    
    輸出:
         factor_test： dataframe, 因子初步篩選打分表格               
    '''
    # 三個表格整合,參考光大打分表格 
    factor_indicator = pd.concat([pre_scoring_data[0].T,pre_scoring_data[1].T, pre_scoring_data[2].T],axis =0)

    # 參照光大打分框架
    factor_test = pd.DataFrame(index = factor_indicator.columns)
    factor_test['Factor_Mean_Return'] = factor_indicator.T['abs_annualized_factor_mean_return']
    factor_test['Factor_Return_tstat'] = factor_indicator.T['T_mean']
    factor_test['IC_mean'] = factor_indicator.T['Normal_IC_mean']
    factor_test['IR'] = factor_indicator.T['Normal_IR']


    # 保存和讀取
    factor_test.to_csv('normal_ic_test.csv')
    factor_test = pd.read_csv('normal_ic_test.csv')
    factor_test.index = factor_test[factor_test.columns[0]]
    del factor_test[factor_test.columns[0]]
    
    return factor_test

# 單調性檢測以及分組回溯函數提取
def get_monotony_and_grouping_return_for_normal_ic(big_data):
    '''
    輸入： 
        big_data: 兩個dict:
                    big_data_t0: 字典，存放T0所有時間點的因子數據
                    big_data_t1: 字典，存放T1所有時間點的因子數據    

    輸出:
        monotony_outcome： dict, key是每個周期的T0時間點
                                 values是指單個周期的所有因子的單調性指標，格式為list
                                 
        group_return_dict: dict, key是每個周期的T0時間點
                                 values是指單個周期所有因子，每個因子按照數值大小排列後，提取對標的的當期收益率進行分組提取，格式為dataframe
        
        
    '''
    # 輔助
    data = big_data_t0[test_timing['T0'][0]]

    print('總的單調性數據提取周期為：'+str(len(test_timing.index)))
    print('總的因子數量為：'+str(len(data.columns)))

    # 創建字典，填入所有因子的單調性和分層回溯周期收益率
    monotony_outcome ={}

    group_return_dict = {}

    # 分組數量
    group_num = 5


    # 遍曆所有T0
    for i in range(len(test_timing['T0'])):

        print('--------第'+str(i+1)+'個周期多空提取中......')
        print('-----第'+str(i+1)+'個周期因子數據表格提取中......')

        # 獲取單周期的因子值
        data = big_data_t0[test_timing['T0'][i]]
        data1 = big_data_t1[test_timing['T1'][i]]

        # 對於T0有因子值，T1沒有的情況，提取T0和T1兩個index的交集
        data_adj = data.loc[data.index & data1.index,:]

        # 創建字典，待填入單個周期，單個因子的單調性指標
        monotony = {}

        # 分層回溯
        group_return = {'group1':[],'group2':[],'group3':[],'group4':[],'group5':[]}

        # 遍曆所有因子
        for num in range(len(data_adj.columns)):

            factor_list = list(data_adj.columns)
            factor = factor_list[num]

            print('---提取'+str(factor)+'因子')

            # 提取單個因子
            factor_test = data_adj[data_adj.columns[num]]

            # 排序分層提取(降序)
            sort_df = factor_test.sort_values(ascending = False)
            # 每組分的標的數量
            code_num = int(len(factor_test)/group_num)
            # 創建字典pool_dict，用來放置分好組的標的池子
            pool_dict = {}
            # 提取降序後的所有index
            sort_index = list(sort_df.index)
            # 分組開始，取得dict
            a = 0
            for q in range((group_num)):

                # 每組命名
                j = 'group'+str((q+1))

                # 提取
                pool_dict[j] = sort_index[a:code_num]
                a= code_num 
                code_num = code_num + int(len(data_adj)/group_num)

            # 提取標的池子名字
            group_name = list(pool_dict.keys())

            # 創建字典mean_return_dict，用來放置分好組的標的池子的周期算術平均收益率
            mean_return_dict = {}

            # 根據每組的index提取所有周期的累計收益率
            for group in group_name:

                stk_group_return = get_group_return(pool_dict[group], test_timing)

                # 提取一個周期的，選出標的池子的算術平均，放到list中
                mean_return_dict[group] = (list(stk_group_return.mean(axis =1))[i])

            # 轉為dataframe格式
            mean_return_df = pd.DataFrame(mean_return_dict, index =[i])

            # 收益率提取
            R1_m = mean_return_df[mean_return_df.columns[0]]
            R2_m = mean_return_df[mean_return_df.columns[1]]
            R3_m = mean_return_df[mean_return_df.columns[2]]
            R4_m = mean_return_df[mean_return_df.columns[3]]
            R5_m = mean_return_df[mean_return_df.columns[4]]

            # 將回溯周期收益率放到字典group_return中
            group_return['group1'].extend(R1_m)
            group_return['group2'].extend(R2_m)
            group_return['group3'].extend(R3_m)
            group_return['group4'].extend(R4_m)
            group_return['group5'].extend(R5_m)

            print("--已經獲取"+factor+"因子第"+str(i+1)+'周期的分組回溯周期收益率')

            # 年化收益率提取
            R1 = (1+R1_m)**12 - 1 
            R2 = (1+R2_m)**12 - 1
            R3 = (1+R3_m)**12 - 1
            R4 = (1+R4_m)**12 - 1
            R5 = (1+R5_m)**12 - 1

            df = pd.DataFrame(index = mean_return_df.index)
            df['R1-R5'] = R1 - R5
            df['R2-R4'] = R2 - R4
            df['Monotony'] = df['R1-R5']/df['R2-R4']

            # 將單調性指標放到字典monotony字典中
            monotony[factor] = float(df['Monotony'])
            print("-已經獲取"+factor+"因子單調性指標")



        # 轉為 dataframe
        monotony_df = pd.DataFrame(monotony,index=[0])
        group_return_df = pd.DataFrame(group_return,index=data.columns)


        # 將單個周期的所有因子單調性指標轉為 list,並放在字典monotony_outcome中    
        monotony_outcome[test_timing['T0'][i]] = list((monotony_df.T)[monotony_df.T.columns[0]])

        # 將單周期的所有因子分組回溯周期收益率，放到字典group_return_dict中
        group_return_dict[test_timing['T0'][i]] = group_return_df

    print("所有因子所有周期的單調性指標已經獲取完畢！")

    return monotony_outcome, group_return_dict

'''
Rank IC框架
'''
# Rank框架下，因子收益率（回歸系數）、因子收益率序列的T值、IC值 三個表格獲取 
def get_rank_ic_original_factor_testing_data(stk_return, test_timing, big_data):
    
    '''
    輸入：
        stk_return: dataframe格式， 所有標的在所有周期的收益率
        
        test_timing: dataframe格式， 時間周期管理函數
        
        big_data: 兩個dict:
                            big_data_t0: 字典，存放T0所有時間點的因子數據
                            big_data_t1: 字典，存放T1所有時間點的因子數據        
    輸出:
        
        rank_ic_df: dataframe格式， Normal_ic表格
        
        rank_t_df： dataframe格式，因子數值（Barra因子暴露）對標的周期收益率RLM回歸後的T值
        
        rank_param_df: dataframe格式， 因子收益率（回歸系數）表格
           
    '''
    # 收益率數據準備
    stk_return1 = stk_return.T

    # 構建字典，待存放因子Rank_IC值
    factor_for_ic = {}

    # 構建字典，待存放因子t_value
    factor_for_t = {}

    # 構建字典，待存放因子parameter
    factor_for_parameter = {}


    # 遍曆所有的T0時間點
    for num in range(len(test_timing['T0'])):

        print('正在進行第'+str(num+1)+'周期因子測試原始數據獲取……')

        # 獲取單周期的因子值
        data = big_data_t0[test_timing['T0'][num]]
        data1 = big_data_t1[test_timing['T1'][num]]

        # 對於T0有因子值，T1沒有的情況，提取T0和T1兩個index的交集
        data_adj = data.loc[data.index & data1.index,:]

        # list1，待存放ic
        list1 = []

        # list2, 待存放t—value
        list2 = []

        # list3, 待存放parameter
        list3 = []

        # 求相關系數和T值,對於第i個因子
        for i in range(len(data_adj.columns)):

            # 提取單個因子
            factor_test = pd.DataFrame(data_adj[data_adj.columns[i]])

            # 提取 X
            factor_test['X_Rank'] = factor_test.rank(ascending = True, method = 'dense')

            X = list(factor_test['X_Rank']) 

            # 提取 Y 
            # 提取收益率
            stk_group_return =  get_single_period_group_return(list(factor_test.index), test_timing)

            Y = list(stk_group_return[test_timing['T0'][num]])

            # 將單個周期所有因子ic值放到cor中
            cor = (list(corrcoef(X, Y)[0])[-1])

            # 回歸
            rlm_model = sm.RLM(Y , X , M=sm.robust.norms.HuberT()).fit()

            # 存放相關系數cor    
            list1.append(cor)

            # 存放t_value
            list2.append(float(rlm_model.tvalues))

            # 存放parameter
            list3.append(float(rlm_model.params))

        # 相關系數，放到字典里
        factor_for_ic[num] = list1

        # t_value， 放到字典里
        factor_for_t[num] = list2

        # parameter, 放到字典里
        factor_for_parameter[num] =list3

    # 整合
    # Normal ICdf整合
    rank_ic_df = pd.DataFrame(factor_for_ic)
    rank_ic_df.index = data.columns
    rank_ic_df.index.name = 'factor'
    rank_ic_df.columns = test_timing['T0']

    # t值整合
    rank_t_df = pd.DataFrame(factor_for_t)
    rank_t_df.index = data.columns
    rank_t_df.index.name ='factor'
    rank_t_df.columns = test_timing['T0']

    # Barra體系下的因子收益率（回歸系數）整合
    rank_param_df = pd.DataFrame(factor_for_parameter)
    rank_param_df.index = data.columns
    rank_param_df.index.name = 'factor'
    rank_param_df.columns = test_timing['T0']

    print('所有周期Rank_IC框架下，因子測試原始數據獲取完畢！')
    
    return rank_ic_df, rank_t_df, rank_param_df

# 根據 Rank_IC原始數據，提取各種打分指標
def get_pre_scoring_data_for_rank_ic(rank_ic_og):
    '''
    輸入：
        rank_ic_og: dataframe，原始數據大表格
    
    輸出:
        factor_mean_return: dataframe, 因子年化收益率均值
        ic_indicator_df： dataframe, normal_ic各項指標
        
               
    '''
    # 初始數據準備
    normal_ic_df = rank_ic_og[0]
    t_df = rank_ic_og[1]
    param_df = rank_ic_og[2]
    
    # 提取因子收益率年化均值
    factor_mean_list =[]
    factor_return = param_df.T
    # 遍曆每個因子
    for i in factor_return.columns:

        # 計算單個因子所有周期的均值
        mean_return = (factor_return[i].mean())

        # 轉為年化收益率
        annualized_mean_return = (1+mean_return)**12 - 1

        # 計算單個因子所有周期年化均值，放到list中去
        factor_mean_list.append((annualized_mean_return))

    # 轉為dataframe格式
    factor_mean_return = pd.DataFrame(factor_return.columns)
    factor_mean_return['abs_annualized_factor_mean_return'] = factor_mean_list
    factor_mean_return.index = factor_mean_return[factor_mean_return.columns[0]]
    del factor_mean_return[factor_mean_return.columns[0]]
    
    # IC各種指標提取 
    ic = normal_ic_df.T
    # 創建字典，待提取每一個因子的所有ic值序列
    features_ic = {}

    # 創建字典，待填入每一個因子的IC各種指標
    ic_indicator ={}

    # 創建dataframe
    indicator = pd.DataFrame(columns = ic.columns)

    # 提取每個factor的ic值，轉為列表
    for i in ic.columns:

        # 提取每一個因子的所有ic值序列，放到字典中
        features_ic[i] = list(ic[i])

        # 提取空值，放到字典ic_indicator中
        ic_indicator[i] = list(indicator[i])


    # 提取每一個因子的所有indicator
    for i in ic.columns:

        # 計算IC值的均值
        ic_indicator[i].append(mean(features_ic[i]))    

        # 計算IC值的標準差
        ic_indicator[i].append(std(features_ic[i]))

        # IC大於0的比例
        ic_indicator[i].append(sum(pd.Series(features_ic[i])>0)/len(features_ic[i]))

        # IC絕對值大於0.02的比例
        abs_list = []
        for d in ic_indicator[i]:
            abs_list.append(abs(d))

        ic_indicator[i].append(sum(pd.Series(abs_list)>0.02)/len(features_ic[i]))

        # IC絕對值
        ic_indicator[i].append(mean(abs_list)) 

        # IR值
        ic_indicator[i].append(mean(features_ic[i])/std(features_ic[i]))

        # IR絕對值
        ic_indicator[i].append(abs(mean(features_ic[i])/std(features_ic[i])))


    # 整合成表格    
    ic_indicator_df = pd.DataFrame(ic_indicator, index = ['Normal_IC_mean','Normal_IC_std','Normal_IC>0_%','Normal_IC>0.02_%','abs_Normal_IC','Normal_IR','abs_IR']).T

    
    # T值各種指標提取 
    t = t_df.T
    # 創建字典，待提取每一個因子的所有t值序列
    features_t = {}

    # 創建字典，待填入每一個因子的t各種指標
    t_indicator ={}

    # 創建dataframe
    indicator = pd.DataFrame(columns = t.columns)

    # 提取每個factor的ic值，轉為列表
    for i in t.columns:

        # 提取每一個因子的所有ic值序列，放到字典中
        features_t[i] = list(t[i])

        # 提取空值，放到字典ic_indicator中
        t_indicator[i] = list(indicator[i])


    # 提取每一個因子的所有indicator
    for i in t.columns:

        # T值均值
        t_indicator[i].append(mean(features_t[i]))

        # 計算T絕對值的均值
        abs_list = []
        for d in features_t[i]:
            abs_list.append(abs(d))

        t_indicator[i].append(mean(abs_list))    

        # T絕對值大於2的比例 
        t_indicator[i].append(sum(pd.Series(abs_list)> 2 )/len(features_t[i]))


    # 整合成表格    
    t_indicator_df = pd.DataFrame(t_indicator, index = ['T_mean','abs(t_value)_mean','abs(t_value)_>_2_%']).T

    return factor_mean_return, t_indicator_df, ic_indicator_df

# 參照光大的因子初步篩選打分表格
def get_scoring_data_for_rank_ic(pre_scoring_data_for_rank_ic):
    '''
    輸入：
        pre_scoring_data: dataframe，原始數據大表格處理過後的三個各種打分指標表格
    
    輸出:
         factor_test： dataframe, 因子初步篩選打分表格               
    '''
    # 三個表格整合,參考光大打分表格 
    
    pre_scoring_data = pre_scoring_data_for_rank_ic
    
    factor_indicator = pd.concat([pre_scoring_data[0].T,pre_scoring_data[1].T, pre_scoring_data[2].T],axis =0)

    # 參照光大打分框架
    factor_test = pd.DataFrame(index = factor_indicator.columns)
    factor_test['Factor_Mean_Return'] = factor_indicator.T['abs_annualized_factor_mean_return']
    factor_test['Factor_Return_tstat'] = factor_indicator.T['T_mean']
    factor_test['IC_mean'] = factor_indicator.T['Normal_IC_mean']
    factor_test['IR'] = factor_indicator.T['Normal_IR']


    # 保存和讀取
    factor_test.to_csv('rank_ic_test.csv')
    factor_test = pd.read_csv('rank_ic_test.csv')
    factor_test.index = factor_test[factor_test.columns[0]]
    del factor_test[factor_test.columns[0]]
    
    factor_test.columns = ['Factor_Mean_Return', 'Factor_Return_tstat', 'Rank_IC', 'Rank_IR']
    
    return factor_test


'''
打分模型
'''
# 打分模型(rank_ir)
def get_score_data_ir(scoring_rank_df):
    '''
    輸入：
        scoring_rank_df： dataframe格式，打分前的數據
       
    輸出:
        
        score_result: dataframe格式，最終打分表格
           
    '''
    a = scoring_rank_df.T
    a=a.T

    # 構建，待填補每個因子的總得分
    total_factor_score = {}

    for i in range(len(a)):


        # 構建list，待填補單個因子的得分
        score_list = []

        factor_return = a.iloc[i,0]
        factor_t = a.iloc[i,1]
        factor_ic = a.iloc[i,2]
        factor_ir = a.iloc[i,3]

        if abs(factor_return) > 0.002:

            score_list.append(1)

        else: 

            score_list.append(0)

        if abs(factor_t) > 2:

            score_list.append(1)

        else: 

            score_list.append(0)

        if abs(factor_ic) > 0.02:

            score_list.append(1)

        else: 

            score_list.append(0)

        if abs(factor_ir) > 0.2:

            score_list.append(1)

        else:

            score_list.append(0)


        total_factor_score[str(a.index[i])] = score_list

    # 轉為dataframe
    score_outcome = pd.DataFrame(total_factor_score, index = a.columns)
    score = score_outcome.T
    score['Total_score'] = score_outcome.sum()
    a['Total_score'] = score['Total_score']
    score_result = a.sort_values(by='Total_score',ascending = False)

    return score_result

'''
分層回溯模型
'''
# 提取每個因子的年化均收益率表格
def get_annualized_factor_return(gr):
    '''
    輸入：
        gr: dict：
                    key為時間周期
                    values為每個周期的所有因子的組合收益率
        
    輸出:
        
        annual_df： dataframe， 所有因子在所有周期的年化收益率
        
    '''
    # 提取keys
    key = list(gr.keys())
    
    
    # 多空組合提取
    for k in list(key):
    
        # 單個多空組合因子收益率提取
        gr[k]['long_short'] = gr[k]['group5']-gr[k]['group1']
    
    # 創建取值為0的表格，待累加
    sum_gr = pd.DataFrame(index = gr[key[2]].index, columns = gr[key[2]].columns)
    sum_gr = sum_gr.fillna(0)

    # 遍曆 key
    for k in list(key):

        # 累加
        sum_gr = sum_gr + gr[k] 

    # 平均值
    mean_gr = sum_gr/len(key)

    # 年化
    annualized_mean = {}

    for i in range(len(mean_gr.index)):

        factor = mean_gr.index[i]

        annualized_return = []

        for j in range(len(mean_gr.columns)):

            # 年化
            annualized_return.append((mean_gr.iloc[i,j] + 1)**12 -1)

        annualized_mean[factor] = annualized_return

    # 轉為dataframe
    annual_df = pd.DataFrame(annualized_mean, index = mean_gr.columns).T
    
    return annual_df
    
# 提取每個因子年化收益率的大表格
def get_annualized_grouping_return(gr):
    '''
    輸入：
        gr: dict：
                    key為時間周期
                    values為每個周期的所有因子的組合收益率
        
    輸出:
        
        factor_dict: 
                    key為因子名字
                    values為單個因子所有周期的組合收益率
        
    '''
    # 提取單個因子的多空組合累計收益率，可視化

    # 創建字典group_dict, keys為因子名字，values為單個因子所有周期內的，分組收益
    group_dict = {}


    # 提取keys
    key = list(gr.keys())

    # 多空組合提取
    for k in list(key):

        # 單個多空組合因子收益率提取
        gr[k]['long_short'] = gr[k]['group5']-gr[k]['group1']


    # 重新提取factor字典

    factor_dict= {}

    factor_list = list(gr[list(gr.keys())[0]].index)

    for factor in factor_list:

        # 創建分組收益率字典，key為時間，values為分組收益
        factor_return_dict = {}

        # 遍曆分組收益率
        for k in list(gr.keys()):

            factor_return_dict[k] = list(gr[k].ix[factor])

        # 轉為dataframe，作為單個因子，所有周期的分組收益率
        return_df = pd.DataFrame(factor_return_dict).T
        return_df.columns = gr[k].ix[factor].index

        # 年化
        annualized = {}

        for i in range(len(return_df.index)):

            factor_time = return_df.index[i]

            annualized_return = []

            for j in range(len(return_df.columns)):

                # 年化
                annualized_return.append((return_df.iloc[i,j] + 1)**12 -1)

            annualized[factor_time] = annualized_return

        # 轉為dataframe，作為單個因子，所有周期的分組收益率
        return_df = pd.DataFrame(annualized).T
        return_df.columns = gr[k].ix[factor].index

        # 將單個因子所有周期的分組收益率表格，存放到字典中
        factor_dict[factor] = return_df
    return factor_dict

# 提取每個因子周期收益率的大表格
def get_grouping_return(gr):
    '''
    輸入：
        gr: dict：
                    key為時間周期
                    values為每個周期的所有因子的組合收益率
        
    輸出:
        
        factor_dict: 
                    key為因子名字
                    values為單個因子所有周期的組合收益率
        
    '''
    # 提取單個因子的多空組合累計收益率，可視化

    # 創建字典group_dict, keys為因子名字，values為單個因子所有周期內的，分組收益
    group_dict = {}


    # 提取keys
    key = list(gr.keys())

    # 多空組合提取
    for k in list(key):

        # 單個多空組合因子收益率提取
        gr[k]['long_short'] = gr[k]['group5']-gr[k]['group1']


    # 重新提取factor字典

    factor_dict= {}

    factor_list = list(gr[list(gr.keys())[0]].index)

    for factor in factor_list:

        # 創建分組收益率字典，key為時間，values為分組收益
        factor_return_dict = {}

        # 遍曆分組收益率
        for k in list(gr.keys()):

            factor_return_dict[k] = list(gr[k].ix[factor])

        # 轉為dataframe，作為單個因子，所有周期的分組收益率
        return_df = pd.DataFrame(factor_return_dict).T
        return_df.columns = gr[k].ix[factor].index

        # 將單個因子所有周期的分組收益率表格，存放到字典中
        factor_dict[factor] = return_df
    return factor_dict