# 导包
import pandas as pd
from pandas import Series, DataFrame
import numpy as np
import statsmodels.api as sm
import scipy.stats as scs
import matplotlib.pyplot as plt


from datetime import timedelta
from six import StringIO

import pickle

from jqdata import *
import time as t

# 得到因子数据
def get_factors(fdate,factors):
    #stock_set = get_index_stocks('000300.XSHG',fdate)
    q = query(
        valuation.code, # 股票代码
        valuation.circulating_market_cap, # CMC 流通市值
        valuation.market_cap, # MC 总市值 
        valuation.circulating_market_cap/valuation.capitalization*10000, # CMC/C 流通市值（亿）/总股本(万) （收盘价） 
        balance.total_owner_equities/valuation.market_cap/100000000, # TOE/MC 每元所有者权益
        valuation.pb_ratio, # PB 市净率
        income.net_profit/valuation.market_cap/100000000, # NP/MC 每元所有者净利润
        income.total_profit/valuation.market_cap/100000000, # TP/MC 每元利润总额
        balance.total_assets/valuation.market_cap/100000000, # TA/MC 每元资产总额
        income.operating_profit/valuation.market_cap/100000000, # OP/MC 每元营业利润
        balance.capital_reserve_fund/valuation.market_cap/100000000, # CRF/MC 每元资本公积
        valuation.ps_ratio, # PS 市销率
        income.operating_revenue/valuation.market_cap/100000000, # OR/MC 每元营业收入
        balance.retained_profit/valuation.market_cap/100000000, # RP/MC 每元未分配利润
        balance.total_liability/balance.total_sheet_owner_equities,# TL/TA 资产负债率
        
        balance.total_current_assets/balance.total_current_liability, # TCA/TCL  流动比率
        valuation.pe_ratio, # PE 市盈率
        income.operating_revenue*indicator.roa/income.net_profit, # OR*ROA/NP  总资产周转率        
        indicator.gross_profit_margin, # GPM 销售毛利率
        
        indicator.inc_revenue_year_on_year, # IRYOY  营业收入同比增长率(%) 
        indicator.inc_revenue_annual, # IRA 营业收入环比增长率(%)
        indicator.inc_net_profit_year_on_year, # INPYOY 净利润同比增长率(%)
        indicator.inc_net_profit_annual, # INPA  净利润环比增长率(%) 
        indicator.net_profit_margin, # NPM  销售净利率(%) 
        indicator.operation_profit_to_total_revenue, # OPTTR  营业利润/营业总收入(%) 
        valuation.capitalization,# C 总股本
        valuation.circulating_cap, # CC 流通股本(万股)
        valuation.pcf_ratio, # PR 市现率
        valuation.pe_ratio_lyr, # PRL 市盈率LYR
        indicator.roe, # ROE  净资产收益率ROE(%) 
        indicator.roa, # ROA  总资产净利率ROA(%) 
        
        indicator.eps, # EPS 每股盈余
        
        # ROIC
        # EBIT = 净利润 + 利息 + 税
        # ROIC
        (income.net_profit+income.financial_expense+income.income_tax_expense)/(balance.total_owner_equities+balance.shortterm_loan+balance.non_current_liability_in_one_year+balance.longterm_loan+balance.bonds_payable+balance.longterm_account_payable),
        
        # 资源占用情况 = 占用资金 / 投入资本
        # 占用资金 = （应付账款accounts_payable+预收款项advance_peceipts+其他应付款other_payable）-（应收账款account_receivable+预付款项advance_payment+其他应收款other_receivable）
        # 投入资本 = 股东权益 + 有息负债
        # 有息负债 = 短期借款shortterm_loan + 1年内到期的长期负债non_current_liability_in_one_year + 长期借款longterm_loan + 应付债券bonds_payable + 长期应付款longterm_account_payable
        (balance.accounts_payable+balance.advance_peceipts+balance.other_payable-balance.account_receivable-balance.advance_payment-balance.other_receivable)/(balance.total_owner_equities+balance.shortterm_loan+balance.non_current_liability_in_one_year+balance.longterm_loan+balance.bonds_payable+balance.longterm_account_payable)
        
        
        ).filter(
        #valuation.code.in_(stock_set),
        valuation.circulating_market_cap
    )
    fdf = get_fundamentals(q, date=fdate)
    fdf.index = fdf['code']
    fdf.columns = ['code'] + factors
    # 行：选择全部，列，返回除了股票代码所有因子
    return fdf.iloc[:,1:]

# 计算股票回报
# 这里的想法很妙，通过对比当前时间的流通市值和下一个时间的流通市值，计算得到涨幅
def caculate_port_return(port,startdate,enddate,CMC):
    close1 = get_price(port, startdate, startdate, 'daily', ['close'])
    close2 = get_price(port, enddate, enddate, 'daily',['close'])
    # 个股涨跌幅*流通市值，得到流通市值涨跌额
    # 所有股票流通市值涨跌额加总，得到流通市值总涨跌额
    # 流通市值总涨跌额 和 之前的流通市值之比，得到流通市值涨跌幅度
    weighted_m_return = ((close2['close'].ix[0,:]/close1['close'].ix[0,:]-1)*CMC).sum()/(CMC.ix[port].sum()) 
    return weighted_m_return

# 计算基准回报
def caculate_benchmark_return(startdate,enddate):
    close1 = get_price(['000001.XSHG'],startdate,startdate,'daily',['close'])['close']
    close2 = get_price(['000001.XSHG'],enddate, enddate, 'daily',['close'])['close']
    benchmark_return = (close2.ix[0,:]/close1.ix[0,:]-1).sum()
    return benchmark_return

# 计算收益并形成dict
def get_return_values(date_list, factors, values_return_dict_history):
    try:
        date_history = values_return_dict_history.keys()
        result = values_return_dict_history
    except:
        date_history = []
        result = {} 
        
    date_temp_list = [ i for i in date_list[:-1] if i not in date_history ]
    print date_temp_list
    
    if len(date_temp_list) == 0 :
        return values_return_dict_history
    
    # 只要date_temp_list在date_list中有上一个，就增加
    for date in date_temp_list:
        if date_list.index(date) > 0 :
            date_add = date_list[date_list.index(date)-1]
            date_temp_list = [date_add] + date_temp_list
            
    # 如果date_temp_list的最后一个在data_list中是倒数第二个，则去掉
    if  date_list.index(date_temp_list[-1]) == len(date_list) - 2 :
        date_temp_list.remove(date_temp_list[-1])
        
    
    for date in date_temp_list:
        startdate = date_list[date_list.index(date)]
        enddate = date_list[date_list.index(date) + 1]
        
        # 如果已经存在，则不计算
        if enddate in result.keys():
            continue
            
        print "回测日期: %s" % enddate
        
        # 因子列表
        fdf = get_factors(startdate,factors)

        # 流通市值
        CMC = fdf['CMC']

        # 5个组合
        df = DataFrame(np.zeros(6*len(factors)).reshape(6,len(factors)),index = ['port1','port2','port3','port4','port5','benchmark'],columns = factors)

        for fac in factors:
            # 根据因子升序排序
            score = fdf[fac].order()

            # 将股票划分为5部分
            port1 = score.index.tolist()[: len(score)/5]
            port2 = score.index.tolist()[ len(score)/5+1: 2*len(score)/5]
            port3 = score.index.tolist()[ 2*len(score)/5+1: -2*len(score)/5]
            port4 = score.index.tolist()[ -2*len(score)/5+1: -len(score)/5]
            port5 = score.index.tolist()[ -len(score)/5+1: ]
            #port1 = list(score.index)[: len(score)/5]
            #port2 = list(score.index)[ len(score)/5+1: 2*len(score)/5]
            #port3 = list(score.index)[ 2*len(score)/5+1: -2*len(score)/5]
            #port4 = list(score.index)[ -2*len(score)/5+1: -len(score)/5]
            #port5 = list(score.index)[ -len(score)/5+1: ]

            # 获得每一部分的收益
            df.ix['port1',fac] = caculate_port_return(port1,startdate,enddate,CMC)
            df.ix['port2',fac] = caculate_port_return(port2,startdate,enddate,CMC)
            df.ix['port3',fac] = caculate_port_return(port3,startdate,enddate,CMC)
            df.ix['port4',fac] = caculate_port_return(port4,startdate,enddate,CMC)
            df.ix['port5',fac] = caculate_port_return(port5,startdate,enddate,CMC)

            # 获得指数的收益
            df.ix['benchmark',fac] = caculate_benchmark_return(startdate,enddate)

        # 赋值给字典
        result[enddate]=df

    return result

# 因子有效性检验
def effect_test(values_return, factors):
    total_return = {} # 总回报
    annual_return = {} # 年化符合收益
    excess_return = {} # 超额收益
    win_prob = {} # 胜率
    loss_prob = {} # 负率
    effect_test = {} # 有效性检验

    MinCorr = 0.3 # 最小相关阀值
    Minbottom = -0.05 # 最小超额亏损
    Mintop = 0.05 # 最小超额收益
    for fac in factors:
        effect_test[fac] = {} # 每个因子的有效性检验建立字典
        daily = values_return[:,:,fac] # 获得各个月回报
        #print daily
        #print "fac ：%s" % fac

        total_return[fac] = (daily+1).T.cumprod().iloc[-1,:]-1 # 总收益
        annual_return[fac] = total_return[fac] # 计算一年
        excess_return[fac] = annual_return[fac]- annual_return[fac][-1] # 减去指数收益，获得超额收益
        #判断因子有效性
        #1.年化收益与组合序列的相关性 大于 阀值
        # 一找你的有效性检验字典-收益相关性
        effect_test[fac]['corr'] = annual_return[fac][0:5].corr(Series([1,2,3,4,5],index = annual_return[fac][0:5].index))

        #2.高收益组合跑赢概率
        #因子小，收益小，port1是输家组合，port5是赢家组合
        if total_return[fac][0] < total_return[fac][-2]:
            loss_excess = daily.iloc[0,:]-daily.iloc[-1,:] # 相对指数，每个月的超额损失
            loss_prob[fac] = loss_excess[loss_excess<0].count()/float(len(loss_excess)) # 出现超额损失的概率
            win_excess = daily.iloc[-2,:]-daily.iloc[-1,:] # 相对指数，每个月的超额收益
            win_prob[fac] = win_excess[win_excess>0].count()/float(len(win_excess)) # 出现超额收益的概率

            # 因子的有效性检验字典-胜负率
            effect_test[fac]['prob_win'] = win_prob[fac]
            effect_test[fac]['prob_lose'] = loss_prob[fac]

            # 超额收益
            # 因子的有效性检验字典-赢家组合的年化收益，输家组合的年化收益
            effect_test[fac]["excess_return_win"] = excess_return[fac][-2]*100
            effect_test[fac]["excess_return_lose"] = excess_return[fac][0]*100

        #因子小，收益大，port1是赢家组合，port5是输家组合
        else:
            loss_excess = daily.iloc[-2,:]-daily.iloc[-1,:] # 相对指数，每个月的超额损失
            loss_prob[fac] = loss_excess[loss_excess<0].count()/float(len(loss_excess)) # 出现超额损失的概率
            win_excess = daily.iloc[0,:]-daily.iloc[-1,:] # 相对指数，每个月的超额收益
            win_prob[fac] = win_excess[win_excess>0].count()/float(len(win_excess)) # 出现超额收益的概率

            # 因子的有效性检验字典-胜负率
            effect_test[fac]['prob_win'] = win_prob[fac]
            effect_test[fac]['prob_lose'] = loss_prob[fac]

            #超额收益
            # 因子的有效性检验字典-赢家组合的年化收益，输家组合的年化收益
            effect_test[fac]["excess_return_win"] = excess_return[fac][0]*100
            effect_test[fac]["excess_return_lose"] = excess_return[fac][-2]*100

    #effect_test[1]记录因子相关性，>0.5或<-0.5合格
    #effect_test[2]记录【赢家组合超额收益，输家组合超额收益】
    #effect_test[3]记录赢家组合跑赢概率和输家组合跑输概率。【>0.5,>0.4】合格(因实际情况，跑输概率暂时不考虑)
    effect_test_df = DataFrame(effect_test)
    effect_test_df_T = effect_test_df.T

    # 条件1 相关性绝对值大于0.5,值越大越有效
    #effect_test_df_T = effect_test_df_T[abs(effect_test_df_T['corr']) > 0.5]

    # 条件3 胜率大于0.5，胜率越大效果越好
    #effect_test_df_T = effect_test_df_T[effect_test_df_T['prob_win'] > 0.4]

    return effect_test_df_T

# 计算得分
def caculate_score(scores_return_panel, date_score, effect_test_df_T, effective_factors):
    # 有效因子年，半年，季，月，周 回报结果
    timesOfReturn = ['year','halfyear','season','month','week']#,'day']
    score_df = DataFrame(np.zeros(len(effective_factors)*len(timesOfReturn)).reshape(len(effective_factors),len(timesOfReturn)),index = effective_factors,columns = timesOfReturn)

    for fac in effective_factors:
        # 相关度为负数，因子小收益大；相关度为正数，因子大收益大
        if effect_test_df_T.ix[fac,"corr"] < 0:
            strCorr = "port1"
        else:
            strCorr = "port5"
            
        returns = []
        date_list = scores_return_panel.items.tolist()
        # 从需要计分的开始日期，一直到最后一天，累积连乘得到这一段时间的收益
        for date in date_score[:-1]:
            i = date_list.index(date)+1
            returns.append((scores_return_panel[date_list[i:],strCorr,fac].T+1).cumprod()[scores_return_panel.items.tolist()[-1]]-1)
            
        score_df.ix[fac] = returns

    # 因子回报排名
    for column in timesOfReturn:
        score_df = score_df.sort(columns=[column], ascending=[False])
        score_df.reset_index(range(1,len(score_df) + 1), inplace = True)
        score_df.index = score_df.index + 1
        score_df[column + '_rank'] = score_df.index
        if column == 'year':
            score_df.rename(columns={"index":"Factors"}, inplace = True)
        else:
            score_df.drop(['index'], axis=1, inplace=True)
        score_df['year_rank'] = score_df.index

    # 计分
    score_df['score'] = score_df['year_rank']*0.1 + score_df['halfyear_rank']*0.2 +  score_df['season_rank']*0.4 + score_df['month_rank']*0.3 + score_df['week_rank']*0.15 #+ score_df['day_rank']*0.1  
    score_df = score_df.sort(columns=['score'], ascending=[True])   
    
    return score_df

def caculate_score_all(scores_return_panel, date_list, effect_test_df_T, effective_factors):
    timesOfReturn = ['all']
    score_df = DataFrame(np.zeros(len(effective_factors)*len(timesOfReturn)).reshape(len(effective_factors),len(timesOfReturn)),index = effective_factors,columns = timesOfReturn)
    for fac in effective_factors:
        # 相关度为负数，因子小收益大；相关度为正数，因子大收益大
        if effect_test_df_T.ix[fac,"corr"] < 0:
            strCorr = "port1"
        else:
            strCorr = "port5"
        
        returns =[ (scores_return_panel[date_list,strCorr,fac].T+1).cumprod()[scores_return_panel.items.tolist()[-1]]-1 ]
            
        score_df.ix[fac] = returns
        
    score_df = score_df.sort(columns=['all'], ascending=[False])
    
    #print score_df
        
    return score_df

print "开始时间 %s " %  t.strftime("%Y-%m-%d %H:%M:%S",t.localtime())  
startTotal = t.time()

# 今天和一年前的今天
today = pd.datetime.today()
yearBefore = today-timedelta(days=365*13) # 前面12年

#start = yearBefore.strftime('%Y-%m-%d')  # 开始日期
start = '2005-01-05'
end = today.strftime('%Y-%m-%d')  # 截止日期

print "start: %s" % start
print "end: %s" % end

# 获得一年的交易日列表
#trade_days = get_trade_days(start_date=start, end_date=end)
#date_list = trade_days.tolist()
date_df = get_price('000001.XSHG', start_date=start, end_date=end, frequency='daily', fields=['close'], fq=None)
date_list = date_df.index.tolist()


# 因子
factors = ['CMC','MC','CMC/C','TOE/MC','PB','NP/MC','TP/MC','TA/MC','OP/MC','CRF/MC','PS','OR/MC','RP/MC','TL/TA','TCA/TCL','PE','OR*ROA/NP','GPM','IRYOY','IRA','INPYOY','INPA','NPM','OPTTR','C','CC','PR','PRL','ROE','ROA','EPS','ROIC','ZYZY']

# 每月回报
filename = "values_return_dict_history_20170621.pkl"
try:
    # 读取文件
    body = read_file(filename)
    values_return_dict_history = pickle.load(StringIO(body))
    values_return_dict = get_return_values(date_list, factors, values_return_dict_history)
except:
    values_return_dict = get_return_values(date_list, factors, 1)

# 记录历史，方便回测
values_return_dict_history = values_return_dict



# 文件写入
#使用pickle模块从文件中重构python对象
content = pickle.dumps(values_return_dict_history) # 该方法返回字符串
write_file(filename, content, append=False)
print "序列化保存对象：values_return_dict_history"

# 变成panel
values_return_panel = pd.Panel(values_return_dict)[date_list,:,:]

# 结果计算用的panel
scores_return_panel = pd.Panel(values_return_dict_history)[date_list,:,:]

# 因子检验
effect_test_df_T = effect_test(scores_return_panel, factors)
# 有效因子
effective_factors = effect_test_df_T.index.tolist()
# 计算得分
score_df = caculate_score_all(scores_return_panel, date_list, effect_test_df_T, effective_factors)

# 前5个
fac_num = 3
list = score_df.index.tolist()[:(fac_num)]
effect_test_df_T['ascending'] = effect_test_df_T['corr'] < 0

resultFac = effect_test_df_T['ascending'].T[list]


print "本次因子: resultFac %s"% resultFac

    

endTotal = t.time()
print "结束时间 %s " %  t.strftime("%Y-%m-%d %H:%M:%S",t.localtime())  
secondUseTotal = endTotal - startTotal
print "所用时间（秒） %s" % secondUseTotal
print "所用时间（分） %s" % (secondUseTotal / 60 )
print "所用时间（时） %s" % (secondUseTotal / 60 /60 )