import numpy as np
import pandas as pd
from jqdata import *
import datetime
import warnings
warnings.filterwarnings("ignore")

start_date = '2005-05'
end_date = '2017-12'
q1 = query(macro.MAC_AREA_GDP_QUARTER.stat_quarter,
          macro.MAC_AREA_GDP_QUARTER.gdp_yoy_sin,
          ).filter(macro.MAC_AREA_GDP_QUARTER.stat_quarter >= start_date,
                   macro.MAC_AREA_GDP_QUARTER.stat_quarter <= end_date,
                   macro.MAC_AREA_GDP_QUARTER.area_name == '中国'
        ).order_by(macro.MAC_AREA_GDP_QUARTER.stat_quarter.asc()
            )

q2 = query(macro.MAC_AREA_CPI_MONTH.stat_month,
           macro.MAC_AREA_CPI_MONTH.item_value
          ).filter(macro.MAC_AREA_CPI_MONTH.stat_month >= start_date,
                   macro.MAC_AREA_CPI_MONTH.stat_month <= end_date,
                   macro.MAC_AREA_CPI_MONTH.area_name == '中国',
                   macro.MAC_AREA_CPI_MONTH.item_name == '居民消费价格指数(上月=100)'
        ).order_by(macro.MAC_AREA_CPI_MONTH.stat_month.asc()
            )

q3 = query(macro.MAC_FIXED_INVESTMENT.stat_month,
           macro.MAC_FIXED_INVESTMENT.fixed_assets_investment_yoy
          ).filter(macro.MAC_FIXED_INVESTMENT.stat_month >= start_date,
                   macro.MAC_FIXED_INVESTMENT.stat_month <= end_date,
        ).order_by(macro.MAC_FIXED_INVESTMENT.stat_month.asc()
            )

q4 = query(macro.MAC_SALE_RETAIL_MONTH.stat_month,
           macro.MAC_SALE_RETAIL_MONTH.retail_sin
          ).filter(macro.MAC_SALE_RETAIL_MONTH.stat_month >= start_date,
                   macro.MAC_SALE_RETAIL_MONTH.stat_month <= end_date,
        ).order_by(macro.MAC_SALE_RETAIL_MONTH.stat_month.asc()
            )

q5 = query(macro.MAC_ECONOMIC_BOOM_IDX.stat_month,
           macro.MAC_ECONOMIC_BOOM_IDX.leading_idx
          ).filter(macro.MAC_ECONOMIC_BOOM_IDX.stat_month >= start_date,
                   macro.MAC_ECONOMIC_BOOM_IDX.stat_month <= end_date
        ).order_by(macro.MAC_ECONOMIC_BOOM_IDX.stat_month.asc()
            )

q6 = query(macro.MAC_MANUFACTURING_PMI.stat_month,
           macro.MAC_MANUFACTURING_PMI.pmi
          ).filter(macro.MAC_MANUFACTURING_PMI.stat_month >= start_date,
                   macro.MAC_MANUFACTURING_PMI.stat_month <= end_date
        ).order_by(macro.MAC_MANUFACTURING_PMI.stat_month.asc()
            )

df_gdp = macro.run_query(q1)
df_cpi = macro.run_query(q2)
df_CI = macro.run_query(q3)
df_Consumption = macro.run_query(q4)
df_BL = macro.run_query(q5)
df_pmi = macro.run_query(q6)

df_gdp.rename(columns=({'stat_quarter': 'stat_month','gdp_yoy_sin':'GDP'}), inplace=True)
df_cpi.rename(columns=({'item_value': 'CPI'}), inplace=True)
df_CI.rename(columns=({'fixed_assets_investment_yoy': 'Capital_Investment'}), inplace=True)
df_Consumption.rename(columns=({'retail_sin': 'Consumption'}), inplace=True)
df_BL.rename(columns=({'leading_idx':'Boom_Lead'}), inplace=True)
df_pmi.rename(columns={'pmi':'PMI'},inplace=True)

import tushare as ts
ts.set_token('3fd8b7ae1b055ce47295b9fe352db0f1f172117f6af30ffec9c4f680')
df_ppi = ts.get_ppi()[['month','ppi']]
date = pd.to_datetime(df_ppi.month,format='%Y.%m')
df_ppi['month'] = date.apply(lambda x: x.strftime('%Y-%m'))
df_ppi.rename(columns={'month':'stat_month','ppi':'PPI'},inplace=True)
df_ppi = df_ppi.sort_values(by=['stat_month'])
df_ppi = df_ppi[(df_ppi.stat_month > '2005-04') & (df_ppi.stat_month < '2018-01')].reset_index(drop=True)

df_macro = pd.DataFrame(df_cpi.stat_month)
for factor in [df_gdp,df_cpi,df_ppi,df_CI,df_Consumption,df_BL,df_pmi]:
    df_macro = pd.merge(df_macro,factor,how = 'outer', on='stat_month')
df_macro = df_macro.sort_values(by="stat_month")
df_macro = df_macro.fillna(method='backfill')
df_macro.head().append(df_macro.tail())

q = query(macro.MAC_MONEY_SUPPLY_MONTH.stat_month,
          macro.MAC_MONEY_SUPPLY_MONTH.m1_yoy,
          macro.MAC_MONEY_SUPPLY_MONTH.m2_yoy
         ).filter(macro.MAC_MONEY_SUPPLY_MONTH.stat_month >= start_date,
                  macro.MAC_MONEY_SUPPLY_MONTH.stat_month <= end_date
        ).order_by(macro.MAC_MONEY_SUPPLY_MONTH.stat_month.asc()
            )
df_M = macro.run_query(q)
df_M.rename(columns={'m1_yoy':'M1','m2_yoy':'M2'},inplace=True)
df_M.head()

def ytm(name):
    temp = pd.read_csv('./data/'+str(name) +'.csv')[['日期','收盘']]
    date = pd.to_datetime(temp.日期,format='%Y年%m月')
    temp['日期'] = date.apply(lambda x: x.strftime('%Y-%m'))
    temp.rename(columns={'日期':'stat_month','收盘':name},inplace=True)
    return temp

ytm1 = ytm('YTM1')
ytm5 = ytm('YTM5')
ytm10 = ytm('YTM10')

df_interest = pd.DataFrame(df_M.stat_month)
for factor in [df_M,ytm1,ytm5,ytm10]:
    df_interest = pd.merge(df_interest,factor,how = 'outer', on='stat_month')
df_interest = df_interest.sort_values(by="stat_month")

def get_index_pricing(code,date):
    '''指定日期的指数PE_PB（等权重）'''
    stocks = get_index_stocks(code, date)
    q = query(valuation).filter(valuation.code.in_(stocks))
    df = get_fundamentals(q, date)
    if len(df)>0:
        pe = len(df)/sum([1/p if p>0 else 0 for p in df.pe_ratio])
        pb = len(df)/sum([1/p if p>0 else 0 for p in df.pb_ratio])
        ps = len(df)/sum([1/p if p>0 else 0 for p in df.ps_ratio])
        pcf = len(df)/sum([1/p if p>0 else 0 for p in df.pcf_ratio])
        return (round(pe,2), round(pb,2),round(ps,2),round(pcf,2))
    else:
        return float('NaN')

def get_index_his_pricing(code, start_date=None, end_date=None):
    '''指数历史Pricing'''
    if start_date is None:
        start_date = get_security_info(code).start_date
    if end_date is None:
        end_date = pd.datetime.today() - timedelta(1)
    x = get_price(code, start_date=start_date, end_date=end_date, frequency='daily', fields='close')
    date_list = x.index.tolist()
    pe_list = []
    pb_list = []
    ps_list = []
    pcf_list = []
    for d in date_list: #交易日
        pricing = get_index_pricing(code,d)
        pe_list.append(pricing[0])
        pb_list.append(pricing[1])
        ps_list.append(pricing[2])
        pcf_list.append(pricing[3])
    df = pd.DataFrame({'PE': pd.Series(pe_list, index=date_list),
                        'PB': pd.Series(pb_list, index=date_list),
                        'PS':pd.Series(pb_list, index=date_list),
                        'PCF':pd.Series(pb_list, index=date_list)})
    return df

index = '000300.XSHG'
df_pricing = get_index_his_pricing(index, '2005-05-09','2017-12-31')

df_pricing_M = df_pricing.groupby(pd.TimeGrouper('M')).apply(lambda i:i.iloc[-1])

df = get_price('000300.XSHG',start_date = '2005-04-09',end_date='2017-12-31',
                 frequency='daily',fields=['close'])
df = df.groupby(pd.TimeGrouper('M')).apply(lambda i:(i.iloc[-1]-i.iloc[1])/i.iloc[1])

df[df.close > 0.0] = 1
df[df.close <= 0.0] = 0

df_stock = df_stock.dropna().reset_index(drop = True)

factor = pd.read_csv('./Data/Factor1.csv')
hs = pd.read_csv('./Data/HS300.csv')
factor.head().append(factor.tail())

hs.head().append(hs.tail())

from sklearn import preprocessing
factor_nor = preprocessing.normalize(factor, norm='l1')

features = factor_nor
target = hs

def roc(test,predictions):
    import sklearn.metrics as metrics
    import matplotlib.pyplot as plt
    fpr, tpr, th = metrics.roc_curve(test, predictions)
    print('-'*50)
    print('FPR = ',fpr)
    print('-'*50)
    print('TPR = ',tpr)
    print('-'*50)
    print('AUC = %.4f' %metrics.auc(fpr, tpr))
    print('-'*50)
    plt.figure(figsize=[6, 6])
    plt.plot(fpr, tpr, 'b--')
    plt.title('ROC curve')
    plt.show()

def accuracy(pred1,stest):
    from sklearn.metrics import accuracy_score
    print(accuracy_score(stest, pred1))

def train_pred(factor,hs,period,alg):
    pred = []
    ytest = []
    alg=alg
    
    for i in np.arange(len(factor)-period):
        x_train, y_train = factor[i:i+period], hs[i:i+period]
        x_test, y_test = np.array(factor[i+period]), hs.close.iloc[i+period]
        
        alg.fit(x_train,y_train)
    
        temp = alg.predict(x_test.reshape(1,16))
        pred.append(temp[0])
        ytest.append(y_test)

    return(pred,ytest)

from sklearn.linear_model import LogisticRegression
alg = LogisticRegression()
lr_pred, ytest = train_pred(features,target,18,alg)

accuracy(ytest,lr_pred)

roc(ytest, lr_pred)

from sklearn import tree 
alg = tree.DecisionTreeClassifier(criterion='gini') 
dt_pred, ytest = train_pred(features,target,18,alg)

accuracy(ytest,dt_pred)

roc(ytest, dt_pred)

from sklearn import svm
alg = svm.SVC() 
svm_pred, ytest = train_pred(features,target,18,alg)

accuracy(ytest,svm_pred)

roc(ytest, svm_pred)