# 导入jqdatasdk
import jqdatasdk as jq
from jqdatasdk import *
from jqdatasdk.technical_analysis import *
# 登录认证
jq.auth('18888888888', 'xxx')

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from xgboost import XGBClassifier
import matplotlib.pyplot as plt
from tpot import TPOTClassifier

import warnings
warnings.filterwarnings("ignore")

#去除上市距beginDate不足n天的股票
def delete_stop(stocks,beginDate,n):
    stockList=[]
    beginDate = datetime.datetime.strptime(beginDate, "%Y-%m-%d")
    for stock in stocks:
        start_date=get_security_info(stock).start_date
        if start_date<(beginDate-datetime.timedelta(days=n)).date():
            stockList.append(stock)
    return stockList

#剔除ST股
def delete_st(stocks,begin_date):
    st_data=get_extras('is_st',stocks, count = 1,end_date=begin_date)
    stockList = [stock for stock in stocks if not st_data[stock][0]]
    return stockList

#按月区间取值
peroid = 'M'
#样本区间（训练集+测试集的区间为2014-1-31到2018-12-31）
start_date = '2014-02-01'
end_date = '2019-01-31'
#训练集长度
train_length = 48
#聚宽一级行业
industry_code = ['HY001', 'HY002', 'HY003', 'HY004', 'HY005', 'HY006', 'HY007', 'HY008', 'HY009', 'HY010', 'HY011']

#股票池，获取中证全指
securities_list = delete_stop(get_index_stocks('000985.XSHG'),start_date,90)
securities_list = delete_st(securities_list,start_date)

jqfactors_list = ['current_ratio',
                  'net_profit_to_total_operate_revenue_ttm',
                  'gross_income_ratio',
                  'roe_ttm',
                  'roa_ttm',
                  'total_asset_turnover_rate',\
                  'net_operating_cash_flow_coverage',
                  'net_operate_cash_flow_ttm',
                  'net_profit_ttm',\
                  'cash_to_current_liability',
                  'operating_revenue_growth_rate',
                  'non_recurring_gain_loss',\
                  'operating_revenue_ttm',
                  'net_profit_growth_rate']

def get_jq_factor(date):
    factor_data = get_factor_values(securities=securities_list, \
                                    factors=jqfactors_list, \
                                    count=1, \
                                    end_date=date)
    df_jq_factor=pd.DataFrame(index=securities_list)
    
    for i in factor_data.keys():
        df_jq_factor[i]=factor_data[i].iloc[0,:]
    
    return df_jq_factor

q = query(valuation.code, 
      valuation.market_cap,#市值
      valuation.circulating_market_cap,
      valuation.pe_ratio, #市盈率（TTM）
      valuation.pb_ratio, #市净率（TTM）
      valuation.pcf_ratio, #CFP
      valuation.ps_ratio, #PS
      balance.total_assets,
      balance.total_liability,
      balance.development_expenditure, #RD
      balance.dividend_payable,
      balance.fixed_assets,  
      balance.total_non_current_liability,
      income.operating_profit,
      income.total_profit, #OPTP
      #
      indicator.net_profit_to_total_revenue, #净利润/营业总收入
      indicator.inc_revenue_year_on_year,  #营业收入增长率（同比）
      indicator.inc_net_profit_year_on_year,#净利润增长率（同比）
      indicator.roe,
      indicator.roa,
      indicator.gross_profit_margin #销售毛利率GPM
    ).filter(
      valuation.code.in_(securities_list)
    )

#获取指定周期的日期列表 'W、M、Q'
def get_period_date(peroid,start_date, end_date):
    #设定转换周期period_type  转换为周是'W',月'M',季度线'Q',五分钟'5min',12天'12D'
    stock_data = get_price('000001.XSHE',start_date,end_date,'daily',fields=['close'])
    #记录每个周期中最后一个交易日
    stock_data['date']=stock_data.index
    #进行转换，周线的每个变量都等于那一周中最后一个交易日的变量值
    period_stock_data=stock_data.resample(peroid).last()
    date = period_stock_data.index
    pydate_array = date.to_pydatetime()
    date_only_array = np.vectorize(lambda s: s.strftime('%Y-%m-%d'))(pydate_array )
    date_only_series = pd.Series(date_only_array)
    start_date = datetime.datetime.strptime(start_date, "%Y-%m-%d")
    start_date = start_date-datetime.timedelta(days=1)
    start_date = start_date.strftime("%Y-%m-%d")
    date_list = date_only_series.values.tolist()
    date_list.insert(0,start_date)
    return date_list

def initialize_df(df,date):
    
    #净资产
    df['net_assets']=df['total_assets']-df['total_liability']
    df_new = pd.DataFrame(index=securities_list)
        
    #估值因子
    df_new['EP'] = df['pe_ratio'].apply(lambda x: 1/x)
    df_new['BP'] = df['pb_ratio'].apply(lambda x: 1/x)
    df_new['SP'] = df['ps_ratio'].apply(lambda x: 1/x)
    df_new['DP'] = df['dividend_payable']/(df['market_cap']*100000000)
    df_new['RD'] = df['development_expenditure']/(df['market_cap']*100000000)
    df_new['CFP'] = df['pcf_ratio'].apply(lambda x: 1/x)
    
    #杠杆因子
    #对数流通市值
    df_new['CMV'] = np.log(df['circulating_market_cap'])
    #总资产/净资产
    df_new['financial_leverage']=df['total_assets']/df['net_assets']
    #非流动负债/净资产
    df_new['debtequityratio']=df['total_non_current_liability']/df['net_assets']
    #现金比率=(货币资金+有价证券)÷流动负债
    df_new['cashratio']=df['cash_to_current_liability']
    #流动比率=流动资产/流动负债*100%
    df_new['currentratio']=df['current_ratio']
    
    #财务质量因子
    # 净利润与营业总收入之比
    df_new['NI'] = df['net_profit_to_total_operate_revenue_ttm']
    df_new['GPM'] = df['gross_income_ratio']
    df_new['ROE'] = df['roe_ttm']
    df_new['ROA'] = df['roa_ttm']
    df_new['asset_turnover'] = df['total_asset_turnover_rate']
    df_new['net_operating_cash_flow'] = df['net_operating_cash_flow_coverage']
    
    #成长因子
    df_new['Sales_G_q'] = df['operating_revenue_growth_rate']
    df_new['Profit_G_q'] = df['net_profit_growth_rate']
    
    #技术指标
    df_new['RSI']=pd.Series(RSI(securities_list, date, N1=20))    
    dif,dea,macd=MACD(securities_list, date, SHORT = 10, LONG = 30, MID = 15)
    df_new['DIF']=pd.Series(dif)
    df_new['DEA']=pd.Series(dea)
    df_new['MACD']=pd.Series(macd)    
    
    return df_new

dateList = get_period_date(peroid,start_date, end_date)

# (jqdata)因子
df_jq_factor = {}
# （财务数据）因子
df_q_factor = {}
# 预处理前的原始因子训练集
df_factor_pre_train = {}

for date in dateList:
    df_jq_factor = get_jq_factor(date)
    df_q_factor = get_fundamentals(q, date = date)
    df_q_factor.index = df_q_factor['code']
    # 合并得大表
    df_factor_pre_train[date] = pd.concat([df_q_factor,df_jq_factor],axis=1)
    # 初始化
    df_factor_pre_train[date] = initialize_df(df_factor_pre_train[date],date)
    
# df_factor_pre_train
print("获取因子完成")

from preprocess import winsorize_med, standardlize

#取股票对应行业
def get_industry_name(i_Constituent_Stocks, value):
    return [k for k, v in i_Constituent_Stocks.items() if value in v]

#缺失值处理
def replace_nan_indu(factor_data,stockList,industry_code,date):
    #把nan用行业平均值代替，依然会有nan，此时用所有股票平均值代替
    i_Constituent_Stocks={}
    data_temp=pd.DataFrame(index=industry_code,columns=factor_data.columns)
    for i in industry_code:
        temp = get_industry_stocks(i, date)
        i_Constituent_Stocks[i] = list(set(temp).intersection(set(stockList)))
        data_temp.loc[i]=mean(factor_data.loc[i_Constituent_Stocks[i],:])
    for factor in data_temp.columns:
        #行业缺失值用所有行业平均值代替
        null_industry=list(data_temp.loc[pd.isnull(data_temp[factor]),factor].keys())
        for i in null_industry:
            data_temp.loc[i,factor]=mean(data_temp[factor])
        null_stock=list(factor_data.loc[pd.isnull(factor_data[factor]),factor].keys())
        for i in null_stock:
            industry=get_industry_name(i_Constituent_Stocks, i)
            if industry:
                factor_data.loc[i,factor]=data_temp.loc[industry[0],factor] 
            else:
                factor_data.loc[i,factor]=mean(factor_data[factor])
    return factor_data

#数据预处理
def data_preprocessing(factor_data,stockList,industry_code,date):
    #去极值
    factor_data = winsorize_med(factor_data, scale=5, inf2nan=False,axis=0)
    #缺失值处理
    factor_data = replace_nan_indu(factor_data,stockList,industry_code,date)
    #标准化处理
    factor_data = standardlize(factor_data,axis=0)
    
    return factor_data

# 预处理后的原始因子训练集
df_factor_train = {}
for date in dateList:
    df_factor_train[date] = data_preprocessing(df_factor_pre_train[date],securities_list,industry_code,date)

# df_factor_train
print("预处理完成")

# 训练集数据
train_data=pd.DataFrame()
for date in dateList[:train_length]:
    traindf=df_factor_train[date]
    #stockList=list(traindf.index)
    #取收益率数据
    data_close=get_price(securities_list,date,dateList[dateList.index(date)+1],'1d','close')['close']
    traindf['pchg']=data_close.iloc[-1]/data_close.iloc[0]-1
    #剔除空值
    traindf=traindf.dropna()   
    #traindf=traindf.sort(columns='pchg')
    traindf=traindf.sort_values(by=['pchg'],ascending=False)
    #选取前后各30%的股票，剔除中间的噪声
    #取0-30%+70%-100%的数据
    traindf=traindf.iloc[:int(len(traindf['pchg'])/10*3),:].append(traindf.iloc[int(len(traindf['pchg'])/10*7):,:])
    #前30%为1，后30%为-1
    traindf['label']=list(traindf['pchg'].apply(lambda x:1 if x>np.mean(list(traindf['pchg'])) else -1))    
    if train_data.empty:
        train_data=traindf
    else:
        train_data=train_data.append(traindf)
        
y_train = train_data['label']  # 分割y
X_train = train_data.copy()
del X_train['pchg']
del X_train['label']

X_train.head()

# 测试集数据        
test_data={}
for date in dateList[train_length:-1]:
    testdf=df_factor_train[date]
    stockList=list(testdf.index)
    # 取收益率数据
    data_close=get_price(stockList,date,dateList[dateList.index(date)+1],'1d','close')['close']
    testdf['pchg']=data_close.iloc[-1]/data_close.iloc[0]-1
    #剔除空值
    testdf=testdf.dropna()   
    testdf=testdf.sort_values(by=['pchg'],ascending=False)
    #选取前后各30%的股票，剔除中间的噪声
    #取0-30%+70%-100%的数据
    testdf=testdf.iloc[:int(len(traindf['pchg'])/10*3),:].append(testdf.iloc[int(len(testdf['pchg'])/10*7):,:])
    testdf['label']=list(testdf['pchg'].apply(lambda x:1 if x>np.mean(list(testdf['pchg'])) else -1)) 
    test_data[date]=testdf
    
for date in dateList[train_length:-1]:
    # 取样本外数据特征及标签
    y_test=test_data[date]['label']
    X_test=test_data[date].copy()
    del X_test['pchg']
    del X_test['label']    
    
X_test.head()

# X_train, X_test, y_train, y_test = X_train, X_test, y_train, y_test

pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,
                                    random_state=42, verbosity=2)
pipeline_optimizer.fit(X_train, y_train)
print(pipeline_optimizer.score(X_test, y_test))
pipeline_optimizer.export('tpot_factor_classifier.py')

# tpot_factor_classifier.py

training_features, testing_features, training_target, testing_target = X_train, X_test, y_train, y_test

exported_pipeline = XGBClassifier(learning_rate=0.01, max_depth=10, min_child_weight=17, n_estimators=100, nthread=1, subsample=0.2)

exported_pipeline.fit(training_features, training_target)
results = exported_pipeline.predict(testing_features)

accuracy = exported_pipeline.score(testing_features, testing_target)
print("accuracy:%s"%accuracy)

#%matplotlib inline
fig = plt.figure(figsize= (20, 12))

n_features = training_features.shape[1]
plt.barh(range(n_features),exported_pipeline.feature_importances_,align='center')
plt.yticks(np.arange(n_features),training_features.columns)
plt.xlabel("Feature importance")
plt.ylabel("Feature")