#获取申万官网申万行业数据

#导入库
import numpy as np
import pandas as pd
import requests
import json
from datetime import timedelta,date


# 获取申万官网申万行业数据
# code：行业代码
# frequency：day/week/month
# start_date：None(表示最早日期)
# end_date：None(表示今天日期)
# fields：None(表示所有字段)
def get_sw_data(code=None,start_date=None,end_date=None,frequency='day',fields=None): 
    #headers
    header={
        'HOST':'www.swsindex.com',
        'Referer':'http://www.swsindex.com/idx0200.aspx?columnid=8838&type=Day',
        'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) \
    Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4482.400 QQBrowser/9.7.13001.400'
        }

    #传入参数
    param={
        'tablename':'V_Report',
        'key':'id',
        #页面序号，每页返回20条数据
        'p':'1',
        #查询语句，查询的代码、日期、数据类型
        "where":"swindexcode in ('801020') and   BargainDate>='2018-04-02' and  BargainDate<='2018-04-24' and type='Day'",
        #排序(swindexcode asc表示按照代码升序，BargainDate_1表示按照日期降序，_2表示按照升序)
        'orderby':'swindexcode asc,BargainDate_2',
        #返回的字段
        'fieldlist':'SwIndexCode,SwIndexName,BargainDate,OpenIndex,CloseIndex,MaxIndex,MinIndex,BargainAmount,BargainSum,Markup,TurnoverRate,\
    PE,PB,MeanPrice,BargainSumRate,NegotiablesShareSum,NegotiablesShareSum2,DP',
        'pagecount':'1',
        'timed':'1524497094532',
        }
    
    #数据表表头
    sw_columns_list=['SwIndexCode','SwIndexName','BargainDate','OpenIndex','CloseIndex','MaxIndex','MinIndex','BargainAmount','BargainSum',
         'Markup','TurnoverRate','PE','PB','MeanPrice','BargainSumRate','NegotiablesShareSum','NegotiablesShareSum2','DP']

    #数据类型（日、周、月）
    frequency_list=['day','week','month']

    #申万一级行业代码列表
    code_list=['801010','801020','801030','801040','801050','801080','801110','801120','801130','801140','801150','801160','801170','801180',
       '801200','801210','801230','801710','801720','801730','801740','801750','801760','801770','801780','801790','801880','801890']    
    
    #配置查询语句
    where="swindexcode in ("
    if code is None:
        #如果代码为空，则代码为代码列表
        code=str(code_list).replace('[','').replace(']','')
    else:    
        if type(code)==list:
            #如果代码为列表，判断代码是否在代码列表中
            if set(code).issubset(set(code_list)):
                code_str=str(code).replace('[','').replace(']','')
            else:
                code_str=str(code_list).replace('[','').replace(']','')
        if type(code)==str:
            if code in code_list:   
                code_str="'"+code+"'"
            else:    
                code_str=str(code_list).replace('[','').replace(']','')
    where+=code_str   
    
    #配置日期
    today_str=pd.datetime.today().strftime('%Y-%m-%d')
    if (start_date is None) or (start_date<'1999-12-30') or (start_date>today_str):
        start_date='1999-12-30'
    where+=") and BargainDate>='"     
    where+=start_date
    if (end_date is None) or (end_date>today_str) or (end_date<'1999-12-30'):
        end_date=today_str
    where+="' and BargainDate<='" 
    where+=end_date  
    
    #配置数据类型
    if not(frequency in frequency_list):  
        frequency='day'
    where+="' and type='"
    where+=frequency
    where+="'"
    param['where']=where 
    
    #配置字段
    columns=sw_columns_list
    fieldlist=str(sw_columns_list).replace(" ","").replace("'","").replace('[',"").replace(']',"")   
    if not(fields is None):
        if(set(fields).issubset(set(sw_columns_list))):  
            if not (['SwIndexCode','SwIndexName','BargainDate'] in fields):
                fields=['SwIndexCode','SwIndexName','BargainDate']+fields
        fieldlist=str(fields).replace(" ","").replace("'","").replace('[',"").replace(']',"") 
        columns=fields
    param['fieldlist']=fieldlist
    
    df=pd.DataFrame()
    #url
    url='http://www.swsindex.com/handler.aspx'
    #页面计数器
    page=1
    while True:
        #获取数据
        ret=requests.get(url,data=param,headers=header)
        if not (ret.ok is True):
            break
        #整理引号、日期格式    
        data=ret.text.replace("'", '"').replace(' 0:00:00','').replace('/','-')
        #解析数据
        data=json.loads(data).get('root')
        if len(data)==0:
            break
        #追加数据表    
        df=df.append(pd.DataFrame(data,columns=columns))
        #设置页面计数器
        page+=1
        param['p']=str(page)    
    if len(df)!=0:   
        df.BargainDate=pd.to_datetime(df.BargainDate,format='%Y-%m-%d')
        
    #返回数据
    return df 

df=get_sw_data('801010',start_date='2018-04-23')
df

df=get_sw_data(code=['801010','801020'],start_date='2018-04-23',fields=['PE','PB'])
df

df=get_sw_data('801010',start_date='2018-03-1',frequency='week')
df