行业类¶

1.申万二级及三级的行情及估值数据¶

#获取申万官网申万行业数据#导入库import numpy as npimport pandas as pdimport requestsimport jsonfrom datetime import timedelta,date# 获取申万官网申万行业数据# code：行业代码  https://www.joinquant.com/help/api/help?name=plateData#申万行业# frequency：day/week/month# start_date：None(表示最早日期)# end_date：None(表示今天日期)# fields：None(表示所有字段)def get_sw_data(code=None,start_date=None,end_date=None,frequency='day',fields=None): #headersheader={'HOST':'www.swsindex.com','Referer':'http://www.swsindex.com/idx0200.aspx?columnid=8838&type=Day','User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) \    Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4482.400 QQBrowser/9.7.13001.400'}#传入参数param={'tablename':'V_Report','key':'id',#页面序号，每页返回20条数据'p':'1',#查询语句，查询的代码、日期、数据类型"where":"swindexcode in ('801020') and   BargainDate>='2018-04-02' and  BargainDate<='2018-04-24' and type='Day'",#排序(swindexcode asc表示按照代码升序，BargainDate_1表示按照日期降序，_2表示按照升序)'orderby':'swindexcode asc,BargainDate_2',#返回的字段'fieldlist':'SwIndexCode,SwIndexName,BargainDate,OpenIndex,CloseIndex,MaxIndex,MinIndex,BargainAmount,BargainSum,Markup,TurnoverRate,\    PE,PB,MeanPrice,BargainSumRate,NegotiablesShareSum,NegotiablesShareSum2,DP','pagecount':'1','timed':'1524497094532',}#数据表表头sw_columns_list=['SwIndexCode','SwIndexName','BargainDate','OpenIndex','CloseIndex','MaxIndex','MinIndex','BargainAmount','BargainSum', 'Markup','TurnoverRate','PE','PB','MeanPrice','BargainSumRate','NegotiablesShareSum','NegotiablesShareSum2','DP']#数据类型（日、周、月）frequency_list=['day','week','month']#配置查询语句where="swindexcode in ("if code is None:#如果代码为空，则代码为代码列表code='801010'else:    if type(code)==list:code_str=str(code).replace('[','').replace(']','')if type(code)==str:code_str="'"+code+"'"where+=code_str   
    #配置日期today_str=pd.datetime.today().strftime('%Y-%m-%d')if (start_date is None) or (start_date<'1999-12-30') or (start_date>today_str):start_date='1999-12-30'where+=") and BargainDate>='"     where+=start_dateif (end_date is None) or (end_date>today_str) or (end_date<'1999-12-30'):end_date=today_strwhere+="' and BargainDate<='" where+=end_date  
    #配置数据类型if not(frequency in frequency_list):  frequency='day'where+="' and type='"where+=frequencywhere+="'"param['where']=where 
    #配置字段columns=sw_columns_listfieldlist=str(sw_columns_list).replace(" ","").replace("'","").replace('[',"").replace(']',"")   if not(fields is None):if(set(fields).issubset(set(sw_columns_list))):  if not (['SwIndexCode','SwIndexName','BargainDate'] in fields):fields=['SwIndexCode','SwIndexName','BargainDate']+fieldsfieldlist=str(fields).replace(" ","").replace("'","").replace('[',"").replace(']',"") columns=fieldsparam['fieldlist']=fieldlistdf=pd.DataFrame()#urlurl='http://www.swsindex.com/handler.aspx'#页面计数器page=1while True:#获取数据ret=requests.get(url,data=param,headers=header)if not (ret.ok is True):break#整理引号、日期格式    data=ret.text.replace("'", '"').replace(' 0:00:00','').replace('/','-')#解析数据data=json.loads(data).get('root')if len(data)==0:break#追加数据表    df=df.append(pd.DataFrame(data,columns=columns))#设置页面计数器page+=1param['p']=str(page)    if len(df)!=0:   df.BargainDate=pd.to_datetime(df.BargainDate,format='%Y-%m-%d')#返回数据return df.set_index('BargainDate')df=get_sw_data('850111',start_date='2019-02-23')df.head()

.dataframe tbody tr th:only-of-type {        vertical-align: middle;    }    .dataframe tbody tr th {        vertical-align: top;    }    .dataframe thead th {        text-align: right;    }

股票类数据¶

1.获取新浪热门股票¶

import requestsimport anyjsonimport pandas as pddef get_hot_stock_from_sina():'''从新浪得到热门数据'''html = requests.get('https://ssl-data.sina.com.cn/api/openapi.php/WeiboReferService.getListSymbol?code=CNHOUR6&callback=var%20AHM=').content.decode()  n = html[html.index('(')+1:html.index(')')]h = anyjson.deserialize(n)data = pd.DataFrame(h['result']['data'])data.SYMBOL = data.SYMBOL.apply(normalize_code)return dataget_hot_stock_from_sina().head()

.dataframe tbody tr th:only-of-type {        vertical-align: middle;    }    .dataframe tbody tr th {        vertical-align: top;    }    .dataframe thead th {        text-align: right;    }

2.获取选股宝涨停原因¶

import urllibimport jsonimport pandas as pddef Xuangubao():url = "https://flash-api.xuangubao.cn/api/pool/detail?pool_name=limit_up"  #涨停#     url = 'https://flash-api.xuangubao.cn/api/pool/detail?pool_name=limit_up_broken'  #炸板header_dict = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko'}#     req = urllib2.Request(url=url, headers=header_dict)#     df = pd.DataFrame(json.loads(urllib2.urlopen(req).read())['data'])req = urllib.request.Request(url,headers = header_dict)df = pd.DataFrame(json.loads(urllib.request.urlopen(req).read())['data'])df['stock_reason'] = df.surge_reason.apply(lambda x: x['stock_reason'])df['plate_name'] = df.surge_reason.apply(lambda x: x['related_plates'][0]['plate_name'])def get_plate_reason(x):try: return x['related_plates'][0][u'plate_reason']except:returndf['plate_reason'] = df.surge_reason.apply(get_plate_reason)df['limit_timeline'] = df.limit_timeline.apply(lambda x: datetime.datetime.fromtimestamp(x['items'][0]['timestamp']))df.index = df.surge_reason.apply(lambda x: normalize_code(x['symbol']))df.index.name=Nonereturn df.drop('surge_reason',axis=1)Xuangubao().head()

.dataframe tbody tr th:only-of-type {        vertical-align: middle;    }    .dataframe tbody tr th {        vertical-align: top;    }    .dataframe thead th {        text-align: right;    }

3.东财股吧热门帖¶

import requestsfrom bs4 import BeautifulSoupdef get_post_data(html):soup = BeautifulSoup(html)post_list = soup.find_all("div", class_="articleh")post_list_convert = []for itm in post_list:single_post = []ll = itm.find_all("span")for idx in range(len(ll)):if idx == 2:# 帖子地址和标题if ll[idx].em:single_post.append(ll[idx].em.string)else:single_post.append("")single_post.append(ll[idx].a.string)single_post.append("http://guba.eastmoney.com" + ll[idx].a['href'])elif idx == 3:# 作者信息single_post.append(ll[idx].a.string)single_post.append(ll[idx].a['href'])else:# 其他信息single_post.append(ll[idx].string)post_list_convert.append(single_post)return post_list_convertdef get_html(urls):g_ret = []for itm in urls:g_ret.append(requests.get(itm))ret = [itm.content.decode("utf-8") for itm in g_ret]return retdef control(page_count=20, step = 50):url_list = []# 构造url列表for idx in range(1, page_count + 1):url_list.append("http://guba.eastmoney.com/list,cjpl,99_{}.html".format(idx))for idx in range(0, page_count, step):# 每次爬step页print("start get html {} -> {}".format(idx, idx + step))all_html_data = get_html(url_list[idx:idx + step])print("start convert data")all_post_data = get_post_data(all_html_data[0])for itm in all_html_data[1:]:for itm2 in get_post_data(itm):all_post_data.append(itm2)return pd.DataFrame(all_post_data)control(page_count=3).head()

start get html 0 -> 50
start convert data

.dataframe tbody tr th:only-of-type {        vertical-align: middle;    }    .dataframe tbody tr th {        vertical-align: top;    }    .dataframe thead th {        text-align: right;    }

4.获取最新公布的机构调研数据¶

def get_jgdy_all(page_count=1,page_size=100):'''获取最近公布的机构调研数据,count为返回页数,一页为page_size条数据(不得超过5000条),默认返回100条,以发布时间排序'''url = 'http://data.eastmoney.com/DataCenter_V3/jgdy/xx.ashx?'l = []for i in range(1,page_count+1):param = {"pagesize":page_size,"page":i,'sortRule':-1,'sortType':0,'rt':51777724,}ret=requests.get(url,params=param)l.append(pd.DataFrame(json.loads(ret.text)['data']))return pd.concat(l,axis=0).set_index('SCode')df = get_jgdy_all(1)df.head()

.dataframe tbody tr th:only-of-type {        vertical-align: middle;    }    .dataframe tbody tr th {        vertical-align: top;    }    .dataframe thead th {        text-align: right;    }

5.获取个股机构调研数据¶

def get_jgdy_one(code):'''获取个股调研数据'''url = 'http://data.eastmoney.com/DataCenter_V3/jgdy/gsjsdy.ashx?'param = {"pagesize":100,"page":1,'sortRule':-1,'sortType':0,'code':code}ret=requests.get(url,params=param)return pd.DataFrame(json.loads(ret.text)['data'])get_jgdy_one('000568').head()

.dataframe tbody tr th:only-of-type {        vertical-align: middle;    }    .dataframe tbody tr th {        vertical-align: top;    }    .dataframe thead th {        text-align: right;    }

6.获取券商月报数据¶

#-*- coding:utf-8 -*-import urllibimport reimport pandas as pddef get_yb(mon):http_url = 'http://data.eastmoney.com/other/qsjy/yb.%s.html'%monheader_dict = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko'}rs = urllib.request.Request(http_url,headers=header_dict)response = urllib.request.urlopen(rs)token = re.search("token=(.*?)&st", response.read().decode('gbk')).group(1)day = mon[0:4]+"-"+mon[4:6]url = "http://dcfm.eastmoney.com/em_mutisvcexpandinterface/api/js/get?type=QSYJBG_MReport&token="+token+"&st=RQ&sr=-1&p=1&ps=1000&js=(x)&filter=(RQ='"+day+"-01T00:00:00')&rt=51785853"rs = urllib.request.Request(url,headers=header_dict)response = urllib.request.urlopen(rs)return pd.read_json(response.read())get_yb('201901').head()

.dataframe tbody tr th:only-of-type {        vertical-align: middle;    }    .dataframe tbody tr th {        vertical-align: top;    }    .dataframe thead th {        text-align: right;    }

抓取港股新股数据统计打新收益¶

内容较多,请直接点击原文链接查

期货类¶

1. 生意社大宗商品报价数据¶

import pandas as pdimport requestsfrom bs4 import BeautifulSoupdef get_sys(symbol='',page=1):"""    查询生意社大宗商品报价数据。    symbol: 生意社网页上对品种的编号,打开对应品种直接可以从url上看到 如806代表*    page: 查询第几页"""txt = requests.get('http://www.100ppi.com/mprice/plist-'+symbol+'-'+str(page)+'.html').texttable = BeautifulSoup(txt).find_all("table", class_="lp-table mb15")df = pd.read_html(str(table),header=0)[0]df['price'] = df['报价'].apply(lambda x: re.findall(r"\d+\.?\d*",x)[0]).astype(int)return dfget_sys('806').head()

.dataframe tbody tr th:only-of-type {        vertical-align: middle;    }    .dataframe tbody tr th {        vertical-align: top;    }    .dataframe thead th {        text-align: right;    }

其他¶

1.获取国债收益率数据¶

import requestsimport jsonimport pandas as pdimport timefrom sqlalchemy import create_enginedef get_bnd_yield(year=10):ids = {10: '29227', 5: '29234', 1: '29231'}url = 'https://cn.investing.com/common/modules/js_instrument_chart/api/data.php?' + \'pair_id={}&pair_id_for_news={}'.format(ids[year], ids[year]) +\'&chart_type=area&pair_interval=month&candle_count=120&events=yes&volume_series=yes&period=5-years'headers = {}headers['X-Requested-With'] = 'XMLHttpRequest'headers['Host'] = 'cn.investing.com'headers['Referer'] = 'https://cn.investing.com/rates-bonds/china-{}-year-bond-yield'.format(year)headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)'res = requests.get(url, headers=headers)res = json.loads(res.content.decode('utf-8').replace("'", "\""))data = pd.DataFrame(res['candles'])data = data.iloc[:, :2]data.columns = ['date', 'y'+str(year)]data['date'] = data['date'].map(lambda x: time.strftime("%Y-%m-%d", time.localtime(int(str(x)[:10]))))data.set_index('date', inplace=True)return datadef get_bnd_yields(years=[1, 5, 10]):bag = pd.DataFrame()for yr in years:bag = pd.concat([bag, get_bnd_yield(year=yr)], axis=1)#print(bag.head())return bagget_bnd_yields().head()

.dataframe tbody tr th:only-of-type {        vertical-align: middle;    }    .dataframe tbody tr th {        vertical-align: top;    }    .dataframe thead th {        text-align: right;    }