上海期货交易所历史期货价格数据

futures.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# `pip install requests`

from datetime import datetime, date
from dateutil.rrule import rrule, DAILY
from enum import Enum
import json

import requests

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

LOGGING_FILE = 'futures.log'
FUTURES_CSV = 'futures.csv'
INSTRUMENTIDS = [ 'CU', 'AL', 'ZN', 'PB', 'NI', 'SN', 'AU', 'AG', 'RB', 'WR', 'HC', 'FU', 'BU', 'RU', ]

class SFE_Suffix(Enum):
    sfe_default = 'defaultTimePrice.dat'
    sfe_main    = 'mainTimePrice.dat'
    sfe_daily   = 'dailyTimePrice.dat'

def generateUrl(date, suffix):
    if date is None:
        date = datetime.today()
    if suffix is None:
        raise AssertionError('Parameter `suffix` is None!')
    print('Input parameters:')
    print('date=', date)
    print('suffix=', suffix.value)

    if isinstance(suffix, SFE_Suffix):
        suffix = suffix.value
    if isinstance(suffix, str):
        result = 'http://www.shfe.com.cn/data/dailydata/ck/{}{}'.format(date.strftime('%Y%m%d'), suffix)
        print('url=', result)
        return result
    else:
        raise AssertionError('Parameter `suffix` is invalid!')

def Session(host='www.shfe.com.cn', referer='http://www.shfe.com.cn/statements/dataview.html?paramid=delaymarket_cu'):
    headers = {
        'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cache-Control': 'max-age=0',
        'Connection': 'keep-alive',
        'Host': host,
        'If-Modified-Since': '0',
        'Referer': referer,
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36',
    }
    session = requests.Session()
    session.headers.update(headers)
    return session

def parseData(date, response):
    print('Response:')
    print(response)
    print()
    result = json.loads(response.text)
    with open(LOGGING_FILE, 'a') as log:
        reportDate = date or pd.to_datetime(result['report_date'], format='%Y%m%d', errors='ignore')
        table = []
        for chunk in result['o_currefprice']:
            if chunk.get('TIME') == '9:00-15:00':
                instrumentId = chunk['INSTRUMENTID'][:2]
                refSettlementPrice = chunk['REFSETTLEMENTPRICE']
                updown = round(chunk['UPDOWN'])
                #log.write('{}\t{}\t{}\t{}\n'.format(reportDate, instrumentId, refSettlementPrice, updown))
                row = [reportDate, instrumentId, refSettlementPrice, updown,]
                table.append(row)

        #print(table)
        #df = pd.DataFrame(table,
        #    columns=['reportDate', 'instrumentId', 'refSettlementPrice', 'updown',],)
        #print(df)

        #log.write('\n')
        #log.write(json.dumps(result, sort_keys=False, indent=2, ensure_ascii=False))
                
        return table

def fetchData(session, date, suffix):
    url = generateUrl(date, suffix)
    response = session.get(url)
    if response.status_code == 200:
        return parseData(date, response)
    else:
        return []

def traverseDate(callback):
    dsrc = date(2002, 1, 1)
    ddst = date.today()
    table = []
    for dt in rrule(DAILY, dtstart=dsrc, until=ddst):
        #print('Fetching data for `{}` ...'.format(dt.strftime('%Y-%m-%d')))
        table.extend(callback(dt))
    df = pd.DataFrame(table,
        columns=['reportDate', 'instrumentId', 'refSettlementPrice', 'updown',],)
    df.to_csv(FUTURES_CSV)
    return df

def plot(dataframe, instrumentId):
    df[df['instrumentId'] == instrumentId].plot(\
        x='reportDate',
        y='refSettlementPrice',
        kind='line',
        title=instrumentId,)
    plt.show()

if __name__ == '__main__':
    # Clean up logging file
    with open(LOGGING_FILE, 'w') as log:
        log.write('')

    print('Running ...')

    with Session() as session:
        df = traverseDate(lambda dt: fetchData(session, dt, SFE_Suffix.sfe_main))
        #df = pd.read_csv(FUTURES_CSV)
        plot(df, 'AU')
        #for instrumentId in INSTRUMENTIDS:
        #    plot(df, instrumentId)

    '''
        date = datetime.strptime('2018-7-2', '%Y-%m-%d')
        print('Date:', date.strftime('%Y-%m-%d'))
        #fetchData(session, date, SFE_Suffix.sfe_default)
        #fetchData(session, date, SFE_Suffix.sfe_daily)
        fetchData(session, date, SFE_Suffix.sfe_main)
    '''

import functools

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

def SeperateByGauge(df, gauge):
    return df[df['gauge'] == gauge].loc[:, ('date', 'count',)].groupby('date').sum().rename(columns={'count': gauge}).copy(deep=False)

if __name__ == '__main__':
    print('Running ...')
    print('-----------------------')

    filepath = 'KOTL-2018-JST-USAGE.csv'
    df = pd.read_csv(filepath,
            header = 0,
            encoding = 'ISO-8859-1',
        )

    gauges = df['gauge'].unique()
    df1 = functools.reduce(lambda left, right: pd.merge(left, right, how='outer', on='date'), map(lambda gauge: SeperateByGauge(df, gauge), iter(gauges)))
    df1.index = pd.to_datetime(df1.index)

    df1.plot(kind='line',
        title='JST Usage',)
    plt.show()

    print('-----------------------')
    print('Done.')