rawdata.py

import pandas as pd
import msgpack
import zlib

with open(f'/data/dataset/qc/msgpack/TXF20190703.msgpack', 'rb') as f:
    rawdata = msgpack.loads(zlib.decompress(f.read()), raw=False)
    
df_bidask = pd.DataFrame(rawdata['Q/TFE/TXFI9'])

df_bidask[df_bidask['Simtrade'] != 1][['Date', 'Time', 'AskPrice', 'AskVolume', 'BidPrice', 'BidVolume']]


def split_data(rec, index):
    sp_rec = {}
    [
        sp_rec.update({k: v[index] if isinstance(v, (list, tuple)) else v})
        for k, v in rec.items()
    ]
    sp_rec['idx'] = index
    return sp_rec
    
df_tick = pd.DataFrame([split_data(rec, ind)
                        for rec in rawdata['L/TFE/TXFI9']
                        for ind, _ in enumerate(rec['Close'])])

df_tick[df_tick['Simtrade']!=1][['Date', 'Time', 'idx', 'Close', 'Volume']]