ftx_trade_historical.py

async def doStuff(urls):
                                
                global ohlcvs, toms
                dodo = True
                mosts = {}
                leasts = {}
                tss = {}

                if urls == None:
                    urls = []
                    
                    for m in toms:
                        try:
                            with open(m + '-trades.json', 'r') as f:
                                ohlcvs[m] = json.loads(f.read())
                                leasts[m] = dateutil.parser.parse(ohlcvs[m][-200]['time'][:-13])
                                dodo = False

                                mosts[m] = dateutil.parser.parse(ohlcvs[m][-1]['time'][:-13])
                                tss[m] = time.mktime(leasts[m].timetuple())
                            ohlcvs[m] = []
                        except Exception as e:
                            print(e)
                            ohlcvs[m] = []
                        urls.append("https://ftx.com/api/markets/" + m + "/trades")
                ran = random.randint(0, 100)
                print(ran)
                if ran < 5:
                    tohlcvs = {}
                    for m in toms:

                        with open(m + '-trades.json', 'r') as f:
                            tohlcvs[m] = (json.loads(f.read()))
                        for ohlcv in tohlcvs[m]:
                            ohlcvs[m].append(ohlcv)
                        with open(m + '-trades.json', 'w') as f:
                            f.write(json.dumps(ohlcvs[m]))
                        tohlcvs[m] = []
                        ohlcvs[m] = []
                loop = asyncio.get_event_loop()
                futuresloop = []
                for u in urls:
                        futuresloop.append(
                                            loop.run_in_executor(
                                                None, 
                                                requests.get, 
                                                u
                                            )
                                        )
                urls = []
                for response in await asyncio.gather(*futuresloop):
                    ftx = response.json()['result']
                    astring = response.url.split('/trades')[0].split('markets/')[1]
                    m = {}
                    m['id'] = astring
                    if len(ftx) > 0:
                        try:
                            futs = []
                            if dodo == True:
                                try:
                                    least = dateutil.parser.parse(ftx[0]['time'][:-13])
                                    most = dateutil.parser.parse(ftx[-1]['time'][:-13]) - timedelta(hours=4)
                                except:
                                    least = dateutil.parser.parse(ftx[0]['time'][:-7])
                                    most = dateutil.parser.parse(ftx[-1]['time'][:-7]) - timedelta(hours=4)
                            else:
                                least = leasts[m['id']]
                                most = mosts[m['id']]
                            ts = time.mktime(least.timetuple())
                            ts2 = time.mktime(most.timetuple())
                            if least > yearago:
                                for candle in ftx:
                                    if m['id'] not in ohlcvs:
                                        ohlcvs[m['id']] = []
                                        ohlcvs2[candle['time'][:-13]] = {}
                                    if candle not in ohlcvs[m['id']]:
                                        ohlcvs[m['id']].append(candle)
                                    try:
                                        d = dateutil.parser.parse(candle['time'][:-13])
                                    except:
                                        d = dateutil.parser.parse(candle['time'][:-7])
                                    least = d
                                    ts = time.mktime(d.timetuple())
                                
                                done = False 
                                while done == False:
                                    try: 
                                        u = "https://ftx.com/api/markets/" + m['id'] + "/trades?limit=5000&end_time=" + str(int(ts2))
                                        urls.append(u)
                                        done = True
                                        sleep(0.01)
                                    except:
                                        sleep(10)
                                if len(ftx) < 10:
                                    least = yearago
                        except Exception as e:
                            print(e)
                await doStuff(urls)