Last active
January 2, 2016 13:49
-
-
Save stephenlb/8312648 to your computer and use it in GitHub Desktop.
Loading History from PubNub Mt.Gox Trades. These are resource files for a StackOverflow answer found here: http://stackoverflow.com/questions/20976433/how-to-retrieve-more-than-100-messages-from-the-history-of-a-pubnub-channel/20988208#20988208
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "ticker.LTCGBP": "0102a446-e4d4-4082-8e83-cc02822f9172", | |
| "ticker.LTCCNY": "0290378c-e3d7-4836-8cb1-2bfae20cc492", | |
| "depth.BTCHKD": "049f65dc-3af3-4ffd-85a5-aac102b2a579", | |
| "depth.BTCEUR": "057bdc6b-9f9c-44e4-bc1a-363e4443ce87", | |
| "ticker.NMCAUD": "08c65460-cbd9-492e-8473-8507dfa66ae6", | |
| "ticker.BTCEUR": "0bb6da8b-f6c6-4ecf-8f0d-a544ad948c15", | |
| "depth.BTCKRW": "0c84bda7-e613-4b19-ae2a-6d26412c9f70", | |
| "depth.BTCCNY": "0d1ecad8-e20f-459e-8bed-0bdcf927820f", | |
| "ticker.BTCCAD": "10720792-084d-45ba-92e3-cf44d9477775", | |
| "depth.BTCCHF": "113fec5f-294d-4929-86eb-8ca4c3fd1bed", | |
| "ticker.LTCNOK": "13616ae8-9268-4a43-bdf7-6b8d1ac814a2", | |
| "ticker.LTCUSD": "1366a9f3-92eb-4c6c-9ccc-492a959eca94", | |
| "ticker.BTCBTC": "13edff67-cfa0-4d99-aa76-52bd15d6a058", | |
| "ticker.LTCCAD": "18b55737-3f5c-4583-af63-6eb3951ead72", | |
| "ticker.NMCCNY": "249fdefd-c6eb-4802-9f54-064bc83908aa", | |
| "depth.BTCUSD": "24e67e0d-1cad-4cc0-9e7a-f8523ef460fe", | |
| "ticker.BTCCHF": "2644c164-3db7-4475-8b45-c7042efe3413", | |
| "depth.BTCAUD": "296ee352-dd5d-46f3-9bea-5e39dede2005", | |
| "ticker.BTCCZK": "2a968b7f-6638-40ba-95e7-7284b3196d52", | |
| "ticker.BTCSGD": "2cb73ed1-07f4-45e0-8918-bcbfda658912", | |
| "ticker.NMCJPY": "314e2b7a-a9fa-4249-bc46-b7f662ecbc3a", | |
| "ticker.BTCNMC": "36189b8c-cffa-40d2-b205-fb71420387ae", | |
| "depth.BTCINR": "414fdb18-8f70-471c-a9df-b3c2740727ea", | |
| "depth.BTCSGD": "41e5c243-3d44-4fad-b690-f39e1dbb86a8", | |
| "ticker.BTCLTC": "48b6886f-49c0-4614-b647-ba5369b449a9", | |
| "ticker.LTCEUR": "491bc9bb-7cd8-4719-a9e8-16dad802ffac", | |
| "ticker.BTCINR": "55e5feb8-fea5-416b-88fa-40211541deca", | |
| "ticker.LTCJPY": "5ad8e40f-6df3-489f-9cf1-af28426a50cf", | |
| "depth.BTCCAD": "5b234cc3-a7c1-47ce-854f-27aee4cdbda5", | |
| "ticker.BTCNZD": "5ddd27ca-2466-4d1a-8961-615dedb68bf1", | |
| "depth.BTCGBP": "60c3af1b-5d40-4d0e-b9fc-ccab433d2e9c", | |
| "depth.BTCNOK": "66da7fb4-6b0c-4a10-9cb7-e2944e046eb5", | |
| "depth.BTCTHB": "67879668-532f-41f9-8eb0-55e7593a5ab8", | |
| "ticker.BTCSEK": "6caf1244-655b-460f-beaf-5c56d1f4bea7", | |
| "ticker.BTCNOK": "7532e866-3a03-4514-a4b1-6f86e3a8dc11", | |
| "ticker.BTCGBP": "7b842b7d-d1f9-46fa-a49c-c12f1ad5a533", | |
| "trade.lag": "85174711-be64-4de1-b783-0628995d7914", | |
| "depth.BTCSEK": "8f1fefaa-7c55-4420-ada0-4de15c1c38f3", | |
| "depth.BTCDKK": "9219abb0-b50c-4007-b4d2-51d1711ab19c", | |
| "depth.BTCJPY": "94483e07-d797-4dd4-bc72-dc98f1fd39e3", | |
| "ticker.NMCUSD": "9aaefd15-d101-49f3-a2fd-6b63b85b6bed", | |
| "ticker.LTCAUD": "a046600a-a06c-4ebf-9ffb-bdc8157227e8", | |
| "ticker.BTCJPY": "a39ae532-6a3c-4835-af8c-dda54cb4874e", | |
| "depth.BTCCZK": "a7a970cf-4f6c-4d85-a74e-ac0979049b87", | |
| "ticker.LTCDKK": "b10a706e-e8c7-4ea8-9148-669f86930b36", | |
| "ticker.BTCPLN": "b4a02cb3-2e2d-4a88-aeea-3c66cb604d01", | |
| "test": "bad99f24-fa8b-4938-bfdf-0c1831fc6665", | |
| "ticker.BTCRUB": "bd04f720-3c70-4dce-ae71-2422ab862c65", | |
| "ticker.NMCGBP": "bf5126ba-5187-456f-8ae6-963678d0607f", | |
| "ticker.BTCKRW": "bf85048d-4db9-4dbe-9ca3-5b83a1a4186e", | |
| "ticker.BTCCNY": "c251ec35-56f9-40ab-a4f6-13325c349de4", | |
| "depth.BTCNZD": "cedf8730-bce6-4278-b6fe-9bee42930e95", | |
| "ticker.BTCHKD": "d3ae78dd-01dd-4074-88a7-b8aa03cd28dd", | |
| "ticker.BTCTHB": "d58e3b69-9560-4b9e-8c58-b5c0f3fda5e1", | |
| "ticker.BTCUSD": "d5f06780-30a8-4a48-a2f8-7ed181b4a13f", | |
| "depth.BTCRUB": "d6412ca0-b686-464c-891a-d1ba3943f3c6", | |
| "ticker.NMCEUR": "d8512d04-f262-4a14-82f2-8e5c96c15e68", | |
| "trade.BTC": "dbf1dee9-4f2e-4a08-8cb7-748919a71b21", | |
| "ticker.NMCCAD": "dc28033e-7506-484c-905d-1c811a613323", | |
| "depth.BTCPLN": "e4ff055a-f8bf-407e-af76-676cad319a21", | |
| "ticker.BTCDKK": "e5ce0604-574a-4059-9493-80af46c776b3", | |
| "ticker.BTCAUD": "eb6aaa11-99d0-4f64-9e8c-1140872a423d" | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <script src="https://cdn.pubnub.com/pubnub.min.js"></script> | |
| <script src="mtgox-history.js"></script> | |
| <script>(function(){ | |
| MTGOX.history.hourly({ | |
| channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f', | |
| data : function(response) { console.log(JSON.stringify(response)) }, | |
| error : function() { console.log('NETWORK ERROR') } | |
| }); | |
| MTGOX.history.full({ | |
| limit : 500, // SET AS HIGH AS NEEDED | |
| channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f', | |
| data : function(messages) { console.log(messages) }, | |
| error : function(e) { console.log("NETWORK ERROR") } | |
| }); | |
| })();</script> |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| (function(){ | |
| // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- | |
| // INITIALIZE PUBNUB | |
| // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- | |
| var pubnub = PUBNUB.init({ | |
| subscribe_key : 'sub-c-50d56e1e-2fd9-11e3-a041-02ee2ddab7fe' | |
| }); | |
| // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- | |
| // MTGOX HISTORY INTERFACE | |
| // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- | |
| window.MTGOX = { | |
| history : { | |
| hourly : hourly, | |
| full : full | |
| } | |
| }; | |
| // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- | |
| // GET ALL DATA FOREVER (WITH LIMIT OF COURSE) | |
| // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- | |
| /* | |
| MTGOX.history.full({ | |
| limit : 1000, | |
| channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f', | |
| data : function(messages) { console.log(messages) }, | |
| error : function(e) { console.log("NETWORK ERROR") } | |
| }); | |
| */ | |
| function full(args) { | |
| var chan = args['channel'] ||'d5f06780-30a8-4a48-a2f8-7ed181b4a13f' | |
| , callback = args['data'] || function(){} | |
| , error = args['error'] || function(){} | |
| , limit = +args['limit'] || 5000 | |
| , start = 0 | |
| , count = 100 | |
| , history = [] | |
| , params = { | |
| channel : chan, | |
| count : count, | |
| callback : function(messages) { | |
| var msgs = messages[0]; | |
| start = messages[1]; | |
| params.start = start; | |
| PUBNUB.each( msgs.reverse(), function(m) {history.push(m)} ); | |
| if (history.length >= limit) return callback(history); | |
| if (msgs.length < count) return callback(history); | |
| count = 100; | |
| add_messages(); | |
| }, | |
| error : function(e) { | |
| callback(history); | |
| error(history); | |
| } | |
| }; | |
| add_messages(); | |
| function add_messages() { pubnub.history(params) } | |
| } | |
| // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- | |
| // GET 24 HOURS IN HOURLY INCREMENTS | |
| // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- | |
| /* | |
| MTGOX.history.hourly({ | |
| channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f', | |
| data : function(response) { console.log(response) }, | |
| error : function() { console.log('ERROR') } | |
| }); | |
| */ | |
| function hourly(setup) { | |
| var limit = 24; | |
| var count = 0; | |
| var chan = setup['channel'] ||'d5f06780-30a8-4a48-a2f8-7ed181b4a13f'; | |
| var cb = setup['data'] || function(){}; | |
| var eb = setup['error'] || function(){}; | |
| var now = new Date(); | |
| now.setUTCHours(0); | |
| now.setUTCMinutes(0); | |
| now.setUTCSeconds(0); | |
| now.setUTCMilliseconds(0); | |
| var utc_now = now.getTime(); | |
| var vectors = []; | |
| PUBNUB.each( (new Array(limit)).join(',').split(','), function( _, d ) { | |
| var day = utc_now - 3600000 * d; | |
| pubnub.history({ | |
| limit : 1, | |
| channel : chan, | |
| start : day * 10000, | |
| error : function() { count++; eb(); }, | |
| callback : function(messages) { | |
| // DONE? | |
| if (++count == limit) return cb(vectors); | |
| // ADD TIME SLICES | |
| var res = +(((messages[0][0]||{}).ticker||{}).avg||{}).value; | |
| res && vectors.push([ new Date(day).getUTCHours(), res ]); | |
| // KEEP IT SORTED | |
| vectors.sort(function(a,b){ return a[0] > b[0] && -1 || 1 }); | |
| } | |
| }) | |
| } ); | |
| } | |
| })(); |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
PubNub Load History More than 100 Messages
Sometimes you want to slice back in time over a linear stream of data. And often you'll want to do this at different levels of granularity. That is why PubNub Storage and Playback APIs provide maximum level of flexibility. However sometimes it ends up being a bit tricky to load data with the preferred result set.
There are several considerations you may be seeking when loading transaction history over timelines that can potentially span millions of message in the transaction set. There are some great options available to you and we will cover two of them right now. The examples will be coded in JavaScript. The first example loads a summary of the data by grabbing the snapshots for the beginning of each hour for the past 24 hours. The second example shows you how to load all transactions in full detail and maximum granularity.
Example PubNub Mt.Gox History JavaScript Usage
This following JavaScript file will provide you the
MTGOXinterface.PubNub Mt.Gox History JavaScript Loader
Mt.Gox PubNub Channel Listing for Tickers, Depth and Trades
The following is a list of channels provided by Mt.Gox data feed options you can use in the history
channelparameter field.