Skip to content

Instantly share code, notes, and snippets.

@mccutchen

mccutchen/example_row.json

Created January 6, 2012 22:51
Show Gist options
  • Save mccutchen/1572822 to your computer and use it in GitHub Desktop.
Save mccutchen/1572822 to your computer and use it in GitHub Desktop.
Download ZIP
Compression test
Raw
example_row.json
[{"privacy": "0", "created": "1325864138", "modified": "1325887899", "global_hash": "bGuRVV", "user_hash": "8rcCNv", "login": "5aQOloVyMze"}, {"privacy": "1", "created": "1325889382", "modified": "1325881184", "global_hash": "7eGqml", "user_hash": "VsrNOH", "login": "87YwHtJVoWthckEYLdum9akgmN4QzB"}, {"privacy": "1", "created": "1325918261", "modified": "1325983013", "global_hash": "a5rlvC", "user_hash": "OTHiE6", "login": "qvIKx3iw"}, {"privacy": "0", "created": "1325946981", "modified": "1325919728", "global_hash": "aZufjE", "user_hash": "7EwMRB", "login": "rpgOLjYkmzFyl2MIiTQAbAv"}, {"privacy": "0", "created": "1325892447", "modified": "1325893896", "global_hash": "rN4QdJ", "user_hash": "wlVmhv", "login": "HEN2KhU"}, {"privacy": "0", "created": "1325822276", "modified": "1325924205", "global_hash": "S2Vcmb", "user_hash": "F5nnD9", "login": "UBoEMP4giVjzo7SJmGG0ptyD"}, {"privacy": "1", "created": "1325890208", "modified": "1325986230", "global_hash": "GvdjO8", "user_hash": "j5qyky", "login": "MgYFhjgp63uo4CIx"}, {"privacy": "0", "created": "1325874825", "modified": "1325915653", "global_hash": "7xyCQl", "user_hash": "YG0GS0", "login": "Yqz6Ufa7s9tZNvPYdiX"}, {"privacy": "1", "created": "1325970439", "modified": "1325922534", "global_hash": "1JCLtB", "user_hash": "HGesPr", "login": "NKcPBsSi0HV9oU5yt9wCPmovM"}, {"privacy": "1", "created": "1325887609", "modified": "1325842871", "global_hash": "hJ4Kny", "user_hash": "O2lYQP", "login": "iUQAWSLAl1"}, {"privacy": "0", "created": "1325828781", "modified": "1325838002", "global_hash": "q5UiiT", "user_hash": "hAytkm", "login": "zy3U"}, {"privacy": "1", "created": "1325960243", "modified": "1325969973", "global_hash": "qtvTAR", "user_hash": "uydXzz", "login": "fXnkJAUvjPM1NWZEgN5HyW"}, {"privacy": "1", "created": "1325925134", "modified": "1325818600", "global_hash": "gRPCk9", "user_hash": "GmXSQZ", "login": "AFwLiugFtxvPj4DVV8ohajXJkXVw"}, {"privacy": "0", "created": "1325952148", "modified": "1325967895", "global_hash": "1ctOB5", "user_hash": "fzRduO", "login": "kSqHumhWVKqUhIaORghDrxOja9a"}, {"privacy": "0", "created": "1325830299", "modified": "1325855409", "global_hash": "8BMBru", "user_hash": "NyCZw0", "login": "MynKbtDyixGyG"}, {"privacy": "1", "created": "1325854015", "modified": "1325803498", "global_hash": "4m0stP", "user_hash": "QOpBN2", "login": "TJoYs1gEA"}, {"privacy": "1", "created": "1325886844", "modified": "1325982895", "global_hash": "UiTD5u", "user_hash": "p3mQr9", "login": "cQUX"}, {"privacy": "1", "created": "1325825189", "modified": "1325963334", "global_hash": "EAGaZI", "user_hash": "5BUere", "login": "y55R1ck7WaBCBIRUnVzJQNGPvd"}, {"privacy": "1", "created": "1325835836", "modified": "1325874634", "global_hash": "3ihdST", "user_hash": "znju9a", "login": "1xt8JTPFhZt7aD9wQSOcg8Y"}, {"privacy": "1", "created": "1325878058", "modified": "1325949796", "global_hash": "6ieEQI", "user_hash": "7Q7Hui", "login": "eVvUPB"}, {"privacy": "0", "created": "1325949034", "modified": "1325988278", "global_hash": "LyuTfR", "user_hash": "7tNQ84", "login": "IBqmaMST8Z1yurAnMqy"}, {"privacy": "1", "created": "1325982826", "modified": "1325852384", "global_hash": "Mq2lFY", "user_hash": "xeXno0", "login": "J6INhdNA421rjAxj"}, {"privacy": "1", "created": "1325959200", "modified": "1325843147", "global_hash": "aZO5rR", "user_hash": "IyvdLJ", "login": "TpiU1pmu3t9SgVCj7GEzMse2wKeyZjf"}, {"privacy": "0", "created": "1325881767", "modified": "1325817712", "global_hash": "gNXzfk", "user_hash": "PiWRxu", "login": "88KYYRIGgvNCIJ7j5rjJmApfFlr9k"}, {"privacy": "0", "created": "1325963097", "modified": "1325912593", "global_hash": "a2Vi3a", "user_hash": "uTx9EA", "login": "0wmOMgTp9bwuGhpIvbOGuQW553"}, {"privacy": "1", "created": "1325977678", "modified": "1325838144", "global_hash": "8ozee2", "user_hash": "21KCOV", "login": "v8j6"}, {"privacy": "1", "created": "1325979550", "modified": "1325798037", "global_hash": "wssHgR", "user_hash": "poFqy1", "login": "n3csrTjmLFzcAjde2L8HFYp"}, {"privacy": "1", "created": "1325820365", "modified": "1325793273", "global_hash": "IHSiiN", "user_hash": "hkhvmB", "login": "Iu3U27DxL05wm4lUTEZmMrOIPTsiffAQ"}, {"privacy": "1", "created": "1325942249", "modified": "1325865216", "global_hash": "B5Uqs6", "user_hash": "8VSxu7", "login": "gSNLbYbaAurh"}, {"privacy": "1", "created": "1325838503", "modified": "1325969819", "global_hash": "ZonNBR", "user_hash": "Ow48ua", "login": "YgsWEefWh"}, {"privacy": "1", "created": "1325941350", "modified": "1325985425", "global_hash": "FQMp6A", "user_hash": "g0uJZb", "login": "VQV8BjDm"}, {"privacy": "0", "created": "1325831402", "modified": "1325816158", "global_hash": "2Hv00F", "user_hash": "XCPnhu", "login": "5OuCTsxIRMAT3C1"}, {"privacy": "1", "created": "1325941794", "modified": "1325806155", "global_hash": "3ypSQT", "user_hash": "0sBLNs", "login": "WDXPHG3UOT1y1I5fY"}, {"privacy": "0", "created": "1325874200", "modified": "1325967939", "global_hash": "5uqFAb", "user_hash": "1YacaS", "login": "Ecnx8lStaiut7wrc0"}, {"privacy": "1", "created": "1325844337", "modified": "1325873360", "global_hash": "UopbR5", "user_hash": "W8WSjU", "login": "ipS7My4oX"}, {"privacy": "1", "created": "1325826526", "modified": "1325897062", "global_hash": "4qaemk", "user_hash": "xRqik5", "login": "JDMJF"}, {"privacy": "0", "created": "1325986638", "modified": "1325830527", "global_hash": "lD3blg", "user_hash": "RjBMr6", "login": "NWqqJgyQxHYj2xfgkvGau"}, {"privacy": "1", "created": "1325943524", "modified": "1325854474", "global_hash": "DISVX0", "user_hash": "QvQjZ6", "login": "pjagg79tZpthwho"}, {"privacy": "1", "created": "1325987048", "modified": "1325953378", "global_hash": "CwHTs4", "user_hash": "iYmhH2", "login": "lrAACGi5"}, {"privacy": "1", "created": "1325799084", "modified": "1325893952", "global_hash": "RiZg6e", "user_hash": "3hzZko", "login": "hHAi1r6P5xJ9D"}, {"privacy": "0", "created": "1325828150", "modified": "1325812800", "global_hash": "fnQlYS", "user_hash": "aoTSIO", "login": "WQXswPnmhsDpWSztmwz"}, {"privacy": "0", "created": "1325831319", "modified": "1325970852", "global_hash": "MUy7lK", "user_hash": "8wxgdw", "login": "iVJorglTqEOHwbBz89x2JdV36"}, {"privacy": "0", "created": "1325882205", "modified": "1325870691", "global_hash": "UiXsn9", "user_hash": "2gENSo", "login": "Uf8MX8rdi"}, {"privacy": "1", "created": "1325921093", "modified": "1325828935", "global_hash": "mSMAJw", "user_hash": "xA1paP", "login": "NevyLbg18jK"}, {"privacy": "0", "created": "1325901333", "modified": "1325957104", "global_hash": "xfbItl", "user_hash": "B9wEJw", "login": "wWAxENwwstjZmZgr"}, {"privacy": "1", "created": "1325989206", "modified": "1325945434", "global_hash": "6ff3O4", "user_hash": "gFkJE3", "login": "D1BiocmHZEEjt5RIN7AE6oXQpMEW"}, {"privacy": "1", "created": "1325843637", "modified": "1325849459", "global_hash": "bNxiMt", "user_hash": "yxZQMJ", "login": "9De4BC"}, {"privacy": "1", "created": "1325888630", "modified": "1325815117", "global_hash": "fGx5Po", "user_hash": "seNbac", "login": "7in6AeSf1c50zmdguJkQi"}, {"privacy": "0", "created": "1325945812", "modified": "1325975009", "global_hash": "QAiS3r", "user_hash": "AFEzO4", "login": "ZhAQWBbVgTi3tLZF7j2FdgXPjheOl"}, {"privacy": "1", "created": "1325969692", "modified": "1325941610", "global_hash": "xQ1YOM", "user_hash": "4a4oWY", "login": "GrPNreFr7Fy3"}]
Raw
example_row.txt
0|1325864138|1325887899|bGuRVV|8rcCNv|5aQOloVyMze,1|1325889382|1325881184|7eGqml|VsrNOH|87YwHtJVoWthckEYLdum9akgmN4QzB,1|1325918261|1325983013|a5rlvC|OTHiE6|qvIKx3iw,0|1325946981|1325919728|aZufjE|7EwMRB|rpgOLjYkmzFyl2MIiTQAbAv,0|1325892447|1325893896|rN4QdJ|wlVmhv|HEN2KhU,0|1325822276|1325924205|S2Vcmb|F5nnD9|UBoEMP4giVjzo7SJmGG0ptyD,1|1325890208|1325986230|GvdjO8|j5qyky|MgYFhjgp63uo4CIx,0|1325874825|1325915653|7xyCQl|YG0GS0|Yqz6Ufa7s9tZNvPYdiX,1|1325970439|1325922534|1JCLtB|HGesPr|NKcPBsSi0HV9oU5yt9wCPmovM,1|1325887609|1325842871|hJ4Kny|O2lYQP|iUQAWSLAl1,0|1325828781|1325838002|q5UiiT|hAytkm|zy3U,1|1325960243|1325969973|qtvTAR|uydXzz|fXnkJAUvjPM1NWZEgN5HyW,1|1325925134|1325818600|gRPCk9|GmXSQZ|AFwLiugFtxvPj4DVV8ohajXJkXVw,0|1325952148|1325967895|1ctOB5|fzRduO|kSqHumhWVKqUhIaORghDrxOja9a,0|1325830299|1325855409|8BMBru|NyCZw0|MynKbtDyixGyG,1|1325854015|1325803498|4m0stP|QOpBN2|TJoYs1gEA,1|1325886844|1325982895|UiTD5u|p3mQr9|cQUX,1|1325825189|1325963334|EAGaZI|5BUere|y55R1ck7WaBCBIRUnVzJQNGPvd,1|1325835836|1325874634|3ihdST|znju9a|1xt8JTPFhZt7aD9wQSOcg8Y,1|1325878058|1325949796|6ieEQI|7Q7Hui|eVvUPB,0|1325949034|1325988278|LyuTfR|7tNQ84|IBqmaMST8Z1yurAnMqy,1|1325982826|1325852384|Mq2lFY|xeXno0|J6INhdNA421rjAxj,1|1325959200|1325843147|aZO5rR|IyvdLJ|TpiU1pmu3t9SgVCj7GEzMse2wKeyZjf,0|1325881767|1325817712|gNXzfk|PiWRxu|88KYYRIGgvNCIJ7j5rjJmApfFlr9k,0|1325963097|1325912593|a2Vi3a|uTx9EA|0wmOMgTp9bwuGhpIvbOGuQW553,1|1325977678|1325838144|8ozee2|21KCOV|v8j6,1|1325979550|1325798037|wssHgR|poFqy1|n3csrTjmLFzcAjde2L8HFYp,1|1325820365|1325793273|IHSiiN|hkhvmB|Iu3U27DxL05wm4lUTEZmMrOIPTsiffAQ,1|1325942249|1325865216|B5Uqs6|8VSxu7|gSNLbYbaAurh,1|1325838503|1325969819|ZonNBR|Ow48ua|YgsWEefWh,1|1325941350|1325985425|FQMp6A|g0uJZb|VQV8BjDm,0|1325831402|1325816158|2Hv00F|XCPnhu|5OuCTsxIRMAT3C1,1|1325941794|1325806155|3ypSQT|0sBLNs|WDXPHG3UOT1y1I5fY,0|1325874200|1325967939|5uqFAb|1YacaS|Ecnx8lStaiut7wrc0,1|1325844337|1325873360|UopbR5|W8WSjU|ipS7My4oX,1|1325826526|1325897062|4qaemk|xRqik5|JDMJF,0|1325986638|1325830527|lD3blg|RjBMr6|NWqqJgyQxHYj2xfgkvGau,1|1325943524|1325854474|DISVX0|QvQjZ6|pjagg79tZpthwho,1|1325987048|1325953378|CwHTs4|iYmhH2|lrAACGi5,1|1325799084|1325893952|RiZg6e|3hzZko|hHAi1r6P5xJ9D,0|1325828150|1325812800|fnQlYS|aoTSIO|WQXswPnmhsDpWSztmwz,0|1325831319|1325970852|MUy7lK|8wxgdw|iVJorglTqEOHwbBz89x2JdV36,0|1325882205|1325870691|UiXsn9|2gENSo|Uf8MX8rdi,1|1325921093|1325828935|mSMAJw|xA1paP|NevyLbg18jK,0|1325901333|1325957104|xfbItl|B9wEJw|wWAxENwwstjZmZgr,1|1325989206|1325945434|6ff3O4|gFkJE3|D1BiocmHZEEjt5RIN7AE6oXQpMEW,1|1325843637|1325849459|bNxiMt|yxZQMJ|9De4BC,1|1325888630|1325815117|fGx5Po|seNbac|7in6AeSf1c50zmdguJkQi,0|1325945812|1325975009|QAiS3r|AFEzO4|ZhAQWBbVgTi3tLZF7j2FdgXPjheOl,1|1325969692|1325941610|xQ1YOM|4a4oWY|GrPNreFr7Fy3
Raw
make_test_data.py
from __future__ import with_statement
import random
import string
import time
import simplejson as json
row_count = 10
record_count = 50
chars = string.letters + string.digits
def random_string(size):
return ''.join(random.choice(chars) for _ in xrange(size))
def login():
return random_string(random.randint(4, 32))
def hash():
return random_string(6)
def timestamp():
return str(int(time.time()) + random.randint(-100000, 100000))
def privacy():
return str(random.randint(0,1))
def row():
return {
'login': login(),
'user_hash': hash(),
'global_hash': hash(),
'created': timestamp(),
'modified': timestamp(),
'privacy': privacy(),
}
rows = [[row() for _ in xrange(record_count)] for _ in xrange(row_count)]
with open('test_data.json', 'w') as f:
for row in rows:
f.write(json.dumps(row) + '\n')
with open('test_data.txt', 'w') as f:
for row in rows:
f.write(','.join('|'.join(record.values()) for record in row) + '\n')
Raw
test_compression.sh
#!/bin/sh
echo "Running 100 tests..."
for i in {0..100}
do
python make_test_data.py
snzip -c test_data.json > test_data.json.sz
snzip -c test_data.txt > test_data.txt.sz
json_size=`wc --bytes test_data.json.sz | cut -d ' ' -f 1`
text_size=`wc --bytes test_data.txt.sz | cut -d ' ' -f 1`
pct=`echo "scale=4; (1 - ($text_size / $json_size)) * 100" | bc`
echo "$pct %"
done
Raw
test_results.log
Running 100 tests...
22.2100 %
22.0900 %
22.2000 %
22.4000 %
22.4300 %
22.2900 %
22.3800 %
22.4400 %
22.2600 %
22.4600 %
22.6000 %
22.6200 %
22.4600 %
22.6700 %
22.5300 %
21.8800 %
22.6600 %
22.8000 %
22.3300 %
22.4400 %
22.8900 %
22.1500 %
22.2800 %
22.7400 %
22.5300 %
22.4600 %
22.9000 %
22.4100 %
22.2000 %
22.7800 %
22.3700 %
22.4700 %
22.2100 %
22.3100 %
22.3300 %
22.5300 %
22.6000 %
22.8800 %
22.5500 %
22.5400 %
22.3300 %
22.5500 %
22.5000 %
22.9800 %
22.7200 %
22.6900 %
22.2100 %
22.2300 %
22.3400 %
22.5600 %
22.5000 %
22.2300 %
21.8900 %
22.1900 %
22.4800 %
22.2700 %
22.6600 %
22.5100 %
22.1800 %
22.5400 %
22.2800 %
22.4700 %
22.2900 %
22.2100 %
22.6400 %
22.2700 %
22.9700 %
22.5600 %
22.3400 %
22.6900 %
22.8200 %
22.4900 %
22.6600 %
22.4600 %
22.4000 %
22.5300 %
22.3000 %
22.2900 %
22.4000 %
22.5100 %
22.6200 %
22.1600 %
22.2600 %
22.0800 %
22.6100 %
22.7200 %
22.5800 %
22.7000 %
22.3800 %
22.2800 %
22.5200 %
22.6000 %
22.3100 %
22.3700 %
22.5400 %
22.6100 %
22.8300 %
22.2700 %
22.1200 %
22.0300 %
23.0200 %
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment