Skip to content

Instantly share code, notes, and snippets.

@wesm

wesm/gist:5100e41173a3b5e53437b7a887e4383a

Created June 13, 2017 03:15
Show Gist options
  • Save wesm/5100e41173a3b5e53437b7a887e4383a to your computer and use it in GitHub Desktop.
Save wesm/5100e41173a3b5e53437b7a887e4383a to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.json
{
"schema": {
"fields": [
{
"name": "dict1",
"nullable": true,
"type": {
"name": "utf8"
},
"dictionary": {
"id": 1,
"indexType": {
"name": "int",
"bitWidth": 32,
"isSigned": true
},
"isOrdered": false
},
"children": [],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 32
}
]
}
},
{
"name": "sparse",
"nullable": true,
"type": {
"name": "utf8"
},
"dictionary": {
"id": 1,
"indexType": {
"name": "int",
"bitWidth": 8,
"isSigned": true
},
"isOrdered": false
},
"children": [],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 8
}
]
}
},
{
"name": "dense",
"nullable": true,
"type": {
"name": "utf8"
},
"dictionary": {
"id": 2,
"indexType": {
"name": "int",
"bitWidth": 32,
"isSigned": true
},
"isOrdered": false
},
"children": [],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 32
}
]
}
},
{
"name": "list of encoded string",
"nullable": true,
"type": {
"name": "list"
},
"children": [
{
"name": "item",
"nullable": true,
"type": {
"name": "utf8"
},
"dictionary": {
"id": 1,
"indexType": {
"name": "int",
"bitWidth": 8,
"isSigned": true
},
"isOrdered": false
},
"children": [],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 8
}
]
}
}
],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "OFFSET",
"typeBitWidth": 32
}
]
}
},
{
"name": "encoded list<int8>",
"nullable": true,
"type": {
"name": "list"
},
"dictionary": {
"id": 3,
"indexType": {
"name": "int",
"bitWidth": 8,
"isSigned": true
},
"isOrdered": false
},
"children": [
{
"name": "item",
"nullable": true,
"type": {
"name": "int",
"bitWidth": 8,
"isSigned": true
},
"children": [],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 8
}
]
}
}
],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 8
}
]
}
}
]
},
"dictionaries": [
{
"id": 3,
"data": {
"count": 3,
"columns": [
{
"name": "dictionary",
"count": 3,
"VALIDITY": [
1,
1,
1
],
"OFFSET": [
0,
2,
2,
3
],
"children": [
{
"name": "item",
"count": 3,
"VALIDITY": [
1,
1,
1
],
"DATA": [
0,
1,
2
],
"children": []
}
]
}
]
}
},
{
"id": 2,
"data": {
"count": 4,
"columns": [
{
"name": "dictionary",
"count": 4,
"VALIDITY": [
1,
1,
1,
1
],
"OFFSET": [
0,
3,
6,
9,
12
],
"DATA": [
"foo",
"bar",
"baz",
"qux"
],
"children": []
}
]
}
},
{
"id": 1,
"data": {
"count": 3,
"columns": [
{
"name": "dictionary",
"count": 3,
"VALIDITY": [
1,
1,
1
],
"OFFSET": [
0,
3,
6,
9
],
"DATA": [
"foo",
"bar",
"baz"
],
"children": []
}
]
}
}
],
"batches": [
{
"count": 6,
"columns": [
{
"name": "dict1",
"count": 6,
"VALIDITY": [
1,
1,
0,
1,
1,
1
],
"DATA": [
1,
2,
0,
0,
2,
0
],
"children": []
},
{
"name": "sparse",
"count": 6,
"VALIDITY": [
1,
1,
0,
1,
1,
1
],
"DATA": [
0,
0,
0,
2,
1,
1
],
"children": []
},
{
"name": "dense",
"count": 6,
"VALIDITY": [
1,
1,
0,
1,
1,
1
],
"DATA": [
3,
0,
0,
1,
0,
2
],
"children": []
},
{
"name": "list of encoded string",
"count": 6,
"VALIDITY": [
1,
1,
0,
1,
1,
1
],
"OFFSET": [
0,
0,
2,
2,
5,
6,
9
],
"children": [
{
"name": "item",
"count": 9,
"VALIDITY": [
1,
1,
1,
1,
1,
1,
1,
1,
1
],
"DATA": [
0,
1,
2,
0,
1,
2,
0,
1,
2
],
"children": []
}
]
},
{
"name": "encoded list<int8>",
"count": 6,
"VALIDITY": [
1,
1,
0,
1,
1,
1
],
"DATA": [
0,
1,
0,
0,
1,
2
],
"children": []
}
]
}
]
}
{
"schema": {
"fields": [
{
"name": "dict1",
"nullable": true,
"type": {
"name": "utf8"
},
"dictionary": {
"id": 1,
"indexType": {
"name": "int",
"bitWidth": 32,
"isSigned": true
},
"isOrdered": false
},
"children": [],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 32
}
]
}
},
{
"name": "sparse",
"nullable": true,
"type": {
"name": "utf8"
},
"dictionary": {
"id": 1,
"indexType": {
"name": "int",
"bitWidth": 8,
"isSigned": true
},
"isOrdered": false
},
"children": [],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 8
}
]
}
},
{
"name": "dense",
"nullable": true,
"type": {
"name": "utf8"
},
"dictionary": {
"id": 2,
"indexType": {
"name": "int",
"bitWidth": 32,
"isSigned": true
},
"isOrdered": false
},
"children": [],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 32
}
]
}
},
{
"name": "list of encoded string",
"nullable": true,
"type": {
"name": "list"
},
"children": [
{
"name": "item",
"nullable": true,
"type": {
"name": "utf8"
},
"dictionary": {
"id": 1,
"indexType": {
"name": "int",
"bitWidth": 8,
"isSigned": true
},
"isOrdered": false
},
"children": [],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 8
}
]
}
}
],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "OFFSET",
"typeBitWidth": 32
}
]
}
},
{
"name": "encoded list<int8>",
"nullable": true,
"type": {
"name": "list"
},
"dictionary": {
"id": 3,
"indexType": {
"name": "int",
"bitWidth": 8,
"isSigned": true
},
"isOrdered": false
},
"children": [
{
"name": "item",
"nullable": true,
"type": {
"name": "int",
"bitWidth": 8,
"isSigned": true
},
"children": [],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 8
}
]
}
}
],
"typeLayout": {
"vectors": [
{
"type": "VALIDITY",
"typeBitWidth": 1
},
{
"type": "DATA",
"typeBitWidth": 8
}
]
}
}
]
},
"dictionaries": [
{
"id": 3,
"data": {
"count": 3,
"columns": [
{
"name": "dictionary",
"count": 3,
"VALIDITY": [
1,
1,
1
],
"OFFSET": [
0,
2,
2,
3
],
"children": [
{
"name": "item",
"count": 3,
"VALIDITY": [
1,
1,
1
],
"DATA": [
0,
1,
2
],
"children": []
}
]
}
]
}
},
{
"id": 2,
"data": {
"count": 4,
"columns": [
{
"name": "dictionary",
"count": 4,
"VALIDITY": [
1,
1,
1,
1
],
"OFFSET": [
0,
3,
6,
9,
12
],
"DATA": [
"foo",
"bar",
"baz",
"qux"
],
"children": []
}
]
}
},
{
"id": 1,
"data": {
"count": 3,
"columns": [
{
"name": "dictionary",
"count": 3,
"VALIDITY": [
1,
1,
1
],
"OFFSET": [
0,
3,
6,
9
],
"DATA": [
"foo",
"bar",
"baz"
],
"children": []
}
]
}
}
],
"batches": [
{
"count": 6,
"columns": [
{
"name": "dict1",
"count": 6,
"VALIDITY": [
1,
1,
0,
1,
1,
1
],
"DATA": [
1,
2,
0,
0,
2,
0
],
"children": []
},
{
"name": "sparse",
"count": 6,
"VALIDITY": [
1,
1,
0,
1,
1,
1
],
"DATA": [
0,
0,
0,
2,
1,
1
],
"children": []
},
{
"name": "dense",
"count": 6,
"VALIDITY": [
1,
1,
0,
1,
1,
1
],
"DATA": [
3,
0,
0,
1,
0,
2
],
"children": []
},
{
"name": "list of encoded string",
"count": 6,
"VALIDITY": [
1,
1,
0,
1,
1,
1
],
"OFFSET": [
0,
0,
2,
2,
5,
6,
9
],
"children": [
{
"name": "item",
"count": 9,
"VALIDITY": [
1,
1,
1,
1,
1,
1,
1,
1,
1
],
"DATA": [
0,
1,
2,
0,
1,
2,
0,
1,
2
],
"children": []
}
]
},
{
"name": "encoded list<int8>",
"count": 6,
"VALIDITY": [
1,
1,
0,
1,
1,
1
],
"DATA": [
0,
1,
0,
0,
1,
2
],
"children": []
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment