Skip to content

Instantly share code, notes, and snippets.

@annanay25

annanay25/nested-vs-flattened.txt

Created February 15, 2022 14:34
Show Gist options
  • Save annanay25/b57b3ff09dedf823adef5b0bc6c7e9a9 to your computer and use it in GitHub Desktop.
Save annanay25/b57b3ff09dedf823adef5b0bc6c7e9a9 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
nested-vs-flattened.txt
======================================================================================
NESTED SCHEMA
======================================================================================
num rows 293741
message SegmentTrace {
repeated group rs {
repeated group ils {
required group il {
required binary name (STRING);
required binary version (STRING);
}
repeated group spans {
repeated group attrs {
required binary key (STRING);
required binary value (STRING);
}
required int32 droppedattributescount (INT(32,true));
required int32 droppedeventscount (INT(32,true));
required int64 end (INT(64,true));
repeated group events {
repeated group attrs {
required binary key (STRING);
required binary value (STRING);
}
required int32 droppedattributescount (INT(32,true));
required binary name (STRING);
required int64 time (INT(64,true));
}
required binary id (STRING);
required int32 kind (INT(8,true));
required binary name (STRING);
required binary parentspanid (STRING);
required int64 start (INT(64,true));
required int32 statuscode (INT(8,true));
required binary statusmsg (STRING);
required binary tracestate (STRING);
}
}
required group resource {
repeated group attrs {
required binary key (STRING);
required binary value (STRING);
}
required binary servicename (STRING);
}
}
required binary traceid (STRING);
}
------------ dumping recursive schema: ------------------
column: SegmentTrace index: -1
column: rs index: -1
column: ils index: -1
column: il index: -1
column: name index: 0
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: version index: 1
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: spans index: -1
column: attrs index: -1
column: key index: 2
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: value index: 3
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: droppedattributescount index: 4
compression: [LZ4_RAW] encoding: [PLAIN RLE]
Go type: int32 Parquet type: INT(32,true)
column: droppedeventscount index: 5
compression: [LZ4_RAW] encoding: [PLAIN RLE]
Go type: int32 Parquet type: INT(32,true)
column: end index: 6
compression: [UNCOMPRESSED] encoding: [RLE DELTA_BINARY_PACKED]
Go type: int64 Parquet type: INT(64,true)
column: events index: -1
column: attrs index: -1
column: key index: 7
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: value index: 8
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: droppedattributescount index: 9
compression: [LZ4_RAW] encoding: [PLAIN RLE]
Go type: int32 Parquet type: INT(32,true)
column: name index: 10
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: time index: 11
compression: [UNCOMPRESSED] encoding: [RLE DELTA_BINARY_PACKED]
Go type: int64 Parquet type: INT(64,true)
column: id index: 12
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: kind index: 13
compression: [LZ4_RAW] encoding: [PLAIN RLE]
Go type: int32 Parquet type: INT(8,true)
column: name index: 14
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: parentspanid index: 15
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: start index: 16
compression: [UNCOMPRESSED] encoding: [RLE DELTA_BINARY_PACKED]
Go type: int64 Parquet type: INT(64,true)
column: statuscode index: 17
compression: [LZ4_RAW] encoding: [PLAIN RLE]
Go type: int32 Parquet type: INT(8,true)
column: statusmsg index: 18
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: tracestate index: 19
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: resource index: -1
column: attrs index: -1
column: key index: 20
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: value index: 21
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: servicename index: 22
compression: [UNCOMPRESSED] encoding: [PLAIN RLE RLE_DICTIONARY]
Go type: string Parquet type: STRING
column: traceid index: 23
compression: [LZ4_RAW] encoding: [DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
------------ row groups -----------------
Num row groups: 1
------------ column sizes -----------------
column.SegmentTrace.rs.ils.il.name size 0 MB
column.SegmentTrace.rs.ils.il.version size 0 MB
column.SegmentTrace.rs.ils.spans.attrs.key size 22 MB
column.SegmentTrace.rs.ils.spans.attrs.value size 32 MB
column.SegmentTrace.rs.ils.spans.droppedattributescount size 2 MB
column.SegmentTrace.rs.ils.spans.droppedeventscount size 2 MB
column.SegmentTrace.rs.ils.spans.end size 41 MB
column.SegmentTrace.rs.ils.spans.events.attrs.key size 28 MB
column.SegmentTrace.rs.ils.spans.events.attrs.value size 39 MB
column.SegmentTrace.rs.ils.spans.events.droppedattributescount size 11 MB
column.SegmentTrace.rs.ils.spans.events.name size 12 MB
column.SegmentTrace.rs.ils.spans.events.time size 41 MB
column.SegmentTrace.rs.ils.spans.id size 164 MB
column.SegmentTrace.rs.ils.spans.kind size 6 MB
column.SegmentTrace.rs.ils.spans.name size 18 MB
column.SegmentTrace.rs.ils.spans.parentspanid size 2 MB
column.SegmentTrace.rs.ils.spans.start size 41 MB
column.SegmentTrace.rs.ils.spans.statuscode size 3 MB
column.SegmentTrace.rs.ils.spans.statusmsg size 2 MB
column.SegmentTrace.rs.ils.spans.tracestate size 2 MB
column.SegmentTrace.rs.resource.attrs.key size 10 MB
column.SegmentTrace.rs.resource.attrs.value size 58 MB
column.SegmentTrace.rs.resource.servicename size 1 MB
column.SegmentTrace.traceid size 4 MB
------------ column sizes -----------------
column: .SegmentTrace.rs.ils.il.name 1412303 values 0.32 MB
column: .SegmentTrace.rs.ils.il.version 1412303 values 0.32 MB
column: .SegmentTrace.rs.ils.spans.attrs.key 12740170 values 22.91 MB
column: .SegmentTrace.rs.ils.spans.attrs.value 12740170 values 32.05 MB
column: .SegmentTrace.rs.ils.spans.droppedattributescount 10847693 values 2.75 MB
column: .SegmentTrace.rs.ils.spans.droppedeventscount 10847693 values 2.75 MB
column: .SegmentTrace.rs.ils.spans.end 10847693 values 41.16 MB
column: .SegmentTrace.rs.ils.spans.events.attrs.key 27920774 values 28.54 MB
column: .SegmentTrace.rs.ils.spans.events.attrs.value 27920774 values 39.92 MB
column: .SegmentTrace.rs.ils.spans.events.droppedattributescount 16278747 values 11.57 MB
column: .SegmentTrace.rs.ils.spans.events.name 16278747 values 12.45 MB
column: .SegmentTrace.rs.ils.spans.events.time 16278747 values 41.49 MB
column: .SegmentTrace.rs.ils.spans.id 10847693 values 164.17 MB
column: .SegmentTrace.rs.ils.spans.kind 10847693 values 6.97 MB
column: .SegmentTrace.rs.ils.spans.name 10847693 values 18.90 MB
column: .SegmentTrace.rs.ils.spans.parentspanid 10847693 values 2.60 MB
column: .SegmentTrace.rs.ils.spans.start 10847693 values 41.08 MB
column: .SegmentTrace.rs.ils.spans.statuscode 10847693 values 3.46 MB
column: .SegmentTrace.rs.ils.spans.statusmsg 10847693 values 2.60 MB
column: .SegmentTrace.rs.ils.spans.tracestate 10847693 values 2.60 MB
column: .SegmentTrace.rs.resource.attrs.key 13338277 values 10.10 MB
column: .SegmentTrace.rs.resource.attrs.value 13338277 values 58.22 MB
column: .SegmentTrace.rs.resource.servicename 1412303 values 1.17 MB
column: .SegmentTrace.traceid 293741 values 4.42 MB
Read : 0.19 MB
======================================================================================
FLATTENED
======================================================================================
num rows 10847693
message SegmentSpanFlattened {
repeated group attrs {
required binary key (STRING);
required binary value (STRING);
}
required int32 droppedattributescount (INT(32,true));
required int32 droppedeventscount (INT(32,true));
required int64 end (INT(64,true));
repeated group events {
repeated group attrs {
required binary key (STRING);
required binary value (STRING);
}
required int32 droppedattributescount (INT(32,true));
required binary name (STRING);
required int64 time (INT(64,true));
}
required binary id (STRING);
required binary il_name (STRING);
required binary il_version (STRING);
required int32 kind (INT(8,true));
required binary name (STRING);
required binary parentspanid (STRING);
repeated group resource_attrs {
required binary key (STRING);
required binary value (STRING);
}
required binary servicename (STRING);
required int64 start (INT(64,true));
required int32 statuscode (INT(8,true));
required binary statusmsg (STRING);
required binary traceid (STRING);
required binary tracestate (STRING);
}
------------ dumping recursive schema: ------------------
column: SegmentSpanFlattened index: -1
column: attrs index: -1
column: key index: 0
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: value index: 1
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: droppedattributescount index: 2
compression: [LZ4_RAW] encoding: [PLAIN]
Go type: int32 Parquet type: INT(32,true)
column: droppedeventscount index: 3
compression: [LZ4_RAW] encoding: [PLAIN]
Go type: int32 Parquet type: INT(32,true)
column: end index: 4
compression: [UNCOMPRESSED] encoding: [DELTA_BINARY_PACKED]
Go type: int64 Parquet type: INT(64,true)
column: events index: -1
column: attrs index: -1
column: key index: 5
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: value index: 6
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: droppedattributescount index: 7
compression: [LZ4_RAW] encoding: [PLAIN RLE]
Go type: int32 Parquet type: INT(32,true)
column: name index: 8
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: time index: 9
compression: [UNCOMPRESSED] encoding: [RLE DELTA_BINARY_PACKED]
Go type: int64 Parquet type: INT(64,true)
column: id index: 10
compression: [LZ4_RAW] encoding: [DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: il_name index: 11
compression: [LZ4_RAW] encoding: [DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: il_version index: 12
compression: [LZ4_RAW] encoding: [DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: kind index: 13
compression: [LZ4_RAW] encoding: [PLAIN]
Go type: int32 Parquet type: INT(8,true)
column: name index: 14
compression: [LZ4_RAW] encoding: [DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: parentspanid index: 15
compression: [LZ4_RAW] encoding: [DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: resource_attrs index: -1
column: key index: 16
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: value index: 17
compression: [LZ4_RAW] encoding: [RLE DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: servicename index: 18
compression: [UNCOMPRESSED] encoding: [PLAIN RLE_DICTIONARY]
Go type: string Parquet type: STRING
column: start index: 19
compression: [UNCOMPRESSED] encoding: [DELTA_BINARY_PACKED]
Go type: int64 Parquet type: INT(64,true)
column: statuscode index: 20
compression: [LZ4_RAW] encoding: [PLAIN]
Go type: int32 Parquet type: INT(8,true)
column: statusmsg index: 21
compression: [LZ4_RAW] encoding: [DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: traceid index: 22
compression: [LZ4_RAW] encoding: [DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
column: tracestate index: 23
compression: [LZ4_RAW] encoding: [DELTA_LENGTH_BYTE_ARRAY]
Go type: string Parquet type: STRING
------------ row groups -----------------
Num row groups: 30
------------ column sizes -----------------
column.SegmentSpanFlattened.attrs.key size 17 MB
column.SegmentSpanFlattened.attrs.value size 26 MB
column.SegmentSpanFlattened.droppedattributescount size 0 MB
column.SegmentSpanFlattened.droppedeventscount size 0 MB
column.SegmentSpanFlattened.end size 38 MB
column.SegmentSpanFlattened.events.attrs.key size 14 MB
column.SegmentSpanFlattened.events.attrs.value size 22 MB
column.SegmentSpanFlattened.events.droppedattributescount size 4 MB
column.SegmentSpanFlattened.events.name size 4 MB
column.SegmentSpanFlattened.events.time size 34 MB
column.SegmentSpanFlattened.id size 161 MB
column.SegmentSpanFlattened.il_name size 0 MB
column.SegmentSpanFlattened.il_version size 0 MB
column.SegmentSpanFlattened.kind size 4 MB
column.SegmentSpanFlattened.name size 16 MB
column.SegmentSpanFlattened.parentspanid size 0 MB
column.SegmentSpanFlattened.resource_attrs.key size 28 MB
column.SegmentSpanFlattened.resource_attrs.value size 107 MB
column.SegmentSpanFlattened.servicename size 7 MB
column.SegmentSpanFlattened.start size 38 MB
column.SegmentSpanFlattened.statuscode size 0 MB
column.SegmentSpanFlattened.statusmsg size 0 MB
column.SegmentSpanFlattened.traceid size 6 MB
column.SegmentSpanFlattened.tracestate size 0 MB
------------ column sizes -----------------
column: .SegmentSpanFlattened.attrs.key 12740170 values 17.26 MB
column: .SegmentSpanFlattened.attrs.value 12740170 values 26.24 MB
column: .SegmentSpanFlattened.droppedattributescount 10847693 values 0.17 MB
column: .SegmentSpanFlattened.droppedeventscount 10847693 values 0.17 MB
column: .SegmentSpanFlattened.end 10847693 values 38.58 MB
column: .SegmentSpanFlattened.events.attrs.key 18518889 values 14.67 MB
column: .SegmentSpanFlattened.events.attrs.value 18518889 values 22.18 MB
column: .SegmentSpanFlattened.events.droppedattributescount 16278747 values 4.01 MB
column: .SegmentSpanFlattened.events.name 16278747 values 4.95 MB
column: .SegmentSpanFlattened.events.time 16278747 values 34.00 MB
column: .SegmentSpanFlattened.id 10847693 values 161.87 MB
column: .SegmentSpanFlattened.il_name 10847693 values 0.01 MB
column: .SegmentSpanFlattened.il_version 10847693 values 0.01 MB
column: .SegmentSpanFlattened.kind 10847693 values 4.39 MB
column: .SegmentSpanFlattened.name 10847693 values 16.63 MB
column: .SegmentSpanFlattened.parentspanid 10847693 values 0.01 MB
column: .SegmentSpanFlattened.resource_attrs.key 99060646 values 28.46 MB
column: .SegmentSpanFlattened.resource_attrs.value 99060646 values 107.97 MB
column: .SegmentSpanFlattened.servicename 10847693 values 7.32 MB
column: .SegmentSpanFlattened.start 10847693 values 38.51 MB
column: .SegmentSpanFlattened.statuscode 10847693 values 0.87 MB
column: .SegmentSpanFlattened.statusmsg 10847693 values 0.01 MB
column: .SegmentSpanFlattened.traceid 10847693 values 6.29 MB
column: .SegmentSpanFlattened.tracestate 10847693 values 0.01 MB
Read : 0.69 MB
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment