Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save toddlipcon/74735cfdeb453420097f93bd836037db to your computer and use it in GitHub Desktop.
Save toddlipcon/74735cfdeb453420097f93bd836037db to your computer and use it in GitHub Desktop.
@Test
public void testLoadTable() throws Exception {
Table table = catalog_.getOrLoadTable("gftrrhrn_managed", "stitched_pts");
if (table instanceof IncompleteTable) {
IncompleteTable it = (IncompleteTable)table;
LOG.warn("failed to load: {}", it.getCause());
}
LOG.info("cols: " + table.getColumnNames());
table.getLock().lock();
try {
TSerializer s = new TSerializer(new TCompactProtocol.Factory());
TCatalogObject obj = table.toTCatalogObject();
byte[] b = s.serialize(obj);
int len = b.length;
int clen = Snappy.compress(b).length;
LOG.info("full size: " + len + " compressed " + clen);
for (Entry<_Fields, FieldMetaData> e : THdfsPartition.metaDataMap.entrySet()) {
if (e.getValue().requirementType == TFieldRequirementType.REQUIRED) {
continue;
}
for (THdfsPartition p : obj.table.hdfs_table.partitions.values()) {
p.setFieldValue(e.getKey(), null);
}
b = s.serialize(obj);
int newLen = b.length;
int newClen = Snappy.compress(b).length;
LOG.info("Field " + e.getValue().fieldName + ": " + (len - newLen) +
" compressed " + (clen - newClen));
len = newLen;
clen = newClen;
}
obj.table.hdfs_table.partitions.clear();
int newLen = s.serialize(obj).length;
LOG.info("Other partition metadata: " + (len - newLen));
len = newLen;
LOG.info("minimal catalog object: " + s.serialize(table.toMinimalTCatalogObject()).length);
} finally {
table.getLock().unlock();
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment