Created
May 4, 2018 22:17
-
-
Save toddlipcon/74735cfdeb453420097f93bd836037db to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
@Test | |
public void testLoadTable() throws Exception { | |
Table table = catalog_.getOrLoadTable("gftrrhrn_managed", "stitched_pts"); | |
if (table instanceof IncompleteTable) { | |
IncompleteTable it = (IncompleteTable)table; | |
LOG.warn("failed to load: {}", it.getCause()); | |
} | |
LOG.info("cols: " + table.getColumnNames()); | |
table.getLock().lock(); | |
try { | |
TSerializer s = new TSerializer(new TCompactProtocol.Factory()); | |
TCatalogObject obj = table.toTCatalogObject(); | |
byte[] b = s.serialize(obj); | |
int len = b.length; | |
int clen = Snappy.compress(b).length; | |
LOG.info("full size: " + len + " compressed " + clen); | |
for (Entry<_Fields, FieldMetaData> e : THdfsPartition.metaDataMap.entrySet()) { | |
if (e.getValue().requirementType == TFieldRequirementType.REQUIRED) { | |
continue; | |
} | |
for (THdfsPartition p : obj.table.hdfs_table.partitions.values()) { | |
p.setFieldValue(e.getKey(), null); | |
} | |
b = s.serialize(obj); | |
int newLen = b.length; | |
int newClen = Snappy.compress(b).length; | |
LOG.info("Field " + e.getValue().fieldName + ": " + (len - newLen) + | |
" compressed " + (clen - newClen)); | |
len = newLen; | |
clen = newClen; | |
} | |
obj.table.hdfs_table.partitions.clear(); | |
int newLen = s.serialize(obj).length; | |
LOG.info("Other partition metadata: " + (len - newLen)); | |
len = newLen; | |
LOG.info("minimal catalog object: " + s.serialize(table.toMinimalTCatalogObject()).length); | |
} finally { | |
table.getLock().unlock(); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment