Created
March 23, 2022 08:58
-
-
Save nsivabalan/0fe5512e91a549dce5655cef57b1c523 to your computer and use it in GitHub Desktop.
MDT records
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 257508 [Spring Shell] WARN org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - XXXX Hoodie rec: HoodieKey { recordKey=__all_partitions__ partitionPath=files} : {"_hoodie_commit_time": null, "_hoodie_commit_seqno": null, "_hoodie_record_key": null, "_hoodie_partition_path": null, "_hoodie_file_name": null, "key": "__all_partitions__", "type": 1, "filesystemMetadata": {".": {"size": 0, "isDeleted": false}}, "BloomFilterMetadata": null, "ColumnStatsMetadata": null} | |
| 257511 [Spring Shell] INFO org.apache.hudi.common.table.log.HoodieLogFormatReader - Moving to the next reader for logfile HoodieLogFile{pathStr='file:/Users/nsb/Documents/personal/temp/onehouse_tbl_debug_mar23_s3/.hoodie/metadata/files/.files-0000_20220323070911012.log.3_0-82-771', fileLen=-1} | |
| 257511 [Spring Shell] INFO org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - Scanning log file HoodieLogFile{pathStr='file:/Users/nsb/Documents/personal/temp/onehouse_tbl_debug_mar23_s3/.hoodie/metadata/files/.files-0000_20220323070911012.log.3_0-82-771', fileLen=-1} | |
| 257511 [Spring Shell] INFO org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - Reading a data block from file file:/Users/nsb/Documents/personal/temp/onehouse_tbl_debug_mar23_s3/.hoodie/metadata/files/.files-0000_20220323070911012.log.3_0-82-771 at instant 20220323073748149 | |
| 257511 [Spring Shell] INFO org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - Number of remaining logblocks to merge 1 | |
| 257528 [Spring Shell] INFO org.apache.hadoop.hbase.io.hfile.CacheConfig - Created cacheConfig: blockCache=LruBlockCache{blockCount=0, currentSize=3133208, freeSize=3051159016, maxSize=3054292224, heapSize=3133208, minSize=2901577472, minFactor=0.95, multiSize=1450788736, multiFactor=0.5, singleSize=725394368, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false | |
| 257529 [Spring Shell] INFO org.apache.hadoop.io.compress.CodecPool - Got brand-new decompressor [.gz] | |
| 257529 [Spring Shell] INFO org.apache.hadoop.io.compress.CodecPool - Got brand-new decompressor [.gz] | |
| 257529 [Spring Shell] INFO org.apache.hadoop.io.compress.CodecPool - Got brand-new decompressor [.gz] | |
| 257529 [Spring Shell] INFO org.apache.hadoop.io.compress.CodecPool - Got brand-new decompressor [.gz] | |
| 257531 [Spring Shell] WARN org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - XXXX Hoodie rec: HoodieKey { recordKey=. partitionPath=files} : {"_hoodie_commit_time": null, "_hoodie_commit_seqno": null, "_hoodie_record_key": null, "_hoodie_partition_path": null, "_hoodie_file_name": null, "key": ".", "type": 2, "filesystemMetadata": {"b014012b-208a-4535-9fff-8a40ab7127c8-0_0-39-386_20220323073328903.parquet": {"size": 77257660, "isDeleted": false}}, "BloomFilterMetadata": null, "ColumnStatsMetadata": null} | |
| 257531 [Spring Shell] WARN org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - XXXX Hoodie rec: HoodieKey { recordKey=__all_partitions__ partitionPath=files} : {"_hoodie_commit_time": null, "_hoodie_commit_seqno": null, "_hoodie_record_key": null, "_hoodie_partition_path": null, "_hoodie_file_name": null, "key": "__all_partitions__", "type": 1, "filesystemMetadata": {"": {"size": 0, "isDeleted": false}}, "BloomFilterMetadata": null, "ColumnStatsMetadata": null} | |
| 257534 [Spring Shell] INFO org.apache.hudi.common.table.log.HoodieLogFormatReader - Moving to the next reader for logfile HoodieLogFile{pathStr='file:/Users/nsb/Documents/personal/temp/onehouse_tbl_debug_mar23_s3/.hoodie/metadata/files/.files-0000_20220323070911012.log.4_0-117-982', fileLen=-1} | |
| 257534 [Spring Shell] INFO org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - Scanning log file HoodieLogFile{pathStr='file:/Users/nsb/Documents/personal/temp/onehouse_tbl_debug_mar23_s3/.hoodie/metadata/files/.files-0000_20220323070911012.log.4_0-117-982', fileLen=-1} | |
| 257534 [Spring Shell] INFO org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - Reading a data block from file file:/Users/nsb/Documents/personal/temp/onehouse_tbl_debug_mar23_s3/.hoodie/metadata/files/.files-0000_20220323070911012.log.4_0-117-982 at instant 20220323074247415 | |
| 257534 [Spring Shell] INFO org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - Number of remaining logblocks to merge 1 | |
| 257554 [Spring Shell] INFO org.apache.hadoop.hbase.io.hfile.CacheConfig - Created cacheConfig: blockCache=LruBlockCache{blockCount=0, currentSize=3133208, freeSize=3051159016, maxSize=3054292224, heapSize=3133208, minSize=2901577472, minFactor=0.95, multiSize=1450788736, multiFactor=0.5, singleSize=725394368, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false | |
| 257554 [Spring Shell] INFO org.apache.hadoop.io.compress.CodecPool - Got brand-new decompressor [.gz] | |
| 257554 [Spring Shell] INFO org.apache.hadoop.io.compress.CodecPool - Got brand-new decompressor [.gz] | |
| 257554 [Spring Shell] INFO org.apache.hadoop.io.compress.CodecPool - Got brand-new decompressor [.gz] | |
| 257555 [Spring Shell] INFO org.apache.hadoop.io.compress.CodecPool - Got brand-new decompressor [.gz] | |
| 257556 [Spring Shell] WARN org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - XXXX Hoodie rec: HoodieKey { recordKey=. partitionPath=files} : {"_hoodie_commit_time": null, "_hoodie_commit_seqno": null, "_hoodie_record_key": null, "_hoodie_partition_path": null, "_hoodie_file_name": null, "key": ".", "type": 2, "filesystemMetadata": {"3a9cf348-ec51-4621-ad62-5c5fea9cb1d3-0_1-71-697_20220323073748149.parquet": {"size": 54128704, "isDeleted": false}, "b014012b-208a-4535-9fff-8a40ab7127c8-0_0-71-696_20220323073748149.parquet": {"size": 104527829, "isDeleted": false}}, "BloomFilterMetadata": null, "ColumnStatsMetadata": null} | |
| 257556 [Spring Shell] WARN org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - XXXX Hoodie rec: HoodieKey { recordKey=__all_partitions__ partitionPath=files} : {"_hoodie_commit_time": null, "_hoodie_commit_seqno": null, "_hoodie_record_key": null, "_hoodie_partition_path": null, "_hoodie_file_name": null, "key": "__all_partitions__", "type": 1, "filesystemMetadata": {"": {"size": 0, "isDeleted": false}}, "BloomFilterMetadata": null, "ColumnStatsMetadata": null} | |
| 257556 [Spring Shell] INFO org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - Merging the final data blocks | |
| 257556 [Spring Shell] INFO org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - Number of remaining logblocks to merge 1 | |
| 257575 [Spring Shell] INFO org.apache.hadoop.hbase.io.hfile.CacheConfig - Created cacheConfig: blockCache=LruBlockCache{blockCount=0, currentSize=3133208, freeSize=3051159016, maxSize=3054292224, heapSize=3133208, minSize=2901577472, minFactor=0.95, multiSize=1450788736, multiFactor=0.5, singleSize=725394368, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false | |
| 257576 [Spring Shell] INFO org.apache.hadoop.io.compress.CodecPool - Got brand-new decompressor [.gz] | |
| 257576 [Spring Shell] INFO org.apache.hadoop.io.compress.CodecPool - Got brand-new decompressor [.gz] | |
| 257576 [Spring Shell] INFO org.apache.hadoop.io.compress.CodecPool - Got brand-new decompressor [.gz] | |
| 257577 [Spring Shell] INFO org.apache.hadoop.io.compress.CodecPool - Got brand-new decompressor [.gz] | |
| 257578 [Spring Shell] WARN org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - XXXX Hoodie rec: HoodieKey { recordKey=. partitionPath=files} : {"_hoodie_commit_time": null, "_hoodie_commit_seqno": null, "_hoodie_record_key": null, "_hoodie_partition_path": null, "_hoodie_file_name": null, "key": ".", "type": 2, "filesystemMetadata": {"3a9cf348-ec51-4621-ad62-5c5fea9cb1d3-0_0-106-910_20220323074247415.parquet": {"size": 83442093, "isDeleted": false}}, "BloomFilterMetadata": null, "ColumnStatsMetadata": null} | |
| 257580 [Spring Shell] WARN org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader - XXXX Hoodie rec: HoodieKey { recordKey=__all_partitions__ partitionPath=files} : {"_hoodie_commit_time": null, "_hoodie_commit_seqno": null, "_hoodie_record_key": null, "_hoodie_partition_path": null, "_hoodie_file_name": null, "key": "__all_partitions__", "type": 1, "filesystemMetadata": {"": {"size": 0, "isDeleted": false}}, "BloomFilterMetadata": null, "ColumnStatsMetadata": null} | |
| 257580 [Spring Shell] INFO org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner - Number of log files scanned => 5 | |
| 257580 [Spring Shell] INFO org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner - MaxMemoryInBytes allowed for compaction => 1073741824 | |
| 257580 [Spring Shell] INFO org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner - Number of entries in MemoryBasedMap in ExternalSpillableMap => 2 | |
| 257580 [Spring Shell] INFO org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner - Total size in bytes of MemoryBasedMap in ExternalSpillableMap => 1200 | |
| 257580 [Spring Shell] INFO org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner - Number of entries in BitCaskDiskMap in ExternalSpillableMap => 0 | |
| 257580 [Spring Shell] INFO org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner - Size of file spilled to disk => 0 | |
| 257581 [Spring Shell] INFO org.apache.hudi.metadata.HoodieBackedTableMetadata - Opened 5 metadata log files (dataset instant=20220323074937289, metadata instant=20220323074937289) in 119 ms | |
| 257583 [Spring Shell] WARN org.apache.hudi.metadata.HoodieBackedTableMetadata - YYYY Final merged Record from MDT : __all_partitions__, HoodieRecord{key=HoodieKey { recordKey=__all_partitions__ partitionPath=files}, currentLocation='null', newLocation='null'} | |
| 257583 [Spring Shell] INFO org.apache.hudi.metadata.BaseTableMetadata - Listed partitions from metadata: #partitions=2 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment