Created
July 28, 2021 22:53
-
-
Save nsivabalan/5147fde404e970fab66515af2eddcdcb to your computer and use it in GitHub Desktop.
abstract log record scanner. createHoodieRecord
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2021-07-28T22:49:24,833 INFO [LocalJobRunner Map Task Executor #0]: log.AbstractHoodieLogRecordScanner (AbstractHoodieLogRecordScanner.java:scan(151)) - Scanning log file HoodieLogFile{pathStr='hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/.3cca15e7-c88d-4ede-ae90-833b06ce52ae-0_20210728224800.log.1_0-16-211', fileLen=0} | |
| 2021-07-28T22:49:24,839 INFO [LocalJobRunner Map Task Executor #0]: log.AbstractHoodieLogRecordScanner (AbstractHoodieLogRecordScanner.java:scan(178)) - Reading a data block from file hdfs://namenode:8020/user/hive/warehouse/stock_ticks_mor/2018/08/31/.3cca15e7-c88d-4ede-ae90-833b06ce52ae-0_20210728224800.log.1_0-16-211 at instant 20210728224829 | |
| 2021-07-28T22:49:24,839 INFO [LocalJobRunner Map Task Executor #0]: log.AbstractHoodieLogRecordScanner (AbstractHoodieLogRecordScanner.java:scan(267)) - Merging the final data blocks | |
| 2021-07-28T22:49:24,840 INFO [LocalJobRunner Map Task Executor #0]: log.AbstractHoodieLogRecordScanner (AbstractHoodieLogRecordScanner.java:processQueuedBlocksForInstant(342)) - Number of remaining logblocks to merge 1 | |
| 2021-07-28T22:49:24,849 ERROR [LocalJobRunner Map Task Executor #0]: log.AbstractHoodieLogRecordScanner (AbstractHoodieLogRecordScanner.java:scan(276)) - Got exception when reading log file | |
| java.lang.NullPointerException | |
| at org.apache.hudi.common.util.SpillableMapUtils.convertToHoodieRecordPayload(SpillableMapUtils.java:120) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.util.SpillableMapUtils.convertToHoodieRecordPayload(SpillableMapUtils.java:113) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.AbstractHoodieLogRecordScanner.createHoodieRecord(AbstractHoodieLogRecordScanner.java:314) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.AbstractHoodieLogRecordScanner.processDataBlock(AbstractHoodieLogRecordScanner.java:308) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.AbstractHoodieLogRecordScanner.processQueuedBlocksForInstant(AbstractHoodieLogRecordScanner.java:347) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.AbstractHoodieLogRecordScanner.scan(AbstractHoodieLogRecordScanner.java:268) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner.performScan(HoodieMergedLogRecordScanner.java:95) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner.<init>(HoodieMergedLogRecordScanner.java:88) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner$Builder.build(HoodieMergedLogRecordScanner.java:239) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.RealtimeCompactedRecordReader.getMergedLogRecordScanner(RealtimeCompactedRecordReader.java:88) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.RealtimeCompactedRecordReader.<init>(RealtimeCompactedRecordReader.java:62) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.HoodieRealtimeRecordReader.constructRecordReader(HoodieRealtimeRecordReader.java:70) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.HoodieRealtimeRecordReader.<init>(HoodieRealtimeRecordReader.java:47) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat.getRecordReader(HoodieParquetRealtimeInputFormat.java:129) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:376) ~[hive-exec-2.3.3.jar:2.3.3] | |
| at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:169) ~[hadoop-mapreduce-client-core-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:432) ~[hadoop-mapreduce-client-core-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) ~[hadoop-mapreduce-client-core-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:270) ~[hadoop-mapreduce-client-common-2.8.4.jar:?] | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_212] | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_212] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_212] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_212] | |
| at java.lang.Thread.run(Thread.java:748) [?:1.8.0_212] | |
| 2021-07-28T22:49:24,850 INFO [Thread-92]: mapred.LocalJobRunner (LocalJobRunner.java:runTasks(483)) - map task executor complete. | |
| 2021-07-28T22:49:24,850 WARN [Thread-92]: mapred.LocalJobRunner (LocalJobRunner.java:run(587)) - job_local1771174113_0002 | |
| java.lang.Exception: java.io.IOException: org.apache.hudi.exception.HoodieException: Exception when reading log file | |
| at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:489) ~[hadoop-mapreduce-client-common-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:549) ~[hadoop-mapreduce-client-common-2.8.4.jar:?] | |
| Caused by: java.io.IOException: org.apache.hudi.exception.HoodieException: Exception when reading log file | |
| at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97) ~[hive-exec-2.3.3.jar:2.3.3] | |
| at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57) ~[hive-exec-2.3.3.jar:2.3.3] | |
| at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:379) ~[hive-exec-2.3.3.jar:2.3.3] | |
| at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:169) ~[hadoop-mapreduce-client-core-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:432) ~[hadoop-mapreduce-client-core-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) ~[hadoop-mapreduce-client-core-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:270) ~[hadoop-mapreduce-client-common-2.8.4.jar:?] | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_212] | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_212] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_212] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_212] | |
| at java.lang.Thread.run(Thread.java:748) [?:1.8.0_212] | |
| Caused by: org.apache.hudi.exception.HoodieException: Exception when reading log file | |
| at org.apache.hudi.common.table.log.AbstractHoodieLogRecordScanner.scan(AbstractHoodieLogRecordScanner.java:277) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner.performScan(HoodieMergedLogRecordScanner.java:95) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner.<init>(HoodieMergedLogRecordScanner.java:88) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner$Builder.build(HoodieMergedLogRecordScanner.java:239) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.RealtimeCompactedRecordReader.getMergedLogRecordScanner(RealtimeCompactedRecordReader.java:88) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.RealtimeCompactedRecordReader.<init>(RealtimeCompactedRecordReader.java:62) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.HoodieRealtimeRecordReader.constructRecordReader(HoodieRealtimeRecordReader.java:70) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.HoodieRealtimeRecordReader.<init>(HoodieRealtimeRecordReader.java:47) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat.getRecordReader(HoodieParquetRealtimeInputFormat.java:129) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:376) ~[hive-exec-2.3.3.jar:2.3.3] | |
| at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:169) ~[hadoop-mapreduce-client-core-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:432) ~[hadoop-mapreduce-client-core-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) ~[hadoop-mapreduce-client-core-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:270) ~[hadoop-mapreduce-client-common-2.8.4.jar:?] | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_212] | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_212] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_212] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_212] | |
| at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_212] | |
| Caused by: java.lang.NullPointerException | |
| at org.apache.hudi.common.util.SpillableMapUtils.convertToHoodieRecordPayload(SpillableMapUtils.java:120) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.util.SpillableMapUtils.convertToHoodieRecordPayload(SpillableMapUtils.java:113) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.AbstractHoodieLogRecordScanner.createHoodieRecord(AbstractHoodieLogRecordScanner.java:314) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.AbstractHoodieLogRecordScanner.processDataBlock(AbstractHoodieLogRecordScanner.java:308) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.AbstractHoodieLogRecordScanner.processQueuedBlocksForInstant(AbstractHoodieLogRecordScanner.java:347) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.AbstractHoodieLogRecordScanner.scan(AbstractHoodieLogRecordScanner.java:268) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner.performScan(HoodieMergedLogRecordScanner.java:95) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner.<init>(HoodieMergedLogRecordScanner.java:88) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner$Builder.build(HoodieMergedLogRecordScanner.java:239) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.RealtimeCompactedRecordReader.getMergedLogRecordScanner(RealtimeCompactedRecordReader.java:88) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.RealtimeCompactedRecordReader.<init>(RealtimeCompactedRecordReader.java:62) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.HoodieRealtimeRecordReader.constructRecordReader(HoodieRealtimeRecordReader.java:70) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.HoodieRealtimeRecordReader.<init>(HoodieRealtimeRecordReader.java:47) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat.getRecordReader(HoodieParquetRealtimeInputFormat.java:129) ~[hoodie-hadoop-mr-bundle.jar:0.9.0-SNAPSHOT] | |
| at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:376) ~[hive-exec-2.3.3.jar:2.3.3] | |
| at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.<init>(MapTask.java:169) ~[hadoop-mapreduce-client-core-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:432) ~[hadoop-mapreduce-client-core-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) ~[hadoop-mapreduce-client-core-2.8.4.jar:?] | |
| at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:270) ~[hadoop-mapreduce-client-common-2.8.4.jar:?] | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_212] | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_212] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_212] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_212] | |
| at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_212] | |
| 2021-07-28T22:49:25,043 INFO [HiveServer2-Handler-Pool: Thread-46]: conf.HiveConf (HiveConf.java:getLogIdVar(3944)) - Using the default value passed in for log id: 6a9501c9-983c-4061-9d63-48e5af923dfa | |
| 2021-07-28T22:49:25,043 INFO [6a9501c9-983c-4061-9d63-48e5af923dfa HiveServer2-Handler-Pool: Thread-46]: conf.HiveConf (HiveConf.java:getLogIdVar(3944)) - Using the default value passed in for log id: 6a9501c9-983c-4061-9d63-48e5af923dfa |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment